^c@svddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZmZmZddlZddlmZdZdZdZdZdadaeZeZded Zddd Z d Z!d Z"d Z#dddZ$dddZ%ddZ&dZ'dZ(dZ)dZ*dZ+dZ,dZ-e j.j/ddZ0dZ1dddZ2ddddZ3dZ4edZ5d Z6ed!Z7d"Z8d#Z9ddd$Z:d%Z;d&Z<d'Z=d(Z>d)Z?d*Z@d+ZAd,ZBd-ZCejDdKd0d1ZEdLd2ZFdd3ZGd4ZHd5ZId6ZJd7ZKd8ZLd9ZMd:ZNd;ZOd<ZPd=ZQd>ZRd?d@ZSedAZTdBZUdCedDZVdEZWdFZXdGZYdHZZdIZ[dJZ\dS(MiN(t LooseVersion(tabspatht expandusertjoin(tSeqIOs == Error == s== Warning == s ERROR s WARN cCs d}|rk|jd||ft|dt|jd||jd|jd|jnetjjd||ftjjd|tjjd tjjd |jtjjtrt j j trt j tntjd dS( NtSPAdess %s %st with_errorsR In case you have troubles running %s, you can write to spades.support@cab.spbu.rusCor report an issue on our GitHub repository github.com/ablab/spadessMPlease provide us with params.txt and %s.log files from the output directory.s %s %s sS In case you have troubles running %s, you can write to spades.support@cab.spbu.ru sDor report an issue on our GitHub repository github.com/ablab/spades sNPlease provide us with params.txt and %s.log files from the output directory. i(tinfot log_warningstTruetlowertsyststderrtwritetflushtcurrent_tmp_dirtostpathtisdirtshutiltrmtreetexit(terr_strtlogtprefixt binary_name((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyterror)s$     cCsK|r |jd||fn'tjjd||ftjjdS(Ns %s %s s %s %s (RR tstdoutR R(twarn_strRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytwarning@scCsd}tjjd}g}xtjD]}|d}d|kr`|jd\}}n-|jdr|d |}}n |}}||}|jd||jddft|t|kot|knr,t Sq,Wt d|d j |fdS( NcSsq|jd}xRttt|D]8}||jr(tt||d||x7|D]/}|j|dkrtd||qqWdS(Nis$file %s was specified at least twice(tcountR(t filenamesRRD((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytcheck_files_duplications c Cs|tjkrtj|}ntjj|d}|jdkrtjj|t| d}||jtjkr||}qtjj|t|| d}||}n|jtjkrtddj tj||f|n| rD|jtj krDtdj tj d||f|n| r|jtj kr|tj krtddj tj ||f|n|j dr|jtjkrtd|dj tj||f|ndS( Nis.gzsGfile with reads has unsupported format (only %s are supported): %s (%s)s, s4 formats supported only for iontorrent mode: %s (%s)sYto run read error correction, reads should be in FASTQ format (%s are supported): %s (%s)tcontigssCfile with %s should be in FASTA format (%s are supported): %s (%s)(R,tdict_of_prefixesRRtsplitextR R"tALLOWED_READS_EXTENSIONSRRt(IONTORRENT_ONLY_ALLOWED_READS_EXTENSIONStBH_ALLOWED_READS_EXTENSIONStLONG_READS_TYPESR.t CONTIGS_ALLOWED_READS_EXTENSIONS( RDRCtonly_assemblert iontorrentt library_typeRtexttpre_extt pre_pre_ext((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytcheck_reads_file_formats,! % ##$cCsd}tjj|\}}|r:||r|SnjdtjkrxXtjdjtjD]:}|jd}tjj||}||rc|SqcWndS(NcSs%tjj|o$tj|tjS(N(RRR:taccesstX_OK(tfpath((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytis_exestPATHt"(RRRtenvirontpathseptstripRtNone(tprogramRdRctfnameRtexe_file((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytwhichs     cCsd}d}tjj|ryXxQt|D]C}|j|r.t|t|jd}|d:}|Sq.WWqtk rdSt k rdSXndS(Ns /proc/meminfos MemTotal:iii( RRR:topent startswithR%R"Rt ValueErrorRjtIOError(tmem_info_filenametavail_mem_headertlinet avail_mem((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_available_memorys     cCs?y|jdWn#tk r%tStk r6tSXtSdS(Ntascii(tencodetUnicodeDecodeErrortFalsetUnicodeEncodeErrorR (Ru((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyRIs  s3.cCs#|rt|djS|jS(Nsutf-8(R$trstrip(Rut is_python3((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytprocess_readlinescCs!d|krd|d}n|S(Nt Rf((R$((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytprocess_spacess c CsQddl}ddl}t|tr0|}n|j|}|j|d|jd|jd|}d}xe|jst |j j }|r|r|j |q||d7}n|j dk roPqoqoWxP|j jD]?}t |}|r|r|j |q#||d7}qqW|j rMtd||j f|n|S(NiRR tcwdR?s s>system call for: "%s" finished abnormally, OS return value: %d(tshlext subprocesst isinstancetlistRtPopentPIPEtSTDOUTtpollRRtreadlineRt returncodeRjt readlinesR( tcmdRRRRtcmd_listtproctoutputRu((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytsys_calls0   '  c Cs+ddl}ddl}t|tr0|}n|j|}|rWt|d}n |j}|rxt|d} n |j} |j|d|d| d|} |r| s| rx| jsC|st | j j } | r|j | qn|s-t | j j } | r-|j | q-n| jdk rPqqW|sx<| j jD](} | dkrZ|j t | qZqZWn|sx<| j jD](} | dkr|j t | qqWqn | j|r|jn|r| jn| jr'td|| jf|ndS( s Runs cmd and redirects stdout to out_filename (if specified), stderr to err_filename (if specified), or to log otherwise iNtwRR RR?s>system call for: "%s" finished abnormally, OS return value: %d(RRRRRRoRRRRRRRR RRjRtwaittcloseR( RRt out_filenamet err_filenameRRRRRR RRu((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytuniversal_sys_callsN     !       cCs^t|d}|j|jWdQXtj|tjtjBtjBtj Btj BdS(Ntwb( RoR treadRtchmodtstattS_IWRITEtS_IREADtS_IXUSRtS_IXGRPtS_IXOTH(tdatatfileR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytsave_data_to_fileNscCsd}|rt}t}n t}t}trt|K}|jt|j}x|jsv|j}q[W|j }WdQXt|j }|j |} || |}nt|j }g} g} d} x|D]} | j | rqn|| krn| | j |t |j}| j|dj} || |s| j| | qq|| kr| | j |t |j}| j} || |s| j| | qqqW| | fS(NcSs(x!|D]}|j|rtSqWtS(N(R.R R{(t list_to_checktsuffixtitem((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt already_savedVs s * R?(tSPADES_PY_WARN_MESSAGEtSPADES_WARN_MESSAGEtSPADES_PY_ERROR_MESSAGEtSPADES_ERROR_MESSAGEtcontinue_logfile_offsetRotseekRRiRtindexRptfindR"R0R/(t log_filenametwarningsRtspades_py_messagetspades_messaget continued_logtcontinued_stage_phrasetlines_to_checkt all_linestfailed_stage_indextspades_py_msgst spades_msgst IMPORTANT_MESSAGE_SUMMARY_PREFIXRuR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_important_messages_from_logUsD      # # cCs@d}x3|jdD]$}|jjdkr|j}qqW|S(Nthandlerst FileHandler(Rjt__dict__t __class__t__name__t baseFilename(Rtlog_fileth((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_logger_filenames c Cst|}|stSx|jdD]}|jq$Wt|dt\}}|s\|r|rr|jdn |jdtjj tjj |d}t j |dd}|j ||jd|r|jd x|D]}|j|qWn|r7|jd x|D]}|j|qWn|jd ||j||rt|dt\} } |jd|jd x"| | D]}|j|qWntStS( NRRs? ======= SPAdes pipeline finished abnormally and WITH WARNINGS!s0 ======= SPAdes pipeline finished WITH WARNINGS!s warnings.logtmodeRR?s=== Pipeline warnings:s-=== Error correction and assembling warnings:s======= Warnings saved to s === ERRORs:(RR{RRRR RRRRRGtloggingRt addHandlert removeHandler( RRRRtspades_py_warnst spades_warnstwarnings_filenametwarnings_handlerRutspades_py_errorst spades_errors((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyRs@   !         cCs\tjjrXttj_t|}|rXt|}|jdd|jaqXndS(Nii( R,targst continue_modeR{RRoRttellR(RRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytcontinue_from_heres    csjdfd}d}xEttj|d|dtD]"}tjj|r@|}Pq@q@W|S(NcSs|jrt|S|S(N(R#R%(ttext((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytatois  cs)gtjd|D]}|^qS(Ns(\d+)(treR(Rtc(R(sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt natural_keysstkeytreverse(RjtsortedtglobR RRR(tpatternRt latest_dirt dir_to_test((Rsk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_latest_dirs (cCsS|stjj}ntjj|s7tj|ntjd|d|a t S(NtdirR( R,Rttmp_dirRRRRLttempfiletmkdtempR(Rtbase_dir((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt get_tmp_dirs cCscx\tjjD]K}|jd|r|td|td|d!jr[|SqqWdS(Ns--i(R,tSHORT_READS_TYPEStkeysRpR"R#Rj(toptiontshort_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_short_reads_types + cCs;x4tjD])}|jdr |d|kr |Sq WdS(Ns--(R,RXRpRj(Rtlong_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_long_reads_typescCs|jdo|djS(Ns--si(RpR#(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytis_single_read_typescCsnd}d}t|rIt|}t|tjd|j}nt|rdt|}n||fS(Ntpeis\d(RR%RtsearchtstartR(Rtlib_typet lib_number((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_lib_type_and_numbers  % cCs|jdrd}n|jdr0d}nu|jdrHd}n]|jdsot|sot|rxd}n-|jd s|jd rd }nd }|S( Ns-12sinterlaced readss-1s left readss-2s right readss-ss single readss-ms-mergeds merged readst orientation(R.RR(Rt data_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt get_data_types   '  cCsid}d|kr_d||jd tjkr_||jd }||jdd}n||fS(Nt:Ri(RjRR,RU(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_option_prefix s ,cCst|\}}d||f}t|}|dkrG|d}n||kri||<|tjkrtj|||d-s, --mp<#>-s, --hqmp<#>-s, or --s<#> option instead of -s!(tonly_old_style_optionstold_style_single_readsR(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytcheck_single_reads_in_optionss cCset|tk r|g}ng}x:t|D],\}}|d|kr1|j|q1q1W|S(NR(RRR+R/(Rttypestlib_idsR-R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_lib_ids_by_types cCs;t||}g}x|D]}|j||qW|S(N(R6R/(RR4tidstresultR-((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytget_libs_by_types  cCs7t||}x!t|dtD] }||=q"W|S(NR(R6RR (RR4R7R-((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytrm_libs_by_types cCsx|D]}|rtSqWtS(N(R{R (RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytdataset_is_emptys cCs`xY|D]Q}xH|D]@}|jdrx(||D]}|jdr4tSq4WqqWqWtS(NRs.gz(R.R R{(RRRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_gzipped_readss  cCs%x|D]}d|krtSqWtS(Nsinterlaced reads(R R{(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_interlaced_readss  cCs,x%|D]}|djdrtSqWtS(NRRR(R.R R{(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_additional_contigss cCs)x"|D]}|ddkrtSqWtS(NRtnxmate(R R{(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_nxmate_readss s cCsdddddg}xt|D]\}}|j|d|d|dfd |kr{|jd ||d fnxM|D]E}||krd }nt||}|jd |||fqWq"WdS( Ns left readss right readssinterlaced readss single readss merged readss$Library number: %d, library type: %siRRs%s orientation: %ss not specifieds %s %s: %s(R+RR$(RRtindentt READS_TYPESR-Rt reads_typeR ((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytpretty_print_readss#    cCsg}g}t}d}|r0tj|}n t|}x|D]}t||odtjjd}|svqCn|ddkr|j|j|s|j|nt }d}qC||j7}qCW|j||j t ||S(NR?s3.it>( R tgzipRoRR R&RpR/RiR{Rtzip(RDtgzippedtres_nametres_seqtfirsttseqt file_handlerRu((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt read_fastas*  !   c Cst|dn}xd|D]\\}}|j|dx<tdt|dD]"}|j|||d!dqOWqWWdQXdS(NRs ii<(RoR R!R"(RDtfastatoutfiletnameRLR(((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt write_fastas tNcCsg}t}xtt||D]\}\}}d} d} d} x-| t|kru|jd| dkru|dkrt}n|jd| } | d} x0| t|kr|| dkr| d7} qW| d} | | |krIt}| | kri|j|jddt| ddj |jd|| | !j d|f| d7} n| } qIqIW| t|kr"|j|jddt| ddj |jd|| j d|fq"q"W||fS(NiiRSit_R( R{R+RNR"RR R/RR$RR0(RBt thresholdt replace_charRHt new_fastatmodifiedR-RQRLR(tcur_contig_numbertcur_contig_startRtend((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytbreak_scaffolds&s2(-   %  <  <!cCs1idd6dd6dd6dd6dd6|jS(NtTtAtGtCRS(tupper(tletter((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytcompCscCs&djtjt|dddS(NR?i(RR$timapRc(RL((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytrev_compGscCss|jd}t|dksA|ddkrS|ddkrStd|dSd|krk|ddS|dS( NRTiis>NODEtNODEscontig %s has unknown ID formatt'i(RR"RRj(tsR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt get_contig_idKs2  cCs|jdr|dS|S(NREi(Rp(Rh((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytremove_fasta_prefUscCs+yt|tSWntk r&tSXdS(N(tfloatR RqR{(R ((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytis_float[s   cCs+yt|tSWntk r&tSXdS(N(R%R RqR{(R ((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pytis_intcs   i'i'(]RRFR$RRRRRRR Rtdistutils.versionRtos.pathRRRR,tcommonRRRRRRjRRR R1R{R2RRR8R9R>RERHR@RMRNRQR`RnRwRIR&RpRRRRRRRRRRRRRRRRRRR Rt READS_TYPES_USED_IN_CONSTRUCTIONRR!RRR0R3R6R9R:R;R<R=R>R@RDRNRRR\RcReRiRjRlRm(((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.14.0-Linux/share/spades/spades_pipeline/support.pyt s                         #7  0  %