O_c@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZmZmZddlZddlmZdZdZdZdZdadaeZeZdedd Z ddd Z!d Z"d Z#d Z$dddZ%ddZ&dddZ'dddZ(ddZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0e j1j2ddZ3dZ4dddZ5ddddZ6d Z7ed!Z8d"Z9ed#Z:d$Z;d%Z<ddd&Z=d'Z>d(Z?d)Z@d*ZAd+ZBd,ZCd-ZDd.ZEd/ZFd0ZGejHdMd3d4ZIdNd5ZJdd6ZKd7ZLd8ZMd9ZNd:ZOd;ZPd<ZQd=ZRd>ZSd?ZTd@ZUdAdBZVedCZWdDZXdEedFZYdGZZdHZ[dIZ\dJZ]dKZ^dLZ_dS(OiN(t LooseVersion(tabspatht expandusertjoin(tSeqIOs == Error == s== Warning == s ERROR s WARN cCs d}|rk|jd||ft|dt|jd||jd|jd|jnetjjd||ftjjd|tjjd tjjd |jtjjtrt j j trt j tntj|dS( NtSPAdess %s %st with_errorsR In case you have troubles running %s, you can write to spades.support@cab.spbu.rusCor report an issue on our GitHub repository github.com/ablab/spadessMPlease provide us with params.txt and %s.log files from the output directory.s %s %s sS In case you have troubles running %s, you can write to spades.support@cab.spbu.ru sDor report an issue on our GitHub repository github.com/ablab/spades sNPlease provide us with params.txt and %s.log files from the output directory. (tinfot log_warningstTruetlowertsyststderrtwritetflushtcurrent_tmp_dirtostpathtisdirtshutiltrmtreetexit(terr_strtlogtprefixt exit_codet binary_name((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyterror*s$     cCsK|r |jd||fn'tjjd||ftjjdS(Ns %s %s s %s %s (RR tstdoutR R(twarn_strRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytwarningAscCsd}tjjd}g}xtjD]}|d}d|kr`|jd\}}n-|jdr|d |}}n |}}||}|jd||jddft|t|kot|knr,t Sq,Wt d|d j |fdS( NcSsq|jd}xRttt|D]8}||jr(tt||d||x7|D]/}|j|dkrtd||qqWdS(Nis$file %s was specified at least twice(tcountR(t filenamesRRE((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytcheck_files_duplications c Cs<|tjkrtj|}ntjj|d}|jdkrtjj|t| d}||jtjkr||}qtjj|t|| d}||}n|jtjkrtddj tj||f|n| rD|jtj krDtdj tj d||f|n| r|jtj kr|tj krtddj tj ||f|n|j dr|jtjkrtd|dj tj||f|n|j d r8|jtjkr8td |dj tj||f|ndS( Nis.gzsGfile with reads has unsupported format (only %s are supported): %s (%s)s, s4 formats supported only for iontorrent mode: %s (%s)sYto run read error correction, reads should be in FASTQ format (%s are supported): %s (%s)tcontigssCfile with %s should be in FASTA format (%s are supported): %s (%s)tgraphsAfile with %s should be in GFA format (%s are supported): %s (%s)(R-RGRRtsplitextR R#tALLOWED_READS_EXTENSIONSRRt(IONTORRENT_ONLY_ALLOWED_READS_EXTENSIONStBH_ALLOWED_READS_EXTENSIONStLONG_READS_TYPESR/t CONTIGS_ALLOWED_READS_EXTENSIONStGRAPH_ALLOWED_READS_EXTENSIONS( RERDtonly_assemblert iontorrentt library_typeRRItpre_extt pre_pre_ext((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytcheck_reads_file_formats2! % ##$&$cCsd}tjj|\}}|r:||r|SnjdtjkrxXtjdjtjD]:}|jd}tjj||}||rc|SqcWndS(NcSs%tjj|o$tj|tjS(N(RRR;taccesstX_OK(tfpath((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytis_exestPATHt"(RRR tenvirontpathseptstripRRQ(tprogramRxRwtfnameRtexe_file((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytwhichs     cCsd}d}tjj|ryXxQt|D]C}|j|r.t|t|jd}|d:}|Sq.WWqtk rdSt k rdSXndS(Ns /proc/meminfos MemTotal:iii( RRR;topent startswithR&R#R t ValueErrorRQtIOError(tmem_info_filenametavail_mem_headertlinet avail_mem((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_available_memorys     cCs?y|jdWn#tk r%tStk r6tSXtSdS(Ntascii(tencodetUnicodeDecodeErrortFalsetUnicodeEncodeErrorR (R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyR]s  s3.cCs#|rt|djS|jS(Nsutf-8(R%trstrip(Rt is_python3((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytprocess_readlinescCs!d|krd|d}n|S(Nt Rz((R%((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytprocess_spacess c CsZddl}ddl}t|tr0|}n|j|}|j|d|jd|jd|}d}xe|jst |j j }|r|r|j |q||d7}n|j dk roPqoqoWxP|j jD]?}t |}|r|r|j |q#||d7}qqW|j rVtd||j f|d|j n|S( NiRR tcwdR@s s>system call for: "%s" finished abnormally, OS return value: %dR(tshlext subprocesst isinstancetlistR tPopentPIPEtSTDOUTtpollRRtreadlineRt returncodeRQt readlinesR( tcmdRRRRtcmd_listtproctoutputR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytsys_calls0   '  &c Cs4ddl}ddl}t|tr0|}n|j|}|rWt|d}n |j}|rxt|d} n |j} |j|d|d| d|} |r| s| rx| jsC|st | j j } | r|j | qn|s-t | j j } | r-|j | q-n| jdk rPqqW|sx<| j jD](} | dkrZ|j t | qZqZWn|sx<| j jD](} | dkr|j t | qqWqn | j|r|jn|r| jn| jr0td|| jf|d | jndS( s Runs cmd and redirects stdout to out_filename (if specified), stderr to err_filename (if specified), or to log otherwise iNtwRR RR@s>system call for: "%s" finished abnormally, OS return value: %dR(RRRRR RRRRRRRRR RRQRtwaittcloseR( RRt out_filenamet err_filenameRRRRRR RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytuniversal_sys_call7sN     !       cCs^t|d}|j|jWdQXtj|tjtjBtjBtj Btj BdS(Ntwb( RR treadRtchmodtstattS_IWRITEtS_IREADtS_IXUSRtS_IXGRPtS_IXOTH(tdatatfileR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytsave_data_to_filenscCsd}|rt}t}n t}t}trt|K}|jt|j}x|jsv|j}q[W|j }WdQXt|j }|j |} || |}nt|j }g} g} d} x|D]} | j | rqn|| krn| | j |t |j}| j|dj} || |s| j| | qq|| kr| | j |t |j}| j} || |s| j| | qqqW| | fS(NcSs(x!|D]}|j|rtSqWtS(N(R/R R(t list_to_checktsuffixtitem((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt already_savedvs s * R@(tSPADES_PY_WARN_MESSAGEtSPADES_WARN_MESSAGEtSPADES_PY_ERROR_MESSAGEtSPADES_ERROR_MESSAGEtcontinue_logfile_offsetRtseekRR}RtindexRtfindR#R1R0(t log_filenametwarningsRtspades_py_messagetspades_messaget continued_logtcontinued_stage_phrasetlines_to_checkt all_linestfailed_stage_indextspades_py_msgst spades_msgst IMPORTANT_MESSAGE_SUMMARY_PREFIXRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_important_messages_from_logusD      # # cCs@d}x3|jdD]$}|jjdkr|j}qqW|S(Nthandlerst FileHandler(RQt__dict__t __class__t__name__t baseFilename(Rtlog_fileth((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_logger_filenames c Cst|}|stSx|jdD]}|jq$Wt|dt\}}|s\|r|rr|jdn |jdtjj tjj |d}t j |dd}|j ||jd|r|jd x|D]}|j|qWn|r7|jd x|D]}|j|qWn|jd ||j||rt|dt\} } |jd|jd x"| | D]}|j|qWntStS( NRRs? ======= SPAdes pipeline finished abnormally and WITH WARNINGS!s0 ======= SPAdes pipeline finished WITH WARNINGS!s warnings.logtmodeRR@s=== Pipeline warnings:s-=== Error correction and assembling warnings:s======= Warnings saved to s === ERRORs:(RRRRRR RRRRR[tloggingRt addHandlert removeHandler( RRRRtspades_py_warnst spades_warnstwarnings_filenametwarnings_handlerRtspades_py_errorst spades_errors((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyRs@   !         cCs\tjjrXttj_t|}|rXt|}|jdd|jaqXndS(Nii( R-RSt continue_modeRRRRttellR(RRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytcontinue_from_heres    csjdfd}d}xEttj|d|dtD]"}tjj|r@|}Pq@q@W|S(NcSs|jrt|S|S(N(R$R&(ttext((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytatois  cs)gtjd|D]}|^qS(Ns(\d+)(treR (Rtc(R(sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt natural_keysstkeytreverse(RQtsortedtglobR RRR(tpatternRt latest_dirt dir_to_test((Rsk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_latest_dirs (cCsS|stjj}ntjj|s7tj|ntjd|d|a t S(NtdirR( R-RSttmp_dirRRRR`ttempfiletmkdtempR(Rtbase_dir((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt get_tmp_dirs cCscx\tjjD]K}|jd|r|td|td|d!jr[|SqqWdS(Ns--i(R-tSHORT_READS_TYPEStkeysRR#R$RQ(toptiontshort_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_short_reads_types + cCs;x4tjD])}|jdr |d|kr |Sq WdS(Ns--(R-RlRRQ(Rtlong_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_long_reads_typescCs;x4tjD])}|jdr |d|kr |Sq WdS(Ns--(R-tGRAPH_READS_TYPESRRQ(Rtgraph_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_graph_type scCs|jdo|djS(Ns--si(RR$(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytis_single_read_typescCsd}d}t|rIt|}t|tjd|j}n6t|rdt|}nt|rt|}n||fS(Ntpeis\d(RR&RtsearchtstartRR(Rtlib_typet lib_number((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_lib_type_and_numbers  %  cCs|jdrd}n|jdr0d}n|jdrHd}ni|jds{t|s{t|s{t|rd}n-|jd s|jd rd }nd }|S( Ns-12sinterlaced readss-1s left readss-2s right readss-ss single readss-ms-mergeds merged readst orientation(R/RRR(Rt data_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt get_data_type$s        cCsid}d|kr_d||jd tjkr_||jd }||jdd}n||fS(Nt:Ri(RQRR-Ri(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_option_prefix7s ,cCs,t|\}}d||f}t|}|dkrG|d}n||kri||<|||d<|tjkrtj|||d-s, --mp<#>-s, --hqmp<#>-s, or --s<#> option instead of -s!(tonly_old_style_optionstold_style_single_readsR(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytcheck_single_reads_in_optionss cCset|tk r|g}ng}x:t|D],\}}|d|kr1|j|q1q1W|S(NR(RRR<R0(Rttypestlib_idsR?R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_lib_ids_by_types cCs;t||}g}x|D]}|j||qW|S(N(RHR0(RRFtidstresultR?((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytget_libs_by_types  cCs7t||}x!t|dtD] }||=q"W|S(NR(RHRR (RRFRIR?((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytrm_libs_by_types cCsx|D]}|rtSqWtS(N(RR (RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytdataset_is_emptys cCs`xY|D]Q}xH|D]@}|jdrx(||D]}|jdr4tSq4WqqWqWtS(NRs.gz(R/R R(RRRR$((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_gzipped_readss  cCs%x|D]}d|krtSqWtS(Nsinterlaced reads(R R(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_interlaced_readss  cCs,x%|D]}|djdrtSqWtS(NRRf(R/R R(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytdataset_has_additional_contigss s cCsdddddg}xt|D]\}}|j|d|d|dfd |kr{|jd ||d fnxM|D]E}||krd }nt||}|jd |||fqWq"WdS( Ns left readss right readssinterlaced readss single readss merged readss$Library number: %d, library type: %siRR s%s orientation: %ss not specifieds %s %s: %s(R<RR%(RRtindentt READS_TYPESR?Rt reads_typeR"((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytpretty_print_reads s#    cCsg}g}t}d}|r0tj|}n t|}x|D]}t||odtjjd}|svqCn|ddkr|j|j|s|j|nt }d}qC||j7}qCW|j||j t ||S(NR@s3.it>( R tgzipRRR R'RR0R}RRtzip(REtgzippedtres_nametres_seqtfirsttseqt file_handlerR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt read_fasta1s*  !   c Cst|dn}xd|D]\\}}|j|dx<tdt|dD]"}|j|||d!dqOWqWWdQXdS(NRs ii<(RR R"R#(REtfastatoutfiletnameR\R)((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt write_fastaLs tNcCsg}t}xtt||D]\}\}}d} d} d} x-| t|kru|jd| dkru|dkrt}n|jd| } | d} x0| t|kr|| dkr| d7} qW| d} | | |krIt}| | kri|j|jddt| ddj |jd|| | !j d|f| d7} n| } qIqIW| t|kr"|j|jddt| ddj |jd|| j d|fq"q"W||fS(NiiRcit_R( RR<R^R#RR R0R R%RR1(RCt thresholdt replace_charRXt new_fastatmodifiedR?RaR\R)tcur_contig_numbertcur_contig_startRtend((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytbreak_scaffoldsTs2(-   %  <  <!cCs1idd6dd6dd6dd6dd6|jS(NtTtAtGtCRc(tupper(tletter((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytcompqscCs&djtjt|dddS(NR@i(RR8timapRs(R\((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytrev_compuscCss|jd}t|dksA|ddkrS|ddkrStd|dSd|krk|ddS|dS( NRdiis>NODEtNODEscontig %s has unknown ID formatt'i(R R#RRQ(tsR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt get_contig_idys2  cCs|jdr|dS|S(NRUi(R(Rx((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytremove_fasta_prefscCs+yt|tSWntk r&tSXdS(N(tfloatR RR(R"((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytis_floats   cCs+yt|tSWntk r&tSXdS(N(R&R RR(R"((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pytis_ints   i'i'(`RRVR8RR*RRRRR RRUtdistutils.versionRtos.pathRRRR-tcommonRRRRRRQRRR RCRRDRRR9R:R?RFRHRYR\RARaRbReRtRRR]R'RRRRRRRRRRRRRRRRR R RRRR&R=R5R7R'R(RBRERHRKRLRMRNRORPRTR^RbRlRsRuRyRzR|R}(((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.0-Linux/share/spades/spades_pipeline/support.pyt s                      #    #7  0  %            &