jac@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZmZmZddlZddlmZdZdZdZd ZdadaeZe Z!dedd Z"ddd Z#d Z$d Z%dZ&dZ'dZ(dZ)dddZ*ddZ+dddZ,dddZ-ddZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5e j6j7dd Z8d!Z9ddd"Z:dddd#Z;d$Z<ed%Z=d&Z>e d'Z?d(Z@d)ZAddd*ZBd+ZCd,ZDd-ZEd.ZFd/ZGd0ZHd1ZId2ZJd3ZKd4ZLejMdQd7d8ZNdRd9ZOdd:ZPd;ZQd<ZRd=ZSd>ZTd?ZUd@ZVdAZWdBZXdCZYdDZZdEdFZ[e dGZ\dHZ]dIe dJZ^dKZ_dLZ`dMZadNZbdOZcdPZddS(SiN(tuname(t LooseVersion(tabspatht expandusertjoin(tSeqIOs == Error == s== Warning == s ERROR s WARN cCs d}|rk|jd||ft|dt|jd||jd|jd|jnetjjd||ftjjd|tjjd tjjd |jtjjtrt j j trt j tntj|dS( NtSPAdess %s %st with_errorsR In case you have troubles running %s, you can write to spades.support@cab.spbu.rusCor report an issue on our GitHub repository github.com/ablab/spadessMPlease provide us with params.txt and %s.log files from the output directory.s %s %s sS In case you have troubles running %s, you can write to spades.support@cab.spbu.ru sDor report an issue on our GitHub repository github.com/ablab/spades sNPlease provide us with params.txt and %s.log files from the output directory. (tinfot log_warningstTruetlowertsyststderrtwritetflushtcurrent_tmp_dirtostpathtisdirtshutiltrmtreetexit(terr_strtlogtprefixt exit_codet binary_name((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyterror+s$     cCsK|r |jd||fn'tjjd||ftjjdS(Ns %s %s s %s %s (RR tstdoutRR(twarn_strRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytwarningBscCsd}|rdSdS(NcSsdtdjkS(Nt microsofti(RR (((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytin_wslKss1. WSL is an unsupported platform 2. If SPAdes crashes, then you might want to compile it from sources 3. If nothing works, run on real Linuxt((R!((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt wsl_checkJs  cCs|dkrtSdS(Ni(R#(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_error_hintsVs cCs6t|}d|||f}t||d|dS(NsAsystem call for: "%s" finished abnormally, OS return value: %d %sR(R$R(tcmdRRt hints_strterr_msg((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt sys_error[s cCsd}tjjd}g}xtjD]}|d}d|kr`|jd\}}n-|jdr|d |}}n |}}||}|jd||jddft|t|kot|knr,t Sq,Wt d|d j |fdS( NcSsq|jd}xRttt|D]8}||jr(tt||d||x7|D]/}|j|dkrtd||qqWdS(Nis$file %s was specified at least twice(tcountR(t filenamesRRN((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytcheck_files_duplications c Cs<|tjkrtj|}ntjj|d}|jdkrtjj|t| d}||jtjkr||}qtjj|t|| d}||}n|jtjkrtddj tj||f|n| rD|jtj krDtdj tj d||f|n| r|jtj kr|tj krtddj tj ||f|n|j dr|jtjkrtd|dj tj||f|n|j d r8|jtjkr8td |dj tj||f|ndS( Nis.gzsGfile with reads has unsupported format (only %s are supported): %s (%s)s, s4 formats supported only for iontorrent mode: %s (%s)sYto run read error correction, reads should be in FASTQ format (%s are supported): %s (%s)tcontigssCfile with %s should be in FASTA format (%s are supported): %s (%s)tgraphsAfile with %s should be in GFA format (%s are supported): %s (%s)(R7RPRRtsplitextR R-tALLOWED_READS_EXTENSIONSRRt(IONTORRENT_ONLY_ALLOWED_READS_EXTENSIONStBH_ALLOWED_READS_EXTENSIONStLONG_READS_TYPESR9t CONTIGS_ALLOWED_READS_EXTENSIONStGRAPH_ALLOWED_READS_EXTENSIONS( RNRMtonly_assemblert iontorrentt library_typeRRRtpre_extt pre_pre_ext((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytcheck_reads_file_formats2! % ##$&$cCsd}tjj|\}}|r:||r|SnjdtjkrxXtjdjtjD]:}|jd}tjj||}||rc|SqcWndS(NcSs%tjj|o$tj|tjS(N(RRREtaccesstX_OK(tfpath((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytis_exestPATHt"(RRR*tenvirontpathseptstripRRZ(tprogramRRtfnameRtexe_file((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytwhichs     cCsd}d}tjj|ryXxQt|D]C}|j|r.t|t|jd}|d:}|Sq.WWqtk rdSt k rdSXndS(Ns /proc/meminfos MemTotal:iii( RRREtopent startswithR0R-R*t ValueErrorRZtIOError(tmem_info_filenametavail_mem_headertlinet avail_mem((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_available_memorys     cCs?y|jdWn#tk r%tStk r6tSXtSdS(Ntascii(tencodetUnicodeDecodeErrortFalsetUnicodeEncodeErrorR (R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyRfs  s3.cCs#|rt|djS|jS(Nsutf-8(R/trstrip(Rt is_python3((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytprocess_readline scCs!d|krd|d}n|S(Nt R((R/((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytprocess_spaces&s c CsJddl}ddl}t|tr0|}n|j|}|j|d|jd|jd|}d}xe|jst |j j }|r|r|j |q||d7}n|j dk roPqoqoWxP|j jD]?}t |}|r|r|j |q#||d7}qqW|j rFt|||j n|S(NiRR tcwdR"s (tshlext subprocesst isinstancetlistR*tPopentPIPEtSTDOUTtpollRRtreadlineRt returncodeRZt readlinesR(( R%RRRRtcmd_listtproctoutputR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytsys_call,s0   '  c Cs$ddl}ddl}t|tr0|}n|j|}|rWt|d}n |j}|rxt|d} n |j} |j|d|d| d|} |r| s| rx| jsC|st | j j } | r|j | qn|s-t | j j } | r-|j | q-n| jdk rPqqW|sx<| j jD](} | dkrZ|j t | qZqZWn|sx<| j jD](} | dkr|j t | qqWqn | j|r|jn|r| jn| jr t||| jndS(s Runs cmd and redirects stdout to out_filename (if specified), stderr to err_filename (if specified), or to log otherwise iNtwRR RR"(RRRRR*RRRRRRRRR RRZRtwaittcloseR(( R%Rt out_filenamet err_filenameRRRRRR RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytuniversal_sys_callOsN     !       cCs^t|d}|j|jWdQXtj|tjtjBtjBtj Btj BdS(Ntwb( RRtreadRtchmodtstattS_IWRITEtS_IREADtS_IXUSRtS_IXGRPtS_IXOTH(tdatatfileR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytsave_data_to_filescCsd}|rt}t}n t}t}trt|K}|jt|j}x|jsv|j}q[W|j }WdQXt|j }|j |} || |}nt|j }g} g} d} x|D]} | j | rqn|| krn| | j |t |j}| j|dj} || |s| j| | qq|| kr| | j |t |j}| j} || |s| j| | qqqW| | fS(NcSs(x!|D]}|j|rtSqWtS(N(R9R R(t list_to_checktsuffixtitem((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt already_saveds s * R"(tSPADES_PY_WARN_MESSAGEtSPADES_WARN_MESSAGEtSPADES_PY_ERROR_MESSAGEtSPADES_ERROR_MESSAGEtcontinue_logfile_offsetRtseekRRRtindexRtfindR-R;R:(t log_filenametwarningsRtspades_py_messagetspades_messaget continued_logtcontinued_stage_phrasetlines_to_checkt all_linestfailed_stage_indextspades_py_msgst spades_msgst IMPORTANT_MESSAGE_SUMMARY_PREFIXRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_important_messages_from_logsD      # # cCs@d}x3|jdD]$}|jjdkr|j}qqW|S(Nthandlerst FileHandler(RZt__dict__t __class__t__name__t baseFilename(Rtlog_fileth((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_logger_filenames c Cst|}|stSx|jdD]}|jq$Wt|dt\}}|s\|r|rr|jdn |jdtjj tjj |d}t j |dd}|j ||jd|r|jd x|D]}|j|qWn|r7|jd x|D]}|j|qWn|jd ||j||rt|dt\} } |jd|jd x"| | D]}|j|qWntStS( NRRs? ======= SPAdes pipeline finished abnormally and WITH WARNINGS!s0 ======= SPAdes pipeline finished WITH WARNINGS!s warnings.logtmodeRR"s=== Pipeline warnings:s-=== Error correction and assembling warnings:s======= Warnings saved to s === ERRORs:(RRRRRR RRRRRdtloggingRt addHandlert removeHandler( RRRRtspades_py_warnst spades_warnstwarnings_filenametwarnings_handlerRtspades_py_errorst spades_errors((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyR s@   !         cCs\tjjrXttj_t|}|rXt|}|jdd|jaqXndS(Nii( R7R\t continue_modeRRRRttellR(RRR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytcontinue_from_heres    csjdfd}d}xEttj|d|dtD]"}tjj|r@|}Pq@q@W|S(NcSs|jrt|S|S(N(R.R0(ttext((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytatois  cs)gtjd|D]}|^qS(Ns(\d+)(treR*(Rtc(R(sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt natural_keysstkeytreverse(RZtsortedtglobR RRR(tpatternRt latest_dirt dir_to_test((Rsk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_latest_dirs (cCsS|stjj}ntjj|s7tj|ntjd|d|a t S(NtdirR( R7R\ttmp_dirRRRRittempfiletmkdtempR(Rtbase_dir((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt get_tmp_dirs cCscx\tjjD]K}|jd|r|td|td|d!jr[|SqqWdS(Ns--i(R7tSHORT_READS_TYPEStkeysRR-R.RZ(toptiontshort_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_short_reads_types + cCs;x4tjD])}|jdr |d|kr |Sq WdS(Ns--(R7RuRRZ(Rtlong_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_long_reads_typescCs;x4tjD])}|jdr |d|kr |Sq WdS(Ns--(R7tGRAPH_READS_TYPESRRZ(Rtgraph_reads_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_graph_type"scCs|jdo|djS(Ns--si(RR.(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytis_single_read_type)scCsd}d}t|rEt|}ttjd|j}n6t|r`t|}nt|r{t|}n||fS(Ntpeis\d+(RR0RtsearchtgroupRR (Rtlib_typet lib_number((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_lib_type_and_number-s  !  cCs|jdrd}n|jdr0d}n|jdrHd}ni|jds{t|s{t|s{t|rd}n-|jd s|jd rd }nd }|S( Ns-12sinterlaced readss-1s left readss-2s right readss-ss single readss-ms-mergeds merged readst orientation(R9R RR (Rt data_type((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt get_data_type<s        cCsid}d|kr_d||jd tjkr_||jd }||jdd}n||fS(Nt:R)i(RZRR7Rr(RR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_option_prefixOs ,cCs,t|\}}d||f}t|}|dkrG|d}n||kri||<|||d<|tjkrtj|||dt itertoolstisliceRRWRXR-R[RR\R]R^R_RR/(R,RR7RStmax_reads_lengthtrecRa((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyR0sJ$c Csg}xt|D]x\}}d}d}d|krH|d|d-s, --mp<#>-s, --hqmp<#>-s, or --s<#> option instead of -s!(tonly_old_style_optionstold_style_single_readsR(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytcheck_single_reads_in_optionss cCset|tk r|g}ng}x:t|D],\}}|d|kr1|j|q1q1W|S(NR(RRRDR:(Rttypestlib_idsRGR$((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_lib_ids_by_types cCs;t||}g}x|D]}|j||qW|S(N(RPR:(RRNtidstresultRG((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytget_libs_by_type s  cCs7t||}x!t|dtD] }||=q"W|S(NR(RPRR (RRNRQRG((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytrm_libs_by_types cCsx|D]}|rtSqWtS(N(RR (RR$((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytdataset_is_emptys cCs`xY|D]Q}xH|D]@}|jdrx(||D]}|jdr4tSq4WqqWqWtS(NRs.gz(R9R R(RR$RR,((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytdataset_has_gzipped_reads s  cCs%x|D]}d|krtSqWtS(Nsinterlaced reads(R R(RR$((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytdataset_has_interlaced_reads*s  cCs,x%|D]}|djdrtSqWtS(NRRo(R9R R(RR$((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytdataset_has_additional_contigs1s s cCsdddddg}xt|D]\}}|j|d|d|dfd |kr{|jd ||d fnxM|D]E}||krd }nt||}|jd |||fqWq"WdS( Ns left readss right readssinterlaced readss single readss merged readss$Library number: %d, library type: %siRRs%s orientation: %ss not specifieds %s %s: %s(RDRR/(RRtindentt READS_TYPESRGR$t reads_typeR*((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytpretty_print_reads8s#    cCsg}g}t}d}|r0tj|}n t|}x|D]}t||odtjjd}|svqCn|ddkr|j|j|s|j|nt }d}qC||j7}qCW|j||j t ||S(NR"s3.it>( R tgzipRRR R1RR:RRRtzip(RNtgzippedtres_nametres_seqtfirsttseqt file_handlerR((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt read_fastaIs*  !   c Cst|dn}xd|D]\\}}|j|dx<tdt|dD]"}|j|||d!dqOWqWWdQXdS(NRs ii<(RRR,R-(RNtfastatoutfiletnameRdR3((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt write_fastads tNcCsg}t}xtt||D]\}\}}d} d} d} x-| t|kru|jd| dkru|dkrt}n|jd| } | d} x0| t|kr|| dkr| d7} qW| d} | | |krIt}| | kri|j|jddt| ddj |jd|| | !j d|f| d7} n| } qIqIW| t|kr"|j|jddt| ddj |jd|| j d|fq"q"W||fS(NiiRkit_R( RRDRfR-RR R:R*R/RR;(RLt thresholdt replace_charR`t new_fastatmodifiedRGRiRdR3tcur_contig_numbertcur_contig_starttstarttend((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytbreak_scaffoldsls2(-   %  <  <!cCs1idd6dd6dd6dd6dd6|jS(NtTtAtGtCRk(tupper(tletter((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytcompscCs&djtjt|dddS(NR"i(RR@timapR|(Rd((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytrev_compscCss|jd}t|dksA|ddkrS|ddkrStd|dSd|krk|ddS|dS( NRliis>NODEtNODEscontig %s has unknown ID formatt'i(R*R-RRZ(tsR"((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt get_contig_ids2  cCs|jdr|dS|S(NR]i(R(R((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytremove_fasta_prefscCs+yt|tSWntk r&tSXdS(N(tfloatR RR(R*((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytis_floats   cCs+yt|tSWntk r&tSXdS(N(R0R RR(R*((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pytis_ints   i'i'(eRR^R@RR2RRRRR RR^tplatformRtdistutils.versionRtos.pathRRRR7tcommonRRRRRRZRRR RKRRLRRR#R$R(RCRDRIRORQRbReRJRjRkRnR}RRRfR1RRRRRRRRR RRRRRR R RRRRR'R.RER=R?R/R0RJRMRPRSRTRURVRWRXR\RfRjRuR|R~RRRR(((sk/oak/stanford/groups/akundaje/marinovg/programs/SPAdes-3.15.4-Linux/share/spades/spades_pipeline/support.pyt s                        #    #7  0  %            &