,\c@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl Z ddlmZmZmZddlmZdZdZdZdZdadadeed Zddd Zd Zd Zd Z ddedZ!ddedZ"ddZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*ej+j,ddZ-dZ.dddZ/ddddZ0dZ1e2dZ3d Z4ed!Z5d"Z6d#Z7d$Z8ddd%Z9d&Z:d'Z;d(Z<d)Z=d*Z>d+Z?d,Z@d-ZAd.ZBdNd1d2ZCd3ZDd4ZEd5ZFd6ZGd7ZHd8ZId9ZJd:ZKd;ZLd<ZMd=ZNd>ZOd?ZPd@ZQdAZRdBdCZSedDZTdEZUdFedGZVdHZWdIZXdJZYdKZZdLZ[dMZ\dS(OiN(tSeqIO(tabspatht expandusertjoin(t LooseVersions == Error == s== Warning == s ERROR s WARN cCs0|sd}nd}|r|jd|d|t|dt|jd|d|jd|jd |jd nstjjd|d|dtjjd|d tjjd tjjd |jd tjjtrt j j trt j tntjddS(NtSPAdest dipSPAdess t t with_errors# In case you have troubles running s-, you can write to spades.support@cab.spbu.rusCor report an issue on our GitHub repository github.com/ablab/spadess&Please provide us with params.txt and s%.log files from the output directory.s., you can write to spades.support@cab.spbu.ru sDor report an issue on our GitHub repository github.com/ablab/spades s&.log files from the output directory. i(tinfot log_warningstTruetlowertsyststderrtwritetflushtcurrent_tmp_dirtostpathtisdirtshutiltrmtreetexit(terr_strtlogt dipspadestprefixt binary_name((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyterror%s"    cCsW|r&|jd|d|dn-tjjd|d|dtjjdS(Ns Rs (R R tstdoutRR(twarn_strRR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytwarning;s  cCsd}tjjd}g}xtjD]}|d}d|kr`|jd\}}n-|jdr|d |}}n |}}||}|jd||jddft|t|kot|knr,t Sq,Wt d|d d d j |dS( NcSsq|jd}xRttt|D]8}||jr(tt||d||x7|D]/}|j|dkrtd||qqWdS(Nis$file %s was specified at least twice(tcountR(t filenamesRRG((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytcheck_files_duplications cCs|tjkrtj|}ntjj|d}|jdkrtjj|t| d}||jtjkr||}qtjj|t|| d}||}n|jtjkrtddj tjd||f|n| r\|jtj kr\|tj kr\tddj tj d||f|n|j dr|jtj krtd|d dj tj d||f|ndS( Nis.gzs-file with reads has unsupported format (only s, s are supported): %s (%s)s?to run read error correction, reads should be in FASTQ format (tcontigss file with s should be in FASTA format ((R/tdict_of_prefixesRRtsplitextR R%tALLOWED_READS_EXTENSIONSRRtBH_ALLOWED_READS_EXTENSIONStLONG_READS_TYPESR1t CONTIGS_ALLOWED_READS_EXTENSIONS(RGRFtonly_assemblert library_typeRtexttpre_extt pre_pre_ext((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytcheck_reads_file_formats*! % $cCsd}tjj|\}}|r:||r|SnjdtjkrxXtjdjtjD]:}|jd}tjj||}||rc|SqcWndS(NcSs%tjj|o$tj|tjS(N(RRR=taccesstX_OK(tfpath((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytis_exestPATHt"(RRR"tenvirontpathseptstripRtNone(tprogramReRdtfnameRtexe_file((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytwhichs     cCsd}d}tjj|ryXxQt|D]C}|j|r.t|t|jd}|d:}|Sq.WWqtk rdSt k rdSXndS(Ns /proc/meminfos MemTotal:iii( RRR=topent startswithR(R%R"t ValueErrorRktIOError(tmem_info_filenametavail_mem_headertlinet avail_mem((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_available_memorys     cCs?y|jdWn#tk r%tStk r6tSXtSdS(Ntascii(tencodetUnicodeDecodeErrortFalsetUnicodeEncodeErrorR (Rv((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyRLs  s3.cCs#|rt|djS|jS(Nsutf-8(R'trstrip(Rvt is_python3((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytprocess_readlinescCs*|jddkr&d|d}n|S(NRiRg(tfind(R'((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytprocess_spacessc CsQddl}ddl}t|tr0|}n|j|}|j|d|jd|jd|}d}xe|jst |j j }|r|r|j |q||d7}n|j dk roPqoqoWxP|j jD]?}t |}|r|r|j |q#||d7}qqW|j rMtd||j f|n|S(NiRRtcwdRBs s7system call for: "%s" finished abnormally, err code: %d(tshlext subprocesst isinstancetlistR"tPopentPIPEtSTDOUTtpollRRtreadlineR t returncodeRkt readlinesR( tcmdRRRRtcmd_listtproctoutputRv((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytsys_calls0   '  c Cs+ddl}ddl}t|tr0|}n|j|}|rWt|d}n |j}|rxt|d} n |j} |j|d|d| d|} |r| s| rx| jsC|st | j j } | r|j | qn|s-t | j j } | r-|j | q-n| jdk rPqqW|sx<| j jD](} | dkrZ|j t | qZqZWn|sx<| j jD](} | dkr|j t | qqWqn | j|r|jn|r| jn| jr'td|| jf|ndS( s Runs cmd and redirects stdout to out_filename (if specified), stderr to err_filename (if specified), or to log otherwise iNtwRRRRBs7system call for: "%s" finished abnormally, err code: %d(RRRRR"RpRRRRRRR RRRkRtwaittcloseR( RRt out_filenamet err_filenameRRRRRRRRv((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytuniversal_sys_call sN     !       cCs_t|d}|j|j|jtj|tjtjBtj Btj Btj BdS(Ntwb( RpRtreadRRtchmodtstattS_IWRITEtS_IREADtS_IXUSRtS_IXGRPtS_IXOTH(tdatatfileR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytsave_data_to_fileDs cCsd}|rt}t}n t}t}trt|d}|jt|j}x|jsv|j}q[W|j }|j t|dj }|j |} || |}nt|dj }g} g} d} x|D]} | j | r qn| j |dkr| | j |t|j}| j|dj} || |s| j| | qq| j |dkr| | j |t|j}| j} || |s| j| | qqqW| | fS(NcSs(x!|D]}|j|rtSqWtS(N(R1R R|(t list_to_checktsuffixtitem((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt already_savedLs trs * iRB(tSPADES_PY_WARN_MESSAGEtSPADES_WARN_MESSAGEtSPADES_PY_ERROR_MESSAGEtSPADES_ERROR_MESSAGEtcontinue_logfile_offsetRptseekRRjRRtindexRqRR%R3R2(t log_filenametwarningsRtspades_py_messagetspades_messaget continued_logtcontinued_stage_phrasetlines_to_checkt all_linestfailed_stage_indextspades_py_msgst spades_msgst IMPORTANT_MESSAGE_SUMMARY_PREFIXRvR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_important_messages_from_logKsF       ## cCs@d}x3|jdD]$}|jjdkr|j}qqW|S(Nthandlerst FileHandler(Rkt__dict__t __class__t__name__t baseFilename(Rtlog_fileth((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_logger_filename|s c Cst|}|stSx|jdD]}|jq$Wt|dt\}}|s\|r|rr|jdn |jdtjj tjj |d}t j |dd}|j ||jd|r|jd x|D]}|j|qWn|r7|jd x|D]}|j|qWn|jd ||j||rt|dt\} } |jd|jd x"| | D]}|j|qWntStS( NRRs? ======= SPAdes pipeline finished abnormally and WITH WARNINGS!s0 ======= SPAdes pipeline finished WITH WARNINGS!s warnings.logtmodeRRBs=== Pipeline warnings:s-=== Error correction and assembling warnings:s======= Warnings saved to s === ERRORs:(RR|RRRR R RRRRJtloggingRt addHandlert removeHandler( RRRRtspades_py_warnst spades_warnstwarnings_filenametwarnings_handlerRvtspades_py_errorst spades_errors((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyR s@   !         cCsYtjrUtt_t|}|rUt|d}|jdd|jaqUndS(NRii(R/t continue_modeR|RRpRttellR(RRR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytcontinue_from_heres   cCs!|jdtjtt_dS(Ns ======= Skipping the rest of SPAdes pipeline (--stop-after was set to '%s'). You can continue later with --continue or --restart-from options (R R/t stop_afterR t run_completed(R((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt finish_heres  csjdfd}d}xEttj|d|dtD]"}tjj|r@|}Pq@q@W|S(NcSs|jrt|S|S(N(R&R((ttext((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytatois  cs)gtjd|D]}|^qS(Ns(\d+)(treR"(Rtc(R(sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt natural_keysstkeytreverse(RktsortedtglobR RRR(tpatternRt latest_dirt dir_to_test((RsW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_latest_dirs (cCsP|stj}ntjj|s4tj|ntjd|d|atS(NtdirR( R/ttmp_dirRRRROttempfiletmkdtempR(Rtbase_dir((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt get_tmp_dirs  cCscx\tjjD]K}|jd|r|td|td|d!jr[|SqqWdS(Ns--i(R/tSHORT_READS_TYPEStkeysRqR%R&Rk(toptiontshort_reads_type((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_short_reads_types + cCs;x4tjD])}|jdr |d|kr |Sq WdS(Ns--(R/RZRqRk(Rtlong_reads_type((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_long_reads_typescCs|jdo|djS(Ns--si(RqR&(R((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytis_single_read_typescCsnd}d}t|rIt|}t|tjd|j}nt|rdt|}n||fS(Ntpeis\d(RR(RtsearchtstartR(Rtlib_typet lib_number((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_lib_type_and_numbers  % cCs|jdrd}n|jdr0d}nu|jdrHd}n]|jdsot|sot|rxd}n-|jd s|jd rd }nd }|S( Ns-12sinterlaced readss-1s left readss-2s right readss-ss single readss-ms-mergeds merged readst orientation(R1RR(Rt data_type((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt get_data_types   '  cCsrd}|jddkrhd||jd tjkrh||jd }||jdd}n||fS(Nt:iR!i(RkRR/RX(RR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_option_prefixs 5cCst|\}}t|}|dkr7|d}n|tjkrvtjttjjj||d}nL|tjkrtjt tjjtjj|}nt d|||s|tjkrtj|||d-s, --mp<#>-s, --hqmp<#>-s, or --s<#> option instead of -s!(R R|R/t reads_optionstOLD_STYLE_READS_OPTIONSR (toptionsRtonly_old_style_optionstold_style_single_readsR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytcheck_single_reads_in_optionss     cCset|tk r|g}ng}x:t|D],\}}|d|kr1|j|q1q1W|S(NR(RRR+R2(Rttypestlib_idsR-R ((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_lib_ids_by_types cCs;t||}g}x|D]}|j||qW|S(N(R9R2(RR7tidstresultR-((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytget_libs_by_types  cCs7t||}x!t|dtD] }||=q"W|S(NR(R9RR (RR7R:R-((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytrm_libs_by_types cCsx|D]}|rtSqWtS(N(R|R (RR ((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytdataset_is_emptys cCs`xY|D]Q}xH|D]@}|jdrx(||D]}|jdr4tSq4WqqWqWtS(NRs.gz(R1R R|(RR RR((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytdataset_has_gzipped_readss  cCs%x|D]}d|krtSqWtS(Nsinterlaced reads(R R|(RR ((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytdataset_has_interlaced_readss  cCs,x%|D]}|djdrtSqWtS(NRRU(R1R R|(RR ((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytdataset_has_additional_contigss cCs)x"|D]}|ddkrtSqWtS(NRtnxmate(R R|(RR ((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytdataset_has_nxmate_readss c Cst}x|D]}t|}|djdrg}xk|dD]_}|tjkrqtj|}|} ntjj|\} }t} |jdrt } |tjkrtjj| \} } qnt |tj ddd| \} } | rtjj |stj |ntjj|tjj| d}|tjkr^tj|=n|jd |tj |ft|| |j|qF|j|qFW||d= %d) Ns fragments (results are in %s directory)(RtdictR1R/RVRRRWR|R tbreak_scaffoldst)THRESHOLD_FOR_BREAKING_ADDITIONAL_CONTIGSRRORtbasenameR t write_fastaR2(RtdstRtnew_dataset_dataR tnew_reads_libraryt new_entryRUR^RJRFt_tmodifiedt new_fastat new_filename((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt process_Ns_in_additional_contigss@     %    c sdttfd}t}xw|D]o}t|}xM|jD]?\}}|dkrGd|krg|ds t@spThe length of sequence and quality lines should be the same! Check read %s (SEQ length is %d, QUAL length is %d)s+ (RkRRRRqRR%R( tin_filetout_filet read_nametis_fastqRt read_valueRvtfpost read_quality(R(sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytwrite_single_reads@  0  # sinterlaced readss left readss right readsis.gzRis.fqs.fastqs.fastat_1t_2s == Splitting s into left and right reads (in s directory)Rs3.is3The number of reads in file with interlaced reads (s) should be EVEN!(RkR|RRGR R/RVRRRWR1R tgzipRpRJR RqRRR=R R R)RRR2(RRLRR^RMR RNRRtinterlaced_readsR^twas_compressedt input_filet ungzippedt out_basenameRZtout_left_filenametout_right_filenamet out_filesR+tnext_read_name((RsW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytsplit_interlaced_readssj$        '$* -    cCs@yddl}t}x|D]}t|}|ddkr|d}|d}g|dNODEtNODEsContig %s has unknown ID formatt'ii(R"R%R RkR(tstvalues((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt get_contig_ids2 cCs|jdr|dS|S(NRUi(Rq(R((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytremove_fasta_prefscCs+yt|tSWntk r&tSXdS(N(tfloatR RrR|(R((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytis_floats   cCs+yt|tSWntk r&tSXdS(N(R(R RrR|(R((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pytis_ints   i'(]RRR RRRRaRRR/R$tcommonRRtos.pathRRRtdistutils.versionRRRRRRkRRR|RR R;R<RARHRKRCRPRQRTRaRoRxRLR)RqRRRRRR RRR RRRRRRRRRRRR RR!RRR0R6R9R<R=R>R?R@RARCRTRkRvRzRRKRHRRRRRR(((sW/srv/scratch/marinovg/spades-cloudspades-paper/assembler/src/spades_pipeline/support.pyt s                         #7  1  %                 % a