B A!p\:@sdZddlmZddlZddlmZddlmZddlZddl m Z ddl m Z ddl mZmZdd lmZGd d d Zd d ZGdddZddiZddZGdddeeZddZdddZdddZedkredS)zGa similarities / code duplication command line tool and pylint checker )print_functionN) defaultdict)groupby)decoding_stream) IRawChecker) BaseCheckertable_lines_from_stats)Tablec@sLeZdZdZdddZdddZd d Zd d Zd dZddZ ddZ dS)Similarz,finds copy-pasted lines of code in a projectFcCs"||_||_||_||_g|_dS)N) min_linesignore_commentsignore_docstringsignore_importslinesets)selfr r rrr6lib/python3.7/site-packages/pylint/checkers/similar.py__init__$s zSimilar.__init__NcCsZ|dkr|j}n t||j}y$|jt|||j|j|jWntk rTYnXdS)z(append a file to search for similaritiesN) readlinesrrappendLineSetr rrUnicodeDecodeError)rZstreamidstreamencodingrrrr append_stream1s zSimilar.append_streamcCs||dS)zisz)Similar._display_sims..z)TOTAL lines=%s duplicates=%s percent=%.2fgY@N)printr,sortedname _real_linesrstripsumr) rr)Znb_lignes_dupliqueesr%r(r.idxlineZnb_total_lignesrrrr\s zSimilar._display_simsccs|j}|j}|j}d}|j}x|t|krd}d} x|||D]} d} xtt|||| D]L\} \\} } \} }| |kr| |kr| |||| fVt|| }P| r`| d7} q`W| d7} | |kr| |||| fVt|| }q@W||7}qWdS)z+find similarities in the two given linesetsrr+N)enumerate_strippedfindr r, enumeratezipmax)rr&r'Zlines1Zlines2r9Zindex1r skipr%Zindex2Z non_blank_Zline1Zline2rrr _find_commonss0(  zSimilar._find_commonccs\xVt|jddD]@\}}x6|j|ddD] }x|||D] }|VqBWq0WqWdS)zWiterate on similarities among all files, by making a cartesian product Nr+)r:rr?)rr6r.r'simrrrr szSimilar._iter_sims)r FFF)N) __name__ __module__ __qualname____doc__rrrrrr?r rrrrr !s  r c Cs|rBtd|}dd|jD}ddt|dddD}d }g}d } xt|d d D]\} } | } |r| s| d s| dr| d d} | dd } | r| | rd } d} |r| | |}|rd} |r| dd d} | | qXW|S)z\return lines with leading/trailing whitespace and any ignored code features removed css&|]}|jt|tjtjffVqdS)N)lineno isinstanceastroidZImportZ ImportFrom)r-noderrr sz!stripped_lines..cSs$i|]\}}tdd|D|qS)css|]\}}|VqdS)Nr)r-r>Z is_importrrrrKsz,stripped_lines...)all)r-rGZnode_is_import_grouprrr sz"stripped_lines..cSs|dS)Nrr)xrrrsz stripped_lines..)keyFNr+)startz"""z'''#r) rIparsejoinZbodyrr:strip startswithendswithgetsplitr) linesr rrZtreeZnode_is_import_by_linenoZline_begins_importZcurrent_line_is_importZ strippedlinesZ docstringrGr7rrrstripped_liness:    r\c@s\eZdZdZdddZddZddZd d Zd d Zd dZ dddZ ddZ ddZ dS)rz7Holds and indexes all the lines of a single source fileFcCs*||_||_t|||||_||_dS)N)r2r3r\_stripped_lines _mk_index_index)rr2r[r rrrrrrs zLineSet.__init__cCs d|jS)Nz)r2)rrrr__str__szLineSet.__str__cCs t|jS)N)r,r3)rrrr__len__szLineSet.__len__cCs |j|S)N)r])rindexrrr __getitem__szLineSet.__getitem__cCs |j|jkS)N)r2)rotherrrr__lt__szLineSet.__lt__cCst|S)N)id)rrrr__hash__szLineSet.__hash__rccsB|}|r|j|d}n|j}x|D]}||fV|d7}q$WdS)zgreturn an iterator on stripped lines, starting from a given index if specified, else 0 Nr+)r])rZstart_atr6r[r7rrrr8s  zLineSet.enumerate_strippedcCs|j|dS)z7return positions of the given stripped line in this setr)r_rY)rZ stripped_linerrrr9sz LineSet.findcCs6tt}x(t|jD]\}}|r|||qW|S)zcreate the index for this set)rrr:r]r)rrbZline_nor7rrrr^s zLineSet._mk_indexN)FFF)r) rBrCrDrErr`rarcrergr8r9r^rrrrrs  rR0801)zSimilar lines in %s files %szduplicate-codezIndicates that a set of similar lines has been detected among multiple file. This usually means that the code should be refactored to avoid this duplication.cCs6ddddg}|t||d7}|t|ddddd S) z/make a layout with some stats about duplicationrFZnowZprevious difference)nb_duplicated_linespercent_duplicated_linesr r+)ZchildrenZcolsZrheadersZcheadersN)rrr )ZsectstatsZ old_statsr[rrrreport_similaritiess  rmc @seZdZdZefZdZeZddddddfd d d d d dfdd d d ddfddd d ddffZ dde ffZ d ddZ d!ddZ ddZddZddZdS)"SimilarCheckerzchecks for similarities and duplicated code. This computation may be memory / CPU intensive, so you should disable it if you experiment some problems. Z similaritieszmin-similarity-linesr intzz%Minimum lines number of a similarity.)defaulttypemetavarhelpzignore-commentsTZynzz,Ignore comments when computing similarities.zignore-docstringsz.Ignore docstrings when computing similarities.zignore-importsFz+Ignore imports when computing similarities.ZRP0801Z DuplicationNcCs(t||tj|ddddd|_dS)Nr T)r r r)rrr rl)rlinterrrrrNs zSimilarChecker.__init__cCsdt||||||dkr&|jj|_n:|dkr:|jj|_n&|dkrN|jj|_n|dkr`|jj|_dS)zmethod called to set an option (registered in the options list) overridden to report options setting to Similar zmin-similarity-lineszignore-commentszignore-docstringszignore-importsN)r set_optionZconfigZmin_similarity_linesr r rr)rZoptnamevalueactionZoptdictrrrruUs   zSimilarChecker.set_optioncCsg|_|jjddd|_dS)z.rz==%s:%srh )argsr+rjgY@rkN) r5rrlrrr2r#r3r4Z add_messager,rU) rZtotalZ duplicatedrlr%r(msgr.r6r7rrrcloseuszSimilarChecker.close)N)NN)rBrCrDrErZ__implements__r2MSGSZmsgsZoptionsrmZreportsrrurxrzr~rrrrrns<     rncCs|t|dS)z.required method to auto register this checker N)Zregister_checkerrn)rtrrrregistersrcCs$tdttdt|dS)z&display command line usage informationz*finds copy pasted blocks in a set of filesz~Usage: symilar [-d|--duplicates min_duplicated_lines] [-i|--ignore-comments] [--ignore-docstrings] [--ignore-imports] file1...N)r0sysexit)Zstatusrrrusages rc Cs|dkrtjdd}ddlm}d}d}d}d}d}d}||||\}} xZ|D]R\} } | d krjt| }qP| d krztqP| d krd }qP| d krd }qP| dkrPd }qPW| stdt||||} x,| D]$} t| }| | |WdQRXqW| t ddS)z$standalone command line access pointNr+r)getoptZhdi)rsz duplicates=zignore-commentszignore-importszignore-docstringsr F)z-dz --duplicates)z-hz--help)z-iz--ignore-commentsT)z--ignore-docstrings)z--ignore-imports) rargvrrorr rxrrr)rrZs_optsZl_optsr r rrZoptsr|ZoptvalrAfilenamerrrrRuns:    r__main__)r)N)rEZ __future__rr collectionsr itertoolsrrIZ pylint.utilsrZpylint.interfacesrZpylint.checkersrrZpylint.reporters.ureports.nodesr r r\rrrmrnrrrrBrrrrs*      z,=  p (