`]c@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z mZddlmZddlmZmZddlmZddlmZmZmZddlmZmZmZmZmZmZmZdd l m!Z!m"Z"dd l#Tdd l$m%Z%d d Z&dZ'dddZ)ddZ*dddZ+dZ,dZ-dZ.defdYZ/dddZ0dZ1e2ddZ3dZ4d Z5d!Z6e7d"Z8d#Z9d$Z:d%Z;d&Z<e7e7ddd'Z=e7e7ddd(Z>d)Z?d*Z@d+dd,ZAd-ZBidd.ZCdd/d0ZDdd1ZEd2ZFdS(3i(tprint_functionN(tislicetchaint combinations(tpprint(t defaultdicttdeque(t version_info(t class_typest string_typest text_type(t build_openertinstall_openert getproxiest ProxyHandlertProxyBasicAuthHandlertProxyDigestAuthHandlertHTTPPasswordMgrWithDefaultRealm(t slice_boundstraise_unorderable_types(t*(tpython_2_unicode_compatibletselfc Cs}t|t|ts%|j}ntd|jx@ttj|j D]#\}}|j drsqRnt |dt rqRnt jddkrtj}n tj}||d \}}}}|r&|ddkr&|dks t|t|kr&|d}d ||f}ntj||||} ttjd || fd d d dt|dqRWdS(Ns%%s supports the following operations:t_t__deprecated__iiiRis%s.%ss%s%stinitial_indents - tsubsequent_indentt i(tstrt isinstanceRt __class__tprintt__name__tsortedtpydoct allmethodstitemst startswithtgetattrtFalsetsysRtinspecttgetfullargspect getargspectNonetlent formatargspecttextwraptfill( tobjtselfnametnametmethodR+targstvarargstvarkwtdefaultstargspec((s(lib/python2.7/site-packages/nltk/util.pytusage-s0  (  $ cCsddl}|jjjdkS(s Return True if this function is run within idle. Tkinter programs that are run in idle should never call ``Tk.mainloop``; so this function should be used to gate all calls to ``Tk.mainloop``. :warning: This function works by checking ``sys.stdin``. If the user has modified ``sys.stdin``, then it may return incorrect results. :rtype: bool iNtPyShelltRPCProxy(R;R<(R(tstdinRR (R(((s(lib/python2.7/site-packages/nltk/util.pytin_idleUs icCs ttt|||dS(s Pretty print a sequence of data items :param data: the data stream to print :type data: sequence or iter :param start: the start position :type start: int :param end: the end position :type end: int N(RtlistR(tdatatstarttend((s(lib/python2.7/site-packages/nltk/util.pytprjs iFcCs&tdjtj|d|dS(s Pretty print a string, breaking lines on whitespace :param s: the string to print, consisting of words and spaces :type s: str :param width: the display width :type width: int s twidthN(RtjoinR/twrap(tsRD((s(lib/python2.7/site-packages/nltk/util.pyt print_stringxs RcCs%djtj|j|d|S(s# Pretty print a list of text tokens, breaking lines on whitespace :param tokens: the tokens to print :type tokens: list :param separator: the string to use to separate tokens :type separator: str :param width: the display width (default=70) :type width: int s RD(RER/RF(ttokenst separatorRD((s(lib/python2.7/site-packages/nltk/util.pyt tokenwraps cCstddkotddkS(Niiii(R(((s(lib/python2.7/site-packages/nltk/util.pytpy25scCstddkotddkS(Niiii(R(((s(lib/python2.7/site-packages/nltk/util.pytpy26scCstddkotddkS(Niiii(R(((s(lib/python2.7/site-packages/nltk/util.pytpy27stIndexcBseZdZRS(cCs<tj|tx%|D]\}}||j|qWdS(N(Rt__init__R?tappend(Rtpairstkeytvalue((s(lib/python2.7/site-packages/nltk/util.pyRPs(R t __module__RP(((s(lib/python2.7/site-packages/nltk/util.pyROst{t}cCs7ttj|tjj|d||jdS(s3 Return a string with markers surrounding the matched substrings. Search str for substrings matching ``regexp`` and wrap the matches with braces. This is convenient for learning about regular expressions. :param regexp: The regular expression. :type regexp: str :param string: The string being matched. :type string: str :param left: The left delimiter (printed before the matched substring) :type left: str :param right: The right delimiter (printed after the matched substring) :type right: str :rtype: str s\g<0>N(RtretcompiletMtsubtrstrip(tregexptstringtlefttright((s(lib/python2.7/site-packages/nltk/util.pytre_showscCs]t|dr|jSt|trMt|d}|jSWdQXn tddS(Ntreadtrs2Must be called with a filename or file-like object(thasattrRbRR topent ValueError(tftinfile((s(lib/python2.7/site-packages/nltk/util.pyt filestrings  c#st|dfg}xk|r|j\}|V|kry'|jfd||DWqtk r{qXqqWdS(sTraverse the nodes of a tree in breadth-first order. (No need to check for cycles.) The first argument should be the tree root; children should be a function taking as argument a tree node and returning an iterator of the node's children. ic3s|]}|dfVqdS(iN((t.0tc(tdepth(s(lib/python2.7/site-packages/nltk/util.pys sN(Rtpoplefttextendt TypeError(ttreetchildrentmaxdepthtqueuetnode((Rls(lib/python2.7/site-packages/nltk/util.pyt breadth_firsts  ' cCsWd}dg}y|jtjtjWntk r?nXy|jtjdWnttfk rtnXy|jtjdWnttfk rnX|jdxN|D]F}|sqnyt ||}|}Wnt t fk rqXPqW|sIt ddj g|D]}|r!t |^q!n ||fSdS(st Given a byte string, attempt to decode it. Tries the standard 'UTF8' and 'latin-1' encodings, Plus several gathered from locale information. The calling program *must* first call:: locale.setlocale(locale.LC_ALL, '') If successful it returns ``(decoded_unicode, successful_encoding)``. If unsuccessful it raises a ``UnicodeError``. sutf-8islatin-1s?Unable to decode input data. Tried the following encodings: %s.s, N(R,RQtlocalet nl_langinfotCODESETtAttributeErrort getlocalet IndexErrortgetdefaultlocaleR t UnicodeErrort LookupErrorREtrepr(R@tsuccessful_encodingt encodingstenctdecoded((s(lib/python2.7/site-packages/nltk/util.pytguess_encodings:      5cCs<t}g|D](}||kr|j| r|^qS(N(tsettadd(txstseentx((s(lib/python2.7/site-packages/nltk/util.pyt unique_list4s cCsktt}xX|D]P}t||drUx4||D]}||j|q7Wq||||_tcSstS(N(R(R((s(lib/python2.7/site-packages/nltk/util.pyRaRc3s%|]}||jfVqdS(N(tcopy(RjR(tgraph(s(lib/python2.7/site-packages/nltk/util.pys csc3s!|]}||fVqdS(N((RjR(tbase_set(s(lib/python2.7/site-packages/nltk/util.pys es(tdicttpopRt setdefaulttget(Rt reflexivet agenda_grapht closure_graphtitagendatclosuretj((RRs(lib/python2.7/site-packages/nltk/util.pyttransitive_closurePs        cCsLi}x?|D]7}x.||D]"}|j|tj|qWq W|S(s Inverts a directed graph. :param graph: the graph, represented as a dictionary of sets :type graph: dict(set) :return: the inverted graph :rtype: dict(set) (RRR(RtinvertedRSRT((s(lib/python2.7/site-packages/nltk/util.pyt invert_graphrs  $cCstddS(Ns>To remove HTML markup, use BeautifulSoup's get_text() function(tNotImplementedError(thtml((s(lib/python2.7/site-packages/nltk/util.pyt clean_htmlscCstddS(Ns>To remove HTML markup, use BeautifulSoup's get_text() function(R(turl((s(lib/python2.7/site-packages/nltk/util.pyt clean_urlscGsg}xx|D]p}t|ttfs4|g}nxF|D]>}t|ttfrl|jt|q;|j|q;Wq W|S(s Flatten a list. >>> from nltk.util import flatten >>> flatten(1, 2, ['b', 'a' , ['c', 'd']], 3) [1, 2, 'b', 'a', 'c', 'd', 3] :param args: items and lists to be combined into a single list :rtype: list (RR?ttupleRntflattenRQ(R5Rtltitem((s(lib/python2.7/site-packages/nltk/util.pyRs    cCsVt|}|r/t|f|d|}n|rRt||f|d}n|S(s Returns a padded sequence of items before ngram extraction. >>> list(pad_sequence([1,2,3,4,5], 2, pad_left=True, pad_right=True, left_pad_symbol='', right_pad_symbol='')) ['', 1, 2, 3, 4, 5, ''] >>> list(pad_sequence([1,2,3,4,5], 2, pad_left=True, left_pad_symbol='')) ['', 1, 2, 3, 4, 5] >>> list(pad_sequence([1,2,3,4,5], 2, pad_right=True, right_pad_symbol='')) [1, 2, 3, 4, 5, ''] :param sequence: the source data to be padded :type sequence: sequence or iter :param n: the degree of the ngrams :type n: int :param pad_left: whether the ngrams should be left-padded :type pad_left: bool :param pad_right: whether the ngrams should be right-padded :type pad_right: bool :param left_pad_symbol: the symbol to use for left padding (default is None) :type left_pad_symbol: any :param right_pad_symbol: the symbol to use for right padding (default is None) :type right_pad_symbol: any :rtype: sequence or iter i(titerR(tsequencetntpad_leftt pad_righttleft_pad_symboltright_pad_symbol((s(lib/python2.7/site-packages/nltk/util.pyt pad_sequences c cst||||||}g}xL|dkroyt|}Wntk rTdSX|j||d8}q$Wx-|D]%}|j|t|V|d=qwWdS(s Return the ngrams generated from a sequence of items, as an iterator. For example: >>> from nltk.util import ngrams >>> list(ngrams([1,2,3,4,5], 3)) [(1, 2, 3), (2, 3, 4), (3, 4, 5)] Wrap with list for a list version of this function. Set pad_left or pad_right to true in order to get additional ngrams: >>> list(ngrams([1,2,3,4,5], 2, pad_right=True)) [(1, 2), (2, 3), (3, 4), (4, 5), (5, None)] >>> list(ngrams([1,2,3,4,5], 2, pad_right=True, right_pad_symbol='')) [(1, 2), (2, 3), (3, 4), (4, 5), (5, '')] >>> list(ngrams([1,2,3,4,5], 2, pad_left=True, left_pad_symbol='')) [('', 1), (1, 2), (2, 3), (3, 4), (4, 5)] >>> list(ngrams([1,2,3,4,5], 2, pad_left=True, pad_right=True, left_pad_symbol='', right_pad_symbol='')) [('', 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, '')] :param sequence: the source data to be converted into ngrams :type sequence: sequence or iter :param n: the degree of the ngrams :type n: int :param pad_left: whether the ngrams should be left-padded :type pad_left: bool :param pad_right: whether the ngrams should be right-padded :type pad_right: bool :param left_pad_symbol: the symbol to use for left padding (default is None) :type left_pad_symbol: any :param right_pad_symbol: the symbol to use for right padding (default is None) :type right_pad_symbol: any :rtype: sequence or iter iNi(Rtnextt StopIterationRQR( RRRRRRthistoryt next_itemR((s(lib/python2.7/site-packages/nltk/util.pytngramss+     cks&xt|d|D] }|VqWdS(s Return the bigrams generated from a sequence of items, as an iterator. For example: >>> from nltk.util import bigrams >>> list(bigrams([1,2,3,4,5])) [(1, 2), (2, 3), (3, 4), (4, 5)] Use bigrams for a list version of this function. :param sequence: the source data to be converted into bigrams :type sequence: sequence or iter :rtype: iter(tuple) iN(R(RtkwargsR((s(lib/python2.7/site-packages/nltk/util.pytbigramsscks&xt|d|D] }|VqWdS(s Return the trigrams generated from a sequence of items, as an iterator. For example: >>> from nltk.util import trigrams >>> list(trigrams([1,2,3,4,5])) [(1, 2, 3), (2, 3, 4), (3, 4, 5)] Use trigrams for a list version of this function. :param sequence: the source data to be converted into trigrams :type sequence: sequence or iter :rtype: iter(tuple) iN(R(RRR((s(lib/python2.7/site-packages/nltk/util.pyttrigrams3sicks_|dkrt|}nx=t||dD](}xt|||D] }|VqHWq/WdS(s Returns all possible ngrams generated from a sequence of items, as an iterator. >>> sent = 'a b c'.split() >>> list(everygrams(sent)) [('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c'), ('a', 'b', 'c')] >>> list(everygrams(sent, max_len=2)) [('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c')] :param sequence: the source data to be converted into trigrams :type sequence: sequence or iter :param min_len: minimum length of the ngrams, aka. n-gram order/degree of ngram :type min_len: int :param max_len: maximum length of the ngrams (set to length of sequence by default) :type max_len: int :rtype: iter(tuple) iiN(R-trangeR(Rtmin_lentmax_lenRRtng((s(lib/python2.7/site-packages/nltk/util.pyt everygramsGs  c ksd|ksd|kr-t|||}nt}xxt|||dtd|D]W}|d }|d}x:t||dD]%}|d|krqn||VqWqVWdS(s Returns all possible skipgrams generated from a sequence of items, as an iterator. Skipgrams are ngrams that allows tokens to be skipped. Refer to http://homepages.inf.ed.ac.uk/ballison/pdf/lrec_skipgrams.pdf >>> sent = "Insurgents killed in ongoing fighting".split() >>> list(skipgrams(sent, 2, 2)) [('Insurgents', 'killed'), ('Insurgents', 'in'), ('Insurgents', 'ongoing'), ('killed', 'in'), ('killed', 'ongoing'), ('killed', 'fighting'), ('in', 'ongoing'), ('in', 'fighting'), ('ongoing', 'fighting')] >>> list(skipgrams(sent, 3, 2)) [('Insurgents', 'killed', 'in'), ('Insurgents', 'killed', 'ongoing'), ('Insurgents', 'killed', 'fighting'), ('Insurgents', 'in', 'ongoing'), ('Insurgents', 'in', 'fighting'), ('Insurgents', 'ongoing', 'fighting'), ('killed', 'in', 'ongoing'), ('killed', 'in', 'fighting'), ('killed', 'ongoing', 'fighting'), ('in', 'ongoing', 'fighting')] :param sequence: the source data to be converted into trigrams :type sequence: sequence or iter :param n: the degree of the ngrams :type n: int :param k: the skip distance :type k: int :rtype: iter(tuple) RRRiiN(RtobjectRtTrueR( RRRRtSENTINELtngramtheadttailt skip_tail((s(lib/python2.7/site-packages/nltk/util.pyt skipgramsas &  c CsC|d}t|}d}d}t|drMtj|jjd}n-|jdd|jd}|jdx||kr>||f}||d} |j| r|| \} } nd} xt rT|jt d| d| dkr|j n|j} |j } | dkr/Pn|| d} | |dkrdSqW||krt| | f|| >> choose(4, 2) 6 >>> choose(6, 2) 15 :param n: The number of things. :type n: int :param r: The number of times a thing is taken. :type r: int iiN(ii(Rtmin(RRtntoktktoktt((s(lib/python2.7/site-packages/nltk/util.pytchoose s '  (Gt __future__RR(R)RvRXttypesR/R"tbisectRt itertoolsRRRRt collectionsRRRtsixRR R tsix.moves.urllib.requestR R R RRRRtnltk.internalsRRtnltk.collectionst nltk.compatRR:R>R,RCRHRKRLRMRNRORaRiRRuRRRR'RRRRRRRRRRRRRRR(((s(lib/python2.7/site-packages/nltk/util.pytsl         4   (        =  "    ( 9   +J*