ó ù`]c@s¬ddlmZmZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZdefd„ƒYZdS( iÿÿÿÿ(tunicode_literalstprint_functionN(t text_type(tZipFilePathPointer(tfind_dir(t TokenizerIt ReppTokenizercBs_eZdZdd„Zd„Zed„Zd„Zed„ƒZ ed„ƒZ d„Z RS( ué A class for word tokenization using the REPP parser described in Rebecca Dridan and Stephan Oepen (2012) Tokenization: Returning to a Long Solved Problem - A Survey, Contrastive Experiment, Recommendations, and Toolkit. In ACL. http://anthology.aclweb.org/P/P12/P12-2.pdf#page=406 >>> sents = ['Tokenization is widely regarded as a solved problem due to the high accuracy that rulebased tokenizers achieve.' , ... 'But rule-based tokenizers are hard to maintain and their rules language specific.' , ... 'We evaluated our method on three languages and obtained error rates of 0.27% (English), 0.35% (Dutch) and 0.76% (Italian) for our best models.' ... ] >>> tokenizer = ReppTokenizer('/home/alvas/repp/') # doctest: +SKIP >>> for sent in sents: # doctest: +SKIP ... tokenizer.tokenize(sent) # doctest: +SKIP ... (u'Tokenization', u'is', u'widely', u'regarded', u'as', u'a', u'solved', u'problem', u'due', u'to', u'the', u'high', u'accuracy', u'that', u'rulebased', u'tokenizers', u'achieve', u'.') (u'But', u'rule-based', u'tokenizers', u'are', u'hard', u'to', u'maintain', u'and', u'their', u'rules', u'language', u'specific', u'.') (u'We', u'evaluated', u'our', u'method', u'on', u'three', u'languages', u'and', u'obtained', u'error', u'rates', u'of', u'0.27', u'%', u'(', u'English', u')', u',', u'0.35', u'%', u'(', u'Dutch', u')', u'and', u'0.76', u'%', u'(', u'Italian', u')', u'for', u'our', u'best', u'models', u'.') >>> for sent in tokenizer.tokenize_sents(sents): # doctest: +SKIP ... print sent # doctest: +SKIP ... (u'Tokenization', u'is', u'widely', u'regarded', u'as', u'a', u'solved', u'problem', u'due', u'to', u'the', u'high', u'accuracy', u'that', u'rulebased', u'tokenizers', u'achieve', u'.') (u'But', u'rule-based', u'tokenizers', u'are', u'hard', u'to', u'maintain', u'and', u'their', u'rules', u'language', u'specific', u'.') (u'We', u'evaluated', u'our', u'method', u'on', u'three', u'languages', u'and', u'obtained', u'error', u'rates', u'of', u'0.27', u'%', u'(', u'English', u')', u',', u'0.35', u'%', u'(', u'Dutch', u')', u'and', u'0.76', u'%', u'(', u'Italian', u')', u'for', u'our', u'best', u'models', u'.') >>> for sent in tokenizer.tokenize_sents(sents, keep_token_positions=True): # doctest: +SKIP ... print sent # doctest: +SKIP ... [(u'Tokenization', 0, 12), (u'is', 13, 15), (u'widely', 16, 22), (u'regarded', 23, 31), (u'as', 32, 34), (u'a', 35, 36), (u'solved', 37, 43), (u'problem', 44, 51), (u'due', 52, 55), (u'to', 56, 58), (u'the', 59, 62), (u'high', 63, 67), (u'accuracy', 68, 76), (u'that', 77, 81), (u'rulebased', 82, 91), (u'tokenizers', 92, 102), (u'achieve', 103, 110), (u'.', 110, 111)] [(u'But', 0, 3), (u'rule-based', 4, 14), (u'tokenizers', 15, 25), (u'are', 26, 29), (u'hard', 30, 34), (u'to', 35, 37), (u'maintain', 38, 46), (u'and', 47, 50), (u'their', 51, 56), (u'rules', 57, 62), (u'language', 63, 71), (u'specific', 72, 80), (u'.', 80, 81)] [(u'We', 0, 2), (u'evaluated', 3, 12), (u'our', 13, 16), (u'method', 17, 23), (u'on', 24, 26), (u'three', 27, 32), (u'languages', 33, 42), (u'and', 43, 46), (u'obtained', 47, 55), (u'error', 56, 61), (u'rates', 62, 67), (u'of', 68, 70), (u'0.27', 71, 75), (u'%', 75, 76), (u'(', 77, 78), (u'English', 78, 85), (u')', 85, 86), (u',', 86, 87), (u'0.35', 88, 92), (u'%', 92, 93), (u'(', 94, 95), (u'Dutch', 95, 100), (u')', 100, 101), (u'and', 102, 105), (u'0.76', 106, 110), (u'%', 110, 111), (u'(', 112, 113), (u'Italian', 113, 120), (u')', 120, 121), (u'for', 122, 125), (u'our', 126, 129), (u'best', 130, 134), (u'models', 135, 141), (u'.', 141, 142)] uutf8cCs.|j|ƒ|_tjƒ|_||_dS(N(tfind_repptokenizertrepp_dirttempfilet gettempdirt working_dirtencoding(tselfRR ((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyt__init__<scCst|j|gƒƒS(uÈ Use Repp to tokenize a single sentence. :param sentence: A single sentence string. :type sentence: str :return: A tuple of tokens. :rtype: tuple(str) (tnextttokenize_sents(R tsentence((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyttokenizeCs c csÖtjddd|jdddtƒ©}x%|D]}|jt|ƒdƒq1W|jƒ|j|jƒ}|j |ƒj |j ƒj ƒ}x:|j |ƒD])}|sÃt|Œ\}}} n|VqŸWWdQXdS( uà Tokenize multiple sentences using Repp. :param sentences: A list of sentence strings. :type sentences: list(str) :return: A list of tuples of tokens :rtype: iter(tuple(str)) tprefixu repp_input.tdirtmodeuwtdeleteu N(R tNamedTemporaryFileR tFalsetwriteRtclosetgenerate_repp_commandtnamet_executetdecodeR tstriptparse_repp_outputstzip( R t sentencestkeep_token_positionst input_filetsenttcmdt repp_outputttokenized_senttstartstends((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyRNs   !cCsH|jdg}|d|jdg7}|ddg7}||g7}|S(u« This module generates the REPP command to be used at the terminal. :param inputfilename: path to the input file :type inputfilename: str u /src/reppu-cu /erg/repp.setu--formatutriple(R(R t inputfilenameR&((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyRhs  cCs7tj|dtjdtjƒ}|jƒ\}}|S(Ntstdouttstderr(t subprocesstPopentPIPEt communicate(R&tpR,R-((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyRus!ccsŽtjdtjƒ}xr|jdƒD]a}g|j|ƒD]*\}}}|t|ƒt|ƒf^q;}td„|Dƒƒ}|Vq%WdS(uZ This module parses the tri-tuple format that REPP outputs using the "--format triple" option and returns an generator with tuple of string tokens. :param repp_output: :type repp_output: type :return: an iterable of the tokenized sentences as tuples of strings :rtype: iter(tuple) u^\((\d+), (\d+), (.+)\)$u css|]}|dVqdS(iN((t.0tt((s1lib/python2.7/site-packages/nltk/tokenize/repp.pys sN(tretcompilet MULTILINEtsplittfindalltintttuple(R't line_regextsectiontstarttendttokentwords_with_positionstwords((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyR {s =cCsitjj|ƒr|}nt|ddƒ}tjj|dƒsIt‚tjj|dƒset‚|S(uX A module to find REPP tokenizer binary and its *repp.set* config file. tenv_varsuREPP_TOKENIZERu /src/reppu /erg/repp.set(uREPP_TOKENIZER(tostpathtexistsRtAssertionError(R t repp_dirnamet _repp_dir((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyRs  ( t__name__t __module__t__doc__RRRRRt staticmethodRR R(((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyRs   (t __future__RRRDR5tsysR.R tsixRt nltk.dataRtnltk.internalsRtnltk.tokenize.apiRR(((s1lib/python2.7/site-packages/nltk/tokenize/repp.pyt s