B P?[K@sddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZmZddlmZddlmZdd lmZd ZGd d d eZGd ddeZGdddeZGdddeZedddZdS))unicode_literalsN)skip)PIPE) text_type) find_jar_iter config_javajava _java_optionsfind_jars_within_path)ParserI)DependencyGraph)Treez2https://nlp.stanford.edu/software/lex-parser.shtmlc@sreZdZdZdZdZdZdZdZdd d Z d dZ dddZ dddZ dddZ dddZd ddZd!ddZdS)"GenericStanfordParserz Interface to the Stanford Parserz+stanford-parser-(\d+)(\.(\d+))+-models\.jarzstanford-parser\.jarz3edu.stanford.nlp.parser.lexparser.LexicalizedParserFN4edu/stanford/nlp/models/lexparser/englishPCFG.ser.gzutf8-mx4gc Cstt|j|ddt|ddddd}tt|j|ddt|ddd dd} tj|d } t| gt | |_ ||_ ||_ ||_ ||_dS) N)ZSTANFORD_PARSERSTANFORD_CORENLPT)Zenv_varsZ searchpathZurlverboseZis_regexcSs tj|S)N)ospathdirname) model_pathrr2lib/python3.7/site-packages/nltk/parse/stanford.pyCsz0GenericStanfordParser.__init__..)key)ZSTANFORD_MODELSrcSs tj|S)N)rrr)rrrrrPsr)maxr_JAR _stanford_url_MODEL_JAR_PATTERNrrsplittupler _classpathr _encodingcorenlp_options java_options) selfZ path_to_jarZpath_to_models_jarrencodingrr&r%Z stanford_jarZ model_jarZ stanford_dirrrr__init__-s4   zGenericStanfordParser.__init__c Csg}g}g}d}x|dD]~}|dkr|rD|t|g}d}q|jrj||d|g}d}q|t|d|gg}q||d}qWt|S)NFr T) splitlinesappenditer_DOUBLE_SPACED_OUTPUT _make_treejoin)r'Zoutput_ZresZ cur_linesZ cur_treesZblanklinerrr_parse_trees_output^s& z)GenericStanfordParser._parse_trees_outputc CsB|jd|jddd|jdddg }|||dd d |D|S) a Use StanfordParser to parse multiple sentences. Takes multiple sentences as a list where each sentence is a list of words. Each sentence will be automatically tagged with this StanfordParser instance's tagger. If whitespaces exists inside a token, then the token will be treated as separate tokens. :param sentences: Input sentences to parse :type sentences: list(list(str)) :rtype: iter(iter(Tree)) z-modelz -sentencesnewlinez -outputFormatz -tokenizedz-escaperz-edu.stanford.nlp.process.PTBEscapingProcessorr*css|]}d|VqdS) N)r0).0sentencerrr sz4GenericStanfordParser.parse_sents..) _MAIN_CLASSr_OUTPUT_FORMATr2_executer0)r' sentencesrcmdrrr parse_sentsusz!GenericStanfordParser.parse_sentscCst||g|S)a& Use StanfordParser to parse a sentence. Takes a sentence as a string; before parsing, it will be automatically tokenized and tagged by the Stanford Parser. :param sentence: Input sentence to parse :type sentence: str :rtype: iter(Tree) )nextraw_parse_sents)r'r6rrrr raw_parses zGenericStanfordParser.raw_parsecCs2|jd|jddd|jg}|||d||S)aI Use StanfordParser to parse multiple sentences. Takes multiple sentences as a list of strings. Each sentence will be automatically tokenized and tagged by the Stanford Parser. :param sentences: Input sentences to parse :type sentences: list(str) :rtype: iter(iter(Tree)) z-modelz -sentencesr3z -outputFormatr*)r8rr9r2r:r0)r'r;rr<rrrr?s z%GenericStanfordParser.raw_parse_sentscCst||g|S)a0 Use StanfordParser to parse a sentence. Takes a sentence as a list of (word, tag) tuples; the sentence must have already been tokenized and tagged. :param sentence: Input sentence to parse :type sentence: list(tuple(str, str)) :rtype: iter(Tree) )r>tagged_parse_sents)r'r6rrrr tagged_parses z"GenericStanfordParser.tagged_parsecsRd|jd|jddd|jdddd d d g}|||d fd d|D|S)ad Use StanfordParser to parse multiple sentences. Takes multiple sentences where each sentence is a list of (word, tag) tuples. The sentences must have already been tokenized and tagged. :param sentences: Input sentences to parse :type sentences: list(list(tuple(str, str))) :rtype: iter(iter(Tree)) /z-modelz -sentencesr3z -outputFormatz -tokenizedz -tagSeparatorz-tokenizerFactoryz,edu.stanford.nlp.process.WhitespaceTokenizerz-tokenizerMethodZnewCoreLabelTokenizerFactoryr*c3s&|]}dfdd|DVqdS)r4c3s|]}|VqdS)N)r0)r5Ztagged) tag_separatorrrr7szEGenericStanfordParser.tagged_parse_sents...N)r0)r5r6)rDrrr7sz;GenericStanfordParser.tagged_parse_sents..)r8rr9r2r:r0)r'r;rr<r)rDrrAs,  z(GenericStanfordParser.tagged_parse_sentsc Cs |j}|d|g|jr&||jdt}t|j|dtj ddd}t |t rf|rf| |}| |||jr|dt||j|ttd\}}n"||jt||jttd \}}|d d }|d d }||}WdQRXt|jt|dd|S) Nz -encodingr4)ZoptionsrwbF)modedeleter) classpathstdinstdoutstderr)rHrJrKs  s)r$extendr%r,r0r rr&tempfileZNamedTemporaryFile isinstancerencodewriteflush _USE_STDINseekrr#rnamereplacedecoderunlink) r'r<Zinput_rr(Zdefault_optionsZ input_filerJrKrrrr:s8          zGenericStanfordParser._execute)NNrrFrr)F)F)F)F)F)F)__name__ __module__ __qualname____doc__r rr8rSr.r)r2r=r@r?rBrAr:rrrrr#s( )   'rcs,eZdZdZdZfddZddZZS)StanfordParsera{ >>> parser=StanfordParser( ... model_path="edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz" ... ) >>> list(parser.raw_parse("the quick brown fox jumps over the lazy dog")) # doctest: +NORMALIZE_WHITESPACE [Tree('ROOT', [Tree('NP', [Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['quick']), Tree('JJ', ['brown']), Tree('NN', ['fox'])]), Tree('NP', [Tree('NP', [Tree('NNS', ['jumps'])]), Tree('PP', [Tree('IN', ['over']), Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['lazy']), Tree('NN', ['dog'])])])])])])] >>> sum([list(dep_graphs) for dep_graphs in parser.raw_parse_sents(( ... "the quick brown fox jumps over the lazy dog", ... "the quick grey wolf jumps over the lazy fox" ... ))], []) # doctest: +NORMALIZE_WHITESPACE [Tree('ROOT', [Tree('NP', [Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['quick']), Tree('JJ', ['brown']), Tree('NN', ['fox'])]), Tree('NP', [Tree('NP', [Tree('NNS', ['jumps'])]), Tree('PP', [Tree('IN', ['over']), Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['lazy']), Tree('NN', ['dog'])])])])])]), Tree('ROOT', [Tree('NP', [Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['quick']), Tree('JJ', ['grey']), Tree('NN', ['wolf'])]), Tree('NP', [Tree('NP', [Tree('NNS', ['jumps'])]), Tree('PP', [Tree('IN', ['over']), Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['lazy']), Tree('NN', ['fox'])])])])])])] >>> sum([list(dep_graphs) for dep_graphs in parser.parse_sents(( ... "I 'm a dog".split(), ... "This is my friends ' cat ( the tabby )".split(), ... ))], []) # doctest: +NORMALIZE_WHITESPACE [Tree('ROOT', [Tree('S', [Tree('NP', [Tree('PRP', ['I'])]), Tree('VP', [Tree('VBP', ["'m"]), Tree('NP', [Tree('DT', ['a']), Tree('NN', ['dog'])])])])]), Tree('ROOT', [Tree('S', [Tree('NP', [Tree('DT', ['This'])]), Tree('VP', [Tree('VBZ', ['is']), Tree('NP', [Tree('NP', [Tree('NP', [Tree('PRP$', ['my']), Tree('NNS', ['friends']), Tree('POS', ["'"])]), Tree('NN', ['cat'])]), Tree('PRN', [Tree('-LRB-', [Tree('', []), Tree('NP', [Tree('DT', ['the']), Tree('NN', ['tabby'])]), Tree('-RRB-', [])])])])])])])] >>> sum([list(dep_graphs) for dep_graphs in parser.tagged_parse_sents(( ... ( ... ("The", "DT"), ... ("quick", "JJ"), ... ("brown", "JJ"), ... ("fox", "NN"), ... ("jumped", "VBD"), ... ("over", "IN"), ... ("the", "DT"), ... ("lazy", "JJ"), ... ("dog", "NN"), ... (".", "."), ... ), ... ))],[]) # doctest: +NORMALIZE_WHITESPACE [Tree('ROOT', [Tree('S', [Tree('NP', [Tree('DT', ['The']), Tree('JJ', ['quick']), Tree('JJ', ['brown']), Tree('NN', ['fox'])]), Tree('VP', [Tree('VBD', ['jumped']), Tree('PP', [Tree('IN', ['over']), Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['lazy']), Tree('NN', ['dog'])])])]), Tree('.', ['.'])])])] Zpenncs&tjdtddtt|j||dS)NzcThe StanfordParser will be deprecated Please use nltk.parse.corenlp.CoreNLPParser instead.) stacklevel)warningswarnDeprecationWarningsuperr]r))r'argskwargs) __class__rrr)Ns zStanfordParser.__init__cCs t|S)N)r Z fromstring)r'resultrrrr/XszStanfordParser._make_tree)rYrZr[r\r9r)r/ __classcell__rr)rfrr]s1 r]cs,eZdZdZdZfddZddZZS)StanfordDependencyParseraT >>> dep_parser=StanfordDependencyParser( ... model_path="edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz" ... ) >>> [parse.tree() for parse in dep_parser.raw_parse("The quick brown fox jumps over the lazy dog.")] # doctest: +NORMALIZE_WHITESPACE [Tree('jumps', [Tree('fox', ['The', 'quick', 'brown']), Tree('dog', ['over', 'the', 'lazy'])])] >>> [list(parse.triples()) for parse in dep_parser.raw_parse("The quick brown fox jumps over the lazy dog.")] # doctest: +NORMALIZE_WHITESPACE [[((u'jumps', u'VBZ'), u'nsubj', (u'fox', u'NN')), ((u'fox', u'NN'), u'det', (u'The', u'DT')), ((u'fox', u'NN'), u'amod', (u'quick', u'JJ')), ((u'fox', u'NN'), u'amod', (u'brown', u'JJ')), ((u'jumps', u'VBZ'), u'nmod', (u'dog', u'NN')), ((u'dog', u'NN'), u'case', (u'over', u'IN')), ((u'dog', u'NN'), u'det', (u'the', u'DT')), ((u'dog', u'NN'), u'amod', (u'lazy', u'JJ'))]] >>> sum([[parse.tree() for parse in dep_graphs] for dep_graphs in dep_parser.raw_parse_sents(( ... "The quick brown fox jumps over the lazy dog.", ... "The quick grey wolf jumps over the lazy fox." ... ))], []) # doctest: +NORMALIZE_WHITESPACE [Tree('jumps', [Tree('fox', ['The', 'quick', 'brown']), Tree('dog', ['over', 'the', 'lazy'])]), Tree('jumps', [Tree('wolf', ['The', 'quick', 'grey']), Tree('fox', ['over', 'the', 'lazy'])])] >>> sum([[parse.tree() for parse in dep_graphs] for dep_graphs in dep_parser.parse_sents(( ... "I 'm a dog".split(), ... "This is my friends ' cat ( the tabby )".split(), ... ))], []) # doctest: +NORMALIZE_WHITESPACE [Tree('dog', ['I', "'m", 'a']), Tree('cat', ['This', 'is', Tree('friends', ['my', "'"]), Tree('tabby', ['the'])])] >>> sum([[list(parse.triples()) for parse in dep_graphs] for dep_graphs in dep_parser.tagged_parse_sents(( ... ( ... ("The", "DT"), ... ("quick", "JJ"), ... ("brown", "JJ"), ... ("fox", "NN"), ... ("jumped", "VBD"), ... ("over", "IN"), ... ("the", "DT"), ... ("lazy", "JJ"), ... ("dog", "NN"), ... (".", "."), ... ), ... ))],[]) # doctest: +NORMALIZE_WHITESPACE [[((u'jumped', u'VBD'), u'nsubj', (u'fox', u'NN')), ((u'fox', u'NN'), u'det', (u'The', u'DT')), ((u'fox', u'NN'), u'amod', (u'quick', u'JJ')), ((u'fox', u'NN'), u'amod', (u'brown', u'JJ')), ((u'jumped', u'VBD'), u'nmod', (u'dog', u'NN')), ((u'dog', u'NN'), u'case', (u'over', u'IN')), ((u'dog', u'NN'), u'det', (u'the', u'DT')), ((u'dog', u'NN'), u'amod', (u'lazy', u'JJ'))]] Z conll2007cs&tjdtddtt|j||dS)NzwThe StanfordDependencyParser will be deprecated Please use nltk.parse.corenlp.CoreNLPDependencyParser instead.r^)r_)r`rarbrcrir))r'rdre)rfrrr)s z!StanfordDependencyParser.__init__cCs t|ddS)Nroot)top_relation_label)r )r'rgrrrr/sz#StanfordDependencyParser._make_tree)rYrZr[r\r9r)r/rhrr)rfrri\s0 ricsJeZdZdZdZdZdZdZdZdZ fddZ dd d Z d d Z Z S)StanfordNeuralDependencyParsera9 >>> from nltk.parse.stanford import StanfordNeuralDependencyParser >>> dep_parser=StanfordNeuralDependencyParser(java_options='-mx4g') >>> [parse.tree() for parse in dep_parser.raw_parse("The quick brown fox jumps over the lazy dog.")] # doctest: +NORMALIZE_WHITESPACE [Tree('jumps', [Tree('fox', ['The', 'quick', 'brown']), Tree('dog', ['over', 'the', 'lazy']), '.'])] >>> [list(parse.triples()) for parse in dep_parser.raw_parse("The quick brown fox jumps over the lazy dog.")] # doctest: +NORMALIZE_WHITESPACE [[((u'jumps', u'VBZ'), u'nsubj', (u'fox', u'NN')), ((u'fox', u'NN'), u'det', (u'The', u'DT')), ((u'fox', u'NN'), u'amod', (u'quick', u'JJ')), ((u'fox', u'NN'), u'amod', (u'brown', u'JJ')), ((u'jumps', u'VBZ'), u'nmod', (u'dog', u'NN')), ((u'dog', u'NN'), u'case', (u'over', u'IN')), ((u'dog', u'NN'), u'det', (u'the', u'DT')), ((u'dog', u'NN'), u'amod', (u'lazy', u'JJ')), ((u'jumps', u'VBZ'), u'punct', (u'.', u'.'))]] >>> sum([[parse.tree() for parse in dep_graphs] for dep_graphs in dep_parser.raw_parse_sents(( ... "The quick brown fox jumps over the lazy dog.", ... "The quick grey wolf jumps over the lazy fox." ... ))], []) # doctest: +NORMALIZE_WHITESPACE [Tree('jumps', [Tree('fox', ['The', 'quick', 'brown']), Tree('dog', ['over', 'the', 'lazy']), '.']), Tree('jumps', [Tree('wolf', ['The', 'quick', 'grey']), Tree('fox', ['over', 'the', 'lazy']), '.'])] >>> sum([[parse.tree() for parse in dep_graphs] for dep_graphs in dep_parser.parse_sents(( ... "I 'm a dog".split(), ... "This is my friends ' cat ( the tabby )".split(), ... ))], []) # doctest: +NORMALIZE_WHITESPACE [Tree('dog', ['I', "'m", 'a']), Tree('cat', ['This', 'is', Tree('friends', ['my', "'"]), Tree('tabby', ['-LRB-', 'the', '-RRB-'])])] Zconllz)edu.stanford.nlp.pipeline.StanfordCoreNLPz%stanford-corenlp-(\d+)(\.(\d+))+\.jarz,stanford-corenlp-(\d+)(\.(\d+))+-models\.jarTcs4tjdtddtt|j|||jd7_dS)Nz}The StanfordNeuralDependencyParser will be deprecated Please use nltk.parse.corenlp.CoreNLPDependencyParser instead.r^)r_z(-annotators tokenize,ssplit,pos,depparse)r`rarbrcrlr)r%)r'rdre)rfrrr)s z'StanfordNeuralDependencyParser.__init__FcCs tddS)z Currently unimplemented because the neural dependency parser (and the StanfordCoreNLP pipeline class) doesn't support passing in pre- tagged tokens. zxtagged_parse[_sents] is not supported by StanfordNeuralDependencyParser; use parse[_sents] or raw_parse[_sents] instead.N)NotImplementedError)r'r;rrrrrAsz1StanfordNeuralDependencyParser.tagged_parse_sentscCs t|ddS)NZROOT)rk)r )r'rgrrrr/sz)StanfordNeuralDependencyParser._make_tree)F)rYrZr[r\r9r8rr rSr.r)rAr/rhrr)rfrrls rlzEdoctests from nltk.parse.stanford are skipped because it's deprecatedcCsBddlm}ytddtWntk r<|dYnXdS)Nr)SkipTestz4edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz)rzndoctests from nltk.parse.stanford are skipped because one of the stanford parser or CoreNLP jars doesn't exist)Znosernr]rl LookupError)modulernrrr setup_modules  rq)Z __future__rrNrr`Zunittestr subprocessrZsixrZnltk.internalsrrrr r Znltk.parse.apir Znltk.parse.dependencygraphr Z nltk.treer rrr]rirlrqrrrr s"       wCBB