ó ù`]c@ sñdZddlmZmZddlmZddlmZmZm Z m Z m Z m Z m Z mZmZmZmZmZmZddlmZmZmZmZmZmZmZmZdefd„ƒYZdeefd „ƒYZd efd „ƒYZd efd „ƒYZ defd„ƒYZ!defd„ƒYZ"defd„ƒYZ#defd„ƒYZ$de fd„ƒYZ%de!fd„ƒYZ&defd„ƒYZ'e ƒeƒe ƒe!ƒe"ƒgZ(e ƒeƒeƒeƒgZ)e ƒeƒe ƒeƒgZ*e ƒeƒe ƒeƒgZ+e ƒeƒe#ƒgZ,defd„ƒYZ-de-fd„ƒYZ.d e-fd!„ƒYZ/d"e-fd#„ƒYZ0d$e-fd%„ƒYZ1d&e-fd'„ƒYZ2e ƒeƒe%ƒe&ƒe'ƒgZ3e ƒeƒeƒe$ƒgZ4e ƒeƒeƒe$ƒgZ5e ƒeƒeƒe$ƒgZ6d(e-efd)„ƒYZ7d*e7fd+„ƒYZ8d,e7fd-„ƒYZ9d.e7fd/„ƒYZ:d0e7fd1„ƒYZ;e<e=e<d2d3d4d5„Z>e?d6kríe>ƒnd7S(8sÕ Data classes and parser implementations for *incremental* chart parsers, which use dynamic programming to efficiently parse a text. A "chart parser" derives parse trees for a text by iteratively adding "edges" to a "chart". Each "edge" represents a hypothesis about the tree structure for a subsequence of the text. The "chart" is a "blackboard" for composing and combining these hypotheses. A parser is "incremental", if it guarantees that for all i, j where i < j, all edges ending at i are built before any edges ending at j. This is appealing for, say, speech recognizer hypothesis filtering. The main parser class is ``EarleyChartParser``, which is a top-down algorithm, originally formulated by Jay Earley (1970). iÿÿÿÿ(tprint_functiontdivision(trange( tChartt ChartParsertEdgeItLeafEdget LeafInitRuletBottomUpPredictRuletBottomUpPredictCombineRuletTopDownInitRuletSingleEdgeFundamentalRuletEmptyPredictRuletCachedTopDownPredictRulet!FilteredSingleEdgeFundamentalRulet"FilteredBottomUpPredictCombineRule(t FeatureCharttFeatureChartParsertFeatureTopDownInitRuletFeatureTopDownPredictRuletFeatureEmptyPredictRuletFeatureBottomUpPredictRulet!FeatureBottomUpPredictCombineRulet FeatureSingleEdgeFundamentalRuletIncrementalChartcB sPeZd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Z RS(cC s5td„|jƒDƒƒ|_i|_i|_dS(Ncs s|] }gVqdS(N((t.0tx((s5lib/python2.7/site-packages/nltk/parse/earleychart.pys Bs(ttuplet _positionst _edgelistst _edge_to_cplst_indexes(tself((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyt initialize@s cC st|jƒƒS(N(tlistt iteredges(R ((s5lib/python2.7/site-packages/nltk/parse/earleychart.pytedgesKscC sd„|jDƒS(Ncs s"|]}|D] }|Vq qdS(N((Rtedgelisttedge((s5lib/python2.7/site-packages/nltk/parse/earleychart.pys Os(R(R ((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR#Nsc s|j|}ˆikr#t|ƒStˆjƒƒ}t|ƒ}||jkr`|j|ƒnt‡fd†|Dƒƒ}t|j||j|gƒƒS(Nc3 s|]}ˆ|VqdS(N((Rtkey(t restrictions(s5lib/python2.7/site-packages/nltk/parse/earleychart.pys `s(RtitertsortedtkeysRRt _add_indextget(R tendR(R%t restr_keystvals((R(s5lib/python2.7/site-packages/nltk/parse/earleychart.pytselectQs    c sÎx0|D](}tt|ƒstd|ƒ‚qqWtd„|jƒDƒƒ}|j|jsc3 s!|]}tˆ|ƒƒVqdS(N(tgetattr(RR'(R&(s5lib/python2.7/site-packages/nltk/parse/earleychart.pys ps( thasattrRt ValueErrorRRRt enumerateRt setdefaulttappend(R R/R'tindexR.R%t this_indexR0((R&s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR,cs '  c siˆjƒ}xV|jjƒD]E\}}t‡fd†|Dƒƒ}||j|gƒjˆƒqWdS(Nc3 s!|]}tˆ|ƒƒVqdS(N(R2(RR'(R&(s5lib/python2.7/site-packages/nltk/parse/earleychart.pys vs(R.RtitemsRR6R7(R R&R.R/R8R0((R&s5lib/python2.7/site-packages/nltk/parse/earleychart.pyt_register_with_indexesss cC s|j|jƒj|ƒdS(N(RR.R7(R R&((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyt _append_edgeyscC st|jƒdƒS(Ni(Rt num_leaves(R ((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR|s( t__name__t __module__R!R$R#R1R,R;R<R(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR?s      tFeatureIncrementalChartcB s#eZd„Zd„Zd„ZRS(c s ˆj|}ˆikr#t|ƒStˆjƒƒ}t|ƒ}|ˆjkr`ˆj|ƒnt‡‡fd†|Dƒƒ}tˆj||j|gƒƒS(Nc3 s"|]}ˆjˆ|ƒVqdS(N(t_get_type_if_possible(RR'(R(R (s5lib/python2.7/site-packages/nltk/parse/earleychart.pys ‘s(RR)R*R+RRR,R-(R R.R(R%R/R0((R(R s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR1s    c sÑx0|D](}tt|ƒstd|ƒ‚qqWtd„ˆjƒDƒƒ}ˆj|‰t‡‡fd†|Dƒƒ}|j|gƒj ˆƒq‡WqjWdS(NsBad restriction: %scs s|] }iVqdS(N((RR((s5lib/python2.7/site-packages/nltk/parse/earleychart.pys œsc3 s*|] }ˆjtˆ|ƒƒƒVqdS(N(RAR2(RR'(R&R (s5lib/python2.7/site-packages/nltk/parse/earleychart.pys £s( R3RR4RRRR5RR6R7(R R/R'R8R.R%R9R0((R&R s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR,•s '   c slˆjƒ}xYˆjjƒD]H\}}t‡‡fd†|Dƒƒ}||j|gƒjˆƒqWdS(Nc3 s*|] }ˆjtˆ|ƒƒƒVqdS(N(RAR2(RR'(R&R (s5lib/python2.7/site-packages/nltk/parse/earleychart.pys ¬s(R.RR:RR6R7(R R&R.R/R8R0((R&R s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR;¨s  (R>R?R1R,R;(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR@€s  tCompleteFundamentalRulecB seZd„ZRS(c c sw|jƒ}xd|jd|d|dtd|jƒƒD]8}|j|jƒƒ}|j|||ƒr7|Vq7q7WdS(NtstartR.t is_completetlhs(R.R1tTruetnextsymtmove_dot_forwardtinsert_with_backpointer(R tcharttgrammart left_edgeR.t right_edgetnew_edge((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyt_apply_incomplete·s   %(R>R?RO(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRB¶st CompleterRulecB seZeƒZd„ZRS(cc s>t|tƒs:x(|jj|||ƒD] }|Vq(WndS(N(t isinstanceRt_fundamental_ruletapply(R RJRKR&RN((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRSÆs(R>R?RBRRRS(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRPÃs t ScannerRulecB seZeƒZd„ZRS(cc s>t|tƒr:x(|jj|||ƒD] }|Vq(WndS(N(RQRRRRS(R RJRKR&RN((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRSÏs(R>R?RBRRRS(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRTÌs t PredictorRulecB seZRS((R>R?(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRUÕstFilteredCompleteFundamentalRulecB seZd„ZRS(cc s8|jƒr4x%|j|||ƒD] }|Vq"WndS(N(RDt_apply_complete(R RJRKR&RN((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRSÚs (R>R?RS(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRVÙstFeatureCompleteFundamentalRulecB seZd„ZRS(c c sv|j}|jƒ}xZ|jd|d|dtd|jƒƒD].}x%|j||||ƒD] }|Vq_Wq@WdS(NRCR.RDRE(RRR.R1RFRGRS(R RJRKRLtfrR.RMRN((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyROès    %(R>R?RO(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRXçstFeatureCompleterRulecB seZeƒZRS((R>R?RXRR(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRZôstFeatureScannerRulecB seZeƒZRS((R>R?RXRR(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR[østFeaturePredictorRulecB seZRS((R>R?(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR\üstIncrementalChartParsercB s/eZdZedded„Zdd„ZRS(s An *incremental* chart parser implementing Jay Earley's parsing algorithm: | For each index end in [0, 1, ..., N]: | For each edge such that edge.end = end: | If edge is incomplete and edge.next is not a part of speech: | Apply PredictorRule to edge | If edge is incomplete and edge.next is a part of speech: | Apply ScannerRule to edge | If edge is complete: | Apply CompleterRule to edge | Return any complete parses in the chart ii2cC s›||_||_||_||_g|_g|_x^|D]V}|jdkre|jj|ƒq=|jdkr‡|jj|ƒq=tdƒ‚q=WdS(sö Create a new Earley chart parser, that uses ``grammar`` to parse texts. :type grammar: CFG :param grammar: The grammar used to parse texts. :type trace: int :param trace: The level of tracing that should be used when parsing a text. ``0`` will generate no tracing output; and higher numbers will produce more verbose tracing output. :type trace_chart_width: int :param trace_chart_width: The default total width reserved for the chart in trace output. The remainder of each line will be used to display edges. :param chart_class: The class that should be used to create the charts used by this parser. iis9Incremental inference rules must have NUM_EDGES == 0 or 1N( t_grammart_tracet_trace_chart_widtht _chart_classt_axiomst_inference_rulest NUM_EDGESR7R4(R RKtstrategyttracettrace_chart_widtht chart_classtrule((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyt__init__5s       cC s½|dkr|j}n|j}t|ƒ}|jj|ƒ|j|ƒ}|j}|j|jƒd}|rˆt |j |ƒƒnx?|j D]4}t|j ||ƒƒ}||||||ƒq’W|j } xãt|jƒdƒD]Ë} |dkrt d| dƒnt|jd| ƒƒ} x‹| r´| jƒ} xr| D]j} t| j ||| ƒƒ}||| |||ƒx0|D](}|jƒ| kr| j|ƒqqWqCWq*WqêW|S(Nis * Processing queue:s R.(tNoneR_t_trace_new_edgesR"R^tcheck_coverageRaR`R=tprinttpretty_format_leavesRbRSRcRR1tpopR.R7(R ttokensRfttrace_new_edgesRJRKttrace_edge_widthtaxiomt new_edgestinference_rulesR.tagendaR&RiRN((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyt chart_parse`s6            N(R>R?t__doc__tBU_LC_INCREMENTAL_STRATEGYRRjRkRx(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR]%s  %tEarleyChartParsercB seZd„ZRS(cK stj||t|dS(N(R]RjtEARLEY_STRATEGY(R RKt parser_args((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRj…s(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR{„stIncrementalTopDownChartParsercB seZd„ZRS(cK stj||t|dS(N(R]RjtTD_INCREMENTAL_STRATEGY(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjŠs(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR~‰stIncrementalBottomUpChartParsercB seZd„ZRS(cK stj||t|dS(N(R]RjtBU_INCREMENTAL_STRATEGY(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRj‘s(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR€st(IncrementalBottomUpLeftCornerChartParsercB seZd„ZRS(cK stj||t|dS(N(R]RjRz(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRj˜s(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR‚—st IncrementalLeftCornerChartParsercB seZd„ZRS(cK s5|jƒstdƒ‚ntj||t|dS(NsNIncrementalLeftCornerParser only works for grammars without empty productions.(t is_nonemptyR4R]RjtLC_INCREMENTAL_STRATEGY(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjŸs   (R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRƒžstFeatureIncrementalChartParsercB seZeded„ZRS(ic K s)tj||d|d|d||dS(NReRgRh(R]Rj(R RKReRgRhR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjÊs(R>R?t"BU_LC_INCREMENTAL_FEATURE_STRATEGYR@Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyR†ÉstFeatureEarleyChartParsercB seZd„ZRS(cK stj||t|dS(N(R†RjtEARLEY_FEATURE_STRATEGY(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjÝs(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRˆÜst$FeatureIncrementalTopDownChartParsercB seZd„ZRS(cK stj||t|dS(N(R†RjtTD_INCREMENTAL_FEATURE_STRATEGY(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjäs(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRŠãst%FeatureIncrementalBottomUpChartParsercB seZd„ZRS(cK stj||t|dS(N(R†RjtBU_INCREMENTAL_FEATURE_STRATEGY(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjës(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRŒêst/FeatureIncrementalBottomUpLeftCornerChartParsercB seZd„ZRS(cK stj||t|dS(N(R†RjR‡(R RKR}((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRjòs(R>R?Rj(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pyRŽñsis$I saw John with a dog with my cookieicC sOddl}ddl}ddlm}|ƒ} |rNtdƒt| ƒntdƒt|ƒ|jƒ} t| ƒtƒt| d|ƒ} |jƒ} | j| ƒ} t | j | j ƒƒƒ}|jƒ| } |rþt |ƒ|ksþt dƒ‚n|r"x.|D]}t|ƒq Wntdt |ƒƒ|rKtd | ƒndS( s0 A demonstration of the Earley parsers. iÿÿÿÿN(t demo_grammars * Grammars * Sentence:RfsNot all parses founds Nr trees:sTime:(tsysttimetnltk.parse.chartRRntsplitR{tclockRxR"tparsesRCtlentAssertionError(t print_timest print_grammart print_treesRftsentt numparsesRR‘RRKRqtearleyttRJR•ttree((s5lib/python2.7/site-packages/nltk/parse/earleychart.pytdemoýs0         ! t__main__N(@Ryt __future__RRt six.movesRR’RRRRRRR R R R R RRtnltk.parse.featurechartRRRRRRRRRR@RBRPRTRURVRXRZR[R\R|RRRzR…R]R{R~R€R‚RƒR‰R‹RR‡R†RˆRŠRŒRŽRFtFalseR R>(((s5lib/python2.7/site-packages/nltk/parse/earleychart.pytsŽX:A6          _      (