`]c @ s d d l m Z d d l Z d d l Z d d l Z d d l m Z d d l m Z m Z m
Z
m Z m Z m
Z
d d l m Z d Z d e f d YZ e e d e j f d
Y Z e e d e j f d Y Z e e d
e j f d Y Z e e d e j f d Y Z e e d e j f d Y Z e e d e j f d Y Z e e d e j f d Y Z d e j f d YZ d S( i( t divisionN( t
add_metaclass( t
Vocabularyt MLEt Lidstonet Laplacet WittenBellInterpolatedt KneserNeyInterpolated( t padded_everygramsc C s_ t d d d d d d d g d d g t d
t d f D] } t t | | ^ q= f S( Nt at bt ct dt zs s t
unk_cutoffi t abcdt egadbe( R t listR ( t ngram_ordert sent( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt _prepare_test_data s $t ParametrizeTestsMetac B s5 e Z d Z d Z e d Z e d Z RS( s, Metaclass for generating parametrized tests.c C s d
d d d d d d d d f } x6 t | D]( \ } } | j | | d
j | R
u R t et rt ws test_sumto1_{0}t score_testss test_score_{0}( R ( R ( u ( R
( u ( R ( R ( R ( R ( t enumeratet add_sum_to_1_testt formatt gett add_score_testt superR t __new__( t clst namet basest dctt contextst iR t scorest wordt contextt expected_score( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR ) s " c s: t j d k r d n d f d } | S( Ni i s word='{word}', context={context}sM {score} != {expected_score} within 4 places, word='{word}', context={context}c sA | j j } | j | d j t d d d S( Nt msgt placesi ( t modelt scoret assertAlmostEqualR t locals( t selfR. ( R) R* t messageR( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_methodJ s ( i i ( t syst version_info( R! R( R) R* R3 ( ( R) R* R2 R( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR > s
c s f d } | S( Nc sH t f d j j D } j | d d d j d S( Nc 3 s$ | ] } j j | Vq d S( N( R- R. ( t .0R ( R) R1 ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pys U s g ?R+ s The context is {}( t sumR- t vocabR/ R ( R1 t s( R) ( R1 s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt testT s %( ( R! R) R: ( ( R) s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR R s ( t __name__t
__module__t __doc__R t classmethodR R ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR & s t MleBigramTestsc B sz e Z d Z d d g d f d d g d f d d d d f d
d d f g Z d Z d
Z d Z d Z d Z RS( s" unit tests for MLENgramModel classR R i R i R
R g @i t yg @c C s; t d \ } } t d d | | _ | j j | d S( Ni t
vocabulary( R R R- t fit( R1 R8 t
training_text( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt setUpl s c C s2 | j j d d g } | j t j | d S( NR R ( R- t logscoret
assertTruet matht isinf( R1 RE ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_logscore_zero_scoreq s c C sl d d d
d d d g } d } d } | j | | j j | d d
| j | | j j | d d
d S( Ns R R
s R s g(\?g_vO @R, i ( s R ( R R
( R
s ( s R ( R R ( R s ( R/ R- t entropyt
perplexity( R1 t trainedt HRK ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_entropy_perplexity_seenw s "c C sZ d d d d g } | j t j | j j | | j t j | j j | d S(
Ns R R R s ( s R ( R R ( R R ( R s ( RF RG RH R- RJ RK ( R1 t untrained( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_entropy_perplexity_unseen s "c C so d } d } d d d
d d d d g } | j | | j j | d d
| j | | j j | d d
d S( Ng~jt@gs @s R R t -R s R, i ( s ( R ( R ( RQ ( R ( R ( s ( R/ R- RJ RK ( R1 RM RK t text( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_entropy_perplexity_unigrams s
"N( R
Ni g$I$I?g۶m۶m?(
R; R< R= t NoneR RD RI RN RP RS ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR? [ s t MleTrigramTestsc B sS e Z d Z d d
d f d d g d f d d d f d d
d d f g Z d Z RS( s MLE trigram model testsR R
R i R g @i R
i R@ g @c C s; t d \ } } t d d | | _ | j j | d S( Ni RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD s ( R
R Ng q q?( R
Ni gUUUUUU?( R; R< R= RT R RD ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRU s t LidstoneBigramTestsc B sb e Z d Z d d g d f d d d f d d d f d
d d f g Z d Z d
Z d Z RS( s unit tests for Lidstone classR R g?g?R g @g-@R
g?R@ g@c C s> t d \ } } t d d d | | _ | j j | d S( Ni g?RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD s c C s | j d | j j d S( Ng?( t assertEqualR- t gamma( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt
test_gamma s c C sl d d d
d d d g } d } d } | j | | j j | d d
| j | | j j | d d
d S( Ns R R s R s g=,Ԛ]@g鷯 1@R, i ( s R ( R R ( R s ( s R ( R R ( R s ( R/ R- RJ RK ( R1 RR RM RK ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_entropy_perplexity s "g88?Ng"u)?gк{?gL?( R; R< R= RT R RD RY RZ ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRV s t LidstoneTrigramTestsc B sY e Z d d g d f d d g d f d d d g d
f d d d g d f g Z d Z RS( R R g?g?R g?R
c C s> t d \ } } t d d d | | _ | j j | d S( Ni g?RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD s g88?g q q?g88?g q q?( R; R< R RD ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR[ s
t LaplaceBigramTestsc B sb e Z d Z d d g d f d d d f d d d f d
d d f g Z d Z d
Z d Z RS( s unit tests for Laplace classR R g @i R g @i R
g ?R@ g @c C s; t d \ } } t d d | | _ | j j | d S( Ni RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD , s c C s | j d | j j d S( Ni ( RW R- RX ( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRY 1 s c C sl d d d
d d d g } d } d } | j | | j j | d d
| j | | j j | d d
d S( Ns R R s R s gQ @gݓz!@R, i ( s R ( R R ( R s ( s R ( R R ( R s ( R/ R- RJ RK ( R1 RR RM RK ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRZ 5 s "g q q?NgtE]t?gF]tE?gF]tE?( R; R< R= RT R RD RY RZ ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR\ s t" WittenBellInterpolatedTrigramTestsc B sl e Z d Z d d d f d d d f d d d f d d g d d f d d d g d d
d d f g Z RS( c C s; t d \ } } t d d | | _ | j j | d S( Ni RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD P s R g ?i R
g R@ g @R
i g镲 q?g ?R gN@?Ng q q?g gUUUUUU?gBiq?gBiq?g镲 q?ghI Hy?gj+?gBiq?gBiq?ghI Hy?( R; R< RD RT R ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR] N s t! KneserNeyInterpolatedTrigramTestsc B s| e Z d Z d d
d f d d
d f d d
d f d d g d d d d
f d d d g d d d d d d
f g Z RS( c C s; t d \ } } t d d | | _ | j j | d S( Ni RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD m s R g ?i R
i R@ R
g?g?i R g?Ng ?g ?g ?g ?g ?( R; R< RD RT R ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR^ k s t NgramModelTextGenerationTestsc B sV e Z d Z d Z d Z d Z d Z d Z d Z d Z d Z
RS( s( Using MLE estimator, generate some text.c C s; t d \ } } t d d | | _ | j j | d S( Ni RA ( R R R- RB ( R1 R8 RC ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyRD s c C s# | j | j j d d d d S( Nt random_seedi s ( RW R- t generate( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_generate_one_no_context s c C sp | j | j j d d g d | j | j j d d d g d | j | j j d d d g d d S( Nt text_seedR R R
R ( RW R- Ra ( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt" test_generate_one_limiting_context s "%c C s) | j | j j d d d d d d S( NRc R s R` i ( R s ( RW R- Ra ( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_generate_one_varied_context s c C su t t | j j t d g } | j j | | j | j j d d d d d d d d d d d d g d S(
Nt bdbdbdi Rc R
R R` i s ( R
R ( R R R- t orderRB RW Ra ( R1 t more_training_text( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_generate_cycle s
$c C s; | j | j j d d d d d d d d d
d g d S( Ni Rc s R R` i s R R R
( s R ( RW R- Ra ( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_generate_with_text_seed s c C s> | j | j j d d d d | j j d d d d d S( NRc t aliensR` i s ( Rk ( s ( RW R- Ra ( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_generate_oov_text_seed s c C sa | j t | j j d d Wd QX| j | j j d d d d | j j d d d S( NRc R` i ( N( t assertRaisest TypeErrorR- Ra RT RW ( R1 ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt test_generate_None_text_seed s
( R; R< R= RD Rb Rd Re Ri Rj Rl Ro ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyR_ s
( t
__future__R RG R4 t unittestt sixR t nltk.lmR R R R R R t nltk.lm.preprocessingR R t typeR t TestCaseR? RU RV R[ R\ R] R^ R_ ( ( ( s< lib/python2.7/site-packages/nltk/test/unit/lm/test_models.pyt s. .
5 O : <