`]c@ s dZddlmZmZmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZmZddlmZmZddlmZddlmZmZddlmZmZyddlZWnek rddlZnXyejejd d Z Wn>e!k rvejej"d d d d d e#Z$dZ nXyddl%m&Z'Wn!ek rddl%m(Z'nXddl)Z)ddl*m+Z+m,Z,m-Z-gZ.ej/j0de1dj2ej3Z4e.ge4D]Z5e5r e5^q7Z.dej/koKej.j6ddkrpe.j7ej.j6e1dne j8j9dr@e.ej.j:e j;e1dej.j:e j;e1de1dej.j:e j;e1de1dej.j:ej/j0e1de1de1de1de1de1dg7Z.ne.ej.j:e j;e1dej.j:e j;e1de1dej.j:e j;e1de1de1d e1d!e1d"e1d#g7Z.d$d%d&e<e<e<d'Z=d(Z>d)Z?e@e<d*ZAeed+eBfd,YZCd-eCefd.YZDd/efd0YZEd1eDfd2YZFd3eCfd4YZGiZHe<d5ZIe<e@d6ZJi d7d86d9d:6d;d<6d=d>6d?d@6dAdB6dCdD6dEdF6dGdH6dIdJ6dKdL6ZKi d8d86d:d:6d<d<6d>d>6d@d@6dBdB6dDdD6dFdF6dHdH6dLdM6dLdL6ZLdNe@e#e<e<e<dOZMdPdQZNdRZOdSZPdTeBfdUYZQdVe jRfdWYZSdXeBfdYYZTdZd[d\d]d^d^d_d`dadbdcdddedfdgd^dhgZUdS(iu Functions to find and load NLTK resource files, such as corpora, grammars, and saved processing objects. Resource files are identified using URLs, such as ``nltk:corpora/abc/rural.txt`` or ``http://nltk.org/sample/toy.cfg``. The following URL protocols are supported: - ``file:path``: Specifies the file whose path is *path*. Both relative and absolute paths may be used. - ``http://host/path``: Specifies the file stored on the web server *host* at path *path*. - ``nltk:path``: Specifies the file stored in the NLTK data package at *path*. NLTK will search for these files in the directories specified by ``nltk.data.path``. If no protocol is specified, then the default protocol ``nltk:`` will be used. This module provides to functions that can be used to access a resource file, given its URL: ``load()`` loads a given resource, and adds it to a resource cache; and ``retrieve()`` copies a given resource to a local file. i(tprint_functiontunicode_literalstdivisionN(tABCMetatabstractmethod(tGzipFiletWRITE(t add_metaclass(t string_typest text_type(turlopent url2pathnametprefixu tinitial_indenttsubsequent_indenttreplace_whitespacecC sdjd|jDS(Nu cs s|]}t|VqdS(N(t textwrap_fill(t.0tline((s(lib/python2.7/site-packages/nltk/data.pys Cs(tjoint splitlines(ttext((s(lib/python2.7/site-packages/nltk/data.pyttextwrap_indentBs(t Z_SYNC_FLUSH(tZ_FINISH(tpy3_datat add_py3_datatBytesIOu NLTK_DATAuuAPPENGINE_RUNTIMEu~/u ~/nltk_datauwinu nltk_dataushareulibuAPPDATAuC:\u C:\nltk_datau D:\nltk_datau E:\nltk_datau/usr/share/nltk_datau/usr/local/share/nltk_datau/usr/lib/nltk_datau/usr/local/lib/nltk_dataurbi uutf-8cC s:|dkr$t||||}ntj||||S(N(tNoneRtiot TextIOWrapper(tfilenametmodet compressleveltencodingtfileobjterrorstnewline((s(lib/python2.7/site-packages/nltk/data.pytgzip_open_unicode}s cC sz|jdd\}}|dkr'nI|dkr[|jdrpd|jd}qpntjdd|}||fS(u Splits a resource url into ":". >>> windows = sys.platform.startswith('win') >>> split_resource_url('nltk:home/nltk') ('nltk', 'home/nltk') >>> split_resource_url('nltk:/home/nltk') ('nltk', '/home/nltk') >>> split_resource_url('file:/home/nltk') ('file', '/home/nltk') >>> split_resource_url('file:///home/nltk') ('file', '/home/nltk') >>> split_resource_url('file:///C:/home/nltk') ('file', '/C:/home/nltk') u:iunltkufileu/u^/{0,2}u(tsplitt startswithtlstriptretsub(t resource_urltprotocoltpath_((s(lib/python2.7/site-packages/nltk/data.pytsplit_resource_urls  cC syt|\}}Wntk r5d}|}nX|dkrotjj|rod}t|td}nU|dkrd}t|td}n.|dkrd}t|t}n |d7}dj ||gS(u Normalizes a resource url >>> windows = sys.platform.startswith('win') >>> os.path.normpath(split_resource_url(normalize_resource_url('file:grammar.fcfg'))[1]) == \ ... ('\\' if windows else '') + os.path.abspath(os.path.join(os.curdir, 'grammar.fcfg')) True >>> not windows or normalize_resource_url('file:C:/dir/file') == 'file:///C:/dir/file' True >>> not windows or normalize_resource_url('file:C:\\dir\\file') == 'file:///C:/dir/file' True >>> not windows or normalize_resource_url('file:C:\\dir/file') == 'file:///C:/dir/file' True >>> not windows or normalize_resource_url('file://C:/dir/file') == 'file:///C:/dir/file' True >>> not windows or normalize_resource_url('file:////C:/dir/file') == 'file:///C:/dir/file' True >>> not windows or normalize_resource_url('nltk:C:/dir/file') == 'file:///C:/dir/file' True >>> not windows or normalize_resource_url('nltk:C:\\dir\\file') == 'file:///C:/dir/file' True >>> windows or normalize_resource_url('file:/dir/file/toy.cfg') == 'file:///dir/file/toy.cfg' True >>> normalize_resource_url('nltk:home/nltk') 'nltk:home/nltk' >>> windows or normalize_resource_url('nltk:/home/nltk') == 'file:///home/nltk' True >>> normalize_resource_url('http://example.com/dir/file') 'http://example.com/dir/file' >>> normalize_resource_url('dir/file') 'nltk:dir/file' unltkufile://ufileunltk:u://uN( R/t ValueErrortostpathtisabstnormalize_resource_nametFalseRtTrueR(R,R-tname((s(lib/python2.7/site-packages/nltk/data.pytnormalize_resource_urls !     cC s6ttjd|p*|jtjj}tjj drQ|j d}ntj dd|}|rtjj |}n9|dkrtj}ntjjtjj||}|jddjtjjd}tjj drtjj|rd|}n|r2|jd r2|d7}n|S(u( :type resource_name: str or unicode :param resource_name: The name of the resource to search for. Resource names are posix-style relative path names, such as ``corpora/brown``. Directory names will automatically be converted to a platform-appropriate path separator. Directory trailing slashes are preserved >>> windows = sys.platform.startswith('win') >>> normalize_resource_name('.', True) './' >>> normalize_resource_name('./', True) './' >>> windows or normalize_resource_name('dir/file', False, '/') == '/dir/file' True >>> not windows or normalize_resource_name('C:/file', False, '/') == '/C:/file' True >>> windows or normalize_resource_name('/dir/file', False, '/') == '/dir/file' True >>> windows or normalize_resource_name('../dir/file', False, '/') == '/dir/file' True >>> not windows or normalize_resource_name('/dir/file', True, '/') == 'dir/file' True >>> windows or normalize_resource_name('/dir/file', True, '/') == '/dir/file' True u[\\/.]$uwinu/u^/+u\N(tboolR*tsearchtendswithR1R2tseptsystplatformR(R)R+tnormpathRtcurdirtabspathRtreplaceR3(t resource_nametallow_relativet relative_pathtis_dir((s(lib/python2.7/site-packages/nltk/data.pyR4s   !$$  t PathPointercB s>eZdZeddZedZedZRS(uq An abstract base class for 'path pointers,' used by NLTK's data package to identify specific paths. Two subclasses exist: ``FileSystemPathPointer`` identifies a file that can be accessed directly via a given absolute path. ``ZipFilePathPointer`` identifies a file contained within a zipfile, that can be accessed by reading that zipfile. cC sdS(u Return a seekable read-only stream that can be used to read the contents of the file identified by this path pointer. :raise IOError: If the path specified by this pointer does not contain a readable file. N((tselfR"((s(lib/python2.7/site-packages/nltk/data.pytopentcC sdS(u Return the size of the file pointed to by this path pointer, in bytes. :raise IOError: If the path specified by this pointer does not contain a readable file. N((RH((s(lib/python2.7/site-packages/nltk/data.pyt file_size(RJcC sdS(uP Return a new path pointer formed by starting at the path identified by this pointer, and then following the relative path given by ``fileid``. The path components of ``fileid`` should be separated by forward slashes, regardless of the underlying file system's path seperator character. N((RHtfileid((s(lib/python2.7/site-packages/nltk/data.pyR2RJN(t__name__t __module__t__doc__RRRIRKR(((s(lib/python2.7/site-packages/nltk/data.pyRGs    tFileSystemPathPointercB s\eZdZedZedZddZdZ dZ dZ dZ RS( um A path pointer that identifies a file which can be accessed directly via a given absolute path. cC sDtjj|}tjj|s7td|n||_dS(u Create a new path pointer for the given absolute path. :raise IOError: If the given path does not exist. uNo such file or directory: %rN(R1R2RAtexiststIOErrort_path(RHRS((s(lib/python2.7/site-packages/nltk/data.pyt__init__CscC s|jS(u2The absolute path identified by this path pointer.(RS(RH((s(lib/python2.7/site-packages/nltk/data.pyR2SscC s4t|jd}|dk r0t||}n|S(Nurb(RIRSRtSeekableUnicodeStreamReader(RHR"tstream((s(lib/python2.7/site-packages/nltk/data.pyRIXs cC stj|jjS(N(R1tstatRStst_size(RH((s(lib/python2.7/site-packages/nltk/data.pyRK^scC s"tjj|j|}t|S(N(R1R2RRSRP(RHRLRS((s(lib/python2.7/site-packages/nltk/data.pyRascC std|jS(NuFileSystemPathPointer(%r)(tstrRS(RH((s(lib/python2.7/site-packages/nltk/data.pyt__repr__escC s|jS(N(RS(RH((s(lib/python2.7/site-packages/nltk/data.pyt__str__ksN( RMRNRORRTtpropertyR2RRIRKRRZR[(((s(lib/python2.7/site-packages/nltk/data.pyRP=s    tBufferedGzipFilecB seZdZd ZdeZedddddZdZdZ dZ dZ e d Z dd Zd d ZRS(u A ``GzipFile`` subclass that buffers calls to ``read()`` and ``write()``. This allows faster reads and writes of data to and from gzip-compressed files at the cost of using more memory. The default buffer size is 2MB. ``BufferedGzipFile`` is useful for loading large gzipped pickle objects as well as writing large encoded feature files for classifier training. iii cK sJtj||||||jd|j|_t|_d|_dS(u! Return a buffered gzip file object. :param filename: a filesystem path :type filename: str :param mode: a file mode which can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb' :type mode: str :param compresslevel: The compresslevel argument is an integer from 1 to 9 controlling the level of compression; 1 is fastest and produces the least compression, and 9 is slowest and produces the most compression. The default is 9. :type compresslevel: int :param fileobj: a BytesIO stream to read from instead of a file. :type fileobj: BytesIO :param size: number of bytes to buffer during calls to read() and write() :type size: int :rtype: BufferedGzipFile usizeiN(RRTtgettSIZEt_sizeRt _nltk_buffert_len(RHRR R!R#tkwargs((s(lib/python2.7/site-packages/nltk/data.pyRT~s cC st|_d|_dS(Ni(RRaRb(RH((s(lib/python2.7/site-packages/nltk/data.pyt _reset_buffers cC s8|dk r4|jj||jt|7_ndS(N(RRatwriteRbtlen(RHtdata((s(lib/python2.7/site-packages/nltk/data.pyt _write_buffers cC s4tj||jj|j|j|dS(N(RReRatgetvalueRdRh(RHRg((s(lib/python2.7/site-packages/nltk/data.pyt _write_gzips cC s6|jtkr)|jd|jntj|S(N(R tGZ_WRITERjRRdRtclose(RH((s(lib/python2.7/site-packages/nltk/data.pyRls  cC s!|jjtj||dS(N(RatflushR(RHtlib_mode((s(lib/python2.7/site-packages/nltk/data.pyRms cC sv|sb|j}t}x=trWtj||}|sG|jPn|j|qW|jStj||SdS(N(R`RR6RtreadRmReRi(RHtsizetcontentstblocks((s(lib/python2.7/site-packages/nltk/data.pyRos     icC sL|s|j}n|jt||kr;|j|n |j|dS(u :param data: bytes to write to file or buffer :type data: bytes :param size: buffer at least size bytes before writing to file :type size: int N(R`RbRfRhRj(RHRgRp((s(lib/python2.7/site-packages/nltk/data.pyRes  iN(RMRNROtMBR_RRRTRdRhRjRltFLUSHRmRoRe(((s(lib/python2.7/site-packages/nltk/data.pyR]os        tGzipFileSystemPathPointercB seZdZddZRS(u A subclass of ``FileSystemPathPointer`` that identifies a gzip-compressed file located at a given absolute path. ``GzipFileSystemPathPointer`` is appropriate for loading large gzip-compressed pickle objects efficiently. cC sgtjjds$tjjdr9t|jd}nt|jd}|rct||}n|S(Nu2.7u3.4urb(R=tversionR(R]RSRRU(RHR"RV((s(lib/python2.7/site-packages/nltk/data.pyRIs $N(RMRNRORRI(((s(lib/python2.7/site-packages/nltk/data.pyRustZipFilePathPointercB sneZdZeddZedZedZd dZ dZ dZ dZ d Z RS( u~ A path pointer that identifies a file contained within a zipfile, which can be accessed by reading that zipfile. ucC st|tr*ttjj|}n|rt|tdjd}y|j |Wqt k r|j drg|j D]}|j |r|^qrqtd|j|fqXn||_||_dS(u Create a new path pointer pointing at the specified entry in the given zipfile. :raise IOError: If the given zipfile does not exist, or if it does not contain the specified entry. u/uZipfile %r does not contain %rN(t isinstanceRtOpenOnDemandZipFileR1R2RAR4R6R)tgetinfot ExceptionR;tnamelistR(RRRt_zipfilet_entry(RHtzipfiletentrytn((s(lib/python2.7/site-packages/nltk/data.pyRTs  . cC s|jS(u The zipfile.ZipFile object used to access the zip file containing the entry identified by this path pointer. (R}(RH((s(lib/python2.7/site-packages/nltk/data.pyRscC s|jS(u_ The name of the file within zipfile that this path pointer points to. (R~(RH((s(lib/python2.7/site-packages/nltk/data.pyRscC s|jj|j}t|}|jjdrtjjdsWtjjdrot|jd|}qt |jd|}n|dk rt ||}n|S(Nu.gzu2.7u3.4R#( R}RoR~RR;R=RvR(R]RRRU(RHR"RgRV((s(lib/python2.7/site-packages/nltk/data.pyRI%s $ cC s|jj|jjS(N(R}RzR~RK(RH((s(lib/python2.7/site-packages/nltk/data.pyRK4scC s#d|j|f}t|j|S(Nu%s/%s(R~RwR}(RHRLR((s(lib/python2.7/site-packages/nltk/data.pyR7scC std|jj|jfS(NuZipFilePathPointer(%r, %r)(RYR}RR~(RH((s(lib/python2.7/site-packages/nltk/data.pyRZ;scC s(tjjtjj|jj|jS(N(R1R2R?RR}RR~(RH((s(lib/python2.7/site-packages/nltk/data.pyR[>sN(RMRNRORRTR\RRRRIRKRRZR[(((s(lib/python2.7/site-packages/nltk/data.pyRws$    cC st|t}|dkr$t}ntjd|}|j\}}x6|D].}|rtjj|r|j dryt ||SWq}t k rqOq}XqO| stjj |rO|dkr"tjj |t|}tjj|rz|j drt|St|Sqzq}tjj |t|}tjj|r}yt ||SWqzt k rvqOqzXq}qOqOW|dkr |jd}xltt|D]U}dj || ||dg||} yt| |SWqtk rqXqWn|jdd} | j drC| jdd} ntdjd | } t| } | d 7} | d jd |7} | d dj d|D7} dd} d| | | f} t| dS(u Find the given resource by searching through the directories and zip files in paths, where a None or empty string specifies an absolute path. Returns a corresponding path name. If the given resource is not found, raise a ``LookupError``, whose message gives a pointer to the installation instructions for the NLTK downloader. Zip File Handling: - If ``resource_name`` contains a component with a ``.zip`` extension, then it is assumed to be a zipfile; and the remaining path components are used to look inside the zipfile. - If any element of ``nltk.data.path`` has a ``.zip`` extension, then it is assumed to be a zipfile. - If a given resource name that does not contain any zipfile component is not found initially, then ``find()`` will make a second attempt to find that resource, by replacing each component *p* in the path with *p.zip/p*. For example, this allows ``find()`` to map the resource name ``corpora/chat80/cities.pl`` to a zip file path pointer to ``corpora/chat80.zip/chat80/cities.pl``. - When using ``find()`` to locate a directory contained in a zipfile, the resource name must end with the forward slash character. Otherwise, ``find()`` will not locate the directory. :type resource_name: str or unicode :param resource_name: The name of the resource to search for. Resource names are posix-style relative path names, such as ``corpora/brown``. Directory names will be automatically converted to a platform-appropriate path separator. :rtype: str u(.*\.zip)/?(.*)$|u.zipu.gzu/iu.iuResource {resource} not found. Please use the NLTK Downloader to obtain the resource: >>> import nltk >>> nltk.download('{resource}') tresourceu< For more information see: https://www.nltk.org/data.html u. Attempted to load {resource_name} RCu Searched in:ucs s|]}d|VqdS(u - %rN((Rtd((s(lib/python2.7/site-packages/nltk/data.pys su*iFu %s %s %s N(R4R6RR2R*tmatchtgroupsR1tisfileR;RwRRtisdirRR RQRuRPR'trangeRftfindt LookupErrort rpartitionRYtformatR(RCtpathstmRtzipentryR.tptpiecestit modified_nametresource_zipnametmsgR<tresource_not_found((s(lib/python2.7/site-packages/nltk/data.pyRMs\%   '      *        ! cC st|}|d krX|jdr@tjj|d}qXtjdd|}ntjj|rtjj |}t d|n|rt d||fnt |}t |d:}x0tr|jd }|j||sPqqWWd QX|jd S( u Copy the given resource to a local file. If no filename is specified, then use the URL's filename. If there is already a file named ``filename``, then raise a ``ValueError``. :type resource_url: str :param resource_url: A URL specifying where the resource should be loaded from. The default protocol is "nltk:", which searches for the file in the the NLTK data package. ufile:iu (^\w+:)?.*/uuFile %r already exists!uRetrieving %r, saving to %ruwbii@Ni(R8RR(R1R2R'R*R+RQRAR0tprintt_openRIR6RoReRl(R,Rtverbosetinfiletoutfilets((s(lib/python2.7/site-packages/nltk/data.pytretrieves$     u;A serialized python object, stored using the pickle module.upickleu9A serialized python object, stored using the json module.ujsonu9A serialized python object, stored using the yaml module.uyamluA context free grammar.ucfguA probabilistic CFG.upcfguA feature CFG.ufcfguZA list of first order logic expressions, parsed with nltk.sem.logic.Expression.fromstring.ufoluA list of first order logic expressions, parsed with nltk.sem.logic.LogicParser. Requires an additional logic_parser parameterulogicu>A semantic valuation, parsed by nltk.sem.Valuation.fromstring.uvalu)The raw (byte string) contents of a file.urawu-The raw (unicode string) contents of a file. utextutxtuautocC st|}t|}|dkr|jd}|d}|dkrV|d}ntj|}|d krtd|qn|tkrtd|fn|rtj||f} | d k r|rt d|fn| Sn|r t d |fnt |} |d kr2| j } ni|d krPt j | } nK|d krdd l} ddlm} | j | } d } t| dkrt| j} n| | krtdqn|dkrdd l}|j | } n| j }|d k r&|j|}n6y|jd}Wn tk r[|jd}nX|dkrq|} n*|dkrtjjj|d|} n|dkrtjjj|d|} n|dkrtjjj|d|d|d|} n|dkr4tjj|dtjjj d|} ng|dkratjj|d|d|} n:|dkrtjj!|d|} nt"d|f| j#|ry| t||f>u<>urawupickleujsonN(t json_tagsiuUnknown json tag.uyamluutf-8ulatin-1utextucfgR"upcfgufcfgt logic_parsertfstruct_readerufolulogicuvalu@Internal NLTK error: Format %s isn't handled by nltk.data.load()(%R8RR't AUTO_FORMATSR^RR0tFORMATSt_resource_cacheRRRotpickletloadtjsont nltk.jsontagsRRftnexttkeystyamltdecodetUnicodeDecodeErrortnltktgrammartCFGt fromstringtPCFGtFeatureGrammartsemt read_logictlogict LogicParsertread_valuationtAssertionErrorRlt TypeError(R,RtcacheRRRR"tresource_url_partstextt resource_valtopened_resourceRRttagRt binary_datat string_data((s(lib/python2.7/site-packages/nltk/data.pyR s7                                    u##cC s|t|}t|dddt}|j}xE|D]=}|j|rRq7ntjd|rjq7nt|q7WdS(u} Write out a grammar file, ignoring escaped and empty lines. :type resource_url: str :param resource_url: A URL specifying where the resource should be loaded from. The default protocol is "nltk:", which searches for the file in the the NLTK data package. :type escape: str :param escape: Prepended string that signals lines to be ignored RutextRu^$N(R8RR5RR(R*RR(R,tescapeRtlinestl((s(lib/python2.7/site-packages/nltk/data.pytshow_cfgs   cC stjdS(uF Remove all objects from the resource cache. :see: load() N(Rtclear(((s(lib/python2.7/site-packages/nltk/data.pyt clear_cachescC st|}t|\}}|dks<|jdkrVt|tdgjS|jdkr~t|dgjSt|SdS(uo Helper function that returns an open file object for a resource, given its resource URL. If the given resource URL uses the "nltk:" protocol, or uses no protocol, then use ``nltk.data.find`` to find its path, and open it with the given mode; if the resource URL uses the 'file' protocol, then open the file with the given mode; otherwise, delegate to ``urllib2.urlopen``. :type resource_url: str :param resource_url: A URL specifying where the resource should be loaded from. The default protocol is "nltk:", which searches for the file in the the NLTK data package. unltkuufileN(R8R/RtlowerRR2RIR (R,R-R.((s(lib/python2.7/site-packages/nltk/data.pyRs t LazyLoadercB s2eZedZdZdZdZRS(cC s ||_dS(N(RS(RHRS((s(lib/python2.7/site-packages/nltk/data.pyRTscC s+t|j}|j|_|j|_dS(N(RRSt__dict__t __class__(RHR((s(lib/python2.7/site-packages/nltk/data.pyt__loads cC s|jt||S(N(t_LazyLoader__loadtgetattr(RHtattr((s(lib/python2.7/site-packages/nltk/data.pyt __getattr__s cC s|jt|S(N(Rtrepr(RH((s(lib/python2.7/site-packages/nltk/data.pyRZs (RMRNRRTRRRZ(((s(lib/python2.7/site-packages/nltk/data.pyRs  RycB sAeZdZedZdZdZdZdZRS(u A subclass of ``zipfile.ZipFile`` that closes its file pointer whenever it is not using it; and re-opens it when it needs to read data from the zipfile. This is useful for reducing the number of open file handles when many zip files are being accessed at once. ``OpenOnDemandZipFile`` must be constructed from a filename, not a file-like object (to allow re-opening). ``OpenOnDemandZipFile`` is read-only (i.e. ``write()`` and ``writestr()`` are disabled. cC s]t|tstdntjj|||j|ksFt|jd|_ dS(Nu+ReopenableZipFile filename must be a stringi( RxRRRtZipFileRTRRRlt _fileRefCnt(RHR((s(lib/python2.7/site-packages/nltk/data.pyRTs  cC s\|jdkstt|jd|_tjj||}|jd7_|j |S(Nurbi( tfpRRRIRRRRoRRl(RHR7tvalue((s(lib/python2.7/site-packages/nltk/data.pyRo&s  cO stddS(u<:raise NotImplementedError: OpenOnDemandZipfile is read-onlyu OpenOnDemandZipfile is read-onlyN(tNotImplementedError(RHtargsRc((s(lib/python2.7/site-packages/nltk/data.pyRe0scO stddS(u<:raise NotImplementedError: OpenOnDemandZipfile is read-onlyu OpenOnDemandZipfile is read-onlyN(R(RHRRc((s(lib/python2.7/site-packages/nltk/data.pytwritestr4scC sttd|jS(NuOpenOnDemandZipFile(%r)(RRYR(RH((s(lib/python2.7/site-packages/nltk/data.pyRZ8s( RMRNRORRTRoReRRZ(((s(lib/python2.7/site-packages/nltk/data.pyRys    RUcB seZdZeZeddZd#dZdZ d#dZ d#edZ dZ dZ d Zd Zd Zed Zed ZedZdZddZdZd#dZdZd#dZdZiejd#fgd6ejdfejdfgd6ejd#fgd6ejd#fgd6ej dfej!dfgd6ej d#fgd 6ej!d#fgd!6Z"d"Z#RS($u A stream reader that automatically encodes the source byte stream into unicode (like ``codecs.StreamReader``); but still supports the ``seek()`` and ``tell()`` operations correctly. This is in contrast to ``codecs.StreamReader``, which provide *broken* ``seek()`` and ``tell()`` methods. This class was motivated by ``StreamBackedCorpusView``, which makes extensive use of ``seek()`` and ``tell()``, and needs to be able to handle unicode-encoded files. Note: this class requires stateless decoders. To my knowledge, this shouldn't cause a problem with any of python's builtin unicode encodings. ustrictcC sq|jd||_||_||_tj||_d|_d|_ d|_ d|_ |j |_ dS(NiRJ(tseekRVR"R$tcodecst getdecoderRt bytebufferRt linebuffert_rewind_checkpointt_rewind_numcharst _check_bomt_bom(RHRVR"R$((s(lib/python2.7/site-packages/nltk/data.pyRTTs       cC sG|j|}|jrCdj|j|}d|_d|_n|S(u6 Read up to ``size`` bytes, decode them using this reader's encoding, and return the resulting unicode string. :param size: The maximum number of bytes to read. If not specified, then read as many bytes as possible. :type size: int :rtype: unicode uN(t_readRRRR(RHRptchars((s(lib/python2.7/site-packages/nltk/data.pyRos    cC sY|jrHt|jdkrH|jjd}|jt|7_n |jjdS(Nii(RRftpopRRVtreadline(RHR((s(lib/python2.7/site-packages/nltk/data.pyt discard_linesc C s|jrIt|jdkrI|jjd}|jt|7_|S|pRd}d}|jr||jj7}d|_nxTtr|jjt|j}|j |}|r|j dr||j d7}n||7}|j t}t|dkrT|d}|d|_t|t|t||_||_ PnLt|dkr|d}|dj t d} || kr|}Pqn| s|dk r|}Pn|dkr|d9}qqW|S( uj Read a line of text, decode it using this reader's encoding, and return the resulting unicode string. :param size: The maximum number of bytes to read. If no newline is encountered before ``size`` bytes have been read, then the returned value may not be a complete line of text. :type size: int iiiHuu i@iN(RRfRRRR6RVttellRRR;RRR5( RHRpRtreadsizeRtstartpost new_charsRt line0withendtline0withoutend((s(lib/python2.7/site-packages/nltk/data.pyRsD        #    cC s|jj|S(u Read this file's contents, decode them using this reader's encoding, and return it as a list of unicode lines. :rtype: list(unicode) :param sizehint: Ignored. :param keepends: If false, then strip newlines. (RoR(RHtsizehinttkeepends((s(lib/python2.7/site-packages/nltk/data.pyt readliness cC s |j}|r|StdS(u8Return the next decoded line from the underlying stream.N(Rt StopIteration(RHR((s(lib/python2.7/site-packages/nltk/data.pyRs cC s |jS(N(R(RH((s(lib/python2.7/site-packages/nltk/data.pyt__next__scC s|S(u Return self((RH((s(lib/python2.7/site-packages/nltk/data.pyt__iter__scC s|js|jndS(N(tclosedRl(RH((s(lib/python2.7/site-packages/nltk/data.pyt__del__s cC s|S(u Return self((RH((s(lib/python2.7/site-packages/nltk/data.pyt xreadlinesscC s |jjS(u(True if the underlying stream is closed.(RVR(RH((s(lib/python2.7/site-packages/nltk/data.pyR scC s |jjS(u"The name of the underlying stream.(RVR7(RH((s(lib/python2.7/site-packages/nltk/data.pyR7scC s |jjS(u"The mode of the underlying stream.(RVR (RH((s(lib/python2.7/site-packages/nltk/data.pyR scC s|jjdS(u. Close the underlying stream. N(RVRl(RH((s(lib/python2.7/site-packages/nltk/data.pyRlsicC s_|dkrtdn|jj||d|_d|_d|_|jj|_dS(u Move the stream to a new file position. If the reader is maintaining any buffers, then they will be cleared. :param offset: A byte count offset. :param whence: If 0, then the offset is from the start of the file (offset should be positive), if 1, then the offset is from the current position (offset may be positive or negative); and if 2, then the offset is from the end of the file (offset should typically be negative). iumRelative seek is not supported for SeekableUnicodeStreamReader -- consider using char_seek_forward() instead.RJN( R0RVRRRRRRR(RHtoffsettwhence((s(lib/python2.7/site-packages/nltk/data.pyR$s     cC s?|dkrtdn|j|j|j|dS(uI Move the read pointer forward by ``offset`` characters. iu"Negative offsets are not supportedN(R0RRt_char_seek_forward(RHR((s(lib/python2.7/site-packages/nltk/data.pytchar_seek_forward<s cC s)|dkr|}nd}xtr$|jj|t|}||7}|j|\}}t||kr|jjt| |ddSt||kr xCt||kr||t|7}|j|| \}}qW|jjt| |ddS||t|7}qWdS(u Move the file position forward by ``offset`` characters, ignoring all buffers. :param est_bytes: A hint, giving an estimate of the number of bytes that will be needed to move forward by ``offset`` chars. Defaults to ``offset``. RJiN(RR6RVRoRft _incr_decodeR(RHRt est_bytestbytestnewbytesRt bytes_decoded((s(lib/python2.7/site-packages/nltk/data.pyRGs"    cC sC|jdkr)|jjt|jS|jj}|t|j|j}td|jD}t||j |j |}|jj |j|j |j ||jj}|j r/|jj ||j |jjdd}dj|j}|j|s/|j|s/tn|jj ||S(u Return the current file position on the underlying byte stream. If this reader is maintaining any buffers, then the returned file position will be the position of the beginning of those buffers. cs s|]}t|VqdS(N(Rf(RR((s(lib/python2.7/site-packages/nltk/data.pys si2iuN(RRRVRRfRRtsumtintRRRtDEBUGRRoRR(R(RHt orig_filepost bytes_readtbuf_sizeRtfilepostcheck1tcheck2((s(lib/python2.7/site-packages/nltk/data.pyRos" 'cC s|dkrdS|jrD|jjdkrD|jj|jn|dkrb|jj}n|jj|}|j|}|j|\}}|dk r| rt|dkrxH|s|jjd}|sPn||7}|j|\}}qWn|||_|S(u Read up to ``size`` bytes from the underlying stream, decode them using this reader's encoding, and return the resulting unicode string. ``linebuffer`` is not included in the result. iuiN(RRVRRoRRRRf(RHRpt new_bytesRRR((s(lib/python2.7/site-packages/nltk/data.pyRs$   %   cC sxtry|j|dSWqtk r}|jt|kr^|j||j |jS|jdkrsq|j||jSqXqWdS(u Decode the given byte string into a unicode string, using this reader's encoding. If an exception is encountered that appears to be caused by a truncation error, then just decode the byte string without the bytes that cause the trunctaion error. Return a tuple ``(chars, num_consumed)``, where ``chars`` is the decoded unicode string, and ``num_consumed`` is the number of bytes that were consumed. ustrictN(R6RRtendRftstartR$(RHRtexc((s(lib/python2.7/site-packages/nltk/data.pyRs uutf8uutf16-leuutf16-beuutf16uutf16leuutf16beuutf32-leuutf32-beuutf32uutf32leuutf32becC stjdd|jj}|jj|}|r|jjd}|jjdxB|D]7\}}|j |r_|r||_nt |Sq_WndS(Nu[ -]uii( R*R+R"Rt _BOM_TABLER^RVRoRR(RfR(RHtenctbom_infoRtbomt new_encoding((s(lib/python2.7/site-packages/nltk/data.pyRs N($RMRNROR6RRRTRRoRRRRRRRRR\RR7R RlRRRRRRRtBOM_UTF8t BOM_UTF16_LEt BOM_UTF16_BEt BOM_UTF32_LEt BOM_UTF32_BER R(((s(lib/python2.7/site-packages/nltk/data.pyRUAs@8   <       ( . & upathu PathPointeruFileSystemPathPointeruBufferedGzipFileuGzipFileSystemPathPointerufinduretrieveuFORMATSu AUTO_FORMATSuloadushow_cfgu clear_cacheu LazyLoaderuOpenOnDemandZipFileuSeekableUnicodeStreamReader(VROt __future__RRRt functoolsttextwrapRR1R*R=RRtabcRRtgzipRRRktsixRRR tsix.moves.urllib.requestR R tcPickleRt ImportErrortpartialtindentRtAttributeErrortfillR5RtzlibRRtRRt nltk.compatRRRR2tenvironR^RYR'tpathsept_paths_from_envRt expandusertappendR>R(RR RR&R/R8R6R4tobjectRGRPR]RuRwRRRRRRRRRRRRyRUt__all__(((s(lib/python2.7/site-packages/nltk/data.pyt s               '#'"$$3  $$       85 )2g^ s)     "1