ó
‡ˆ\c           @   sc  d  Z  d d l m Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l	 Z	 d d l
 m Z y e Wn e k
 r­ e d ƒ Z n Xd d l m Z y$ d d l m Z d d l m Z Wn1 e k
 rd d l m Z d d l m Z n Xd d l m Z m Z y  d d l Z d d	 l m Z Wn e k
 red Z n Xd
 d l m Z d
 d l m Z d
 d l m  Z  d
 d l! m" Z" d Z# e$ d ƒ Z% e j& e j' Be j( BZ) e j& e j' BZ* d d d „  ƒ  YZ+ d „  Z, d d „ Z- d „  Z. d „  Z/ d „  Z0 d „  Z1 d e2 f d „  ƒ  YZ3 d d d d d d e4 d „ Z5 d S(    s/   
Reducer using memory mapping for numpy arrays
iÿÿÿÿ(   t   mmapN(   t   uuid4(   t   whichmodule(   t   loads(   t   dumps(   t   HIGHEST_PROTOCOLt   PicklingError(   t
   as_stridedi   (   t   load(   t   dump(   t   make_memmap(   t   delete_folders   /dev/shmg    eÍÝAt   _WeakArrayKeyMapc           B   s2   e  Z d  Z d „  Z d „  Z d „  Z d „  Z RS(   s  A variant of weakref.WeakKeyDictionary for unhashable numpy arrays.

    This datastructure will be used with numpy arrays as obj keys, therefore we
    do not use the __get__ / __set__ methods to avoid any conflict with the
    numpy fancy indexing syntax.
    c         C   s   i  |  _  d  S(   N(   t   _data(   t   self(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   __init__E   s    c         C   s;   |  j  t | ƒ \ } } | ƒ  | k	 r7 t | ƒ ‚ n  | S(   N(   R   t   idt   KeyError(   R   t   objt   reft   val(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   getH   s    c            s   t  | ƒ ‰  y5 ˆ j ˆ  \ } } | ƒ  | k	 r@ t | ƒ ‚ n  Wn5 t k
 rx ‡  ‡ f d †  } t j | | ƒ } n X| | f ˆ j ˆ  <d  S(   Nc            s   ˆ j  ˆ  =d  S(   N(   R   (   t   _(   t   keyR   (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt
   on_destroy\   s    (   R   R   R   t   weakrefR   (   R   R   t   valueR   R   R   (    (   R   R   sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   setP   s    c         C   s   t  d ƒ ‚ d  S(   Ns"   _WeakArrayKeyMap is not pickleable(   R   (   R   (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   __getstate__a   s    (   t   __name__t
   __module__t   __doc__R   R   R   R   (    (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyR   =   s
   			c         C   sC   t  |  d d ƒ } | d k r" d St | t ƒ r5 |  St | ƒ Sd S(   s@   Recursively look up the original np.memmap instance base if any.t   baseN(   t   getattrt   Nonet
   isinstanceR    t   _get_backing_memmap(   t   at   b(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyR$   i   s    c         C   s;  t  } | d k r* t j j d d ƒ } n  | d k rã t j j t ƒ rã yu t j t ƒ } | j	 | j
 } | t k r¼ t } t j j | |  ƒ } t j j | ƒ s³ t j | ƒ n  t } n  Wqà t t f k
 rÜ d } qà Xqã n  | d k rþ t j ƒ  } n  t j j t j j | ƒ ƒ } t j j | |  ƒ } | | f S(   s  Get the full path to a subfolder inside the temporary folder.

    Parameters
    ----------
    pool_folder_name : str
        Sub-folder name used for the serialization of a pool instance.

    temp_folder: str, optional
        Folder to be used by the pool for memmapping large arrays
        for sharing memory with worker processes. If None, this will try in
        order:

        - a folder pointed by the JOBLIB_TEMP_FOLDER environment
          variable,
        - /dev/shm if the folder exists and is writable: this is a
          RAMdisk filesystem available by default on modern Linux
          distributions,
        - the default system temporary folder that can be
          overridden with TMP, TMPDIR or TEMP environment
          variables, typically /tmp under Unix operating systems.

    Returns
    -------
    pool_folder : str
       full path to the temporary folder
    use_shared_mem : bool
       whether the temporary folder is written to the system shared memory
       folder or some other temporary folder.
    t   JOBLIB_TEMP_FOLDERN(   t   FalseR"   t   ost   environR   t   patht   existst   SYSTEM_SHARED_MEM_FSt   statvfst   f_bsizet   f_bavailt   SYSTEM_SHARED_MEM_FS_MIN_SIZEt   joint   makedirst   Truet   IOErrort   OSErrort   tempfilet
   gettempdirt   abspatht
   expanduser(   t   pool_folder_namet   temp_foldert   use_shared_memt	   shm_statst   available_nbytest   pool_folder(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   _get_temp_dirz   s*    c         C   s   t  |  ƒ d k	 S(   s?   Return True if a is backed by some mmap buffer directly or not.N(   R$   R"   (   R%   (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   has_shareable_memoryµ   s    c   	      C   s   | d k r d } n  | d	 k rI t |  d | d | d | d | d | ƒSt |  d | d | d | d | d | ƒ} t | d | d | ƒSd	 S(
   s2   Reconstruct an array view on a memory mapped file.s   w+s   r+t   dtypet   shapet   modet   offsett   ordert   stridesN(   R"   R
   R   (	   t   filenameRC   RE   RF   RG   RD   RH   t   total_buffer_lenR    (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   _strided_from_memmapº   s    	c   	   	   C   sÌ   t  j |  ƒ \ } } t  j | ƒ d } | | } | | j 7} | j d rU d } n d } |  j d su |  j d r„ d } d } n |  j } | | |  j } t | j |  j	 | j
 | | |  j | | f f S(   s  Pickling reduction for memmap backed arrays.

    a is expected to be an instance of np.ndarray (or np.memmap)
    m is expected to be an instance of np.memmap on the top of the ``base``
    attribute ancestry of a. ``m.base`` should be the real python mmap object.
    i    t   F_CONTIGUOUSt   Ft   Ct   C_CONTIGUOUSN(   t   npt   byte_boundsRF   t   flagsR"   RH   t   itemsizeRK   RI   RC   RE   RD   (	   R%   t   mt   a_startt   a_endt   m_startRF   RG   RH   RJ   (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   _reduce_memmap_backedÍ   s    
			!c         C   sK   t  |  ƒ } | d k	 r% t |  | ƒ St t t j |  ƒ d t ƒf f Sd S(   sC   Pickle the descriptors of a memmap instance to reopen on same file.t   protocolN(   R$   R"   RX   R   R   RP   t   asarrayR   (   R%   RT   (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   reduce_memmapñ   s    t   ArrayMemmapReducerc           B   s/   e  Z d  Z d e d „ Z d „  Z d „  Z RS(   sã  Reducer callable to dump large arrays to memmap files.

    Parameters
    ----------
    max_nbytes: int
        Threshold to trigger memmapping of large arrays to files created
        a folder.
    temp_folder: str
        Path of a folder where files for backing memmapped arrays are created.
    mmap_mode: 'r', 'r+' or 'c'
        Mode for the created memmap datastructure. See the documentation of
        numpy.memmap for more details. Note: 'w+' is coerced to 'r+'
        automatically to avoid zeroing the data on unpickling.
    verbose: int, optional, 0 by default
        If verbose > 0, memmap creations are logged.
        If verbose > 1, both memmap creations, reuse and array pickling are
        logged.
    prewarm: bool, optional, False by default.
        Force a read on newly memmapped array to make sure that OS pre-cache it
        memory. This can be useful to avoid concurrent disk access when the
        same data array is passed to different worker processes.
    i    c         C   sC   | |  _  | |  _ | |  _ t | ƒ |  _ | |  _ t ƒ  |  _ d  S(   N(   t   _max_nbytest   _temp_foldert
   _mmap_modet   intt   verboset   _prewarmR   t   _memmaped_arrays(   R   t
   max_nbytesR<   t	   mmap_modeRa   t   prewarm(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyR     s    				c         C   s?   |  j  |  j |  j f } i |  j d 6|  j d 6} t | | f S(   NRa   Rf   (   R]   R^   R_   Ra   Rb   R\   (   R   t   argst   kwargs(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt
   __reduce__   s
    
c         C   sB  t  | ƒ } | d  k	 r% t | | ƒ S| j j rü|  j d  k	 rü| j |  j k rüy' t j |  j	 ƒ t j
 |  j	 t ƒ Wn. t k
 rª } | j t j k r« | ‚ q« n Xy |  j j | ƒ } WnQ t k
 rd j t j ƒ  t t j ƒ  ƒ t ƒ  j ƒ } |  j j | | ƒ n Xt j j |  j	 | ƒ } t j j | ƒ s½|  j d k rkd j | j | j | ƒ GHn  x' t | | ƒ D] } t j
 | t  ƒ q{W|  j! rét" | d |  j# ƒj$ ƒ  qén, |  j d k réd j | j | j | ƒ GHn  t" | |  j# f f S|  j d k r%d j | j | j ƒ GHn  t% t& | d t' ƒf f Sd  S(	   Ns   {}-{}-{}.pkli    s.   Memmapping (shape={}, dtype={}) to new file {}Re   i   s.   Memmapping (shape={}, dtype={}) to old file {}s$   Pickling array (shape={}, dtype={}).RY   ((   R$   R"   RX   RC   t	   hasobjectR]   t   nbytesR)   R3   R^   t   chmodt   FOLDER_PERMISSIONSR6   t   errnot   EEXISTRc   R   R   t   formatt   getpidR   t	   threadingt   current_threadR   t   hexR   R+   R2   R,   Ra   RD   R	   t   FILE_PERMISSIONSRb   R   R_   t   maxR   R   R   (   R   R%   RT   t   et   basenameRI   t   dumped_filename(    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   __call__,  sD    '	(   R   R   R   R4   R   Ri   Rz   (    (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyR\   ÿ   s
   	g    €„.At   ri    c            s!  | d k r t ƒ  } n  | d k r0 t ƒ  } n  d j t j ƒ  |  ƒ }	 t |	 | ƒ \ ‰  }
 t t d ƒ ‰ ‡  ‡ f d †  } t j	 | ƒ t
 d k	 r| d k r­ |
 } n  t | ˆ  | | d | ƒ} | | t
 j <t | t
 j <t d ˆ  | | ƒ } | | t
 j <t | t
 j <n  | | ˆ  f S(   s	  Construct a pair of memmapping reducer linked to a tmpdir.

    This function manage the creation and the clean up of the temporary folders
    underlying the memory maps and should be use to get the reducers necessary
    to construct joblib pool or executor.
    s   joblib_memmapping_folder_{}_{}R   c             sT   t  ˆ d d g ƒj }  y |  ˆ  ƒ Wn' t k
 rO t j d j ˆ  ƒ ƒ n Xd  S(   Nt   fromlistR   s$   Failed to clean temporary folder: {}(   t
   __import__R   t   WindowsErrort   warningst   warnRp   (   R   (   R@   t   pool_module_name(    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   _cleanupŠ  s    	t   autoRf   N(   R"   t   dictRp   R)   Rq   RA   R   R   t   atexitt   registerRP   R\   t   ndarrayR[   t   memmap(   t   pool_idt   forward_reducerst   backward_reducersR<   Rd   Re   Ra   Rf   Rh   R;   R=   R‚   t   forward_reduce_ndarrayt   backward_reduce_ndarray(    (   R@   R   sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   get_memmapping_reducersl  s0    

	(    (6   R   R    Rn   R)   t   statRr   R…   R7   R   R   t   uuidR   R~   t	   NameErrort   typeR"   t   pickleR   t   cPickleR   R   t   ImportErrorR   R   t   numpyRP   t   numpy.lib.stride_tricksR   t   numpy_pickleR   R	   t	   backportsR
   t   diskR   R-   R`   R1   t   S_IRUSRt   S_IWUSRt   S_IXUSRRm   Ru   R   R$   RA   RB   RK   RX   R[   t   objectR\   R(   RŽ   (    (    (    sK   lib/python2.7/site-packages/sklearn/externals/joblib/_memmapping_reducer.pyt   <module>   s\   
,	;			$	n