ó
ßüÚ\c           @` s  d  Z  d d l m Z m Z m Z d d l Z d d l Z d d l Z d d l m	 Z	 m
 Z
 y> d d l Z e j d ƒ Z e d d ƒ r™ e j e ƒ n  Wn e k
 r³ e Z n Xd d l m Z m Z i i e	 d	 6e	 d
 6d 6Z e d e e <d Z e j e ƒ Z e e ƒ / d d l Z i e j d	 6e j d
 6e d <Wd QXe e ƒ ; d d l Z d „  Z  i e j d	 6e  d
 6e d <d Z Wd QXe e ƒ  d d l! Z! y( d d l" Z! e! j# j Z$ e! j# j Z% Wn# e k
 rôe! j& Z$ e! j' Z% n Xd „  Z( d „  Z) i e( d	 6e) d
 6e d <d Z Wd QXe e ƒ A d d l Z i e
 e j d d d d ƒd	 6e j d
 6e d <Wd QXe j* j+ d ƒ Z, e, d k rÞe, e k r­e, Z qÞe- d e, d j. e/ e0 e1 e ƒ ƒ ƒ f ƒ ‚ n  d „  Z2 d d d d „ Z3 d „  Z d S(   sY   
Record known compressors

Includes utilities for determining whether or not to compress
i    (   t   print_functiont   divisiont   absolute_importN(   t   identityt   partiali   t   blosct
   releasegil(   t   ignoringt   ensure_bytest   compresst
   decompresst   zlibc         C` s1   t  |  t t f ƒ r$ t |  ƒ }  n  t j |  ƒ S(   N(   t
   isinstancet
   memoryviewt	   bytearrayt   bytest   snappyR
   (   t   data(    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyt   _fixed_snappy_decompress-   s    R   c         C` sN   y t  |  ƒ SWn9 t k
 rI t |  t t f ƒ rC t  t |  ƒ ƒ S‚  n Xd  S(   N(   t   lz4_compresst	   TypeErrorR   R   R   R   (   R   (    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyt   _fixed_lz4_compressJ   s    c         C` sT   y t  |  ƒ SWn? t t f k
 rO t |  t t f ƒ rI t  t |  ƒ ƒ S‚  n Xd  S(   N(   t   lz4_decompresst
   ValueErrorR   R   R   R   R   (   R   (    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyt   _fixed_lz4_decompressS   s    t   lz4t   cleveli   t   cnames   distributed.comm.compressiont   autos<   Default compression '%s' not found.
Choices include auto, %ss   , c   
      C` sØ   g  t  | ƒ D]" } t j d t |  ƒ | ƒ ^ q } g  } x@ t | d  ƒ D]. \ } } | j t | | | | d ƒ ƒ qL W| j | d | ƒ g  t | | ƒ D] \ } } |  | | !^ q£ }	 d j t	 t
 |	 ƒ ƒ S(   sÌ    Sample a bytestring from many locations

    Parameters
    ----------
    b: bytes or memoryview
    size: int
        size of each sample to collect
    n: int
        number of samples to collect
    i    iÿÿÿÿi   t    (   t   ranget   randomt   randintt   lent	   enumeratet   appendt   mint   zipt   joint   mapR   (
   t   bt   sizet   nt   jt   startst   endst   it   startt   endt   parts(    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyt   byte_samplew   s    5&/g     ˆÃ@c   	      C` s’  t  j j d ƒ } | d k r' t } n  | s7 d |  f St |  ƒ | k  rS d |  f St |  ƒ d k ro d |  f St | ƒ } t | ƒ } t | d } t |  | | ƒ } t | | ƒ ƒ d t | ƒ k rÓ d |  f St	 |  ƒ t
 k rû |  j t |  ƒ } n t |  ƒ } t rRt rRt	 |  ƒ t
 k rRt j |  d |  j d d	 d
 d ƒ} d } n | t |  ƒ ƒ } t | ƒ d | k r„d |  f S| | f Sd S(   sX  
    Maybe compress payload

    1.  We don't compress small messages
    2.  We sample the payload in a few spots, compress that, and if it doesn't
        do any good we return the original
    3.  We then compress the full original, it it doesn't compress well then we
        return the original
    4.  We return the compressed result
    s   distributed.comm.compressionR   i   i   R	   gÍÌÌÌÌÌì?t   typesizeR   R   R   i   R   NI   €    (   t   daskt   configt   gett   default_compressiont   NoneR!   t   intt   compressionsR2   t   typeR   t   itemsizeR   R	   R   (	   t   payloadt   min_sizet   sample_sizet   nsamplest   compressionR	   t   samplet   nbytest
   compressed(    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyt   maybe_compressŒ   s4    	


"
	
c         C` s8   g  t  |  d | ƒ D]  \ } } t | d | ƒ ^ q S(   s:    Decompress frames according to information in the header RA   R
   (   R%   R:   (   t   headert   framest   ct   frame(    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyR
   À   s    (4   t   __doc__t
   __future__R    R   R   t   loggingR   R4   t   toolzR   R   R   t   set_nthreadsR*   t   hasattrt   set_releasegilt   Truet   ImportErrort   Falset   utilsR   R   R8   R:   R7   t	   getLoggert   __name__t   loggerR   R	   R
   R   R   R   t	   lz4.blockt   blockR   R   t   LZ4_compresst   LZ4_uncompressR   R   R5   R6   t   defaultR   R&   t   sortedR'   t   strR2   RE   (    (    (    s?   lib/python2.7/site-packages/distributed/protocol/compression.pyt   <module>   sr   
$	
						+	4