ó
¦–Õ\c           @   sq   d  d l  Z  d d l m Z d d l m Z d d l m Z d e f d „  ƒ  YZ d d d i  d d d	 „ Z	 d S(
   iÿÿÿÿNi   (   t   ignoring(   t   is_dask_collection(   t   Mappingt   HighLevelGraphc           B   sz   e  Z d  Z d „  Z e d „  ƒ Z e d d „ ƒ Z d „  Z d „  Z	 d „  Z
 d „  Z e d „  ƒ Z d	 d d
 „ Z RS(   sÉ	   Task graph composed of layers of dependent subgraphs

    This object encodes a Dask task graph that is composed of layers of
    dependent subgraphs, such as commonly occurs when building task graphs
    using high level collections like Dask array, bag, or dataframe.

    Typically each high level array, bag, or dataframe operation takes the task
    graphs of the input collections, merges them, and then adds one or more new
    layers of tasks for the new operation.  These layers typically have at
    least as many tasks as there are partitions or chunks in the collection.
    The HighLevelGraph object stores the subgraphs for each operation
    separately in sub-graphs, and also stores the dependency structure between
    them.

    Parameters
    ----------
    layers : Dict[str, Mapping]
        The subgraph layers, keyed by a unique name
    dependencies : Dict[str, Set[str]]
        The set of layers on which each layer depends

    Examples
    --------

    Here is an idealized example that shows the internal state of a
    HighLevelGraph

    >>> import dask.dataframe as dd

    >>> df = dd.read_csv('myfile.*.csv')  # doctest: +SKIP
    >>> df = df + 100  # doctest: +SKIP
    >>> df = df[df.name == 'Alice']  # doctest: +SKIP

    >>> graph = df.__dask_graph__()  # doctest: +SKIP
    >>> graph.layers  # doctest: +SKIP
    {
     'read-csv': {('read-csv', 0): (pandas.read_csv, 'myfile.0.csv'),
                  ('read-csv', 1): (pandas.read_csv, 'myfile.1.csv'),
                  ('read-csv', 2): (pandas.read_csv, 'myfile.2.csv'),
                  ('read-csv', 3): (pandas.read_csv, 'myfile.3.csv')},
     'add': {('add', 0): (operator.add, ('read-csv', 0), 100),
             ('add', 1): (operator.add, ('read-csv', 1), 100),
             ('add', 2): (operator.add, ('read-csv', 2), 100),
             ('add', 3): (operator.add, ('read-csv', 3), 100)}
     'filter': {('filter', 0): (lambda part: part[part.name == 'Alice'], ('add', 0)),
                ('filter', 1): (lambda part: part[part.name == 'Alice'], ('add', 1)),
                ('filter', 2): (lambda part: part[part.name == 'Alice'], ('add', 2)),
                ('filter', 3): (lambda part: part[part.name == 'Alice'], ('add', 3))}
    }

    >>> graph.dependencies  # doctest: +SKIP
    {
     'read-csv': set(),
     'add': {'read-csv'},
     'filter': {'add'}
    }

    See Also
    --------
    HighLevelGraph.from_collections :
        typically used by developers to make new HighLevelGraphs
    c         C   s   | |  _  | |  _ d  S(   N(   t   layerst   dependencies(   t   selfR   R   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   __init__G   s    	c         C   s   |  j  S(   N(   R   (   R   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   dictsK   s    c   	   
   C   s  i | | 6} i  } t  ƒ  | | <xß t j | d t ƒD]È } t | ƒ rì | j ƒ  } t | t ƒ rµ | j | j	 ƒ | j | j
 ƒ t t ƒ ! | | c t  | j ƒ  ƒ O<Wd QXqþ t | ƒ } | | | <| | j | ƒ t  ƒ  | | <q6 t t | ƒ ƒ ‚ q6 W|  | | ƒ S(   sc   Construct a HighLevelGraph from a new layer and a set of collections

        This constructs a HighLevelGraph in the common case where we have a single
        new layer and a set of old collections on which we want to depend.

        This pulls out the ``__dask_layers__()`` method of the collections if
        they exist, and adds them to the dependencies for this new layer.  It
        also merges all of the layers from all of the dependent collections
        together into the new layers for this graph.

        Parameters
        ----------
        name : str
            The name of the new layer
        layer : Mapping
            The graph layer itself
        dependencies : List of Dask collections
            A lit of other dask collections (like arrays or dataframes) that
            have graphs themselves

        Examples
        --------

        In typical usage we make a new task layer, and then pass that layer
        along with all dependent collections to this method.

        >>> def add(self, other):
        ...     name = 'add-' + tokenize(self, other)
        ...     layer = {(name, i): (add, input_key, other)
        ...              for i, input_key in enumerate(self.__dask_keys__())}
        ...     graph = HighLevelGraph.from_collections(name, layer, dependencies=[self])
        ...     return new_collection(name, graph)
        t   keyN(   t   sett   toolzt   uniquet   idR   t   __dask_graph__t
   isinstanceR   t   updateR   R   R    t   AttributeErrort   __dask_layers__t   addt	   TypeErrort   type(	   t   clst   namet   layerR   R   t   depst
   collectiont   graphR	   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   from_collectionsP   s"    #%
c         C   s>   x+ |  j  j ƒ  D] } | | k r | | Sq Wt | ƒ ‚ d  S(   N(   R   t   valuest   KeyError(   R   R	   t   d(    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   __getitem__ˆ   s    c         C   s   t  d „  |  Dƒ ƒ S(   Nc         s   s   |  ] } d  Vq d S(   i   N(    (   t   .0t   _(    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pys	   <genexpr>   s    (   t   sum(   R   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   __len__Ž   s    c         c   sc   t  ƒ  } xS |  j j ƒ  D]B } x9 | D]1 } | | k r& | j | ƒ | | | f Vq& q& Wq Wd  S(   N(   R
   R   R   R   (   R   t   seenR   R	   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   items‘   s    	c         C   s   t  j t  j |  j j ƒ  ƒ ƒ S(   N(   R   R   t   concatR   R   (   R   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   __iter__™   s    c         G   s   i  } i  } x | D]y } t  | t ƒ rK | j | j ƒ | j | j ƒ q t  | t ƒ r€ | | t | ƒ <t ƒ  | t | ƒ <q t | ƒ ‚ q W|  | | ƒ S(   N(	   R   R   R   R   R   R   R   R
   R   (   R   t   graphsR   R   t   g(    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   mergeœ   s    s   dask.pdfc         K   s/   d d l  m } t |  |  } | | | | ƒ S(   Ni   (   t   graphviz_to_file(   t   dotR,   t   to_graphviz(   R   t   filenamet   formatt   kwargsR,   R*   (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt	   visualize«   s    (    N(   t   __name__t
   __module__t   __doc__R   t   propertyR   t   classmethodR   R    R$   R&   R(   R+   t   NoneR2   (    (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyR      s   >	7				t   BTc         K   sX  d d l  m } m }	 m }
 | d  k r1 i  } n  | d  k rF i  } n  | pO i  } | | d <| j | ƒ | j d | d | d | ƒ } i  } xk |  j D]` } |	 | ƒ } | j | i  ƒ } | j	 d |
 | d | ƒƒ | j	 d	 d
 ƒ | j
 | |  q— WxV |  j j ƒ  D]E \ } } |	 | ƒ } x* | D]" } |	 | ƒ } | j | | ƒ q*WqW| S(   Ni   (   t   graphvizR   t   labelt   rankdirt
   graph_attrt	   node_attrt	   edge_attrR;   t   cachet   shapet   box(   R-   R:   R   R;   R8   R   t   DigraphR   t   gett
   setdefaultt   nodeR&   t   edge(   t   hgt   data_attributest   function_attributesR<   R=   R>   R?   R1   R:   R   R;   R*   R@   t   kt   k_namet   attrsR   t   dept   dep_name(    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyR.   ±   s0    		
	(
   R   t   utilsR    t   baseR   t   compatibilityR   R   R8   R.   (    (    (    s2   lib/python2.7/site-packages/dask/highlevelgraph.pyt   <module>   s   ©