B
    x\B             
   @   s  d Z ddlZddlmZmZ ddlmZ ddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ dd
l%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z> ddl?m@Z@mAZA ddlBmCZC ddlDmE  mFZG ddlHmIZImJZJ ddlKmLZL ddlMmNZN ddlOmPZPmQZQmRZRmSZSmTZT ddlUmVZV ddlWmXZXmYZY dZZdZ[dd Z\dd Z]dd Z^eIZ_dd  Z`G d!d" d"eaZbG d#d$ d$eaZcG d%d& d&edZed'ZfG d(d) d)edZgd*ZhG d+d, d,edZid-Zjd.Zkd/d/d0d0d1Zld2Zme/d3e1d4e(d5e0d6e-d7iZnd8d9d9d:d;d<d=d>d?Zod@dAdBdCdDdEdFdGdHdI	Zpe(dge-dJdKgiZqdLZrdMZse8tdN8 e8judOdPere8jvdQ e8judRdese8wd/d0dgdQ W dQ R X daxdPaydSdT ZzddUdVZ{ddXdYZ|dZd[ Z}G d\d] d]eCZ~G d^d_ d_eZG d`da daeCZG dbdc dceZG ddde deeZG dfdg dgeZG dhdi dieZG djdk dkeCZG dldm dmeZG dndo doeZG dpd8 d8eZG dqd9 d9eZG drd: d:eZG dsdt dteZG dud; d;eZG dvd= d=eZG dwdx dxeZG dyd< d<eZG dzd> d>eZG d{d| d|eZG d}dF dFeZG d~d deZG ddG dGeZG ddH dHeZG dd deZG ddC dCeZG ddA dAeZG ddB dBeZG dd@ d@eZG ddD dDeZG ddE dEeZdddZdd Zdd ZdddZdddZdddZdddZdddZdddZdd Zdd Zdd ZG dd deZdddZdS )zY
High level interface to PyTables for reading and writing pandas data structures
to disk
    N)datedatetime)LooseVersion)algoslibwriters)	timezones)PY3filterlrangerangestring_types)PerformanceWarning)ensure_int64ensure_objectensure_platform_intis_categorical_dtypeis_datetime64_dtypeis_datetime64tz_dtypeis_list_likeis_timedelta64_dtype)array_equivalent)	DataFrameDatetimeIndexIndex
Int64Index
MultiIndexPanelPeriodIndexSeriesSparseDataFrameSparseSeriesTimedeltaIndexcompatconcatisnato_datetime)config)matchunique)Categorical_factorize_from_iterables)
BlockIndexIntIndex)StringMixin)Exprmaybe_expression)
get_option)ensure_index)BlockManager_block2d_to_blocknd_block_shape_factor_indexer
make_block)_stringify_path)adjoinpprint_thingz0.15.2zUTF-8c             C   s   t | tjr| d} | S )z* if we have bytes, decode them to unicode zUTF-8)
isinstancenpZbytes_decode)s r?   1lib/python3.7/site-packages/pandas/io/pytables.py_ensure_decoded:   s    
rA   c             C   s   | d krt rt} | S )N)r	   _default_encoding)encodingr?   r?   r@   _ensure_encodingA   s    rD   c             C   s   t | tjrt| } | S )zEnsure that an index / column name is a str (python 3) or
    unicode (python 2); otherwise they may be np.string dtype.
    Non-string dtypes are passed through unchanged.

    https://github.com/pandas-dev/pandas/issues/13492
    )r;   r#   r   Z	text_type)namer?   r?   r@   _ensure_strI   s    
rF   c             C   sv   |d }t | ttfr^g }x<tdd | D ]*}t|sB|| q*|t||d q*W |} nt| rrt| |d} | S )z
    ensure that the where is a Term or a list of Term
    this makes sure that we are capturing the scope of variables
    that are passed
    create the terms here with a frame_level=2 (we are 2 levels down)
       c             S   s   | d k	S )Nr?   )xr?   r?   r@   <lambda>e   s    z_ensure_term.<locals>.<lambda>)scope_level)r;   listtupler
   r0   appendTerm)whererJ   levelZwlistwr?   r?   r@   _ensure_termX   s    
rR   c               @   s   e Zd ZdS )PossibleDataLossErrorN)__name__
__module____qualname__r?   r?   r?   r@   rS   p   s   rS   c               @   s   e Zd ZdS )ClosedFileErrorN)rT   rU   rV   r?   r?   r?   r@   rW   t   s   rW   c               @   s   e Zd ZdS )IncompatibilityWarningN)rT   rU   rV   r?   r?   r?   r@   rX   x   s   rX   z
where criteria is being ignored as this version [%s] is too old (or
not-defined), read the file in and write it out to a new file to upgrade (with
the copy_to method)
c               @   s   e Zd ZdS )AttributeConflictWarningN)rT   rU   rV   r?   r?   r?   r@   rY      s   rY   zu
the [%s] attribute of the existing index is [%s] which conflicts with the new
[%s], resetting the attribute to None
c               @   s   e Zd ZdS )DuplicateWarningN)rT   rU   rV   r?   r?   r?   r@   rZ      s   rZ   z;
duplicate entries in table, taking most recently appended
z
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->%s,key->%s] [items->%s]
fixedtable)fr[   tr\   a  
the table keyword has been deprecated
use the format='fixed(f)|table(t)' keyword instead
  fixed(f) : specifies the Fixed format
             and is the default for put operations
  table(t) : specifies the Table format
             and is the default for append operations
seriessparse_seriesframesparse_framewideLegacySeriesFixedLegacyFrameFixedSeriesFixedSparseSeriesFixed
FrameFixedSparseFrameFixed
PanelFixed)r   r   Z
DataMatrixr_   r`   ra   rb   rc   GenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTableAppendablePanelTable	WORMTableLegacyFrameTableLegacyPanelTable)	generic_tableappendable_seriesappendable_multiseriesappendable_frameappendable_multiframeappendable_panelwormlegacy_framelegacy_panelrG      z;
: boolean
    drop ALL nan rows when appending to a table
z~
: format
    default format writing format, if None, then
    put will default to 'fixed' and append will default to 'table'
zio.hdfZdropna_tableF)Z	validatorZdefault_formatc              C   sX   t d krTdd l} | a t| jtdk r.tdy| jjdkaW n tk
rR   Y nX t S )Nr   z3.0.0z%PyTables version >= 3.0.0 is requiredstrict)	
_table_modtablesr   __version__ImportErrorfileZ_FILE_OPEN_POLICY!_table_file_open_policy_is_strictAttributeError)r   r?   r?   r@   _tables   s    r   c       
   	      sj   |r fdd}n fdd}t | } t| tr^t| |||d}	||	 W dQ R X n||  dS )z- store this object, close it if we opened it c                s   | j  fS )N)rM   )store)keykwargsvaluer?   r@   rI   
  s    zto_hdf.<locals>.<lambda>c                s   | j  fS )N)put)r   )r   r   r   r?   r@   rI     s    )mode	complevelcomplibN)r8   r;   r   HDFStore)
path_or_bufr   r   r   r   r   rM   r   r]   r   r?   )r   r   r   r@   to_hdf  s    

r   rc       
      K   s  |dkrt d|d|kr2t|d dd|d< t| trT| jsJtd| }d}nvt| } t| tsnt	dyt
j| }W n tt fk
r   d}Y nX |std	j| d
t| fd|i|}d}yv|dkr,| }t|dkrt d|d }x*|dd D ]}	t|	|st dqW |j}|j|fd|i|S  t tfk
r   y|  W n tk
rx   Y nX  Y nX dS )a  
    Read from the store, close it if we opened it.

    Retrieve pandas object stored in file, optionally based on where
    criteria

    Parameters
    ----------
    path_or_buf : string, buffer or path object
        Path to the file to open, or an open :class:`pandas.HDFStore` object.
        Supports any object implementing the ``__fspath__`` protocol.
        This includes :class:`pathlib.Path` and py._path.local.LocalPath
        objects.

        .. versionadded:: 0.19.0 support for pathlib, py.path.
        .. versionadded:: 0.21.0 support for __fspath__ protocol.

    key : object, optional
        The group identifier in the store. Can be omitted if the HDF file
        contains a single pandas object.
    mode : {'r', 'r+', 'a'}, optional
        Mode to use when opening the file. Ignored if path_or_buf is a
        :class:`pandas.HDFStore`. Default is 'r'.
    where : list, optional
        A list of Term (or convertible) objects.
    start : int, optional
        Row number to start selection.
    stop  : int, optional
        Row number to stop selection.
    columns : list, optional
        A list of columns names to return.
    iterator : bool, optional
        Return an iterator object.
    chunksize : int, optional
        Number of rows to include in an iteration when using an iterator.
    errors : str, default 'strict'
        Specifies how encoding and decoding errors are to be handled.
        See the errors argument for :func:`open` for a full list
        of options.
    **kwargs
        Additional keyword arguments passed to HDFStore.

    Returns
    -------
    item : object
        The selected object. Return type depends on the object stored.

    See Also
    --------
    pandas.DataFrame.to_hdf : Write a HDF file from a DataFrame.
    pandas.HDFStore : Low-level access to HDF files.

    Examples
    --------
    >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z'])
    >>> df.to_hdf('./store.h5', 'data')
    >>> reread = pd.read_hdf('./store.h5')
    )r   zr+azOmode {0} is not allowed while performing a read. Allowed modes are r, r+ and a.rO   rG   )rJ   z&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile {path} does not exist)pathr   TNr   zNo dataset in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.
auto_close)
ValueErrorformatrR   r;   r   is_openIOErrorr8   r   NotImplementedErrorosr   exists	TypeErrorr#   FileNotFoundErrorgroupslen_is_metadata_of_v_pathnameselectcloser   )
r   r   r   r   r   r   r   r   Zcandidate_only_groupZgroup_to_checkr?   r?   r@   read_hdf  sN    <



r   c             C   sJ   | j |j krdS | }x0|j dkrD|j}||kr<|jdkr<dS |j}qW dS )zDCheck if a given group is a metadata group for a given parent_group.FrG   metaT)Z_v_depthZ	_v_parent_v_name)groupZparent_groupcurrentparentr?   r?   r@   r     s    
r   c               @   s  e Zd ZdZdZddZdd Zedd	 Zed
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# ZeZd[d%d&Zd'd( Zed)d* Zd\d+d,Zd-d. Zd]d/d0Zd^d1d2Zd3d4 Zd_d5d6Zd`d7d8Zdad9d:Z dbd<d=Z!dcd>d?Z"d@dA Z#dBdC Z$dddEdFZ%dGdH Z&dIdJ Z'dedLdMZ(dNdO Z)dPdQ Z*dRdS Z+dfdTdUZ,dgdVdWZ-dXdY Z.dS )hr   aC  
    Dict-like IO interface for storing pandas objects in PyTables
    either Fixed or Table format.

    Parameters
    ----------
    path : string
        File path to HDF5 file
    mode : {'a', 'w', 'r', 'r+'}, default 'a'

        ``'r'``
            Read-only; no data can be modified.
        ``'w'``
            Write; a new file is created (an existing file with the same
            name would be deleted).
        ``'a'``
            Append; an existing file is opened for reading and writing,
            and if the file does not exist it is created.
        ``'r+'``
            It is similar to ``'a'``, but the file must already exist.
    complevel : int, 0-9, default None
            Specifies a compression level for data.
            A value of 0 disables compression.
    complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
            Specifies the compression library to be used.
            As of v0.20.2 these additional compressors for Blosc are supported
            (default if no compressor specified: 'blosc:blosclz'):
            {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
             'blosc:zlib', 'blosc:zstd'}.
            Specifying a compression library which is not available issues
            a ValueError.
    fletcher32 : bool, default False
            If applying compression use the fletcher32 checksum

    Examples
    --------
    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5')
    >>> store['foo'] = bar   # write to HDF5
    >>> bar = store['foo']   # retrieve
    >>> store.close()
    NFc       	   
   K   s   d|krt dydd l}W n2 tk
rN } ztdj|dW d d }~X Y nX |d k	rx||jjkrxt dj|jjd|d kr|d k	r|jj}t|| _|d krd}|| _	d | _
|r|nd| _|| _|| _d | _| jf d	|i| d S )
Nr   z-format is not a defined argument for HDFStorer   z6HDFStore requires PyTables, "{ex!s}" problem importing)exz)complib only supports {libs} compression.)Zlibsr   r   )r   r   r   r   filtersZall_complibsZdefault_complibr8   _path_mode_handle
_complevel_complib_fletcher32_filtersopen)	selfr   r   r   r   
fletcher32r   r   r   r?   r?   r@   __init__  s.    
zHDFStore.__init__c             C   s   | j S )N)r   )r   r?   r?   r@   
__fspath__  s    zHDFStore.__fspath__c             C   s   |    | jjS )z return the root node )_check_if_openr   root)r   r?   r?   r@   r     s    zHDFStore.rootc             C   s   | j S )N)r   )r   r?   r?   r@   filename  s    zHDFStore.filenamec             C   s
   |  |S )N)get)r   r   r?   r?   r@   __getitem__  s    zHDFStore.__getitem__c             C   s   |  || d S )N)r   )r   r   r   r?   r?   r@   __setitem__  s    zHDFStore.__setitem__c             C   s
   |  |S )N)remove)r   r   r?   r?   r@   __delitem__  s    zHDFStore.__delitem__c          	   C   s@   y
|  |S  ttfk
r"   Y nX tdjt| j|ddS )z& allow attribute access to get stores z+'{object}' object has no attribute '{name}')objectrE   N)r   KeyErrorrW   r   r   typerT   )r   rE   r?   r?   r@   __getattr__   s    
zHDFStore.__getattr__c             C   s8   |  |}|dk	r4|j}||ks0|dd |kr4dS dS )z| check for existence of this key
              can match the exact pathname or the pathnm w/o the leading '/'
              NrG   TF)get_noder   )r   r   noderE   r?   r?   r@   __contains__
  s    
zHDFStore.__contains__c             C   s   t |  S )N)r   r   )r   r?   r?   r@   __len__  s    zHDFStore.__len__c             C   s   dj t| t| jdS )Nz{type}
File path: {path}
)r   r   )r   r   r:   r   )r   r?   r?   r@   __unicode__  s    zHDFStore.__unicode__c             C   s   | S )Nr?   )r   r?   r?   r@   	__enter__  s    zHDFStore.__enter__c             C   s   |    d S )N)r   )r   exc_type	exc_value	tracebackr?   r?   r@   __exit__  s    zHDFStore.__exit__c             C   s   dd |   D S )z
        Return a (potentially unordered) list of the keys corresponding to the
        objects stored in the HDFStore. These are ABSOLUTE path-names (e.g.
        have the leading '/'
        c             S   s   g | ]
}|j qS r?   )r   ).0nr?   r?   r@   
<listcomp>(  s    z!HDFStore.keys.<locals>.<listcomp>)r   )r   r?   r?   r@   keys"  s    zHDFStore.keysc             C   s   t |  S )N)iterr   )r   r?   r?   r@   __iter__*  s    zHDFStore.__iter__c             c   s"   x|   D ]}|j|fV  q
W dS )z'
        iterate on key->group
        N)r   r   )r   gr?   r?   r@   items-  s    zHDFStore.itemsr   c          
   K   s  t  }| j|krL| jdkr$|dkr$n"|dkrF| jrFtd| j| j|| _| jrZ|   | jr| jdkrt  j| j| j	| j
d| _y|j| j| jf|| _W n tk
r } z:dt|krtdj| jd	 |j| jd
f|| _n W dd}~X Y n tk
rB } z.dt|kr.tdj|j| d}|W dd}~X Y nL tk
r } z,| jd
krzdt|krztt| W dd}~X Y nX dS )z
        Open the file in the specified mode

        Parameters
        ----------
        mode : {'a', 'w', 'r', 'r+'}, default 'a'
            See HDFStore docstring or tables.open_file for info about modes
        )r   rQ   )r   zr+)rQ   zGRe-opening the file [{0}] with mode [{1}] will delete the current file!r   )r   zcan not be writtenz Opening {path} in read-only mode)r   r   NZFILE_OPEN_POLICYa3  PyTables [{version}] no longer supports opening multiple files
even in read-only mode on this HDF5 version [{hdf_version}]. You can accept this
and not open the same file multiple times at once,
upgrade the HDF5 version, or downgrade to PyTables 3.0.0 which allows
files to be opened multiple times at once
)versionZhdf_versionzUnable to open/create file)r   r   r   rS   r   r   r   r   Filtersr   r   r   Z	open_filer   r   strprintr   r   Zget_hdf5_version	Exception)r   r   r   r   er?   r?   r@   r   6  sD    	
zHDFStore.openc             C   s   | j dk	r| j   d| _ dS )z0
        Close the PyTables file handle
        N)r   r   )r   r?   r?   r@   r     s    

zHDFStore.closec             C   s   | j dkrdS t| j jS )zF
        return a boolean indicating whether the file is open
        NF)r   boolZisopen)r   r?   r?   r@   r     s    
zHDFStore.is_openc             C   sF   | j dk	rB| j   |rByt| j   W n tk
r@   Y nX dS )a  
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        N)r   flushr   fsyncfilenoOSError)r   r   r?   r?   r@   r     s    

zHDFStore.flushc             C   s,   |  |}|dkr"tdj|d| |S )z
        Retrieve pandas object stored in file

        Parameters
        ----------
        key : object

        Returns
        -------
        obj : same type as object stored in file
        Nz!No object named {key} in the file)r   )r   r   r   _read_group)r   r   r   r?   r?   r@   r     s    
zHDFStore.getc	                st   |  |}
|
dkr"tdj|dt|dd}| |
   fdd}t| ||j|||||d
}| S )	a  
        Retrieve pandas object stored in file, optionally based on where
        criteria

        Parameters
        ----------
        key : object
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        columns : a list of columns that if not None, will limit the return
            columns
        iterator : boolean, return an iterator, default False
        chunksize : nrows to include in iteration, return an iterator
        auto_close : boolean, should automatically close the store when
            finished, default is False

        Returns
        -------
        The selected object
        Nz!No object named {key} in the file)r   rG   )rJ   c                s   j | || dS )N)startstoprO   columns)read)_start_stop_where)r   r>   r?   r@   func  s    zHDFStore.select.<locals>.func)rO   nrowsr   r   iterator	chunksizer   )	r   r   r   rR   _create_storer
infer_axesTableIteratorr   
get_result)r   r   rO   r   r   r   r   r   r   r   r   r   itr?   )r   r>   r@   r     s    


zHDFStore.selectc             K   s*   t |dd}| |jf |||d|S )aA  
        return the selection as an Index

        Parameters
        ----------
        key : object
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        rG   )rJ   )rO   r   r   )rR   
get_storerread_coordinates)r   r   rO   r   r   r   r?   r?   r@   select_as_coordinates  s    zHDFStore.select_as_coordinatesc             K   s   |  |jf d|i|S )a  
        return a single column from the table. This is generally only useful to
        select an indexable

        Parameters
        ----------
        key : object
        column: the column of interest

        Exceptions
        ----------
        raises KeyError if the column is not found (or key is not a valid
            store)
        raises ValueError if the column can not be extracted individually (it
            is part of a data block)

        column)r   read_column)r   r   r   r   r?   r?   r@   select_column  s    zHDFStore.select_columnc
                sz  t |dd}t|ttfr.t|dkr.|d }t|trXjf ||||||dS t|ttfsntdt|s~td|dkr|d }fdd	|D 	|}d}xzt
||fgt|D ]^\}}|dkrtd
j|d|jstdj|jd|dkr|j}q|j|krtdqW tdd D d   fdd}t|||||||||	d
}|jddS )a   Retrieve pandas objects from multiple tables

        Parameters
        ----------
        keys : a list of the tables
        selector : the table to apply the where criteria (defaults to keys[0]
            if not supplied)
        columns : the columns I want back
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        iterator : boolean, return an iterator, default False
        chunksize : nrows to include in iteration, return an iterator

        Exceptions
        ----------
        raises KeyError if keys or selector is not found or keys is empty
        raises TypeError if keys is not a list or tuple
        raises ValueError if the tables are not ALL THE SAME DIMENSIONS
        rG   )rJ   r   )r   rO   r   r   r   r   r   zkeys must be a list/tuplez keys must have a non-zero lengthNc                s   g | ]}  |qS r?   )r   )r   k)r   r?   r@   r   4  s    z/HDFStore.select_as_multiple.<locals>.<listcomp>zInvalid table [{key}])r   zKobject [{obj}] is not a table, and cannot be used in all select as multiple)objz,all tables must have exactly the same nrows!c             S   s   h | ]}|j d  d  qS )r   )non_index_axes)r   r^   r?   r?   r@   	<setcomp>I  s    z.HDFStore.select_as_multiple.<locals>.<setcomp>c                s,    fddD }t |dd S )Nc          	      s&   g | ]}|j f  d qS ))rO   r   r   r   )r   )r   r^   )r   r   r   r   r   r?   r@   r   O  s   z=HDFStore.select_as_multiple.<locals>.func.<locals>.<listcomp>F)axisverify_integrity)r$   _consolidate)r   r   r   objs)r   r   r   tbls)r   r   r   r@   r   K  s    z)HDFStore.select_as_multiple.<locals>.func)rO   r   r   r   r   r   r   T)coordinates)rR   r;   rK   rL   r   r   r   r   r   r   	itertoolschainzipr   r   is_tablepathnamer   r   r   )r   r   rO   selectorr   r   r   r   r   r   r   r>   r   r^   r   r   r   r?   )r   r   r   r   r  r@   select_as_multiple
  sD    

"



zHDFStore.select_as_multiplec             K   s<   |dkrt dpd}| ||}| j||fd|i| dS )ag  
        Store object in HDFStore

        Parameters
        ----------
        key      : object
        value    : {Series, DataFrame, Panel}
        format   : 'fixed(f)|table(t)', default is 'fixed'
            fixed(f) : Fixed format
                       Fast writing/reading. Not-appendable, nor searchable
            table(t) : Table format
                       Write as a PyTables Table structure which may perform
                       worse but allow more flexible operations like searching
                       / selecting subsets of the data
        append   : boolean, default False
            This will force Table format, append the input data to the
            existing.
        data_columns : list of columns to create as data columns, or True to
            use all columns. See
            `here <http://pandas.pydata.org/pandas-docs/stable/io.html#query-via-data-columns>`__ # noqa
        encoding : default None, provide an encoding for strings
        dropna   : boolean, default False, do not write an ALL nan row to
            the store settable by the option 'io.hdf.dropna_table'
        Nzio.hdf.default_formatr[   rM   )r1   _validate_format_write_to_group)r   r   r   r   rM   r   r?   r?   r@   r   ]  s    zHDFStore.putc             C   s   t |dd}y| |}W nZ tk
r0    Y nF tk
rt   |dk	rNtd| |}|dk	rp|jdd dS Y nX t|||r|j	jdd n|j
std|j|||dS dS )	aE  
        Remove pandas object partially by specifying the where condition

        Parameters
        ----------
        key : string
            Node to remove or delete rows from
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection

        Returns
        -------
        number of rows removed (or None if not a Table)

        Exceptions
        ----------
        raises KeyError if key is not a valid store

        rG   )rJ   Nz5trying to remove a node with a non-None where clause!T)	recursivez7can only remove with where on objects written as tables)rO   r   r   )rR   r   r   r   r   r   Z	_f_removecom	_all_noner   r  delete)r   r   rO   r   r   r>   r?   r?   r@   r   {  s&    

zHDFStore.removeTc             K   s^   |dk	rt d|dkr td}|dkr4tdp2d}| ||}| j||f||d| dS )a  
        Append to Table in file. Node must already exist and be Table
        format.

        Parameters
        ----------
        key : object
        value : {Series, DataFrame, Panel}
        format : 'table' is the default
            table(t) : table format
                       Write as a PyTables Table structure which may perform
                       worse but allow more flexible operations like searching
                       / selecting subsets of the data
        append       : boolean, default True, append the input data to the
            existing
        data_columns :  list of columns, or True, default None
            List of columns to create as indexed data columns for on-disk
            queries, or True to use all columns. By default only the axes
            of the object are indexed. See `here
            <http://pandas.pydata.org/pandas-docs/stable/io.html#query-via-data-columns>`__.
        min_itemsize : dict of columns that specify minimum string sizes
        nan_rep      : string to use as string nan represenation
        chunksize    : size to chunk the writing
        expectedrows : expected TOTAL row size of this table
        encoding     : default None, provide an encoding for strings
        dropna       : boolean, default False, do not write an ALL nan row to
            the store settable by the option 'io.hdf.dropna_table'

        Notes
        -----
        Does *not* check if data being appended overlaps with existing
        data in the table, so be careful
        Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tablezio.hdf.default_formatr\   )rM   dropna)r   r1   r
  r  )r   r   r   r   rM   r   r  r   r?   r?   r@   rM     s    #zHDFStore.appendc                s  |dk	rt dt|ts"td||kr2tdttt jttt	   d }d}	g }
x<|
 D ]0\}}|dkr|	dk	rtd|}	qj|
| qjW |	dk	rڈ j| }|t|
}t||}||||	< |dkr|| }|r2 fdd| D }t|}x|D ]}||}qW  j|  xN|
 D ]B\}}||krR|nd} j||d	}| j||fd
|i| q<W dS )a  
        Append to multiple tables

        Parameters
        ----------
        d : a dict of table_name to table_columns, None is acceptable as the
            values of one node (this will get all the remaining columns)
        value : a pandas object
        selector : a string that designates the indexable table; all of its
            columns will be designed as data_columns, unless data_columns is
            passed, in which case these are used
        data_columns : list of columns to create as data columns, or True to
            use all columns
        dropna : if evaluates to True, drop rows from all tables if any single
                 row in each table has all NaN. Default False.

        Notes
        -----
        axes parameter is currently not accepted

        Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictr   z<append_to_multiple can only have one value in d that is Nonec             3   s    | ]} | j d djV  qdS )all)ZhowN)r  index)r   cols)r   r?   r@   	<genexpr>  s    z.HDFStore.append_to_multiple.<locals>.<genexpr>)r   data_columns)r   r;   dictr   rK   setr   ndim	_AXES_MAPr   r   extendaxes
differencer   sortedget_indexertakevaluesnextintersectionlocreindexrM   )r   dr   r  r  r  r  r   r   Z
remain_keyZremain_valuesr   vorderedZorddZidxsZvalid_indexr  dcvalr?   )r   r@   append_to_multiple  sF    
&


zHDFStore.append_to_multiplec             K   s:   t   | |}|dkrdS |js*td|jf | dS )z Create a pytables index on the table
        Parameters
        ----------
        key : object (the node to index)

        Exceptions
        ----------
        raises if the node is not a table

        Nz1cannot create table index on a Fixed format store)r   r   r  r   create_index)r   r   r   r>   r?   r?   r@   create_table_index.  s    
zHDFStore.create_table_indexc             C   s"   t   |   dd | j D S )zkreturn a list of all the top-level nodes (that are not themselves a
        pandas storage object)
        c             S   sP   g | ]H}t |tjjst|jd dsHt|ddsHt |tjjr|jdkr|qS )pandas_typeNr\   )	r;   r   linkZLinkgetattr_v_attrsr\   Tabler   )r   r   r?   r?   r@   r   L  s    z#HDFStore.groups.<locals>.<listcomp>)r   r   r   walk_groups)r   r?   r?   r@   r   E  s    zHDFStore.groups/c             c   s   t   |   x| j|D ]}t|jdddk	r4qg }g }xP|j D ]B}t|jdd}|dkr~t|t	j
jr||j qH||j qHW |jd||fV  qW dS )a   Walk the pytables group hierarchy for pandas objects

        This generator will yield the group path, subgroups and pandas object
        names for each group.
        Any non-pandas PyTables objects that are not a group will be ignored.

        The `where` group itself is listed first (preorder), then each of its
        child groups (following an alphanumerical order) is also traversed,
        following the same procedure.

        .. versionadded:: 0.24.0

        Parameters
        ----------
        where : str, optional
            Group where to start walking.
            If not supplied, the root group is used.

        Yields
        ------
        path : str
            Full path to a group (without trailing '/')
        groups : list of str
            names of the groups contained in `path`
        leaves : list of str
            names of the pandas objects contained in `path`
        r-  Nr3  )r   r   r   r2  r/  r0  _v_childrenr   r;   r   r   ZGrouprM   r   r   rstrip)r   rO   r   r   ZleavesZchildr-  r?   r?   r@   walkT  s    zHDFStore.walkc             C   sH   |    y"|dsd| }| j| j|S  tjjk
rB   dS X dS )z; return the node with the key or None if it does not exist r3  N)r   
startswithr   r   r   r   
exceptionsZNoSuchNodeError)r   r   r?   r?   r@   r     s    
zHDFStore.get_nodec             C   s8   |  |}|dkr"tdj|d| |}|  |S )z> return the storer object for a key, raise if not in the file Nz!No object named {key} in the file)r   )r   r   r   r   r   )r   r   r   r>   r?   r?   r@   r     s    

zHDFStore.get_storerrQ   c	          	   C   s   t |||||d}	|dkr&t|  }t|ttfs:|g}x|D ]}
| |
}|dk	r@|
|	krl|rl|	|
 | |
}|jrd}|rdd |j	D }|	j
|
||t|dd|jd q@|	j|
||jd q@W |	S )	a,   copy the existing store to a new file, upgrading in place

            Parameters
            ----------
            propindexes: restore indexes in copied file (defaults to True)
            keys       : list of keys to include in the copy (defaults to all)
            overwrite  : overwrite (remove and replace) existing nodes in the
                new store (default is True)
            mode, complib, complevel, fletcher32 same as in HDFStore.__init__

            Returns
            -------
            open file handle of the new store

        )r   r   r   r   NFc             S   s   g | ]}|j r|jqS r?   )
is_indexedrE   )r   r   r?   r?   r@   r     s    z!HDFStore.copy.<locals>.<listcomp>r  )r  r  rC   )rC   )r   rK   r   r;   rL   r   r   r   r  r  rM   r/  rC   r   )r   r   r   Zpropindexesr   r   r   r   Z	overwriteZ	new_storer   r>   datar  r?   r?   r@   copy  s6    




zHDFStore.copyc             C   s   dj t| t| jd}| jrtt|  }t|rg }g }x|D ]}y<| 	|}|dk	r|
t|jpj| |
t|p|d W qD tk
r } z$|
| |
dj t|d W dd}~X Y qDX qDW |td||7 }q|d7 }n|d	7 }|S )
z\
        Print detailed information on the store.

        .. versionadded:: 0.21.0
        z{type}
File path: {path}
)r   r   Nzinvalid_HDFStore nodez![invalid_HDFStore node: {detail}])detail   ZEmptyzFile is CLOSED)r   r   r:   r   r   r  rK   r   r   r   rM   r  r   r9   )r   outputZlkeysr   r   r   r>   r<  r?   r?   r@   info  s.    


$
zHDFStore.infoc             C   s   | j std| jd S )Nz{0} file is not open!)r   rW   r   r   )r   r?   r?   r@   r     s    zHDFStore._check_if_openc             C   sD   |  }yt|  |d< W n" tk
r>   td|Y nX |S )z5 validate / deprecate formats; return the new kwargs r   z'invalid HDFStore format specified [{0}])r;  _FORMAT_MAPlowerr   r   r   )r   r   r   r?   r?   r@   r
    s    zHDFStore._validate_formatc                s:   fdd}t tjdd}t tjdd}|dkrdkr|t  tddshttjjrrd}d}qtd	n>yt	t
 }W n tk
r   |d
 Y nX dkr|d7 }d|kryt t|  | fS  tk
r   |d Y nX |dkrdk	r|dkrPtdd}	|	dk	r|	jdkr>d}n|	jdkrd}nb|dkrtdd}	|	dk	r|	jdkrd}n|	jdkrd}n|dkrd}n|dkrd}nJd}y.jjj}
t|
dkr|
d dkrd}W n tk
r   Y nX yt t|  | fS  tk
r4   |d Y nX dS )z$ return a suitable class to operate c          	      s"   t dj| t dd S )Nzcannot properly create the storer for: [{t}] [group->{group},value->{value},format->{format},append->{append},kwargs->{kwargs}])r^   r   r   r   rM   r   )r   r   r   )r^   )rM   r   r   r   r   r?   r@   error  s
    z&HDFStore._create_storer.<locals>.errorr-  N
table_typer\   frame_tablert   zKcannot create a storer if the object is not existing nor a value are passed	_TYPE_MAPZ_table_STORER_MAPseries_tabler  rG   ru   rv   rw   rx   
wide_tablery   Z
ndim_tableZappendable_ndimr|   r   r   r{   
_TABLE_MAP)rA   r/  r0  r   r;   r   r\   r1  r   rE  r   r   globalsrF  nlevelsfieldsr   
IndexErrorrI  )r   r   r   r   rM   r   rB  ZptZttr  rL  r?   )rM   r   r   r   r   r@   r     sl    








zHDFStore._create_storerc             K   s`  |  |}	|	d k	r*|s*| jj|	dd d }	t|dd rF|dksB|rFd S |	d kr|d}
d}xX|
D ]P}t|spqb|}|ds|d7 }||7 }|  |}	|	d kr| j||}	|}qbW | j|	||f||d|}|r|j	r|j	r|dkr|j
rtd|j
s|  n|  |j	s*|r*td	|jf |||d
| |j	r\|r\|j|d d S )NT)r  emptyr\   r3  )rM   rC   r[   zCan only append to Tablesz0Compression not supported on Fixed format stores)r   rM   r   )r   )r   r   remove_noder/  splitr   endswithcreate_groupr   r  	is_existsr   set_object_infowriter+  )r   r   r   r   r  rM   r   rC   r   r   pathsr   pnew_pathr>   r?   r?   r@   r  U  sF    





zHDFStore._write_to_groupc             K   s   |  |}|  |jf |S )N)r   r   r   )r   r   r   r>   r?   r?   r@   r     s    
zHDFStore._read_group)NNNF)r   )F)NNNNFNF)NNN)NNNNNFNF)NF)NNN)NTNN)NNF)r3  )rQ   TNNNFT)NNF)TFNN)/rT   rU   rV   __doc__r   r   propertyr   r   r   r   r   r   r   r   r   r   r   r   r   r   	iteritemsr   r   r   r   r   r   r   r   r	  r   r   rM   r*  r,  r   r6  r   r   r;  r?  r   r
  r   r  r   r?   r?   r?   r@   r     sf   + 


I	
 
-
  
Q

2 
. 
Q
.

 
2$
W 
6r   c               @   s4   e Zd ZdZdddZdd Zdd	 Zdd
dZdS )r   a   define the iteration interface on a table

        Parameters
        ----------

        store : the reference store
        s     : the referred storer
        func  : the function to execute the query
        where : the where of the query
        nrows : the rows to iterate on
        start : the passed start value (default is None)
        stop  : the passed stop value (default is None)
        iterator : boolean, whether to use the default iterator
        chunksize : the passed chunking value (default is 50000)
        auto_close : boolean, automatically close the store at the end of
            iteration, default is False
        kwargs : the passed kwargs
        NFc             C   s   || _ || _|| _|| _| jjrN|d kr,d}|d kr8d}|d krD|}t||}|| _|| _|| _d | _	|sr|	d k	r|	d kr~d}	t
|	| _nd | _|
| _d S )Nr   i )r   r>   r   rO   r  minr   r   r   r  intr   r   )r   r   r>   r   rO   r   r   r   r   r   r   r?   r?   r@   r     s,    
zTableIterator.__init__c             c   sh   | j }xT|| jk rZt|| j | j}| d d | j|| }|}|d kst|sRq|V  qW |   d S )N)r   r   r\  r   r   r  r   r   )r   r   r   r   r?   r?   r@   r     s    
zTableIterator.__iter__c             C   s   | j r| j  d S )N)r   r   r   )r   r?   r?   r@   r     s    zTableIterator.closec             C   sr   | j d k	r0| jjstd| jj| jd| _| S |rN| jj| j| j| jd}n| j}| 	| j| j|}| 
  |S )Nz0can only use an iterator or chunksize on a table)rO   )rO   r   r   )r   r>   r  r   r   rO   r  r   r   r   r   )r   r  rO   Zresultsr?   r?   r@   r     s    
zTableIterator.get_result)NNFNF)F)rT   rU   rV   rY  r   r   r   r   r?   r?   r?   r@   r     s    
r   c               @   s6  e Zd ZdZdZdZdddgZdEddZdFd	d
Zdd Z	dd Z
dd Zdd Zdd Zdd Zedd Zdd Zdd Zdd Zdd  Zed!d" Zed#d$ Zed%d& Zed'd( Zd)d* ZdGd+d,Zd-d. Zd/d0 Zd1d2 ZdHd3d4Zd5d6 Zd7d8 Z d9d: Z!d;d< Z"d=d> Z#d?d@ Z$dAdB Z%dCdD Z&dS )IIndexCola3   an index column description class

        Parameters
        ----------

        axis   : axis which I reference
        values : the ndarray like converted values
        kind   : a string description of this type
        typ    : the pytables type
        pos    : the position in the pytables

        Tfreqtz
index_nameNc             K   s   || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|| _
|| _d | _d | _d | _|d k	rn| || |	d k	r| |	 d S )N)r   kindtypitemsizerE   cname	kind_attrr   posr_  r`  ra  r\   r   metadataset_nameset_pos)r   r   rb  rc  re  rd  rE   r   rf  rg  r_  r`  ra  r   r?   r?   r@   r     s&    zIndexCol.__init__c             C   s,   || _ |pdj|d| _| jdkr(|| _| S )z set the name of this indexer z{name}_kind)rE   N)rE   r   rf  re  )r   rE   rf  r?   r?   r@   ri    s
    
zIndexCol.set_namec             C   s
   || _ | S )z! set the axis over which I index )r   )r   r   r?   r?   r@   set_axis&  s    zIndexCol.set_axisc             C   s$   || _ |dk	r | jdk	r || j_| S )z. set the position of this column in the Table N)rg  rc  Z_v_pos)r   rg  r?   r?   r@   rj  ,  s    zIndexCol.set_posc             C   s
   || _ | S )N)r\   )r   r\   r?   r?   r@   	set_table3  s    zIndexCol.set_tablec          	   C   sF   t tt| j| j| j| j| jf}ddd t	dddddg|D S )	N,c             s   s    | ]\}}d j ||dV  qdS )z{key}->{value})r   r   N)r   )r   r   r   r?   r?   r@   r  ?  s   z'IndexCol.__unicode__.<locals>.<genexpr>rE   re  r   rg  rb  )
rL   mapr:   rE   re  r   rg  rb  joinr  )r   tempr?   r?   r@   r   7  s    
zIndexCol.__unicode__c                s   t  fdddD S )z compare 2 col items c             3   s&   | ]}t |d t  |d kV  qd S )N)r/  )r   r   )otherr   r?   r@   r  E  s   z"IndexCol.__eq__.<locals>.<genexpr>)rE   re  r   rg  )r  )r   rq  r?   )rq  r   r@   __eq__C  s    zIndexCol.__eq__c             C   s   |  | S )N)rr  )r   rq  r?   r?   r@   __ne__H  s    zIndexCol.__ne__c             C   s,   yt | jj| jjS  tk
r&   Y nX dS )z' return whether I am an indexed column N)r/  r\   r  re  r9  r   )r   r?   r?   r@   r9  K  s    zIndexCol.is_indexedc             C   s   t  | }|S )N)r;  )r   new_selfr?   r?   r@   r;  S  s    
zIndexCol.copyc             C   s.   |j }|  }|| |  || |S )z@infer this column from the table: create and return a new object)r\   r;  rl  get_attrread_metadata)r   handlerr\   rt  r?   r?   r@   inferW  s    

zIndexCol.inferc             C   s   |j jdk	r|| j }t|| j||}t }| jdk	rDt| j|d< | jdk	r\t| j|d< yt	|f|| _
W n2 tk
r   d|krd|d< t	|f|| _
Y nX t| j
| j| _
| S )z; set the values from this selection: take = take ownership Nr_  rE   )dtyperL  re  _maybe_convertrb  r  r_  rA   ra  r   r   r   _set_tzr`  )r   r   nan_reprC   errorsr   r?   r?   r@   convert`  s     


zIndexCol.convertc             C   s   d| j  | _ }|S )z( return the values & release the memory N)r   )r   r   r?   r?   r@   	take_data}  s    zIndexCol.take_datac             C   s   | j jS )N)r\   r0  )r   r?   r?   r@   attrs  s    zIndexCol.attrsc             C   s   | j jS )N)r\   description)r   r?   r?   r@   r    s    zIndexCol.descriptionc             C   s   t | j| jdS )z# return my current col description N)r/  r  re  )r   r?   r?   r@   col  s    zIndexCol.colc             C   s   | j S )z return my cython values )r   )r   r?   r?   r@   cvalues  s    zIndexCol.cvaluesc             C   s
   t | jS )N)r   r   )r   r?   r?   r@   r     s    zIndexCol.__iter__c             C   sP   t | jdkrLt|tr$|| j}|dk	rL| jj|k rLt j	|| j
d| _dS )z maybe set a string col itemsize:
               min_itemsize can be an integer or a dict with this columns name
               with an integer size stringN)rd  rg  )rA   rb  r;   r  r   rE   rc  rd  r   	StringColrg  )r   min_itemsizer?   r?   r@   maybe_set_size  s    
zIndexCol.maybe_set_sizec             C   s   |    d S )N)validate_names)r   rw  rM   r?   r?   r@   validate  s    zIndexCol.validatec             C   s   d S )Nr?   )r   r?   r?   r@   r    s    zIndexCol.validate_namesc             C   s>   |  |j |   | | | | | | |   d S )N)rl  r\   validate_colvalidate_attrvalidate_metadatawrite_metadataset_attr)r   rw  rM   r?   r?   r@   validate_and_set  s    


zIndexCol.validate_and_setc             C   sV   t | jdkrR| j}|dk	rR|dkr*| j}|j|k rLtdj|| j|jd|jS dS )z< validate this column: return the compared against itemsize r  NzTrying to store a string with len [{itemsize}] in [{cname}] column but
this column has a limit of [{c_itemsize}]!
Consider using min_itemsize to preset the sizes on these columns)rd  re  Z
c_itemsize)rA   rb  r  rd  r   r   re  )r   rd  cr?   r?   r@   r    s    
zIndexCol.validate_colc             C   s>   |r:t | j| jd }|d k	r:|| jkr:tdj|| jdd S )Nz3incompatible kind in col [{existing} - {self_kind}])existingZ	self_kind)r/  r  rf  rb  r   r   )r   rM   Zexisting_kindr?   r?   r@   r    s    zIndexCol.validate_attrc             C   s   x| j D ]}t| |d}t|| j}||}||kr|dk	r||kr|dkrt|||f }tj|tdd d||< t	| |d qt
dj| j|||dq|dk	s|dk	r|||< qW | S )zs set/update the info for this indexable with the key/value
            if there is a conflict raise/warn as needed N)r_  ra     )
stacklevelzkinvalid info for [{name}] for [{key}], existing_value [{existing_value}] conflicts with new value [{value}])rE   r   existing_valuer   )_info_fieldsr/  	_get_inforE   r   attribute_conflict_docwarningswarnrY   setattrr   r   )r   r?  r   r   idxr  wsr?   r?   r@   update_info  s"    
zIndexCol.update_infoc             C   s$   | | j}|dk	r | j| dS )z# set my state from the passed info N)r   rE   __dict__update)r   r?  r  r?   r?   r@   set_info  s    zIndexCol.set_infoc             C   s   t | j| jd| _dS )z set the kind for this column N)r/  r  rf  rb  )r   r?   r?   r@   ru    s    zIndexCol.get_attrc             C   s   t | j| j| j dS )z set the kind for this column N)r  r  rf  rb  )r   r?   r?   r@   r    s    zIndexCol.set_attrc             C   s   | | j| _dS )z( retrieve the metadata for this columns N)rv  re  rh  )r   rw  r?   r?   r@   rv    s    zIndexCol.read_metadatac             C   sB   | j dkr>| j}|| j}|dk	r>|dk	r>t||s>tddS )z< validate that kind=category does not change the categories categoryNzEcannot append a categorical with different categories to the existing)r   rh  rv  re  r   r   )r   rw  Znew_metadataZcur_metadatar?   r?   r@   r    s    

zIndexCol.validate_metadatac             C   s   | j dk	r|| j| j  dS )z set the meta data N)rh  r  re  )r   rw  r?   r?   r@   r    s    
zIndexCol.write_metadata)NNNNNNNNNNNN)N)N)N)'rT   rU   rV   rY  is_an_indexableis_data_indexabler  r   ri  rk  rj  rl  r   rr  rs  rZ  r9  r;  rx  r~  r  r  r  r  r  r   r  r  r  r  r  r  r  r  ru  r  rv  r  r  r?   r?   r?   r@   r^    sJ   
  

		


"
r^  c               @   s4   e Zd ZdZedd Zdd Zdd Zdd	 Zd
S )GenericIndexColz< an index which is not represented in the data of the table c             C   s   dS )NFr?   )r   r?   r?   r@   r9    s    zGenericIndexCol.is_indexedc             C   s   t t| jj| _| S )z; set the values from this selection: take = take ownership )r   r<   aranger\   r   r   )r   r   r|  rC   r}  r?   r?   r@   r~    s    zGenericIndexCol.convertc             C   s   d S )Nr?   )r   r?   r?   r@   ru  #  s    zGenericIndexCol.get_attrc             C   s   d S )Nr?   )r   r?   r?   r@   r  &  s    zGenericIndexCol.set_attrN)	rT   rU   rV   rY  rZ  r9  r~  ru  r  r?   r?   r?   r@   r    s
   r  c                   s   e Zd ZdZdZdZddgZed=ddZd> fdd		Z	d
d Z
dd Zd?ddZdd Zdd Zdd Zd@ddZdd Zdd ZdAddZdBdd Zd!d" Zd#d$ ZdCd%d&Zd'd( ZdDd)d*ZdEd+d,Zd-d. ZdFd/d0Zed1d2 Zed3d4 Zd5d6 Z d7d8 Z!d9d: Z"d;d< Z#  Z$S )GDataColaM   a data holding column, by definition this is not indexable

        Parameters
        ----------

        data   : the actual data
        cname  : the column name in the table to hold the data (typically
                 values)
        meta   : a string description of the metadata
        metadata : the actual metadata
        Fr`  r'  Nc             K   s   |dkr|pdj |d}|dkr$|}yL|d dkrn|d dkrn|d dkrntd|}|rnd	j | d d
}W n tk
r   Y nX | f ||d|S )z' return a new datacol with the block i Nzvalues_block_{idx})r  r   rG   
   r}   zvalues_block_(\d+)zvalues_{group})r   )rE   re  )r   researchr   rM  )clsirE   re  r   r   mr?   r?   r@   create_for_block;  s    $zDataCol.create_for_blockc	       
         sd   t t| jf ||||d|	 d | _dj| jd| _|| _dj| jd| _| 	| | 
| d S )N)r   rb  rc  re  z{name}_dtype)rE   z{name}_meta)superr  r   ry  r   rE   
dtype_attrr   	meta_attrset_dataset_metadata)
r   r   rb  rc  re  r:  r   rh  blockr   )	__class__r?   r@   r   Q  s    
zDataCol.__init__c          	   C   sF   t tt| j| j| j| j| jf}ddd t	dddddg|D S )	Nrm  c             s   s    | ]\}}d j ||dV  qdS )z{key}->{value})r   r   N)r   )r   r   r   r?   r?   r@   r  e  s   z&DataCol.__unicode__.<locals>.<genexpr>rE   re  ry  rb  shape)
rL   rn  r:   rE   re  ry  rb  r  ro  r  )r   rp  r?   r?   r@   r   ]  s    
zDataCol.__unicode__c                s   t  fdddD S )z compare 2 col items c             3   s&   | ]}t |d t  |d kV  qd S )N)r/  )r   r   )rq  r   r?   r@   r  k  s   z!DataCol.__eq__.<locals>.<genexpr>)rE   re  ry  rg  )r  )r   rq  r?   )rq  r   r@   rr  i  s    zDataCol.__eq__c             C   sF   || _ |d k	rB|d k	r&|| _|   n| jd krB|jj| _|   d S )N)r:  ry  set_kindrE   )r   r:  ry  r?   r?   r@   r  n  s    


zDataCol.set_datac             C   s   d| j  | _ }|S )z& return the data & release the memory N)r:  )r   r:  r?   r?   r@   r  x  s    zDataCol.take_datac             C   s$   |dk	rt j|dd }|| _dS )z record the metadata NF)r;  )r<   arrayravelrh  )r   rh  r?   r?   r@   r  }  s    zDataCol.set_metadatac             C   s   | j d k	rt| j }|ds(|dr0d| _n|drBd| _nv|drTd| _nd|dsh|drpd| _nH|drd	| _n6|d
rd
| _n$|drd| _ntdj|| d| jd krt| j| j	d | _d S )Nr  bytesfloatcomplexr]  uintintegerr   r   Z	timedeltar   z.cannot interpret dtype of [{dtype}] in [{obj}])ry  r   )
ry  rA   r7  rb  AssertionErrorr   rc  r/  r  re  )r   ry  r?   r?   r@   r    s*    







zDataCol.set_kindr~   c	          	   C   s   t || _|jr | j|||dS |jr4| j||dS |jrD| |S |jrT| 	|S |j
rd| |S |jj}	tj|jdd}
|
dkrtdnV|
dkrtdnD|
d	krtd
n2|
dks|	dkr| ||||||| n
| | dS )z+ create and setup my atom from the block b )r   r?  )r?  F)skipnar   z+[date] is not implemented as a table columnr   z>too many timezones in this block, create separate data columnsunicodez.[unicode] is not implemented as a table columnr  r   N)rK   r   Zis_categoricalset_atom_categoricalZis_datetimetzset_atom_datetime64tzZis_datetimeset_atom_datetime64Zis_timedeltaset_atom_timedelta64Z
is_complexset_atom_complexry  rE   r   infer_dtyper   set_atom_stringset_atom_data)r   r  block_itemsexisting_colr  r|  r?  rC   r}  ry  inferred_typer?   r?   r@   set_atom  s@    



zDataCol.set_atomc             C   s   t  j||jd dS )Nr   )rd  r  )r   r  r  )r   r  rd  r?   r?   r@   get_atom_string  s    zDataCol.get_atom_stringc             C   s,  |j |dd}t|tr |d }|j}tj| dd}	|	dkrxJt|D ]>\}
}||
}tj| dd}	|	dkrJt	dj
||	dqJW t|||}|j}t|trt|| jp|dpd}t|pd|}|d k	r||}||kr|}|| _d| _| ||| _| |jd	j
|d
dd d S )NF)Zdowncastr   )r  r  zXCannot serialize the column [{item}] because
its data contents are [{type}] object dtype)itemr   r   z|S{size})size)r;  )Zfillnar;   rK   r   r   r  r  	enumerateZigetr   r   _convert_string_arrayrd  r  r]  r   rE   maxr  rb  r  rc  r  astype)r   r  r  r  r  r|  rC   r}  r:  r  r  r  r  Zdata_convertedrd  Zecir?   r?   r@   r    s8    



zDataCol.set_atom_stringc             C   sL   |dkr| j }| j dr0dj|dd d}ndj| d}tt |S )z2 return the PyTables column class for this column Nr  zUInt{name}Col   )rE   z	{name}Col)rb  r7  r   
capitalizer/  r   )r   rb  Zcol_namer?   r?   r@   get_atom_coltype  s    zDataCol.get_atom_coltypec             C   s   | j |d|jd dS )N)rb  r   )r  )r  r  )r   r  rb  r?   r?   r@   get_atom_data  s    zDataCol.get_atom_datac             C   sX   |j j| _t| jdd d }t j||jd d| _| 	|j
j| jjdd d S )Nr     r   )rd  r  F)r;  )ry  rE   rb  r]  rP  r   Z
ComplexColr  rc  r  r   r  r   )r   r  rd  r?   r?   r@   r    s
    
zDataCol.set_atom_complexc             C   s4   |j j| _| || _| |jj| jjdd d S )NF)r;  )	ry  rE   rb  r  rc  r  r   r  r   )r   r  r?   r?   r@   r    s    
zDataCol.set_atom_datac             C   s   |j }|j}d| _|jj| _|jdkr.tdt|dkrBtd|j| _| j	||jjd| _
| t| d| _| |j j | | d S )Nr  rG   zonly support 1-d categoricalsz&only support single block categoricals)rb  r  )r   codesrb  ry  rE   r  r   r   r'  r  rc  r  r5   r   r  
categoriesr  )r   r  r   r?  r   r  r?   r?   r@   r  "  s    

zDataCol.set_atom_categoricalc             C   s   t  j|jd dS )Nr   )r  )r   Int64Colr  )r   r  r?   r?   r@   get_atom_datetime64;  s    zDataCol.get_atom_datetime64c             C   s6   d| _ | || _|d kr&|jd}| |d d S )N
datetime64i8)rb  r  rc  r   viewr  )r   r  r   r?   r?   r@   r  >  s
    zDataCol.set_atom_datetime64c             C   sV   |d kr|j }|j|j}t|j j| _| | d| _| || _	| 
|d d S )Nr  )r   asi8reshaper  _get_tzr`  r  rb  r  rc  r  )r   r  r?  r   r?   r?   r@   r  E  s    
zDataCol.set_atom_datetime64tzc             C   s   t  j|jd dS )Nr   )r  )r   r  r  )r   r  r?   r?   r@   get_atom_timedelta64U  s    zDataCol.get_atom_timedelta64c             C   s6   d| _ | || _|d kr&|jd}| |d d S )Ntimedelta64r  )rb  r  rc  r   r  r  )r   r  r   r?   r?   r@   r  X  s
    zDataCol.set_atom_timedelta64c             C   s   t | jdd S )Nr  )r/  r:  )r   r?   r?   r@   r  _  s    zDataCol.shapec             C   s   | j S )z return my cython values )r:  )r   r?   r?   r@   r  c  s    zDataCol.cvaluesc             C   s`   |r\t | j| jd}|dk	r2|t| jkr2tdt | j| jd}|dk	r\|| jkr\tddS )zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r/  r  rf  rK   r   r   r  ry  )r   rM   Zexisting_fieldsZexisting_dtyper?   r?   r@   r  h  s    
zDataCol.validate_attrc       
      C   s  |j jdk	r|| j }| | t| j}| j dk	rt| j }|dkr`t| j| jdd| _nh|dkr~t	j
| jdd| _nJ|dkry t	j
d	d
 | jD td| _W n0 tk
r   t	j
dd
 | jD td| _Y nX n|dkrt	j
dd
 | jD td| _n|dkr| j}| j }|dkr6tg t	jd}n<t|}	|	 rr||	  }||dk  |	t j8  < tj||| jd| _n@y| jj|dd| _W n( tk
r   | jjddd| _Y nX t| jdkrt| j|||d| _| S )z^set the data from this selection (and convert to the correct dtype
        if we can)
        Nr  T)coercer  zm8[ns])ry  r   c             S   s   g | ]}t |qS r?   )r   fromordinal)r   r&  r?   r?   r@   r     s    z#DataCol.convert.<locals>.<listcomp>c             S   s   g | ]}t |qS r?   )r   fromtimestamp)r   r&  r?   r?   r@   r     s    r   c             S   s   g | ]}t |qS r?   )r   r  )r   r&  r?   r?   r@   r     s    r  r  )r  r'  F)r;  Or  )r|  rC   r}  )ry  rL  re  r  rA   r   r{  r:  r`  r<   asarrayr   r   rh  r  r   float64r%   anyr  r]  Zcumsumr   r*   Z
from_codesr'  r   rb  _unconvert_string_array)
r   r   r|  rC   r}  r   ry  r  r  maskr?   r?   r@   r~  w  sT    









 zDataCol.convertc             C   sB   t | j| jd| _t | j| jd| _t | j| jd| _|   dS )z get the data for this column N)	r/  r  rf  r   r  ry  r  r   r  )r   r?   r?   r@   ru    s    zDataCol.get_attrc             C   sD   t | j| j| j t | j| j| j | jdk	r@t | j| j| j dS )z set the data for this column N)r  r  rf  r   r  r   ry  r  )r   r?   r?   r@   r    s    
zDataCol.set_attr)NNNN)NNNNNNNN)N)Nr~   )N)N)NN)N)N)N)%rT   rU   rV   rY  r  r  r  classmethodr  r   r   rr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rZ  r  r  r  r~  ru  r  __classcell__r?   r?   )r  r@   r  *  sD     



40





Nr  c               @   s>   e Zd ZdZdZdd Zdd Zddd	Zd
d Zdd Z	dS )DataIndexableColz- represent a data column that can be indexed Tc             C   s   t | j stdd S )Nz-cannot have non-object label DataIndexableCol)r   r   Z	is_objectr   )r   r?   r?   r@   r    s    zDataIndexableCol.validate_namesc             C   s   t  j|dS )N)rd  )r   r  )r   r  rd  r?   r?   r@   r    s    z DataIndexableCol.get_atom_stringNc             C   s   | j |d S )N)rb  )r  )r   r  rb  r?   r?   r@   r    s    zDataIndexableCol.get_atom_datac             C   s
   t   S )N)r   r  )r   r  r?   r?   r@   r    s    z$DataIndexableCol.get_atom_datetime64c             C   s
   t   S )N)r   r  )r   r  r?   r?   r@   r    s    z%DataIndexableCol.get_atom_timedelta64)N)
rT   rU   rV   rY  r  r  r  r  r  r  r?   r?   r?   r@   r    s   
r  c               @   s   e Zd ZdZdd ZdS )GenericDataIndexableColz* represent a generic pytables data column c             C   s   d S )Nr?   )r   r?   r?   r@   ru    s    z GenericDataIndexableCol.get_attrN)rT   rU   rV   rY  ru  r?   r?   r?   r@   r    s   r  c               @   sB  e Zd ZdZdZdZdZdZd=ddZe	dd Z
d	d
 Ze	dd Ze	dd Zdd Zdd Zdd Ze	dd Ze	dd Ze	dd Ze	dd Ze	dd Ze	dd  Ze	d!d" Ze	d#d$ Ze	d%d& Zd'd( Zd)d* Ze	d+d, Ze	d-d. Ze	d/d0 Zd1d2 Zd>d3d4Z d5d6 Z!d7d8 Z"d9d: Z#d?d;d<Z$dS )@Fixeda   represent an object in my store
        facilitate read/write of various types of objects
        this is an abstract base class

        Parameters
        ----------

        parent : my parent HDFStore
        group  : the group node where the table resides
        NFr~   c             K   s(   || _ || _t|| _|| _|   d S )N)r   r   rD   rC   r}  set_version)r   r   r   rC   r}  r   r?   r?   r@   r   	  s
    
zFixed.__init__c             C   s*   | j d dko(| j d dko(| j d dk S )Nr   rG   r  r}   )r   )r   r?   r?   r@   is_old_version	  s    zFixed.is_old_versionc             C   sl   t t| jjdd}y8tdd |dD | _t| jdkrJ| jd | _W n tk
rf   d| _Y nX dS )	z compute and set our version pandas_versionNc             s   s   | ]}t |V  qd S )N)r]  )r   rH   r?   r?   r@   r  	  s    z$Fixed.set_version.<locals>.<genexpr>.r}   )r   )r   r   r   )	rA   r/  r   r0  rL   rP  r   r   r   )r   r   r?   r?   r@   r  	  s    zFixed.set_versionc             C   s   t t| jjdd S )Nr-  )rA   r/  r   r0  )r   r?   r?   r@   r-  	  s    
zFixed.pandas_typec             C   s   dS )Nr[   r?   )r   r?   r?   r@   format_type 	  s    zFixed.format_typec             C   sV   |    | j}|dk	rPt|ttfr@djddd |D d}dj| j|dS | jS )	z* return a pretty representation of myself Nz	[{shape}]rm  c             s   s   | ]}t |V  qd S )N)r:   )r   rH   r?   r?   r@   r  +	  s    z$Fixed.__unicode__.<locals>.<genexpr>)r  z{type:12.12} (shape->{shape}))r   r  )r   r  r;   rK   rL   r   ro  r-  )r   r>   r?   r?   r@   r   $	  s    zFixed.__unicode__c             C   s&   t | j| j_t t| j_|   dS )z set my pandas type & version N)r   pandas_kindr  r-  _versionr  r  )r   r?   r?   r@   rT  0	  s    zFixed.set_object_infoc             C   s   t  | }|S )N)r;  )r   rt  r?   r?   r@   r;  6	  s    
z
Fixed.copyc             C   s   | j S )N)obj_type)r   r?   r?   r@   storage_obj_type:	  s    zFixed.storage_obj_typec             C   s   | j S )N)r   )r   r?   r?   r@   r  >	  s    zFixed.shapec             C   s   | j jS )N)r   r   )r   r?   r?   r@   r  B	  s    zFixed.pathnamec             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   F	  s    zFixed._handlec             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   J	  s    zFixed._filtersc             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   N	  s    zFixed._complevelc             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   R	  s    zFixed._fletcher32c             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   V	  s    zFixed._complibc             C   s   | j jS )N)r   r0  )r   r?   r?   r@   r  Z	  s    zFixed.attrsc             C   s   dS )z set our object attributes Nr?   )r   r?   r?   r@   	set_attrs^	  s    zFixed.set_attrsc             C   s   dS )z get our object attributes Nr?   )r   r?   r?   r@   	get_attrsb	  s    zFixed.get_attrsc             C   s   | j S )z return my storable )r   )r   r?   r?   r@   storablef	  s    zFixed.storablec             C   s   dS )NFr?   )r   r?   r?   r@   rS  k	  s    zFixed.is_existsc             C   s   t | jdd S )Nr   )r/  r  )r   r?   r?   r@   r   o	  s    zFixed.nrowsc             C   s   |dkrdS dS )z' validate against an existing storable NTr?   )r   rq  r?   r?   r@   r  s	  s    zFixed.validatec             C   s   dS )z- are we trying to operate on an old version? Tr?   )r   rO   r?   r?   r@   validate_versiony	  s    zFixed.validate_versionc             C   s   | j }|dkrdS |   dS )zh infer the axes of my storer
              return a boolean indicating if we have a valid storer or not NFT)r  r  )r   r>   r?   r?   r@   r   }	  s
    zFixed.infer_axesc             K   s   t dd S )Nz>cannot read on an abstract storer: subclasses should implement)r   )r   r   r?   r?   r@   r   	  s    z
Fixed.readc             K   s   t dd S )Nz?cannot write on an abstract storer: sublcasses should implement)r   )r   r   r?   r?   r@   rU  	  s    zFixed.writec             K   s0   t |||r$| jj| jdd dS tddS )zs
        support fully deleting the node in its entirety (only) - where
        specification must be None
        T)r  Nz#cannot delete on an abstract storer)r  r  r   rO  r   r   )r   rO   r   r   r   r?   r?   r@   r  	  s    zFixed.delete)Nr~   )N)NNN)%rT   rU   rV   rY  r  r  r  r  r   rZ  r  r  r-  r  r   rT  r;  r  r  r  r   r   r   r   r   r  r  r  r  rS  r   r  r   r   r   rU  r  r?   r?   r?   r@   r    sB   


r  c               @   s   e Zd ZdZedediZdd eeD Z	g Z
dd Zdd	 Zd
d Zdd Zedd Zdd Zdd Zdd Zd1ddZdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd2d)d*Zd+d, Zd-d. Zd3d/d0Z dS )4GenericFixedz a generified fixed version r   Zperiodc             C   s   i | ]\}}||qS r?   r?   )r   r   r&  r?   r?   r@   
<dictcomp>	  s    zGenericFixed.<dictcomp>c             C   s   | j |dS )N )_index_type_mapr   )r   r  r?   r?   r@   _class_to_alias	  s    zGenericFixed._class_to_aliasc             C   s   t |tr|S | j|tS )N)r;   r   _reverse_index_mapr   r   )r   aliasr?   r?   r@   _alias_to_class	  s    
zGenericFixed._alias_to_classc             C   s0   |t krddd}|S |tkr,ddd}|S |S )Nc             S   s.   t j| jd |d}|d k	r*|d|}|S )N)rE   r_  UTC)r   _simple_newr   tz_localize
tz_convert)r   r_  r`  resultr?   r?   r@   r]   	  s
    
z*GenericFixed._get_index_factory.<locals>.fc             S   s   t j| d |dS )N)rE   r_  )r   r
  )r   r_  r`  r?   r?   r@   r]   	  s    )NN)NN)r   r   )r   klassr]   r?   r?   r@   _get_index_factory	  s    

zGenericFixed._get_index_factoryc             C   sF   t  |}|dd}|dk	r&td|dd}|dk	rBtd|S )zz
        remove table keywords from kwargs and return
        raise if any keywords are passed which are not-None
        r   Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyrO   zucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r;  popr   )r   r   r   rO   r?   r?   r@   validate_read	  s    
zGenericFixed.validate_readc             C   s   dS )NTr?   )r   r?   r?   r@   rS  	  s    zGenericFixed.is_existsc             C   s   | j | j_ | j| j_dS )z set our object attributes N)rC   r  r}  )r   r?   r?   r@   r  	  s    
zGenericFixed.set_attrsc          	   C   sV   t t| jdd| _tt| jdd| _x(| jD ]}t| |tt| j|d q0W dS )z retrieve our attributes rC   Nr}  r~   )rD   r/  r  rC   rA   r}  
attributesr  )r   r   r?   r?   r@   r  	  s    zGenericFixed.get_attrsc             K   s   |    d S )N)r  )r   r   r   r?   r?   r@   rU  	  s    zGenericFixed.writeNc             C   s   ddl }t| j|}|j}t|dd}t||jrD|d || }nrt|dd}	t|dd}
|
dk	rttj|
|	d}n||| }|	dkrt|t|d	dd
d}n|	dkrtj	|dd}|r|j
S |S dS )z4 read an array for the specified node (off of group r   N
transposedF
value_typer  )ry  r  r`  T)r  r  zm8[ns])r   r/  r   r0  r;   ZVLArrayr<   rN  r{  r  T)r   r   r   r   r   r   r  r  Zretry  r  r?   r?   r@   
read_array	  s$    zGenericFixed.read_arrayc             K   s   t t| jdj|d}|dkr.| j|f|S |dkrD| j|f|S |dkrZ| j|f|S |dkr| jt| j|f|\}}|S t	dj|dd S )	Nz{key}_variety)r   multir  	sparseintregularz%unrecognized index variety: {variety})variety)
rA   r/  r  r   read_multi_indexread_block_indexread_sparse_intindexread_index_noder   r   )r   r   r   r  _r  r?   r?   r@   
read_index
  s    zGenericFixed.read_indexc             C   sL  t |tr0t| jdj|dd | || nt |tr^t| jdj|dd | || nt |trt| jdj|dd | 	|| nt| jdj|dd t
|| j| j| jd}| ||j t| j|}|j|j_|j|j_t |ttfr| t||j_t|dr"|j|j_t|d	rH|jd k	rHt|j|j_d S )
Nz{key}_variety)r   r  r  r  r  r  r_  r`  )r;   r   r  r  r   write_multi_indexr,   write_block_indexr-   write_sparse_intindex_convert_indexrC   r}  r  ri  write_arrayr   r/  r   rb  r0  rE   r   r   r  r   index_classhasattrr_  r`  r  )r   r   r  	convertedr   r?   r?   r@   write_index
  s,    





zGenericFixed.write_indexc             C   sH   |  dj|d|j |  dj|d|j t| jdj|d|j d S )Nz{key}_blocs)r   z{key}_blengthsz{key}_length)r%  r   blocsblengthsr  r  length)r   r   r  r?   r?   r@   r"  4
  s    zGenericFixed.write_block_indexc             K   sL   t | jdj|d}| jdj|df|}| jdj|df|}t|||S )Nz{key}_length)r   z{key}_blocsz{key}_blengths)r/  r  r   r  r,   )r   r   r   r,  r*  r+  r?   r?   r@   r  9
  s    zGenericFixed.read_block_indexc             C   s2   |  dj|d|j t| jdj|d|j d S )Nz{key}_indices)r   z{key}_length)r%  r   indicesr  r  r,  )r   r   r  r?   r?   r@   r#  ?
  s    z"GenericFixed.write_sparse_intindexc             K   s4   t | jdj|d}| jdj|df|}t||S )Nz{key}_length)r   z{key}_indices)r/  r  r   r  r-   )r   r   r   r,  r-  r?   r?   r@   r  C
  s    z!GenericFixed.read_sparse_intindexc             C   s   t | jdj|d|j xtt|j|j|jD ]\}\}}}dj||d}t	|| j
| j| j|}| ||j t| j|}	|j|	j_||	j_t |	jdj||d| dj||d}
| |
| q0W d S )Nz{key}_nlevels)r   z{key}_level{idx})r   r  z{key}_name{name})r   rE   z{key}_label{idx})r  r  r   rK  r  r  levelsr  namesr$  rC   r}  r  ri  r%  r   r/  r   rb  r0  rE   )r   r   r  r  levlevel_codesrE   	level_keyZ
conv_levelr   	label_keyr?   r?   r@   r!  H
  s    


zGenericFixed.write_multi_indexc             K   s   t | jdj|d}g }g }g }xrt|D ]f}dj||d}| jt | j|f|\}	}
||
 ||	 dj||d}| j|f|}|| q*W t|||ddS )Nz{key}_nlevels)r   z{key}_level{idx})r   r  z{key}_label{idx}T)r.  r  r/  r   )	r/  r  r   r   r  r   rM   r  r   )r   r   r   rK  r.  r  r/  r  r2  rE   r0  r3  r1  r?   r?   r@   r  _
  s    


zGenericFixed.read_multi_indexc             C   s"  ||| }d|j krF| t|j drFtjt|j dt|j dd}t|j j}d }d|j krtt|j j}t|}| 	tt|j dd}| 
|}i }	d|j kr|j d |	d< d|j kr|j d |	d< |d	kr|t||| j| jd
fdti|	}
n|t||| j| jd
f|	}
||
_||
fS )Nr  r  )ry  rE   r&  r  r_  r`  )r   r   )rC   r}  ry  )r0  _is_empty_arrayr/  r<   rN  rA   rb  rF   rE   r  r  _unconvert_indexrC   r}  r   )r   r   r   r   r:  rb  rE   r&  factoryr   r  r?   r?   r@   r  s
  s8    




zGenericFixed.read_index_nodec             C   sN   t d|j }| j| j|| t|jt| j|j	_
|jt| j|j	_dS )z write a 0-len array )rG   N)r<   rN  r  r   create_arrayr   r   ry  r/  r0  r  r  )r   r   r   Zarrr?   r?   r@   write_array_empty
  s    zGenericFixed.write_array_emptyc             C   s   t dd |D S )z(Returns true if any axis is zero length.c             s   s   | ]}|d kV  qdS )r   Nr?   )r   rH   r?   r?   r@   r  
  s    z/GenericFixed._is_empty_array.<locals>.<genexpr>)r  )r   r  r?   r?   r@   r4  
  s    zGenericFixed._is_empty_arrayc             C   sf  || j kr| j| j | | |j}d}t|r:td|sRt|drR|j}d}| j	d k	rd }yt
 j|j}W n tk
r   Y nX |d k	r|s| jj| j |||j| j	d}||d d < |t| j |j_n| || d S |jjtjkr|tj| dd}|r
nN|dkrnByt|}W n tk
r8   Y nX t|||f }	tj|	tdd	 | j | j |t
 ! }
|
"| n|r| || nt#|jr| j$| j ||%d
 dt| j |j_&nt'|jr
| j$| j ||j( t| j |}t)|j*|j_*d|j_&nHt+|jr@| j$| j ||%d
 dt| j |j_&n| j$| j || |t| j |j_d S )NFz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".r  T)r   )r  r     )r  r  r  r  ),r   r   rO  r4  r  r   r   r'  r  r   r   ZAtomZ
from_dtypery  r   Zcreate_carrayr/  r0  r  r8  r   r<   Zobject_r   r  r  rK   r   performance_docr  r  r   Zcreate_vlarray
ObjectAtomrM   r   r7  r  r  r   r  r  r`  r   )r   r   r   r   Zempty_arrayr  atomZcar  r  Zvlarrr   r?   r?   r@   r%  
  st    





zGenericFixed.write_array)NN)NN)N)!rT   rU   rV   rY  r   r   r  r#   r[  r  r  r  r  r  r  rZ  rS  r  r  rU  r  r   r)  r"  r  r#  r  r!  r  r  r8  r4  r%  r?   r?   r?   r@   r  	  s2   
!
(	r  c               @   s   e Zd ZdddZdS )LegacyFixedNc             C   s4   t | j|}||| }|jj}t||| j| jdS )N)rC   r}  )r/  r   r0  rb  _unconvert_index_legacyrC   r}  )r   r   r   r   r   r:  rb  r?   r?   r@   read_index_legacy  s
    
zLegacyFixed.read_index_legacy)NN)rT   rU   rV   r?  r?   r?   r?   r@   r=    s   r=  c               @   s   e Zd Zdd ZdS )rd   c             K   s*   |  |}| d}| d}t||dS )Nr  r   )r  )r  r?  r  r   )r   r   r  r   r?   r?   r@   r     s    


zLegacySeriesFixed.readN)rT   rU   rV   r   r?   r?   r?   r@   rd     s   c               @   s   e Zd Zdd ZdS )re   c             K   s6   |  |}| d}| d}| d}t|||dS )Nr  r   r   )r  r   )r  r?  r  r   )r   r   r  r   r   r?   r?   r@   r     s
    



zLegacyFrameFixed.readN)rT   rU   rV   r   r?   r?   r?   r@   re     s   c                   s:   e Zd ZdZdgZedd Zdd Z fddZ  Z	S )	rf   r_   rE   c          	   C   s0   yt t| jdfS  ttfk
r*   d S X d S )Nr   )r   r/  r   r   r   )r   r?   r?   r@   r  "  s    zSeriesFixed.shapec             K   s2   |  |}| jd|}| jd|}t||| jdS )Nr  r   )r  rE   )r  )r   )r  r   r  r   rE   )r   r   r  r   r?   r?   r@   r   )  s    
zSeriesFixed.readc                s>   t t| j|f| | d|j | d|j |j| j_d S )Nr  r   )	r  rf   rU  r)  r  r%  r   rE   r  )r   r   r   )r  r?   r@   rU  /  s    zSeriesFixed.write)
rT   rU   rV   r  r  rZ  r  r   rU  r  r?   r?   )r  r@   rf     s
   c                   s   e Zd Z fddZ  ZS )SparseFixedc                s,   t t| |}d|ks d|kr(td|S )z=
        we don't support start, stop kwds in Sparse
        r   r   z;start and/or stop are not supported in fixed Sparse reading)r  r@  r  r   )r   r   )r  r?   r@   r  8  s    zSparseFixed.validate_read)rT   rU   rV   r  r  r?   r?   )r  r@   r@  6  s   r@  c                   s2   e Zd ZdZdddgZdd Z fddZ  ZS )	rg   r`   rE   
fill_valuerb  c             K   sF   |  |}| d}| d}| d}t|||| jp8d| j| jdS )Nr  	sp_valuessp_indexr  )r  Zsparse_indexrb  rA  rE   )r  r   r  r!   rb  rA  rE   )r   r   r  rB  rC  r?   r?   r@   r   G  s    



zSparseSeriesFixed.readc                s`   t t| j|f| | d|j | d|j | d|j |j| j	_|j
| j	_
|j| j	_d S )Nr  rC  rB  )r  rg   rU  r)  r  rC  r%  rB  rE   r  rA  rb  )r   r   r   )r  r?   r@   rU  Q  s    

zSparseSeriesFixed.write)rT   rU   rV   r  r  r   rU  r  r?   r?   )r  r@   rg   C  s   

c                   s0   e Zd ZdZddgZdd Z fddZ  ZS )ri   rb   default_kinddefault_fill_valuec             K   sn   |  |}| d}i }x@|D ]8}dj|d}t| jt| j|}|  | ||< qW t	||| j
| jdS )Nr   zsparse_series_{columns})r   )r   rD  rE  )r  r   r   rg   r   r/  r   r   r   r    rD  rE  )r   r   r   Zsdictr  r   r>   r?   r?   r@   r   _  s    


zSparseFrameFixed.readc                s   t t| j|f| xbt|D ]T\}}dj|d}|| jjkrR| j	| j|}nt
| j|}t| j|}|| q W |j| j_|j| j_| d|j dS )z6 write it as a collection of individual sparse series zsparse_series_{name})rE   r   N)r  ri   rU  r#   r[  r   r   r4  r   rR  r/  rg   r   rE  r  rD  r)  r   )r   r   r   rE   Zssr   r   r>   )r  r?   r@   rU  l  s    

zSparseFrameFixed.write)rT   rU   rV   r  r  r   rU  r  r?   r?   )r  r@   ri   [  s   c                   s>   e Zd ZddgZdZedd ZdddZ fd	d
Z  Z	S )BlockManagerFixedr  nblocksFc             C   s   y| j }d}xFt| jD ]8}t| jdj|d}t|dd }|d k	r||d 7 }qW t| jd}t|dd }|d k	rt|d|d  }ng }|| | jr|d d d }|S  t	k
r   d S X d S )Nr   zblock{idx}_items)r  r  Zblock0_valuesrG   r  )
r  r   rG  r/  r   r   rK   rM   is_shape_reversedr   )r   r  r   r  r   r  r?   r?   r@   r    s&    
zBlockManagerFixed.shapeNc             K   s   |  |}|  d}g }xLt| jD ]>}||kr<||fnd\}}| jdj|d||d}	||	 q(W |d }
g }xXt| jD ]J}| dj|d}| j	dj|d||d}t
||
|d}|| qW | t||S )	Nr   )NNz	axis{idx})r  )r   r   zblock{idx}_itemszblock{idx}_values)	placement)r  r  Z_get_block_manager_axisr   r  r   r   rM   rG  r  r7   r  r3   )r   r   r   r   Zselect_axisr  r  r   r   axr   blocks	blk_itemsr   blkr?   r?   r@   r     s$    

zBlockManagerFixed.readc                s   t t| j|f| |j}| s*| }|j| j_x@t|j	D ]2\}}|dkr^|j
s^td| dj|d| q@W t|j| j_xRt|jD ]D\}}|j|j}| jdj|d|j|d | dj|d| qW d S )Nr   z/Columns index has to be unique for fixed formatz	axis{idx})r  zblock{idx}_values)r   zblock{idx}_items)r  rF  rU  _dataZis_consolidatedZconsolidater  r  r  r  Z	is_uniquer   r)  r   r   rK  rG  r   r  mgr_locsr%  r   )r   r   r   r:  r  rJ  rM  rL  )r  r?   r@   rU    s"    
zBlockManagerFixed.write)NN)
rT   rU   rV   r  rH  rZ  r  r   rU  r  r?   r?   )r  r@   rF  |  s
   
rF  c               @   s   e Zd ZdZeZdS )rh   ra   N)rT   rU   rV   r  r   r  r?   r?   r?   r@   rh     s   c                   s(   e Zd ZdZeZdZ fddZ  ZS )rj   rc   Tc                s   |   tt| j|f|S )N)Z_consolidate_inplacer  rj   rU  )r   r   r   )r  r?   r@   rU    s    zPanelFixed.write)	rT   rU   rV   r  r   r  rH  rU  r  r?   r?   )r  r@   rj     s   c                   s  e Zd ZdZdZdZdZdZdZ fddZ	e
d	d
 Ze
dd Zdd Zdd Zdd Ze
dd Zdd Zdd Ze
dd Ze
dd Ze
dd Ze
dd  Ze
d!d" Ze
d#d$ Ze
d%d& Ze
d'd( Ze
d)d* Ze
d+d, Zd-d. Zd/d0 Zd1d2 Zd3d4 Z d5d6 Z!d7d8 Z"d9d: Z#d;d< Z$d=d> Z%dWd?d@Z&dAdB Z'e
dCdD Z(dXdEdFZ)dGdH Z*dIdJ Z+dKdL Z,dYdMdNZ-dZdOdPZ.d[dQdRZ/d\dSdTZ0d]dUdVZ1  Z2S )^r1  a   represent a table:
          facilitate read/write of various types of tables

        Attrs in Table Node
        -------------------
        These are attributes that are store in the main table node, they are
        necessary to recreate these tables when read back in.

        index_axes    : a list of tuples of the (original indexing axis and
            index column)
        non_index_axes: a list of tuples of the (original index axis and
            columns on a non-indexing axis)
        values_axes   : a list of the columns which comprise the data of this
            table
        data_columns  : a list of the columns that we are allowing indexing
            (these become single columns in values_axes), or True to force all
            columns
        nan_rep       : the string to use for nan representations for string
            objects
        levels        : the names of levels
        metadata      : the names of the metadata columns

        rH  NrG   TFc                sH   t t| j|| g | _g | _g | _g | _g | _t | _	d | _
d | _d S )N)r  r1  r   
index_axesr   values_axesr  rh  r  r?  r|  	selection)r   argsr   )r  r?   r@   r      s    zTable.__init__c             C   s   | j dd S )Nr  r   )rC  rP  )r   r?   r?   r@   table_type_short  s    zTable.table_type_shortc             C   s   dS )Nr\   r?   )r   r?   r?   r@   r    s    zTable.format_typec          
   C   s   |    djt| jr"d| jndd}d}| jrRdjddd | jD d	}d
j| j|| j| j	| j
ddd | jD |dS )z+ return a pretty representatgion of myself z,dc->[{columns}]rm  r  )r   z[{version}]r  c             s   s   | ]}t |V  qd S )N)r   )r   rH   r?   r?   r@   r    s    z$Table.__unicode__.<locals>.<genexpr>)r   zg{pandas_type:12.12}{ver} (typ->{table_type},nrows->{nrows},ncols->{ncols},indexers->[{index_axes}]{dc})c             s   s   | ]}|j V  qd S )N)rE   )r   r   r?   r?   r@   r  $  s    )r-  verrC  r   ncolsrP  r(  )r   r   r   r  ro  r  r   r-  rT  r   rV  rP  )r   r(  rU  r?   r?   r@   r     s    zTable.__unicode__c             C   s"   x| j D ]}||jkr|S qW dS )z return the axis for c N)r  rE   )r   r  r   r?   r?   r@   r   '  s    
zTable.__getitem__c             C   s   |dkrdS |j | j kr.tdj|j | j dxzdD ]r}t| |d}t||d}||kr4x8t|D ],\}}|| }||krbtdj|||dqbW tdj|||dq4W dS )	z$ validate against an existing table Nz8incompatible table_type with existing [{other} - {self}])rq  r   )rP  r   rQ  zMinvalid combinate of [{c}] on appending data [{sax}] vs current table [{oax}])r  saxoaxzKinvalid combinate of [{c}] on appending data [{sv}] vs current table [{ov}])r  svov)rC  r   r   r/  r  r   r   )r   rq  r  rY  rZ  r  rW  rX  r?   r?   r@   r  .  s&    
zTable.validatec             C   s   t | jtS )z@the levels attribute is 1 or a list in the case of a multi-index)r;   r.  rK   )r   r?   r?   r@   is_multi_indexL  s    zTable.is_multi_indexc             C   s   dd | j D | _dS )z create / validate metadata c             S   s   g | ]}|j d k	r|jqS )N)rh  rE   )r   r  r?   r?   r@   r   T  s    z+Table.validate_metadata.<locals>.<listcomp>N)rQ  rh  )r   r  r?   r?   r@   r  Q  s    zTable.validate_metadatac             C   sD   dd t |jjD }y| |fS  tk
r>   tdY nX dS )z\validate that we can store the multi-index; reset and return the
        new object
        c             S   s&   g | ]\}}|d k	r|nd |qS )Nz	level_{0})r   )r   r  lr?   r?   r@   r   Z  s   z-Table.validate_multiindex.<locals>.<listcomp>zBduplicate names/columns in the multi-index when storing as a tableN)r  r  r/  Zreset_indexr   )r   r   r.  r?   r?   r@   validate_multiindexV  s    zTable.validate_multiindexc             C   s   t dd | jD S )z/ based on our axes, compute the expected nrows c             S   s   g | ]}|j jd  qS )r   )r  r  )r   r  r?   r?   r@   r   e  s    z(Table.nrows_expected.<locals>.<listcomp>)r<   prodrP  )r   r?   r?   r@   nrows_expectedb  s    zTable.nrows_expectedc             C   s
   d| j kS )z has this table been created r\   )r   )r   r?   r?   r@   rS  g  s    zTable.is_existsc             C   s   t | jdd S )Nr\   )r/  r   )r   r?   r?   r@   r  l  s    zTable.storablec             C   s   | j S )z. return the table group (this is my storable) )r  )r   r?   r?   r@   r\   p  s    zTable.tablec             C   s   | j jS )N)r\   ry  )r   r?   r?   r@   ry  u  s    zTable.dtypec             C   s   | j jS )N)r\   r  )r   r?   r?   r@   r  y  s    zTable.descriptionc             C   s   t | j| jS )N)r  r  rP  rQ  )r   r?   r?   r@   r  }  s    z
Table.axesc             C   s   t dd | jD S )z0 the number of total columns in the values axes c             s   s   | ]}t |jV  qd S )N)r   r   )r   r   r?   r?   r@   r    s    zTable.ncols.<locals>.<genexpr>)sumrQ  )r   r?   r?   r@   rV    s    zTable.ncolsc             C   s   dS )NFr?   )r   r?   r?   r@   is_transposed  s    zTable.is_transposedc             C   s(   t tdd | jD dd | jD S )z@return a tuple of my permutated axes, non_indexable at the frontc             S   s   g | ]}t |d  qS )r   )r]  )r   r   r?   r?   r@   r     s    z*Table.data_orientation.<locals>.<listcomp>c             S   s   g | ]}t |jqS r?   )r]  r   )r   r   r?   r?   r@   r     s    )rL   r  r  r   rP  )r   r?   r?   r@   data_orientation  s    zTable.data_orientationc                s<   t dd  jD  fdd jD   fdd jD  S )z> return a dict of the kinds allowable columns for this object c             S   s   g | ]}|j |fqS r?   )re  )r   r   r?   r?   r@   r     s    z$Table.queryables.<locals>.<listcomp>c                s    g | ]\}} j j| d fqS )N)r  _AXIS_NAMES)r   r   r   )r   r?   r@   r     s   c                s&   g | ]}|j t jkr|j|fqS r?   )rE   r  r  re  )r   r&  )r   r?   r@   r     s    )r  rP  r   rQ  )r   r?   )r   r@   
queryables  s    "zTable.queryablesc             C   s   dd | j D S )z  return a list of my index cols c             S   s   g | ]}|j |jfqS r?   )r   re  )r   r  r?   r?   r@   r     s    z$Table.index_cols.<locals>.<listcomp>)rP  )r   r?   r?   r@   
index_cols  s    zTable.index_colsc             C   s   dd | j D S )z! return a list of my values cols c             S   s   g | ]
}|j qS r?   )re  )r   r  r?   r?   r@   r     s    z%Table.values_cols.<locals>.<listcomp>)rQ  )r   r?   r?   r@   values_cols  s    zTable.values_colsc             C   s   dj | jj|dS )z+ return the metadata pathname for this key z{group}/meta/{key}/meta)r   r   )r   r   r   )r   r   r?   r?   r@   _get_metadata_path  s    
zTable._get_metadata_pathc             C   s0   t |}| jj| ||d| j| j| jd dS )z
        write out a meta data array to the key as a fixed-format Series

        Parameters
        ----------
        key : string
        values : ndarray

        r\   )r   rC   r}  r|  N)r   r   r   rg  rC   r}  r|  )r   r   r   r?   r?   r@   r    s    
zTable.write_metadatac             C   s0   t t | jdd|ddk	r,| j| |S dS )z) return the meta data array for this key r   N)r/  r   r   r   rg  )r   r   r?   r?   r@   rv    s    zTable.read_metadatac             C   s   | j | j_ dS )z update our table index info N)r?  r  )r   r?   r?   r@   r    s    zTable.set_infoc             C   sx   t | j| j_|  | j_|  | j_| j| j_| j| j_| j| j_| j| j_| j	| j_	| j
| j_
| j| j_|   dS )z! set our table type & indexables N)r   rC  r  re  rf  r   r  r|  rC   r}  r.  rh  r  )r   r?   r?   r@   r    s    






zTable.set_attrsc                s   t  jddpg  _t  jddp$g  _t  jddp:t  _t  jdd _tt  jdd _t	t  jdd _
t  jd	dpg  _ fd
d jD  _ fdd jD  _t  jddpg  _dS )z retrieve our attributes r   Nr  r?  r|  rC   r}  r~   r.  c                s   g | ]}|j r| qS r?   )r  rx  )r   r   )r   r?   r@   r     s    z#Table.get_attrs.<locals>.<listcomp>c                s   g | ]}|j s| qS r?   )r  rx  )r   r   )r   r?   r@   r     s    rh  )r/  r  r   r  r  r?  r|  rD   rC   rA   r}  r.  
indexablesrP  rQ  rh  )r   r?   )r   r@   r    s     zTable.get_attrsc             C   s\   |dk	rX| j d dkrX| j d dkrX| j d dk rXtddd | j D  }t|t dS )	z- are we trying to operate on an old version? Nr   rG   r  r}   r  c             S   s   g | ]}t |qS r?   )r   )r   rH   r?   r?   r@   r     s    z*Table.validate_version.<locals>.<listcomp>)r   incompatibility_docro  r  r  rX   )r   rO   r  r?   r?   r@   r     s    zTable.validate_versionc             C   s^   |dkrdS t |tsdS |  }x6| D ]*\}}|dkr>q,||kr,tdj|dq,W dS )zvalidate the min_itemisze doesn't contain items that are not in the
        axes this needs data_columns to be defined
        Nr   zDmin_itemsize has the key [{key}] which is not an axis or data_column)r   )r;   r  rd  r   r   r   )r   r  qr   r&  r?   r?   r@   validate_min_itemsize  s    
zTable.validate_min_itemsizec                sz   j dkrtg _ j dd tjjD  tjtj   fddj fddtjjD  j S )z1 create/cache the indexables if they don't exist Nc             S   s"   g | ]\}\}}t |||d qS ))rE   r   rg  )r^  )r   r  r   rE   r?   r?   r@   r     s   z$Table.indexables.<locals>.<listcomp>c                s(   t }|krt}|j| | |  jdS )N)r  rE   rg  r   )r  r  r  r   )r  r  r  )base_posr(  r   r?   r@   r]     s
    zTable.indexables.<locals>.fc                s   g | ]\}} ||qS r?   r?   )r   r  r  )r]   r?   r@   r     s    )	_indexablesr  r  r  re  r  r  r   rf  )r   r?   )rl  r(  r]   r   r@   rh    s    


zTable.indexablesc             C   s$  |   sdS |dkrdS |dks(|dkr8dd | jD }t|ttfsL|g}t }|dk	rb||d< |dk	rr||d< | j}x|D ]}t|j|d}|dk	r~|j	r|j
}|j}	|j}
|dk	r|
|kr|  n|
|d< |dk	r|	|kr|  n|	|d< |j	s~|jdrtd	|jf | q~W dS )
a  
        Create a pytables index on the specified columns
          note: cannot index Time64Col() or ComplexCol currently;
          PyTables must be >= 3.0

        Parameters
        ----------
        columns : False (don't create an index), True (create all columns
            index), None or list_like (the indexers to index)
        optlevel: optimization level (defaults to 6)
        kind    : kind of index (defaults to 'medium')

        Exceptions
        ----------
        raises if the node is not a table

        NFTc             S   s   g | ]}|j r|jqS r?   )r  re  )r   r   r?   r?   r@   r   ;  s    z&Table.create_index.<locals>.<listcomp>optlevelrb  r  zColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.)r   r  r;   rL   rK   r  r\   r/  r  r9  r  rn  rb  Zremove_indexr   r7  r   r+  )r   r   rn  rb  kwr\   r  r&  r  Zcur_optlevelZcur_kindr?   r?   r@   r+  !  s@    


zTable.create_indexc             K   sn   |  | |  sdS t| fd|i|| _| j }x2| jD ](}|| j |j|| j	| j
| jd q>W dS )z^create and return the axes sniffed from the table: return boolean
        for success
        FrO   )r|  rC   r}  T)r   r   	SelectionrR  r   r  r  r?  r~  r|  rC   r}  )r   rO   r   r   r   r?   r?   r@   	read_axesf  s    

zTable.read_axesc             C   s   |S )z return the data for this obj r?   )r   r   r?   r?   r@   
get_object~  s    zTable.get_objectc                s   t | jsg S | jd \} | j|t }|ddkrN|rNtd|||dkr`t }n|dkrlg }t|trt	||
fdd| D   fd	d|D S )
z[take the input data_columns and min_itemize and create a data
        columns spec
        r   r   r   z<cannot use a multi-index on axis [{0}] with data_columns {1}TNc                s    g | ]}|d kr| kr|qS )r   r?   )r   r   )existing_data_columnsr?   r@   r     s    z/Table.validate_data_columns.<locals>.<listcomp>c                s   g | ]}| kr|qS r?   r?   )r   r  )axis_labelsr?   r@   r     s    )r   r   r?  r   r  r   r   rK   r;   r  r  r   )r   r  r  r   r?  r?   )rt  rs  r@   validate_data_columns  s     



zTable.validate_data_columnsc       %         sf  |dkrHyt t }W n. tk
rF   tdjjjtdY nX fdd|D } r }|  dd |j	D }|j
}|j}|j_|j_t|j_nd}t|jd krtdg _g _
|dkrd	}|_t  xtjD ]\}	}
|	|kr>j|	 }t|
jjj||	 |	< qt|
}|dk	rtj}|j| d }tt|t|sttt|tt|r|}t j|	}t|
j!|d
< |
j"j#|d< j$|	|f qW  fddt|D _	tj	}xjD ]}
|
j%|d q
W x$jD ]}
t&|
d |
d q(W dd }'( }|j)j*}||j)|}tjrjd \}}+||}t|r|j,t-|.t-||dj)}t|j*}|||}x>|D ]6}|j,|g|dj)}|/|j* |/|||j* qW |dk	rdd t0||D }g }g }x||j1D ]r}t2|j3}y&|4|\}}|$| |$| W n: t5tfk
r   tdjd6dd |D dY nX q@W |}|}g _1xvtt0||D ]b\}	\}}t7} d}|r&t|dkr&|d |kr&t8} |d }j
$| |dk	rv|rvy|j1|	 }!W n. t5tfk
rr   tdj||j1dY nX nd}!yL| j9|	|j:d}"|"j;|||!||jjjd |"<| j1$|" W nh t=ttfk
r }# z|#W dd}#~#X Y n< t>k
r. }$ zt>dj|j?j@||$dW dd}$~$X Y nX |d7 }qW A| B| |rbC| dS )a   create and return the axes
        leagcy tables create an indexable column, indexable index,
        non-indexable fields

            Parameters:
            -----------
            axes: a list of the axes in order to create (names or numbers of
                the axes)
            obj : the object to create axes on
            validate: validate the obj against an existing object already
                written
            min_itemsize: a dict of the min size for a column in bytes
            nan_rep : a values to use for string column nan_rep
            encoding : the encoding for string values
            data_columns : a list of columns that we want to create separate to
                allow indexing (or True will force all columns)

        NzFcannot properly create the storer for: [group->{group},value->{value}])r   r   c                s   g | ]}  |qS r?   )_get_axis_number)r   r   )r   r?   r@   r     s    z%Table.create_axes.<locals>.<listcomp>c             S   s   g | ]
}|j qS r?   )r   )r   r   r?   r?   r@   r     s    rG   z<currently only support ndim-1 indexers in an AppendableTablenanr/  r   c                s&   g | ]\}} |  |jqS r?   )rj  r  r?  )r   jr   )index_axes_mapr   r?   r@   r     s   )r  r   c                s    fdd|D S )Nc                s   g | ]} j |jqS r?   )r   r  rO  )r   rM  )mgrr?   r@   r     s    z<Table.create_axes.<locals>.get_blk_items.<locals>.<listcomp>r?   )rz  rK  r?   )rz  r@   get_blk_items  s    z(Table.create_axes.<locals>.get_blk_items)r   c             S   s"   i | ]\}}||ft | qS r?   )rL   tolist)r   bb_itemsr?   r?   r@   r  *  s   z%Table.create_axes.<locals>.<dictcomp>zEcannot match existing table structure for [{items}] on appending datarm  c             s   s   | ]}t |V  qd S )N)r:   )r   r  r?   r?   r@   r  8  s   z$Table.create_axes.<locals>.<genexpr>)r   zCIncompatible appended table [{blocks}]with existing table [{table}])rK  r\   )r  rE   r   )r  r  r  r  r|  rC   r}  r?  zNcannot find the correct atom type -> [dtype->{name},items->{items}] {detail!s})rE   r   r<  )Dr  r   r   r   r   r   r   r   r;  rP  r  r|  rC   r}  r?  r   r  r   r   r  r  r  rc  r$  r  ri  rk  rK   r   r<   r  r  r  r/  r  rT   rM   r  _reindex_axisrr  r   rN  rK  ru  r$  r   r  r  r  rQ  rL   r   r  rM  ro  r  r  r  r   r  rj  r   r   ry  rE   rk  r  r  )%r   r  r   r  r|  r  r  r   Zexisting_tabler  r   rE   Zappend_axisindexerZ
exist_axisr?  rx  r{  Z	block_objrK  rL  r   rt  rz  r  Zby_itemsZ
new_blocksZnew_blk_itemsZear   r}  r~  r  r  r  r   r<  r?   )ry  r   r   r@   create_axes  s   














 

$

zTable.create_axesc       	         s   |dk	rt |}|dk	rBjrBx"jD ]}||kr&|d| q&W x jD ]\}}t ||| qJW jjdk	rx4jj D ]$\}} fdd}||| q~W  S )z process axes filters Nr   c                s   x j  D ]} |} |}| |kr\jrB|tj}||} jj	||dS | |krt
t | j}t
|}t trd| }||} jj	||dS qW tdj| dd S )N)r   rG   z.cannot find the field [{field}] for filtering!)field)rc  r   rv  	_get_axisr[  unionr   r.  r#  Z_getitem_axisr2   r/  r;   r   r   r   )r  filtZ	axis_nameZaxis_numberZaxis_valuesZtakersr   )r   opr   r?   r@   process_filter  s&    




z*Table.process_axes.<locals>.process_filter)	rK   r[  r.  insertr   r  rR  r
   r   )	r   r   r   r   r   labelsr  r  r  r?   )r   r  r   r@   process_axesz  s    #zTable.process_axesc             C   s   |dkrt | jd}td|d}dd | jD |d< |rl|dkrJ| jpHd}t j|||p\| jd	}||d
< n| jdk	r| j|d
< |S )z< create the description of the table from the axes & values Ni'  r\   )rE   expectedrowsc             S   s   i | ]}|j |jqS r?   )rc  re  )r   r   r?   r?   r@   r    s    z,Table.create_description.<locals>.<dictcomp>r  	   )r   r   r   r   )	r  r_  r  r  r   r   r   r   r   )r   r   r   r   r  r%  r   r?   r?   r@   create_description  s    



zTable.create_descriptionc       
      K   s   |  | |  sdS t| f|||d|| _| j }| jjdk	rxT| jj D ]D\}}}| j|| |	 d d}	|||	j
||   |j }qTW t|S )z]select coordinates (row numbers) from a table; return the
        coordinates object
        F)rO   r   r   NrG   )r   r   )r   r   rp  rR  select_coordsr
   r   r   r\  r  ilocr   r   )
r   rO   r   r   r   Zcoordsr  r  r  r:  r?   r?   r@   r     s    

 zTable.read_coordinatesc          	   C   s   |    |  sdS |dk	r$tdx~| jD ]t}||jkr,|jsPtdj|dt| j	j
|}|| j tt|j||| | j| j| jd |jd|dS q,W td	j|ddS )
zareturn a single column from the table, generally only indexables
        are interesting
        FNz4read_column does not currently accept a where clausezMcolumn [{column}] can not be extracted individually; it is not data indexable)r   )r|  rC   r}  T)rE   z(column [{column}] not found in the table)r   r   r   r  rE   r  r   r   r/  r\   r  r  r?  r   r{  r~  r|  rC   r}  r  r`  r   )r   r   rO   r   r   r   r  r?   r?   r@   r     s(    

zTable.read_column)N)NNN)TNNN)N)NNFN)NNN)NNN)3rT   rU   rV   rY  r  rC  r.  r  rH  r   rZ  rT  r  r   r   r  r[  r  r]  r_  rS  r  r\   ry  r  r  rV  ra  rb  rd  re  rf  rg  r  rv  r  r  r  r   rk  rh  r+  rq  rr  ru  r  r  r  r   r   r  r?   r?   )r  r@   r1    sb   
	
E! 
 W
< 

r1  c               @   s$   e Zd ZdZdZdd Zdd ZdS )rq   z a write-once read-many table: this format DOES NOT ALLOW appending to a
         table. writing is a one-time operation the data are stored in a format
         that allows for searching the data on disk
         rz   c             K   s   t ddS )zS read the indices and the indexing array, calculate offset rows and
        return z!WORMTable needs to implement readN)r   )r   r   r?   r?   r@   r     s    zWORMTable.readc             K   s   t ddS )z write in a format that we can search later on (but cannot append
               to): write out the indices and the values using _write_array
               (e.g. a CArray) create an indexing table so that we can search
        z"WORKTable needs to implement writeN)r   )r   r   r?   r?   r@   rU    s    zWORMTable.writeN)rT   rU   rV   rY  rC  r   rU  r?   r?   r?   r@   rq     s   c               @   sV   e Zd ZdZeddddeddddd	ed
dd
ddgZdZdZdd Z	dddZ
dS )LegacyTablea   an appendable table: allow append/query/delete operations to a
          (possibly) already existing appendable table this table ALLOWS
          append (but doesn't require them), and stores the data in a format
          that can be easily searched

    r  rG   r   )rE   r   rg  r   r}   Zcolumns_kind)rE   r   rg  Z
index_kindrL  r   )rE   re  rf  rg  legacy   c             K   s   t dd S )Nz2write operations are not allowed on legacy tables!)r   )r   r   r?   r?   r@   rU  4  s    zLegacyTable.writeNc                sL  | j f d|i|sdS dd | jD }t|\}}t|}t|}dd |D }t|dd |}g }	tt|t|krdtt	|t
|\ }
t  x| jD ]}| j dd}|jdkr||jd df} fd	d|D }t|j}t|t
t|t|||d
}t|g|g| }| |}| jrT|jtt| j  }|	| qW nt j!t"t#dd t$%dd | jD }x| jD ]z}t&|j'||jd}|j}t|}t()|}t*||}t|}||}|jj|dd}t&|||j+d}|	|,  qW t|	dkr(|	d }nt-|	ddd. }| j/||d}|S )zSwe have n indexable columns, with an arbitrary number of data
        axes
        rO   Nc             S   s   g | ]
}|j qS r?   )r   )r   r   r?   r?   r@   r   ?  s    z$LegacyTable.read.<locals>.<listcomp>c             S   s   g | ]}t |qS r?   )r   )r   Zlvlr?   r?   r@   r   D  s    rG   r   )r   c                s   g | ]}|  qS r?   )r  )r   r\  )sorterr?   r@   r   Y  s    )r   rI  r  r  Z	ref_items   )r  c             S   s   g | ]
}|j qS r?   )r   )r   r  r?   r?   r@   r   o  s    )r  r   F)r   r   )r   )0rq  rP  r+   rK   r6   r   r)   r   Zgroupsort_indexerr   r<   r^  r   rQ  r  r  r  r  r  r   r   r4   r  rL   r3   r  ra  	transposer   rb  ZargsortrM   r  r  duplicate_docrZ   r   Zfrom_arraysr   r:  r  Zasarray_tuplesafer(   r   Zto_panelr$   r   r  )r   rO   r   r   Zlst_valsr  r.  Nr   r   r  r  Zsorted_valuesZtake_labelsr   r  rz  r   Z
long_indexZlpZtuple_indexZunique_tuplesr  Z	new_indexZ
new_valuesZwpr?   )r  r@   r   7  s`    






zLegacyTable.read)NN)rT   rU   rV   rY  r^  r  rm  rC  r  rU  r   r?   r?   r?   r@   r  $  s   r  c                   s,   e Zd ZdZdZdZeZ fddZ  Z	S )rr   z  support the legacy frame table rD  r{   c                s   t t| j||d S )Nr   )r  rr   r   )r   rS  r   )r  r?   r@   r     s    zLegacyFrameTable.read)
rT   rU   rV   rY  r  rC  r   r  r   r  r?   r?   )r  r@   rr     s
   c               @   s   e Zd ZdZdZeZdS )rs   z  support the legacy panel table r|   N)rT   rU   rV   rY  rC  r   r  r?   r?   r?   r@   rs     s   c            	   @   s>   e Zd ZdZdZdZdddZdddZd	d
 ZdddZ	dS )AppendableTablez) suppor the new appendable table formats NZ
appendableFc             K   s   |s| j r| j| jd | jf ||||d| x| jD ]}|| | q<W | j s| j||||	d}|   | jj	| jf| n | 
  x| jD ]}|| | qW | j||
d d S )Nr\   )r  r   r  r  )r   r   r   r  )r  )rS  r   rO  r   r  r  r  r  r  Zcreate_tabler  r  
write_data)r   r   r  rM   r   r   r   r  r   r  r  r   r   Zoptionsr?   r?   r@   rU    s$    
zAppendableTable.writec                sh  | j j}| j}g }|rXx@| jD ]6}t|jjdd}t|tj	r|
|jddd qW t|r|d }x|dd D ]}||@ }qvW | }nd}dd	 | jD t}	g }
xtD ]\}}|dkr||	k rtfd
d	td|D }t||}||	d k r8tfdd	t|d |	D }t||}|

| qW dd	 | jD }dd	 |D }g }xDt|D ]8\}}|f| j ||	|   j }|
|| | qrW |dkrd}tjt||| j d}t|| d }xt|D ]t}|| t|d | |  krP | j| fdd	|
D |dk	rD|  nd fdd	|D d qW dS )z\ we form the data into a 2-d including indexes,values,mask
            write chunk-by-chunk r   )r   Zu1F)r;  rG   Nc             S   s   g | ]
}|j qS r?   )r  )r   r   r?   r?   r@   r     s    z.AppendableTable.write_data.<locals>.<listcomp>c                s   g | ]} | j d  qS )r   )r  )r   bi)indexesr?   r@   r     s    c                s   g | ]} | j d  qS )r   )r  )r   r  )r  r?   r@   r     s   c             S   s   g | ]}|  qS r?   )r  )r   r   r?   r?   r@   r     s    c          	   S   s,   g | ]$}| tt|j|jd  qS )rG   )r  r<   Zrollr  r  )r   r&  r?   r?   r@   r     s   i )ry  c                s   g | ]}|  qS r?   r?   )r   r   )end_istart_ir?   r@   r     s    c                s   g | ]}|  qS r?   r?   )r   r&  )r  r  r?   r@   r     s    )r  r  r   )ry  r/  r_  rQ  r%   r:  r  r;   r<   ndarrayrM   r  r   r  rP  r  r^  r   Ztilerepeatr  r  rN  r\  r]  write_data_chunk)r   r   r  r/  r   Zmasksr   r  r  nindexesZbindexesr  r  Zrepeaterr   Zbvaluesr&  Z	new_shaperowsZchunksr?   )r  r  r  r@   r    s`    


zAppendableTable.write_datac          
   C   sd  x|D ]}t |jsdS qW y|d jd }|t|krJt j|| jd}| jj}t|}x t|D ]\}	}
|
|||	 < qdW x$t|D ]\}	}||||	|  < qW |dk	r| j	t
dd }| s|| }W n4 tk
r } ztdj|dW dd}~X Y nX y$t|r(| j| | j  W n4 tk
r^ } ztdj|dW dd}~X Y nX dS )	z
        Parameters
        ----------
        rows : an empty memory space where we are putting the chunk
        indexes : an array of the indexes
        mask : an array of the masks
        values : an array of the values
        Nr   )ry  F)r;  z"cannot create row-data -> {detail})r<  z)tables cannot write this data -> {detail})r<   r^  r  r   rN  ry  r/  r  r  r  r   r  r   r   r\   rM   r   r   )r   r  r  r  r   r&  r   r/  r  r  r  r  r<  r?   r?   r@   r    s8    
 
z AppendableTable.write_data_chunkc             K   sr  |d kst |sf|d kr:|d kr:| j}| jj| jdd n(|d krH| j}| jj||d}| j  |S |  srd S | j}t	| |f||d|| _
| j
 }t| }t |}	|	rn| }
t|
|
dk j}t |sdg}|d |	kr||	 |d dkr|dd | }xLt|D ]@}|t||}|j||jd  ||jd  d d |}q W | j  |	S )NT)r  )r   r   rG   r   r  )r   r   r   rO  r   r\   Zremove_rowsr   r   rp  rR  r  r   Zsort_valuesdiffrK   r  rM   r  r  reversedr  r   )r   rO   r   r   r   r   r\   r   Zsorted_seriesZlnr  r   Zpgr   r  r?   r?   r@   r  M  sD    




zAppendableTable.delete)	NFNNNNNNF)F)NNN)
rT   rU   rV   rY  rm  rC  rU  r  r  r  r?   r?   r?   r@   r    s     
(
K1r  c               @   s>   e Zd ZdZdZdZdZeZe	dd Z
dd Zdd
dZd	S )rn   z) suppor the new appendable table formats rD  rw   r}   c             C   s   | j d jdkS )Nr   rG   )rP  r   )r   r?   r?   r@   ra    s    z"AppendableFrameTable.is_transposedc             C   s   | j r|j}|S )z these are written transposed )ra  r  )r   r   r?   r?   r@   rr    s    zAppendableFrameTable.get_objectNc             K   s  | j f d|i|sd S t| jr<| j| jd d t nt }| jd j}g }x| jD ]}|ddkrzt	
|j}n
t|j}|d}	|	d k	r|j|	dd | jr|j}
|}t|t|dd d	}n |jj}
t|t|dd d	}|}|
jd
krt|
tjr|
d
|
jd f}
t|
tt|d}t|g||g}|t| qZW t|d
krh|d }nt|d
d}| j||d}|S )NrO   r   r   r   r/  T)inplacerE   )rE   rG   )rI  )r   )r   )rq  r   r   r?  r   r  rP  r   rQ  r   Zfrom_tuplesr   	set_namesra  r  r/  r  r  r;   r<   r  r  r  r7   r  r3   rM   r   r$   r  )r   rO   r   r   r?  r  Zframesr   r  r/  r   Zindex_Zcols_r  rz  dfr?   r?   r@   r     s:    *


zAppendableFrameTable.read)NN)rT   rU   rV   rY  r  rC  r  r   r  rZ  ra  rr  r   r?   r?   r?   r@   rn     s   c                   sX   e Zd ZdZdZdZdZeZe	Z
edd Zdd Zd fd
d	Zd fdd	Z  ZS )rl   z* support the new appendable table formats rG  ru   r}   c             C   s   dS )NFr?   )r   r?   r?   r@   ra    s    z#AppendableSeriesTable.is_transposedc             C   s   |S )Nr?   )r   r   r?   r?   r@   rr    s    z AppendableSeriesTable.get_objectNc                sP   t |ts.|jpd}t||i|jd}|g|_tt| jf ||j d|S )z- we are going to write this as a frame table r   )r  )r   r  )	r;   r   rE   r  r   r  rl   rU  r|  )r   r   r  r   rE   )r  r?   r@   rU    s    

zAppendableSeriesTable.writec                s   | j }|d k	r6|r6x"| jD ]}||kr|d| qW tt| jf d|i|}|rd|j| jdd |jd d df }|jdkrd |_|S )Nr   r   T)r  r   )	r[  r.  r  r  rl   r   	set_indexr  rE   )r   r   r   r[  r   r>   )r  r?   r@   r     s    
zAppendableSeriesTable.read)N)N)rT   rU   rV   rY  r  rC  r  r   r  r   r  rZ  ra  rr  rU  r   r  r?   r?   )r  r@   rl     s   	c                   s(   e Zd ZdZdZdZ fddZ  ZS )rm   z* support the new appendable table formats rG  rv   c                sN   |j pd}| |\}| _t| j}|| ||_tt| jf d|i|S )z- we are going to write this as a frame table r   r   )	rE   r]  r.  rK   rM   r   r  rm   rU  )r   r   r   rE   r  )r  r?   r@   rU    s    


z AppendableMultiSeriesTable.write)rT   rU   rV   rY  r  rC  rU  r  r?   r?   )r  r@   rm     s   c               @   sT   e Zd ZdZdZdZdZeZe	dd Z
e	dd Zd	d
 Ze	dd Zdd ZdS )rk   z< a table that read/writes the generic pytables table format rD  rt   r}   c             C   s   | j S )N)r  )r   r?   r?   r@   r-    s    zGenericTable.pandas_typec             C   s   t | jdd p| jS )Nr\   )r/  r   )r   r?   r?   r@   r    s    zGenericTable.storablec                sT   g  _ d _g  _ fdd jD  _ fdd jD  _dd  jD  _dS )z retrieve our attributes Nc                s   g | ]}|j r| qS r?   )r  rx  )r   r   )r   r?   r@   r     s   z*GenericTable.get_attrs.<locals>.<listcomp>c                s   g | ]}|j s| qS r?   )r  rx  )r   r   )r   r?   r@   r     s   c             S   s   g | ]
}|j qS r?   )rE   )r   r   r?   r?   r@   r      s    )r   r|  r.  rh  rP  rQ  r  )r   r?   )r   r@   r    s    

zGenericTable.get_attrsc             C   s^   | j dkrX| j}tdddg| _ x6t|jD ](\}}t|||g| jd}| j | q,W | j S )z2 create the indexables from the table description Nr  r   )rE   r   )rE   rg  r   r   )rm  r  r  r  Z_v_namesr  r   rM   )r   r%  r  r   r(  r?   r?   r@   rh  "  s    
zGenericTable.indexablesc             K   s   t dd S )Nz cannot write on an generic table)r   )r   r   r?   r?   r@   rU  4  s    zGenericTable.writeN)rT   rU   rV   rY  r  rC  r  r   r  rZ  r-  r  r  rh  rU  r?   r?   r?   r@   rk     s   c                   sP   e Zd ZdZdZeZdZe	dZ
edd Zd fdd		Z fd
dZ  ZS )ro   z a frame with a multi-index rx   r}   z^level_\d+$c             C   s   dS )NZappendable_multir?   )r   r?   r?   r@   rT  @  s    z*AppendableMultiFrameTable.table_type_shortNc                sp   |d krg }n|dkr |j  }| |\}| _x"| jD ]}||kr8|d| q8W tt| jf ||d|S )NTr   )r   r  )r   r|  r]  r.  r  r  ro   rU  )r   r   r  r   r   )r  r?   r@   rU  D  s    
zAppendableMultiFrameTable.writec                sB   t t jf |}| j}|j fdd|jjD |_|S )Nc                s    g | ]} j |rd n|qS )N)
_re_levelsr  )r   r\  )r   r?   r@   r   W  s    z2AppendableMultiFrameTable.read.<locals>.<listcomp>)r  ro   r   r  r.  r  r  r/  )r   r   r  )r  )r   r@   r   P  s
    zAppendableMultiFrameTable.read)N)rT   rU   rV   rY  rC  r   r  r  r  compiler  rZ  rT  rU  r   r  r?   r?   )r  r@   ro   8  s   
c               @   s0   e Zd ZdZdZdZeZdd Ze	dd Z
dS )	rp   z) suppor the new appendable table formats ry   r  c             C   s   | j r|j| j }|S )z these are written transposed )ra  r  rb  )r   r   r?   r?   r@   rr  d  s    zAppendablePanelTable.get_objectc             C   s   | j tt| jkS )N)rb  rL   r   r  )r   r?   r?   r@   ra  j  s    z"AppendablePanelTable.is_transposedN)rT   rU   rV   rY  rC  r  r   r  rr  rZ  ra  r?   r?   r?   r@   rp   ]  s   c             C   s   |  |}t|}|d k	r"t|}|d ks4||rB||rB| S t| }|d k	rlt| j|dd}||std d g| j }|||< | jt| } | S )NF)sort)	r  r2   Zequalsr)   r"  slicer  r#  rL   )r   r   r  rq  rJ  Zslicerr?   r?   r@   r  o  s    

r  c             C   s4   y| | }W n" t k
r.   t  }| |< Y nX |S )z# get/create the info for this name )r   r  )r?  rE   r  r?   r?   r@   r    s
    r  c             C   s"   t | }|dkr|   }|S )z- for a tz-aware type, return an encoded zone N)r   get_timezoneZ	utcoffsetZtotal_seconds)r`  Zzoner?   r?   r@   r    s    
r  c             C   s|   |dk	rft | dd}|  } tt|}t| |d} | jdkrP| d|} |rx|dkrxt	| } n|rxt
j| dd} | S )aH  
    coerce the values to a DatetimeIndex if tz is set
    preserve the input shape if possible

    Parameters
    ----------
    values : ndarray
    tz : string/pickled tz object
    preserve_UTC : boolean,
        preserve the UTC of the result
    coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray
    NrE   )rE   r	  zM8[ns])ry  )r/  r  r   r  rA   r   r`  r  r  rK   r<   r  )r   r`  Zpreserve_UTCr  rE   r?   r?   r@   r{    s    

r{  r~   c       
   	   C   s  t | dd }t| trF| j}t|dt  t | dd t | dd |dS t| trv| j}t|dt  t | dd |dS t| tt	frt  }t| j
d|t | dd |dS t| trtd	tj| d
d}t| }|dkr|d}t|dt  t | dd t | dd |dS |dkrF|d}t|dt  t | dd |dS |dkrtjdd |D tjd}t|dt  |dS |dkrtjdd |D tjd}t|dt  |dS |dkrt|||}|jj}	t|dt |	|	|dS |dkr:|dkr*t  }ttj|ddd||dS td|n|dkrjt  }ttj|tjdd||dS |dkrt  }ttj|tjdd||dS t  }ttj|ddd||dS d S )NrE   r  r_  r`  )r_  r`  ra  r  )r_  ra  r  zMultiIndex not supported here!F)r  r  r   c             S   s$   g | ]}t | |jd   qS )g    .A)timeZmktimeZ	timetupleZmicrosecond)r   r&  r?   r?   r@   r     s   z"_convert_index.<locals>.<listcomp>)ry  )ra  r   c             S   s   g | ]}|  qS r?   )Z	toordinal)r   r&  r?   r?   r@   r     s    r  )rd  ra  r  r[   r  r   z?[unicode] is not supported as a in index type for [{0}] formatsZfloatingr  )r/  r;   r   r  r^  r   r  r"   r   r   Z_ndarray_valuesr   r   r   r  r<   r  r  r  Z	Time64ColZint32Z	Time32Colr  ry  rd  r  r;  r   Zint64Z
Float64Col)
r  rC   r}  r  ra  r(  r<  r  r   rd  r?   r?   r@   r$    s    
































r$  c             C   s   t |}|dkrt| }n|dkr,t| }n|dkrNtjdd | D td}n|dkrytjdd | D td}W q tk
r   tjd	d | D td}Y qX nV|d
krt| }nB|dkrt| d ||d}n(|dkrt| d }ntdj|d|S )Nr  r  r   c             S   s   g | ]}t |qS r?   )r   r  )r   r&  r?   r?   r@   r     s    z$_unconvert_index.<locals>.<listcomp>)ry  r   c             S   s   g | ]}t |qS r?   )r   r  )r   r&  r?   r?   r@   r     s    c             S   s   g | ]}t |qS r?   )r   r  )r   r&  r?   r?   r@   r     s    )r  r  r  )r|  rC   r}  r   r   zunrecognized index type {kind})rb  )	rA   r   r"   r<   r  r   r   r  r   )r:  rb  rC   r}  r  r?   r?   r@   r5  	  s0    



r5  c             C   s`   t |}|dkrt| }nB|dkr2tj| td}n*|dkrLt| d ||d}ntdj|d|S )Nr   r  )ry  r  )r|  rC   r}  zunrecognized index type {kind})rb  )rA   r&   r<   r  r   r  r   r   )r:  rb  r  rC   r}  r  r?   r?   r@   r>  %  s    

r>  c             C   sn   |dk	r0t | r0t|  j||j| j} |dkrTt|  }t	dt
|}tj| dj|dd} | S )a  
    we take a string-like that is object dtype and coerce to a fixed size
    string type

    Parameters
    ----------
    data : a numpy array of object dtype
    encoding : None or string-encoding
    errors : handler for encoding errors
    itemsize : integer, optional, defaults to the max length of the strings

    Returns
    -------
    data in a fixed-length string dtype, encoded to bytes if needed
    NrG   zS{size})r  )ry  )r   r   r  r   encoder   r  r  r   r  
libwritersmax_len_string_arrayr<   r  r   )r:  rC   r}  rd  Zensuredr?   r?   r@   r  4  s    r  c             C   s   | j }tj|  td} t|}|dk	rt| rtt	| }t
jrPd|}n
d|}t| d t
jrt| jj||dj} n| j|ddjtdd} |dkrd	}t| |} | |S )
aT  
    inverse of _convert_string_array

    Parameters
    ----------
    data : fixed length string dtyped array
    nan_rep : the storage repr of NaN, optional
    encoding : the encoding of the data, optional
    errors : handler for encoding errors, default 'strict'

    Returns
    -------
    an object array of the decoded data

    )ry  NzU{0}zS{0}r   )r}  F)r;  rw  )r  r<   r  r  r   rD   r   r  r  r   r#   r	   r   r;   Zbinary_typer   r   r=   r   r  Z!string_array_replace_from_nan_repr  )r:  r|  rC   r}  r  rd  ry  r?   r?   r@   r  S  s    
r  c             C   s    t |rt|||}|| } | S )N)_need_convert_get_converter)r   Zval_kindrC   r}  Zconvr?   r?   r@   rz  ~  s    rz  c                sR   t | } | dkrdd S | dkr(dd S | dkr> fddS tdj| d	d S )
Nr  c             S   s   t j| ddS )NzM8[ns])ry  )r<   r  )rH   r?   r?   r@   rI     s    z _get_converter.<locals>.<lambda>r   c             S   s   t | dd S )NT)cache)r&   Zto_pydatetime)rH   r?   r?   r@   rI     s    r  c                s   t |  dS )N)rC   r}  )r  )rH   )rC   r}  r?   r@   rI     s   zinvalid kind {kind})rb  )rA   r   r   )rb  rC   r}  r?   )rC   r}  r@   r    s    r  c             C   s   t | } | dkrdS dS )N)r   r  r  TF)rA   )rb  r?   r?   r@   r    s    r  c               @   s2   e Zd ZdZdddZdd Zdd Zd	d
 ZdS )rp  z
    Carries out a selection operation on a tables.Table object.

    Parameters
    ----------
    table : a Table object
    where : list of Terms (or convertible to)
    start, stop: indices to start and/or stop selection

    Nc             C   sP  || _ || _|| _|| _d | _d | _d | _d | _t|ryt	j
|dd}|dksZ|dkrt|}|jtjkr| j| j }}|d krd}|d kr| j j}t||| | _nNt|jjtjr| jd k	r|| jk  s| jd k	r|| jk rtd|| _W n tk
r   Y nX | jd krL| || _| jd k	rL| j \| _| _d S )NF)r  r  Zbooleanr   z3where must have index locations >= start and < stop)r\   rO   r   r   	conditionr
   Ztermsr  r   r   r  r<   r  ry  Zbool_r   r  
issubclassr   r  r  r   generateZevaluate)r   r\   rO   r   r   Zinferredr?   r?   r@   r     sD    




zSelection.__init__c             C   s\   |dkrdS | j  }yt||| j jdS  tk
rV   td|d| Y nX dS )z) where can be a : dict,list,tuple,string N)rd  rC   a  The passed where expression: {0}
            contains an invalid variable reference
            all of the variable references must be a reference to
            an axis (e.g. 'index' or 'columns'), or a data_column
            The currently defined references are: {1}
rm  )	r\   rd  r/   rC   	NameErrorr   r   ro  r   )r   rO   rj  r?   r?   r@   r    s    
zSelection.generatec             C   sX   | j dk	r(| jjj| j  | j| jdS | jdk	rB| jj| jS | jjj| j| jdS )z(
        generate the selection
        N)r   r   )	r  r\   Z
read_wherer   r   r   r  r   r   )r   r?   r?   r@   r     s    


zSelection.selectc             C   s   | j | j }}| jj}|dkr$d}n|dk r4||7 }| jdkrD|}n|dk rT||7 }| jdk	rz| jjj| j ||ddS | jdk	r| jS t	||S )z(
        generate the selection
        Nr   T)r   r   r  )
r   r   r\   r   r  Zget_where_listr   r  r<   r  )r   r   r   r   r?   r?   r@   r    s"    


zSelection.select_coords)NNN)rT   rU   rV   rY  r   r  r   r  r?   r?   r?   r@   rp    s
   
.rp  Tc             K   sB   |d krd}t |dd}|j| |f| |  |r>t| d S )Nz	timeit.h5rQ   )r   )r   rM   r   r   r   )r   r  fnr   r   r   r?   r?   r@   timeit  s    r  )NNNN)Nr   )N)FF)Nr~   N)Nr~   )FNr~   )N)NNr~   )NT)rY  r;  r   r   Zdistutils.versionr   r  r   r  r  r  Znumpyr<   Zpandas._libsr   r   r   r  Zpandas._libs.tslibsr   Zpandas.compatr	   r
   r   r   r   Zpandas.errorsr   Zpandas.core.dtypes.commonr   r   r   r   r   r   r   r   Zpandas.core.dtypes.missingr   Zpandasr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   Zpandas.corer'   Zpandas.core.algorithmsr(   r)   Zpandas.core.arrays.categoricalr*   r+   Zpandas.core.arrays.sparser,   r-   Zpandas.core.baser.   Zpandas.core.commonZcorecommonr  Z pandas.core.computation.pytablesr/   r0   Zpandas.core.configr1   Zpandas.core.indexr2   Zpandas.core.internalsr3   r4   r5   r6   r7   Zpandas.io.commonr8   Zpandas.io.formats.printingr9   r:   r  rB   rA   rD   rF   rN   rR   r   rS   rW   WarningrX   ri  rY   r  rZ   r  r:  r@  Zformat_deprecate_docrE  rF  rI  r  Z
dropna_docZ
format_docZconfig_prefixZregister_optionZis_boolZis_one_of_factoryr   r   r   r   r   r   r   r   r   r^  r  r  r  r  r  r  r=  rd   re   rf   r@  rg   ri   rF  rh   rj   r1  rq   r  rr   rs   r  rn   rl   rm   rk   ro   rp   r  r  r  r{  r$  r5  r>  r  r  rz  r  r  rp  r  r?   r?   r?   r@   <module>   s:  (D


 

y       {a  $   - *  h
	
!V
      4k jC+1%


U
 

 
*x