B \s@sXdZddlmZddlZddlZddlmZddlm Z ddl Z ddl m Z ddl Z ddlmZmZmZmZmZmZmZd d lmZmZmZmZmZd d lmZd d lmZd d lmZ d dl!m"Z"d dl#m$Z$d dl%m&Z&m'Z'ddl(m)Z)e*e+dZ,e-j.Z/ddZ0dddZ1d ddZ2Gddde3Z4e/rDGddde3Z5GdddeZ6dS)!z7 Implements support for high-level dataset access. )absolute_importN)warn)local)xrange)h5h5sh5th5rh5dh5ph5fd)HLObjectphil with_philEmptyis_empty_dataspace)filters) selections) selections2)Datatype)filename_decode)VDSmap vds_support)H5pyDeprecationWarning cs^t|dkrSjdkr"tdx"|D]}|jkr(td|q(Wtfdd|DS)z, Make a NumPy dtype appropriate for reading rNz+Field names only allowed for compound typesz&Field %s does not appear in this type.csg|]}|j|dfqS)r)fields).0name)basetype/lib/python3.7/site-packages/h5py/_hl/dataset.py 6sz"readtime_dtype..)lennames ValueErrornumpydtype)r r%rr!)r r"readtime_dtype)s    r)c Cs4|dk rpt|tspddlm}|dk r2t|}nd}|dk oP|jdkoP|jdk}tj|d|rb|n| |d}|dkr|dkr|dkrt dt|}|j }n:t |}|dk rtj |tjd tj |j tjd krtd |dk r|n|}t|t r$td d t||Dr$d ||}t|t|tr>|j}|j}nT|dkr^|dkr^td}n&|dkrz|dk rz|j}n t|}tj|dd}t||||| fr|dkrtd|dkr| dkrd} d}|tkr| dk rt d|} d}t|p ttj||||| |||| | }| dk rBt| } || | dkrX|| n| dk rjt d|dkr| tj!tj"Bn(|dkr| dn|dk rt d|dk rt dd |D}t|trt#t#j$}n t#%||}t&j|jd|||d}|dk r0t|ts0|'t#j(t#j(||S)zU Return a new low-level dataset identifier Only creates anonymous datasets. Nr)basefrC)orderr(z-One of data, shape or dtype must be specified)r(z%Shape tuple is incompatible with datacss"|]\}}|dk r||kVqdS)Nr!)rZdimchunkr!r!r" dsz make_new_dset..z^Chunk shape must not be greater than data shape in any dimension. {} is not compatible with {}z=f4)logicalFz1Chunked format required for given storage optionsTgzipzConflict in compression options)TFz(track_times must be either True or Falserz(track_order must be either True or Falsecss |]}|dk r|ntjVqdS)N)r UNLIMITED)rmr!r!r"r/s)dcpl)) isinstancerr*r'r(kinditemsizeasarrayZ guess_dtype TypeErrorshapetupleproduct ulonglongr&anyzipformatridr py_create_LEGACY_GZIP_COMPRESSION_VALSrZ fill_dcplr createDATASET_CREATEarrayset_fill_valueZset_obj_track_timesZset_attr_creation_orderZCRT_ORDER_TRACKEDZCRT_ORDER_INDEXEDrZNULL create_simpler writeZALL)parentr<r(datachunks compressionshuffle fletcher32maxshapecompression_opts fillvalue scaleoffsetZ track_timesexternalZ track_orderr5r*_dtypeZis_small_floatZ tmp_shapeerrmsgtidsidZdset_idr!r!r" make_new_dset9s     *                      r[c Csttj}|dk r&|t|g|dk r@tdd|D}t||}x$|D]\}} } } | || | | qRW|}t |t r|j } n,|dkrt d}n t |}tj|dd} tj|j d| ||dS)zReturn a new low-level dataset identifier for a virtual dataset Like make_new_dset(), this creates an anonymous dataset, which can be given a name later. Ncss |]}|dk r|ntjVqdS)N)rr3)rr4r!r!r"r/sz(make_new_virtual_dset..z=f4r)r0)rrYspacer5)r rFrGrIr'rHr=rrJZ set_virtualr6rrCr(r rDr ) rLr<Zsourcesr(rRrTr5Z virt_dspaceZvspaceZfpathdsetZ src_dspacerYr!r!r"make_new_virtual_dsets"     r^c@s(eZdZdZddZddZddZdS) AstypeContextzQ Context manager which allows changing the type read from a dataset. cCs||_t||_dS)N)_dsetr'r(rW)selfr]r(r!r!r"__init__szAstypeContext.__init__cCs|j|jj_dS)N)rWr`_localastype)rar!r!r" __enter__szAstypeContext.__enter__cGsd|jj_dS)N)r`rcrd)raargsr!r!r"__exit__szAstypeContext.__exit__N)__name__ __module__ __qualname____doc__rbrergr!r!r!r"r_sr_c@s(eZdZdZddZddZddZdS) CollectiveContextz$ Manages collective I/O in MPI mode cCs ||_dS)N)r`)rar]r!r!r"rbszCollectiveContext.__init__cCs|jjtjdS)N)r`_dxpl set_dxpl_mpior ZMPIO_COLLECTIVE)rar!r!r"reszCollectiveContext.__enter__cGs|jjtjdS)N)r`rmrnr ZMPIO_INDEPENDENT)rarfr!r!r"rgszCollectiveContext.__exit__N)rhrirjrkrbrergr!r!r!r"rlsrlc@seZdZdZddZer(eeddZeddZ eedd Z eed d Z e j ed d Z eed dZ eeddZeeddZeeddZeeddZeeddZeeddZeeddZeeddZedd Zeed!d"Zeed#d$Zed%d&ZdFd(d)Zed*d+Zd,d-Zed.d/Zed0d1Zed2d3ZdGd4d5Z dHd6d7Z!edId8d9Z"ed:d;Z#e$e%j&d<red=d>Z'e$e%j&d?red@dAZ(e)redBdCZ*dDdEZ+d'S)JDatasetz( Represents an HDF5 dataset cCs t||S)z Get a context manager allowing you to perform reads to a different destination type, e.g.: >>> with dataset.astype('f8'): ... double_precision = dataset[0:100:2] )r_)rar(r!r!r"rdszDataset.astypecCst|S)z3 Context manager for MPI collective reads & writes )rl)rar!r!r" collectiveszDataset.collectivec Cs$ddlm}t ||SQRXdS)z3 Access dimension scales attached to this dataset. r)DimensionManagerN)dimsrqr)rarqr!r!r"rrs z Dataset.dimscCs|jjS)z5Numpy-style attribute giving the number of dimensions)rCrank)rar!r!r"ndimsz Dataset.ndimcCs|jjS)z1Numpy-style shape tuple giving dataset dimensions)rCr<)rar!r!r"r<sz Dataset.shapecCs||dS)N)resize)rar<r!r!r"r< scCs t|jrdStj|jtjdS)z3Numpy-style attribute giving the total dataset sizeN)r()rrCr'Zprodr<Zintp)rar!r!r"size&s z Dataset.sizecCs|jjS)z%Numpy dtype representing the datatype)rCr()rar!r!r"r(.sz Dataset.dtypecCstdt|dS)z Alias for dataset[()] z;dataset.value has been deprecated. Use dataset[()] instead.r!)rr)rar!r!r"value4sz Dataset.valuecCs |j}|tjkr|SdS)zDataset chunks (or None)N)_dcpl get_layoutr ZCHUNKEDZ get_chunk)rar5r!r!r"rN<szDataset.chunkscCs xdD]}||jkr|SqWdS)zCompression strategy (or None))r2ZlzfZszipN)_filters)raxr!r!r"rOEs  zDataset.compressioncCs|j|jdS)z< Compression setting. Int(0-9) for gzip, 2-tuple for szip. N)rzgetrO)rar!r!r"rSNszDataset.compression_optscCs d|jkS)zShuffle filter present (T/F)rP)rz)rar!r!r"rPTszDataset.shufflecCs d|jkS)z"Fletcher32 filter is present (T/F)rQ)rz)rar!r!r"rQZszDataset.fletcher32cCs(y|jddStk r"dSXdS)aScale/offset filter settings. For integer data types, this is the number of bits stored, or 0 for auto-detected. For floating point data types, this is the number of decimal places retained. If the scale/offset filter is not in use, this is None.rUrN)rzKeyError)rar!r!r"rU`szDataset.scaleoffsetcCsX|j}|dkrdSt}x6t|D]*}|j|\}}}|t|||fq&W|S)zExternal file settings. Returns a list of tuples of (name, offset, size) for each external file entry, or returns None if no external files are used.rN)rxZget_external_countlistrZ get_externalappendr)racountZext_listr{roffsetrvr!r!r"rVls zDataset.externalcCs&|j}|d}tdd|DS)zcShape up to which this dataset can be resized. Axes with value None have no resize limit. Tcss |]}|tjkr|ndVqdS)N)rr3)rr{r!r!r"r/sz#Dataset.maxshape..)rC get_spaceZget_simple_extent_dimsr=)rar\rrr!r!r"rRzs  zDataset.maxshapecCs$tjd|jd}|j||dS)z*Fill value for this dataset (0 by default))r)r(r)r'ndarrayr(rxZget_fill_value)raarrr!r!r"rTs zDataset.fillvaluecCs`t|tjstd|t|||j|_t t j |_ t |j|_t|_d|j_dS)zJ Create a new Dataset object by binding to a low-level DatasetID. z%s is not a DatasetIDN)r6r DatasetIDr&rrbrCZget_create_plistrxr rFZ DATASET_XFERrmrZ get_filtersrzrrcrd)raZbindr!r!r"rbs    zDataset.__init__Nc Cst|jdkrtd|dk r|dkr4||jjksHtd|jjdy t|}Wntk rptdYnXt|j}|||<t |}|j |WdQRXdS)aE Resize the dataset, or the specified axis. The dataset must be stored in chunked format; it can be resized up to the "maximum shape" (keyword maxshape) specified at creation time. The rank of the dataset cannot be changed. "Size" should be a shape tuple, or if an axis is specified, an integer. BEWARE: This functions differently than the NumPy resize() method! The data is not "reshuffled" to fit in the new shape; each axis is grown or shrunk independently. The coordinates of existing data are fixed. Nz$Only chunked datasets can be resizedrzInvalid axis (0 to %s allowed)rz2Argument must be a single int if axis is specified) rrNr;rCrsr&intr~r<r=Z set_extent)rarvZaxisZnewlenr!r!r"rus   zDataset.resizecCs|}|tjkrtd|S)z The size of the first axis. TypeError if scalar. Limited to 2**32 on 32-bit systems; Dataset.len() is preferred. z>Value too big for Python's __len__; use Dataset.len() instead.)r$sysmaxsize OverflowError)rarvr!r!r"__len__s zDataset.__len__c Cs2t$|j}t|dkr td|dSQRXdS)z The size of the first axis. TypeError if scalar. Use of this method is preferred to len(dset), as Python's built-in len() cannot handle values greater then 2**32 on 32-bit systems. rz'Attempt to take len() of scalar datasetN)rr<r$r;)rar<r!r!r"r$s  z Dataset.lenccs>|j}t|dkrtdxt|dD]}||Vq(WdS)z Iterate over the first axis. TypeError if scalar. BEWARE: Modifications to the yielded data are *NOT* written to file. rz#Can't iterate over a scalar datasetN)r<r$r;r)rar<ir!r!r"__iter__s  zDataset.__iter__cCsFt|tr|n|f}t|jrD|tks:|tfks:tdt|jStdd|D}tdd|D}tj rtdd|D}t |j dd}|dk rt ||}nt |jj|}t |}t|dkrt|d tjrt|d |j}||jkrtd t|d |j}t|}|dkr2tjd |d Stj|tjd d krVtj||d Stj||d }t|} | |j| ||||Stj|jtjd d kr|tfks|tkrtj|j|d S|jd krV|j } t!"| |} tj#| j$|d } x$| D]\} } |j| | | |qWt|dkr>| |d } | j$dkrR| d S| Stj%|j||jd} | j&d krtj#| j$|d S| j$d k}|rdn| j$}tj#||dd} t|t|jkrdt|jt||}t|} | j} |jj| | | ||j'dt|dkr | |d } | jd kr4| (} |rB| d } | S)a Read a slice from the HDF5 dataset. Takes slices and recarray-style field names (more than one is allowed!) in any order. Obeys basic NumPy rules, including broadcasting. Also supports: * Boolean "mask" array indexing zEmpty datasets cannot be slicedcss|]}t|tjr|VqdS)N)r6six string_types)rr{r!r!r"r/sz&Dataset.__getitem__..css|]}t|tjs|VqdS)N)r6rr)rr{r!r!r"r/scss(|] }t|tjr|dn|VqdS)zutf-8N)r6r text_typeencode)rr{r!r!r"r/srdNrrz+Region reference must point to this dataset)r)r(r!)dsid)rr,)r-)dxpl))r6r=rrCEllipsisr&rr(rPY2getattrrcr)r rDr$r ZRegionReferenceZ dereferenceZ get_regionselZ guess_shaper'rHr>r?emptyrrJZ select_allreadr<rsel2Z select_readrmshapeselectnselectrmitem)rarfr%Z new_dtypemtypeobjrZroutZsid_outfspace selectionrmspaceZsingle_elementr!r!r" __getitem__sz                    zDataset.__getitem__c s(t|tr|n|f}tdd|Dtdd|D}tjrPtddDtjjddk rjttjfkrjyt j |d}WnLt k ry"t j fdd |Djd}Wnt k rYnXYnX|jkrx|j d krJt j|jdd td }d d |t j|jdd t jd|jd fD|dd<nt j dgtd}||d<|}njjdksjjdkrjt|t jr|jjdkrjjjdkrjtd krjjdk rdjjkrt ddjjdd}d}n j}d}t j ||jdd}|rx|t d|fg}||jdt|jt|j}nt j |dd}jjdk rjjd }|jt| d}||krtd||ftt |j|f}|jdt|jt|} n.css|]}t|tjs|VqdS)N)r6rr)rr{r!r!r"r/Tscss(|] }t|tjr|dn|VqdS)zutf-8N)r6rrr)rr{r!r!r"r/Vs)vlenN)r(csg|]}tj|dqS))r()r'rH)rr{)rr!r"r#`sz'Dataset.__setitem__..r)r<r(cSsg|]}|qSr!r!)rrr!r!r"r#gsrOVzNo such field for indexing: %sTFr,)r(r-)r-zWWhen writing to array types, last N dimensions have to match (got %s, but should be %s)z1Illegal slicing argument (not a compound dataset)csg|]}|jjkr|qSr!)r(r)rr{)rar!r"r#sz, css|]}d|VqdS)z"%s"Nr!)rr{r!r!r"r/sz8Illegal slicing argument (fields %s not in dataset type)csg|]}|kr|qSr!r!)rr{)r%r!r"r#s)rr!z5Scalar broadcasting is not supported for array dtypes.)r)r)1r6r=rrr Z check_dtyper(bytesrr'r:r&rHrtrr<objectZreshaper>r?Zravelr8rZsubdtyper$rr*Zviewr;rDjoinrFZCOMPOUNDZget_sizeinsertZ_er%r9rrrCrrrrJr3 broadcastrKrm)rarfvalZtmpr(Z cast_compoundZshpZvalshprrZmismatchZsubtypeZ fieldnamesZ fieldnamerrZval2Z mshape_padrrr!)r%rarr" __setitem__Hs    8&       zDataset.__setitem__c Cstt|jrtd|dkr.t|j}nt|j||j}|j}|dkr\t|j}nt|j||j}x*||j D]}|jj ||||j dq|WWdQRXdS)z Read data directly from HDF5 into an existing NumPy array. The destination array must be C-contiguous and writable. Selections must be the output of numpy.s_[]. Broadcasting is supported for simple indexing. z+Empty datasets have no numpy representationN)r) rrrCr;rSimpleSelectionr<rrrrrm)radest source_seldest_selrrr!r!r" read_directs zDataset.read_directc Cstt|jrtd|dkr.t|j}nt|j||j}|j}|dkr\t|j}nt|j||j}x*||j D]}|jj ||||j dq|WWdQRXdS)z Write data directly to HDF5 from a NumPy array. The source array must be C-contiguous. Selections must be the output of numpy.s_[]. Broadcasting is supported for simple indexing. z#Empty datasets cannot be written toN)r) rrrCr;rrr<rrrrKrm)rasourcerrrrr!r!r" write_directs zDataset.write_directcCsFtj|j|dkr|jn|d}tj|jtjddkr8|S|||S)z Create a Numpy array containing the whole dataset. DON'T THINK THIS MEANS DATASETS ARE INTERCHANGEABLE WITH ARRAYS. For one thing, you have to read the whole dataset every time this method is called. N)r(r)r'rr<r(r>r?r)rar(rr!r!r" __array__s  zDataset.__array__cCsh|s d}nJ|jdkrd}n&tt|j}d|dkr:|nd}d||j|jjf}tjrd| dS|S)Nzz ("anonymous")z"%s"r7/z&utf8) rppbasenamenormpathr<r(strrrr)rarZnamestrrr!r!r"__repr__s  zDataset.__repr__refreshcCs|jdS)z Refresh the dataset metadata by reloading from the file. This is part of the SWMR features and only exist when the HDF5 library version >=1.9.178 N)_idr)rar!r!r"rszDataset.refreshflushcCs|jdS)a Flush the dataset data and metadata to the file. If the dataset is chunked, raw data chunks are written to the file. This is part of the SWMR features and only exist when the HDF5 library version >=1.9.178 N)rr)rar!r!r"r#sz Dataset.flushcCs|jtjkS)N)rxryr ZVIRTUAL)rar!r!r" is_virtual.szDataset.is_virtualcs.|jstd|jfddtDS)NzNot a virtual datasetc s2g|]*}t||||qSr!)rZget_virtual_vspaceZget_virtual_filenameZget_virtual_dsetnameZget_virtual_srcspace)rj)r5r!r"r#7sz+Dataset.virtual_sources..)r RuntimeErrorrxrangeZget_virtual_count)rar!)r5r"virtual_sources2s  zDataset.virtual_sources)N)NN)NN)N),rhrirjrkrdMPIpropertyrrprrrtr<setterrvr(rwrNrOrSrPrQrUrVrRrTrbrurr$rrrrrrrhasattrr rrrrrrr!r!r!r"rosl      l ~       ro)NNNNNNNNNNNNNNN)NNN)7rkZ __future__r posixpathrrwarningsrZ threadingrrZ six.movesrr'r7rrr r r r r r*rrrrrrrrrrZdatatypercompatrZvdsrrZ h5py_warningsr frozensetrrEZ get_configZmpirr)r[r^rr_rlror!r!r!r" s>    $         r "