B
    ÛT•\E*  ã               @   st   d dl mZmZmZ d dlZd dlZddlmZ dZG dd„ de	ƒZ
e
ejd< e
ejd	< G d
d„ de	ƒZdd„ ZdS )é    )Úprint_functionÚdivisionÚabsolute_importNé   )Úcorei  P c               @   sF   e Zd ZdZdZdd„ Zdd„ Zdd„ Zddd„Zdd„ Z	dd„ Z
d
S )ÚHTTPFileSystemz¤
    Simple File-System for fetching data via HTTP(S)

    Unlike other file-systems, HTTP is limited in that it does not provide glob
    or write capability.
    ú/c             K   s"   |  dt¡| _|| _t ¡ | _dS )am  
        Parameters
        ----------
        block_size: int
            Blocks to read bytes; if 0, will default to raw requests file-like
            objects instead of HTTPFile instances
        storage_options: key-value
            May be credentials, e.g., `{'auth': ('username', 'pword')}` or any
            other parameters passed on to requests
        Ú
block_sizeN)ÚpopÚDEFAULT_BLOCK_SIZEr	   ÚkwargsÚrequestsÚSessionÚsession)ÚselfZstorage_options© r   ú.lib/python3.7/site-packages/dask/bytes/http.pyÚ__init__   s    zHTTPFileSystem.__init__c             C   s   t ‚dS )z*For a template path, return matching filesN)ÚNotImplementedError)r   Úurlr   r   r   Úglob#   s    zHTTPFileSystem.globc             C   s   t ‚dS )z7Make any intermediate directories to make path writableN)r   )r   r   r   r   r   Úmkdirs'   s    zHTTPFileSystem.mkdirsÚrbNc             K   sr   |dkrt ‚|dk	r|n| j}|r6t|| j|f| jŽS | j ¡ }d|d< | jj|f|Ž}| ¡  d|j_	|jS dS )zþMake a file-like object

        Parameters
        ----------
        url: str
            Full URL with protocol
        mode: string
            must be "rb"
        kwargs: key-value
            Any other parameters, passed to requests calls
        r   NTÚstream)
r   r	   ÚHTTPFiler   r   ÚcopyÚgetÚraise_for_statusÚrawZdecode_content)r   r   Úmoder	   r   ÚkwÚrr   r   r   Úopen+   s    
zHTTPFileSystem.openc             C   s
   t  ¡ jS )z=Unique identifier, implied file might have changed every time)ÚuuidZuuid1Úhex)r   r   r   r   r   ÚukeyD   s    zHTTPFileSystem.ukeyc             C   s   t |fd| ji| j—ŽS )z!Size in bytes of the file at pathr   )Ú	file_sizer   r   )r   r   r   r   r   ÚsizeH   s    zHTTPFileSystem.size)r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úsepr   r   r   r"   r%   r'   r   r   r   r   r      s   
r   ZhttpZhttpsc               @   s–   e Zd ZdZd%dd„Zd&dd„Zdd	„ Zd'dd„Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!d"„ Zd#d$„ ZdS )(r   aø  
    A file-like object pointing to a remove HTTP(S) resource

    Supports only reading, with read-ahead of a predermined block-size.

    In the case that the server does not supply the filesize, only reading of
    the complete file in one go is supported.

    Parameters
    ----------
    url: str
        Full URL of the remote resource, including the protocol
    session: requests.Session or None
        All calls will be made within this session, to avoid restarting
        connections where the server allows this
    block_size: int or None
        The amount of read-ahead to do, in bytes. Default is 5MB, or the value
        configured for the FileSystem creating this file.
    kwargs: all other key-values are passed to reqeuests calls.
    Nc             K   s’   || _ || _d| _|d k	r|nt ¡ | _|d k	r4|nt| _y t|| jfddi| j—Ž| _	W n t
k
rt   d | _	Y nX d | _d| _d | _d | _d S )Nr   Úallow_redirectsTF)r   r   Úlocr   r   r   r   Ú	blocksizer&   r'   Ú
ValueErrorÚcacheÚclosedÚstartÚend)r   r   r   r	   r   r   r   r   r   g   s    zHTTPFile.__init__r   c             C   sl   | j dkrtdƒ‚|dkr |}n2|dkr2||7 }n |dkrF| j | }ntd| ƒ‚|dk rbtdƒ‚|| _|S )at  Set file position

        Parameters
        ----------
        where: int
            Location to set
        whence: int (default 0)
            If zero, set from start of file (value should be positive); if 1,
            set relative to current position; if 2, set relative to end of file
            (value shoulf be negative)

        Returns the position.
        Nz+Cannot seek since size of file is not knownr   r   é   z'Whence must be in [1, 2, 3], but got %szSeek before start of file)r'   r0   r.   )r   ÚwhereÚwhenceZnlocr   r   r   Úseeky   s    

zHTTPFile.seekc             C   s   | j S )zGet current file byte position)r.   )r   r   r   r   Útell–   s    zHTTPFile.telléÿÿÿÿc             C   s¸   |dkrdS | j dkr0|dkr(tdƒ‚n|  ¡ S |dk rJ| jdkrJ|  ¡ S |dk sb| j| | j krj| j }n
| j| }| j| j kr„dS |  | j|¡ | j| j| j || j … }|| _|S )a5  Read bytes from file

        Parameters
        ----------
        length: int
            Read up to this many bytes. If negative, read all content to end of
            file. If the server has not supplied the filesize, attempting to
            read only part of the data will raise a ValueError.
        r   ó    Nz(File size is unknown, must read all data)r'   r0   Ú
_fetch_allr.   Ú_fetchr1   r3   )r   Úlengthr4   Údatar   r   r   Úreadš   s"    



zHTTPFile.readc             C   s  | j dkr8| jdkr8|| _ || j | _|  || j¡| _nà|| j k rš| j| | jkrx|| _ || j | _|  | j | j¡| _n |  || j ¡}|| _ || j | _n~|| jkr| j| jkr¶dS || j | jkrì|| _ || j | _|  | j | j¡| _n,|  | j|| j ¡}|| j | _| j| | _dS )z9Set new bounds for data cache and fetch data, if requiredN)r3   r4   r/   Ú_fetch_ranger1   r'   )r   r3   r4   Únewr   r   r   r=   ¾   s,    
zHTTPFile._fetchc             C   sZ   | j j| jf| jŽ}| ¡  |j}t|ƒ}|| jk rLd| _|| _	|| _
|| _t|ƒ| _|S )zµRead whole file in one shot, without caching

        This is only called when size is None or position is still at zero,
        and read() is called without a byte-count.
        r   )r   r   r   r   r   ÚcontentÚlenr/   r3   r4   r1   r'   r.   )r   r!   ÚoutÚlr   r   r   r<   Ú   s    

zHTTPFile._fetch_allc       	      C   s  | j  ¡ }| di ¡}d||d f |d< | jj| jf|ddœ|—Ž}| ¡  |jdkr^|jS d|j	krœt
|j	d ƒ}||| krˆ|jS td	||| f ƒ‚d
}g }xR|jddD ]B}|rò| |¡ |t|ƒ7 }||| krôtd||| f ƒ‚q²P q²W d |¡S )a3  Download a block of data

        The expectation is that the server returns only the requested bytes,
        with HTTP code 206. If this is not the case, we first check the headers,
        and then stream the output - if the data size is bigger than we
        requested, an exception is raised.
        Úheaderszbytes=%i-%ir   ZRangeT)rG   r   éÎ   zContent-Lengthz'Got more bytes (%i) than requested (%i)r   i   )Z
chunk_sizez/Got more bytes so far (>%i) than requested (%i)r;   )r   r   r
   r   r   r   r   Zstatus_coderC   rG   Úintr0   Ziter_contentÚappendrD   Újoin)	r   r3   r4   r   rG   r!   ZclrE   Úchunkr   r   r   rA   î   s2    



zHTTPFile._fetch_rangec             C   s
   d| _ | S )Nr   )r.   )r   r   r   r   Ú	__enter__  s    zHTTPFile.__enter__c             G   s   |   ¡  d S )N)Úclose)r   Úargsr   r   r   Ú__exit__  s    zHTTPFile.__exit__c             C   s   t ‚d S )N)r   )r   r   r   r   Ú__iter__  s    zHTTPFile.__iter__c             C   s   t ‚d S )N)r   )r   r   r   r   Úwrite   s    zHTTPFile.writec             C   s   d S )Nr   )r   r   r   r   Úflush#  s    zHTTPFile.flushc             C   s
   d| _ d S )NT)r2   )r   r   r   r   rN   &  s    zHTTPFile.closec             C   s   dS )NTr   )r   r   r   r   Úseekable)  s    zHTTPFile.seekablec             C   s   dS )NFr   )r   r   r   r   Úwritable,  s    zHTTPFile.writablec             C   s   dS )NTr   )r   r   r   r   Úreadable/  s    zHTTPFile.readable)NN)r   )r:   )r(   r)   r*   r+   r   r8   r9   r@   r=   r<   rA   rM   rP   rQ   rR   rS   rN   rT   rU   rV   r   r   r   r   r   Q   s"   


$'r   c             K   sv   |  ¡ }| dd¡}| di ¡}d|kr0d|d< |j| fd|i|—Ž}| ¡  d|jkrft|jd ƒS td|  ƒ‚dS )	zºCall HEAD on the server to get file size

    Default operation is to explicitly allow redirects and use encoding
    'identity' (no compression) to get the true size of the target.
    r-   TrG   zAccept-EncodingZidentityzContent-Lengthz Server did not supply size of %sN)r   r
   r   Úheadr   rG   rI   r0   )r   r   r   ZarrW   r!   r   r   r   r&   3  s    
r&   )Z
__future__r   r   r   r   r#   Ú r   r   Úobjectr   Z_filesystemsr   r&   r   r   r   r   Ú<module>   s   B

 c