๓
าbc           @@  sถ   d  d l  m Z d  d l Z d  d l m Z m Z m Z m Z d d l m	 Z	 d d l
 m Z d d l m Z m Z m Z m Z d d l m Z m Z d Z d
 e f d     YZ d S(   i    (   t   absolute_importN(   t   debugt   infot   warningt   errori   (   t   ParameterError(   t   S3UriS3(   t   getTextFromXmlt   getTreeFromXmlt   s3_quotet
   parseNodes(   t
   formatSizet   calculateChecksumi   t   MultiPartUploadc           B@  s   e  Z d  Z d Z d Z d Z d d d  Z d   Z d   Z	 d   Z
 d d  Z d d d	  Z d d
  Z d   Z d   Z RS(   s<   Supports MultiPartUpload and MultiPartUpload(Copy) operationi   i   c         C@  sภ   | |  _  d  |  _ d  |  _ | |  _ | |  _ i  |  _ | p? i  |  _ t | t	  r | |  _ | sr t
 d   n  |  j  j j t } n | |  _ |  j  j j t } | |  _ |  j   |  _ d  S(   Ns8   Source size is missing for MultipartUploadCopy operation(   t   s3t   Nonet   file_streamt   src_urit   src_sizet   dst_urit   partst   headers_baselinet
   isinstanceR   R   t   configt   multipart_copy_chunk_size_mbt   SIZE_1MBt   multipart_chunk_size_mbt
   chunk_sizet   initiate_multipart_uploadt	   upload_id(   t   selfR   t   srcR   R   R   t   c_size(    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   __init__   s     									c         C@  su   |  j  j | |  } t   } xP | D]H } y. i | d d 6| d d 6| t | d  <Wq% t k
 rl q% Xq% W| S(   Nt   ETagt   checksumt   Sizet   sizet
   PartNumber(   R   t   list_multipartt   dictt   intt   KeyError(   R   t   uriR   t	   part_listR   t   elem(    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   get_parts_information2   s    	 c         C@  sท   d } |  j  j |  } x | D] } yv | d } | d } t d | | j   f  | | j   k r | r t d | t j d | f   n  | } n  Wq t k
 rฎ q Xq W| S(   Nt    t   UploadIdt   Keys   mp_path: %s, object: %ss   More than one UploadId for URI %s.  Disable multipart upload, or use
 %s multipart %s
to list the Ids, then pass a unique --upload-id into the put command.i    (   R   t   get_multipartR   t   objectt
   ValueErrort   syst   argvR*   (   R   R+   R   t   multipart_listt   mpuploadt   mp_upload_idt   mp_path(    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   get_unique_upload_idA   s     

c      
   C@  sฦ   |  j  j j r$ |  j  j j |  _ n0 |  j  j j rK |  j |  j  |  _ n	 d |  _ |  j sฟ |  j  j d d |  j d |  j d i d	 d 6} |  j  j	 |  } | d } t
 | d  |  _ n  |  j S(
   s   
        Begin a multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html
        R/   t   OBJECT_POSTR+   t   headerst
   uri_paramst   uploadst   dataR0   N(   R   R   R   t   put_continueR;   R   t   create_requestR   R   t   send_requestR   (   R   t   requestt   responseR@   (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyR   V   s    			
R/   c      	   C@  s%  |  j  s t d   n  i  } |  j r9 |  j j   } n |  j j } |  j j j ro |  j	 |  j
 |  j   } n  | r d | } n  i | d 6|  j
 j   d 6} d } |  j r |  j } |  j |  j |  j |  j oื d } t d | | f  x| d k r
|  j | d } t |  j | |  j  }	 | |	 8} d | | d	 t |	 d
 t | f | d <yZ |  j r|  j | | |	 | d | j |  n% |  j | | |	 | d | j |  WnD t d | | t j d |  j
 |  j  t j d |  j  f    n X| d 7} q๒ Wt d | d  d St d |  x฿ t r|  j j |  j  }
 d } t |
  }	 d | d	 t |	 d
 t | f | d <|
 sPn  y, |  j | | |	 | |
 d | j |  WnD t d | | t j d |  j
 |  j  t j d |  j  f    n X| d 7} q1Wt d | d  d S(   s   
        Execute a full multipart upload on a file
        Returns the seq/etag dict
        TODO use num_processes to thread it
        sA   Attempting to use a multipart upload that has not been initiated.u    t   sourcet   destinationi   s#   MultiPart: Uploading %s in %d partsi    s   [part %d of %d, %s]%ss   %d%sBt   human_readablet   extrat   remote_statusu   
Upload of '%s' part %d failed. Use
  %s abortmp %s %s
to abort the upload, or
  %s --upload-id %s put ...
to continue the upload.s$   MultiPart: Upload finished: %d partsNs   MultiPart: Uploading from %ss   [part %d of -, %s]%suw   
Upload of '%s' part %d failed. Use
  %s abortmp %s %s
to abort, or
  %s --upload-id %s put ...
to continue the upload.(   R   R   R   R+   R   t   stream_nameR   R   RA   R.   R   R   R   R   t   minR   t   Truet   upload_partt   gett	   copy_partR   R5   R6   t   readt   len(   R   t   extra_labelt   remote_statusest   filenamet   labelst   seqt	   size_leftt   nr_partst   offsett   current_chunk_sizet   buffer(    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   upload_all_partsl   s    				
		c      	   C@  s  t  d | |  j | f  | d k	 r๕ t | d  | k rห t | |  j | | |  j j j  } | d j	 d  } | | k rซ t
 d |  j | f  | d |  j | <d St
 d | | |  j | f  q๕ t
 d t | d  | |  j | f  n  i t |  d 6}	 i d	 | d
 6|  j d 6}
 |  j j d d |  j d |	 d |
 } |  j j | |  j | | d | d | } | d j d d  j	 d  |  j | <| S(   s   
        Upload a file chunk
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
        s"   Uploading part %i of %r (%s bytes)R%   R#   s   "'s:   MultiPart: size and md5sum match for %s part %d, skipping.sJ   MultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading.sF   MultiPart: size (%d vs %d) does not match for %s part %d, reuploading.s   content-lengths   %st
   partNumbert   uploadIdt
   OBJECT_PUTR+   R=   R>   RZ   R   t   etagR/   N(   R   R   R   R)   R   R   R   R   t
   send_chunkt   stripR   R   R   t   strRB   t	   send_fileRO   (   R   RW   RZ   R   RV   R\   RJ   R#   t   remote_checksumR=   t   query_string_paramsRD   RE   (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyRN   ฬ   s8    	&c   
      C@  s๙   t  d | |  j | f  i t d |  j j   |  j j   f d t d t d 6} d | | | d f | d <i d	 | d
 6|  j d 6} |  j j	 d d |  j
 d | d | } d | d <|  j j | | |  }	 t |	 d d  p๋ d |  j | <|	 S(   sๅ   
        Copy a remote file chunk
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
        http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html
        s    Copying part %i of %r (%s bytes)s   /%s/%st   quote_backslashest   unicode_outputs   x-amz-copy-sources   bytes=%d-%di   s   x-amz-copy-source-ranges   %sR^   R_   R`   R+   R=   R>   u   remote copyu   actionR@   R"   R/   (   R   R   R	   R   t   bucketR3   t   FalseRM   R   RB   R   t   send_request_with_progressR   R   (
   R   RW   RZ   R   RV   RJ   R=   Rg   RD   RE   (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyRP   ๕   s$    
	
	 c   	      C@  sอ   t  d |  j  g  } d } x4 |  j j   D]# \ } } | j | | | f  q- Wd d j |  } i t t |   d 6} |  j j	 d d |  j
 d | d	 | d
 i |  j d 6} |  j j |  } | S(   s   
        Finish a multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
        s    MultiPart: Completing upload: %ss7   <Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>s5   <CompleteMultipartUpload>%s</CompleteMultipartUpload>R/   s   content-lengthR<   R+   R=   t   bodyR>   R_   (   R   R   R   t   itemst   appendt   joinRd   RR   R   RB   R   RC   (	   R   t	   parts_xmlt   part_xmlRW   Ra   Rm   R=   RD   RE   (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   complete_multipart_upload#  s    	c         C@  s   t  d |  j  d } | S(   s   
        Abort multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadAbort.html
        s   MultiPart: Aborting upload: %sN(   R   R   R   (   R   RE   (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   abort_upload9  s    i   i   i  P N(   t   __name__t
   __module__t   __doc__t   MIN_CHUNK_SIZE_MBt   MAX_CHUNK_SIZE_MBt   MAX_FILE_SIZER   R!   R.   R;   R   R]   RN   RP   Rs   Rt   (    (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyR      s   			`(.	i   (   t
   __future__R    R5   t   loggingR   R   R   R   t
   ExceptionsR   t   S3UriR   t	   BaseUtilsR   R   R	   R
   t   UtilsR   R   R   R3   R   (    (    (    sL   /oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.pyt   <module>   s   ""