3
b>5                 @   s   d dl mZ d dlZd dlmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZmZmZmZ ddlmZmZ dZG d
d deZdS )    )absolute_importN)debuginfowarningerror   )ParameterError)S3UriS3)getTextFromXmlgetTreeFromXmls3_quote
parseNodes)
formatSizecalculateChecksumi   c               @   sl   e Zd ZdZdZdZdZdddZdd Zd	d
 Z	dd Z
dddZdddZdddZdd Zdd ZdS )MultiPartUploadz<Supports MultiPartUpload and MultiPartUpload(Copy) operation   i   Nc             C   s   || _ d | _d | _|| _|| _i | _|p*i | _t|trZ|| _|sJt	d| j j
jt }n|| _| j j
jt }|| _| j | _d S )Nz8Source size is missing for MultipartUploadCopy operation)s3file_streamsrc_urisrc_sizedst_uripartsheaders_baseline
isinstancer	   r   configmultipart_copy_chunk_size_mbSIZE_1MBmultipart_chunk_size_mb
chunk_sizeinitiate_multipart_upload	upload_id)selfr   srcr   r   r   Zc_size r#   L/oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/MultiPart.py__init__   s     

zMultiPartUpload.__init__c             C   s^   | j j||}t }xD|D ]<}y"|d |d d|t|d < W q tk
rT   Y qX qW |S )NETagSize)checksumsize
PartNumber)r   list_multipartdictintKeyError)r!   urir    	part_listr   elemr#   r#   r$   get_parts_information2   s    

z%MultiPartUpload.get_parts_informationc             C   s   d}| j j|}xv|D ]n}yT|d }|d }td||j f  ||j krl|rhtd|tjd |f |}W q tk
r   Y qX qW |S )N UploadIdKeyzmp_path: %s, object: %szMore than one UploadId for URI %s.  Disable multipart upload, or use
 %s multipart %s
to list the Ids, then pass a unique --upload-id into the put command.r   )r   get_multipartr   object
ValueErrorsysargvr.   )r!   r/   r    Zmultipart_listmpuploadZmp_upload_idZmp_pathr#   r#   r$   get_unique_upload_idA   s     

z$MultiPartUpload.get_unique_upload_idc             C   s   | j jjr| j jj| _n | j jjr2| j| j| _nd| _| jsz| j jd| j| jddid}| j j|}|d }t	|d| _| jS )z
        Begin a multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html
        r3   OBJECT_POSTuploadsN)r/   headers
uri_paramsdatar4   )
r   r   r    put_continuer<   r   create_requestr   send_requestr
   )r!   requestresponserA   r#   r#   r$   r   V   s    

z)MultiPartUpload.initiate_multipart_uploadr3   c             C   sZ  | j stdi }| jr$| jj }n| jj}| jjjrF| j	| j
| j }|rRd| }|| j
j d}d}| jr| j}| j| j | j| j od }td||f  x|dkr|| j|d  }t| j| | j}	||	8 }d||dt|	d	d
 |f |d< yB| jr| j|||	||j|d n| j|||	||j|d W n:   td||tjd | j
| j tjd | j f   Y nX |d7 }qW td|d  dS td|  x| jj| j}
d}t|
}	d|dt|	d	d
 |f |d< |
sP y | j|||	||
|j|d W n:   td||tjd | j
| j tjd | j f   Y nX |d7 }qW td|d  dS )z
        Execute a full multipart upload on a file
        Returns the seq/etag dict
        TODO use num_processes to thread it
        zAAttempting to use a multipart upload that has not been initiated. )sourcedestinationr   z#MultiPart: Uploading %s in %d partsr   z[part %d of %d, %s]%sz%d%sBT)human_readableextra)remote_statusz
Upload of '%s' part %d failed. Use
  %s abortmp %s %s
to abort the upload, or
  %s --upload-id %s put ...
to continue the upload.z$MultiPart: Upload finished: %d partsNzMultiPart: Uploading from %sz[part %d of -, %s]%szw
Upload of '%s' part %d failed. Use
  %s abortmp %s %s
to abort, or
  %s --upload-id %s put ...
to continue the upload.)r    r   r   r/   r   stream_namer   r   rB   r2   r   r   r   r   minr   upload_partget	copy_partr   r9   r:   readlen)r!   extra_labelZremote_statusesfilenamelabelsseq	size_leftZnr_partsoffsetZcurrent_chunk_sizebufferr#   r#   r$   upload_all_partsl   s    



z MultiPartUpload.upload_all_partsc             C   s   t d|| j|f  |dk	rt|d |krt|| j||| jjj}|d jd}||kr~t	d| j
|f  |d | j|< dS t	d||| j
|f  nt	dt|d || j
|f  d	t|i}	d
| | jd}
| jjd| j
|	|
d}| jj|| j||||d}|d jddjd| j|< |S )z
        Upload a file chunk
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
        z"Uploading part %i of %r (%s bytes)Nr)   r(   z"'z:MultiPart: size and md5sum match for %s part %d, skipping.zJMultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading.zFMultiPart: size (%d vs %d) does not match for %s part %d, reuploading.zcontent-lengthz%s)
partNumberuploadId
OBJECT_PUT)r/   r?   r@   )rY   r   r?   etagr3   )r   r    r-   r   r   r   r   
send_chunkstripr   r   r   strrC   	send_filerP   )r!   rW   rY   r   rV   rZ   rL   r(   remote_checksumr?   query_string_paramsrE   rF   r#   r#   r$   rO      s8    





zMultiPartUpload.upload_partc       
      C   s   t d|| j|f  dtd| jj | jj f dddi}d||| d f |d	< d
| | jd}| jjd| j||d}d|d< | jj	|||}	t
|	d dpd| j|< |	S )z
        Copy a remote file chunk
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
        http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html
        z Copying part %i of %r (%s bytes)zx-amz-copy-sourcez/%s/%sFT)quote_backslashesunicode_outputzbytes=%d-%dr   zx-amz-copy-source-rangez%s)r\   r]   r^   )r/   r?   r@   zremote copyactionrA   r&   r3   )r   r    r   r   bucketr7   r   rC   r   send_request_with_progressr
   r   )
r!   rW   rY   r   rV   rL   r?   re   rE   rF   r#   r#   r$   rQ      s"    




zMultiPartUpload.copy_partc       	      C   s   t d| j  g }d}x(| jj D ]\}}|j|||f  q"W ddj| }dtt|i}| jj	d| j
||d| jid}| jj|}|S )	z
        Finish a multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
        z MultiPart: Completing upload: %sz7<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>z5<CompleteMultipartUpload>%s</CompleteMultipartUpload>r3   zcontent-lengthr=   r]   )r/   r?   bodyr@   )r   r    r   itemsappendjoinrb   rS   r   rC   r   rD   )	r!   Z	parts_xmlZpart_xmlrW   r_   rk   r?   rE   rF   r#   r#   r$   complete_multipart_upload#  s    
z)MultiPartUpload.complete_multipart_uploadc             C   s   t d| j  d}|S )z
        Abort multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadAbort.html
        zMultiPart: Aborting upload: %sN)r   r    )r!   rF   r#   r#   r$   abort_upload9  s    zMultiPartUpload.abort_uploadi   i   i  P )NN)r3   )r3   N)N)__name__
__module____qualname____doc__MIN_CHUNK_SIZE_MBMAX_CHUNK_SIZE_MBZMAX_FILE_SIZEr%   r2   r<   r   r[   rO   rQ   ro   rp   r#   r#   r#   r$   r      s    

` 
(
.r   i   )
__future__r   r9   loggingr   r   r   r   
Exceptionsr   S3Urir	   	BaseUtilsr
   r   r   r   Utilsr   r   r   r7   r   r#   r#   r#   r$   <module>   s   