3 ÒbZ>m?Z?yÆddl@mAZAddlBZByeBjCddZDdd„ZEWn”eFk r†y eBjGeBjHƒZDeDjIƒdd„ZEWn,eJk r€eBjCeBjHdZDdd„ZEYnXYn2eJk r¶eBjGeBjHƒZDeDjIƒd d„ZEYnXWn\eeKfk rZLz:eMeLƒZNd!eNkrêd"ZOnd#eNZOeOd$7ZOd%aPd&d„ZEWYddZL[LXnXd'd(„ZQd)ZRd3ZSgZTGd+d,„d,eUƒZVGd-d.„d.eUƒZWeTjXd.ƒd/d0„ZYd1d2„ZZdS)4é)Úabsolute_importÚdivisionN)Úsaxutils)Útimeout)ÚdebugÚinfoÚwarningÚerror)ÚST_SIZE)Úurlparse)Úencodestring)Úencodebytes)Úmd5é)ÚgetListFromXmlÚgetTextFromXmlÚgetRootTagNameÚdecode_from_s3Úencode_to_s3Ús3_quote)ÚconvertHeaderTupleListToDictÚ hash_file_md5Ú unicodiseÚdeunicodiseÚcheck_bucket_nameÚcheck_bucket_name_dns_supportÚgetHostnameFromBucketÚcalculateChecksum)Ú SortedDict)Ú AccessLog)ÚACLÚGranteeLogDelivery)ÚBidirMap)ÚConfig)Ú*)ÚMultiPartUpload)ÚS3Uri)ÚConnMan)Úsign_request_v2Úsign_request_v4Úchecksum_sha256_fileÚchecksum_sha256_bufferÚformat_param_str)Ú ArgumentErrorT)ÚmimecCs tj|ƒS)N)Úmagic_Z from_file)Úfile©r1úE/oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/S3.pyÚmime_magic_fileCsr3c Cs4y tj|ƒStttfk r.tjt|ƒƒSXdS)N)r/r0ÚUnicodeDecodeErrorÚUnicodeEncodeErrorr-r)r0r1r1r2r3Ps )ÚflagscCs tj|ƒS)N)r/Zid_filename)r0r1r1r2r3ZscCstjt|ƒƒS)N)r/r0r)r0r1r1r2r3csÚmagicz%Module python-magic is not available.z'Module python-magic can't be used (%s).z. Guessing MIME types based on file extensions.FcCststtƒdatj|ƒdS)NTr)Úmagic_warnedrÚ magic_messageÚ mimetypesÚ guess_type)r0r1r1r2r3nscCsjdd„}||ƒ}|dk rZt|tƒrZd|krR|jdƒ\}}|tdƒd…}||f}n|df}|dkrfd}|S)NcSst|ƒ}|S)N)r3)r0Z magictyper1r1r2Ú_mime_magicwszmime_magic.._mime_magicú;Úcharset)NN)Ú isinstanceÚstrÚsplitÚlen)r0r<ÚresultÚmimetyper>r1r1r2Ú mime_magicus rEéic@sFeZdZiZiZddd„Zdd„Zdd„Zdd „Zd d„Z dd „Z dS)Ú S3RequestNcCsn||_t|pidd|_t|jjjƒdkrF|jjjƒ|jjj|jd<||_||_|pXi|_ ||_ |jƒdS)NT)Úignore_caserzx-amz-security-token)Ús3rÚheadersrBÚconfigÚaccess_tokenÚrole_refreshÚresourceÚ method_stringÚparamsÚbodyÚrequester_pays)ÚselfrIrOrNrJrQrPr1r1r2Ú__init__“s zS3Request.__init__cCs"|jjjr|jdkrd|jd<dS)NÚGETÚPOSTÚPUTÚHEADZ requesterzx-amz-request-payer)rUrVrWrX)rIrKrRrOrJ)rSr1r1r2rRŸszS3Request.requester_payscCs,d|jkr|jd=tjdtjƒƒ|jd<dS)NÚdatez%a, %d %b %Y %H:%M:%S +0000z x-amz-date)rJÚtimeÚstrftimeÚgmtime)rSr1r1r2Úupdate_timestamp£s zS3Request.update_timestampcCs&|jjrdS|jjjs|jjr"dSdS)NFT)rIÚendpoint_requires_signature_v4rKÚsignature_v2Úfallback_to_signature_v2)rSr1r1r2Úuse_signature_v2¨s zS3Request.use_signature_v2cCs |jjdƒ}|jƒrVtdƒ|r4d||jdf}n |jd}t|j||j|jƒ|_n°tdƒ|jj |jdƒ}|rÆ|t jkr˜t jj|dƒjd|ƒs²|t jkrÆt tƒj|ƒrÆd||jdf}n |jd}t jj|jdtƒjƒ}t|j|||j||j|jƒ|_dS)NÚbucketzUsing signature v2z/%s%sÚurizUsing signature v4Úz%s.)rNÚgetrarr(rOrPrJrIÚget_hostnamerGÚ redir_mapÚ startswithrr#Úhost_bucketÚ region_mapÚbucket_locationr)rQ)rSÚbucket_nameZresource_uriÚhostnameZ bucket_regionr1r1r2Úsign±s& zS3Request.signcCsT|jƒ|jƒt|jƒ}t|dddd|d<|dt|jƒ7<|j||jfS)NrcFT)Úquote_backslashesÚunicode_output) r]rnÚdictrNrr,rPrOrJ)rSrNr1r1r2Úget_tripletÒs zS3Request.get_triplet)N)Ú__name__Ú __module__Ú__qualname__rjrgrTrRr]rarnrrr1r1r1r2rGŽs !rGc@sFeZdZedddddddZedd d ddd ZedededBededBededBededBededBededBededBededBededBededBdZddddœZdZdd„Z d d!„Z d"d#„Zd$d%„Zd©d'd(„Z d)d*„Zd«d+d,„Zdd-d.„Zd¯d/d0„Zd°d1d2„Zd3d4„Zd±d6d7„Zd8d9„Zd:d;„Zd²dd?„Zd´d@dA„ZdµdBdC„Zd¶dDdE„ZdFdG„ZdHdI„ZdJdK„Zd·dLdM„ZdNdO„Z d¸dQdR„Z!d¹dSdT„Z"dUdV„Z#dWdX„Z$dYdZ„Z%d[d\„Z&d]d^„Z'dºd_d`„Z(d»dadb„Z)d¼dcdd„Z*dedf„Z+dgdh„Z,didj„Z-dkdl„Z.dmdn„Z/dodp„Z0dqdr„Z1dsdt„Z2dudv„Z3dwdx„Z4dydz„Z5d{d|„Z6d}d~„Z7d¾dd€„Z8dÀdd‚„Z9dÂdƒd„„Z:dÄd…d†„Z;dÆd‡dˆ„ZddŽ„Z?dÇdd„Z@dÈd‘d’„ZAd“d”„ZBd•d–„ZCd—d˜„ZDd™dš„ZEd›dœ„ZFefddž„ZGdÉdŸd „ZHdPdeddÊd&fd¡d¢„ZIdËd£d¤„ZJdÌd¥d¦„ZKdefd§d¨„ZLd&S)ÍÚS3rrFéééé)rUrWrXÚDELETErVÚMASKéiiii)ÚSERVICEÚBUCKETÚOBJECTÚBATCHr|rr~rUrrWr{r€rXrVr)ZUNDFINEDÚLIST_ALL_BUCKETSÚ BUCKET_CREATEÚBUCKET_LISTÚ BUCKET_DELETEÚ OBJECT_PUTÚ OBJECT_GETÚOBJECT_HEADÚ OBJECT_DELETEÚOBJECT_POSTÚBATCH_DELETEzBucket '%s' does not existz Access to bucket '%s' was deniedzBucket '%s' already exists)ÚNoSuchBucketÚAccessDeniedZBucketAlreadyExistsécCs||_d|_d|_d|_dS)NF)rKr`r^Úexpect_continue_not_supported)rSrKr1r1r2rTszS3.__init__cCs(d}|jjdkr|jjS|jjr$d}|S)NÚSTANDARDrdZREDUCED_REDUNDANCY)rKÚ storage_classÚreduced_redundancy)rSÚclsr1r1r2r‘ szS3.storage_classcCsŒ|r|tjkrtj|}n(|r6t|jj|ƒr6t|ƒ}n|jjjƒ}|jjrb|j dƒrx|dd…}n|j dƒrx|dd…}t d||fƒ|S)Nz:443rwz:80ézget_hostname(%s): %séüÿÿÿéýÿÿÿ)rGrgrrKrirÚ host_baseÚlowerÚ use_httpsÚendswithr)rSrbÚhostr1r1r2rfs zS3.get_hostnamecCs|jƒtj|<dS)N)r˜rGrg)rSrbÚredir_hostnamer1r1r2Úset_hostname,szS3.set_hostnameNcCs¸|jdƒ}|rh|tjkr2tjj|dƒjd|ƒsL|tjkrht|jj|ƒrhdt|ddd|df}n|d}|r€d ||f}|jjdkr¨|jj r¨d |j |ƒ|f}td|ƒ|S)Nrbrdz%s.z/%s%sFT)rorprcz%s%szhttp://%s%szformat_uri(): )rerGrgrhrrKrirÚ proxy_hostr™rfr)rSrNÚ base_pathrlrcr1r1r2Ú format_uri/s z S3.format_uricCs*|jdƒ}|j|ƒ}t|ddƒ|d<|S)Nr‚ÚdataZBucketÚlist)Úcreate_requestÚsend_requestr)rSÚrequestÚresponser1r1r2Úlist_all_bucketsDs zS3.list_all_bucketscCs^g}g}x4|j|||||ƒD]\}} } |j| ƒ|j| ƒqWi}||d<||d<||d<|S)Nr¢Úcommon_prefixesÚ truncated)Úbucket_list_streamingÚextend)rSrbÚprefixÚ recursiveÚ uri_paramsÚlimitZ item_listÚprefixesr©ÚdirsÚobjectsr¦r1r1r2Úbucket_listJs zS3.bucket_listc#sDdd„}dd„}dd„}‡fdd„} |r0|jƒp2i}d } g}d }d } |}xö| r>|j|||||ƒ‰|ˆdƒ}|ˆdƒ}|t|ƒ7}| t|ƒ7} ||| krª||| }|ˆdƒ} | r0|dksÒ|| |kr"|rê| ˆd|ƒ|d <n&|r|dd|d <nd||fVPtd|d ƒn| ||fVP| ||fVqJWdS)ze Generator that produces , pairs of groups of content of a specified bucket. cSst|dƒpd}|jƒdkS)Nz.//IsTruncatedÚfalse)rr˜)r¡Zis_truncatedr1r1r2Ú_list_truncatedYsz1S3.bucket_list_streaming.._list_truncatedcSs t|dƒS)NZContents)r)r¡r1r1r2Ú _get_contents^sz/S3.bucket_list_streaming.._get_contentscSs t|dƒS)NZCommonPrefixes)r)r¡r1r1r2Ú_get_common_prefixesasz6S3.bucket_list_streaming.._get_common_prefixescstˆddƒp|ddS)Nr¡Z NextMarkerrÚKeyéÿÿÿÿ)r)r¡Úcurrent_list)r¦r1r2Ú_get_next_markerdsz2S3.bucket_list_streaming.._get_next_markerTrr¡rZmarkerÚPrefixFzListing continues after '%s'Nr¹r¹)ÚcopyÚbucket_list_noparserBr)rSrbr¬rr®r¯rµr¶r·r»r©r°Únum_objectsZnum_prefixesÚmax_keysrºZcurrent_prefixesr1)r¦r2rªWs@ zS3.bucket_list_streamingcCsb|dkri}|r||d<|jjr0|r0d|d<|dkrDt|ƒ|d<|jd||d}|j|ƒ}|S) Nr¬ú/Ú delimiterrzmax-keysr„)rbr®r¹)rKrr@r£r¤)rSrbr¬rr®rÀr¥r¦r1r1r2r¾‹s zS3.bucket_list_noparsecCsÆtdd}|r|j|ƒd}|rŠ|jƒjƒdkrŠ|jƒjƒdkrŠ|jƒ}|jƒdkr\|jƒ}d}||7}|d7}td |ƒt|dd nt|dd |jjr¦d|d <|j d|||d}|j |ƒ}|S)NT)rHrdÚUSz us-east-1ÚEUz/z1zbucket_location: )Ú dns_strictFzpublic-readz x-amz-aclrƒ)rbrJrQ)rÚupdateÚstripÚupperr˜rrrKÚ acl_publicr£r¤)rSrbrkÚ extra_headersrJrQr¥r¦r1r1r2Ú bucket_create™s& $ zS3.bucket_createcCs|jd|d}|j|ƒ}|S)Nr…)rb)r£r¤)rSrbr¥r¦r1r1r2Ú bucket_delete±s zS3.bucket_deleteFc Csò|jƒ}|jd|jƒddid}tjj|dƒ}tjj|dƒ}z4|rd|oH|rd|jjtj|<dtj|<|j|ƒ}Wd||kr†|tj|<n|tjkr˜tj|=||kr¬|tj|<n|tjkr¾tj|=Xt |ddƒ}|sÜ|dkrâd}n|d krîd }|S)Nr„Úlocation)rbr®rdz us-east-1r¡ZLocationConstraintrÃrÄz eu-west-1)rdrÃ) rbr£rGrgrerjrKr—r¤r) rSrcÚforce_us_defaultrbr¥Zsaved_redir_mapZsaved_region_mapr¦rÍr1r1r2Úget_bucket_location¶s0 zS3.get_bucket_locationcCsJ|jd|jƒddid}|j|ƒ}|jddƒ}|rBt|ddƒ}nd}|S)Nr„ÚrequestPayment)rbr®r¡rdZPayer)r£rbr¤rer)rSrcr¥r¦Z resp_dataÚpayerr1r1r2Úget_bucket_requester_pays×s zS3.get_bucket_requester_payscCsTi}|j|ƒ|d<y|j|ƒ|d<Wn*tk rN}zd|d<WYdd}~XnX|S)Nzbucket-locationzrequester-pays)rÏrÒÚS3Error)rSrcr¦Úer1r1r2Úbucket_infoâszS3.bucket_infocCs®|jƒ}|jd|ddid}yR|j|ƒ}t|ddƒ|d<t|ddƒ|d<|jj|jƒ|j|ƒd œ|d <|Stk r¨}z|jdkr–t dƒdS‚WYdd}~XnXdS) Nr„Úwebsite)rbr®r¡z.//IndexDocument//SuffixÚindex_documentz.//ErrorDocument//KeyÚerror_document)rbrÍÚwebsite_endpointi”zICould not get /?website - website probably not configured for this bucket) rbr£r¤rrKrÙrÏrÓÚstatusr)rSrcrkrbr¥r¦rÔr1r1r2Úwebsite_infoës zS3.website_infocCsŒ|jƒ}d}|d7}|d|jj7}|d7}|jjrT|d7}|d|jj7}|d7}|d7}|jd ||d did}|j|ƒ}td|ƒ|S) NzFz z %sz z z %sz zrƒrÖ)rbrQr®zReceived response '%s')rbrKÚ website_indexÚ website_errorr£r¤r)rSrcrkrbrQr¥r¦r1r1r2Úwebsite_createþs zS3.website_createcCsN|jƒ}|jd|ddid}|j|ƒ}td|ƒ|ddkrJtd|ƒ‚|S)Nr…rÖ)rbr®zReceived response '%s'rÚéÌzExpected status 204: %s)rbr£r¤rÚS3ResponseError)rSrcrkrbr¥r¦r1r1r2Úwebsite_deletes zS3.website_deletecCsØ|jƒ}|jd|ddid}y|j|ƒ}WnPtk rz}z4|jdkrRtdƒdS|jdkrhtdƒdS‚WYdd}~XnXt|dƒ}|d kržtd |ƒdSt|ddƒ|d<t|dd ƒ|d<t|ddƒ|d<|S)Nr„Ú lifecycle)rbr®i”zMCould not get /?lifecycle - lifecycle probably not configured for this bucketiõzKCould not get /?lifecycle - lifecycle support not implemented by the serverr¡ZLifecycleConfigurationz7Could not get /?lifecycle - unexpected xml response: %sz.//Rule//Prefixr¬z.//Rule//Expiration//DaterYz.//Rule//Expiration//DaysÚdays)rbr£r¤rÓrÚrrr)rSrcrkrbr¥r¦rÔZ root_tag_namer1r1r2Úexpiration_infos2 zS3.expiration_infocCs‚|jjr|jjrtdƒ‚|jjp&|jjs^|jjr8tdƒ‚tdƒ|jƒ}|jd|ddid}n |j|ƒ}|j |ƒ}td|ƒ|S)Nz+Expect either --expiry-day or --expiry-datezdel bucket lifecycler…râ)rbr®zReceived response '%s') rKÚexpiry_dateÚexpiry_daysÚParameterErrorÚ expiry_prefixrrbr£Ú_expiration_setr¤)rSrcrkrbr¥r¦r1r1r2Úexpiration_set=s zS3.expiration_setcCs¸tdƒd}|d7}|d|jj7}|d7}|d7}|jjrN|d|jj7}n|jjrf|d|jj7}|d 7}|d 7}|d7}tdd }t|ƒ|d<|jƒ}|jd|||ddid}|S)Nzput bucket lifecyclezz z %sz Enabledz z %sz %sz z zT)rHzcontent-md5rƒrâ)rbrJrQr®) rrKrèrårærÚcompute_content_md5rbr£)rSrcrQrJrbr¥r1r1r2réMs( zS3._expiration_setcCs‚|jj}d}|dkr&|jjr&tdƒ‚|jjrn|jjrJttjjt |ƒƒƒ}|jj r`t|ƒ\}}ntj |ƒ\}}|sz|jj}||fS)Nú-zRYou must specify --mime-type or --default-mime-type for files uploaded from stdin.)rKÚdefault_mime_typerçÚguess_mime_typeÚfollow_symlinksrÚosÚpathÚrealpathrÚuse_mime_magicrEr:r;)rSÚfilenameÚcontent_typeÚcontent_charsetr1r1r2Ú_guess_content_typedszS3._guess_content_typecCs,|jj}|s|jj}|d|jjjƒ7}|S)Nz ; charset=)rKÚ mime_typeríÚencodingrÈ)rSrõr1r1r2Ústdin_content_typevs zS3.stdin_content_typecCsb|jj}d}|dkr|jƒS|s.|j|ƒ\}}|s>|jjjƒ}|j||ƒr^|dk r^|d|}|S)Nrìz ; charset=)rKrørúr÷rùrÈÚadd_encoding)rSrôrõrör1r1r2rõ~szS3.content_typecCsbd|krdS|jjjdƒ}|ddkr*dS|jddƒ}t|ƒdkrFdS|d}||krZd SdSdS) Nzcharset=Fú,rrdÚ.rFrT)rKÚadd_encoding_extsrAÚrsplitrB)rSrôrõZextsÚpartsÚextr1r1r2rûszS3.add_encodingrdcCsÎ|jdkrtd|jƒ‚|dkr:tjjt|ƒƒr:tdƒ‚yX|dkrftjt j jƒddd}d|_d }n*tjt|ƒdd }||_tj t|ƒƒt}Wn4ttfk rÆ}ztd|jƒ‚WYdd}~XnXtdd }|rà|j|ƒ|jjrðd|d<|jjrd|d<|jj|d<|j|d|d<|jjr0d|d<|jƒ|d<d} |jjr^|dkr^tdƒ‚|jjrº||jjtks„|dkrºd} ||jj|jjtkrºtd|jj|jjfƒ‚| rÒ|j|||||ƒS|jj ry|j!|ƒ} Wnt"k rd} YnX| dk rt#| ddƒ}| ddj$dƒ}||kr~t%d|d ||jj&ƒ} || krjt'd|ƒdSt'd|| |fƒnt'd |||fƒt(|ƒ|d<|j)d!||d"}||j*ƒ|d#œ}|j+|||ƒ}|S)$NrIz Expected URI type 's3', got '%s'rìzNot a regular fileÚrbF)ÚmodeÚclosefdzr)rz%sT)rHÚAES256zx-amz-server-side-encryptionzaws:kmsz+x-amz-server-side-encryption-aws-kms-key-id)rôzcontent-typezpublic-readz x-amz-aclzx-amz-storage-classz2Multi-part upload is required to upload from stdinzZChunk size %d MB results in more than %d chunks. Please increase --multipart-chunk-size-mbrJzcontent-lengthÚetagz"'rdz,Put: size and md5sum match for %s, skipping.zBMultiPart: checksum (%s vs %s) does not match for %s, reuploading.z>MultiPart: size (%d vs %d) does not match for %s, reuploading.r†)rcrJ)ÚsourceÚdestinationÚextra),ÚtypeÚ ValueErrorrðrñÚisfilerÚInvalidFileErrorÚioÚopenÚsysÚstdinÚfilenoÚstream_nameÚstatr ÚIOErrorÚOSErrorÚstrerrorrrÆrKÚserver_side_encryptionÚkms_keyrõrÉr‘Úenable_multipartrçÚmultipart_chunk_size_mbÚSIZE_1MBÚmultipart_max_chunksÚsend_file_multipartÚput_continueÚobject_infoÚ ExceptionÚintrÇrÚ send_chunkrr@r£rcÚ send_file)rSrôrcrÊÚextra_labelZ src_streamÚsizerÔrJÚ multipartrZremote_sizeZremote_checksumZchecksumr¥Úlabelsr¦r1r1r2Ú object_putŸsv z S3.object_putc CsJ|jdkrtd|jƒ‚|jd|d}|jƒ||dœ}|j||||ƒ}|S)NrIz Expected URI type 's3', got '%s'r‡)rc)rrr )r rr£rcÚ recv_file) rSrcÚstreamÚ dest_nameÚstart_positionr%r¥r(r¦r1r1r2Ú object_getös z S3.object_getcs ‡fdd„ˆDƒ}|j|ƒdS)z" Batch delete given a remote_list csg|]}ˆ|d‘qS)Úobject_uri_strr1)Ú.0Úitem)Úremote_listr1r2ú sz*S3.object_batch_delete..N)Úobject_batch_delete_uri_strs)rSr2Úurisr1)r2r2Úobject_batch_deleteþszS3.object_batch_deletec Csvdd„}|}t|ƒdkr tdƒ‚t|dƒjƒ}|||ƒ}tt|ƒddœdd}|jd |||d did}|j|ƒ}|S) z* Batch delete given a list of object uris cSsˆd}xn|D]f}t|ƒ}|jdkr.td|jƒ‚|jƒsBtd|ƒ‚|jƒ|krVtdƒ‚tj|jƒƒ}|d|7}q W|d7}t|ƒ}|S)Nz.rIz Expected URI type 's3', got '%s'zURI '%s' has no objectz2The batch should contain keys from the same bucketz

z ) r&r rÚ has_objectrbrÚescapeÚobjectr)rbZkey_listrQÚkeyrcr9r1r1r2Úcompose_batch_del_xmls z>S3.object_batch_delete_uri_strs..compose_batch_del_xmlrzKey list is emptyzapplication/xml)zcontent-md5zcontent-typeT)rHr‹ÚdeleteN)rbrJrQr®)rBrr&rbrrër£r¤) rSr5r;ZbatchrbZrequest_bodyrJr¥r¦r1r1r2r4s zS3.object_batch_delete_uri_strscCs4|jdkrtd|jƒ‚|jd|d}|j|ƒ}|S)NrIz Expected URI type 's3', got '%s'r‰)rc)r rr£r¤)rSrcr¥r¦r1r1r2Ú object_delete"s zS3.object_deletecCs¬|jdkrtd|jƒ‚|jjdkr,tdƒ‚|jjdkr@tdƒ‚d }|d |jj7}|d7}|d|jj7}|d 7}|d7}|jd||ddid}|j|ƒ}td|ƒ|S)NrIz Expected URI type 's3', got '%s'rz*You must restore a file for 1 or more daysÚStandardÚ ExpeditedÚBulkz7Valid restoration priorities: bulk, standard, expeditedz@z %sz z %sz zrŠÚrestore)rcrQr®zReceived response '%s')r>r?r@) r rrKÚrestore_daysrçÚrestore_priorityr£r¤r)rSrcrQr¥r¦r1r1r2Úobject_restore)s" zS3.object_restorec CsNddddddddd d ddd g }x*||jjD]}|jƒ|kr,||jƒ=q,W|S)NrYzcontent-lengthz last-modifiedzcontent-md5zx-amz-version-idzx-amz-delete-markerz accept-rangesÚ connectionrZserverz x-amz-id-2zx-amz-request-idzx-amz-storage-class)rKÚremove_headersr˜)rSrJZ to_removeÚhr1r1r2Ú_sanitize_headers<s"zS3._sanitize_headerscCsæ|jdkrtd|jƒ‚|jdkr0td|jƒ‚|jjdkr€y|j|ƒ}Wn4tk r~}z|jdkrj|‚d}WYdd}~XnXd} d} |s”|jjr˜d}|rº|j|ƒ}|d} t | dƒ}|jj rJ| }|dkrî|j|ƒ}|d}t |dƒ}||krtjt } n|jjt } || krJ|dkrB|j|ƒ}|d}t |dƒ}|} d} | rh|j| ƒt| dd } n tdd } |jjr„d | d<|jƒ| d<|jjr¢d | d<|jjrÀd| d<|jj| d<|rÐ| j|ƒ|jjræ|jj| d<|söd| d<nd| d<| r|j|||| |ƒ}n zRequester zBucketOwner z rƒrÐ)rcrQr®)rrKrRr£r¤)rSrcrJrQr¥r¦r1r1r2Ú set_payer]s zS3.set_payercCsB|jd|jƒddid}td|ƒ|j|ƒ}td|dƒ|S)Nr„râ)rbr®zget_lifecycle_policy(%s)z%s: Got Lifecycle PolicyrÚ)r£rbrr¤)rSrcr¥r¦r1r1r2Úget_lifecycle_policyks zS3.get_lifecycle_policycCs.|jd|ddid}td|ƒ|j|ƒ}|S)Nr…râ)rcr®zdelete_lifecycle_policy(%s))r£rr¤)rSrcr¥r¦r1r1r2Údelete_lifecycle_policyts zS3.delete_lifecycle_policycCs.g}x$|j|||ƒD]\}}|j|ƒqW|S)N)Úget_multipart_streamingr«)rSrcr®r¯Úupload_listr©Úuploadsr1r1r2Ú get_multipart{szS3.get_multipartccs<|r|jƒpi}|jƒ}d}d}|}d|d<x|r6|j|d||d}|d} t| dƒ} |t| ƒ7}||krv||}t| dƒ}|s’|jƒdkr–d }|r*|dks®||kr| rt| dƒ}|sÎ| dd}||d <t| dƒ} | rî| |d<nd|kr|d=nd | fVPtd|d ƒn|| fVP|| fVq0WdS)NTrri)rr®rÀr¡ZUploadz.//IsTruncatedr´FrZ NextKeyMarkerr¸Z KeyMarkerZNextUploadIdMarkerZUploadIdMarkerzListing continues after '%s'r¹r¹)r½rbr¾rrBrr˜r)rSrcr®r¯rbr©r¿rÀr¦Úxml_datarhÚ xml_truncatedZnext_keyZupload_id_markerr1r1r2rg„sL zS3.get_multipart_streamingcCs0g}x&|j||||ƒD]\}}|j|ƒqW|S)N)Úlist_multipart_streamingr«)rSrcÚ upload_idr®r¯Ú part_listr©rr1r1r2Úlist_multipartºszS3.list_multipartc csø|r|jƒpi}d}d}|}xÖ|rò|j||||ƒ}|d} t| dƒ} |t| ƒ7}||kr`||}t| dƒ}|s||jƒdkr€d}|ræ|d ks”||krÚ| r¼t| d ƒ}|s²| dd }||d<nd| fVPtd|dƒn|| fVP|| fVqWdS)NTrr¡ZPartz.//IsTruncatedr´FrZNextPartNumberMarkerÚ PartNumberzpart-number-markerz!Listing continues after Part '%s'r¹r¹)r½Úlist_multipart_noparserrBrr˜r) rSrcrnr®r¯r©r¿Ú max_partsr¦rkrorlZnext_part_numberr1r1r2rmÄs< zS3.list_multipart_streamingcCsF|dkri}|dkr t|ƒ|d<||d<|jd||d}|j|ƒ}|S)Nrz max-partsÚuploadIdr‡)rcr®r¹)r@r£r¤)rSrcrnr®rsr¥r¦r1r1r2rrïs zS3.list_multipart_noparsecCs"|jd|d|id}|j|ƒ}|S)Nr‰rt)rcr®)r£r¤)rSrcÚidr¥r¦r1r1r2Úabort_multipartûs zS3.abort_multipartcCs2|jd|jƒddid}|j|ƒ}t|dƒ}|S)Nr„Úlogging)rbr®r¡)r£rbr¤r)rSrcr¥r¦Ú accesslogr1r1r2Ú get_accesslogs zS3.get_accesslogcCs^|j|ƒ}td|jƒ|fƒ|jtdƒƒ|jtdƒƒtd|jƒ|fƒ|j||ƒdS)NzCurrent ACL(%s): %sZREAD_ACPZWRITEzUpdated ACL(%s): %s)rKrrcZ appendGranteer!rO)rSrcrTr1r1r2Úset_accesslog_acls zS3.set_accesslog_aclc CsÎtƒ}|r |j|ƒ|j|ƒn|jƒd|}td||fƒ|jd|jƒ|ddid}y|j|ƒ}Wn\tk rÄ} z@| j ddkr²t dƒ|j td |jƒƒƒ|j|ƒ}n‚WYdd} ~ XnX||fS) Nz%sz$set_accesslog(%s): accesslog-xml: %srƒrw)rbrQr®ÚCodeZInvalidTargetBucketForLoggingz.Setting up log-delivery ACL for target bucket.zs3://%s)rZ enableLoggingZsetAclPublicZdisableLoggingrr£rbr¤rÓrrzr&) rSrcÚenableÚlog_target_prefix_urirÉrxrQr¥r¦rÔr1r1r2Ú set_accesslogs$ zS3.set_accesslogcCsšdddœ}|r|s|rtdƒ‚|r>|jƒ}|jƒr:|jƒpBoth 'uri' and either 'bucket' or 'object' parameters suppliedrbrcr|zCreateRequest: resource[uri]=%s) rrbr7r9rvÚhttp_methodsÚgetkeyÚ operationsrGr)rSÚ operationrcrbr9rJrQr®rNrOr¥r1r1r2r£(s zS3.create_requestcCs|j|ddS)Nrr”)Ú_max_retries)rSÚretriesr1r1r2Ú _fail_wait>sz S3._fail_waitc Os0|djdƒ}d|krzt|dƒdkrzt|ddƒ}t|ddƒ}|j||ƒtd|ƒ|rp|tj|<td|ƒ|||ŽS|jd kr$|djd ƒ} | r| jdƒr²| dd…} n| jd ƒrÈ| dd…} t d | ƒj } |jd}|j|| ƒtd| ƒ|r|tj|<td|ƒ|||ŽStd|ƒt |ƒ‚dS)NrJzx-amz-bucket-regionr¡rz .//Bucketz.//EndpointzRedirected to: %szRedirected to region: %srXrÍzhttp://ézhttps://rxrbzƒRedirection error: No info provided by the server to where should be forwarded the request (HEAD request). (Hint target region: %s))rerBrrrrGrjrOrhrrmrNrrÓ) rSr¥r¦ÚfnÚargsÚkwargsZredir_regionZredir_bucketrœZlocation_urlr1r1r2Ú_http_redirection_handlerBs6 zS3._http_redirection_handlerc Osd|krÞt|dƒdkrÞt|ddƒ}|dkrrt|ddƒ}|dk rh|tj|jd<td|ƒ|||ŽStd ƒnj|d krªt|ddƒ}|dkrÜtd ƒd|_|||ŽSn2|dkrÜ|j ƒrÜ|j rÜtdƒd|_ |||ŽSn,|j ƒoî|j r tdƒd|_ |||ŽSdS)zR Returns None if no handler available for the specific error code r¡rr{ZAuthorizationHeaderMalformedZRegionNrbzForwarding request to %szVCould not determine bucket the location. Please consider using the --region parameter.ZInvalidRequestÚMessagez\The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.zEndpoint requires signature v4TZInvalidArgumentzFalling back to signature v2)rBrrGrjrNrrrr^rar`) rSr¥r¦r‡rˆr‰ÚfailureCodeÚregionÚmessager1r1r2Ú_http_400_handlergs2 zS3._http_400_handlercOszd|krnt|dƒdkrnt|ddƒ}|dkrnt|ddƒ}|dkrn|jƒrn|jrntdƒd|_|||ŽSt|ƒ‚dS) Nr¡rr{rr‹z=AWS authentication requires a valid Date or x-amz-date headerzFalling back to signature v2T)rBrrar`rrÓ)rSr¥r¦r‡rˆr‰rŒrŽr1r1r2Ú_http_403_handler‘s zS3._http_403_handlercCsº|jjdƒr¶|jƒr¶tjj|jdtƒjƒdkr¶tdƒyFtd|jdƒ}|j |dd}|dk rt|tj|jd<td|ƒWn4t k r´}ztd |ƒtd ƒWYdd}~XnXdS)a?Get and update region for the request if needed. Signature v4 needs the region of the bucket or the request will fail with the indication of the correct region. We are trying to avoid this failure by pre-emptively getting the correct region to use, if not provided by the user. rbrÃz====== SEND Inner request to determine the bucket region =====zs3://T)rÎNzE===== SUCCESS Inner request to determine the bucket region (%r) =====z,getlocation inner request failure reason: %sz?===== FAILED Inner request to determine the bucket region =====)rNrerarGrjr#rkrr&rÏr!)rSr¥Zs3_urirrUr1r1r2Úupdate_region_inner_requestŸs zS3.update_region_inner_requestcCsÔ|j|ƒt|jƒ|_|j}|jƒ\}}}i}tdƒd}yÂtj|j|dƒƒ}|j ||j ƒ}td|||t|jprdƒfƒ|jj |||j|ƒ|jjƒ} | j|d<| j|d<t| jƒƒ|d<| jƒ|d<d |dkrît|dd ƒ} | |d <tj|ƒWn2ttfk r‚Ynttfk r}zÖtdtj|ƒƒt|dƒrn|jrn|jtjtjtj fksŠd t!|ƒksŠdt!|ƒkršt"|t#ƒrš‚tj$|ƒ|rðt%d|d|fƒt%d|j&|ƒƒt'j(|j&|ƒƒ|j)||dƒSt*d|dƒ‚WYdd}~Xn tdtj|ƒƒ‚YnXtdtj|ƒƒ|dd#krb|j+|||j)|ƒS|ddkrð|j,|||j)|ƒ}|rŒ|St-|ƒ} |rì| j.d$krìt%d|d| fƒt%d|j&|ƒƒt'j(|j&|ƒƒ|j)||dƒS| ‚|ddkr|j/|||j)|ƒS|ddkr&t-|ƒ‚|ddksB|ddkr¬t-|ƒ}|ddkr\d }|r¨t%d|d|fƒt%d|j&|ƒƒt'j(|j&|ƒƒ|j)||dƒS|‚|dd!ksÈ|dd"krÐt-|ƒ‚|S)%Nz"Processing request, please wait...rbzESending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)rdrÚÚreasonrJr¡zx-amz-meta-s3cmd-attrszs3cmd-attrsz Response: Úerrnoz[Errno 104]z [Errno 32]z Retrying failed request: %s (%s)rczWaiting %d sec...rzRequest failed for: %sé-é3iÚ BadDigestÚOperationAbortedÚTokenRefreshRequiredÚRequestTimeouti“i•iôiiõréÈi+)r”r•)r–r—r˜r™)0r‘rrQrJrrrr'rerfr rñrBÚcr¥ÚgetresponserÚr’rÚ getheadersÚreadÚparse_attrs_headerÚputÚ S3SSLErrorÚS3SSLCertificateErrorrr!ÚpprintÚpformatÚhasattrr“ÚEPIPEÚ ECONNRESETÚ ETIMEDOUTr@r?ÚSocketTimeoutExceptionÚcloserr…rZÚsleepr¤ÚS3RequestErrorrŠrrÓÚcoder)rSr¥r„rJrOrNr¦ÚconnrcÚ http_responseÚattrsrÔÚ handler_fnÚerrr1r1r2r¤¼s’ zS3.send_requestcCs|jjstdƒ|j|ƒSd|kr*d|d<|jj||ƒ}y|j|ƒ}Wn.tk rt}z|jdƒ‚WYdd}~XnX|j|d|jdƒ|S)zrWrapper around send_request for slow requests. To be able to show progression for small requests z$Sending slow request, please wait...Úactionr¥ÚfailedN)Úcurrent_positionÚdone)rKÚprogress_meterrr¤Úprogress_classr!r¶rÆ)rSr¥r(Zoperation_sizeÚprogressr¦rUr1r1r2Úsend_request_with_progress"s zS3.send_request_with_progressc ).Cs, |j|ƒ| dkr|jj} |jr(| r(d} |j} t| dƒ}}|j} |jjrdd|d<|jj||ƒ}nt d| ƒt j ƒ}|rŠt|||ƒ}nt| ||ƒ}||_ | r²|sªd} nd| d<|jƒ\}}} yftj|j|dƒƒ}|jj||j||jƒƒx*| jƒD]}|jjt|ƒt| |ƒƒqøW|jjƒWnÄtk rL}z‚WYdd}~Xn tk rê}z‚|jjrr|jd ƒ|rÊtd |d|fƒtd|j|ƒƒt j|j|ƒƒ|j ||||||d ||ƒSt!d|dƒ‚WYdd}~XnX|dkr|j"|ƒt#ƒ}yðd}| rft$j$|jj%gggt&ƒ\}}}|rB|jj'ƒ}n$|rf|rftdt&ƒd|_d} | s‚|rœ|j(tj)krœ|rš|j*ƒtj+|j_,xî|dkrˆt-|jj.|ƒ}|dkrÊ|j*|ƒ}n|}|sÜt/dƒ‚|jj0dkròt j ƒ}|j1|ƒ|jj2|ƒ|jjr"|j1t3|ƒd|t3|ƒ8}|}|jj0dkrjt j ƒ|}t4|ƒ|jj0} t5| ||ƒ}|rœt jt-||jj6ƒƒqœW|j7ƒ}!|jj'ƒ}i}"|j(|"d<|j8|"d<t9|j:ƒƒ|"d<|j*ƒ|"d<||"d<tj;|ƒt|"ƒƒWnätk r }z‚WYdd}~Xn¾t/k rZ}z|jjrF|jd ƒ‚WYdd}~Xn„tk rÜ}zd|jjr‚|jd ƒ|r¬d}#t?|dƒr¸|j@r¸|j@t@jAt@jBt@jCfksÔdtD|ƒksÔdtD|ƒkrPtE|tFƒrPyN|jj'ƒ}i}"|j(|"d<|j8|"d<t9|j:ƒƒ|"d<|j*ƒ|"d<||"d<d}#Wntk rNtGdƒYnX|#sÌtd|d|fƒtd|j|ƒƒt j|j|ƒƒ|j ||||||d ||| ƒ Sn tƒ| rtd?| ƒ|j ||||||d ||| ƒ Std8| ƒt!d8| ƒ‚|"S)DNFzcontent-lengthÚuploadr³z!Sending file '%s', please wait...z100-continueZexpectrbr´z Retrying failed request: %s (%s)rczWaiting %d sec...rzUpload failed for: %srdzQHTTP Expect Continue feature disabled because of no reply of the server in %.2fs.Trz3File smaller than expected. Was the file truncated?)Údelta_positionrÚr’rJr¡r&z Response: r“z[Errno 104]z [Errno 32]zXCannot retrieve any response status before encountering an EPIPE or ECONNRESET exceptionzUpload failed: %s (%s)zGiving up on '%s' %sÚelapsedÚspeedr¶é-é3)ÚoffsetÚ chunk_sizeÚuse_expect_continueir–r—r˜r™i“i¡)rÃrrši+iôi÷rŽg{®Gáz„?iz(Retrying on lower speed (throttle=%0.2f)z$Too many failures. Giving up on '%s'z"MD5 sums: computed=%s, received=%sz"'rìzx-amz-server-side-encryptionzaws:kmszMD5 Sums don't match!zRetrying upload of %sr¹)r¿rÀ)r–r—r˜r™)r–r—r˜r™)Nr‘rKÚuse_http_expectrrJr"rr·r¸rrZr+r*rQrrr'rerfr›Ú putrequestr rñÚkeysÚ putheaderrÚ endheadersrçr!r¶rr…r«r$Ú S3UploadErrorÚseekrÚselectÚsockÚEXPECT_CONTINUE_TIMEOUTrœrÚÚCONTINUEržÚ_CS_REQ_SENTÚ_HTTPConnection__stateÚminr#r Ú limitraterÆZwrapper_send_bodyrBÚfloatÚmaxÚthrottle_maxÚ hexdigestr’rrr rr£r¤r¥r“r¦r§r¨r@r?r©r rŠrrÓrrrÇ))rSr¥r+r(ÚbufferZthrottler„rÁrÂrÃrJÚ size_leftÚ size_totalrôr¹Útimestamp_startZsha256_hashrOrNr®ÚheaderrÔÚmd5_hashr¯ÚreadableÚwritableZexceptionalÚlr¡Ú start_timeZlimitrate_throttleÚ real_durationÚexpected_durationZmd5_computedr¦Zknown_errorÚ timestamp_endr±r²Z try_retryÚmd5_from_s3r1r1r2r$:st " ", ",zS3.send_filec Csœtjƒ}t|||||ƒ}|j|ƒ|jƒ}tjƒ} | ||d<||d<|drbt|dƒ|dphtdƒ|d<|dr˜t|dƒdkr˜tt|ddƒƒ‚|S) Nr½r&rr¾r¡rJr‹r¹)rZr%Zupload_all_partsZcomplete_multipart_uploadrÓrrÉr) rSr+rJrcr&r%rÚr»r¦rãr1r1r2r7s (zS3.send_file_multipartcCs|j|||||ƒS)N)r)rSrPrQr&rJr%r1r1r2rNGszS3.copy_file_multipartc8CsÔ|j|ƒ|jƒ\}}}|j} |jjr>d|d<|jj|dƒ} ntd| ƒtjƒ}d}yøtj |j |dƒƒ}|jj||j ||jƒƒx*|jƒD]} |jjt| ƒt|| ƒƒqŽW|dkrÖtd|ƒ|jjdd|ƒ|jjƒi}|jjƒ}|j|d <|j|d <t|jƒƒ|d<d|dkr8t|ddƒ}||d <tdtj|ƒƒWn&tk rv}z‚WYdd}~Xnttfk rt}zÞ|jjr | jdƒt |dƒrÌ|j!rÌ|j!t!j"t!j#t!j$fksèdt%|ƒksèdt%|ƒkrøt&|t'ƒrø‚tj(|ƒ|rTt)d|d|fƒt)d|j*|ƒƒtj+|j*|ƒƒ|j,|||||dƒSt-d|dƒ‚WYdd}~XnX|d d5kr¨|j.ƒ|d<|j/|||j,||||ƒS|d dkrì|j.ƒ|d<|j0|||j,||||ƒ}|rä|St1|ƒ‚|d dkr|j.ƒ|d<|j2|||j,||||ƒS|d dkr@|j.ƒ|d<t1|ƒ‚|d dks\|d dkrp|j.ƒ|d<t1|ƒ‚|dkr€t3ƒ}t4|dd ƒ}||}|}|jjr¸|| _5|| _6|| _7y |dkrâ|j.dƒ}t8|ƒdksât9‚xè||krÊ||jj:kr|jj:p|}|jj;dkr tjƒ}|j.|ƒ}t8|ƒdkr@t|ƒ|dkr |j?|ƒ|t8|ƒ7}|jjrä| j?t8|ƒd"qäWtj@|ƒWntAk rô‚Ynttfk rò}zÞ|jjr| jdƒt |dƒrJ|j!rJ|j!t!j"t!j#t!j$fksfdt%|ƒksfdt%|ƒkrvt&|t'ƒrv‚tj(|ƒ|rÒt)d|d|fƒt)d|j*|ƒƒtj+|j*|ƒƒ|j,|||||dƒSt-d|dƒ‚WYdd}~XnX|jBƒtjƒ}|jjr | j?ƒ| jd#ƒ|dj d$d%ƒjCd&ƒ}d'|dkrly|d d(}WntDk rjYnXd)|krê|dkrŽ|jEƒ|d(<n\ytF| ƒ|d(<WnJtk rè}z,|j!t!jGkrÐt)d*| |fƒt)d+ƒWYdd}~XnX|j d(ƒ|k|d,<|||d-<||d.<|d-r0t=|d.ƒ|d-p6t=d6ƒ|d/<|d.|t4|dd ƒkr~t)d0|t4|dd ƒ|d.fƒtd1|j d(ƒƒd)|krÐ|d,rÐ|dj d2ƒd3krÐt)d4|j d(ƒ|fƒ|S)7NZdownloadr³rz#Receiving file '%s', please wait...rbzRequesting Range: %d .. endZRangez bytes=%d-rÚr’rJzx-amz-meta-s3cmd-attrszs3cmd-attrsz Response: r´r“z[Errno 104]z [Errno 32]z Retrying failed request: %s (%s)rczWaiting %d sec...rzDownload failed for: %sé-é3r¡ii“i•rši+zcontent-lengthzEOF from S3!)r¼r¶rrdz"'zx-amz-meta-s3tools-gpgencrrìzUnable to open file: %s: %sz(Unable to verify MD5. Assume it matches.Zmd5matchr½r&r¾z4Reported size (%s) does not match received size (%s)zReceiveFile: Computed MD5 = %szx-amz-server-side-encryptionzaws:kmsz5MD5 signatures do not match: computed=%s, received=%s)rårær¹)Hr‘rrrrKr·r¸rrZr'rerfr›rÅr rñrÆrÇrrrÈrœrÚr’rrrŸr£r¤rçrr!r¶r¥r“r¦r§r¨r@r?r©rªrr…r«r*ÚS3DownloadErrorržrŠrrÓrrr"Ú total_sizeÚinitial_positionrµrBÚAssertionErrorÚ recv_chunkrÒràrÓÚwriterÆr rÚflushrÇÚKeyErrorrÖrÚENOENT)rSr¥r+r(r-r„rOrNrJrôr¹rÚr®rÛr¦r¯r°rÔr±rÜrØrÙrµr¡Z this_chunkràrárârãrär1r1r2r*Ls& " " , *zS3.recv_file)Nr¹)NNNr¹r¹)NNNr¹r¹)NNNr¹)NN)F)N)N)N)N)N)N)Nrd)rrd)NNrdF)NNrd)NNrdr¹)Nr¹r¹)Nr¹r¹)Nr¹r¹)Nr¹r¹)Nr¹)NF)NNNNrdN)rr¹)rd)rd)Mrsrtrur"rZtargetsrÚcodesrƒrTr‘rfrr r§r³rªr¾rËrÌrÏrÒrÕrÛrÞrárärêrér÷rúrõrûr)r.r6r4r=rDrHrVrWrZr rKrOr\r]r^r`rarbrcrdrerfrjrgrprmrrrvryrzr~r£r…rŠrrr‘r¤rºr$rrNr*r1r1r1r2rvÞsÔ 4 ! W 6 * %*f | rvcCs2i}x(|jdƒD]}|jdƒ\}}|||<qW|S)NrÁú:)rA)Zattrs_headerr°Úattrr:Úvalr1r1r2rŸs rŸcCs@tt|ƒƒ}t|jƒƒ}t|ƒ}|ddkr8|dd…}t|ƒS)NrÚ rr¹r¹)rrrÚdigestr)rQÚmZ base64md5r1r1r2rësrëi)[Ú __future__rrrrðrZr“r:rr£Zxml.saxrÚsocketrr©rwrrrr rr rÚImportErrorÚurllib.parseÚbase64rr rËÚhashlibrÚ BaseUtilsrrrrrrÚUtilsrrrrrrrrrrr r!r"r#Ú ExceptionsZ MultiPartr%r&r'ÚCryptor(r)r*r+r,Zctypesr-r7ZMagicr/r3Ú TypeErrorrZ MAGIC_MIMEÚloadÚAttributeErrorrrÔr@Z error_strr9r8rErÍrÚ__all__r9rGrvÚappendrŸrër1r1r1r2Ú sª ( P>