3
bk              $   @   s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZmZ ddlmZ yd dlZd dlZW n4 ek
r   ejjd ejj  eje Y nX yd dlZW n ek
r   d dljZY nX y d dlmZmZmZmZmZ  W n2 ek
r8   d dlmZmZmZmZmZ  Y nX ddl!m"Z" dd	l#m#Z# dd
l$m%Z%m&Z&m'Z'm(Z(m)Z) ye* W n e+k
r   e,Z*Y nX dd Z-dd Z.dd Z/G dd de0Z1G dd de0ZG dd de0Z2dS )    )absolute_importN)debugwarning   )	EX_OSFILEa:  
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
ImportError trying to import dateutil.parser and dateutil.tz.
Please install the python dateutil module:
$ sudo apt-get install python-dateutil
  or
$ sudo yum install python-dateutil
  or
$ pip install python-dateutil
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
)NoOptionErrorNoSectionErrorMissingSectionHeaderErrorParsingErrorConfigParser)Progress)
SortedDict)s3_quotegetTreeFromXmlgetDictFromTreebase_unicodisedateRFC822toPythonc             C   s8   t | tkr| j dkS t | tkr0| dkr0dS dS dS )	zsCheck to see if a string is true, yes, on, or 1

    value may be a str, or unicode.

    Return True if it is
    trueyeson1TFN)r   r   r   r   )typeunicodelowerbool)value r   I/oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/Config.pyis_bool_trueI   s
    r   c             C   s8   t | tkr| j dkS t | tkr0| dkr0dS dS dS )	ztCheck to see if a string is false, no, off, or 0

    value may be a str, or unicode.

    Return True if it is
    falsenooff0FTN)r   r    r!   r"   )r   r   r   r   )r   r   r   r   is_bool_falseX   s
    r#   c             C   s   t | pt| S )z)Check a string value to see if it is bool)r   r#   )r   r   r   r   is_boolg   s    r$   c            	   @   s<  e Zd ZdZg Zi ZdZdZdZdZ	dZ
dZdZdZdZdZdZejZejj ZejZd;Zd<Zd
Zd
Zd
Ze ddZ!d
Z"d
Z#dZ$d
Z%d
Z&dZ'd
Z(d
Z)dZ*dZ+dZ,g Z-g Z.dZ/dZ0d
Z1d
Z2dZ3dZ4dddddddddg	Z5d
Z6d
Z7d
Z8d=Z9d>Z:ded< d
Z;dZ<dZ=dZ>dZ?dZ@dZAdZBdZCdZDdZEdZFdZGdZHdZIdZJdZKdZLd?ZMdZNd dgZOg ZPg ZQi ZRi ZSeTjU pfd!ZVd"ZWdZXd
ZYdZZd
Z[d#Z\d
Z]d
Z^dZ_d$Z`dZad%Zbg Zcg ZddZedZfg ZgdZhdZidZjd
Zkd&Zld
Zmd
ZndZodZpd
Zqd
Zrd
Zsd'Ztd
ZudZvd(Zwd@d)d*ZxdAd+d,Zyd-d. Zzd/d0 Z{d1d2 Z|d3d4 Z}d5d6 Z~d7d8 Zd9d: ZdS )BConfigN Tzs3.amazonaws.comz%(bucket)s.s3.amazonaws.comzsdb.amazonaws.comzcloudfront.amazonaws.com@   i   F)Zignore_caser   Standardi8  unameuidgnamegidatimemtimectimemodemd5z@[sync] Remove remote S3 objects when local file has been deleteddelete_removedz}%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)sz}%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)sZUSzbinary/octet-stream   i'  sizezUTF-8Znormali,  z
index.htmlz8http://%(bucket)s.s3-website-%(location)s.amazonaws.com/r   d      c             C   s   | j d krtj| | _ | j S )N)	_instanceobject__new__)self
configfile
access_key
secret_keyaccess_tokenr   r   r   r9      s    
zConfig.__new__c             C   s2  |r.y| j | W n0 tk
rD   dtjks8dtjkr@| j  Y nX |rZ|rZ|| _|| _|rj|| _d| _t	| jdkrtj
dptj
d}tj
dptj
d}tj
d	ptj
d
}|r|stdt|| _t|| _|rd| _t|| _n| j  | jr| jdkrtd | jr.| jdkr.tdd S )NAWS_CREDENTIAL_FILEAWS_PROFILEFr   ZAWS_ACCESS_KEYZAWS_ACCESS_KEY_IDZAWS_SECRET_KEYZAWS_SECRET_ACCESS_KEYZAWS_SESSION_TOKENZAWS_SECURITY_TOKENzRAWS_ACCESS_KEY environment variable is used but AWS_SECRET_KEY variable is missingTzCannot have server_side_encryption (S3 SSE) and KMS_key set (S3 KMS). KMS encryption will be used. Please set server_side_encryption to FalsezFKMS encryption requires signature v4. Please set signature_v2 to False)read_config_fileIOErrorosenvironaws_credential_filer<   r=   r>   _access_token_refreshlengetenv
ValueErrorr   role_configkms_keyserver_side_encryptionr   signature_v2	Exception)r:   r;   r<   r=   r>   Zenv_access_keyZenv_secret_keyZenv_access_tokenr   r   r   __init__   s<    

zConfig.__init__c             C   s  yt jjd}|rdttj  }dd||d}t jjd}|rpt|}|j }W dQ R X d|d	< ||d
< djdd |j D }d}t jjddkrt jjd}	|	s| j	}	d|	 }t
j|dd}
|
jdd|  |
j }|j }|jdkot|dkrt|}t|}|jdkr*|d d }n(|jdkrD|d d }ntd|j t jd|d   t jd!|d"  t jd#|d$  tt|d% }|tjd&d' | _tjjtjj | _ntnt
j d(dd}
|
jd)d* |
j }|j }|jdkrt|dkr|
jd)d+|j!d,  |
j }|jdkrt|j }t"j#|}t jdt|d   t jd!t|d"  t jd#t|d-  tt|d% }|tjd&d' | _tt|d. | _ntntW n    Y nX dS )/zL
        Get credentials from IAM authentication and STS AssumeRole
        ZAWS_ROLE_ARNzrole-session-%sZ
AssumeRolez
2011-06-15)ActionZVersionZRoleArnZRoleSessionNameZAWS_WEB_IDENTITY_TOKEN_FILENZAssumeRoleWithWebIdentityrP   ZWebIdentityToken&c             S   s$   g | ]\}}d |t |ddf qS )z%s=%sT)unicode_output)r   ).0kvr   r   r   
<listcomp>>  s   z&Config.role_config.<locals>.<listcomp>zsts.amazonaws.comZAWS_STS_REGIONAL_ENDPOINTSZregionalZ
AWS_REGIONzsts.%s.amazonaws.com   )hosttimeoutZPOSTz/?   r   ZAssumeRoleResponseZAssumeRoleResultZCredentialsZ!AssumeRoleWithWebIdentityResponseZAssumeRoleWithWebIdentityResultz.Unexpected XML message from STS server: <%s />r<   ZAccessKeyIdr=   ZSecretAccessKeyr>   ZSessionTokenZ
Expirationr3   )minutesz169.254.169.254ZGETz+/latest/meta-data/iam/security-credentials/z-/latest/meta-data/iam/security-credentials/%szutf-8TokenZLastUpdated)$rC   rD   getinttimeopenreadjoinitemsbucket_locationhttplibZHTTPSConnectionrequestZgetresponsestatusrG   r   r   tagrB   r%   update_optionr   r   datetime	timedelta_access_token_expirationnowdateutiltztzutc_access_token_last_updateZHTTPConnectiondecodejsonloads)r:   Zrole_arnZrole_session_nameparamsZweb_identity_token_filefZweb_identity_tokenZencoded_paramsZsts_endpointZregionZconnZrespZresp_contenttreeZresult_dictZcreds
expirationfilesr   r   r   rJ   )  s    

zConfig.role_configc             C   sj   | j rftjjtjj }| jr<|| jk r<| jr<| j|kr<d S y| j  W n t	k
rd   t
d Y nX d S )NzCould not refresh role)rF   rj   rm   rn   ro   rp   rl   rq   rJ   rN   r   )r:   rm   r   r   r   role_refresh}  s    

zConfig.role_refreshc                 s  yt jjd t jjd}|r6t jj|r6t| nt jj sFd S t td   t	j
 dt| ddd}|j }W d Q R X yFyjt	j| W n, tk
r   d| }jt	j| Y nX W n8 tk
r } ztd	 t|f W Y d d }~X nX tt jjd
d}td|  d fdd	}||dd}|rVt jdt| ||dd}|rzt jdt| ||dd d}	|	rt jdt|	 W n` tk
r }
 ztd|
j  W Y d d }
~
X n0 tk
r }
 ztd|  W Y d d }
~
X nX d S )Nz~/.aws/credentialsr?   zReading AWS credentials from %srencodingzUTF-8)r|   z
[default]
z*Error reading aws_credential_file (%s): %sr@   defaultzUsing AWS profile '%s'Tc                s   d }yj | |}W n tk
r } zf|r<td|j|j  |ry"|}d} j | |}td| W n" tk
r } zW Y d d }~X nX W Y d d }~X nX |rtd||   |S )NzLCouldn't find key '%s' for the AWS Profile '%s' in the credentials file '%s'r}   zLegacy configuratin key '%s' used, please use the standardized config format as described here: https://aws.amazon.com/blogs/security/a-new-and-standardized-way-to-manage-credentials-in-the-aws-sdks/zWFound the configuration option '%s' for the AWS Profile '%s' in the credentials file %s)r]   r   r   optionsectionr   )profilekeyZ
legacy_keyZprint_warningresulte)rE   configr   r   get_key  s*    
$
z+Config.aws_credential_file.<locals>.get_keyZaws_access_key_idZAWSAccessKeyIdr<   Zaws_secret_access_keyZAWSSecretKeyr=   Zaws_session_tokenFr>   z&Errno %d accessing credentials file %sz;Couldn't find AWS Profile '%s' in the credentials file '%s')T)rC   path
expanduserrD   r]   isfiler   PyConfigParserr   ior`   getattrra   ZreadfpStringIOr	   r
   rI   strr%   ri   rB   r   errnor   )r:   Zcredential_file_from_envfpZconfig_stringexcr   r   Zprofile_access_keyZprofile_secret_keyZprofile_access_tokenr   r   )rE   r   r   rE     s^    
" 

zConfig.aws_credential_filec             C   sX   g }xNt | D ]B}ttt|}|jds|tdtdtdfkrFq|j| qW |S )N_string*   T)dirr   r   r%   
startswithappend)r:   retvalr~   Zoption_typer   r   r   option_list  s    
zConfig.option_listc             C   s   t |}x6| j D ]*}|j|}|d k	r0|j }| j|| qW |jdr\| jd|jd |jdrx:|jdjdD ]&}|jdd\}}|j | j|j < qxW | jj| d S )N
acl_publicadd_headers,:r   )	r   r   r]   stripri   splitextra_headers_parsed_filesr   )r:   r;   cpr~   Z_optionr   r   r   r   r   rA     s    


zConfig.read_config_filec             C   s   t |jd|  d S )Nr}   )ConfigDumperdump)r:   streamr   r   r   dump_config  s    zConfig.dump_configc             C   sH  |d krd S t |jdr4| j|tj|dd  S |dkryt|}W nh tk
r   y2ytj| }W n t	k
r   tj
| }Y nX W n  tk
r   td| Y nX Y nX n|dkrJ|jds|jdrd}n|jd	s|jd
rd}nd}y(|rt|d d |> pt|}W n& tk
rF   td||f Y nX nttt|tdks|tt|d krt|rt|rd}n t|rd}ntd||f nttt|tdkryt|}W n& tk
r   td||f Y nX n<|dkr8|jdr |dd  }n|jdr8|dd  }tt|| d S )N$r   	verbosityz)Config: verbosity level '%s' is not valid	limitraterT   K
   mM   r   zFConfig: value of option %s must have suffix m, k, or nothing, not '%s'TFz8Config: value of option '%s' must be Yes or No, not '%s'r   z9Config: value of option '%s' must be an integer, not '%s'	host_basehost_bucketcloudfront_hostzhttp://   zhttps://   )r   r   r   )r   r   ri   rC   rH   r^   rI   logging_levelNamesAttributeError_nameToLevelKeyErrorendswithrN   r   r   r%   r$   r   r#   setattr)r:   r~   r   shiftr   r   r   ri     sV    
(


zConfig.update_optioni   i   r   r   i   )NNNN)NNNN)__name__
__module____qualname__r7   r   Z_docr<   r=   r>   rF   rl   rq   r   r   rK   Zsimpledb_hostr   r   WARNINGr   sysstdoutisattyprogress_meterr   Z
ProgressCRZprogress_class
send_chunkZ
recv_chunklist_md5long_listinghuman_readable_sizesr   r   forcerL   enableget_continueput_continue	upload_idskip_existing	recursiverestore_daysrestore_priorityr   
acl_grantsacl_revokes
proxy_host
proxy_portencryptdry_runadd_encoding_extspreserve_attrspreserve_attrs_listr2   delete_afterdelete_after_fetch
max_deletelimitdelay_updatesgpg_passphrasegpg_commandgpg_encryptgpg_decrypt	use_httpsca_certs_filessl_client_key_filessl_client_cert_filecheck_ssl_certificatecheck_ssl_hostnamerd   default_mime_typeguess_mime_typeuse_mime_magic	mime_typeenable_multipartmultipart_chunk_size_mbZmultipart_copy_chunk_size_mbZmultipart_max_chunkssync_checksexcludeincludedebug_excludedebug_includelocalegetpreferredencodingr|   urlencoding_modelog_target_prefixreduced_redundancystorage_classfollow_symlinkssocket_timeoutinvalidate_on_cfinvalidate_default_index_on_cf#invalidate_default_index_root_on_cfwebsite_indexwebsite_errorwebsite_endpointadditional_destinations
files_from
cache_filer   remove_headersexpiry_daysexpiry_dateexpiry_prefixrM   r   requester_paysstop_on_errorcontent_dispositioncontent_typestatsZuse_http_expectZsignurl_use_httpsZthrottle_maxZpublic_url_use_httpsconnection_poolingZconnection_max_ager9   rO   rJ   rz   rE   r   rA   r   ri   r   r   r   r   r%   l   s   



+Tar%   c               @   s>   e Zd Zg fddZg fddZdd Zdd Zdd
dZd	S )r   c             C   s   i | _ | j|| d S )N)cfg
parse_file)r:   filesectionsr   r   r   rO   R  s    zConfigParser.__init__c             C   s  t d|  t|tg kr"|g}d}tjd}tjd}tjd}tjd}tjd}tj|d| jd	d
d"}	x|	D ]}
|j|
s||j|
rq||j|
}|r|j d }||kpt	|dk}q||j|
}|o|r||j
 }|j|d r|d dd |d< | j|d |d  |d dkr^d|d d d t	|d d |d dd  f }n|d }t d|d |f  q|td||
f  q|W W d Q R X d S )NzConfigParser: Reading file '%s'Tz^\s*#.*z^\s*$z^\[([^\]]+)\]z$^\s*(?P<key>\w+)\s*=\s*(?P<value>.*)z^"(.*)"\s*$r{   r|   zUTF-8)r|   r   r   r   r   r<   r=   r   z%s...%d_chars...%srW      zConfigParser: %s->%sz!Ignoring invalid line in '%s': %sr   )r<   r=   r   r   )r   r   recompiler   r`   r]   matchgroupsrG   	groupdict__setitem__r   )r:   r  r  Zin_our_sectionZ	r_commentZr_emptyZ	r_sectionZr_dataZr_quotesr   lineZ
is_sectionr   Zis_datadataZprint_valuer   r   r   r  V  s<    







4zConfigParser.parse_filec             C   s
   | j | S )N)r  )r:   namer   r   r   __getitem__w  s    zConfigParser.__getitem__c             C   s   || j |< d S )N)r  )r:   r  r   r   r   r   r  z  s    zConfigParser.__setitem__Nc             C   s   || j kr| j | S |S )N)r  )r:   r  r}   r   r   r   r]   }  s    

zConfigParser.get)N)r   r   r   rO   r  r  r  r]   r   r   r   r   r   Q  s
   !r   c               @   s   e Zd Zdd Zdd ZdS )r   c             C   s
   || _ d S )N)r   )r:   r   r   r   r   rO     s    zConfigDumper.__init__c             C   s   | j jd|  x|j D ]|}t||}|dkrt|try2ytj| }W n tk
rh   tj	| }Y nX W n t
k
r   Y nX | j jd||f  qW d S )Nz[%s]
r   z%s = %s
)r   writer   r   
isinstancer^   r   r   r   _levelToNamer   )r:   r   r   r~   r   r   r   r   r     s    

zConfigDumper.dumpN)r   r   r   rO   r   r   r   r   r   r     s   r   )3
__future__r   r   rj   r   r  rC   r   r   rs   r_   r   r   	ExitCodesr   dateutil.parserrn   Zdateutil.tzImportErrorstderrr  flushexitre   Zhttp.clientZclientconfigparserr   r   r	   r
   r   r   r&   r   r   	BaseUtilsr   r   r   r   r   r   	NameErrorr   r   r#   r$   r8   r%   r   r   r   r   r   <module>	   sV   

 "
   h1