3
bf                 @   s*  d dl mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZmZmZ dd	lmZ dd
lmZ d dlmZmZmZ d dlZd dlZd dlZd dlZd dlZd dlZejd!kZdddgZdd Zdd Zdd Z dd Z!dd Z"dd Z#d"ddZ$ddi fddZ%d d Z&dS )#    )absolute_import   )S3)Config)S3Uri)FileDict)dateS3toUnixdateRFC822toUnix)	unicodisedeunicodisedeunicodise_sreplace_nonprintables)ParameterError)	HashCache)debuginfowarningN   fetch_local_listfetch_remote_listcompare_filelistsc             c   s   yt jt| }W n tk
r&   dS X g g  }}xN|D ]F}t|}t jjtt jj| |rtt| |s~|j	| q8|j	| q8W | ||fV  xB|D ]:}t jj| |}t jj
t|sxt|D ]
}|V  qW qW dS )z_
    Reimplementation of python's os.walk to nicely support unicode in input as in output.
    N)oslistdirr   	Exceptionr
   pathisdirjoinhandle_exclude_include_walk_dirappendislink_os_walk_unicode)topnamesdirsnondirsnamenew_pathx r(   L/oak/stanford/groups/akundaje/marinovg/programs/s3cmd-master/S3/FileLists.pyr    !   s"    



r    c                s   t  }d}tjj| |d}td|  d}xH|jD ]> t fdd	|D sNq2 j|r2d
}td|j    P q2W |rxZ|j	D ]P t fdd	|D sqtd| j
f   j|rd}td|j    P qW |rtd|  ntd|  |S )a
  
    Should this root/dirname directory be excluded? (otherwise included by default)
    Exclude dir matches in the current directory
    This prevents us from recursing down trees we know we want to ignore
    return True for excluding, and False for including
    /)$/)\Z\/$	\/\Z(?ms) z	CHECK: %rFc             3   s   | ]} j j|V  qd S )N)patternendswith).0dp)rr(   r)   	<genexpr>I   s    z2handle_exclude_include_walk_dir.<locals>.<genexpr>TzEXCL-MATCH: '%s'c             3   s   | ]} j j|V  qd S )N)r/   r0   )r1   r2   )r3   r(   r)   r4   S   s    zINCL-TEST: %s ~ %szINCL-MATCH: '%s'zEXCLUDE: %rzPASS: %r)r*   r+   r,   r-   )r   r   r   r   r   excludeanysearchdebug_excludeincluder/   debug_include)rootdirnamecfgZdirectory_patternsdexcludedr(   )r3   r)   r   :   s2    

r   c       
      c   s   t jjt| stt| g}xt| D ]\}}}tt jjt|}xp|D ]h}t jj	||}tt jjt|}t jj
t|rL||ks|j|t jj rtd|  qL|j| qLW q(W x0|D ](}	x"t|	D ]\}}}|||fV  qW qW dS )z
    Walk filesystem, following symbolic links (but without recursion), on python2.4 and later

    If a symlink directory loop is detected, emit a warning and skip.
    E.g.: dir1/dir2/sym-dir -> ../dir2
    z+Skipping recursively symlinked directory %sN)r   r   r   r   AssertionErrorsetr    r
   realpathr   r   
startswithsepr   add)
r   Zwalkdirsdirpathdirnames	filenamesZreal_dirpathr<   currentZreal_currentZwalkdirr(   r(   r)   _fswalk_follow_symlinksb   s    


rJ   c             c   s(   x"t | D ]\}}}|||fV  q
W dS )z\
    Directory tree generator

    path (str) is the root of the directory tree to walk
    N)r    )r   rF   rG   rH   r(   r(   r)   _fswalk_no_symlinksz   s    rK   c             C   s   t d t }tdd}x| j D ]}t d|  d}x0|jD ]&}|j|r>d}t d|j|   P q>W |rx0|jD ]&}|j|rtd}t d|j|   P qtW |rt d|  | | ||< | |= q"q"t d	|  q"W | |fS )
NzApplying --exclude/--includeF)ignore_casez	CHECK: %sTzEXCL-MATCH: '%s'zINCL-MATCH: '%s'zEXCLUDE: %szPASS: %r)	r   r   r   keysr5   r7   r8   r9   r:   )src_listr=   exclude_listfiler?   r3   r(   r(   r)   filter_exclude_include   s0    


rQ   c             C   sD  dd }i }x| j D ]}zd }|dkr>tjtjj ddd}nNytjt|dd}W n6 tk
r } ztd||j	f  wW Y d d }~X nX xb|D ]Z}t
|j }tjjtjj||}t
tjjt|}t
tjjt|}	||||	 qW W d |r|j  X qW g }
x2t|D ]&}|| }|j  |
j|g |f qW |
S )	Nc             S   s&   || kr|g| |< n| | j | d S )N)r   )r>   keyvaluer(   r(   r)   _append   s    z(_get_filelist_from_file.<locals>._append-r3   F)modeclosefd)rV   zJ--files-from input file %s could not be opened for reading (%s), skipping.)
files_fromioopensysstdinfilenor   IOErrorr   strerrorr
   stripr   r   normpathr   r<   basenameclosesortedsortr   )r=   
local_pathrT   filelistfnamefeliner<   rb   resultrR   valuesr(   r(   r)   _get_filelist_from_file   s4    
rn   Fc                s\   fdd}fdd}fdd}t  t  jrptjjtjrptjjtjdkrp jj nt	d g }t
d	d
}d	}t| tttfkr| g} |d krj}xN| D ]F}	t|	}
|
jdkstd|	 |
j r| rtd|	 |j|
 qW x|D ]}
|||
 \}}qW t|dkr2d	}t|\}}||}| | ||||fS )Nc       	         st  t | }d}td|  d}xP| D ]F}|d7 }|d dkrNtd||f  |dkrXq$| | d }ytjtjt|}W n4 tk
r } z|jtjkrw$n W Y d d }~X nX | | j	|j
|j|j|j|j|j|d ||j
7 }d	jkr$ j|j|j|j|j
}|d krTy| j|}W n tk
r8   w$Y nX  j|j|j|j|j
| | j||j|j||j
 q$W |S )
Nr   zYRunning stat() and reading/calculating MD5 values on %d files, this may take some time...r   i  z[%d/%d]rU   	full_name)sizemtimedevinodeuidgidsrmd5)lenr   r   stat_resultstatr   OSErrorerrnoENOENTupdatest_sizest_mtimest_devst_inost_uidst_gidsync_checksrw   get_md5r^   rE   record_hardlink)	loc_listZlen_loc_list
total_sizecounterrelative_filero   rv   rj   rw   )cacher=   r(   r)   _fetch_local_list_info   sH    



z0fetch_local_list.<locals>._fetch_local_list_infoc                s  t d |j dkrfytj }tj }W n tk
rD   d}d}Y nX ddd||ddd| d< | dfS |j r|j }|j }rt j	rt
 |}d}q܈ jrt|}nt|}d}n"d}|j }|g |j gfg}d}x|D  ]\}	}
}|	j||d}x|D ]}tjj|	|}tjjt|sJtjjt|rtd	|  qtjjt|rt jsttd
|  qtjj||}tjjdkrdj|jtjj} jdkrt|}|jdr|dd  }d|i| |< qW qW | |fS )Nz Compiling list of local files...rU   r   r   )ro   rp   rq   rt   ru   rr   rs   TFr.   z"Skipping over non regular file: %szSkipping over symbolic link: %s/normalz./   ro   r   )r   rb   r   geteuidgetegidr   r   r   rx   rX   rn   follow_symlinksrJ   rK   r<   replacer   isfiler   existsr   r   rD   spliturlencoding_moder   rC   )r   Z	local_urir   rt   ru   Z
local_baserf   rg   single_filer;   r#   filesZrel_rootri   ro   r   )r=   is_srcr(   r)   _get_filelist_local   sf    



z-fetch_local_list.<locals>._get_filelist_localc                s    j rt jdkr| j  tr*|j }n|j }x:|D ]2}|jdd}|sNq8| j|d ||d |d  q8W | j	  | j
 j  d S )Nr   rs   rr   rq   rp   )
cache_filerx   rX   Zmark_all_for_purgePY3rm   
itervaluesgetZunmark_for_purgepurgeZsave)r   
local_listZlocal_list_val_iterZf_infors   )r=   r(   r)   _maintain_cache5  s    

z)fetch_local_list.<locals>._maintain_cacher   z6Cache file not found or empty, creating/populating it.F)rL   rP   z.Expecting filename or directory instead of: %sz)Use --recursive to upload a directory: %sr   )r   r   r   r   r   r   r   getsizeloadr   r   typelisttuplerA   	recursiver   r   r   r   rx   rQ   )argsr   r   r   r   r   Z
local_urisr   r   arguriZlist_for_urirO   r   r(   )r   r=   r   r)   r      s<    -?.




c                s   fdd d fdd	}t  g }tdd}t| tttfkrH| g} |d krVj}x4| D ],}t|}|jdks~td	| |j	| q\W d
}	|rxP|D ]H}||dd\}
}|	|7 }	x*|
D ]"}|
| ||< |j
||
j| qW qW n x|D ]}|j }tjd|dd}t|dkr|\}}d|k}|t||d\}
}|	|7 }	x|
D ]*}tjj|
| d |rP|
| ||< qPW qttjjt|j }|std|j  ||j |j d}|r҈ || |||< |jd}|r|j
|| |	|jdd
7 }	qW t|\}}|||	fS )Nc                s   t  j| }|jdsd S |jt|d d |d d jdt|d d d y*|d d }|jd|i td	|  W n tk
r   Y nX d S )
Nheaderszcontent-lengthetagz"'zlast-modified)rp   rw   	timestampzs3cmd-attrsrw   zretreived md5=%s from headers)	r   object_infor   r~   intr`   r	   r   KeyError)r   remote_itemresponserw   )r=   r(   r)   _get_remote_attribst  s    
z.fetch_remote_list.<locals>._get_remote_attribsTc          
      s  t d|   tjd}d}tt }|j| j | j |d}| j  }}| }|dkr|d dkr|d |jdd  }t	d| j |f } t
|}	td	d
}
d	}x|d D ]}|d |ko|d d dkrttjjt|d }|j }d}td	d
}
n|d |	d  }| j | }|j|r4td qt|d t|d |d jd|d || d d d|
|< d|
| d kr t	||
|  |
| d }|
j|| |t|d 7 }|rP qW |
|fS )Nz*Retrieving list of remote files for %s ...z\A\s*\Zr   )prefixr   
uri_paramsr.   r   r   z
s3://%s/%sF)rL   r   KeyTz(Empty object name on S3 found, ignoring.SizeLastModifiedETagz"')rp   r   rw   
object_keyobject_uri_strbase_urirr   rs   rU   rw   r   r   )r   recompiler   r   bucket_listbucketobjectrfindr   rx   r   r
   r   r   rb   r   r   matchr   r   r   r`   
record_md5)
remote_urir   Zempty_fname_rer   s3r   Zrem_base_originalZrem_baseZremote_uri_originalZrem_base_lenZrem_listZ	break_nowr   rR   r   rw   )r   r   r(   r)   _get_filelist_remote  sR    





z/fetch_remote_list.<locals>._get_filelist_remoteF)rL   r   z Expecting S3 URI instead of '%s'r   )r   z\*|\?r   )maxsplitr   r   r   z3Expecting S3 URI with a filename or --recursive: %s)r   r   r   rw   rp   )T)r   r   r   r   r   rA   r   r   r   r   r   r   r   r   r   rx   globfnmatchr
   r   r   rb   r   r   r   rQ   )r   require_attribsr   r   r   Zremote_urisremote_listr   r   r   Z
objectlistZtmp_total_sizerR   uri_strZwildcard_split_resultr   restZneed_recursionr   rw   rO   r(   )r   r=   r   r)   r   s  s`    B







c                sv  dd } fdd}|dko$|dk s,t td t  tdd}g }td||||f  x| j D ]}td	|  |kr jrtd
|  | |= |= qly|| |||}	W n> ttfk
r   td|  t	d|  | |= |= wlY nX |	rtd|  | |= |= ny| j
|}
W n tk
rD   d }
Y nX |
d k	r|
jkrj|
}td||f  |j| | |||
f | |= |= n$j||
 | | ||< | |= |= qly| j
|}
W n tk
r   d }
Y nX j|
}|d k	r,td||f  |j| | |||
f | |= qlj||
 qlW x,j D ] }|| ks^||krF|= qFW | ||fS )Nc             S   s   | rdp
dS )NZremotelocalr(   )Z	is_remoter(   r(   r)   __direction_str	  s    z*compare_filelists.<locals>.__direction_strc       	         s\  d}|| ko|ks2t d||| k|kf  dS d jkrd| krd| | kr| d | | d krtd|| | d | d f  d}d jk}|r|dkrd| | d ks|dkrd| d krd}t d|  |o|rXy| j|}j|}W n4 ttfk
r6   td	|  td
|   Y nX ||krXd}td|||f  |S )z@Return True if src_list[file] matches dst_list[file], else FalseTzE%s: does not exist in one side or the other: src_list=%s, dst_list=%sFrp   z'xfer: %s (size mismatch: src=%s dst=%s)rw   rU   zdisabled md5 check for %szIGNR: %s (disappeared)z%s: file disappeared, ignoring.z&XFER: %s (md5 mismatch: src=%s dst=%s))r   r   r   r   r^   r{   r   )	rN   Zdst_lst
src_remote
dst_remoterP   Zattribs_matchZcompare_md5src_md5Zdst_md5)r=   dst_listr(   r)   _compare  s4    
"
0


z#compare_filelists.<locals>._compareFzVerifying attributes...)rL   z)Comparing filelists (direction: %s -> %s)z	CHECK: %szIGNR: %s (used --skip-existing)zIGNR: %s (disappeared)z%s: file disappeared, ignoring.zIGNR: %s (transfer not needed)zDST COPY src: %s -> %szDST COPY dst: %s -> %s)r@   r   r   r   r   rM   skip_existingr^   r{   r   r   by_md5find_md5_oner   r   )rN   r   r   r   r   r   update_list
copy_pairsr   Z	same_filerw   dst1ri   r(   )r=   r   r)   r     sr    '







)r   r   )FN)'
__future__r   r   r   r   r   	BaseUtilsr   r	   Utilsr
   r   r   r   
Exceptionsr   r   loggingr   r   r   r   r[   r   r   r|   rY   version_infor   __all__r    r   rJ   rK   rQ   rn   r   r   r   r(   r(   r(   r)   <module>	   s8   

(	&
 - 