B
    XM\+                 @   s  d dl mZmZmZmZ d dlZd dlmZmZ d dl	m	Z	 d dl
Z
d dlmZ d dlZd dlmZmZmZmZmZmZmZmZmZmZ d dlmZmZ d dlZd dlZd dlmZ d dl Z d d	l!m"Z" d d
l#m$Z$ d dl%Z%d dl&m'Z'm(Z( d dl)m)Z) d dl*Z*d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1Z1d dl2Z2d dl3m4Z4 d dl5Z5d dl6Z6ddl7m8Z8m9Z9 ddl8m:Z:m;Z;m<Z<m=Z= ddl8m>Z>m?Z?m@Z@mAZA ddl8mBZBmCZC ddl9mDZDmEZEmFZFmGZGmHZHmIZI yd dlJmKZK W n( eLk
r   d dlJmMZM eMfZKY nX yd dlNmOZO W n ePk
r$   eQZOY nX eEeRZSyd dlTmUZUmVZV W nF ePk
r   d dlWmXZXmYZYmVZV d dlZm[Z[ G dd deXZUY nX yd dlJm\Z\m]Z] W nR ePk
r   d d!d!d!d!d"d#d#d$d%d&d'd(d)d*d+d,d-Z\e^e\Z]e^e\_ Z`Y nX d aadabd.acg adi aeefed/r(eg dkr(eg ndZhd0Zid1Zjd2Zkyd d3llmmZmmnZnmoZo W n* ePk
rz   d d3lpmmZmmnZnmoZo Y nX d4d5 Zqdvd8d9Zrd:d; Zsd6ddehd6d6ddd7f	d<d=Ztd>d? Zud@dA ZvdZwdZxdBZydCZzdDdE Z{dFdG Z|dHdI Z}dwdJdKZ~dLdM ZdNdO ZdPdQ ZdRdS ZdTdU ZdVdW ZdXdY ZdZd[ Zd\d] Zd^d_ Zd`da Zdbdc Zddde Zdfdg Zdhdi Zdjdk Zdldm Zdndo Ze1jdpdq Zdrds ZG dtdu dueZdS )x    )absolute_importdivisionprint_functionunicode_literalsN)OrderedDictdefaultdict)datetime)Number)
abspathbasenamegetmtimegetsizeisdirisfilejoinlexistssplitextdirname)copy2move)
gettempdir)uuid4)ensure_binary)EnvironmentPackageLoader)tqdm)ConstructorError)ParserError)ScannerError)partial   )conda_interfaceutils)	MatchSpecVersionOrderhuman_bytescontext)
CondaErrorCondaHTTPError	get_indexurl_path)downloadTemporaryDirectory)glob
get_loggerFileNotFoundErrorPermissionErrortar_xf_filetar_xf_getnames)CONDA_TARBALL_EXTENSIONS)CONDA_TARBALL_EXTENSION)JSONDecodeError)ThreadLimitedThreadPoolExecutoras_completed)ThreadPoolExecutor_baser7   )	_WorkItemc                   s&   e Zd Zd fdd	Zdd Z  ZS )r6   
   c                s   t t| | d S )N)superr6   __init__)selfZmax_workers)	__class__ 0lib/python3.7/site-packages/conda_build/index.pyr=   D   s    z(ThreadLimitedThreadPoolExecutor.__init__c          	   O   sz   | j j | jrtdt }t||||}| j| y|   W n& tk
rj   t	| j
dkrdn Y nX |S Q R X dS )a-  
            This is an exact reimplementation of the `submit()` method on the parent class, except
            with an added `try/except` around `self._adjust_thread_count()`.  So long as there is at
            least one living thread, this thread pool will not throw an exception if threads cannot
            be expanded to `max_workers`.

            In the implementation, we use "protected" attributes from concurrent.futures (`_base`
            and `_WorkItem`). Consider vendoring the whole concurrent.futures library
            as an alternative to these protected imports.

            https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131  # NOQA
            https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124
            z*cannot schedule new futures after shutdownr   N)Z_shutdown_lock	_shutdownRuntimeErrorr9   ZFuturer:   Z_work_queueZputZ_adjust_thread_countlenZ_threads)r>   fnargskwargsfwr@   r@   rA   submitG   s    z&ThreadLimitedThreadPoolExecutor.submit)r;   )__name__
__module____qualname__r=   rJ   __classcell__r@   r@   )r?   rA   r6   B   s   r6   )NAMESPACES_MAPNAMESPACE_PACKAGE_NAMESpythonrerlangjavajulialatexluaZjsperlphprubyZm2Zm2w64)rQ   rR   zr-basezmro-basezmro-base_implrS   rT   ZopenjdkrU   rV   rW   ZnodejsrX   rY   rZ   zm2-basezmsys2-conda-epoch 	cpu_counti0*  z.lock)zlinux-64zlinux-32zlinux-ppc64lezlinux-armv6lzlinux-armv7lzlinux-aarch64zwin-64zwin-32zosx-64zzos-znoarch)concatconcatvgroupbyc          
   C   sj   t  Z}tj|d}t| | y"t|}t|}W d Q R X W n tk
rZ   i }Y nX W d Q R X |S )Nzchanneldata.json)	r,   ospathr   r+   openjsonloadr5   )channel_urlZtdZtfrH   datar@   r@   rA   _download_channeldata   s    

rh   FTc                s|  d}	t t|}|st|}tj|| d}
tj|
rFtj|
}	|sptj|
rpt	| ksp|	t
kspt|krrt |}tjjtg }|rttjtj|d}n4|rttjtj|d}nttjtjd |d}tj}| ~ tj|rt|  fdd|D } |kr|d  t||  t||d tdd	 }| l | d
krVtj} yt|| dd| daW n< t k
r   d|kr|!d t||dd| daY nX W d Q R X dd t" D }i }x||D ]r}|j#dkr|j$}tj%r |&d}nBtj'|j$sBtj(tjtjj)|j$rBtjtjj)|j$}tj||j*d}d}d}x||k ry,t+|d}t,-|t.|j*< W d Q R X P W n, t/t0fk
r   t12d |d7 }Y nX q^W n<t3j4syt5|j6d t.|j*< W n t k
r   wY nX |j6t3j7krt.8|j*r|8di }|9t.|j*  ||d< qW |t.d< W d Q R X tj|
a
| a	|att
t.fS )Nr   zrepodata.json)loggersr    c                s   g | ]}|d kr|n qS )Zlocalr@   ).0Zurl)
local_pathr@   rA   
<listcomp>   s    z#get_build_index.<locals>.<listcomp>)verbosec               s   s   d V S )Nr@   r@   r@   r@   rA   <lambda>   s    z!get_build_index.<locals>.<lambda>r]   F)channel_urlsZprependZ	use_localZ	use_cacheplatformdefaultsc             S   s   h | ]
}|j qS r@   )channel)rj   recr@   r@   rA   	<setcomp>   s    z"get_build_index.<locals>.<setcomp>file/zchanneldata.jsonr;   zr+g?z/channeldata.jsonpackages):listr"   Zensure_listr   ra   rb   r   r   r   local_subdirlocal_index_timestampcached_channelsLoggingContextZdefault_loggersrK   r   loggingDEBUGZWARNZCRITICALcapturer   r*   insert_ensure_valid_channelupdate_index
contextlibcontextmanagerr!   subdirr)   cached_indexr(   removevaluesZschemelocationZon_winlstripisabsexistssepnamerc   rd   re   channel_dataIOErrorr5   timeZsleepr&   Zofflinerh   Zbase_urlZdefault_channelsgetupdate)r   Zbldpkgs_dirZoutput_folderZclear_cacheZomit_defaultsro   debugrm   rG   mtimeZ
index_fileZurlsri   Zlog_contextr   Zexpanded_channelsZsuperchannelrr   r   Zchanneldata_fileZretryZmax_retriesrH   rw   r@   )rk   rA   get_build_index   s    








r   c             C   s:   x4|dhD ](}t j| |}t j|s
t | q
W d S )Nr]   )ra   rb   r   r   makedirs)Zlocal_folderr   Zfolderrb   r@   r@   rA   r   '  s    r   c
          	   C   s\   t j| \}
}|tkr<|	r&td t|
||||||dS t| ||||dj||||dS )a  
    If dir_path contains a directory named 'noarch', the path tree therein is treated
    as though it's a full channel, with a level of subdirs, each subdir having an update
    to repodata.json.  The full channel will also have a channeldata.json file.

    If dir_path does not contain a directory named 'noarch', but instead contains at least
    one '*.tar.bz2' file, the directory is assumed to be a standard subdir, and only repodata.json
    information will be updated.

    zThe update_index function has changed to index all subdirs at once.  You're pointing it at a single subdir.  Please update your code to point it at the channel root, rather than a subdir.)	check_md5channel_namethreadsrm   progresshotfix_source_repo)subdirsr   deep_integrity_check)patch_generatorrm   r   r   )	ra   rb   splitDEFAULT_SUBDIRSlogwarnr   ChannelIndexindex)Zdir_pathr   r   r   r   rm   r   r   r   r   	base_pathr   r@   r@   rA   r   .  s    

r   c          	   C   s   |  dr| d }ntt }x>|  dg D ].}y|t|j W q( tk
rT   Y q(X q(W |t@ }t|dkr|t|	  }nd}|| d< |  dsd| d kr| d 
dd\}}||kr| d | d< || d< ||  d| d | d fS )	N	namespacedependsr    globalZnamespace_in_name-r   name_in_channel)r   setaddr#   r   r'   rP   rD   rO   popr   )infor   Zdepends_namesspecZspacesZnamespace_prefixZreduced_namer@   r@   rA   _determine_namespaceH  s&    


r   c             C   s   t | } | dkr| d } | S )Nl   A i  )int)	timestampr@   r@   rA   _make_secondsb  s    r   zrepodata.json)descriptionZdev_urlZdoc_urlZdoc_source_urlhomelicensereference_packageZ
source_urlZsource_git_urlZsource_git_tagZsource_git_revsummaryversionr   icon_url	icon_hashrun_exportsbinary_prefixtext_prefixz
activate.dzdeactivate.dpre_link	post_link
pre_unlinkZtagsZidentifierskeywordsrecipe_origincommitsc             C   sZ   || krVy| |   dd| |< W n0 tk
rT   | | d   dd| |< Y nX d S )N
 r   )stripreplaceAttributeError)record
field_namer@   r@   rA   _clear_newline_chars  s
    r   c             C   s   | dg  tj|di |di ddd x:|ddD ]*}d|d | d< |d | d	 d
 q<W x6|ddD ]&}|d |d }|rx|d | qxW |d   |S )Nremovedrw   F)mergeZadd_missing_keysZrevoker@   Trevokedr   Zpackage_has_been_revokedr   )
setdefaultr"   Zmerge_or_update_dictr   appendr   sort)r   repodatainstructionsrE   poppedr@   r@   rA   _apply_instructions  s    r   c              C   sN   dd } dd }t tddd}t|jd< | |jd	< ||jd
< d|_d|_|S )Nc             S   s<   t | tr2| dkr| d } t| jtdd} | |S )Nl   A i  UTC)tzinfo)
isinstancer	   r   Zutcfromtimestampr   pytztimezonestrftime)ZdtZ	dt_formatr@   r@   rA   _filter_strftime  s
    
z1_get_jinja2_environment.<locals>._filter_strftimec             [   sP   |rHd |g}|d |  |dd | D 7 }d d|| S | S d S )Nz
href="{0}"z	alt="{0}"c             S   s   g | ]\}}d  ||qS )z	{0}="{1}")format)rj   kvr@   r@   rA   rl     s    zE_get_jinja2_environment.<locals>._filter_add_href.<locals>.<listcomp>z<a {0}>{1}</a>r   )r   r   itemsr   )textlinkrG   Zkwargs_listr@   r@   rA   _filter_add_href  s    z1_get_jinja2_environment.<locals>._filter_add_hrefZconda_buildZ	templates)loaderr%   r   Zadd_hrefT)r   r   r%   filterstrim_blocksZlstrip_blocks)r   r   environmentr@   r@   rA   _get_jinja2_environment  s    	


r   c          	   C   s   t t tt }|st|}t|d}|| |rB|d W d Q R X t| rvt	|t	| krvt
| dS yt||  W n* tk
r   t||  t
| Y nX dS )Nwb   
FT)r   r   strr   r   rc   writer   r"   md5_filera   unlinkr   r0   Z	copy_into)rb   contentwrite_newline_endcontent_is_binaryZ	temp_pathfhr@   r@   rA   _maybe_write  s"    

r   c       
      C   s   t d| }g }x| D ]}yt d|}t|tdd }t d|| }t|d }t|| dd dd }ttdd	 |D |d
< d|d |d f |d< || W q tk
r }	 ztd	||	 W d d }	~	X Y qX qW |S )Nr   r   )keybuild_numberc             S   s   | d S )Nr   r@   )xr@   r@   rA   rn     s    z8_gather_channeldata_reference_packages.<locals>.<lambda>c             s   s   | ]}|d  V  qdS )r   Nr@   )rj   rs   r@   r@   rA   	<genexpr>  s    z9_gather_channeldata_reference_packages.<locals>.<genexpr>r   z%s/%sr   rE   r   zKgroup {} failed to gather channeldata.  Error was {}.  Skipping this one...)
r`   r   sortedr$   r   r   KeyErrorr   r   r   )
all_repodata_packagesgroupsreference_packagesgroupZversion_groupsZlatest_versionZbuild_number_groupsZlatest_build_numberZref_pkger@   r@   rA   &_gather_channeldata_reference_packages  s     

(r  c             C   s   dd |  D }i }|| tt}x| D ]z}|| }xl|d   D ]\}t|\}	}
}|	d | }|
|kr|||
 kr||
 ||
  ||
 | qH|||
< qHW q.W ||fS )Nc             S   s,   i | ]$}| d i  D ]\}}||qqS )external_dependencies)r   r   )rj   Zpir   namekeyr@   r@   rA   
<dictcomp>  s   z$_collect_namemap.<locals>.<dictcomp>rw   :)r   r   r   r   r   r   )r   patched_repodatapatch_instructionsr  namemapambiguous_namekeysr   r   r   r   r   r   r  r@   r@   rA   _collect_namemap  s     

r  c             C   s8  | r4t dd }x\|D ]T}|| }xF|d  D ]6\}}|d | kr2||d  |d  |d |  q2W qW dg}xt|D ]}	|d|	  xt||	 D ]r}
|d	|
  x^t||	 |
 D ]J}|d
|  |d\}}|| d |d}|r|| d | qW qW qW |d td| dS )ab  
    The following packages ambiguously straddle namespaces and require metadata correction:
        package_name:
        namespace1:
            - subdir/fn1.tar.bz2
            - subdir/fn2.tar.bz2
        namespace2:
            - subdir/fn3.tar.bz2
            - subdir/fn4.tar.bz2

    The associated packages are being removed from the index.
    c               S   s   t tS )N)r   rx   r@   r@   r@   rA   rn     s    z-_warn_on_ambiguous_namekeys.<locals>.<lambda>rw   r   r   rv   z`WARNING: The following packages ambiguously straddle namespaces and require metadata correction:z  %s:z    %s:z
      - %sNr   r[   r   )	r   r   r   r   r   r   r   r   r   )r
  r   r  abcr   r   rE   r   builderpackage_namer   	subdir_fnr   r@   r@   rA   _warn_on_ambiguous_namekeys  s(    
*
r  c       
      C   s   t js
|S t|}t|dr&|jr&|S |j|krL||j |d |   |S ||j }|dd\}}	yt|||	d}W n  tk
r   t||	d}Y nX |	 S d S )Nr   rv   r  r    )r   r   )r   )
r!   Zconda_47r#   hasattrr   r   r   r   r'   Zconda_build_form)
rE   r   Zdep_strr	  missing_dependenciesr   r   r  r   r   r@   r@   rA   _add_namespace_to_spec4  s    

r  c             C   s6   t |}| |r.| d t|  } | d} | }|S )N_)r   endswithrD   rstrip)buildr   Zbuild_number_as_stringZbuild_stringr@   r@   rA   _make_build_stringL  s    

r  c             C   s   | rddg}xzt | D ]n}|d|  xZt | | D ]J}|d|  |d\}}|| d |d}|r6|| d | q6W qW |d	 |d
 td| dS )aI  
    The following dependencies do not exist in the channel and are not declared
    as external dependencies:

    dependency1:
        - subdir/fn1.tar.bz2
        - subdir/fn2.tar.bz2
    dependency2:
        - subdir/fn3.tar.bz2
        - subdir/fn4.tar.bz2

    The associated packages are being removed from the index.
    z?WARNING: The following dependencies do not exist in the channelz2    and are not declared as external dependencies:z  %sz    - %srv   rw   Nr   z9The associated packages are being removed from the index.r[   r   )r   r   r   r   r   r   r   )r  r  r  Zdep_namer  r   rE   r   r@   r@   rA   _warn_on_missing_dependenciesU  s    

r  c                s  i }t | ||\}t|| | ttxX| D ]N| }x"|d  D ]\dd< dkrtdd d D  yJfddd D d	<  fd
dd D d< W n4 tk
r } ztd	| W d d }~X Y nX qNy&fddd D d< W qN tk
r^ } ztd	| W d d }~X Y qNX qNW | 
dg |d< ||< q0W t| |S )Nrw   r    record_version
constrainsc             s   s   | ]}|  d  V  qdS )r   N)r   )rj   depr@   r@   rA   r     s    z$_augment_repodata.<locals>.<genexpr>c          	      s   g | ]}t  |qS r@   )r  )rj   r  )rE   r   r  r	  r   r@   rA   rl     s   z%_augment_repodata.<locals>.<listcomp>constrains2c          	      s.   g | ]&}|  d   krt|qS )r   )r   r  )rj   r  )constrains_namesrE   r   r  r	  r   r@   rA   rl     s   r   depends2zWEncountered a file ({}) that conda does not like.  Error was: {}.  Skipping this one...c          	      s   g | ]}t  |qS r@   )r  )rj   r  )rE   r   r  r	  r   r@   rA   rl     s   r   r   )r  r  r   rx   r   r   r'   r   r   r   r   r  )r   r  r  augmented_repodatar
  r   r  r@   )r  rE   r   r  r	  r   rA   _augment_repodataw  s4    
&(
r!  c          	   C   s  ddd}t | dr| d} xdt| dg D ]N}|drn|ddkrXd	|d
< n|ddkrnd	|d< |d
 r2|d r2P q2W tdd |D |d< tdd |D |d< tdd |D |d< tdd |D |d< tdd |D |d< t|d}t|| W d Q R X d S )NF)r   r   decodezutf-8pathsZprefix_placeholderZ	file_modeZbinaryTr   r   r   c             s   s   | ]}| d V  qdS )zetc/conda/activate.dN)
startswith)rj   rE   r@   r@   rA   r     s    z._cache_post_install_details.<locals>.<genexpr>z
activate.dc             s   s   | ]}| d V  qdS )zetc/conda/deactivate.dN)r$  )rj   rE   r@   r@   rA   r     s    zdeactivate.dc             s   s   | ]}t  |d V  qdS )z*/.*-pre-link.*N)fnmatch)rj   rE   r@   r@   rA   r     s    r   c             s   s   | ]}t  |d V  qdS )z*/.*-post-link.*N)r%  )rj   rE   r@   r@   rA   r     s    r   c             s   s   | ]}t  |d V  qdS )z*/.*-pre-unlink.*N)r%  )rj   rE   r@   r@   rA   r     s    r   rI   )r  r"  rd   loadsr   anyrc   dump)Zloaded_json_text	all_pathspost_install_cache_pathZpost_install_details_jsonrH   r   r@   r@   rA   _cache_post_install_details  s0    




r+  c       	   
      s   d}t  fdd|D d }|r,t| |}nd}yt|}W n tttfk
r\   i }Y nX ytj|dd}W n4 t	k
r   |
di d tj|dd}Y nX t|d	}|| W d Q R X |S )
N)zinfo/recipe/meta.yaml.renderedzinfo/recipe/meta.yamlzinfo/meta.yamlc             3   s   | ]}| kr|V  qd S )Nr@   )rj   p)r)  r@   rA   r     s    z _cache_recipe.<locals>.<genexpr>z{}T)ZskipkeysZrequirementsr  rI   )nextr1   yaml	safe_loadr   r   r   rd   dumps	TypeErrorr   r   rc   r   )	Ztarballr)  recipe_cache_pathZrecipe_path_search_orderZrecipe_pathZrecipe_yaml_binaryrecipe_jsonZrecipe_json_strr   r@   )r)  rA   _cache_recipe  s"    
r4  c          	   C   sZ   yt | d}W n& tk
r4   td|   d}Y nX t|d}|| W d Q R X d S )Nzinfo/about.jsonz%s has no file info/about.jsons   {}r   )r1   r   r   r   rc   r   )tar_pathabout_cache_pathZbinary_about_jsonr   r@   r@   rA   _cache_about_json  s    
r7  c          	   C   sZ   yt | d}W n& tk
r4   td|   d}Y nX t|d}|| W d Q R X d S )Nzinfo/recipe_log.jsonz0%s has no file info/recipe_log.json (this is OK)s   {}r   )r1   r   r   r   rc   r   )r5  recipe_log_pathZbinary_recipe_logr   r@   r@   rA   _cache_recipe_log  s    
r9  c             C   s6  i }t j| ryt| d}t|d}W nP tk
r~   yt| d}t	|}W n" tk
rx   t
d|   Y nX Y nX nt j| r2y.tt j| dd}t|}W d Q R X W nr ttfk
r0   y.tt j| dd}t	|}W d Q R X W n( ttfk
r*   t
d|   Y nX Y nX |S )Nzinfo/run_exports.jsonzutf-8zinfo/run_exports.yamlz'%s has no run_exports file (this is OK)r   zrun_exports.jsonzrun_exports.yaml)ra   rb   r   r1   rd   r&  r"  r   r.  r/  r   r   r   rc   r   re   r   r/   )Ztar_or_folder_pathr   Zbinary_run_exportsrH   r@   r@   rA   get_run_exports  s,    

r:  c          	   C   s.   t | }t|d}t|| W d Q R X d S )NrI   )r:  rc   rd   r(  )r5  run_exports_cache_pathr   r   r@   r@   rA   _cache_run_exports  s    r<  c          	   C   sZ   yt | d}W n& tk
r4   td|   d}Y nX t|d}|| W d Q R X |S )Nzinfo/paths.jsonz%s has no file info/paths.jsons   {}r   )r1   r   r   r   rc   r   )r5  paths_cache_pathbinary_paths_jsonr   r@   r@   rA   _cache_paths_json
  s    
r?  c          	   C   s\   | di  dpd}||krX|t|d 7 }t| d}t|d}|| W d Q R X d S )NZappiconzinfo/icon.pngr   r   )r   r   r1   rc   r   )r5  r3  r)  icon_cache_pathZapp_icon_pathZbinary_icon_datar   r@   r@   rA   _cache_icon  s    
rB  c             C   sD   t  }|d}|jd| pd|f |t jtdd|d}|S )Nzsubdir-index.html.j2z%s/%sr[   r   )r   )titlerw   current_timeextra_paths)r   get_templaterenderr   utcnowr   r   r   )r   r   repodata_packagesrE  r   templaterendered_htmlr@   r@   rA   _make_subdir_index_html#  s    
rL  c             C   s@   t  }|d}|j| |d |d t jtddd}|S )Nzchanneldata-index.html.j2rw   r   r   )r   )rC  rw   r   rD  )r   rF  rG  r   rH  r   r   r   )r   channeldatar   rJ  rK  r@   r@   rA   _make_channeldata_index_html/  s    
rN  c       	      C   s   t jdddg| d}| ddkrt jdddd	g| d}g }xD|d  D ].}|d
\}}}}||t|||d qTW |S )NZgitz	rev-parsez--is-inside-work-tree)cwdzutf-8truer   z--pretty=format:'%h|%ad|%an|%s'z--date=unix|)hashr   Zauthorr   )
subprocessZcheck_outputr   r"  
splitlinesr   r   r   )	rb   Zis_repooutputr   line_hashZ_timeZ_authorZ_descr@   r@   rA   _get_source_repo_git_info;  s    rX  c          	   c   s.   t  }zt |  d V  W d t | X d S )N)ra   getcwdchdir)destcurdirr@   r@   rA   
_tmp_chdirI  s
    

r]  c       
         s   i  x|   D ]z\}}|dg }x|D ]}t|d |d< q(W fdd|D }|jdd dd |r|d	|d
 d||d f < qW |r|tfddt|D dd ddd
 d< t }t  fdddd}x|D ]}	 |	 ||	< qW |S )Nr   r   c                s   g | ]}|d   kr|qS )r   r@   )rj   commit)cutoff_timer@   rA   rl   Z  s    z$_collect_commits.<locals>.<listcomp>c             S   s   | d S )Nr   r@   )r   r@   r@   rA   rn   [  s    z"_collect_commits.<locals>.<lambda>T)r   reverser   )r   r   z%s (%s)r   c                s   g | ]}|d   kr|qS )r   r@   )rj   r^  )r_  r@   rA   rl   a  s    c             S   s   | d S )Nr   r@   )r   r@   r@   rA   rn   c  s    zindex hotfixesc                s    |  d d d S )Nr   r   r   r@   )r   )commit_infor@   rA   rn   f  s    )r   r   r   r   r   rX  r   )
Zpackage_orderr   r_  r  r   r   r^  Zsorted_commit_infoorderr   r@   )ra  r_  rA   _collect_commitsS  s&    

rc  c               @   s   e Zd ZdedfddZd-ddZd.ddZd	d
 Zd/ddZdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd0d!d"Zd#d$ Zd%d& Zd1d'd(Zd)d* Zd+d, ZdS )2r   NFc             C   s8   t || _|pt|d| _|| _t|| _|| _d S )Nrv   )	r
   channel_rootr   r  r   _subdirsr6   thread_executorr   )r>   rd  r   r   r   r   r@   r@   rA   r=   n  s
    

zChannelIndex.__init__c                s*  |rt j}nt j}tj|tgd  jsnt fddt	 j
D }td|  t|dhB   _}ntt jdhB   _}tjt j
gddt i }tt||p| dJ}	xB|D ]:}
|	d	|
  |	  t j
|
  j|
||d
||
< qW W d Q R X i }i }x,|D ]$}
 |
||
 |\||
< ||
< qW x|D ]}
 |
||
  qJW t|||}i }xH|D ]@}
 |
||
 ||
<  |
||
 }|rz |
||
  qzW ttdd | D }t |} !||\}} "|  #|||  $| W d Q R X W d Q R X d S )N)ri   c             3   s*   | ]"}|t krtt j|r|V  qd S )N)r   r   r   rd  )rj   r   )r>   r@   rA   r   ~  s    z%ChannelIndex.index.<locals>.<genexpr>zfound subdirs %sr]   i  )Ztimeout)totaldisablez
Subdir: %s)rm   r   c             s   s   | ]}|d  V  qdS )rw   Nr@   )rj   r   r@   r@   rA   r     s    )%r}   r~   ZERRORr"   r|   rK   re  r   ra   listdirrd  r   r   r   r   Ztry_acquire_locksZget_lockr   rD   set_descriptionr   r   index_subdir_patch_repodata_write_repodatar!  _create_repodata2_write_repodata2_write_subdir_index_htmltupler^   r   r  _build_channeldata_write_channeldata_index_html_write_channeldata_rss_write_channeldata)r>   r   r   rm   r   levelZdetected_subdirsr   Zrepodata_from_packagestr   r  r  r   	repodata2Zchangedr   r   r   package_mtimesr@   )r>   rA   r   v  sJ    
"
 


zChannelIndex.indexc                s  t  j}  t |t}t |dd}|r>td|  tdd t|D }t	dt
|f  y&t|}t|pi }	W d Q R X W n ttfk
r   i }	Y nX |	di }
t|
}i } jsy&t|}t|pi }W d Q R X W n ttfk
r   Y nX | }zl|| }|| } j|||||d}t|| | }|| }x|D ]}||krb||= qbW i }xJt|D ]>}y ||||< W n  tk
r   || Y nX qW ttt||}t fd	d|D }td
 t
||p| d\}xTt|D ]H}| \}}}}|r |d|  |  ||d||< |||< q W W d Q R X |ditd}W d ||krt|d}t || W d Q R X X |S )Nz.cachez	stat.jsonzBuilding repodata for %sc             s   s   | ]}| tr|V  qd S )N)r  r3   )rj   rE   r@   r@   rA   r     s    z,ChannelIndex.index_subdir.<locals>.<genexpr>zfound %d conda packages in %srw   )rm   r   c             3   s    | ]} j  j|V  qd S )N)rf  rJ   _extract_to_cache)rj   rE   )r>   r   r@   rA   r     s   zhash & extract packages for %s)descrg  rh  zHash & extract: %s)r   sizer   )rw   r   repodata_versionrI   )!r   rd  _ensure_dirsREPODATA_JSON_FNr   r   r   ra   ri  r   rD   rc   rd   re   EnvironmentErrorr5   r   r   copy_calculate_update_setr   _load_index_from_cacher   r   r_   rq  r   r7   resultrj  r   REPODATA_VERSIONr(  )r>   r   rm   r   subdir_pathrepodata_json_pathZstat_cache_pathfns_in_subdirr   Zold_repodataZold_repodata_packagesold_repodata_fns
stat_cacheZstat_cache_originalZadd_setZ
remove_set
update_setZunchanged_setZremoved_setrE   Znew_repodata_packagesZhash_extract_setfuturesrw  futurer   r|  
index_jsonnew_repodatar@   )r>   r   rA   rk    st    



	


	

zChannelIndex.index_subdirc             C   s   dd }t | j|d}|| |t |d |t |d |t |d |t |d |t |d |t |d	 |t |d
 |t | jd |t |d d S )Nc             S   s   t | pt| S )N)r   ra   r   )rb   r@   r@   rA   rn   -  s    z+ChannelIndex._ensure_dirs.<locals>.<lambda>z.cacher   aboutr#  reciper   post_installr@  icons
recipe_log)r   rd  )r>   r   Zensure
cache_pathr@   r@   rA   r~  +  s    zChannelIndex._ensure_dirsc       
         sT   ||@ }t | j|tfdd|D }t fddt|d|pD| dD }	|	S )Nc             3   s"   | ]}|t t |fV  qd S )N)ra   lstatr   )rj   rE   )r  r@   rA   r   ?  s    z5ChannelIndex._calculate_update_set.<locals>.<genexpr>c             3   sF   | ]>\}}|j  |i d ks:|j |i dkr|V  qdS )r   r|  N)st_mtimer   st_size)rj   rE   stat_result)r  r@   rA   r   @  s    zFinding updated files)r{  rh  )r   rd  rq  r   r   )
r>   r   r  r  r  rm   r   Zcandidate_fnsZstat_resultsr  r@   )r  r  rA   r  :  s    z"ChannelIndex._calculate_update_setc             C   s  t | j|}t ||}|d d d f}tj|rt |dd|d }t |dd|d }t |dd|d }t |dd|d }	t |dd|d }
t |dd|d }t |dd	|}t |dd
|d }td|  yt|d}t	|
d}tt|}t|| t||
 t||}t||| t|||	}t|| t|||| t|}|j |d< }|j}t||d< t||d< ddddddddddh
}x|t|@ D ]}||= qW t|d}t|| W d Q R X ||||f}W n6 tjjtj t!t"fk
r   t#d||f  Y nX |S )Nz.cacher   z.jsonr  r#  r  r   r  r@  r  z#hashing, extracting, and caching %szinfo/index.jsonzutf-8r|  md5Zsha256archZ
has_prefixr   rp   ZucsZrequires_featuresZbinstarztarget-tripletmachineZoperatingsystemrI   zIPackage %s/%s appears to be corrupt.  Please remove it and re-download it)$r   rd  ra   rb   r   r   r   r1   rd   r&  r"  r   r2   r7  r<  r?  r+  r4  r9  rB  statr  r  r"   r   Zsha256_checksumrc   r(  
libarchiveZ	exceptionZArchiveErrortarfileZ	ReadErrorr   EOFErrorerror)r>   r   rE   r  r5  Zretvalindex_cache_pathr6  r=  r2  r;  r*  rA  r8  Zbinary_index_jsonr  r)  r>  r3  r  r|  r   Zfilter_fieldsr   r   r@   r@   rA   rz  F  s^    






zChannelIndex._extract_to_cachec          	   C   sF   t | j|dd|d }td|  t|}t|}W d Q R X |S )Nz.cacher   z.jsonzloading index cache %s)r   rd  r   r   rc   rd   re   )r>   r   rE   r  r  r   r  r@   r@   rA   r    s
    
z#ChannelIndex._load_index_from_cachec             C   s  t | j|}ytt ||}W n tk
r2   i S X t |dd|d }t |dd|d }t |dd|d }t |dd|d }t |dd|d }	t |dd|d	 }
t |dd
|d }i }xn||||	|fD ]\}y:tj|dkrt|}|t	
| W d Q R X W q tttfk
r    Y qX qW t|
}|rt|d }|ddd }d|d |f }d| }t | jd|}t|}d|t|f }|j||d t|rt||krt| t|st|| ||d< |di }y|dd | D  W n tk
r   Y nX t|d t|d y&t|}t	
||d< W d Q R X W n" ttfk
rz   i |d< Y nX |S )Nz.cacher   z.jsonr  r  r   r  r@  z.*r  r   r   .r    z%s.%sr   zicons/r  z	md5:%s:%s)r   r   r   sourcec             S   s   i | ]\}}|d | qS )Zsource_r@   )rj   r   r   r@   r@   rA   r    s    z5ChannelIndex._load_all_from_cache.<locals>.<dictcomp>r   r   )r   rd  r   r/   ra   rb   r   rc   r   rd   re   OSErrorr  r   r-   r   rsplitr"   r   r   r   r   r   r   r   r   )r>   r   rE   r  r   r  r6  r2  r;  r*  Zicon_cache_path_globr8  rg   rb   r   Zicon_cache_pathsrA  Zicon_extZchannel_icon_fnr   Zicon_channel_pathZicon_md5r   r  r@   r@   rA   _load_all_from_cache  s`    








z!ChannelIndex._load_all_from_cachec             C   s\   t | j|t}tj|ddddd}t||dd}|rX|d }t|}t||dd |S )	N   T),z: )indent	sort_keys
separatorszutf-8)r   z.bz2)r   )	r   rd  r  rd   r0  encoder   bz2compress)r>   r   r   r  Znew_repodata_binaryZwrite_resultZrepodata_bz2_pathZbz2_contentr@   r@   rA   rm    s    

zChannelIndex._write_repodatac       	         s   |d }t  j|} fdd}t }||t |t ||t |td  ||t |d ||t |d t j|||}t |d}t||S )Nrw   c                s:   t t j|r6t|tt|t|d| t|< d S )N)r|  r   r  )	r   r   rd  r   r   r   r"   r   r   )rE  rb   )r>   r@   rA   _add_extra_path  s    
z>ChannelIndex._write_subdir_index_html.<locals>._add_extra_pathz.bz2zrepodata2.jsonzpatch_instructions.jsonz
index.html)r   rd  r   r  rL  r   r   )	r>   r   r   rI  r  r  rE  rK  
index_pathr@   )r>   rA   rp    s    
z%ChannelIndex._write_subdir_index_htmlc       
         s   t   d  fdd| D }t||}t }|d}|j| jdt j	t
dd|dd	}t| jd
}	t|	| |S )Ni u c                s&   i | ]\}}|kr d  | |qS )rw   r@   )rj   r   r   )rM  r_  r@   rA   r    s    z7ChannelIndex._write_channeldata_rss.<locals>.<dictcomp>z
rss.xml.j2zhttps://anaconda.orgr   )r   T)r   rf   rD  ra  r   zrss.xml)r   r   rc  r   rF  rG  r   r   rH  r   r   r   r   rd  r   )
r>   rM  ry  r   Zcurrentra  r   rJ  Zrendered_xmlZrss_pathr@   )rM  r_  rA   rt    s    

z#ChannelIndex._write_channeldata_rssc             C   s&   t | j|}t| jd}t|| d S )Nz
index.html)rN  r   r   rd  r   )r>   rM  rK  r  r@   r@   rA   rs    s    
z*ChannelIndex._write_channeldata_index_htmlc                s   t  i }i }tfdd|D }xZt||D ]L\}}| }|r.|| |d }	 fdd| D ||	< |d ||	< q.W t||d}
|
|fS )Nc             3   s(   | ] } j  j|d  |d V  qdS )r   rE   N)rf  rJ   r  )rj   rs   )r>   r@   rA   r     s   z2ChannelIndex._build_channeldata.<locals>.<genexpr>r   c                s   i | ]\}}| kr||qS r@   r@   )rj   r   r   )_CHANNELDATA_FIELDSr@   rA   r    s    z3ChannelIndex._build_channeldata.<locals>.<dictcomp>r   )Zchanneldata_versionr   rw   )CHANNELDATA_FIELDSrq  zipr  r   r   CHANNELDATA_VERSION)r>   r   r   Zpackage_datary  r  rs   r  rg   r   rM  r@   )r  r>   rA   rr    s     

zChannelIndex._build_channeldatac             C   sZ   x*| di  D ]\}}d|kr|d= qW t| jd}tj|dddd}t||d d S )Nrw   r   zchanneldata.jsonr  T)r  z: )r  r  r  )r   r   r   rd  rd   r0  r   )r>   rM  Z_pkgZpkg_dictZchanneldata_pathr   r@   r@   rA   ru  "  s    
zChannelIndex._write_channeldatac             C   s,   t |tj|d}i }|r(t|}|S )Nzpatch_instructions.json)r"   Zpackage_has_filera   rb   r   rd   r&  )r>   r   r   Zpatch_instructions_filer   r@   r@   rA    _load_patch_instructions_tarball+  s    
z-ChannelIndex._load_patch_instructions_tarballc             C   s   |pt | jd}t|rtd||f  y2ddlm}m} |d|}||}|j	| W n( t
k
r   dd l}	|	d|}Y nX |||}
|
dddkrtd|
S |rtd	|i S d S )
Nzgen_patch.pyzusing patch generator %s for %sr   )spec_from_file_locationmodule_from_specZa_bpatch_instructions_versionr    z'Incompatible patch instructions versionzSpecified metadata patch file '{}' does not exist.  Please try an absolute path, or examine your relative path carefully with respect to your cwd.)r   rd  r   r   r   importlib.utilr  r  r   exec_moduleImportErrorimpZload_sourcerl  r   rC   
ValueErrorr   )r>   r   r   r   Zgen_patch_pathr  r  r   modr  r   r@   r@   rA   _create_patch_instructions3  s&    
z'ChannelIndex._create_patch_instructionsc             C   s0   t j|dddd}t| j|d}t||d d S )Nr  T)r  z: )r  r  r  zpatch_instructions.json)rd   r0  r   rd  r   )r>   r   r   Z	new_patchpatch_instructions_pathr@   r@   rA   _write_patch_instructionsQ  s    z&ChannelIndex._write_patch_instructionsc          	   C   s^   t | j|d}t|rZtd|  t|(}t|}|dddkrPt	d|S Q R X i S )Nzpatch_instructions.jsonzusing patch instructions %sr  r   r    z'Incompatible patch instructions version)
r   rd  r   r   r   rc   rd   re   r   rC   )r>   r   r  r   r   r@   r@   rA   _load_instructionsV  s    


zChannelIndex._load_instructionsc             C   sn   |r| dr| ||}n| |||}|r<| || n
| |}|dddkr^tdt||||fS )Nr  r  r   r    z'Incompatible patch instructions version)r  r  r  r  r  r   rC   r   )r>   r   r   r   r   r@   r@   rA   rl  a  s    
zChannelIndex._patch_repodatac       
      C   s  |}d|d< t  }| j}x&|d  D ]\}}d|d< d|krDq(|d |d< |d= |d= d|krv|d |d	< |d= ||d
< ||d< d|krt|d |d< d|kr|d dkrd|d< |d= |dd  d|kr|d d|d< d|kstd|ks
t|||f|dd  |dd  ||d< |dr(|| q(W dd }t	dd |d 
 }	t|	dd|d|d< t|	d d|d|d< |S )!Nr  r}  rw   r  r  Zrequiresr   r  r  rE   r   r   r]   rQ   Znoarch_pythonZpackage_typeZfeaturesZtrack_featuresr   r  r|  r  rp   r   r   c             S   s6   | d dkrdp| d | d t | d | d | d fS )Nr   r   0r   r   r   r  )r$   )r   r@   r@   rA   rn     s
    
z0ChannelIndex._create_repodata2.<locals>.<lambda>c             S   s   |  ddS )Nr   F)r   )r   r@   r@   rA   rn     s    Fr@   )r   T)r   r   r   r   r   r   AssertionErrorr   r   r`   r   r   )
r>   r   r   rx  Zrevoked_setr   rE   r   Zsort_keyZpackage_groupsr@   r@   rA   rn  o  sJ    
	zChannelIndex._create_repodata2c             C   s,   t | j|d}tj|dddd}t||dS )Nzrepodata2.jsonr  T)r  z: )r  r  r  )r   rd  rd   r0  r   )r>   r   rx  r  r  r@   r@   rA   ro    s    zChannelIndex._write_repodata2)NFF)FF)FF)N)N)rK   rL   rM   MAX_THREADS_DEFAULTr=   r   rk  r~  r  rz  r  r  rm  rp  rt  rs  rr  ru  r  r  r  r  rl  rn  ro  r@   r@   r@   rA   r   l  s,   
:
{
?<	

Br   )NFFNFT)FF)Z
__future__r   r   r   r   r  collectionsr   r   r   rd   Znumbersr	   ra   os.pathr
   r   r   r   r   r   r   r   r   r   Zshutilr   r   rS  r  Ztempfiler   r   Zuuidr   Zconda.common.compatr   r   Zjinja2r   r   r   r.  Zyaml.constructorr   Zyaml.parserr   Zyaml.scannerr   r   r%  	functoolsr   r}   r  r[   r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   Zconda.base.constantsr3   	Exceptionr4   Zjson.decoderr5   r  r  rK   r   Zconda.common.ior6   r7   Zconcurrent.futuresr8   r9   Zconcurrent.futures.threadr:   rO   rP   	frozensetr   Z
NAMESPACESrz   r   ry   r{   r   r  r\   r  ZLOCK_TIMEOUT_SECSZLOCKFILE_NAMEr   Zcytoolz.itertoolzr^   r_   r`   Zconda._vendor.toolz.itertoolzrh   r   r   r   r   r   r  r  r  r  r   r   r   r   r  r  r  r  r  r  r!  r+  r4  r7  r9  r:  r<  r?  rB  rL  rN  rX  r   r]  rc  objectr   r@   r@   r@   rA   <module>   s   0 
(& 
u
 	
&	"& 

