B
    \1                 @   s2  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZ ddlmZ yddlmZ W n$ ek
r   G d	d
 d
eZY nX ddlmZ ddlmZ ddlmZmZmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$ e%e&Z'G dd dej(Z)dd Z*G dd deZ+dd Z,dS )z
    sphinx.builders.linkcheck
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    The CheckExternalLinksBuilder class.

    :copyright: Copyright 2007-2019 by the Sphinx team, see AUTHORS.
    :license: BSD, see LICENSE for details.
    N)path)nodes)	HTTPError)queuehtml_parser)unquote)HTMLParseErrorc               @   s   e Zd ZdS )r   N)__name__
__module____qualname__ r   r   8lib/python3.7/site-packages/sphinx/builders/linkcheck.pyr      s   r   )Builder)__)
encode_urirequestslogging)purplered	darkgreendarkgraydarkred	turquoise)is_ssl_errorc               @   s    e Zd ZdZdd Zdd ZdS )AnchorCheckParserz9Specialized HTML parser that looks for a specific anchor.c             C   s   t j|  || _d| _d S )NF)r   
HTMLParser__init__search_anchorfound)selfr   r   r   r   r   6   s    zAnchorCheckParser.__init__c             C   s0   x*|D ]"\}}|dkr|| j krd| _P qW d S )N)idnameT)r   r   )r   tagZattrskeyvaluer   r   r   handle_starttag=   s    z!AnchorCheckParser.handle_starttagN)r	   r
   r   __doc__r   r%   r   r   r   r   r   3   s   r   c             C   sZ   t |}y6x(| jdddD ]}|| |jrP qW |  W n tk
rR   Y nX |jS )zReads HTML data from a response object `response` searching for `anchor`.
    Returns True if anchor was found, False otherwise.
    i   T)Z
chunk_sizeZdecode_unicode)r   Ziter_contentZfeedr   closer   )responseanchorparserchunkr   r   r   check_anchorE   s    
r,   c               @   sf   e Zd ZdZdZedZdd Zdd Zdd	 Z	dddZ
dd Zdd Zdd Zdd Zdd Zd
S )CheckExternalLinksBuilderz+
    Checks for broken external links.
    Z	linkcheckzCLook for any errors in the above output or in %(outdir)s/output.txtc             C   s   dd | j jjD | _dd | j jjD | _t | _i | _i | _	t
d tt| jdd  t | _t | _g | _xBt| j jjD ]0}tj| jd}|d |  | j| qW d S )	Nc             S   s   g | ]}t |qS r   )recompile).0xr   r   r   
<listcomp>d   s    z2CheckExternalLinksBuilder.init.<locals>.<listcomp>c             S   s   g | ]}t |qS r   )r.   r/   )r0   r1   r   r   r   r2   e   s   g      @z
output.txtw)targetT)appconfiglinkcheck_ignore	to_ignorelinkcheck_anchors_ignoreanchors_ignoresetgoodbroken
redirectedsocketZsetdefaulttimeoutopenr   joinoutdirr'   r   ZQueuewqueuerqueueworkersrangelinkcheck_workers	threadingZThreadcheck_threadZ	setDaemonstartappend)r   iZthreadr   r   r   initb   s     



zCheckExternalLinksBuilder.initc                s   dddidj jjr&j jjd< fdd  fdd	}xBj \}}d krbP | \}}}j|||||f qHW d S )
NTZAcceptz/text/html,application/xhtml+xml;q=0.9,*/*;q=0.8)Zallow_redirectsZheadersZtimeoutc           
      s  dkr: dd\} }x(jD ]}||r d }P q W n} d }y| d W n tk
rl   t| } Y nX y|rjjjrt	j
| fdjjd }t|t|}|sttd| nby&t	j| fdjji }|  W n: tk
r   t	j
| fdjjd }|  Y nX W n tk
rf } z$|jjdkrHd	S d
t|dfS W d d }~X Y nJ tk
r } z*t|rdt|dfS d
t|dfS W d d }~X Y nX |jd| dkrdS |j}|r|d| 7 }|jr|jd j}d||fS d|dfS d S )N#   asciiT)streamr6   zAnchor '%s' not foundr6   i  )workingz - unauthorizedr   r=   r   ignored/)rR    r   r>   )splitr:   matchencodeUnicodeErrorr   r5   r6   linkcheck_anchorsr   getr,   r   	Exceptionr   headZraise_for_statusr   r(   Zstatus_codestrr   Zurlrstriphistory)Zreq_urlr)   rexr(   r   errZnew_urlcode)kwargsr   urir   r   	check_uri   sV    

 
z9CheckExternalLinksBuilder.check_thread.<locals>.check_uric                 s  t dksdrdS ds(dS jkr6dS jkrPdj dfS jkrxdj d j d	 fS xjD ]} | rd
S qW x,tjj	j
D ]}  \}}}|dkrP qW |dkrڈj n,|dkr|j< n|dkr||fj< |||fS )Nr   )rN   zmailto:zftp:)	uncheckedrU   r   )zhttp:zhttps:)localrU   r   )rR   oldr   r=   r>   rO   )rS   rU   r   rR   )len
startswithr<   r=   r>   r8   rX   rF   r5   r6   linkcheck_retriesadd)rb   _statusinford   )rg   r   rf   r   r   check   s0    





z5CheckExternalLinksBuilder.check_thread.<locals>.check)r5   r6   linkcheck_timeoutrC   r\   rD   put)r   rr   docnamelinenorp   rq   rd   r   )rg   re   r   rf   r   rI   y   s    
@ z&CheckExternalLinksBuilder.check_threadc       
      C   s  |\}}}}}}|dkrd S |dkr0|dkr0d S |rDt jd|dd |dkr|rlt td| d	 |  nt td|  n@|d
krt td|  | d
||| n|dkrt td| |  n|dkrJ| d|||d	 |  | jjs| jjr*t jt	d||| j
||fd nt td| td|   nx|dkrdtfdtfdtfdtfdtfd| \}}	| d| |||d |  t |	d| |	d| d |   d S )Nrh   rR   rj   z(line %4d) rO   )ZnonlrS   z
-ignored- z: ri   z
-local-   z
ok        r=   zbroken link: %s (%s))locationz
broken    z - r>   Zpermanentlyz
with Foundzwith See OtherZtemporarilyzwith unknown code)i-  i.  i/  i3  r   zredirected z to z
redirect  )loggerrq   r   write_entryr   r5   quietZwarningiserrorZwarningr   envdoc2pathr   r   r   r   )
r   resultrf   ru   rv   rp   rq   rd   textZcolorr   r   r   process_result   s@    
 


z(CheckExternalLinksBuilder.process_resultNc             C   s   dS )NrU   r   )r   ru   typr   r   r   get_target_uri  s    z(CheckExternalLinksBuilder.get_target_uric             C   s   | j jS )N)r{   Z
found_docs)r   r   r   r   get_outdated_docs  s    z+CheckExternalLinksBuilder.get_outdated_docsc             C   s   d S )Nr   )r   Zdocnamesr   r   r   prepare_writing  s    z)CheckExternalLinksBuilder.prepare_writingc             C   s   t d d}xj|tjD ]Z}d|kr*q|d }d }x"|d krX|j}|d krPP |j}q8W | j|||fd |d7 }qW d}x$||k r| 	| j
  |d7 }qW | jrd| j_d S )NrU   r   ZrefuriFrO   )rx   rq   Ztraverser   Z	referenceparentlinerC   rt   r   rD   r\   r=   r5   Z
statuscode)r   ru   ZdoctreenZnoderf   rv   Zdoner   r   r   	write_doc  s(    



z#CheckExternalLinksBuilder.write_docc          	   C   sH   t t| jddd&}|d| j|d |||f  W d Q R X d S )Nz
output.txtazutf-8z%s:%s: [%s] %s
)codecsr@   r   rA   rB   writer{   r|   )r   Zwhatru   r   rf   outputr   r   r   ry   5  s    z%CheckExternalLinksBuilder.write_entryc             C   s"   x| j D ]}| jdd qW d S )N)NNNF)rE   rC   rt   )r   Zworkerr   r   r   finish;  s    z CheckExternalLinksBuilder.finish)N)r	   r
   r   r&   r!   r   ZepilogrM   rI   r   r   r   r   r   ry   r   r   r   r   r   r-   Z   s   r&
r-   c             C   sp   |  t | dg d  | ddd  | dd d tg | ddd  | ddd  | d	d
gd  ddddS )Nr7   rm   rO   rs   rG      r[   Tr9   z^!Zbuiltin)versionZparallel_read_safeZparallel_write_safe)Zadd_builderr-   Zadd_config_valueint)r5   r   r   r   setupA  s    
r   )-r&   r   r.   r?   rH   osr   Zdocutilsr   Zrequests.exceptionsr   Z	six.movesr   r   Zsix.moves.urllib.parser   Zsix.moves.html_parserr   ImportErrorr]   Zsphinx.buildersr   Zsphinx.localer   Zsphinx.utilr   r   r   Zsphinx.util.consoler   r   r   r   r   r   Zsphinx.util.requestsr   Z	getLoggerr	   rx   r   r   r,   r-   r   r   r   r   r   <module>
   s0    	
 h