ó
›ßÈ[c           @` s¿  d  Z  d d l m Z m Z m Z m Z d d l m Z d d l m	 Z	 d d l
 Z
 d d l Z d d l Z d d l m Z d d	 l m Z m Z d
 d l m Z d
 d l m Z d d l m Z d d l m Z d d l m Z m Z d d l m Z d d l m  Z  d d l! m" Z" d d l# m$ Z$ d d l% m& Z& d d l' m( Z( d d l) m* Z* d g Z+ e" d d d ƒe& d ƒ e j, e- e- d d „ ƒ ƒ Z. d „  Z/ d „  Z0 d „  Z1 d  „  Z2 d S(!   u   Validate VO Services.i    (   t   absolute_importt   divisiont   print_functiont   unicode_literalsi   (   t   six(   t   mapN(   t   OrderedDicti   (   t   ValidationMultiprocessingErrort   InvalidValidationAttributei   (   t   vos_catalog(   t   VOSError(   t   votable(   t   E19(   t   htmlt   result(   t   log(   t   data(   t
   deprecated(   t   AstropyUserWarning(   t   timefunc(   t   unescape_all(   t   parse_csu   check_conesearch_sitesu   2.0t   alternativeuB   astroquery.vo_conesearch.validator.validate.check_conesearch_sitesu   defaultc   $   
   C` s;  d d l  m } | d k r( | j } n  t |  t j ƒ sr t |  ƒ d k sr t j j	 |  ƒ r t j j
 |  ƒ r t d ƒ ‚ n  t j j	 |  ƒ s£ t j |  ƒ n  t j j |  d ƒ } t j j	 | ƒ sÚ t j | ƒ n  t ƒ  } t j j |  d ƒ | d <t j j |  d	 ƒ | d
 <t j j |  d ƒ | d <t j j |  d ƒ | d <i  } xn | D]f } t j j ƒ  | | <t j j	 | | ƒ rTt j | | ƒ | rºt j d j | | ƒ ƒ qºqTqTWt j ƒ  3 t j d ƒ t j j | j d d d | ƒ}	 Wd QX| d k	 rŽt t t g  | D]* }
 t |
 t ƒ r?|
 j  d ƒ n |
 ^ qƒ ƒ } t | ƒ } g  } | ršt j d j | t |	 ƒ ƒ ƒ qšn t |	 ƒ } i  } x× |	 j! ƒ  D]É \ } } | d }
 | d d k s­| d k	 rñ|
 | k rñq­n  t" | d ƒ } g  | D]( } d j | | | g ƒ j  d ƒ ^ q} | d g 7} | | |
 d j | ƒ <| d k	 r­| j# |
 ƒ q­q­W| d k	 r÷| t | ƒ } t | ƒ } | d k r÷d j | ƒ } x! | D] }
 | d j |
 ƒ 7} qÄWt j$ | t% ƒ q÷n  t& | ƒ } t' j j( } g  | D] } | | | f ^ q} | r‡t) j* ƒ  } y | j t+ | ƒ } Wq–t, k
 rƒ} t- d  j | ƒ ƒ ‚ q–Xn t t+ | ƒ } xU | D]M } | d! } | | j. } |	 j/ | ƒ } t0 | | ƒ | | j1 | | ƒ qWt2 j3 | | ƒ } t4 j5 | | | ƒ | rNg  | D] }  | |  | f ^ q }! | j t6 |! ƒ n$ x! | D] }  t6 | |  | f ƒ qUWi  }" d }# xq | D]i } t | | ƒ |" | <|# |" | 7}# | | j7 | | d" t8 ƒ| r…t j d# j | |" | ƒ ƒ q…q…W| rt j d$ j |# | ƒ ƒ n  |" d d k r7t j$ d% t% ƒ n  d S(&   uÐ  Validate Cone Search Services.

    .. note::

        URLs are unescaped prior to validation.

        Only check queries with ``<testQuery>`` parameters.
        Does not perform meta-data and erroneous queries.

    Parameters
    ----------
    destdir : str, optional
        Directory to store output files. Will be created if does
        not exist. Existing files with these names will be deleted
        or replaced:

            * conesearch_good.json
            * conesearch_warn.json
            * conesearch_exception.json
            * conesearch_error.json

    verbose : bool, optional
        Print extra info to log.

    parallel : bool, optional
        Enable multiprocessing.

    url_list : list of string, optional
        Only check these access URLs against
        `astropy.vo.validator.Conf.conesearch_master_list` and ignore
        the others, which will not appear in output files.  By
        default, check those in
        `astropy.vo.validator.Conf.conesearch_urls`.  If `None`, check
        everything.

    Raises
    ------
    IOError
        Invalid destination directory.

    timeout
        URL request timed out.

    ValidationMultiprocessingError
        Multiprocessing failed.

    i   (   t   confu   defaulti    u   Invalid destination directoryu   resultsu   conesearch_good.jsonu   goodu   conesearch_warn.jsonu   warnu   conesearch_exception.jsonu   excpu   conesearch_error.jsonu   nerru   Existing file {0} deletedu   ignoret   encodingu   binaryt   show_progressNu   utf-8u"   Only {0}/{1} site(s) are validatedu   urlu   capabilityClasst
   ConeSearchu
   resourceIDu   =s   VERB=3t   &u$   {0} not found in registry! Skipped:
u   	{0}
uK   An exception occurred during parallel processing of validation results: {0}u   out_db_namet	   overwriteu   {0}: {1} catalog(s)u    total: {0} out of {1} catalog(s)u(   No good sites available for Cone Search.(9   t    R   t   conesearch_urlst
   isinstanceR   t   string_typest   lent   ost   patht   existst   isdirt   IOErrort   mkdirt   joinR   R	   t   VOSDatabaset   create_emptyt   removeR   t   infot   formatt   warningst   catch_warningst   simplefiltert   from_registryt   conesearch_master_listt   Nonet   setR   R   t   strt   encodet   get_catalogsR   t   appendt   warnR   t   listR   t   remote_timeoutt   multiprocessingt   Poolt   _do_validationt	   ExceptionR   t   urlt   get_catalogt   _copy_r_to_catt   add_catalogR   t   get_result_subsetsR   t   write_indext   _html_subindext   to_jsont   True($   t   destdirt   verboset   parallelt   url_listR   t   out_dirt   db_filet   js_treet   keyt   js_mstrt   cur_urlt	   uniq_rowst   url_list_processedt   key_lookup_by_urlt   cur_keyt   cur_catt   testquery_parst   cs_pars_arrt   url_list_skippedt	   n_skippedt   warn_strt   all_urlst   timeoutR@   t   map_argst   poolt   mp_listt   exct   rt   db_keyt   cat_keyt   html_subsetst   html_subsett   html_subindex_argst   nt   n_tot(    (    s<   lib/python2.7/site-packages/astropy/vo/validator/validate.pyt   check_conesearch_sites%   sÄ    4%%	$		:
2"
$c         C` sþ  |  \ } } } t  j j ƒ  t j | d | d | ƒ} | j ƒ  t | ƒ | d d k rí| d d k ríd } d } g  } t j d t	 ƒ w } y4 t
 j t  j j | j ƒ  d	 t ƒ| j i  ƒ }	 Wn9 t t t f k
 r}
 | j t |
 ƒ ƒ | d
 7} n XWd QXg  | D] } t | j ƒ ^ q| } t ƒ  } x_ | D]W } t  j j | ƒ } | d rm| d
 7} n  | d r„| d
 7} n  | j | d ƒ q>W| d c | 7<| d c | 7<| d c | 7<| d j | ƒ | d <t | ƒ n  t j | ƒ | S(   u'   Validation for multiprocessing support.t   rootR^   u   expectedu   goodu	   incorrectu   nexceptionsi    t   recordt   pedantici   Nu
   is_warningu   is_exceptionu   warningu	   nwarningsu   warningsu   warning_types(   u   goodu	   incorrect(   R   t   tablet   reset_vo_warningsR   t   Resultt   validate_vot   _categorize_resultR.   R/   RH   R	   t   vo_tab_parset   parset   get_vo_xml_patht   FalseR@   R   t
   IndexErrorR
   R8   R5   t   messageR4   t
   exceptionst   parse_vowarningt   addt   unionR   t   write_result(   t   argsRl   R@   R^   Rc   t   nexceptionst	   nwarningst   linest   warning_linest   tabt   et   xt   warning_typest   linet   w(    (    s<   lib/python2.7/site-packages/astropy/vo/validator/validate.pyR>   í   s@    

 %&	

c         C` sú   d d l  m } d |  k rC |  d d k	 rC d |  d <d |  d <n³ |  d d	 k rc |  d
 d	 k sy |  d j | j ƒ r d |  d <d |  d <nf |  d d	 k r· d |  d <d |  d <n? |  d
 d	 k rÞ d |  d <d |  d <n t d j |  j ƒ ƒ ‚ d S(   uÒ   Set success codes.

    Parameters
    ----------
    r : `astropy.io.votable.validator.result.Result`

    Raises
    ------
    InvalidValidationAttribute
        Unhandled validation result attributes.

    i   (   R   u   network_erroru   nerru   out_db_nameu   brokenu   expectedu   nexceptionsi    u	   nwarningsu   warning_typesu   goodu   excpu	   incorrectu   warnu+   Unhandled validation result attributes: {0}N(   R   R   R3   t   issubsett   noncritical_warningsR   R-   t   _attributes(   Rc   R   (    (    s<   lib/python2.7/site-packages/astropy/vo/validator/validate.pyRs   '  s     
 


c         C` s)   |  \ } } } t  j | d | | Œd S(   u(   HTML writer for multiprocessing support.t   totalN(   R   t   write_index_table(   R   RM   t   subsetR   (    (    s<   lib/python2.7/site-packages/astropy/vo/validator/validate.pyRF   H  s    c         C` s3   x, |  j  D]! } d | } |  j  | | | <q
 Wd S(   uÅ   Copy validation result attributes to given VO catalog.

    Parameters
    ----------
    r : `astropy.io.votable.validate.result.Result`

    cat : `astropy.vo.client.vos_catalog.VOSCatalog`

    u	   validate_N(   RŒ   (   Rc   t   catRP   t   new_key(    (    s<   lib/python2.7/site-packages/astropy/vo/validator/validate.pyRB   N  s    

(3   t   __doc__t
   __future__R    R   R   R   t   externR   t   extern.six.movesR   R<   R"   R.   t   collectionsR   Rz   R   R   t   clientR	   t   client.exceptionsR
   t   ioR   t   io.votable.exceptionsR   t   io.votable.validatorR   R   t   loggerR   t   utilsR   t   utils.decoratorsR   t   utils.exceptionsR   t   utils.timerR   t   utils.xml.unescaperR   t   tstqueryR   t   __all__t   curdirRH   Rk   R>   Rs   RF   RB   (    (    (    s<   lib/python2.7/site-packages/astropy/vo/validator/validate.pyt   <module>   s:   "		Ä	:	!	