o eiC@sndZddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z zddl Z Wne yAedejdedYnwzddlmZWne y_ed ejdedYnwzddlZd ZWne yzd Zed ejdYnwd ede e fddZd ede e e fddZd ede e e fddZddefddZddZedkredSdS)u Scraper for HDMA motif lexicon from the MOTIFS.html page. Extracts the pattern → motif_name_safe → annotation mapping from: https://greenleaflab.github.io/HDMA/MOTIFS.html The page uses DataTables with embedded JavaScript data. N)Path)DictListOptionalzDError: requests module not found. Install with: pip install requestsfile) BeautifulSoupzPError: beautifulsoup4 module not found. Install with: pip install beautifulsoup4TFz:Warning: pandas not available, will use CSV writer instead html_contentreturnc Csd}t||tj}|D]1}z#|d}t|}t|tr0t|dkr0t|dtr0|WSWq tj t fy=Yq wt |d}| d}|D]}|j durRqJ|j } gd} | D]R} t| | tj}|D]E}z.|d}tdd |}td d |}t|}t|trt|dkr|WSWqgtj t fy} zWYd} ~ qgd} ~ wwq[gd } | D]I} t| | tj}|D]<}z.|d}tdd |}td d |}t|}t|trt|dkr|WSWqtj t fyYqwqqJdS) z Extract DataTable data from HTML page. The data is embedded in JavaScript as a DataTable initialization. We need to find the data array in the script tags. z,var\s+\w*[Dd]ata\w*\s*=\s*(\[\[[\s\S]*?\]\])rr html.parserscriptN)"data"\s*:\s*(\[\[[\s\S]*?\]\])z'data'\s*:\s*(\[\[[\s\S]*?\]\])zdata\s*:\s*(\[\[[\s\S]*?\]\])z,\s*\]]z ,\s*\]\s*\]z]])z"var\s+\w+\s*=\s*(\[\[[\s\S]*?\]\])z$const\s+\w+\s*=\s*(\[\[[\s\S]*?\]\])z"let\s+\w+\s*=\s*(\[\[[\s\S]*?\]\]))refinditer MULTILINEgroupjsonloads isinstancelistlenJSONDecodeError ValueErrorr find_allstringDOTALLsub)r json_patternmatchesmatchdata_strdatasoupscriptsr script_content data_patternspatterne var_patternsr+/oak/stanford/groups/akundaje/airanman/projects/lab/rare-disease-manuscript/curation/broad/varbook-container/snakemake/./scrape_hdma_motifs.pyextract_datatable_data&sj   $           r-c Csd}t||}|D]8}|d}td|}|rBz|d}t|}t|tr4t|dkr4|WSWq tj t fyAYq wq dS)z Alternative method: Extract data from DataTable initialization. Look for the full DataTable config object and extract the data array. z1\$\([^)]+\)\.DataTable\s*\(\s*(\{[\s\S]*?\})\s*\)rrrN) rrrsearchrrrrrrr)r r(r r! config_str data_matchr"r#r+r+r,extract_from_datatable_init}s"       r1c Cst|d}|d}|durdSg}|d}|r,dd|ddgD}|r,|||d }|rN|d D]}d d|ddgD}|rM||q8|rR|SdS) zQ Fallback: Parse the HTML table directly if JavaScript extraction fails. r tableNtheadcSg|]}|jddqST)stripget_text).0thr+r+r, z$parse_html_table..r:tdtbodytrcSr4r5r7)r9r=r+r+r,r;r<)r findrappend) r r$r2rows header_rowheadersr>r?cellsr+r+r,parse_html_tables$       rF/https://greenleaflab.github.io/HDMA/MOTIFS.htmlurlcstd|dtj|dd}||j}d}tdt|}|dur,tdt|}|dur8tdt|}|dur@td gd }t |d t sUtd t |d t |}|d kratdt |d t rnt |d nd}|t |dkr|t |dkrtd|d|dt j dg}t|D]fdd|D}||q|}t |}|rt |d nd }|d krt |d t rt |d dkrtdd|d ddDnd} tdd|d ddD } | r | r t |d t |dkr |d } |dd}n|t |kr|d|n|} |}n|} |}|rt |d fdd|D} t | t |krKtdt |t | dt j d| }t | krt | kru| ddtt | D} td d!t j dn| d} td"d!t j d|rtt | d krtt |d krt |d d krtd#t |d d!t j dtd$| dd%t j dtd&|d dd%t j dd} d}d}t| D]\}|d'kr} q|d(kr}q|d)kr}qt |d d kr=d*d|D}t | d kr=| dd} | dur!| d kr!| d8} |dur/|d kr/|d8}|dur=|d kr=|d8}g}t| D].\}| krS|d+qC|kr_|d,qC|krk|d(qC||qC|} tr|rtj|| d-Stj| d-S| |d.S)/z Scrape the HDMA motif lexicon from the HTML page. Returns either a pandas DataFrame (if available) or a list of lists with headers. z Fetching z...)timeoutNz7Attempting to extract DataTable data from JavaScript...z'Trying alternative extraction method...z%Falling back to HTML table parsing...z%Could not extract data from HTML page) pattern_classidx_uniq motif_namemotif_name_safe annotationannotation_broadcategoryquery_consensuscwm_fwdcwm_rev total_hitstotal_n_seqletsn_component_celltypes top_organ cwm_entropy entropy_ratior( best_matchbest_match_TOMTOM_qvalrz%Data appears to be a flat array with z1 elements. Expected nested array (list of lists).zExtracted data is emptyrzDetected transposed data: z columns with z rows each. Transposing...rcs$g|]}t|kr|ndqS)rr9col)ir+r,r;s$z&scrape_hdma_motifs..css|]}t|tVqdS)N)rstrr9xr+r+r, sz%scrape_hdma_motifs..Fcss6|]}t|trt|ddddVqdS).r_-N)rrdreplaceisdigitrer+r+r,rgs csg|] }t|kr|qSr+r`r9row) expected_lenr+r,r;szWarning: Filtered out z rows with incorrect lengthcSsg|]}d|qS)col_r+r9rcr+r+r,r;szWarning: Extended headers to z columnszWarning: Truncated headers to zDebug: First row has zDebug: Headers: zDebug: First row values: r[rNrOcSsg|]}|ddqS)rNr+rlr+r+r,r;Cr<r(rMcolumns)rDrB)printrequestsgetraise_for_statustextr-r1rFrrrrsysstderrrangerAallany enumerate HAS_PANDASpd DataFrame)rHresponser r#rs num_top_levelnum_elements_firstrBrmfirst_row_all_stringsfirst_row_not_numericrD valid_rowsbest_match_idxmotif_name_safe_idxannotation_idxcol_name new_headersr+)rnrcr,scrape_hdma_motifss    2 & "             rc s&tjdd}|jdddd|jdtdd |jd d gd d d|}zAt|j}trt|t j r|t dt dt dt jfdd|jD}|r]|}n t dtjd}|jrqt|j}ntd}|j|dddt dt |d|t dt |dWdS|}|d}|d}t dt |dt d|g} g} |jD]} | |vr|| } | | | | q| st dtjdt tt |} |} |jrt|j}ntd}t|d d!d"/} tj| dd#}|| |D]t t| kr|fd$d| DqWdn 1s+wYt dt |d|t d%t|ddD]\}t t| krct d fd&d| DqHWdSt!y}zt d'|tjdd(dl"}|#t$d)WYd}~dSd}~ww)*Nz/Scrape HDMA motif lexicon from MOTIFS.html page) descriptionz--urlrGzURL of the HDMA MOTIFS page)defaulthelpz--outputz6Output TSV file path (default: hdma_motif_mapping.tsv))typerz --columns+)r(rNrOrPr[zfColumns to include in output (default: pattern motif_name_safe annotation annotation_broad best_match))nargsrrz Extracted z motifsz Columns: csg|] }|jvr|qSr+rrra)dfr+r,r;szmain..z@Warning: None of the requested columns found. Using all columns.rzhdma_motif_mapping.tsv F)sepindexzSaved z motifs to z Sample data:rqrDrBwr_)newline) delimitercsg|]}|qSr+r+rprmr+r,r;sz Sample data (first 10 rows):csg|]}t|qSr+)rdrprr+r,r;r<zError: rr)%argparseArgumentParser add_argumentrd parse_argsrrHrrrrrtrrrscopyryrzoutputrto_csvhead to_stringrrAr{opencsvwriterwriterowmaxr~join Exception traceback print_excexit)parserargsresultavailable_columns df_output output_path data_dictrDrB col_indicesoutput_headersrbidxfrrcr)rr+)rrmr,mainds        r__main__)rG)__doc__rrryrrpathlibrtypingrrrru ImportErrorrtrzrbs4r pandasrrrdr-r1rFrr__name__r+r+r+r,sH      W.j