o Uݢgg @sddlmZddlZddlZddlZddlmZddlZddlm Z ddl m Z ddlmZddlmmZddlmmZddlmmZddlmZddlm Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)m*Z*ddl+m,Z,m-Z-dd l.m/Z/m0Z0m1Z1dd l2m3Z3ej4ej5ej6ej7ej8ej9gZ:ej4ej5ej6ej9gZ;ej4ej6gZd dZ?ddZ@ddZAddZBdVddZCgdZDdWddZEddZFdd dXd#d$ZGd%d&ZHd'd(ZIGd)d*d*eZJeJd+d+d,gd-eJd.d.d,gd-eJd/d/d,gd-eJd0gd1d-eJd2gd3d-eJd4gd5d-eJd6gd7d-eJd8gd9d-eJd:d:d,gd-eJd;d;d,gd-g ZKejLfdd?ZNd@dAZOdBZPdYdJdKZQgfdLdMZRdNdOZS dZdPdQZTdRdSZUdTdUZVdS)[) annotationsN) NamedTuple) csv_utils)validate_reference)FeatureDefException)LIBRARIES_METRICMoleculeCounter)get_ref_name_from_genomesget_reference_genomes)TARGETING_METHOD_FILE_NAMESTARGETING_METHOD_TL_FILE_FORMATTargetingMethod)%SPATIAL_TARGET_DISALLOWED_PANEL_TYPEScs$eZdZfddZddZZS)PreflightExceptioncst|||_dSN)super__init__msg)selfr __class__c/oak/stanford/groups/akundaje/marinovg/programs/cellranger-9.0.1/lib/python/cellranger/preflight.pyr<s  zPreflightException.__init__cCs|jSr)r)rrrr__str__@szPreflightException.__str__)__name__ __module__ __qualname__rr __classcell__rrrrr;s rcCs|\}}|s t|dS)zTranslate (ok,msg) style from tenkit into an exception. Raises: PreflightException: The message as an exception. Nr)resultokrrrrcheckDsr!cCs$zt|WdStyYdSw)NFT)int ValueError)srrris_intOs   r%cCsLtdd|Dr dStdd|Ds$tdtdtjtjfdS)Ncss |] }|tjdkVqdS)ZVDJN)get rna_library LIBRARY_TYPE.0xrrr Ysz*check_gex_or_ab_present..cssD|]}|tjtjkp|tjtjkp|tjduVqdSr)r&r'r(GENE_EXPRESSION_LIBRARY_TYPEANTIBODY_LIBRARY_TYPEr)rrrr,^s zeYou must specify >= 1 input library with either library_type == '%s' or library_type == '%s' to run 'z count')allanyrcr_envproductr'r-r.) sample_defrrrcheck_gex_or_ab_presentWs r4cCs tdSr) tk_preflightZwarn_deprecated_osrrrrcheck_osjs r6Fc st}tt||tjkrt|tdd|D}t t |dD]!}||||dkrFd}|d||d7}|d7}t |q%t dd d |D]}|d } | d kr`t d | dslt d| tj| s|t d|d| tj| rt d| dt| tjst d|dtd| t| st d| |d} | dur| D] } t| st dqtt||tjkr@|rdddtD} n dddtD} |tj dd krd| }t |durt!vs |rtvrt dd| }t ||dur?dur?tj"kr?t#fdd|j$Ds?t dd qO|tjkra|tj ddtj%fvrat dd!tj%d"qOdS)#Ncss4|]}|d|d|d|dfVqdS) read_path sample_namessample_indiceslanesN)r&)r*Zsdrrrr,us " z#check_sample_def..zhDuplicated entry in the input FASTQ data. Please use a unique combination of fastq path and sample name.z Path: rz\ Note in demux mode, a unique combination fastq path, sample indices, and lanes is required.zChecking FASTQ folder...Tflushr7z;Empty fastq path specifed. Please specify an absolute path./z1Specified FASTQ folder must be an absolute path: z On machine: z), specified FASTQ folder does not exist: zSpecified FASTQ path z3 is a file. The path is expected to be a directory.z, z0 does not have permission to open FASTQ folder: z!Specified FASTQ folder is empty: r:z0Lanes must be a comma-separated list of numbers.cs|] }d|dVqdS'Nrr)rrrr,csr@rArr)rrrr,rCzllibrary_type field may not be an empty string. The 'library_type' field in the libraries csv must be one of zUnknown library_type: 'z@'. The 'library_type' field in the libraries csv must be one of c3s|]}|jkVqdSr)Z feature_typer) library_typerrr,sz,You declared a library with library_type = 'z', but there are no features declared with that feature_type in the feature reference. Check that the 'library_type' field in the libraries csv matches at least 1 entry in the 'feature_type' field in the feature reference CSVzR'. For the VDJ pipeline, the library_type field in sample_def must be missing or 'rB)&socket gethostnamer!r5Zcheck_gem_groups cr_constantsZ PIPELINE_VDJr4sortedrangelenrprintstrip startswithospathexistsisfileaccessX_OKr1r2listdirr%Zcheck_sample_indicesPIPELINE_COUNTjoinSPATIAL_ALLOWED_LIBRARY_TYPESPUBLIC_LIBRARY_TYPESr&r'r(ALLOWED_LIBRARY_TYPESr-r0 feature_defsZVDJ_LIBRARY_TYPE) Z sample_defs feature_refpipeline is_spatialhostnameZ sd_entriesirr3r7r:ZlaneoptionsrrDrcheck_sample_defns                rb)z chrLength.txtzchrNameLength.txtz chrName.txtz chrStart.txtZGenomezgenomeParameters.txtZSAZSAindexreference_pathstrcCs|dusJt}td|d|dddtjtjg}|D]}tj||}tj |s6t d|dqtj|tj }tj|tj d}tj |sdtj |sdt d |d tj d |dt D]}tj|tj |}tj|s{t d qft|dS) NzChecking reference_path (z) on z...Tr<ztYour reference does not contain the expected files, or they are not readable. Please check your reference folder on .z.gzzEYour reference is missing gene annotations that should be present at r?zG[.gz], or they are not readable. Please check your reference folder on zLYour reference doesn't appear to be indexed. Please run the mkreference tool)rFrGrLrHZREFERENCE_METADATA_FILEZREFERENCE_FASTA_PATHrOrPrWrRrZREFERENCE_GENES_GTF_PATHSTAR_REQUIRED_FILESZREFERENCE_STAR_PATHrQr)rcr_Zrequired_filesfilenamepZp_gtfZp_gtf_gzrrr check_refdatas2      ric Csddtjh}z t|d|}Wntjy#}ztt||d}~wwg}|D]S}t||D]}|dus>||durGd|j }t||| ||<q2|d dkr\tdddd dd dd |dd |dgtj|tjd dgi}| |q(|S)NZfastqssample librarieszjInvalid libraries CSV file: incorrrect number of columns on line number (after excluding comment lines) %dr>z_Empty sample field in libraries csv. Please specify an non-empty sample value for each library.Z fastq_modeZILMN_BCL2FASTQZ gem_groupr:r7r8r9r0) r'r(rload_csv_filter_commentsCSVParseExceptionrrdrLkeysline_numrMappend) Zcsv_path required_colsreaderexcrkrowkeyrlibraryrrrexpand_libraries_csvs@     rwinput file_descrPrzcCsHt|tjst|d|t|jdkr"t|d|ddS)z1Check that the input file exists and is readable.z6 file is not readable, please check file permissions: r file z is empty.N)rOrSR_OKrstatst_size)rPrzrrrcheck_file_properties+s rc CsDzt||}t||WSty!}ztt||d}~ww)zuTry creating the feature reference object. Raises: PreflightException: wrapped FeatureDefException. N)rna_feature_refZfrom_transcriptome_and_csvZFeatureExtractorrrrd)Ztranscriptome_ref_pathfeature_ref_pathr\exrrrtry_load_feature_ref6s rcCsttdSr)r!r5Z check_open_fhrrrrcheck_environmentHsrc@seZdZUded<ded<dS) _VersionCmdrdnamez list[str]cmdN)rrr__annotations__rrrrrLs  rmroz --version)rrZmrppythonnumpy)r-cz&import numpy; print(numpy.__version__)Zscipy)rrz&import scipy; print(scipy.__version__)Zpysam)rrz&import pysam; print(pysam.__version__)Zh5py)rrz$import h5py; print(h5py.__version__)Zpandas)rrz(import pandas; print(pandas.__version__)STARZsamtoolscCsLtD]!}|j}|j}t|jdddd}t||d|ddqdS) zPrint the versions of various dependencies to the given destination. Args: file (file-like object): The destination to which to print the versions.  r;)maxsplitrz: T)sepfiler=N) _PACKAGE_VERSION_CMDSrr tk_subproc check_outputrMsplitdecoderL)destpackagerrversionrrrrecord_package_versions_s  rcCs|dkrtddS)Nr;z9Specified read length must be greater than or equal to 1.r)r+rrrcheck_read_lengthmsrc sddtfdd|Dr|durtdddd d tfd d|Dr.td |raz tj|d d\}}WntyN}ztt||d}~wwtfdd|Drct|dSdSdS)zTIf any non "Gene Expression" libraries are present then the feature-ref is required.cSs |d}|dur dS|tjkSNrEF)r&r'r-rjrErrris_not_gene_expressionv  z8check_feature_preflights..is_not_gene_expressionc3|]}|VqdSrrr))rrrr,|z+check_feature_preflights..NzFYou must specify --feature-ref when using feature barcoding libraries.cS |d}|dur dS|tjkSr)r&r'CRISPR_LIBRARY_TYPErrrris_crispr_guide_capturerz9check_feature_preflights..is_crispr_guide_capturecSrr)r&r'ANTIGEN_LIBRARY_TYPErrrris_antigen_capturerz4check_feature_preflights..is_antigen_capturec3rrrr))rrrr,rzEAnalysis of Antigen Capture libraries is unsupported in this product.rZ index_offsetc3rrrr))rrrr,r)r0rrparse_feature_def_filerrdZcheck_crispr_target_gene_name)r3rr[_rsr)rrrrcheck_feature_preflightsss2 r target_panel str | Nonefeature_reference_path parse_filesboolexpected_targeting_methodr^c s|dur|dur|durtd|durdSt|ddztj|t|ddgdWntjy?}ztt||d}~ww|durJt||g|r<|durVdd}nt j |dd j D}zt j|||d \}} } Wnty}ztt||d}~ww|durt||nd} t| tkrtd | d t| d t|vrtt||} |d} | | kstd| d| ddurtfdd| Drtd| d |dd}|r|tvrtd|ddur>|dur@dd j Dfdd| D}dt|i}tj|tj d\}} z t|j |WdSty;}ztt||d}~wwdSdSdS)a3Check preflights for the target panel. Args: target_panel (str | None): Target panel csv reference_path (str | None): Location of the reference feature_reference_path (str | None): Location of the feature reference path parse_files (bool): Flag to parse the GTF to verify that the gene is present expected_targeting_method (str): Type of targeting method used (Need more details) is_spatial (bool): Is this a spatial sample or not? Raises: PreflightException: Any Preflight specific Exception NzDMust specify a transcriptome or probe set or feature reference path.zThe probe set CSVryztarget panel or probe setgene_id)rgdescriptive_namerqcSsi|]}|j|jqSr)ridr*Zgenerrr sz.check_targeting_preflights..)Zref_gene_indexgene_name_to_idrz+Ten or more genes must be specified in the zI file for compatibility with downstream analysis. Number of genes found: reZreference_genomez+The reference genome of the transcriptome 'z' and probe set 'z' must be identical.c3s|] }|duVqdSr)Zgene_id_to_intr*r) gene_indexrrr,s z-check_targeting_preflights..zLThere are no gene IDs in common between the reference transcriptome and the panel_typez"The provided targeted panel type "z>" is UNSUPPORTED in this product and results may be incorrect.cSsi|]}|jt|qSr)rrrrrrscsg|] }|vr|qSrrr)gene_id_to_indexrr s z.check_targeting_preflights..r>r)rrrrlr r&rmrdcheck_targeting_method cr_referenceZ NewGeneIndexZload_from_referenceZgenestgt_simple_utilsZparse_target_csv ExceptionrK$MINIMUM_TARGET_PANEL_GENES_PREFLIGHTr r r r/rmartianalarmrIrrZcheck_crispr_target_gener)rrcrrrr^errrZtarget_panel_metadataZtarget_panel_genesrrZtranscriptome_reference_genomeZprobe_set_reference_genomerZtarget_set_gene_indicesZtarget_sets_dictZcsv_feature_defser)rrrcheck_targeting_preflightss               5rcCst|d;}|}|dkrtd|d|D]}tt|t}|| vr8td|d|dqWddS1sDwYdS)NrrzThe molecule info file z has no associated libraries.z& is too old and is missing the metric z. Please rerun.) ropenZget_library_infornextiterZget_all_metricsrvaluesrn)Z mol_info_fnZmetrics_expectedmcZ num_librariesZmetricZlibrary_metricsrrr,check_molecule_info_contains_library_metricss  "rcCshd}d}t}|D](}|dkr|}n||kr-||||||td||d7}q dS)zTakes in a list of target sets.rNz8Target sets are not the same. Found inconsistent genes: r;)setupdate differencer) target_setscounterZglobal_target_setZinconsistent_genes target_setrrrcheck_target_features_same(s rcCs\d}|durtdddt|dd|r|durt||}tdddt||tj|ddS)Nz#Checking feature definition file...Tr<zfeature referenceryzChecking sample info...)r]r^)rLrrrbrHrV)r3rc full_checkrr^r\rrrcheck_sample_info;s       rc CsD|durdSt|dkrtd|t|dkrtdt|}t|dd|ddz_t|d|}t|dkrJtd|d |dd t||}|dur_td |d |d |j|krstd|d|j d|d t ||j \}}}}t |dt|||WdSt jy}ztt||d}~ww)zyDetermines the targeting method specified by a given panel or target file and compares the result to the specified value.Nr;z'Multiple different target sets found: rz1A target or probe set was expected, but not foundzThe z csvryr{z contains no metadata.z(No file format version specified in the z file header. Please check your z file.zThe header in the z file indicates this is a z file. Please check your )rKrr Z get_file_namerrZload_target_csv_metadataZ-determine_targeting_method_info_from_metadatamethodZfile_format_tagZ.get_target_panel_or_probe_set_file_format_specZ file_versionZcheck_target_csv_metadatarrmrd) targeting_methodrZ expected_nameZtarget_set_metadataZtarget_method_inforZrequired_metadataZconflicting_metadatarrrrrMsZ       rcCstddd|durt||durtdddt||dur*tdddt||r5tdddttddd|rA|dnd}|rI|dnd}|durW|durYtd dSdS) NzChecking reference...Tr<zChecking read 1 length...zChecking read 2 length...zChecking system environment...zChecking optional arguments...Z per_gem_wellzECannot specify both --force-cells and --expect-cells in the same run.)rLrirrr)rrc r1_length r2_lengthrecovered_cells force_cellsrrrcheck_common_preflightss(     r)NNF)rcrd)rPrdrzrd) rrrcrrrrrrrdr^r)NF)W __future__rrOrFsystypingrrZcellranger.constants constantsrHZcellranger.envenvr1Zcellranger.referenceZ referencerZcellranger.rna.feature_refZrnar\rZcellranger.rna.libraryrvr'Z cellranger.targeted.simple_utilsZtargetedZ simple_utilsrZtenkit.log_subprocessZlog_subprocessrZtenkit.preflight preflightr5Z cellrangerrZcellranger.fast_utilsrcellranger.feature_refrZcellranger.molecule_counterrrZcellranger.reference_pathsr r Z&cellranger.targeted.targeted_constantsr r r Z$cellranger.targeted.targeted_spatialrr-rr.rZMULTIPLEXING_LIBRARY_TYPEZCUSTOM_LIBRARY_TYPErZrYrXrrr!r%r4r6rbrfrirwrrrrrstdoutrrrrrrrrrrrrrrs               h  ) -  u  8