o Uݢg< @sddlmZddlZddlZddlZddlmZmZddlm Z m Z ddl m Z m Z mZddlZddlZddlmZddlmmZddlmZddlmZddlmmZ ddl!mZ"ddl#m$Z$ddl%m&Z&dd lm'Z'dd l(m)Z)dd l*m+Z+ed Z,d Z-dZ.dddddej/e.gZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;dZe?d"Z@e=e@iZAee;ejBfe6ejCfe7ej?fe8ejBfed?ZbdidBdCZcdjdEdFZddkdldJdKZeGdLdMdMejfZgdmdPdQZhdRdSZidndVdWZjdodZd[Zkdpd\d]Zldqdrd`daZmdbdcZndddeZoGdfdAdAZpGdgdhdheqZrdS)s) annotationsN) OrderedDict defaultdict)Callable Generator)Any NamedTupleTypeVar) ensure_binary)concatenate_molecule_infos)FeatureReference LIBRARY_TYPE)TARGETING_METHOD_TL)tables_T1Zmolecule file_versionTITLECLASSVERSIONFILTERSPYTABLES_FORMAT_VERSION metrics_jsonmetrics barcode_infoZprobes barcode_idxZ feature_idx library_idxZ probe_idxcount gem_groupZumiZumi_typebarcodes library_info gem_groups libraries is_aggregatedtargeting_methodZmolecule_info_typerawanalysis_parametersZraw_read_pairsZ#raw_read_pairs_in_filtered_barcodesZdownsampled_readsZusable_read_pairsZfeature_read_pairsZdownsampled_feature_read_pairsZon_target_usable_read_pairsrecovered_cells force_cellsZinclude_intronsF filter_probesTZno_probe_filtergzipic@s*eZdZUdZdZdZded<ded<dS) BarcodeInforr! np.ndarray pass_filter list[str]genomesN)__name__ __module__ __qualname__PASS_FILTER_BARCODE_IDXPASS_FILTER_LIBRARY_IDXPASS_FILTER_GENOME_IDX__annotations__r:r:j/oak/stanford/groups/akundaje/marinovg/programs/cellranger-9.0.1/lib/python/cellranger/molecule_counter.pyr-s  r-uint64strr0r2mcMoleculeCountercCst|tjr|jjtjur|jjt|dd St |dkr0tj gddd}|jjt|t dd St dd|D}td |}|jjttj ||t |d t dd S) zWrites the barcodes to the molecule info. Args: mc: MoleculeCounter instance barcodes: a numpy array of strings or iterable of strings )datarSF)dtypecopyrA compressioncs|]}t|VqdSNlen.0xr:r:r; z"_write_barcodes..S%drN) isinstancenpndarrayrCtypebytes_h5create_datasetBARCODE_DS_NAMErJarrayHDF5_COMPRESSIONmaxfromiter)r?r Zbc_arraymax_barcode_len barcode_dtyper:r:r;_write_barcodess    r`namecCs,t|}|jj|dd|ttfd|j|<dS)N)rrH)maxshaperCrFchunks)MOLECULE_INFO_COLUMNSrWrXr[HDF5_CHUNK_SIZEcolumns)r?racol_typer:r:r;rXsrX逄returnr/cCs\|}|jddtjf}|j|ddD]\}}t||t|||}qt |S)zAGet barcode indices which have nonzero counts or pass the filter.NFpreserve_boundaries) get_barcode_infor0r-r6 get_chunksrSunion1dget_column_lazyBARCODE_IDX_COL_NAMErZ)r?Z tgt_chunk_lenrZ unique_bc_idx chunk_start chunk_lenr:r:r;get_barcode_index_to_retains rsc@seZdZdZddZdS) NumpyEncoderz$Custom encoder for numpy data types.cCst|tjtjBtjBtjBtjBtjBtjBtj Btj Btj Btj Br(t |St|tjtjBtjBr8t|St|tjtjBrH|j|jdSt|tjrR|St|tjr\t|St|tjrddStj||S)zEncode the object. Args: o: The object to encode. Returns: A transformed version of the object to encode. )realimagN)rRrSint_intcintpint8int16int32int64uint8uint16uint32r<intfloat16float32float64float complex64 complex128rurvrTtolistbool_boolvoidjson JSONEncoderdefault)selfor:r:r;rsH      zNumpyEncoder.defaultN)r3r4r5__doc__rr:r:r:r;rts rtversionrcCs||_t|jt|dS)zqSet the file version. Args: mc: an moleculecounter instance version: The version to set N)rcr_h5 set_hdf5_attrrWFILE_VERSION_KEY)r?rr:r:r;set_file_versionsrcs.t|dd}|jfddDS)Nrz/metricscsi|]}||qSr:r:)rLkattrsetr:r; z"get_v2_metrics..)r open_fileget_node_v_attrs_f_list)Zh5_filegroupr:rr;get_v2_metricssrv2_mc_in h5py.Filec Csfddd}tj|tttd}g}||}|D]\}}}}|tt|dt|ttj iq||fS) aMethod to generate library_info information for molecule info files with versions prior to this field. being introduced. Args: hdf_file: an open V2 or earlier molecule info file Returns: A tuple containing a list of library infos, as well as a corresponding list of chunks, one for each gem group with a tuple defining it as (gem_group, chunk_start, chunk_length, lib_idx) gem_group_arrnp.arrayc sttdks Jjd}t}fdd}t|||}g}tt||D]\}\}}| ||d|d|fq,|S)z'Return exactly one chunk per gem group.rcs|SrHr:)irr:r;gg_key*szOget_v2_library_info_and_chunks..get_chunks_by_gem_group..gg_keyr!) rSalldiffshapeuniquer@ get_chunks_from_partition_static enumeratezipappend) rnum_rowsZ unique_ggsr chunk_iterrclib_idxggchunkr:rr;get_chunks_by_gem_group#s   z?get_v2_library_info_and_chunks..get_chunks_by_gem_grouprC library_idN)rr) rSasarrayGEM_GROUP_COL_NAMErdrrr=r rna_libraryGENE_EXPRESSION_LIBRARY_TYPE)rrZ v2_gem_groupsr"rcr_rr:r:r;get_v2_library_info_and_chunkss  rh5flist[dict[str, Any]]cCst|ddS)z}Get the library info. Args: mc: MoleculeCounter instance Returns: Dictionary of molecule infos r"r)rloads)rr:r:r;_get_library_infoFs rcCs(t|\}}|dkrt|dSt|S)avTakes a molecule info filename and loads the library info This method. allows one to either read the `library_info` from new files, or generate one on the fly from older molecule info files without running into version compatibility errors Args: mol_info_fname: file name of a molecule info file Returns: Dictionaries of library infos r)get_h5py_file_and_versionrr)mol_info_fnamer?rr:r:r;get_library_infoRs  rrtuple[h5py.File, int]c Csz tj||d}Wnty}z td|d|d}~wwt|s+td|dt|jvr;t|jt}||fSd}||fS)zOpens a molecule info h5py.File. Args: mol_info_fname: Path to a molecule info file to open Returns: A tuple with objects (h5py.File, file_version_number) modezThe molecule info HDF5 file (z/) is invalid. Please provide a valid HDF5 file.Nz#The input molecule info HDF5 file (zH) does not appear to be properly formatted. Please provide a valid file.r!)h5pyFileOSErroris_valid_mol_info_h5 ValueErrorrattrsr)rrZmc_h5exrr:r:r;rds&    rcCstdd|jDS)adChecks whether a file is a valid molecule info file. The molecule info HDF5 file is deemed valid if and only if all of its attribute keys exist in VALID_MOL_INFO_H5_ATTRIBUTE_KEYS Args: mol_info: Open h5py.File for the molecule info HDF5 file to be validated Returns: A boolean with True for valid files, False otherwise css|]}|tvVqdSrH) VALID_MOL_INFO_H5_ATTRIBUTE_KEYS)rLar:r:r;rNrOz'is_valid_mol_info_h5..)rr)Zmol_infor:r:r;r~s rcCsld}||vr ||St|dtr|ddS|dd}d}|d|kr4|d}t|tr2|dS|SdS)NZchemistry_barcode_whitelistchemistry_barcode whitelistrZgel_beadkindra)rRdict)rZold_wl_metric_key barcode_defZGB_BARCODE_KINDrr:r:r;barcode_whitelist_from_metricss    rc@seZdZdZddZddZddZdd d Zdd dZdddZ dddZ dddZ e dddZ e dddZe ddd#d$Ze dd*d+Ze , ,ddd0d1Ze dd5d6Ze dd7d8Zdd9d:Ze , , , ,ddd?d@ZddAdBZddCdDZddGdHZdIdJZdddNdOZdPdQZddSdTZddWdXZddZd[Zdd^d_Z ddadbZ!ddcddZ"ddedfZ#ddgdhZ$ddjdkZ%ddldmZ&dndoZ' pdddsdtZ(ddvdwZ)ddydzZ*dd{d|Z+dd~dZ,ddZ-ddZ.ddZ/ddZ0e dddZ1e ddddZ2dddZ3dddZ4e dddZ5dddZ6e dddZ7 ddddZ8e dddZ9e dddZ:e dddZ;e dddZ ,ddddZ?dddZ@dddZAdddĄZBdddƄZCe dddɄZDe dd˄ZEe ddd̈́ZFe dddτZGd,S(r@z@Streams a list of tuples w/named elements to or from an h5 file.cCs,d|_d|_t|_t|_d|_d|_dSrH)rrWrrf ref_columnsr"feature_referencerr:r:r;__init__s  zMoleculeCounter.__init__cCs t|SrH)rget_all_metricsrr:r:r;get_gb_barcode_whitelists z(MoleculeCounter.get_gb_barcode_whitelistcCsH|}|dd}|r"t|tr"|dd}t|tr"|ddSdS)NrrrZslide)rgetrRlistr)rrrrr:r:r;get_visium_hd_slide_names    z(MoleculeCounter.get_visium_hd_slide_nameri list[int]cCsdd|tDS)NcSsg|]}t|qSr:)rrKr:r:r; z2MoleculeCounter.get_gem_groups..) get_metricGEM_GROUPS_METRICkeysrr:r:r;get_gem_groupszMoleculeCounter.get_gem_groupsr1cCs|jjtjdS)N) feature_type)r get_genomesrDEFAULT_LIBRARY_TYPErr:r:r;rzMoleculeCounter.get_genomesr=cCs|ttSrH)rrMOLECULE_INFO_TYPE_METRICMOLECULE_INFO_TYPE_COUNTrr:r:r;get_molecule_info_typerz&MoleculeCounter.get_molecule_info_typercCs|tjdSNr)get_ref_column_lazyBARCODESrrr:r:r;get_barcode_list_sizerz%MoleculeCounter.get_barcode_list_sizercCs|t}|dur |SdS)NF)rIS_AGGREGATED_METRIC)rretr:r:r;r%s zMoleculeCounter.is_aggregatednp.dtypecCstt|SrH)rSrCrd)rr:r:r;get_column_dtypez MoleculeCounter.get_column_dtypecCstddtDS)Ncss|] }t|jVqdSrH)rSrCitemsizerKr:r:r;rNz3MoleculeCounter.get_record_bytes..)sumrdvaluesr:r:r:r;get_record_bytessz MoleculeCounter.get_record_bytes?TrrscalercapcCs6tdt}t|||}|rttj|S|S)zOEstimate memory usage in GB (not GiB) of this object given a number of records.geA)rr@rmathceilr\ h5_constants MIN_MEM_GB)rrrrZmol_entries_per_gbmem_gbr:r:r;estimate_mem_gbs  zMoleculeCounter.estimate_mem_gbfiltered_barcodes_by_genomedict[bytes, list[bytes]]r list[bytes]r-cCsHddtt|D}tt}t|D] \}}||d|qddt|D}g}|D])\} } || } | D]} t| \} }|| }||}|D] }|||| fqOq.rcSr r:r:)rLrbcr:r:r;rrrr0r)rrr!rN)r2)rsortedrrrritemscr_utilssplit_barcode_seqrJrSrZBARCODE_INFO_DTYPESzerosrargsortr-)r r"r Z genome_to_idxZlibraries_for_gem_grouprlibZ bc_seq_to_idxZ pf_tuplesgenomebcs genome_idxZbc_strseqrr library_indsrr0r:r:r;build_barcode_infos4 " z"MoleculeCounter.build_barcode_infoNrr int | Nonecs|j}|ddtjf}|ddtjf}|ddtjf}tj|jdtd} |dur2| ||kM} durPtj fddt |Dt dd} | t || M} t | } tj dd|Ddd} | || } tjt| || fdd fd dtjdDS) aGet a list of filtered barcode strings e.g. ['ACGT-1',...]. Args: barcode_info (BarcodeInfo): Barcode info object. library_info (list of dict): Library info. barcodes (np.array): Barcode sequences. genome_idx (int): Restrict passing definition to this genome. None for no restriction. library_type (str): Restrict passing definition to this library type. None for no restriction. Returns: list of str Nrrcs g|] \}}|tkr|qSr:r )rLrr) library_typer:r;r2 z9MoleculeCounter.get_filtered_barcodes..rcSsg|]}|dqS)rr:)rLrr:r:r;r8rr<axiscs,g|]}t|df|dfqS)r!r)rformat_barcode_seq)rLr)r gg_bcsr:r;r@s)r0r-r6r7r8rSonesrrrZrrdisin flatnonzeror column_stackrange)rr"r rrr0Zpf_barcode_idxZpf_library_idxZ pf_genome_idxmaskrindsZ lib_to_ggZ pf_gem_groupr:)r r#rr;get_filtered_barcodess(     z%MoleculeCounter.get_filtered_barcodesbc_infor h5py.GroupcCs<|jd|jd|jjdftddtj|d|jtdddS)zSave barcode info to HDF5. Args: barcode_info (BarcodeInfo): Data. group (h5py.Group): Output group. r0Nr!T)rArbrFshuffler2)rFr.)rXr0rr[rcreate_hdf5_string_datasetr2)r,rr:r:r;save_barcode_infoEs  z!MoleculeCounter.save_barcode_infocCs"t|dddt|ddS)zLoad barcode info from an HDF5 group. Args: group (h5py.Group): Input group. Returns: BarcodeInfo: object r0Nr2r>)r-rread_hdf5_string_dataset)rr:r:r;load_barcode_infoXs  z!MoleculeCounter.load_barcode_infocCst|jtSrH)r@r2rWBARCODE_INFO_GROUP_NAMErr:r:r;rlgrz MoleculeCounter.get_barcode_infor feature_refFeatureReference | NoneBarcodeInfo | Nonec Cs|dvsJ|}|dkr||durtd|durtd|dur%td|dur-tdt|d|_t|tt|jtj t |j tj }| |t||tj|dd d } t|jd | g|j t} t|| tD]} t|| qr|St||d \|_|_t|jtsJ|jtkrtd |d|jd|jdkrtd |d|jd|jD]B} | tvr|j| |j| <q| tvr|j| |j| <q| tj krt |j| |_!q| t"ks| tks| t#ks| t$vrqt%d| t&t'|jd d|_(|S)a Open a molecule info object. Args: filename (str): Filename to open or create mode (str): 'r' for reading, 'w' for writing. feature_ref (FeatureReference): Required when mode is 'w'. barcodes (list of str): All possible barcode sequences. Required when mode is 'w'. library_info (list of dict): Library metadata. Required when mode is 'w'. barcode_info (BarcodeInfo): Per-barcode metadata. Returns: MoleculeInfo: A new object )rzr+wr7NzSFeature reference must be specified when opening a molecule info object for writingzJBarcodes must be specified when opening a molecule info object for writingzNLibrary info must be specified when opening a molecule info object for writingzNBarcode info must be specified when opening a molecule info object for writingT)indent sort_keysr"rz#The molecule info HDF5 file (file: z, format version zO) was produced by a newer software version. Reading these files is unsupported.rzP) was produced by an older software version. Reading these files is unsupported.zUnrecognized dataset key: r))rrrrWrCURR_FILE_VERSIONrrrH5_FILETYPE_KEYMOLECULE_H5_FILETYPE create_groupH5_FEATURE_REF_ATTRto_hdf5r`rdumpsr/r3r@r0rdrXrrrRrrrfMOLECULE_REF_COLUMNSrr from_hdf5rV3_METRICS_GROUP_NAMEMETRICS_JSON_DATASET_NAMEUNIMPLEMENTED_PROBE_KEYSAttributeErrorrr1r") clsfilenamerr4r r"rr?Z fref_groupZ lib_info_jsonr rakeyr:r:r;openjs       %   zMoleculeCounter.opencCst|p |duS)zUsing the chemistry whitelist, determine if this is Spatial (Visium) data. Returns: bool: True if the known barcode list is for Spatial data. N)bc_utilsZis_whitelist_spatialrrrr:r:r;is_spatial_datas zMoleculeCounter.is_spatial_datacCs|tttjdSr)ronextiterrdrrr:r:r;nrowsrzMoleculeCounter.nrowsidxtuple[Any, Any]cstfddtDS)Nc3s|] }|VqdSrH)ro)rLcolrQrr:r;rNsz0MoleculeCounter.get_chunk_key..)tuple CHUNK_COLUMNS)rrQr:rTr; get_chunk_keyrzMoleculeCounter.get_chunk_keycCs|}|||<||dS)z4Set a metric. Serialize to Pickle. N)rset_all_metrics)rrJvaluerr:r:r; set_metricszMoleculeCounter.set_metricrJrrcCs|||S)z Get a metric.)rr)rrJrr:r:r;rszMoleculeCounter.get_metriccCs tj|ddtd}||jt<dS)NT),:)r: separatorsrH)rrArtrWrE)rrrr:r:r;rXszMoleculeCounter.set_all_metricsdict[str, Any]cCs4t|jvrt|jtdSt|jvr|SiS)zReturn a dictionary of metrics.r:)rErWrrrD_legacy_get_all_metricsrr:r:r;rs  zMoleculeCounter.get_all_metricsrrcCsdd|jtjDS)NcSs i|] \}}|tt|qSr:)picklerr rLrvr:r:r;rsz;MoleculeCounter._legacy_get_all_metrics..)rWrDrrrr:r:r;r_sz'MoleculeCounter._legacy_get_all_metricsracCsD|j|}t|}|t|}||f||||<|jdS)z&Append an array of values to a column.N)rfrJresizerWflush)rrardsstartendr:r:r; append_columns    zMoleculeCounter.append_columncol_name h5py.DatasetcC |j|S)zRetrieve column. Does not handle missing columns. Depending on how the file was opened, this may only be a file view instead of a full array. )rfrrir:r:r;ro s zMoleculeCounter.get_column_lazyr/cCs8||jvr||ddStj|t|t|dS)zULoad an entire column of data into memory. Handles missing columns. Nr)rfrorSfullrPMOLECULE_INFO_DEFAULT_VALUESrdrlr:r:r; get_columns zMoleculeCounter.get_columncCs2|tvsJ|jj|jj|t|d|j|<dS)N)obj)rBrW create_carrayrootrSrZr)rrirr:r:r;set_ref_column&s zMoleculeCounter.set_ref_columncCs||ddS)z4Load a reference array into memory as a numpy array.N)rrlr:r:r;get_ref_column-zMoleculeCounter.get_ref_columncCrk)z-Get a reference array as a lazy h5py Dataset.)rrlr:r:r;r1s z#MoleculeCounter.get_ref_column_lazyr cCst|jtjS)z(Returns the feature reference from HDF5.)r rCrWrr?rr:r:r;get_feature_ref5ruzMoleculeCounter.get_feature_refcCs|jtddS)z!Returns fully loaded barcode set.N)rWrYrr:r:r; get_barcodes9ruzMoleculeCounter.get_barcodescs|}|}|j}t|}||}tt}tjt||dt |D]\}}||<q$|j dddD]\} } |j t| | | } | |j t| | | <q4fdd|ddt j fD|ddt j f<t ||jd} |j t=t|||j t=|j t} t| | |S) zChanges the data in the dataset so that only barcodes with have > 0 counts and/or were pass filtered are. stored in the file. Updates the barcode_idx file to indicate the new indices Returns: rrhFrjcsg|]}|qSr:r:rKZ new_positionsr:r;rTsz1MoleculeCounter.trim_barcodes..Nr>)rwrlr0rsrdrprSrrJrrmrWr-r6r2rYr`r3r>r@r0)rZ old_barcodesZold_barcode_infoZnew_pass_filterZbc_idx_to_retainZ new_barcodesZ col_dtypenrZ chnk_startZchnk_lenZfutureZnew_barcode_infor r:rxr; trim_barcodes=s6     zMoleculeCounter.trim_barcodesidxs chunk_sizec Cs||}|}|j|fkrT|jtkrTtj|f|jd}d}td||D]*}t |||} |||| } || kr?q'||| ||| ||| <| }q'|S|jddkrctjg|jdSt d)z5Get the column values at row indices (boolean array).rrz$index value arrays not yet supported) rorPrrCrrSemptyrr(minNotImplementedError) rrir|r}rSZnmolvalsZlwrbeginrgZuprr:r:r;get_column_with_indicesds  z'MoleculeCounter.get_column_with_indicesrcCsJ|jtddd}t|dddf|k}||df}tt|S)zCount the number of barcodes passing filter for a library. Args: library_idx (int): Index of library to count. Returns: int: Number of filtered barcodes for this library. r0Nr!r)rWr3rSr&rJr)rrr0Zthis_libZ barcode_indsr:r:r;%get_num_filtered_barcodes_for_library|s  z5MoleculeCounter.get_num_filtered_barcodes_for_library library_idxscCsF|jtddd}|jdksJtj|ddddf|kddS)aCount the number of barcodes passing filter for a range of libraries. This is the vectorized equivalent to .. code-block:: python np.array( [ mc.get_num_filtered_barcodes_for_library(lib_idx) for lib_idx in library_idxs ] ) Args: library_idxs (np.ndarray[int]): 1d array of indicies to count. Returns: np.ndarray[int]: Number of filtered barcodes for each library. r0Nr!r.rr )rWr3ndimrS count_nonzero)rrr0r:r:r;'get_num_filtered_barcodes_for_librariess"z7MoleculeCounter.get_num_filtered_barcodes_for_librariescCs0|jdusJt|jtddddfjS)z'Return the number of filtered barcodes.Nr0r)rWrSrr3sizerr:r:r;get_num_filtered_barcodess"z)MoleculeCounter.get_num_filtered_barcodesrcCs t|jSrH)rrWrr:r:r;rs z MoleculeCounter.get_library_infocCs|SrHr:rr:r:r; __enter__szMoleculeCounter.__enter__cCs |dSrH)close)rrUrY tracebackr:r:r;__exit__ zMoleculeCounter.__exit__cC|jdSrHrWrrr:r:r;rzMoleculeCounter.closecCrrHrrr:r:r;saverzMoleculeCounter.savebc_infoslist[BarcodeInfo]cCst|dksJ|dj}g}|D]}|jjddksJ|j|ks$J||jqtj|dd}|jddkr@tj|dd}t||dS)zMerge a BarcodeInfo into another BarcodeInfo. Args: bc_infos (list of BarcodeInfo): Input BarcodeInfos. Returns: BarcodeInfo: rr!rr r>) rJr2r0rrrS concatenaterr-)rr2Zpfsr,Znew_pfr:r:r;merge_barcode_infoss z#MoleculeCounter.merge_barcode_infosr dict | NoneNonec Cst|dd}|}|}|}dd|jD}g}|D]} t| d} || Wdn1s:wYq t|} tj|d|||| d} d} |D]I} tj| dd8}||ksfJt ||spJ|}d d|jD|ksJ|dur| }| | 7} Wdn1swYqT| || t||t|d} | | ksJd WddS1swYdS) zConcatenate MoleculeCounter HDF5 files. Args: out_filename (str): Output HDF5 filename in_filenames (list of str): Input HDF5 filenames metrics (dict): Metrics to write rrcSg|]}|jqSr:idrLfr:r:r;rz/MoleculeCounter.concatenate..Nr7)rr4r r"rrcSrr:rrr:r:r;rrz/Concatenation did not produce expected results.)r@rKrvrwr feature_defsrrlrrS array_equalrrPrXrr ) out_filename in_filenamesrZfirst_mcr4r r"Z feature_idsrrIr?Zmerged_bc_infoZout_mcZ total_rowsZin_mcZfrefr:r:r;rsL     "zMoleculeCounter.concatenatefrom_rowcCsH|}||}t||D]}||}||kr|dSq|dS)Nr!)rPrWr()rrrZinitial_chunk_keyrZ chunk_keyr:r:r;!find_last_occurrence_of_chunk_keys   z1MoleculeCounter.find_last_occurrence_of_chunk_keyqueryrkey_funcCallable[[int], _T1]cCt|||SrH)r@ bisect_staticrP)rrrr:r:r;bisectszMoleculeCounter.bisectrc Csd}|}d} ||d}||}||krn||dkrd}n ||kr&|}n|}q|rBt|ddD]}||}||krA|dSq1dS)zPerforms a binary search to find the leftmost insertion point of query. Args: key_func: A function, where `key_func(i)` is the value to compare to at index i. rTr.r!F)r() rrrlohiexistsrcurrjr:r:r;rs,   zMoleculeCounter.bisect_staticr list[_T1]&Generator[tuple[int, int], None, None]cCrrH)r@rrP)rrrr:r:r;get_chunks_from_partition>sz)MoleculeCounter.get_chunks_from_partitionc#spdgfdd|ddD}t|}t|D]}||}|d|kr,||dn}|||fVqdS)z3Get chunks by partitioning on the specified values.rcsg|] }t|qSr:)r@r)rLvalrrr:r;rHszDMoleculeCounter.get_chunks_from_partition_static..r!N)rJr()rrrstartsryrrq chunk_endr:rr;rCs  z0MoleculeCounter.get_chunks_from_partition_statictarget_chunk_lenrkccsx|}d\}}||dkr:t|d||d}|r!||n|}d||}||fVd|}||dksdSdS)zGet chunks, optionally preserving boundaries defined by get_chunk_key(). Yields: (int, int): (chunk_start, chunk_len) which are closed intervals )rrr!N)rPrr)rrrkrrqrZtarget_chunk_endrrr:r:r;rmQs    zMoleculeCounter.get_chunks np.uint16cCs tt|SrH)rdr)rMr:r:r;compress_gem_groupfrz"MoleculeCounter.compress_gem_grouprMbytesumi_bitscCs t||SrH)r compress_seq)rMrr:r:r;compress_umi_seqjrz MoleculeCounter.compress_umi_seq metrics_list list[dict]csd}i}i}g|D]@}|durdd|D}dd|D|t}|t}|dur>|D] }|||q4||||tq |dusQJ||t<||t<|tdtfddDrq|d|S)NcSs i|] \}}|ds||qStarget startswithrar:r:r;rvs zAMoleculeCounter.naive_concatenate_metric_list..cSs i|] \}}|dr||qSrrrar:r:r;r|rc3s|] }|dkVqdS)rNr:rKZtargeted_metricsr:r;rNrz@MoleculeCounter.naive_concatenate_metric_list..r) rrrrANALYSIS_PARAMETERS_METRICupdateLIBRARIES_METRICpopr)rcombined_metricsZ gg_metricsZ lib_metricssingle_metricsZsingle_gg_metricsr(rr:rr;naive_concatenate_metric_listns4    z-MoleculeCounter.naive_concatenate_metric_list mol_h5_listc Cs\g}|D]"}tj|dd}|}||Wdn1s!wYqt|}|S)Nrr)r@rKrrr)rrmol_h5counterrrr:r:r;"naive_concatenate_metrics_from_h5ss  z2MoleculeCounter.naive_concatenate_metrics_from_h5s all_metricsc lz$|jdusJ|dur|tn|tfddtt|jDWSty5}ztd|d}~ww)zGet raw read pairs per library. Returns: list of int: Order is by library index Raises: ValueError if expected metrics are missing. Ncg|] }t|tqSr:)r=TOTAL_READS_METRICrLlimetricr:r;rzBMoleculeCounter.get_raw_read_pairs_per_library..#Missing metrics in molecule counterr"rrrr(rJKeyErrorrrrexcr:rr;get_raw_read_pairs_per_librarys   z.MoleculeCounter.get_raw_read_pairs_per_librarycs`z$|jdusJ|dur|tn|tfddtt|jDWSty/tdw)zGet read pairs in filtered barcodes per library. Returns: list of int: Order is by library index Raises: ValueError if expected metrics are missing. Ncrr:)r='TOTAL_READS_IN_FILTERED_BARCODES_METRICrrr:r;rzSMoleculeCounter.get_read_pairs_in_filtered_barcodes_per_library..rr)rrr:rr;/get_read_pairs_in_filtered_barcodes_per_librarys     z?MoleculeCounter.get_read_pairs_in_filtered_barcodes_per_libraryc r)zGet transcriptomic read pairs per library (cell + non-cell). Returns: list of int: Order is by library index Raises: ValueError if expected metrics are missing. Ncrr:)r=FEATURE_READS_METRICrrr:r;rrzMMoleculeCounter.get_transcriptomic_read_pairs_per_library..rrrr:rr;)get_transcriptomic_read_pairs_per_librarys     z9MoleculeCounter.get_transcriptomic_read_pairs_per_libraryc Zz|jdusJ|tfddtt|jDWSty,}ztd|d}~ww)zGet usable read pairs per library. Returns: list of int: Order is by library index Raises: ValueError if expected metrics are missing. Ncrr:)r=USABLE_READS_METRICrrr:r;rrzEMoleculeCounter.get_usable_read_pairs_per_library..rr"rrr(rJrrrrr:rr;!get_usable_read_pairs_per_librarys   z1MoleculeCounter.get_usable_read_pairs_per_libraryc r)zGet usable on-target read pairs per library. Returns: list of int: Order is by library index Raises: ValueError if expected metrics are missing. Ncrr:)r=ON_TARGET_USABLE_READS_METRICrrr:r;rrzOMoleculeCounter.get_on_target_usable_read_pairs_per_library..rrrr:rr;+get_on_target_usable_read_pairs_per_librarys     z;MoleculeCounter.get_on_target_usable_read_pairs_per_librarydict[str, list[int]]cCs8|}tt}t|D] \}}||t|q |S)z0Get indices of libraries that are GEX libraries.)rrrrrr)rr$Zlibrary_indices_by_typerrr:r:r;get_library_indices_by_types z+MoleculeCounter.get_library_indices_by_typecCs|ttkS)zCReturn True if metrics_json/targeting_method is templated_ligation.)rTARGETING_METHOD_METRICrrr:r:r;is_templated_ligationrz%MoleculeCounter.is_templated_ligation metric_typec Cs|tus |tus Jtt}|D]/}tj|dd}||D]\}}||||7<q!Wdn1s:wYq|S)zNCombine a library- or gemgroup- level integer metric across multiple h5 files.rrN)rrrrr@rKrr)r metric_namercombinedrrrJrr:r:r; _sum_metricszMoleculeCounter._sum_metriccCst||tSrH)r@rr)rrr:r:r;sum_library_metricrz"MoleculeCounter.sum_library_metricc Csd}t|d||-}t|d|d|dD]\}}} | dkr$q||f|vr+q|| 7}qWdn1s:wY||dS)Nrrbarcoderreadsr!)r@rKrroput) rIZfiltered_bcs_setrflengthqueueZtotal_mapped_readsr?rrrr:r:r;6get_total_conf_mapped_reads_in_filtered_barcodes_chunk#s   zFMoleculeCounter.get_total_conf_mapped_reads_in_filtered_barcodes_chunkcCs|tjtjko t|SrH)rrrhas_target_set)libraryr:r:r;is_targeted_library3sz#MoleculeCounter.is_targeted_library)rir)rir1)rir=)rir)rir)rir)rT)rrrrrrrrir)r r r r rir-)NN)rr-rr)r,r-rr-)rr-rir-)rir-)NNNN)rr=r4r5rr6rir@)rQrrirRrH)rJr=rr)rir^)rr@rir)rar=)rir=rirj)rir=rir/)rir=)rir )rir/)r{)r|r/r}rrir/)rrrir)rr/rir/rir)rrrir-)rrrir)rrrir)rrrrrir)rrrrrrrir)rrrrrir)rrrrrrrir)T)rrrkrrir)rir)rMrrrrir)rr)rr1)rrrir)rir)rr=)rir)Hr3r4r5rrrrrrrrr% staticmethodrrrrr+r0r2rl classmethodrKrMrPrWrZrrXrr_rhrororsrtrrvrwrzrrrrrrrrrrrrrrrrrmrrrrrrrrrrrrrrrr:r:r:r;r@s       45    h         (      :        %         c@s$eZdZdZddZeddZdS)MergedBarcodesz!Class to hold a list of barcodes.cCsztdd|D}td|}t|d}|jttj||t|ddd| WddS1s6wYdS) zWrite to disk.csrGrHrIrKr:r:r;rN?rOz/MergedBarcodes.write_to_disk..rPr7rQr,rEN) r\rSrCrrrXrYr]rJr)rrIr^r_rWr:r:r; write_to_disk=s "zMergedBarcodes.write_to_diskcCsXt|d}t}||tdd||WdS1s%wYdS)zIDeserialize the data. :param filename: :return: rN)rrrextendrYr)rIrWresultr:r:r;load_from_diskIs $zMergedBarcodes.load_from_diskN)r3r4r5rrrrr:r:r:r;r:s  r)r?r@)r?r@rar=)rh)r?r@rir/)r?r@rr)rr)rrrirr)r)rir)s __future__rrrr` collectionsrrcollections.abcrrtypingrrr rnumpyrSsixr Zcellranger.barcodes.utilsr utilsrLcellranger.h5_constantsrcellranger.hdf5hdf5rcellranger.rna.libraryrnarrcellranger.utilsrcellranger.fast_utilsr cellranger.feature_refr r&cellranger.targeted.targeted_constantsrcellranger.wrapped_tablesrrr=rr<rr;rErDr3ZPROBE_GROUP_NAMErpZFEATURE_IDX_COL_NAMEZLIBRARY_IDX_COL_NAMEZPROBE_IDX_COL_NAMEZCOUNT_COL_NAMErZ UMI_COL_NAMEZUMI_TYPE_COL_NAMErYrZUMI_TYPE_TXOMICrnrr<rdrFrrBrVrrrrrZMOLECULE_INFO_TYPE_RAWrrrrZDOWNSAMPLED_READS_METRICrrZ DOWNSAMPLED_FEATURE_READS_METRICrZGG_RECOVERED_CELLS_METRICZGG_FORCE_CELLS_METRICZINTRON_MODE_PARAMZINTRON_MODE_HISTORIC_DEFAULTZ FILTER_PROBESZFILTER_PROBES_PARAM_DEFAULTZNO_PROBE_FILTERr[rer-rr`rXrsrrtrrrrrrrrr@rrr:r:r:r;s                .  0 "