o Uݢg@s0ddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z mZmZddlmZddlmZmZmZddlZddlZddlmZddlmZmZddl m!Z!ddl"m#Z#ddl$m%Z&ddl'm(m)Z*ddl+mZ,ddl-m.Z/ddl0m1Z2ddl3m4Z4dd l5m6Z6m7Z7m8Z8dd l9m:Z:d Z;d ZdZ?dSdTddZ@edejAdZBdUddZCdZDdZEdZFd ZGd!ZHe#jIeFge#jJZKGd"d#d#eLZMdVd'd(ZNdWd.d/ZOdXd2d3ZPGd4d5d5ZQGd6d7d7ZRdYd;d<ZS dZd=d>ZTd?d@ZUdAdBZVd[dEdFZWdGdHZXdIdJZYd\dLdMZZd]dOdPZ[dQdRZ\dS)^) annotationsN) OrderedDict)Callable Collection ContainerIterableMappingSequence)BufferedIOBase)AnyTypeVaroverload) ensure_binary ensure_str)MatrixBarcodeIndex)GENOME_FEATURE_TAG FeatureDefFeatureReference)tablesgzipi8int32zfeatures.tsv.gz normalizedaxisintreturn np.ndarraycCs*t|j|d}t|j}||fS)z"Sum a sparse matrix along an axis.r)npasarraysumprodshapeZreshape)matrixrZaxis_sumZmax_dimr#`/oak/stanford/groups/akundaje/marinovg/programs/cellranger-9.0.1/lib/python/cellranger/matrix.pysum_sparse_matrix/s  r%_T)boundarraynp.ndarray[Any, np.dtype[_T]]nIterable[tuple[int, _T]]cCs8t|| | d}|t||}t|||S)zRetrieve the N largest elements and their positions in a numpy ndarray. Args: array (numpy.ndarray): Array n (int): Number of elements Returns: list of tuple of (int, x): Tuples are (original index, value). N)rZ argpartitionZargsortzip)r(r*indicesr#r#r$top_n9s r.r"versionZsoftware_versionc@s eZdZdS)NullAxisMatrixErrorN)__name__ __module__ __qualname__r#r#r#r$r1Usr1originalsp_sparse.spmatrixsubsetcCs|j|jkr ||j}|S)aScipy has the fantastic habit of subsetting a matrix of one dtype down to a float64 if no elements are selected. This function makes the new matrix have the same type as the old matrix, and can be called after subsetting. Args: original: The original matrix subset: The matrix after subsetting Returns: A type correct version of the matrix )dtypeastype)r5r7r#r#r$_ensure_types_matchYs  r:fileh5.File extra_attrs@None | Mapping[str | bytes, str | bytes | Iterable[str | bytes]]NonecCs,|r|D] \}}t|||qdSdS)z"Set optional top level attributes.N)itemscr_h5Z set_hdf5_attr)r;r=kvr#r#r$_save_extra_attrsls rD sw_versionNone | str | bytescCs|r ||jt<dSdSN)attrsSOFTWARE_H5_VERSION_KEY)r;rEr#r#r$_save_sw_versionusrJc@s0eZdZdZ  dYdZdd Zed[d d Zed[ddZd\ddZd\ddZ d]d^ddZ e d_ddZ e d`ddZ dad dZ dbd#d$Z dcd'd(Zddd*d+Zded,d-Zdfd.d/Zdgd2d3Zdhd6d7Zdid8d9Zd:d;Zdjd>d?ZdkdBdCZdldEdFZdmdIdJZdndLdMZdodPdQZdpdSdTZdUdVZdWdXZdS)qCountMatrixViewzBSupports summing a sliced CountMatrix w/o copying the whole thing.Nr" CountMatrixfeature_indicesCollection[int] | None bc_indicescCs~tj|jdd|_tj|jdd|_||_|dur'|jdd|jt|<|dur9|jdd|jt|<| dS)Nboolr8FT) rZones features_dim feature_maskbcs_dimbc_maskr"fillr_update_feature_ref)selfr"rMrOr#r#r$__init__}s   zCountMatrixView.__init__rrcC t|jSrG)r count_nonzerorUrXr#r#r$rT zCountMatrixView.bcs_dimcCrZrG)rr[rSr\r#r#r$rRr]zCountMatrixView.features_dimcCs2t|j}t|j|_t|j|_||SzReturn a copy of this view.)rKr"rcopyrUrSrW)rXviewr#r#r$_copys zCountMatrixView._copycCs|Sr^)rar\r#r#r$r`zCountMatrixView.viewr int | NonecCstj|jj|j|j|dS)zSum across an axis.r) cr_sparseZ sum_maskedr"mrSrU)rXrr#r#r$rszCountMatrixView.sumr? threshold np.uint64cCdSrGr#rXrrfr#r#r$count_gezCountMatrixView.count_ge$np.ndarray[int, np.dtype[np.uint64]]cCrhrGr#rir#r#r$rjrk0np.uint64 | np.ndarray[int, np.dtype[np.uint64]]cCst|jj|j|j||S)z+Count number of elements >= X over an axis.)rdZcount_ge_maskedr"rerSrUrir#r#r$rjsr-list[int] | np.ndarraycCs@|}tj|t|jdt}d||dk<|j|M_|S)z\Select a subset of barcodes (by index in the original matrix) and return the resulting view.Z minlength)rarbincountlenrUr9rPrXr-r`maskr#r#r$select_barcodess  zCountMatrixView.select_barcodes barcode_seqs list[bytes]cCs|j|}||SrG)r" bcs_to_intsrurXrvr-r#r#r$select_barcodes_by_seq  z&CountMatrixView.select_barcodes_by_seq gem_groupcs|fdd|jjDS)Ncs"g|] }t|dkr|qSrp)cr_utilsZsplit_barcode_seq.0bcr|r#r$ s"z@CountMatrixView.select_barcodes_by_gem_group..)rzr"bcsrXr|r#rr$select_barcodes_by_gem_groupsz,CountMatrixView.select_barcodes_by_gem_groupcs:tj}tfdd|Djjjjjjd_dS)z.)r all_tag_keystarget_featuresN)r flatnonzerorSrr"rrr)rXr-r#r\r$rWs  z#CountMatrixView._update_feature_refcCsH|}tj|t|jdt}d||dk<|j|M_||S)z:Select a subset of features and return the resulting view.rorp)rarrqrrrSr9rPrWrsr#r#r$select_featuress  zCountMatrixView.select_featuresgenomestrcCsDg}|jjjD]}|jtjkr|jd|kr||jq| |SzMSelect the subset of gene-expression features for genes in a specific genome.r r"rr feature_type rna_libraryDEFAULT_LIBRARY_TYPEtagsappendindexrrXrr-featurer#r#r$select_features_by_genomes   z)CountMatrixView.select_features_by_genome feature_typesContainer[str]cCsZg}|jjjD] }|j|vr'd}|jtjkr|jd|krd}|r'||jq| |S)zSubset the features by types and genome. Select the subset of gene-expression features for genes in a specific genome and matching one of the types listed in feature_types. TrFr)rXrrr-rZinclude_featurer#r#r$#select_features_by_genome_and_typess    z3CountMatrixView.select_features_by_genome_and_typescCs4g}|jjjD] }|j|vr||jq||S)zSubset the features by type. Select the subset of gene-expression features for genes in a specific genome and matching one of the types listed in feature_types. )r"rrrrrrrXrr-rr#r#r$select_features_by_typess    z(CountMatrixView.select_features_by_typescC|||SzVSelect the subset of features with a particular feature type (e.g. "Gene Expression").)rget_feature_indices_by_typerXrr#r#r$select_features_by_typez'CountMatrixView.select_features_by_type feature_idsIterable[bytes]cCs||j|SrG)rr"feature_ids_to_intsrXrr#r#r$select_features_by_idsz&CountMatrixView.select_features_by_idsr list[int]cCs|jj|S)zDReturn the list of indices of features corresponding a feature type.)r"rZget_indices_for_typerr#r#r$rz+CountMatrixView.get_feature_indices_by_type list[str]cCrZKGet a list of the distinct genomes represented by gene expression features.rL_get_genomes_from_feature_refrr\r#r#r$ get_genomes r]zCountMatrixView.get_genomesrlist[bytes] | set[bytes]cCs(t|j|jjkrtd|j|S)NzDCalling bcs_to_ints on a barcode-sliced matrix view is unimplemented)rr[rUr"rTNotImplementedErrorrx)rXrr#r#r$rxs  zCountMatrixView.bcs_to_intsbc_intscsD|dus t|dkr gStj}|t|}fdd|DS)Nrcsg|]}jj|qSr#)r"rrr\r#r$rz/CountMatrixView.ints_to_bcs..)rrrrrUr)rXrZsliced_bc_intsZ orig_bc_intsr#r\r$ ints_to_bcss  zCountMatrixView.ints_to_bcsrbytescC|jj|jSrGrridrXrr#r#r$int_to_feature_idz!CountMatrixView.int_to_feature_idtuple[int, int]cCst|jt|jfSz&Return the shape of the sliced matrix.)rr[rSrUr\r#r#r$ get_shape!szCountMatrixView.get_shapecCs|jdddS):Return the number of nonzero entries in the sliced matrix.Nrp)rrf)rjr\r#r#r$get_num_nonzero%rzCountMatrixView.get_num_nonzerocCs |jddSNrr)rr\r#r#r$get_counts_per_bc)s z!CountMatrixView.get_counts_per_bcNN)r"rLrMrNrOrN)rr)rrKrG)rrc)rr?rfrrrg)rrrfrrrl)rrcrfrrrm)r-rn)rvrwrrK)r|rrrKrr?)r-rnrrK)rr)rrrr)rr)rrrrK)rrrr)rr)rrrr)rrnrrwrrrr)rr)r2r3r4__doc__rYpropertyrTrRrar`rr rjrurzrrWrrrrrrrrrxrrrrrr#r#r#r$rKzsF                      rKc@seZdZd ddZd d Zd d Zd dZeefdddZ e dddZ dddZ ddd Z dd"d#Zdd%d&Zdd'd(Zdd*d+Zddd0d1Zdd3d4Zdd8d9Zdddd?ZddAdBZddCdDZ E EdddHdIZd dLdMZe d!dOdPZe d"dQdRZe d#dTdUZe d"dVdWZe d$d[d\Ze e j!d,fd%dadbZ"e e j!d,fd&dcddZ#e e j!d,fd'dedfZ$e e j!d,fd(didjZ%e d)dkdlZ&e d*dmdnZ'e d+dodpZ(e dqdrZ)ed,dsdtZ*ed-dwdxZ+e d.dydzZ,e d*d{d|Z-e d*d}d~Z.e d*ddZ/e d*ddZ0e d/ddZ1d/ddZ2e d0ddZ3e d1ddZ4ddZ5ddZ6ddZ7ddZ8 d2d3ddZ9d2d4ddZ:d5ddZ;d6ddZ<d7ddZ=d8ddZ>d9ddZ?d:ddZ@d;ddZAdddZDd?ddZEd@ddZFdAddZGdBdd„ZHdCddńZIe dDddȄZJddʄZKe dEdd̄ZLe dd΄ZMdFddфZNdFddӄZO EdGdHddքZPdFdd؄ZQdFddڄZRdIdd݄ZS EdJdFddZTdFddZUdKddZV E ,dLdMddZWe dNddZXe dOddZYe ddZZe ddPddZ[e ddZ\e ddZ]dQddZ^e ddZ_dRd d Z`dES(SrLrrr#Collection[bytes] | Collection[str]r"r6cCs||_t|j|_dd|jD|_t|tjr$|jj tj ur$| }n(t|dkr3tj gddd}nt dd|D}tj|t|ttj |fd }~d|j_||_|jj\|_t|j|jjj|_||_|jjd t|jksyJd dS) NcSsi|]}|j|jqSr#)rrrfr#r#r$ 8z(CountMatrix.__init__..rSF)r8r_cs|]}t|VqdSrGrrrr#r#r$ @z'CountMatrix.__init__..countr8rpz(Barcodes must be equal to cols of matrix)rrrrrRfeature_ids_map isinstancerZndarrayr8typeZbytes_r_r(maxfromiterflagsZ writeablerr!rTrZfrom_raw_bytestobytesitemsizebcs_idxre)rXrrr"Zbc_arrayZmax_lenr#r#r$rY/s$     "zCountMatrix.__init__cC|jjSr)rer!r\r#r#r$rNrbzCountMatrix.get_shapecCr)r)rennzr\r#r#r$rRrbzCountMatrix.get_num_nonzerocCst|S)zReturn a view on this matrix.)rKr\r#r#r$r`VrbzCountMatrix.viewCollection[bytes]cCs*tjt|jt|f|d}||||dS)zCreate an empty matrix.rQrrr") sp_sparse lil_matrixrrr)clsrrr8r"r#r#r$emptyZszCountMatrix.emptyh5_filer<rc sg}g}g}g}g}g}tt|D]\}}||} tdd|D} || ddd| | ddd} | D] } | vrIt| <q=tjfdd| Dt| dd} | d dd}t|d t| ksoJ|d krt| t t|d sJt |}t|t| dksJt | |}t|t| dksJt|t| d ksJ|||| d dd|| d dd|| ddd|t |t| d qt |}t |}t |}g}t |||D]\}}}|tt|||tjt|idqt|tg}t |}t |}t |}tts0Jtj|||fft|tfd}t||S)z@Create a CountMatrix from a legacy h5py.File (format version 1).csrrGrrxr#r#r$rqrz0CountMatrix.from_legacy_v1_h5..r-Nbarcodesc3s|]}|VqdSrGr#rZ barcode_mapr#r$rruint64rindptrrprdatagenes gene_names)rrnamerrr!)r enumeratekeysrrrrrrZ array_equalZarangediffrepeatZ concatenater,rrGENE_EXPRESSION_LIBRARY_TYPErrrr csc_matrixrL)rZ genome_arraysZgene_id_arraysZgene_name_arraysZ bc_idx_arraysZfeat_idx_arraysZ data_arraysZ genome_idxrgZn_genesrrZremapped_col_indsrZnz_elems_per_bcZbc_idxgenomesZgene_idsrrZgene_idZ gene_namerrjrr"r#rr$from_legacy_v1_h5`sl          "zCountMatrix.from_legacy_v1_h5 feature_idrrcCsHt|tstd|dt|||jvrtd||j|S)Nz feature_id z must be bytes, but was z*Specified feature ID not found in matrix: )rrKeyErrorrrdecode)rXrr#r#r$feature_id_to_ints   zCountMatrix.feature_id_to_intrrrcstfdd|DS)Nc3s|]}|VqdSrG)r)rZfidr\r#r$rsz2CountMatrix.feature_ids_to_ints..)sortedrr#r\r$rszCountMatrix.feature_ids_to_intsrcCs||}|jj|jSrG)rrrr)rXridxr#r#r$feature_id_to_name zCountMatrix.feature_id_to_namercCrrGrrr#r#r$rrzCountMatrix.int_to_feature_idcCrrG)rrrrr#r#r$int_to_feature_namerzCountMatrix.int_to_feature_namercC |j|S)a!Get the integer index for a barcode. Args: bc (bytes): The barcode to search for. Raises: ValueError: `barcode` was not bytes. KeyError: `barcode` was not found in the set. Returns: int: the barcode index. )r bc_to_int)rXrr#r#r$r s zCountMatrix.bc_to_intTSequence[bytes] return_sortedrPcCs"t|tr t|}|jj||dS)N)sort)rsetlistrrx)rXrr r#r#r$rxs zCountMatrix.bcs_to_intsrcCs |j|SrGr)rXrr#r#r$ int_to_bcs zCountMatrix.int_to_bcjj Iterable[int]rwcsfdd|DS)Ncsg|]}|qSr#)r)rrr\r#r$rrz+CountMatrix.ints_to_bcs..r#)rXrr#r\r$rrzCountMatrix.ints_to_bcsrpr?cCs0||||}}|j||f|7<dS)z Add a count.Nrr re)rXrrvaluerrr#r#r$addszCountMatrix.addcCs$||||}}|j||fSrGr)rXrrrrr#r#r$getszCountMatrix.getothercCs$|j|jksJ|j|j7_dS)zmMerge this matrix with another CountMatrix. Works by addition, dimensions must be the same. N)rRre)rXrr#r#r$mergeszCountMatrix.mergecCs"||jjs|jdSdSrG)tocscreZhas_sorted_indices sort_indicesr\r#r#r$rszCountMatrix.sort_indicesNr=r>cCsttt|d(}t|jtj<t|jt<t ||t ||| t }| |WddS1s3wYdS)z=Save this matrix to an HDF5 file, optionally with SW version.wN)h5FilerMATRIX_H5_FILETYPErH h5_constantsH5_FILETYPE_KEYMATRIX_H5_VERSIONMATRIX_H5_VERSION_KEYrJrD create_groupMATRIX save_h5_group)rXfilenamer=rErgroupr#r#r$ save_h5_files      "zCountMatrix.save_h5_filer)h5.Groupc Cs|||tj}|j|tj|tj|j ddtj D]\}}t j t|j||d}|j||tfdtddq dS)z^Save this matrix to an HDF5 (h5py) group and converts the matrix to csc format if not already.T) compressionrQrG)rchunksZmaxshaper,shuffleN)rr%r!H5_FEATURE_REF_ATTRrZto_hdf5rAZcreate_hdf5_string_dataset H5_BCS_ATTRrZH5_MATRIX_ATTRSr@rr(getattrreZcreate_datasetHDF5_CHUNK_SIZEHDF5_COMPRESSION)rXr)Zfeature_ref_groupattrr8Zarrr#r#r$r' s"   zCountMatrix.save_h5_grouptuple[int, int, int]cCs6|tjdd\}}t|tj}t|t||fS)z)Load the matrix shape from an HDF5 group.N)r!H5_MATRIX_SHAPE_ATTRrrH5_MATRIX_DATA_ATTRr)r)rowscolsentriesr#r#r$ load_dims$szCountMatrix.load_dimscCstt|S)(Load the matrix shape from an HDF5 file.)rLload_dims_from_h5_file_handler)r(r#r#r$load_dims_from_h5+zCountMatrix.load_dims_from_h5rcCsd|}d}d}t}|D]}||}|t|d7}|t|d7}||dq |t||fS)Nrrrr)rrrrupdate)rrZnum_nonzero_entriesZ num_gene_idsrrrr#r#r$#_load_dims_from_legacy_v1_h5_handle0sz/CountMatrix._load_dims_from_legacy_v1_h5_handlecC>tt|d }t|WdS1swYdS)zALoad the matrix shape from a legacy h5py.File (format version 1).rN)rrrrLrAr(rr#r#r$_load_dims_from_legacy_v1_h5>$z(CountMatrix._load_dims_from_legacy_v1_h5r( str | bytesfloatcCs8t|\}}}|tj|tj|tjtjdS)z8Estimate memory usage of anndata object from the matrix.i@)rLr>r!Z!MEM_BYTES_PER_MATRIX_FEATURE_H5ADZ!MEM_BYTES_PER_MATRIX_BARCODE_H5ADZMEM_BYTES_PER_MATRIX_NNZ_H5ADZMEM_BYTES_CONSTANT_H5AD)r(Z num_featuresnum_bcsnonzero_entriesr#r#r$!get_anndata_mem_gb_from_matrix_h5Dsz-CountMatrix.get_anndata_mem_gb_from_matrix_h5 num_barcodesrJscaleceilcCs:t|tj}|t|tj7}|r|t|S||S)z*Estimate memory usage of loading a matrix.)rHr!ZNUM_MATRIX_ENTRIES_PER_MEM_GBZNUM_MATRIX_BARCODES_PER_MEM_GBrrN)rLrJrMrNZ matrix_mem_gbr#r#r$get_mem_gb_from_matrix_dimQsz&CountMatrix.get_mem_gb_from_matrix_dimcC t|\}}}t||||S)z)Estimate memory usage from an HDF5 group.)rLr;rO)r)rMrN_rIrJr#r#r$get_mem_gb_from_group^z!CountMatrix.get_mem_gb_from_groupcCstt|||Sz(Estimate memory usage from an HDF5 file.)rL%get_mem_gb_from_matrix_h5_file_handler)r(rMrNr#r#r$get_mem_gb_from_matrix_h5hs z%CountMatrix.get_mem_gb_from_matrix_h5 file_handlestr | bytes | BufferedIOBasecCrPrT)rLr=rO)rWrMrNrQrIrJr#r#r$rUsrSz1CountMatrix.get_mem_gb_from_matrix_h5_file_handlecCht|d$}t|}|dkrt|WdSt|tWdS1s-wYdS)r<rCrpN)rrrL_get_format_version_from_handlerAr;r&rWr h5_versionr#r#r$r=}s  $z)CountMatrix.load_dims_from_h5_file_handlecCrY)z8Load just the barcode sequences from an HDF5 file hadle.rCrpN)rrrLrZ'_load_bcs_from_legacy_v1_h5_file_handleload_bcs_from_h5_groupr&r[r#r#r$load_bcs_from_h5_file_handles  $z(CountMatrix.load_bcs_from_h5_file_handlecCs|tjddSrG)r!H5_MATRIX_INDPTR_ATTRr)r#r#r$_load_indptr_from_matrix_groupz*CountMatrix._load_indptr_from_matrix_groupcCsdt|\}}|tkrtd|t|d}|t}t|WdS1s+wYdS)NUMatrix HDF5 file format version (%d) is an older version that is no longer supported.rC)rL_validate_h5_filer# ValueErrorrrr&rb)r(fnr/rZ mat_groupr#r#r$load_indptr_from_files$z!CountMatrix.load_indptr_from_filec Cst|}||}|tjdd}|tjdd}|tjdd}t|}t t |dks6Jt j |||f|d}||||dS)zLoad from an HDF5 group.Nrrr) rLload_feature_ref_from_h5_groupr^r!r6r7H5_MATRIX_INDICES_ATTRrbrallrrr) rr)rrr!rr-rr"r#r#r$loads   zCountMatrix.loadstartendcCst|}||}||\}}}|dks||ks||kr+td|d|d||tj||d} | d} | d} t|||g} |tj | | } |tj | | }| | d} t t | dkskJt j| || f| d}||||||dS) aBLoad from an HDF5 group only a subset of the columns. Very similar to CountMatrix.load(). Args: group: The matrix H5 group start: the first column to select end: the column to select up to. Returns: A CountMatrix with the relevant columns rz>The column range you've specified is invalid. Start Column = z, End = z, Total Columns = rprr)rLrir^r;rfr!r`rr(r7rjrkrrr)rr)rmrnrrZn_rowsZn_colsrQr col_startcol_endr!rr-r"r#r#r$load_columns_from_files"   z"CountMatrix.load_columns_from_filecCs*|tjjdurt|tjddSgS)z1Load just the barcode sequences from an h5 group.N)r!r0r!rrar#r#r$r^sz"CountMatrix.load_bcs_from_h5_groupcCsbt|}t|}|dkrt|St|d}t|tWdS1s*wYdS)z3Load just the barcode sequences from an HDF5 group.rprCN)rrLget_format_version_from_h5_load_bcs_from_legacy_v1_h5rrr^r&)r(r\rr#r#r$load_bcs_from_h5s   $zCountMatrix.load_bcs_from_h5cCrB)KLoad just the barcode sequences from a legacy h5py.File (format version 1).rCN)rrrrLr]rDr#r#r$rtrFz'CountMatrix._load_bcs_from_legacy_v1_h5cCs6|}t}|D] }||}||dq t|S)rvr)rrr@r)rWrrrr)r#r#r$r]s z3CountMatrix._load_bcs_from_legacy_v1_h5_file_handlecCstt|dL}tj|jvs|jtjtkrtdt|jvr&|jt}nd}|t kr2td||t krCould not find the "matrix" group inside the matrix HDF5 file.) rrrr!r"rHr rfr$r#rrLr^r(rr/r#r#r$load_bcs_from_h5_files.     $z!CountMatrix.load_bcs_from_h5_fileset[str]cCstt|dA}t|}|tkr3t|\}}}|dkr)tjhWdSt WdSt |t }dd|j DWdS1sLwYdS)zCReturn a set of all library types defined in the Feature Reference.rCrNcSh|]}|jqSr#rrr#r#r$ $z>CountMatrix.load_library_types_from_h5_file..) rrrrLrZr#rArrrrir&r)r(rr/Z gene_countrQrr#r#r$load_library_types_from_h5_files  $z+CountMatrix.load_library_types_from_h5_filecCsdd|jjDS)zGet the list of feature types.cSr}r#r~rr#r#r$r(rz0CountMatrix.get_library_types..)rrr\r#r#r$get_library_types&rczCountMatrix.get_library_typescCs|tj}t|S)z,Load just the FeatureRef from an h5py.Group.)r!r/rZ from_hdf5)r)Z feature_groupr#r#r$ri*s  z*CountMatrix.load_feature_ref_from_h5_groupcCs\tt|d}t|}|tkrtdt|tWdS1s'wYdS)z1Load just the FeatureRef from a matrix HDF5 file.rCzrzCountMatrix.tocoocCrrG)rrerrrr\r#r#r$rBszCountMatrix.tocsccCs$t|jtjur|j|_dSdS)zeConvert to a csr matrix if not already. Returns: None, mutates in place N)rrerZ csr_matrixtocsrr\r#r#r$rGszCountMatrix.tocsrrrf*tuple[CountMatrix, np.ndarray, np.ndarray]cCst|}t||k}|jt|kr||}t||k}|j t|kr/| |}t|dks;t|dkr>t |||fS)zSelect axes with sums greater than the threshold value. Returns: (CountMatrix, np.array of int, np.array of int): New count matrix, non-zero bc indices, feat indices r) r_deepcopyrrrrTrrruget_counts_per_featurerRrr1)rXrfnew_matZ nonzero_bcsZnonzero_featuresr#r#r$select_axes_above_thresholdQs    z'CountMatrix.select_axes_above_thresholdrcCsX|dkr|}t||k}||S|dkr(|}t||k}||Std)aSelect axis given a threshold of features per barcode or total counts per feature. Args: axis (int): 0 for rows (features) and 1 for columns (barcodes). Returns: (CountMatrix): New count matrix rrpaxis out of rangerrrrrrurf)rXrrfcountsr-r#r#r$select_axis_above_thresholdis   z'CountMatrix.select_axis_above_thresholdcCs |dS)zSelect axes with nonzero sums. Returns: (CountMatrix, np.array of int, np.array of int): New count matrix, non-zero bc indices, feat indices r)rr\r#r#r$select_nonzero_axess zCountMatrix.select_nonzero_axestuple[CountMatrix, np.ndarray]cCs`|dkr|}t|dk}|||fS|dkr,|}t|dk}|||fStd)aSelect axis with nonzero sums. Args: axis (int): 0 for rows (features) and 1 for columns (barcodes). Returns: (CountMatrix, np.array of int, np.array of int): New count matrix, selected indices rrprr)rXrrr-r#r#r$select_nonzero_axiss zCountMatrix.select_nonzero_axisr- Sequence[int]cs<jdd|f}tj|}tjfdd|D|dS)zASelect a subset of barcodes and return the resulting CountMatrix.Ncsg|]}j|qSr#rrr\r#r$rrz/CountMatrix.select_barcodes..r)rer:rLr)rXr-submatr#r\r$rus zCountMatrix.select_barcodesrvcCs||d}||S)NF)rxruryr#r#r$rzr{z"CountMatrix.select_barcodes_by_seqr|cs|fdd|jDS)Ncsg|] }t|kr|qSr#)r~Zget_gem_group_from_barcoderrr#r$rsz.)rzrrr#rr$rsz(CountMatrix.select_barcodes_by_gem_groupcCs:|j|}|j|ddf}t|j|}t||j|dS)zSelect a subset of features and return the resulting matrix. We also update FeatureDefs to keep their indices consistent with their new position. Nr)rrrer:rLr)rXr-rrr#r#r$rs  zCountMatrix.select_featurescCrrG)rrrr#r#r$rsz"CountMatrix.select_features_by_idsgene_indices_to_keepcCs8t|}|jjD]}|jtjkr||jq||S)axRemoves all features that are GEX and not on this list, keeping all others. Used to subset the matrix down in the targeted assay Args: gene_indices_to_keep: list of indices of the GEX features to keep Returns: A copy of this matrix subset to the GEX feature indices requested and all other feature types. ) rrrrrrrrr)rXrr-rr#r#r$remove_genes_not_on_lists     z$CountMatrix.remove_genes_not_on_listrcCsBg}|jjD]}|jtjkr|jd|kr||jq||Sr) rrrrrrrrrrr#r#r$rs    z%CountMatrix.select_features_by_genomerrcCs2g}|jjD] }|j|vr||jq||S)zISelect the subset of gene-expression features by genome and feature type.rrrrrrrr#r#r$r     z$CountMatrix.select_features_by_typesrcCs2g}|jjD] }|j|kr||jq||Srr)rXrr-rr#r#r$rrz#CountMatrix.select_features_by_typetag_typecCs<g}|jjD]}|j|kr||jvr||jq||S)zpSelect the subset of features with a particular feature type (e.g. "Antibody Capture") and tag (e.g. "Hashtag").)rrrrrrr)rXrrr-rr#r#r$select_features_by_type_and_tags    z+CountMatrix.select_features_by_type_and_tagcCr )zSReturn a list of feature ids of a particular feature type (e.g. "Gene Expression").)rget_feature_ids_by_typerr#r#r$rr]z#CountMatrix.get_feature_ids_by_typecCr )zeCount how many features in the matrix are of a given type. (e.g. "Gene Expression") )rget_count_of_feature_typerr#r#r$rs z%CountMatrix.get_count_of_feature_typedict[str, int]csfddDS)z"Get count of each feature by type.csi|]}||qSr#)r)rftr\r#r$rrz:CountMatrix.get_count_of_feature_types..)rr\r#r\r$get_count_of_feature_typessz&CountMatrix.get_count_of_feature_typesrcCs|jtjdS)rr~)rrr)rr#r#r$rr?z)CountMatrix._get_genomes_from_feature_refcCrZrrr\r#r#r$rr]zCountMatrix.get_genomescCslt|}t|}|dkrt|St|d}t|t}t|WdS1s/wYdS)z;Get a list of the distinct genomes from a matrix HDF5 file.rprCN) rrLrs_get_genomes_from_legacy_v1_h5rrrir&r)r(r\rrr#r#r$get_genomes_from_h5 s  $zCountMatrix.get_genomes_from_h5cCs@tt|d}t|WdS1swYdS)zNGet a list of the distinct genomes from a legacy h5py.File (format version 1).rCN)rrrrrrDr#r#r$rs $z*CountMatrix._get_genomes_from_legacy_v1_h5"np.ndarray[int, np.dtype[np.int_]]cCs|j|jjddSr)reZeliminate_zerosgetnnzr\r#r#r$get_numfeatures_per_bcrz"CountMatrix.get_numfeatures_per_bccCt|jddSrr%rer\r#r#r$r!rzCountMatrix.get_counts_per_bc str | Nonecsd|jjvs Jdrfddt|jjD}n fddt|jjD}|r7t||dd}|jdd Stjg|jj d S) zSum the count matrix across feature rows with a given genome tag. The feature reference must contain a 'genome' tag. If feature_type is not null filter on it as well. rz&feature reference missing 'genome' tagcs,g|]\}}|jdkr|jkr|qSr)rrrrZfdefrrr#r$r.s zACountMatrix.get_counts_per_barcode_for_genome..cs"g|] \}}|jdkr|qSr)rrrr#r$r4s N)rMrOrrrQ) rrrrrKrrr(rer8)rXrrr-r`r#rr$!get_counts_per_barcode_for_genome$s     z-CountMatrix.get_counts_per_barcode_for_genomecCrNrprrr\r#r#r$r>rz"CountMatrix.get_counts_per_featurecCs tt|j}t|jdd|Sr)rr%re)rXZ total_umir#r#r$get_frac_counts_per_featureAsz'CountMatrix.get_frac_counts_per_featureStuple[np.ndarray[int, np.dtype[np.float64]], np.ndarray[int, np.dtype[np.float64]]]cCsht|jtjs J|jjdd}|j}|d}|jdd|jdt |d}t |}||fS)zCalculate the mean and variance on the sparse matrix efficiently. :return: a tuple with numpy arrays for mean and var rprg@) rrerrmeanr_Zpowerrr!rr)rXZmean_per_featureZ second_momentZvar_per_featurer#r#r$get_mean_and_var_per_featureEs   z(CountMatrix.get_mean_and_var_per_featureFcCsj|}|dur|tjvsJd||}|dur||}|dur&||}|}|r3td|S|S)zGet counts per barcode, sliced various ways. - subset by list of feature IDs - subset by list of barcodes - subset by library_type Nzlibrary_type not recognizedg?)rZRECOGNIZED_FEATURE_TYPESrrrzrrlog10)rXZlist_feature_idsZ list_barcodesZ log_transform library_typeZsubselect_matrixZcounts_featurer#r#r$get_subselected_countsVs     z"CountMatrix.get_subselected_countscCst|jdkddS)Nrrprrr\r#r#r$get_numbcs_per_featuresrz"CountMatrix.get_numbcs_per_featurecutoffrcCs@|}tdt|j|d}t|dd|}t||kdS)NrrpT)reverse)rrminsizerrnonzero)rXrZ reads_per_bcrrr#r#r$ get_top_bcsvszCountMatrix.get_top_bcsbase_dirsave_features_func-Callable[[FeatureReference, str, bool], None]metadata dict | Nonecompressc Cs||tj|ddtj|d}tj|d}|r'|d7}|d7}|jjdvr2d}d}n|jjd vr=d }d }n td |jjt |jt j j sPJ|jj \} } d } d} |p]i}t|d<dt|} t|d\}|d| d|d| d| dD]}|d|||dq|d| | |jjft|jjd|jjd|jjD]\}}}|||||fqWdn1swY||j||dt|d}|jD] }||dqWddS1swYdS)a^Save in Matrix Market Exchange format. Note: This operation modifies the matrix by converting to a coordinate representation by calling scipy.sparse.csc_matrix.tocoo(). Args: base_dir (str): Path to directory to write files in. save_features_func (func): Func that takes (FeatureReference, base_dir, compress) and writes a file describing the features. metadata (dict): Optional metadata to encode into the comments as JSON. compress: Whether to compress output files with gzip. T)exist_okz matrix.mtxz barcodes.tsvz.gz)Zuint32rrZint64integers %i %i %i )rHZdoublereals %i %i %15g z&Unsupported data type for the matrix: Z coordinateZgeneralZformat_versionsmetadata_json: wbz%%MatrixMarket matrix    %rpN)r)rrosmakedirspathjoinrer8rfrrZcoorr!r# tk_safe_jsonZ safe_jsonifyencodecr_ioZopen_maybe_gzipwritesplitrr,rowcolrrr)rXrrrrZ out_matrix_fnZout_barcodes_fnfieldfmtr8r9ZrepZsymmetrycommentstreamlinerCcdrrr#r#r$save_mex|sN   "   * "zCountMatrix.save_mexofilecCs t|jvr |jt}|Sd}|SNrp)r$rH)rr/r#r#r$rZs  z+CountMatrix._get_format_version_from_handle bytes | strcCs>tt|d }t|WdS1swYdS)NrC)rrrrLrZrDr#r#r$rss$z&CountMatrix.get_format_version_from_h5cCst|tjr |}nt|}t|dF}tj|jvs$t |jtjt kr(t dt |}|tkr7t d||tkrEt|vrOt dWd||fSWd||fS1s\wY||fS)NrCrwrxry)rpathlib PosixPathrrrr!r"rHrr rfrLrZr#r&r)r(rgrr/r#r#r$res0     zCountMatrix._validate_h5_filerprcrqcCst|\}}t|dO}|tkr*|dus|durtdt|WdS|dus2|durN|dus:|dur>tdt|t||WdSt |tWdS1s_wYdS)aPLoad a matrix H5 file, optionally subsetting down to a particular range of columns if requests. Args: filename: The name of the H5 file col_start: (Optional) The column to select col_end: (Optional) End of column select range Returns: Instance of a CountMatrix rCNzASubsetting columns when loading legacy H5 files is not supported.z)r(rQrr#r#r$count_cells_from_h5szCountMatrix.count_cells_from_h5c Csptt|d&}z |dtj}Wn tyd}Yn wWd|SWd|S1s1wY|S)NrC/Unknown)r open_filer get_node_attrr!H5_CHEMISTRY_DESC_KEYAttributeError)r(r chemistryr#r#r$load_chemistry_from_h5 s   z"CountMatrix.load_chemistry_from_h5bcs_per_genomedict[Any, Iterable[bytes]]cCs2t}|D]}||O}qtt|}||S)zReturn CountMatrix containing only the specified barcodes. Args: bcs_per_genome (dict of str to list): Maps genome to cell-associated barcodes. Returns: CountMatrix w/ the specified barcodes. )rvaluesrrrz)rXrrrr#r#r$filter_barcodess     zCountMatrix.filter_barcodescCstj|ddS)Nhdf5z matrices.hdf5)rrr) base_pathr#r#r$h5_path&rzCountMatrix.h5_pathr scale_factor int | floatcontrol_feature_nameslist[str] | list[bytes]c shd}|vsJd|ddfdd|D}|r.|d}n |d}t|}t j j j j }j |ddft||tj} | jdk| j|dkB}t| jd|d| _| j j j j |ddf<t|} d d | D} tjjvrjt| d |Sjt| |S) aNormalise the features of a library type by control features in the matrix. scale by the scale_factor. The features in the library are all 1. divided by 1 plus the sum of control features in each barcode 2. multiplied by scale factor 3. floored to be an integer If the control features provided are not in the matrix, the division is by the sum of all features in the library provided. The features that are normalized have NORMALIZATION_TAG_IN_H5 set to TRUE. If this tag was not in the H5, all other features have the tag set to FALSE. If it was present in the H5, its values for other features is unchanged. Args: library_type (str): string showing library type scale_factor (Union[int, float]): factor to scale by control_feature_names (list[Union[str, bytes]]): feature names that are used as control Returns: int: Number of entries clipped due to overflow rz@Library type to normalise not in matrix. Library type passed in z. zLibrary types in matrix csg|] }|jvr|qSr#)rrr\r#r$rOrzJCountMatrix.normalise_library_type_by_control_features..rpNcSsi|]}|dqS)TRUEr#rr#r#r$rlrzJCountMatrix.normalise_library_type_by_control_features..FALSE)rr`rrrrrrrZiinfor"rer8rdotrZdiagsr9Zfloat64rrZcliprNORMALIZATION_TAG_IN_H5rrZadd_tagZ update_tag) rXrrrZnumber_of_overflow_entriesZcontrol_features_in_matrixZ size_factorZindices_in_libraryZ max_valueZtmp_normalised_matrixZfeatures_normalized_setZfeatures_label_dictr#r\r$*normalise_library_type_by_control_features*s@  " z6CountMatrix.normalise_library_type_by_control_features)rrrrr"r6)rrrr)rr<rrL)rrrr)rrrr)rrrrr)rrrr)rrrr)T)rr r rPrr)rrrr)rrrrwr})rrrrrr?)rrLrr?rr)r=r>)r)r+rr?)r)r+rr5)rr5)rr<rr5)r(rGrrH) rLrrJrrMrHrNrPrrH)r)r+rMrHrNrPrrH)r(rGrMrHrNrPrrH)rWrXrMrHrNrPrrH)rWrXrr5)rrw)r)r+)r)r+rrL)r)r+rmrrnrrrL)r)r+rrw)rr|)r)r+rr)rrr)rfrrr)rrrfrrrL)rr)rrrr)r-rrrL)rvr rrL)r|rrrL)r-rrrL)rrrrL)rrrrL)rrrrL)rrrrL)rrrrL)rrrrrrL)rrrrw)rrrr)rr)rrrr)r(rGrr)rrrG)rrrrrr)rr)NNFN)rrrr)NT)rrrrrrrrP)rr<rr)r(r)rprcrqrc)rrrrL)rrrrrrrr)ar2r3r4rYrrr` classmethodDEFAULT_DATA_DTYPEr staticmethodrrrrrr r rxrrrrrrr*r'r;r>rArErKr!ZMATRIX_MEM_GB_MULTIPLIERrOrRrVrUr=r_rbrhrlrrr^rurtr]r{rrrirrrrrrrrrrurzrrrrrrrrrrrrrrrrrrrrrrrrrrZrsrerrrrrrr#r#r#r$rL-s  U                                               K   rL h5_filenamesrCountMatrix | NonecCsFd}|D]}|durt|}q|t|q|dur!||S)z-Merge multiple matrices into a single matrix.N)rLrrr)rr"Z h5_filenamer#r#r$merge_matricesus rcsdd|DtddD}tfddDs JdtfddDs0Jdg}|D]*}t|}|d d |d d }t|} t| } t| } ||| | fq4|j d d dd } d } |D]\}} } | | ks~| | ks~| | krt d| } | } qm|dd}t ||t jt jf}t|d}|rt|||rt|||t}t|tj}t||d} |D] }||}||fq|d d D]T\}}}t|}| } t|jj| } |jj|t j| | <|jj|t j| | <|jjtj}|| 7}||d ||}|dks!Jd||d ||d <~q||t jd d <Wd d S1sBwYd S)aMerge several h5 files into one larger matrix file. An efficient method for doing column concatenation of H5 files. Assumes the input files all have the same features/barcodes in their matrix, and that each only contains a non-overlapping subset of the columns. Strategy is to append the column data directly, rather than merging arbitrary non-distinct columns (As is done in merge_matrices). Args: in_h5_filenames: A list of filenames to column distinct sets of a larger matrix out_h5_filename: The desired output filename extra_attrs: Similar to the argument to save_h5_file sw_version: A version of the software to add into the attributes. Returns: Nothing cSsg|]}t|qSr#rrr#r#r$rrz8create_merged_matrix_from_col_concat..css|]}|dVqdS)r0Nr#rr#r#r$rrz7create_merged_matrix_from_col_concat..c3s$|] }|dddkVqdS)rNr#rZ dimensionsr#r$r z$Not all row dimensions were the samec3s$|] }|dddkVqdS)rprNr#rrr#r$rrz$Not all col dimensions were the samerpNrocSs|dSrr#)rr#r#r$sz6create_merged_matrix_from_col_concat..keyz;H5 files to be concatenated were not unique sets of columnsraz4Data was not cleanly separated into distinct columns) rrrkrLrhrrrrrrfshutilcopyfiler!r7rjrrrJrDr&rbr9rrrresizerrerr-rr`)Zin_h5_filenamesZout_h5_filenamer=rEZ total_nnzZ fn_start_endsrgrZ col_spansZ active_colsrmrnZlast_endZ last_startrQZ start_fileZ to_appendoutfiler"Zind_ptrZ dset_nameZdsetZ small_fileZs_startZs_endrZ cur_indptrZnum_entries_beforer#rr$$create_merged_matrix_from_col_concatsr            $rcCst||}||tj<|SrG)make_library_map_countr!r) sample_id gem_groupsrZ matrix_attrsr#r#r$make_matrix_attrs_counts  rcCst|}t|}|dkrt|Si}t|d/}tjD]!}|j |}|dur@t |rtt|}tjtj|gt|ddtjtj|tdi}|S)NrrQ) rrr!H5_LIBRARY_ID_MAPPING_KEYrr(rrH5_ORIG_GEM_GROUP_MAPPING_KEYr)rrZunique_gem_groups library_mapr#r#r$rs rdict[str, np.ndarray]cCsfg}g}t|dddD]\}\}}||||qtjtj|ddtjtj|tdi}|S)NcSs t|dS)Nr)r)pairr#r#r$r (s z'make_library_map_aggr..r rrQ) rr@rr!r!rr(r"r)Zgem_group_index library_idsoriginal_gem_groupsrQlidogr#r#r#r$make_library_map_aggr#s   r* matrix_h5c Cstj|dd)}z|dtj}|dtj}Wnty(YWddSwWdn1s3wYi}tt||ddD] \}\}}||f||<qC|S)NrC)moderrp)rm) rrrr!r!r"rrr,)r+rr&r'r#Zngr(r)r#r#r$get_gem_group_index2s r-cCs|dkst|jdtjtBsJt|tjsJt|j dD]C}d}t|j ||j |dD]}||j||j|7}q3|dkrGq"t |}t|j ||j |dD] }|j||<qYq"dS)afPerform in-place column L2-normalization of input matrix X. >>> import numpy as np >>> import scipy.sparse as sp >>> from sklearn.preprocessing import normalize >>> a = np.arange(12, dtype='float').reshape((3, 4)) >>> b = sp.csc_matrix(a) >>> inplace_csc_column_normalize_l2(b) >>> np.all(normalize(a, axis=0) == b) True rrpgN) rrrrZfloat32rHrrranger!rsqrt)Xrsrr#r#r$inplace_csc_column_normalize_l2?s&  r2r)rrrr)r(r)r*rrr+)r5r6r7r6rr6)r;r<r=r>rr?)r;r<rErFrr?)rrrrr)r(r)rr$)r+r)] __future__rr_Zos.pathrrr  collectionsrcollections.abcrrrrrr ior typingr r r Zh5pyrZnumpyrZ scipy.sparseZsparserZsixrrZcellranger.cr_iorZcellranger.h5_constantsr!Zcellranger.hdf5rrAZcellranger.rna.libraryZrnaZlibraryrZcellranger.sparserdZcellranger.utilsutilsr~Ztenkit.safe_jsonZ safe_jsonrZcellranger.fast_utilsrZcellranger.feature_refrrrZcellranger.wrapped_tablesrr3r2rZFEATURES_TSV_GZrr%Zgenericr&r.r r&r$r#rIr"rr Exceptionr1r:rDrJrKrLrrrrrr rr*r-r2r#r#r#r$s                  4 P U