U fv@s*ddlZddlZddlmZddlmZddlmZddl Z da da da da GdddeZdd Zd d Zd d ZddZddZddZddZddZddZddZddZddZd d!Zd"d#Zd$d%Zd&d'Zd(d)ZiZ ee d*<ee d+<ee d,<ee d-<ee d.<ee d/<ee d0<ee d1<ee d2<ee d3<ee d4<ee d5<ee d6<ee d7<ee d8<ee d9<ee d:<ee d;<ee d<<ee d=<ee d><ee d?<ee d@<ee dA<ede dB<ede dC<ede dD<edEge dF<ede dG<ede dH<ede dI<ede dJ<ede dK<ede dL<ede dM<ede dN<ede dO<ede dP<ede dQ<ede dR<ede dS<ede dT<ede dU<edddVdWe dX<edddYdWe dZ<eddd[dWe d\<eddd]dWe d^<eddd_dWe d`<edddadWe db<ee dc<ee dd<ee de<ee df<dS)gN) Explainer)!standard_combine_mult_and_diffref) LooseVersionc@sJeZdZdZddefddZddZddZdd d Zd d Z ddZ dS)TFDeepExplainera Using tf.gradients to implement the backgropagation was inspired by the gradient based implementation approach proposed by Ancona et al, ICLR 2018. Note that this package does not currently use the reveal-cancel rule for ReLu units proposed in DeepLIFT. Nc Csb||_tdkr`ddlmaddlmattds| D|_Your TensorFlow version is older than 1.4.0 and not supported.z2.1.0z9Your Keras version is older than 2.1.0 and not supported.z$keras.engine.sequential.Sequential'>zkeras.models.Sequential'>zkeras.engine.training.Model'>ztuple'>rFz) is not currently a supported model type!z9The model output to be explained must be a single tensor!z4The model output must be a vector or a single value!T__call___SESSIONZlearning_phaseZConstcSsg|]}|jdqSr)outputs.0oprC/tmp/pip-target-lpfmz8o1/lib/python/shap/explainers/deep/deep_tf.py sz,TFDeepExplainer.__init__..cSsg|] }|jqSr)rrtrrrrsizjYou have provided over 5k background samples! For better performance consider using smaller random sample.cSsg|]}t|tkr|qSr) op_handlersbreak_dependencerkrrrrs cSsg|]}|D]}|qqSr) consumers)rinputrrrrrs ) within_opsz Warning: zWused in model but handling of op is not specified by shap; will use original gradientscSsg|]}dqSNrrirrrrszTThe model output tensor to be explained cannot have a static shape in dim 1 of None!)@combine_mult_and_diffreftfZtensorflow.python.frameworkrtf_opsZtensorflow.python.opsrtf_gradients_implhasattrr Z tensorflowr __version__warningswarnkerasstrtypeendswithinputs model_inputsZlayersoutput model_outputAssertionErrorlistlenshape multi_output multi_inputdata_vinputs orig_gradsbackendZtensorflow_backendrZ get_sessionsessioncompatZv1Zlearning_phase_opsgraphZget_operationsnamerZdtypeboolappendlearning_phase_flagsZexpected_valuerunZmeantensors_blocked_by_falserbackward_walk_opsrforward_walk_ops between_ops used_typesprint phi_symbolicsas_listrange Exception) selfmodelr9r=rCr#rtensor_blacklistZdependence_breakersZback_opsZnoutputsrrr__init__s1           * .       zTFDeepExplainer.__init__cs:|jjkr.tfdd|jDj|j<j|jS)z_ Return which inputs of this operation are variable (i.e. depend on the model inputs). cs.g|]&}|jjkp(|jddjDkqS)cSsg|] }|jqSrr@)rxrrrrsz?TFDeepExplainer._variable_inputs...)rrHr@r0rrOrrrsz4TFDeepExplainer._variable_inputs..)r@r:nparrayr/)rOrrrUr_variable_inputss "z TFDeepExplainer._variable_inputsc Cs|j|dkrtjj}tD]P}||krT||d|j|<t|tk rj|j||d<q||jkrt |dqt t drt j }ddt _ z6|j r|jdd|fn|j}t||j|j|<W5t t dr|t _ tD]}||kr|j|||d<qX|j|S)zH Get the SHAP value computation graph for a given model output. Nr-z; was used in the model but is not in the gradient registry!r cSsdS)NTr)Ztensorrrrz.TFDeepExplainer.phi_symbolic..)rKr%Z_gradient_registry _registryrr; passthrough custom_gradrIrNr'r&r r7r2r$ gradientsr0)rOr"regnZorig_IsBackpropagatableoutrrr phi_symbolics*     zTFDeepExplainer.phi_symbolicmaxcs |js@ttkr,tdkr,ds>tdqTttkrTgnttksTtdt|jtkstdt|jtf|dk r |jr ||j|j}|dkrt | }n:|dkrt |}n&|dkrt t |}n dstd |ddd|f}n&t t t|jd jd df}g}t|jdD]}g} ttD]} | t | jqXtd jd D]L|dk rȈ|d krtd d d jd tjt|jd r|fddttDttkrgn|jfddttDfddttD} ||f} ||| |j| |jfddttDfddttDd} ttD]}| || |<qq||js| d n| qB|js|d S|dk r||fS|SdS)NrFz(Expected a single tensor as model input!z Expected a list of model inputs!zANumber of model inputs (%d) does not match the number given (%d)!rcminZmax_absz/output_rank_order must be max, min, or max_abs!rDonez examples ofr csg|]}|qSrrrlXjrrrsz/TFDeepExplainer.shap_values..c sXg|]P}t|d|jdftddtt|jdDqS)rrcSsg|]}dqSrrrrrrr"sz:TFDeepExplainer.shap_values...)rVtiler6tuplerMr5rf)ribg_datarjrrr"scs$g|]}t||gdqSr)rVZ concatenaterf)rntiled_Xrrr#scs(g|] }|d|jd qS)Nr)r6rf)rn sample_phisrrr+scsg|]}|qSrrrfrhrrr-s)ZmultZorig_inprn)r8r-r4r5r3r0r7rDr2rVZargsortabsrlZarangerKr6rMrBZzerosrJsysstdoutflushr'r9rbr#)rOriZranked_outputsZoutput_rank_orderZprogress_messageZmodel_output_valuesZmodel_output_ranksZ output_phisr"ZphisrZ joint_inputZ feature_indZphis_jrgr)rirnrjrpror shap_valuessl ,  &   "    zTFDeepExplainer.shap_valuescCs0tt||}|jD] }d||<q|j||S)zN Runs the model while also setting the learning phase flags to False. F)dictziprCr=rD)rOrar0riZ feed_dictrrrrrD<s  zTFDeepExplainer.runcGst|j||f|S)zG Passes a gradient op creation request to the correct handler. )rr-)rOrgradsrrrr]DszTFDeepExplainer.custom_grad)NrcN) __name__ __module__ __qualname____doc__rrRrXrbrurDr]rrrrr s 3& Ircs(gfdd|D] }|qS)z Follows a set of ops assuming their value is False and find blocked Switch paths. This is used to prune away parts of the model graph that are only used during the training phase (like dropout, batch norm, etc.). csB|jdkr|jdn"|jD]}|D] }|q.q"dS)NSwitchr)r-rBrr)rracZblockedrecurserrrQs    z)tensors_blocked_by_false..recurser)rrrrrrEJs  rEcCshg}dd|D}t|dkrd|}|j|kr||kr|||jD]}||krH||jqHq|S)NcSsg|]}|qSrrrrrrr_sz%backward_walk_ops..r)r5popr-rBr/r) start_opsrQop_type_blacklist found_opsop_stackrrrrrrF]s   rFc Cs|g}dd|D}t|dkrx|}|j|kr||kr||kr|||jD]$}||krP|D]}||qdqPq|S)NcSsg|]}|qSrrrrrrrksz$forward_walk_ops..r)r5rr-rBrr) rrQrrrrrrar~rrrrGis    rGc Gs|jd}tj|dddd}||}tj|dd}tj|ddd}||}|j|j|j|j|jgtj|||dd d} |jd d =t |d \} } t |d \} } | | }d gd d|j dd D}t t t ||dk| | t | | ||S)a Just decompose softmax into its components and recurse, we can handle all of them :) We assume the 'axis' is the last dimension because the TF codebase swaps the 'axis' to the last dimension before the softmax op if 'axis' is not already the last dimension. We also don't subtract the max before tf.exp for numerical stability since that might mess up the attributions and it seems like TensorFlow doesn't define softmax that way (according to the docs) rr Tin0_max)axiskeepdimsr@Z custom_exprSrr)Zgrad_ysNcSsg|]}dqSrkrr!rrrrszsoftmax..rư>)r/r$Z reduce_maxexp reduce_sumrHextendrr^splitr6whererlrq) explainerrrxZin0rZ in0_centeredZevalsZrsumdivraxin0rin0Z xin0_centeredZ rin0_centered delta_in0dup0rrrsoftmaxws$  rc Gst|jdd\}}t|jdd\}}||}dgdd|jddD}t||} t| ||| gd} t|j|j||d| d\} } t t t |dkt || | ||S)NrrcSsg|]}dqSrkrr!rrrrszmaxpool..rHz>) r$rr/rr6maximumconcatr;r-rlrrq zeros_like) rrrxrrxoutroutrrZ cross_maxZdiffsZxmax_posZrmax_posrrrmaxpools &  rc Gs|jd}||}|drP|dsPt|jdks>tdtt|jdd\}}t|jdd\}}dgdd|jddD} dgdd|jddD} t ||| } t |dt ||| t t t|jt|dj} |j dkr(dtt| d kt| | | gSdtt| d kt| | | dgS|dr~|ds~|j|j ||ddgSd std dS) Nrrrz4Only scalar indices supported right now in GatherV2!cSsg|]}dqSrkrr!rrrrszgather..cSsg|]}dqSrkrr!rrrrsResourceGatherrFz3Axis not yet supported to be varying for gather op!)r/rXr5r6r3r$rZto_floatrrlrr4rMr-rrqrr;) rrrxindicesvarxin1rin1rrZdup_in1Zdup_outZ delta_in1_tZout_sumrrrgathers4  :   rcsfdd}|S)Ncs||}|r,|s,t||f|S|rN|sNt||f|S|rt|rtt||f|Sdd|jDSdS)NcSsg|]}dqSr rr_rrrrszAlinearity_1d_nonlinearity_2d..handler..)rXlinearity_1d_handlernonlinearity_2d_handlerr/rrrxr input_ind0 input_ind1op_funcrrhandlers z-linearity_1d_nonlinearity_2d..handlerrrrrrrrrlinearity_1d_nonlinearity_2ds rcsfdd}|S)Ncs||}|r,|s,t||f|S|rN|sNt||f|S|rt|rtt||f|Sdd|jDSdS)NcSsg|]}dqSr rrrrrrszDnonlinearity_1d_nonlinearity_2d..handler..)rXnonlinearity_1d_handlerrr/rrrrrs z0nonlinearity_1d_nonlinearity_2d..handlerrrrrrnonlinearity_1d_nonlinearity_2ds rcsfdd}|S)Ncst||f|Sr )rrrrx input_indrrrsz nonlinearity_1d..handlerrrrrrrnonlinearity_1ds rc Gstt|jD]4}||kr|||rtt|d|jdqt|j|d\}}t|j |d\}}||} dgdd| j ddD} dd|jD} |j |j ||d} t tt| | d kt|jdkr| |n| |dt||| | | |<| S) N th input to cannot vary!rcSsg|]}dqSrkrr!rrrrsz+nonlinearity_1d_handler..rcSsg|]}dqSr rrrrrrsrr)rMr5r/rXr3r,r@r$rrr6r;r-rrlrq) rrrrxr"rrrrrrrar;rrrrs*rcGs|dkr|dkstdt|jdd\}}t|j|d\}} t|j|d\} } || } | | } dgdd| jddD}||| }|| | }||}}d||||}|dt|| |}d||||}|dt|| |}tt |jt | jkrVt t |jt | jkdd}tj ||dd }nVtt |jt | jkrt t |jt | jkdd}tj ||dd }t t t| |d kt ||}t t t| |d kt ||}||gS) Nrrz8TODO: Can't yet handle double inputs that are not first!rcSsg|]}dqSrkrr!rrrrsz+nonlinearity_2d_handler..g?Trr)r3r$rrr/r6rlrVanyrWrrrqr)rrrrrrxrrrrrrrZ delta_in1rZout10Zout01Zout11Zout00Zout0Zout1Zbroadcast_indexrrrrs.    & &&&rcsfdd}|S)Ncst||f|Sr )rrrrrrszlinearity_1d..handlerrrrrr linearity_1ds rcGsZtt|jD]4}||kr|||rtt|d|jdq|j|j|f|SNrr rMr5r/rXr3r,r@r;r-)rrrrxr"rrrrs*rcsfdd}|S)Ncst||f|Sr )linearity_with_excluded_handlerr input_indsrrrsz(linearity_with_excluded..handlerr)rrrrrlinearity_with_excludeds rcGsltt|jD]F}||ks,|t|j|kr|||rtt|d|jdq|j|j|f|Srr)rrrrxr"rrrr#s*rcGs|j|j|f|Sr )r;r-rrrrr\*sr\cGsdd|jDS)a This function name is used to break attribution dependence in the graph traversal. These operation types may be connected above input data values in the graph but their outputs don't depend on the input values (for example they just depend on the shape). cSsg|]}dqSr rrrrrr3sz$break_dependence..)r/rrrrr-srZIdentityZ StridedSliceZSqueezeZ ExpandDimsZPackZBiasAddZUnpackAddSubZMergeZSumZMeanZCastZ TransposeZEnterZExitZ NextIterationZTileZTensorArrayScatterV3ZTensorArrayReadV3ZTensorArrayWriteV3ZShapeZ RandomUniformZ ZerosLikeZReshapeZPadZ ReverseV2r ZConcatV2ZConv2Dr}ZAvgPoolZFusedBatchNormZReluZEluZSigmoidZTanhZSoftplusZExpLogZ ClipByValueZRsqrtZSquareZMaxcCs||||Sr rrTyrrrrYmrZrYZSquaredDifferencecCs t||Sr )r$minimumrrrrrYnrZZMinimumcCs t||Sr )r$rrrrrrYorZZMaximumcCs||Sr rrrrrrYrrZZMulcCs||Sr rrrrrrYsrZZRealDivcCs t||Sr )r$matmulrrrrrYtrZZMatMulZGatherV2rZMaxPoolZSoftmax)!numpyrVr)Zshap.explainers.explainerrmiscrZdistutils.versionrrrr+r$r%r&rrErFrGrrrrrrrrrrrrr\rrrrrrs   ?