B \X @sdZddlmZmZmZddlZddlZddlZddlm Z m Z m Z m Z m Z mZmZddlZddlmZddlmZddlmZmZdd lmZmZmZeeZdZdZ d Z!d Z"d Z#e Z$e Z%d &Z'x&e(e'D]\Z)Z*e+ej,ee*e)qWddZ-e.Z/Gddde0Z1Gddde0Z2dddZ3e3e4j5dZ6ye17Z8WndZ8YnXe8dkrfdZ9ndZ9ddZ:ddZ;d Zd$Z?d%Z@d&ZAd'ZBd(d)ZCd*d+ZDd,d-ZEeFd.ZGeFd/ZHeFd0ZIeFd1Jd2d3ZKeFd4ZLd5d6d7d8d9d:d;dd?d@h ZMeFdAZNeFdBZOeFdCZPeFdDZQeFdEZReFdFZSeFdGZTeFdHZUdIdJZVdKdLZWdMdNZXdOdPZYdQdRZZdS)Sz( This is a direct translation of nvvm.h )print_functionabsolute_importdivisionN)c_void_pc_intPOINTERc_char_pc_size_tbyrefc_char)ir)config) NvvmErrorNvvmSupportError) get_libdeviceopen_libdevice open_cudaliba NVVM_SUCCESS NVVM_ERROR_OUT_OF_MEMORY NVVM_ERROR_PROGRAM_CREATION_FAILURE NVVM_ERROR_IR_VERSION_MISMATCH NVVM_ERROR_INVALID_INPUT NVVM_ERROR_INVALID_PROGRAM NVVM_ERROR_INVALID_IR NVVM_ERROR_INVALID_OPTION NVVM_ERROR_NO_MODULE_IN_PROGRAM NVVM_ERROR_COMPILATION cCs(y tWntk rdSXdSdS)z( Return if libNVVM is available FTN)NVVMrrr6lib/python3.7/site-packages/numba/cuda/cudadrv/nvvm.py is_available1s  rc @seZdZdZeeeeefeeefeeefeeee efeeeeefeeee feeefeeee feeefd Z dZ ddZ ddZ d d d ZdS) rzProcess-wide singleton. ) nvvmVersionnvvmCreateProgramnvvmDestroyProgramnvvmAddModuleToProgramnvvmCompileProgramnvvmGetCompiledResultSizenvvmGetCompiledResultnvvmGetProgramLogSizenvvmGetProgramLogNc Cst|jdkrt||_}ytddd|_Wn8tk rj}zd|_d}t||Wdd}~XYnXxF|j D]8\}}t |j|}|d|_ |dd|_ t |||qxWWdQRX|jS)NZnvvmT)Zcccz;libNVVM cannot be found. Do `conda install cudatoolkit`: %srr) _nvvm_lock_NVVM__INSTANCEobject__new__rdriverOSErrorr _PROTOTYPESitemsgetattrZrestypeZargtypessetattr)clsZinsteerrmsgnameprotofuncrrrr'js   z NVVM.__new__cCs8t}t}|t|t|}||d|j|jfS)NzFailed to get version.)rrr check_errorvalue)selfmajorminorerrrrr get_versions  zNVVM.get_versionFcCs2|r.t|t|}|r*t|tdn|dS)Nr)rRESULT_CODE_NAMESprintsysexit)r6errormsgr>excrrrr4s  zNVVM.check_error)F)__name__ __module__ __qualname____doc__ nvvm_resultrr nvvm_programrr r*r%r'r:r4rrrrr?s     rc@s<eZdZddZddZddZddZd d Zd d Zd S)CompilationUnitcCs4t|_t|_|jt|j}|j|ddS)NzFailed to create CU)rr(rG_handlerr r4)r6r9rrr__init__szCompilationUnit.__init__cCs*t}|t|j}|j|ddddS)NzFailed to destroy CUT)r>)rrr rIr4)r6r(r9rrr__del__szCompilationUnit.__del__cCs*|j|j|t|d}|j|ddS)z Add a module level NVVM IR to a compilation unit. - The buffer should contain an NVVM module IR either in the bitcode representation (LLVM3.0) or in the text representation. NzFailed to add module)r(rrIlenr4)r6bufferr9rrr add_modules  zCompilationUnit.add_modulec Kshg}d|kr |dr |d|dr>|d|d|dr\|d|dd}x@|D]8}||krftt||}|d|d d |fqfW|rd tt| }t d |t t |d d|D}|j|jt ||}||dt} |j|jt| }||dt| j} |j|j| }||d||_| ddS)aFPerform Compliation The valid compiler options are * - -g (enable generation of debugging information) * - -opt= * - 0 (disable optimizations) * - 3 (default, enable optimizations) * - -arch= * - compute_20 (default) * - compute_30 * - compute_35 * - -ftz= * - 0 (default, preserve denormal values, when performing * single-precision floating-point operations) * - 1 (flush denormal values to zero, when performing * single-precision floating-point operations) * - -prec-sqrt= * - 0 (use a faster approximation for single-precision * floating-point square root) * - 1 (default, use IEEE round-to-nearest mode for * single-precision floating-point square root) * - -prec-div= * - 0 (use a faster approximation for single-precision * floating-point division and reciprocals) * - 1 (default, use IEEE round-to-nearest mode for * single-precision floating-point division and reciprocals) * - -fma= * - 0 (disable FMA contraction) * - 1 (default, enable FMA contraction) * debugz-gZoptz-opt=%darchz-arch=%s)ZftzZ prec_sqrtZprec_divZfmaz-%s=%d_-z, zunsupported option {0}cSsg|]}t|dqS)utf8)rencode).0xrrr sz+CompilationUnit.compile..zFailed to compile z&Failed to get size of compiled result.zFailed to get compiled result.N)popappendgetintboolreplacejoinmapreprkeysrformatrrLr(rrI _try_errorr r r r r5r!get_loglog) r6ZoptionsoptsZ other_optionskvZoptstrZc_optsr9reslenZptxbufrrrcompiles8#           zCompilationUnit.compilecCs|j|d||fdS)Nz%s %s)r(r4rd)r6r9r@rrrrcszCompilationUnit._try_errorcCslt}|j|jt|}|j|d|jdkrht|j}|j|j|}|j|d|j dSdS)Nz#Failed to get compilation log size.rzFailed to get compilation log.rS) r r(r"rIr r4r5r r#decode)r6rir9Zlogbufrrrrds   zCompilationUnit.get_logN) rBrCrDrJrKrNrjrcrdrrrrrHs  TrHze-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64ze-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64) @)rr)rr) )r)rpr)rr)rr)rr)rrp)rr)r)rqr)rqrp) )rr)rr)rr)rrp)rr)rqr)rqr)rqrp)rcCsXxNttD]B\}}||kr|S||kr |dkr@td||q t|dSq WtdS)Nrz@GPU compute capability %d.%d is not supported (requires >=%d.%d)r) enumerate SUPPORTED_CCr)ZmycciZccrrr _find_arch$srwcCs"tjrtj}n t||f}d|S)z1Matches with the closest architecture option z compute_%d%d)r Z FORCE_CUDA_CCrw)r7r8rPrrrget_arch_option7s rxz Please define environment variable NUMBAPRO_LIBDEVICE=/path/to/libdevice /path/to/libdevice -- is the path to the directory containing the libdevice.*.bc files in the installation of CUDA. (requires CUDA >=8.0) zMissing libdevice file for {arch}. Please ensure you have package cudatoolkit 8.0. Install package by: conda install cudatoolkit=8.0 c@s4eZdZiZddddgZddZddZd d Zd S) LibDevice compute_20Z compute_30Z compute_35Z compute_50cCsT||jkr>||}t|dkr0ttj|dt||j|<||_|j||_dS)z@ arch --- must be result from get_arch_option() N)rP) _cache__get_closest_archr RuntimeErrorMISSING_LIBDEVICE_FILE_MSGrbrrPbc)r6rPrrrrJXs   zLibDevice.__init__cCs*|jd}x|jD]}||kr|}qW|S)Nr) _known_arch)r6rPZresZ potentialrrrr|es   zLibDevice._get_closest_archcCs|jS)N)r)r6rrrrZlsz LibDevice.getN)rBrCrDr{rrJr|rZrrrrryOs ryz define internal i32 @___numba_cas_hack(i32* %ptr, i32 %cmp, i32 %val) alwaysinline { %out = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %val monotonic ret i32 %out } ae define internal double @___numba_atomic_double_add(double* %ptr, double %val) alwaysinline { entry: %iptr = bitcast double* %ptr to i64* %old2 = load volatile i64, i64* %iptr br label %attempt attempt: %old = phi i64 [ %old2, %entry ], [ %cas, %attempt ] %dold = bitcast i64 %old to double %dnew = fadd double %dold, %val %new = bitcast double %dnew to i64 %cas = cmpxchg volatile i64* %iptr, i64 %old, i64 %new monotonic %repeat = icmp ne i64 %cas, %old br i1 %repeat, label %attempt, label %done done: %result = bitcast i64 %old to double ret double %result } a define internal {T} @___numba_atomic_{T}_max({T}* %ptr, {T} %val) alwaysinline {{ entry: %ptrval = load volatile {T}, {T}* %ptr ; Check if val is a NaN and return *ptr early if so %valnan = fcmp uno {T} %val, %val br i1 %valnan, label %done, label %lt_check lt_check: %dold = phi {T} [ %ptrval, %entry ], [ %dcas, %attempt ] ; Continue attempts if dold < val or dold is NaN (using ult semantics) %lt = fcmp ult {T} %dold, %val br i1 %lt, label %attempt, label %done attempt: ; Attempt to swap in the larger value %iold = bitcast {T} %dold to {Ti} %iptr = bitcast {T}* %ptr to {Ti}* %ival = bitcast {T} %val to {Ti} %cas = cmpxchg volatile {Ti}* %iptr, {Ti} %iold, {Ti} %ival monotonic %dcas = bitcast {Ti} %cas to {T} br label %lt_check done: ; Return max %ret = phi {T} [ %ptrval, %entry ], [ %dold, %lt_check ] ret {T} %ret }} a define internal {T} @___numba_atomic_{T}_min({T}* %ptr, {T} %val) alwaysinline{{ entry: %ptrval = load volatile {T}, {T}* %ptr ; Check if val is a NaN and return *ptr early if so %valnan = fcmp uno {T} %val, %val br i1 %valnan, label %done, label %gt_check gt_check: %dold = phi {T} [ %ptrval, %entry ], [ %dcas, %attempt ] ; Continue attempts if dold > val or dold is NaN (using ugt semantics) %lt = fcmp ugt {T} %dold, %val br i1 %lt, label %attempt, label %done attempt: ; Attempt to swap in the smaller value %iold = bitcast {T} %dold to {Ti} %iptr = bitcast {T}* %ptr to {Ti}* %ival = bitcast {T} %val to {Ti} %cas = cmpxchg volatile {Ti}* %iptr, {Ti} %iold, {Ti} %ival monotonic %dcas = bitcast {Ti} %cas to {T} br label %gt_check done: ; Return min %ret = phi {T} [ %ptrval, %entry ], [ %dold, %gt_check ] ret {T} %ret }} cCsF|}x2t|D]&\}}|drd}|t||<PqWd|S)z@ Find the line containing the datalayout and replace it ztarget datalayoutztarget datalayout = "{0}" ) splitlinesrt startswithrbdefault_data_layoutr^)llvmirlinesrvZlnZtmprrr_replace_datalayouts rc Kst}t|ddd}t|}dtfdtfdtjddd fd tjd d d fd tjddd fdtjd d d fg}x|D]\}}| ||}qvWt |}| | d| ||j f|}t|S)NrPrz)rPz.declare i32 @___numba_cas_hack(i32*, i32, i32)z;declare double @___numba_atomic_double_add(double*, double)z7declare float @___numba_atomic_float_max(float*, float)floati32)TZTiz;declare double @___numba_atomic_double_max(double*, double)ZdoubleZi64z7declare float @___numba_atomic_float_min(float*, float)z;declare double @___numba_atomic_double_min(double*, double)rS)rHryrZrir_numba_cas_hackir_numba_atomic_double_addir_numba_atomic_maxrbir_numba_atomic_minr]llvm39_to_34_irrNrTrjpatch_ptx_debug_pubnames)rrfZcuZ libdeviceZ replacementsZdeclfnptxrrr llvm_to_ptxs, rcCsVxP|d}|dkrP|d|}|dkr2td|d|||dd}qW|S)z Patch PTX to workaround .debug_pubnames NVVM error:: ptxas fatal : Internal error: overlapping non-identical data s.section .debug_pubnamesr}z missing "}"Nr)find ValueError)rstartstoprrrrs   rz \!\d+\s*=zmetadata\s*\![{'\"0-9]z\!\d+z,\!{i32 \d, \!\"Debug Info Version\", i32 \d} z\s+z"^attributes #\d+ = \{ ([\w\s]+)\ }Z alwaysinlineZcoldZ inlinehintZminsizeZ noduplicateZnoinlineZnoreturnZnounwindZoptnoneZoptiszeZreadnonereadonlyz"\bgetelementptr\s(?:inbounds )?\(?z=\s*\bload\s(?:\bvolatile\s)?z(call\s[^@]+\))(\s@)z\s*!range\s+!\d+z [,{}()[\]]z \bnonnull\bz"\b(local_unnamed_addr|writeonly)\bz\((.*)\)c CsTdd}g}x:|D],}|dr4|dd}|drVd|krV|dd}t|rdt|kr|d d }|d d }|d }|d|d||dd}}dd}d |t ||f}|drqt |dk rt dd|}|drDt |}|d} d dd| D} ||d| }d|krt|}|dkrptd|f|} |d| ||| d}d|krt|}|r|} |d| ||| d}d|krt d|}t d|d}d|krt t|}d |kr0d|kr0t t|}t d|}||qWd! |S)"z+ Convert LLVM 3.9 IR for LLVM 3.4. cSsd}d}xpt||}|dkr.td|fP|}|d}|dkrT|dkrvPq |dkrf|d7}q |dkr |d8}q W||dS)Nrzfailed parsing leading type: %s,z{[(rz)]}) re_type_toksearchr}endgrouplstrip)sZ par_levelposmtokrrrparse_out_leading_type,s"    z/llvm39_to_34_ir..parse_out_leading_typez!numba.llvm.dbg.cuz !llvm.dbg.cuz%tail call void asm sideeffect "// dbgz !numba.dbgz!dbgNz!{z metadata !{z!"z metadata !"=rcSsd|dS)Nz metadata r)r)rrrrfix_metadata_refYsz)llvm39_to_34_ir..fix_metadata_refrzsource_filename =cSsdS)Nrkr)rrrr_sz!llvm39_to_34_ir..z attributes #css|]}|tkr|VqdS)N)supported_attributes)rUarrr esz"llvm39_to_34_ir..zgetelementptr z failed parsing getelementptr: %szload zcall z\1*\2rkrz @llvm.memsetZdeclarer)rrr]rre_metadata_defmatchre_metadata_correct_usagerrr^re_metadata_refsubre_unsupported_keywordsre_attributes_defrsplitre_getelementptrr}rre_loadre_callre_rangerstripre_parenthesized_list_replace_llvm_memset_usage _replace_llvm_memset_declarationre_annotationsrY) r rZbuflineZassigposZlhsZrhsrrZattrsrrrrr(sf       "              rcCsPt|dd}td|dd}|dd|d|}d|S) zNReplace `llvm.memset` usage for llvm7+. Used as functor for `re.sub. rrz align (\d+)rrszi32 {}z, z({}))listrrrerinsertrbr^)rparamsZalignoutrrrrs  rcCs4t|dd}|ddd|}d|S)zTReplace `llvm.memset` declaration for llvm7+. Used as functor for `re.sub. rrrsrz, z({}))rrrrr^rb)rrrrrrrs  rc Csddlm}m}m}m}|j}|||d||df}|||}|d}| |t d} | | d| d| d| dg} | d| dS) Nr)MetaDataMetaDataStringConstantTypeZkernelrznvvm.annotationsrmrpznvvmir.version)Zllvmlite.llvmpy.corerrrrmodulerZr[Zget_or_insert_named_metadataaddr ZIntTypeZ add_metadataZadd_named_metadata) ZlfuncrrrrrZopsZmdZnmdrZmd_verrrrset_cuda_kernels    "rcCs t|_dS)N)r data_layout)rrrrfix_data_layoutsr)[rEZ __future__rrrr=ZloggingrZctypesrrrrr r r Z threadingZllvmliter Znumbar r?rrZlibsrrrZ getLoggerrBZloggerZADDRSPACE_GENERICZADDRSPACE_GLOBALZADDRSPACE_SHAREDZADDRSPACE_CONSTANTZADDRSPACE_LOCALrGrFrr;rtrvrgr-modulesrZLockr$r&rrHrtuple __itemsize__rr:Z NVVM_VERSIONrurwrxZMISSING_LIBDEVICE_MSGr~ryrrrrrrrrjrrrr]Zre_metadata_debuginforrrrrrrrrrrrrrrrrrrs$     Q}  & "             j