9iGiSSKrSSKrSSKrSSKrSSKJr SSKJr SSKJ r SSK J r SSK J r SSKJr SSKJr \R$R'\R$R)\55r\R$R-\S 5/rS r\R2"5S 5r"S S \5rSrSSSSSS.rSSSSSS.rSr Sr!Sr""SS\5r#"SS\ 5r$g)N)Path)knobs) GPUTarget) GPUDriver) _allocation)compile_module_from_src)TensorDescriptorincludec^^ ^ ^ ^SSKnUR5S:wagSSKm SSKJnJnJnJm Jm Jn "U U 4SjST R5nT RX5"U5U"U5U"U55nT RS5RnUT /UlX8lSmT R!TS-5n U UU4S jn U"U"U 5U 5(a%["R$"T R'U 55$g![a gf=f) NrLinux)c_charc_intc_size_tc_void_pc_char_pPOINTERc*>\rSrSrSY4SY4/rSrg)8_find_already_mmapped_dylib_on_linux..DlPhdrInfo dlpi_addr dlpi_nameN)__name__ __module__ __qualname____firstlineno___fields___static_attributes__)rrsZ/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/triton/backends/amd/driver.py DlPhdrInfors ( # ( # r z libc.so.6ic >URRn[[R"U55nTUR ;a&TR X#[T[U555 gg)Nr"r) contentsrrosfsdecodenamememmoveminlen)infosizedatarpctypeslib_namemax_path_lengths rcallback6_find_already_mmapped_dylib_on_linux..callback3sPMM++ Y' ( qvv  NN4CY,P Qr!)platformsystemr/r rrrrr Structure CFUNCTYPECDLLdl_iterate_phdr Exceptionargtypesrestypecreate_string_bufferr%r& string_at)r0r4r rrrr callback_tr9pathr2rrr/r1s` @@@@r$_find_already_mmapped_dylib_on_linuxrAsG# KK  V%% !!%)OQXY_Q`aJ ++k2BB!+H5O#O  & &': ;Dz(+T22{{6++D122 + s2C44 DDcV Sn[RR=n(aMURU5(a&[R R U5(aU$[SUSU35e[U5nU(a7[R R U5(aU$[SUSU35e/n[R R[R R[5SU5n[R R U5(aU$URU5 SSK nUR5nUR5nUR (aU/U-nUH^n[R RUSSU5n[R R U5(aUs $URU5 M` [R""S 5n U (apU R%S 5H[n [R RX5n [R R U 5(aU s $URU 5 M] [R""S 5n U (aX[R RU SU5n [R R U 5(aU $URU 5 [&R("S S /5R+5R-5nU(aX[R RUSU5n [R R U 5(aU $URU 5 [R""S5nU(aX[R RUSU5n[R R U5(aU$URU5 [&R("SS/5R+SS9nUR35Vs/sH<nUR-5RU5(dM)UR%5SPM> nnUH<n[R R U5(aUs $URU5 M> [R RSU5n[R R U5(aU$URU5 [SUSU35e![&R.[04a GNf=fs snf)Nzlibamdhip64.sozTRITON_LIBHIP_PATH 'z' does not point to a valid zmemory mapped 'z'' in process does not point to a valid librtorchLD_LIBRARY_PATH:HIP_PATH hipconfigz--path ROCM_PATHz/sbin/ldconfigz-pignore)errorsz/opt/rocm/lib/zcannot locate z after attempted paths )ramd libhip_pathendswithr%r@exists RuntimeErrorrAjoindirname__file__appendsitegetsitepackagesgetusersitepackagesENABLE_USER_SITEgetenvsplit subprocess check_outputdecodestripCalledProcessErrorFileNotFoundError splitlines)r0env_libhip_path mmapped_pathpaths local_librV site_packages user_siter@env_ld_library_pathdf env_hip_path hip_lib_pathhip_root env_rocm_path rocm_lib_pathlibslinelocsloccommon_install_paths r_get_path_to_hip_runtime_dylibrvAsH ))////  # #H - -"''..2Q2Q" "1/1BB^_g^hijj8AL 77>>, ' ' _\N:abjaklmm E RWW__X6xHI ww~~i   LL((*M((*I " m3 ww||D'5(; 77>>$  K T ))$56$**3/A Q)Aww~~a   LLO 099Z(Lww||L%B 77>>, ' '  \"  **K+BCJJLRRT 77<<%BLww~~l++## LL & IIk*M ]E8D 77>>- ( (  ]#  " "$4d#; < C C8 C TD*.): ^):djjl>S>ST\>] DJJL ):D ^ 77>>#  J S '',,'7B ww~~)**"" LL$% z1HP QQ;  ) )+< =   " _s%BTT(T&T&T#"T#c.^\rSrSrU4SjrSrSrU=r$)HIPUtilscn>[US5(d[[U]U5UlUR$)Ninstance)hasattrsuperrx__new__r{)cls __class__s rr~HIPUtils.__new__s-sJ'' 37%>"r!)rr)rrrrr~rr __classcell__rs@rrxrxs ? ?r!rxc@USS:XagSSSSSSSS S S S S S S S S .U$)Nr*hipDeviceptr_tint8_tint16_tint32_tint64_tuint8_tuint16_tuint32_tuint64_tdouble)i1i8i16i32i64u1u8u16u32u64fp16bf16fp32f32fp64r)tys r ty_to_cpprsQ !u|   !  r!rrr)rrrrr pack_fp16 pack_bf16 pack_fp32 pack_fp64 piiiKKOOOOOc".^^^SnU4SjmU4SjmU4Sjm[U"UR555VVs0sHupEXE_M nnnSRUR5Vs/sH nT"U5PM sn5n[U-nSR[ TUR555n[ [ [URS555n[U5V Vs0sHupX_M nn n[U5S:a)SSRS UR555-OSn /n UR5HUupUS :XaM U[;aU R[US U 35 M6U R[U5S U 35 MW SRU 5n /n UR5HbupUSS :XaU RS U S35 M%U[;aU RSU S35 MFUS :wdMNU RSU 35 Md UR5V Vs/sH-upU[;dM[USU S[USU S3PM/ nn n[5n[ [![U555nUR5V Vs/sHupUS :wdM SU 3PM nn nURS5 URS5 SUS[U 5S:aSU -OSSSRU5SUSUSSRUR5V Vs/sHupT"U5SU S3PM snn 5S US!U S"SRU5S#SRUR5V Vs/sHupUSS :XaS$U S%U SU S&U S'3 OSPM! snn 5S([U 5S:aSSRU 5-OSS)3nU$s snnfs snfs snn fs snn fs snn fs snn fs snn f)*Nc0/nUGH n[U[5(aURS5(aURS5S-n[R "SU5R 5nURSU-5 [SU-5HnURS5 M URS5 [U5HnURS 5 M [U5HnURS5 M MURU5 GM U$) N tensordesc,r"ztensordesc<([^[>]*)rrrr) isinstancestr startswithcountrematchgrouprUrange) signatureoutputsigndimdtype_s r_expand_signature(make_launcher.._expand_signaturesC#s##|(D(Dyy~)!6<BBD cEk*q4xAMM%() d# tAMM%(%tAMM%(% c"'* r!ch>[U[5(aSR[TU55$U$)Nr)rtuplerRmap)r_serialize_signatures rr+make_launcher.._serialize_signatures, c5 ! !88C 4c:; ; r!c>[U[5(a!SR[TU55nSUS3$USS:XagUS:Xag[ U5$)Nr[]rrz PyObject* constexprrrrRrr)rval_extracted_types rr&make_launcher.._extracted_typesV b% ((334Cse1:  a5C<  }r!c >[U[5(a!SR[TU55nSUS3$USS:XagUS:XagSS S S S S SSSSS. [ U5$)N()rrOrrjlbhiLBHIK) rlongrrrrrrrrr)rr format_ofs rr make_launcher..format_of s b% ''#i,-Cse1:  a5C<     B-  r!rrrz, c30# UH upSU3v M g7f)z&_argNr).0rrs r make_launcher..%s L:K5:Ksrz argrptr_infoz.dev_ptr_arg_storagez _argz _storage = z(_argz);z&argz&global_scratchz&profile_scratcha\ #define __HIP_PLATFORM_AMD__ #include #include #include #include #include #include // The list of paths to search for the HIP runtime library. The caller Python // code should substitute the search path placeholder. static const char *hipLibSearchPaths[] = {"a"}; // The list of HIP dynamic library symbols and their signature we are interested // in this file. #define HIP_SYMBOL_LIST(FOR_EACH_ERR_FN, FOR_EACH_STR_FN) \ FOR_EACH_STR_FN(hipGetLastError) \ FOR_EACH_STR_FN(hipGetErrorString, hipError_t hipError) \ FOR_EACH_ERR_FN(hipModuleLaunchKernel, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipModuleLaunchCooperativeKernel, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipPointerGetAttribute, void *data, \ hipPointer_attribute attribute, hipDeviceptr_t ptr) // The HIP symbol table for holding resolved dynamic library symbols. struct HIPSymbolTable { #define DEFINE_EACH_ERR_FIELD(hipSymbolName, ...) \ hipError_t (*hipSymbolName)(__VA_ARGS__); #define DEFINE_EACH_STR_FIELD(hipSymbolName, ...) \ const char *(*hipSymbolName)(__VA_ARGS__); HIP_SYMBOL_LIST(DEFINE_EACH_ERR_FIELD, DEFINE_EACH_STR_FIELD) }; static struct HIPSymbolTable hipSymbolTable; bool initSymbolTable() { // Use the HIP runtime library loaded into the existing process if it exits. void *lib = dlopen("libamdhip64.so", RTLD_NOLOAD); // Otherwise, go through the list of search paths to dlopen the first HIP // driver library. if (!lib) { int n = sizeof(hipLibSearchPaths) / sizeof(hipLibSearchPaths[0]); for (int i = 0; i < n; ++i) { void *handle = dlopen(hipLibSearchPaths[i], RTLD_LAZY | RTLD_LOCAL); if (handle) { lib = handle; } } } if (!lib) { PyErr_SetString(PyExc_RuntimeError, "cannot open libamdhip64.so"); return false; } typedef hipError_t (*hipGetProcAddress_fn)( const char *symbol, void **pfn, int hipVersion, uint64_t hipFlags, hipDriverProcAddressQueryResult *symbolStatus); hipGetProcAddress_fn hipGetProcAddress; dlerror(); // Clear existing errors const char *error = NULL; *(void **)&hipGetProcAddress = dlsym(lib, "hipGetProcAddress"); error = dlerror(); if (error) { PyErr_SetString(PyExc_RuntimeError, "cannot query 'hipGetProcAddress' from libamdhip64.so"); dlclose(lib); return false; } // Resolve all symbols we are interested in. int hipVersion = HIP_VERSION; uint64_t hipFlags = 0; hipDriverProcAddressQueryResult symbolStatus; hipError_t status = hipSuccess; #define QUERY_EACH_FN(hipSymbolName, ...) status = hipGetProcAddress(#hipSymbolName, (void **)&hipSymbolTable.hipSymbolName, hipVersion, hipFlags, &symbolStatus); if (status != hipSuccess) { PyErr_SetString(PyExc_RuntimeError, "cannot get address for '" #hipSymbolName "' from libamdhip64.so"); dlclose(lib); return false; } HIP_SYMBOL_LIST(QUERY_EACH_FN, QUERY_EACH_FN) return true; } static inline void gpuAssert(hipError_t code, const char *file, int line) { if (code != HIP_SUCCESS) { const char* prefix = "Triton Error [HIP]: "; const char* str = hipSymbolTable.hipGetErrorString(code); char err[1024] = {0}; snprintf(err, 1024, "%s Code: %d, Messsage: %s", prefix, code, str ); PyErr_SetString(PyExc_RuntimeError, err); } } #define HIP_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int launch_cooperative_grid, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, hipStream_t stream, hipFunction_t function, hipDeviceptr_t profile_scratchz>) { hipDeviceptr_t global_scratch = 0; void *params[] = { z }; if (gridX*gridY*gridZ > 0 && launch_cooperative_grid) { HIP_CHECK(hipSymbolTable.hipModuleLaunchCooperativeKernel(function, gridX, gridY, gridZ, z*num_warps, 1, 1, shared_memory, stream, params, 0)); return; } if (gridX*gridY*gridZ > 0) { HIP_CHECK(hipSymbolTable.hipModuleLaunchKernel(function, gridX, gridY, gridZ, ae *num_warps, 1, 1, shared_memory, stream, params, 0)); } } typedef struct _DevicePtrInfo { hipDeviceptr_t dev_ptr; bool valid; } DevicePtrInfo; static PyObject* data_ptr_str = NULL; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; hipError_t status = hipSuccess; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ret = PyObject_CallMethodNoArgs(obj, data_ptr_str); if (!ret) { PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); ptr_info.valid = false; goto cleanup; } if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; goto cleanup; } ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(ret); if (!ptr_info.dev_ptr) goto cleanup; uint64_t dev_ptr; status = hipSymbolTable.hipPointerGetAttribute(&dev_ptr, HIP_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == hipErrorInvalidValue) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; // Clear and ignore HIP error (void)hipSymbolTable.hipGetLastError(); } ptr_info.dev_ptr = (hipDeviceptr_t)dev_ptr; cleanup: Py_DECREF(ret); return ptr_info; } static uint16_t pack_fp16(double f) { uint16_t result; // from https://github.com/python/pythoncapi-compat/blob/5e317108f872c904eb726cb8d560dcadbdf88a72/pythoncapi_compat.h#L482-L492 #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) _PyFloat_Pack2(f, (unsigned char*)&result, 1); #else PyFloat_Pack2(f, (char*)&result, 1); #endif return result; } static uint16_t pack_bf16(double f) { float f32 = (float)f; uint32_t u32 = *(uint32_t*)&f32; return (uint16_t)(u32 >> 16); } static uint32_t pack_fp32(double f) { float f32 = (float)f; return *(uint32_t*)&f32; } static uint64_t pack_fp64(double f) { return *(uint64_t*)&f; } static PyObject* launch(PyObject* self, PyObject* args) { int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; int launch_cooperative_grid; PyObject *profile_scratch_obj = NULL; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL;  z; z if(!PyArg_ParseTuple(args, "a,", &launch_cooperative_grid, &gridX, &gridY, &gridZ, &_stream, &_function, &profile_scratch_obj, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hook z)) { return NULL; } a // extract kernel metadata int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ; if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) { return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_enter_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } hipDeviceptr_t profile_scratch = 0; if (profile_scratch_obj != Py_None) { DevicePtrInfo profile_scratch_info = getPointer(profile_scratch_obj, -1); if (!profile_scratch_info.valid) { return NULL; } profile_scratch = profile_scratch_info.dev_ptr; } // raise exception asap zDevicePtrInfo ptr_infoz = getPointer(_argz); if (!ptr_infoz.valid) return NULL;z; _launch(gridX, gridY, gridZ, num_warps, num_ctas, launch_cooperative_grid, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (hipStream_t)_stream, (hipFunction_t)_function, (hipDeviceptr_t)profile_scratcha); if(launch_exit_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_exit_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } if(PyErr_Occurred()) { return NULL; } Py_RETURN_NONE; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { if (!initSymbolTable()) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } data_ptr_str = PyUnicode_InternFromString("data_ptr"); if(data_ptr_str == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } ) enumeratevaluesrR_BASE_ARGS_FORMATrlistfilterboolr[r*itemsFLOAT_STORAGE_TYPErUrFLOAT_PACK_FUNCTIONrvr) constantsr warp_sizeridxsr args_formatformatr args_list arg_decl_list arg_declsinternal_args_listfloat_storage_declsrNparamsrrrrs @@@r make_launcherrs 6 *'00A)BRBRBT0U&VW&VFC&VIW''93C3C3EF3ER9R=3EFGK  ,F193C3C3EFGIVD)//#"678I"+I"67"6$!"6I7PST]P^abPbtyy L)//:K LLLhjIM"    # #  $6r$:#;4s!C D  IbM?$qc!: ; # -(I" a5C<  % %8&< = % %  % %QCx&8 9 ;   % %QCj 1 #__&&EA # # Z b ! "%s+6I"6M5NeTUSVVXY& 12K%I' (F&/oo&7 M&7UQ2;LjQCj&7F M MM#$ MM$% -.9Mi:@RUXYbUcfgUgAEHQAQmo@ppyy()*^^g]hiST]R]Y^r88Y__=N O=NEA#$E!B /=N OPQR &x(RS\Q\]88  !"299R[RaRaRcdRcINIJoqrsotx{o{&qc);A3bCSTUSVVjkBDDRcdeffS|@R|SVW|WTX[_[d[dew[xTx]_S`+`ob CF JY XF8,Np PBds5O.)O4*O9<O? O?- P= P<P &P0c^U4SjnU$)zF Replace all tensor descriptors with the base ptr, shape, and strides c|>US[[5nU[[5Sn/nUHn[U[5(a]UR UR /UR QURQURS:HPUR QURQ5 MuURU5 M T"/UQUQ76$)Nnan) r*rrr extendbaseshapestridespaddingrU)args meta_argsraw_kernel_args final_argsarglaunchers rinner,wrap_handle_tensor_descriptor..innerrs0#/01 s#4567 "C#/00!!388"vcii"v#++"vs{{V[G["v^a^g^g"vjmjuju"vw!!#&#00Z00r!r)rrs` rwrap_handle_tensor_descriptorrms 1" Lr!c \rSrSrSrSrSrg) HIPLauncherict^[TS5(a TRO [5nU4SjnUR5VVs0sHupVU"U5U_M nnnTRR5VVs0sHupVXV_M nnn[ X7UR 5m[TS[S9n[SUR555n U (a[UR5O URUl URUl URUlURUlgs snnfs snnf)Nrc~>[U[5(a&TRRR U54$U$N)rrfn arg_namesindex)xrs r&HIPLauncher.__init__..s2Z3=O=OSVV--33A69VUVVr!__triton_launcherrc3r# UH-n[U[5=(a URS5v M/ g7f)rN)rrr)rrs rr'HIPLauncher.__init__..s+!vcu\_*S#"6"W3>>,;W"Wcus57)r|rdictrrrrrranyrrlaunchlaunch_cooperative_gridprofile_scratch_sizeprofile_scratch_align) rrmetadatararg_idxrvaluerrhas_tensor_desc_args ` rrHIPLauncher.__init__s%,S+%>%>CMMDF V;D??;LM;LZSWS\5(;L M25--2E2E2GH2GJCSZ2G HI(2D2DE%#4GVbc!!vclcscscu!vvCV3CJJ?\_\f\f '/'G'G$$,$A$A!%-%C%C"NHs D.7D4c ^^^^UUUU4SjnU"URUR[R5nUR"UR TTTTXX/UQ76 g)Nc\>US:a%TT-T-nX0-nUR5nU"XAT 5$gNr)get) r,align allocator grid_size alloc_sizealloc_fngridXgridYgridZstreams rallocate_scratch.HIPLauncher.__call__..allocate_scratchs;ax!EME1 &- $==? 6::r!)r'r(r_profile_allocatorr%r&) rr7r8r9r:functionr r;profile_scratchs ```` r__call__HIPLauncher.__call__sW  +4+D+DdF`F`+6+I+IK D00%vxpkopr!)r%r&r(r'N)rrrrrr@rrr!rrrs D qr!rcn^\rSrSrU4SjrSr\S5rS\S\4Sjr Sr S r S r S r S rS rU=r$) HIPDrivericV>[TU]5 [5Ul[Ulgr)r}rrxutilsr launcher_cls)rrs rrHIPDriver.__init__s Z 'r!c"SSKnUR$r0)rDcudarrDs rget_device_interfaceHIPDriver.get_device_interfaceszzr!cSSKnURR5=(a URRSL$![ a gf=f)NrF)rDrI is_availableversionhip ImportError)rDs r is_activeHIPDriver.is_activesC  ::**,P%--2C2C42O P  s>r!cJSSKnURSUR55$)NrrI)rDr^r[rJs rget_active_torch_device!HIPDriver.get_active_torch_devices||FD$;$;$=>>r!cSSKJn U$)Nr)do_bench)triton.testingrf)rrfs rget_benchmarkerHIPDriver.get_benchmarkers +r!c\SSKnSnUR[US-5URSS9$)NrirI)rr^)rDemptyint)rrD cache_sizes rget_empty_cache_for_benchmark'HIPDriver.get_empty_cache_for_benchmarks.' {{3zQ/uyy{PPr!c$UR5 gr)zero_)rcaches r clear_cacheHIPDriver.clear_caches  r!)rFrE)rrrrrrK staticmethodrRrrVr`rcrhrortrrrs@rrCrCsS( ?? Qr!rC)% functoolsr%r\rpathlibrtritonrtriton.backends.compilerrtriton.backends.driverrtriton.runtimertriton.runtime.buildrtriton.tools.tensor_descriptorr r@rSrealpathrTrRrrA lru_cachervobjectrxrrrrrrrrCrr!rrs  .,&8; ''//"''**84 5 Wi01 -` \R\R~?v?( .            "M` 2q&q@. .r!