Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP DNM] L0 always sysman #595

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

[WIP DNM] L0 always sysman #595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
36d3b61
levelzero: disable ZES_ENABLE_SYSMAN=1 when zesInit() is available
bgoglin Jun 9, 2023
09b5f13
levelzero: only get memory info from sysman
bgoglin Jun 12, 2023
1b3f6c7
levelzero: only get pci locality info from sysman
bgoglin Jun 12, 2023
c003c2f
levelzero: remove some warnings about missing sysman
bgoglin Jun 12, 2023
980e829
levelzero: remove the setting of ZES_ENABLE_SYSMAN=1
bgoglin Jun 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 0 additions & 2 deletions config/hwloc.m4
View file
Open in desktop
Expand Up @@ -1410,7 +1410,6 @@ return clGetDeviceIDs(0, 0, 0, NULL, NULL);
[hwloc_levelzero_happy=yes
HWLOC_LEVELZERO_REQUIRES=libze_loader
AC_CHECK_LIB([ze_loader], [zesInit], [AC_DEFINE(HWLOC_HAVE_ZESINIT, 1, [Define to 1 if zesInit is available])])
AC_CHECK_LIB([ze_loader], [zeDevicePciGetPropertiesExt], [AC_DEFINE(HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT, 1, [Define to 1 if zeDevicePciGetPropertiesExt is available])])
], [hwloc_levelzero_happy=no])
if test x$hwloc_levelzero_happy = xno; then
hwloc_levelzero_happy=yes
Expand All @@ -1421,7 +1420,6 @@ return clGetDeviceIDs(0, 0, 0, NULL, NULL);
[zesDevicePciGetProperties],
[HWLOC_LEVELZERO_LIBS="-lze_loader"
AC_CHECK_LIB([ze_loader], [zesInit], [AC_DEFINE(HWLOC_HAVE_ZESINIT, 1, [Define to 1 if zesInit is available])])
AC_CHECK_LIB([ze_loader], [zeDevicePciGetPropertiesExt], [AC_DEFINE(HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT, 1, [Define to 1 if zeDevicePciGetPropertiesExt is available])])
], [hwloc_levelzero_happy=no])
], [hwloc_levelzero_happy=no])
], [hwloc_levelzero_happy=no])
Expand Down
163 changes: 19 additions & 144 deletions hwloc/topology-levelzero.c
View file
Open in desktop
Expand Up @@ -63,7 +63,6 @@ hwloc__levelzero_osdev_array_find(struct hwloc_osdev_array *array,

static void
hwloc__levelzero_properties_get(ze_device_handle_t h, hwloc_obj_t osdev,
int sysman_maybe_missing,
int *is_integrated_p)
{
ze_result_t res;
Expand Down Expand Up @@ -139,11 +138,8 @@ hwloc__levelzero_properties_get(ze_device_handle_t h, hwloc_obj_t osdev,
hwloc_obj_add_info(osdev, "LevelZeroBoardNumber", (const char *) prop2.boardNumber);
} else {
static int warned = 0;
if (!warned) {
if (sysman_maybe_missing == 1 && HWLOC_SHOW_ALL_ERRORS())
fprintf(stderr, "hwloc/levelzero: zesDeviceGetProperties() failed (ZES_ENABLE_SYSMAN=1 set too late?).\n");
else if (sysman_maybe_missing == 2 && HWLOC_SHOW_ALL_ERRORS())
fprintf(stderr, "hwloc/levelzero: zesDeviceGetProperties() failed (ZES_ENABLE_SYSMAN=0).\n");
if (!warned && HWLOC_SHOW_ALL_ERRORS()) {
fprintf(stderr, "hwloc/levelzero: zesDeviceGetProperties() failed (sysman missing?).\n");
warned = 1;
}
/* continue in degraded mode, we'll miss locality and some attributes */
Expand Down Expand Up @@ -182,9 +178,9 @@ hwloc__levelzero_cqprops_get(ze_device_handle_t h,
}

static int
hwloc__levelzero_memory_get_from_sysman(zes_device_handle_t h,
hwloc_obj_t root_osdev,
unsigned nr_osdevs, hwloc_obj_t *sub_osdevs)
hwloc__levelzero_memory_get(zes_device_handle_t h,
hwloc_obj_t root_osdev,
unsigned nr_osdevs, hwloc_obj_t *sub_osdevs)
{
zes_mem_handle_t *mh;
uint32_t nr_mems;
Expand Down Expand Up @@ -289,94 +285,6 @@ hwloc__levelzero_memory_get_from_sysman(zes_device_handle_t h,
return 0;
}

static void
hwloc__levelzero_memory_get_from_coreapi(ze_device_handle_t h,
hwloc_obj_t osdev,
int ignore_ddr)
{
ze_device_memory_properties_t *mh;
uint32_t nr_mems;
ze_result_t res;

nr_mems = 0;
res = zeDeviceGetMemoryProperties(h, &nr_mems, NULL);
if (res != ZE_RESULT_SUCCESS || !nr_mems)
return;
hwloc_debug("L0/CoreAPI: found %u memories in osdev %s\n",
nr_mems, osdev->name);

mh = malloc(nr_mems * sizeof(*mh));
if (mh) {
res = zeDeviceGetMemoryProperties(h, &nr_mems, mh);
if (res == ZE_RESULT_SUCCESS) {
unsigned m;
for(m=0; m<nr_mems; m++) {
const char *_name = mh[m].name;
char name[300], value[64];
/* FIXME: discrete GPUs report 95% of the physical memory (what sysman sees)
* while integrated GPUs report 80% of the host RAM (sysman sees 0), adjust?
*/
hwloc_debug("L0/CoreAPI: found memory name %s size %llu in osdev %s\n",
mh[m].name, (unsigned long long) mh[m].totalSize, osdev->name);
if (!mh[m].totalSize)
continue;
if (ignore_ddr && !strcmp(_name, "DDR"))
continue;
if (!_name[0])
_name = "Memory";
snprintf(name, sizeof(name), "LevelZero%sSize", _name); /* HBM or DDR, or Memory if unknown */
snprintf(value, sizeof(value), "%llu", (unsigned long long) mh[m].totalSize >> 10);
hwloc_obj_add_info(osdev, name, value);
}
}
free(mh);
}
}


static void
hwloc__levelzero_memory_get(zes_device_handle_t h, hwloc_obj_t root_osdev, int is_integrated,
unsigned nr_subdevices, zes_device_handle_t *subh, hwloc_obj_t *sub_osdevs)
{
static int memory_from_coreapi = -1; /* 1 means coreapi, 0 means sysman, -1 means sysman if available or coreapi otherwise */
static int first = 1;

if (first) {
char *env;
env = getenv("HWLOC_L0_COREAPI_MEMORY");
if (env)
memory_from_coreapi = atoi(env);

if (memory_from_coreapi == -1) {
int ret = hwloc__levelzero_memory_get_from_sysman(h, root_osdev, nr_subdevices, sub_osdevs);
if (!ret) {
/* sysman worked, we're done, disable coreapi for next time */
hwloc_debug("levelzero: sysman/memory succeeded, disabling coreapi memory queries\n");
memory_from_coreapi = 0;
return;
}
/* sysman failed, enable coreapi */
hwloc_debug("levelzero: sysman/memory failed, enabling coreapi memory queries\n");
memory_from_coreapi = 1;
}

first = 0;
}

if (memory_from_coreapi > 0) {
unsigned k;
int ignore_ddr = (memory_from_coreapi != 2) && is_integrated; /* DDR ignored in integrated GPUs, it's like the host DRAM */
hwloc__levelzero_memory_get_from_coreapi(h, root_osdev, ignore_ddr);
for(k=0; k<nr_subdevices; k++)
hwloc__levelzero_memory_get_from_coreapi(subh[k], sub_osdevs[k], ignore_ddr);
} else {
hwloc__levelzero_memory_get_from_sysman(h, root_osdev, nr_subdevices, sub_osdevs);
/* no need to call hwloc__levelzero_memory_get() on subdevices,
* the call on the root device is enough (and identical to a call on subdevices)
*/
}
}

struct hwloc_levelzero_ports {
unsigned nr_allocated;
unsigned nr;
Expand Down Expand Up @@ -604,8 +512,6 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
uint32_t nbdrivers, i, k, zeidx;
struct hwloc_osdev_array oarray;
struct hwloc_levelzero_ports hports;
int sysman_maybe_missing = 0; /* 1 if ZES_ENABLE_SYSMAN=1 was NOT set early, 2 if ZES_ENABLE_SYSMAN=0 */
char *env;

assert(dstatus->phase == HWLOC_DISC_PHASE_IO);

Expand All @@ -620,30 +526,15 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
#ifdef HWLOC_HAVE_ZESINIT
res = zesInit(0);
if (res != ZE_RESULT_SUCCESS) {
hwloc_debug("hwloc/levelzero: Failed to initialize LevelZero Sysman in zesInit(): 0x%x\n", (unsigned)res);
hwloc_debug("hwloc/levelzero: Continuing. Hopefully ZES_ENABLE_SYSMAN=1\n");
}
#endif /* HWLOC_HAVE_ZESINIT */

/* Tell L0 to create sysman devices.
* If somebody already initialized L0 without Sysman,
* zesDeviceGetProperties() will fail and warn in hwloc__levelzero_properties_get().
* The lib constructor and Windows DllMain tried to set ZES_ENABLE_SYSMAN=1 early (see topology.c),
* we try again in case they didn't.
*/
env = getenv("ZES_ENABLE_SYSMAN");
if (!env) {
/* setenv() is safer than putenv() but not available on Windows */
#ifdef HWLOC_WIN_SYS
putenv("ZES_ENABLE_SYSMAN=1");
#else
setenv("ZES_ENABLE_SYSMAN", "1", 1);
#endif
/* we'll warn in hwloc__levelzero_properties_get() if we fail to get zes devices */
sysman_maybe_missing = 1;
} else if (!atoi(env)) {
sysman_maybe_missing = 2;
if (res == ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) {
fprintf(stderr, "hwloc/levelzero: Unsupported zesInit(), please either an older or a more recent level-zero runtime.\n");
} else if (HWLOC_SHOW_ALL_ERRORS()) {
fprintf(stderr, "hwloc/levelzero: Failed to initialize LevelZero in zesInit(): 0x%x\n", (unsigned)res);
}
return 0;
}
#endif /* !HWLOC_HAVE_ZESINIT */
/* if !HWLOC_HAVE_ZESINIT, the user must set ZES_ENABLE_SYSMAN=1 in his environment */

res = zeInit(0);
if (res != ZE_RESULT_SUCCESS) {
Expand Down Expand Up @@ -716,7 +607,7 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
snprintf(buffer, sizeof(buffer), "%u", j);
hwloc_obj_add_info(osdev, "LevelZeroDriverDeviceIndex", buffer);

hwloc__levelzero_properties_get(dvh[j], osdev, sysman_maybe_missing, &is_integrated);
hwloc__levelzero_properties_get(dvh[j], osdev, &is_integrated);

hwloc__levelzero_cqprops_get(dvh[j], osdev);

Expand All @@ -742,7 +633,7 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
snprintf(tmp, sizeof(tmp), "%u", k);
hwloc_obj_add_info(subosdevs[k], "LevelZeroSubdeviceID", tmp);

hwloc__levelzero_properties_get(subh[k], subosdevs[k], sysman_maybe_missing, NULL);
hwloc__levelzero_properties_get(subh[k], subosdevs[k], NULL);

hwloc__levelzero_cqprops_get(subh[k], subosdevs[k]);
}
Expand All @@ -755,32 +646,16 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
}

/* get all memory info at once */
hwloc__levelzero_memory_get(dvh[j], osdev, is_integrated, nr_subdevices, subh, subosdevs);
hwloc__levelzero_memory_get(dvh[j], osdev, nr_subdevices, subosdevs);
/* no need to call hwloc__levelzero_memory_get() on subdevices,
* the call on the root device is enough (and identical to a call on subdevices)
*/

/* get all ports info at once */
if (!(hwloc_topology_get_flags(topology) & HWLOC_TOPOLOGY_FLAG_NO_DISTANCES))
hwloc__levelzero_ports_get(dvh[j], osdev, nr_subdevices, subosdevs, &hports);

parent = NULL;
#ifdef HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT
{ /* try getting PCI BDF+speed from core extension */
ze_pci_ext_properties_t ext_pci;
ext_pci.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES;
ext_pci.pNext = NULL;
res = zeDevicePciGetPropertiesExt(dvh[j], &ext_pci);
if (res == ZE_RESULT_SUCCESS) {
parent = hwloc_pci_find_parent_by_busid(topology,
ext_pci.address.domain,
ext_pci.address.bus,
ext_pci.address.device,
ext_pci.address.function);
if (parent && parent->type == HWLOC_OBJ_PCI_DEVICE) {
if (ext_pci.maxSpeed.maxBandwidth > 0)
parent->attr->pcidev.linkspeed = ((float)ext_pci.maxSpeed.maxBandwidth)/1000/1000/1000;
}
}
}
#endif /* HWLOC_HAVE_LEVELZERO_CORE_PCI_EXT */
if (!parent) {
/* try getting PCI BDF+speed from sysman */
zes_pci_properties_t pci;
Expand Down
50 changes: 0 additions & 50 deletions hwloc/topology.c
View file
Open in desktop
Expand Up @@ -54,56 +54,6 @@
#endif


#ifdef HWLOC_HAVE_LEVELZERO
/*
* Define ZES_ENABLE_SYSMAN=1 early so that the LevelZero backend gets Sysman enabled.
*
* Only if the levelzero was enabled in this build so that we don't enable sysman
* for external levelzero users when hwloc doesn't need it. If somebody ever loads
* an external levelzero plugin in a hwloc library built without levelzero (unlikely),
* he may have to manually set ZES_ENABLE_SYSMAN=1.
*
* Use the constructor if supported and/or the Windows DllMain callback.
* Do it in the main hwloc library instead of the levelzero component because
* the latter could be loaded later as a plugin.
*
* L0 seems to be using getenv() to check this variable on Windows
* (at least in the Intel Compute-Runtime of March 2021),
* but setenv() doesn't seem to exist on Windows, hence use putenv() to set the variable.
*
* For the record, Get/SetEnvironmentVariable() is not exactly the same as getenv/putenv():
* - getenv() doesn't see what was set with SetEnvironmentVariable()
* - GetEnvironmentVariable() doesn't see putenv() in cygwin (while it does in MSVC and MinGW).
* Hence, if L0 ever switches from getenv() to GetEnvironmentVariable(),
* it will break in cygwin, we'll have to use both putenv() and SetEnvironmentVariable().
* Hopefully L0 will provide a way to enable Sysman without env vars before it happens.
*/
#if HWLOC_HAVE_ATTRIBUTE_CONSTRUCTOR
static void hwloc_constructor(void) __attribute__((constructor));
static void hwloc_constructor(void)
{
if (!getenv("ZES_ENABLE_SYSMAN"))
#ifdef HWLOC_WIN_SYS
putenv("ZES_ENABLE_SYSMAN=1");
#else
setenv("ZES_ENABLE_SYSMAN", "1", 1);
#endif
}
#endif
#ifdef HWLOC_WIN_SYS
BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved)
{
if (fdwReason == DLL_PROCESS_ATTACH) {
if (!getenv("ZES_ENABLE_SYSMAN"))
/* Windows does not have a setenv, so use putenv. */
putenv((char *) "ZES_ENABLE_SYSMAN=1");
}
return TRUE;
}
#endif
#endif /* HWLOC_HAVE_LEVELZERO */


unsigned hwloc_get_api_version(void)
{
return HWLOC_API_VERSION;
Expand Down
9 changes: 4 additions & 5 deletions tests/hwloc/levelzero.c
View file
Open in desktop
Expand Up @@ -26,19 +26,18 @@ int main(void)
#ifdef HWLOC_HAVE_ZESINIT
res = zesInit(0);
if (res != ZE_RESULT_SUCCESS) {
fprintf(stderr, "Failed to initialize LevelZero Sysman in zesInit(): %d\n", (int)res);
/* continuing, assuming ZES_ENABLE_SYSMAN=1 will be enough */
fprintf(stderr, "Failed to initialize LevelZero in zesInit(): %d\n", (int)res);
return 0;
}
#endif

putenv((char *) "ZES_ENABLE_SYSMAN=1");
/* if !HWLOC_HAVE_ZESINIT, the user must set ZES_ENABLE_SYSMAN=1 in his environment */

res = zeInit(0);
if (res != ZE_RESULT_SUCCESS) {
fprintf(stderr, "Failed to initialize LevelZero in zeInit(): %d\n", (int)res);
return 0;
}

hwloc_topology_init(&topology);
hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
hwloc_topology_load(topology);
Expand Down
1 change: 1 addition & 0 deletions tests/hwloc/ports/include/levelzero/level_zero/ze_api.h
View file
Open in desktop
Expand Up @@ -8,6 +8,7 @@

typedef int ze_result_t;
#define ZE_RESULT_SUCCESS 0
#define ZE_RESULT_ERROR_UNSUPPORTED_FEATURE 0x78000003

#define ZE_MAX_DEVICE_NAME 64

Expand Down