utils/calc+info: rework --best-memattr to allow multiple nodes

--best-memattr was very strict when selecting best nodes.
The node had to be the best for the entire input CPUs.
On a dual-socket machine with HBM in each socket, each HBM
is the best local but not the best remote. Hence we'd report
no best for the entire machine.

Now we return both HBM for the entire machine by default,
but may go back to the previous behavior by adding ",strict"
after the attribute name.

Adding ",default" also allows to return all nodes if no best
was found (if no attribute values are found).

Thanks to Antoine Morvan for the report.

Refs #652

Signed-off-by: Brice Goglin <Brice.Goglin@inria.fr>
(cherry picked from commit 313de56)

(with compile fixes forgotten in intermediate commits in master)

NEWS

@@ -1,5 +1,5 @@
 Copyright © 2009 CNRS
-Copyright © 2009-2023 Inria.  All rights reserved.
+Copyright © 2009-2024 Inria.  All rights reserved.
 Copyright © 2009-2013 Université Bordeaux
 Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
 Copyright © 2020 Hewlett Packard Enterprise.  All rights reserved.
@@ -21,6 +21,9 @@ Version 2.11.0
 --------------
 * API
   + Add hwloc_obj_set_subtype().
+* Tools/
+  + Option --best-memattr may now return multiple nodes. Additional
+    configuration flags may be given to tweak its behavior.
 
 
 Version 2.10.0

utils/hwloc/hwloc-calc.1in

@@ -1,5 +1,5 @@
 .\" -*- nroff -*-
-.\" Copyright © 2010-2023 Inria.  All rights reserved.
+.\" Copyright © 2010-2024 Inria.  All rights reserved.
 .\" Copyright © 2009-2020 Cisco Systems, Inc.  All rights reserved.
 .\" See COPYING in top-level directory.
 .TH HWLOC-CALC "1" "%HWLOC_DATE%" "%PACKAGE_VERSION%" "%PACKAGE_NAME%"
@@ -248,7 +248,7 @@ This option enables \fB\-\-local\-memory\fR.
 .TP
 \fB\-\-best\-memattr\fR <name>
 Enable the listing of local memory nodes with \fB\-\-local\-memory\fR,
-but only display the local node that has the best value for the memory
+but only display the local nodes that have the best value for the memory
 attribute given by \fI<name>\fR (or as an index).
 
 If the memory attribute values depend on the initiator, the hwloc-calc
@@ -262,6 +262,15 @@ All existing attributes in the current topology may be listed with
 
 If combined with \fB\-\-object\-output\fR, the object index is prefixed
 with its type (e.g. \fINUMANode:0\fR instead of \fI0\fR).
+
+\fI<name>\fR may be suffixed with flags to tune the selection of best nodes,
+for instance as \fBbandwidth,strict,default\fR.
+\fBdefault\fR means that all local nodes are reported if no best could be found.
+\fBstrict\fR means that nodes are selected only if their performance is the best
+for all the input CPUs. On a dual-socket machine with HBM in each socket,
+both HBMs are the best for their local socket, but not for the remote socket.
+Hence both HBM are also considered best for the entire machine by default,
+but none if \fBstrict\fR.
 .TP
 \fB\-\-sep <sep>\fR
 Change the field separator in the output.
@@ -393,7 +402,7 @@ whose locality is exactly equal to a Package:
     $ hwloc-calc --local-memory-flags 0 --physical-output pack:1
     4,7
 
-To display the best-capacity NUMA node, by physical indexes,
+To display the best-capacity NUMA node(s), by physical indexes,
 whose locality is exactly equal to a Package:
 
     $ hwloc-calc --local-memory-flags 0 --best-memattr capacity --physical-output pack:1

utils/hwloc/hwloc-calc.c

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009 CNRS
- * Copyright © 2009-2022 Inria.  All rights reserved.
+ * Copyright © 2009-2024 Inria.  All rights reserved.
  * Copyright © 2009-2011 Université Bordeaux
  * Copyright © 2009-2010 Cisco Systems, Inc.  All rights reserved.
  * Copyright © 2023 Université de Reims Champagne-Ardenne.  All rights reserved.
@@ -79,6 +79,7 @@ static struct hwloc_calc_level *hierlevels;
 static int local_numanodes = 0;
 static unsigned long local_numanode_flags = HWLOC_LOCAL_NUMANODE_FLAG_SMALLER_LOCALITY | HWLOC_LOCAL_NUMANODE_FLAG_LARGER_LOCALITY;
 static hwloc_memattr_id_t best_memattr_id = (hwloc_memattr_id_t) -1;
+static unsigned long best_node_flags = 0;
 static int showobjs = 0;
 static int no_smt = -1;
 static int singlify = 0;
@@ -232,35 +233,33 @@ hwloc_calc_output(hwloc_topology_t topology, const char *sep, hwloc_bitmap_t set
   } else if (local_numanodes) {
     unsigned nrnodes;
     hwloc_obj_t *nodes;
+    hwloc_nodeset_t nodeset = hwloc_bitmap_alloc_full(); /* show all nodes by default */
     nrnodes = hwloc_bitmap_weight(hwloc_topology_get_topology_nodeset(topology));
     nodes = malloc(nrnodes * sizeof(*nodes));
-    if (nodes) {
+    if (nodeset && nodes) {
       int err;
       struct hwloc_location loc;
       loc.type = HWLOC_LOCATION_TYPE_CPUSET;
       loc.location.cpuset = set;
       err = hwloc_get_local_numanode_objs(topology, &loc, &nrnodes, nodes, local_numanode_flags);
       if (!err) {
-        unsigned i;
+        unsigned i, first = 1;
         if (best_memattr_id != (hwloc_memattr_id_t) -1) {
-          int best = hwloc_utils_get_best_node_in_array_by_memattr(topology, best_memattr_id, nrnodes, nodes, &loc);
-          if (best == -1) {
-            /* no perf info found, report nothing */
-            nrnodes = 0;
-          } else {
-            /* only report the best nodes */
-            nodes[0] = nodes[best];
-            nrnodes = 1;
-          }
+          err = hwloc_utils_get_best_node_in_array_by_memattr(topology, best_memattr_id, nrnodes, nodes, &loc, best_node_flags, nodeset);
+          /* on error, nodeset is zeroed, and we report nothing below (except if default flag is set) */
         }
         if (!sep)
           sep = ",";
         for(i=0; i<nrnodes; i++) {
           char type[64];
           unsigned idx;
+          if (!hwloc_bitmap_isset(nodeset, nodes[i]->os_index))
+            continue;
           hwloc_obj_type_snprintf(type, sizeof(type), nodes[i], 1);
           idx = logicalo ? nodes[i]->logical_index : nodes[i]->os_index;
-          if (i>0)
+          if (first)
+            first = 0;
+          else
             printf("%s", sep);
           if (objecto) {
             char types[64];
@@ -270,8 +269,9 @@ hwloc_calc_output(hwloc_topology_t topology, const char *sep, hwloc_bitmap_t set
           printf("%u", idx);
         }
       }
-      free(nodes);
     }
+    free(nodes);
+    hwloc_bitmap_free(nodeset);
     printf("\n");
 
   } else {
@@ -695,6 +695,17 @@ int main(int argc, char *argv[])
   }
 
   if (best_memattr_str) {
+    char *tmp;
+    tmp = strstr(best_memattr_str, ",default");
+    if (tmp) {
+      memmove(tmp, tmp+8, strlen(tmp+8)+1);
+      best_node_flags |= HWLOC_UTILS_BEST_NODE_FLAG_DEFAULT;
+    }
+    tmp = strstr(best_memattr_str, ",strict");
+    if (tmp) {
+      memmove(tmp, tmp+7, strlen(tmp+7)+1);
+      best_node_flags |= HWLOC_UTILS_BEST_NODE_FLAG_STRICT;
+    }
     best_memattr_id = hwloc_utils_parse_memattr_name(topology, best_memattr_str);
     if (best_memattr_id == (hwloc_memattr_id_t) -1) {
       fprintf(stderr, "unrecognized memattr %s\n", best_memattr_str);

utils/hwloc/hwloc-info.1in

@@ -136,10 +136,19 @@ This option enables \fB\-\-local\-memory\fR.
 .TP
 \fB\-\-best\-memattr\fR <name>
 Enable the listing of local memory nodes with \fB\-\-local\-memory\fR,
-but only display the local node that has the best value for the memory
+but only display the local nodes that have the best value for the memory
 attribute given by \fI<name>\fR (or as an index).
 If the memory attribute values depend on the initiator, the object given
 to hwloc-info is used as the initiator.
+
+\fI<name>\fR may be suffixed with flags to tune the selection of best nodes,
+for instance as \fBbandwidth,strict,default\fR.
+\fBdefault\fR means that all local nodes are reported if no best could be found.
+\fBstrict\fR means that nodes are selected only if their performance is the best
+for all the input CPUs. On a dual-socket machine with HBM in each socket,
+both HBMs are the best for their local socket, but not for the remote socket.
+Hence both HBM are also considered best for the entire machine by default,
+but none if \fBstrict\fR.
 .TP
 \fB\-\-first\fR
 For each input object, only report the first matching output object
@@ -310,7 +319,7 @@ To list the NUMA nodes that are local a PU:
      type = NUMANode
     ...
 
-To show the best-bandwidth node among NUMA nodes local to a PU:
+To show the best-bandwidth node(s) among NUMA nodes local to a PU:
 
     $ hwloc-info --local-memory --best-memattr bandwidth pu:25
     NUMANode L#7 = local memory #1 of PU L#25

utils/hwloc/hwloc-info.c

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009 CNRS
- * Copyright © 2009-2023 Inria.  All rights reserved.
+ * Copyright © 2009-2024 Inria.  All rights reserved.
  * Copyright © 2009-2012 Université Bordeaux
  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright © 2023 Université de Reims Champagne-Ardenne.  All rights reserved.
@@ -83,6 +83,7 @@ static int show_first_only = 0;
 static int show_local_memory = 0;
 static int show_local_memory_flags = HWLOC_LOCAL_NUMANODE_FLAG_SMALLER_LOCALITY | HWLOC_LOCAL_NUMANODE_FLAG_LARGER_LOCALITY;
 static hwloc_memattr_id_t best_memattr_id = (hwloc_memattr_id_t) -1;
+static unsigned long best_node_flags = 0;
 static unsigned current_obj;
 
 void usage(const char *name, FILE *where)
@@ -582,9 +583,10 @@ hwloc_calc_process_location_info_cb(struct hwloc_calc_location_context_s *lconte
   } else if (show_local_memory) {
     unsigned nrnodes;
     hwloc_obj_t *nodes;
+    hwloc_nodeset_t nodeset = hwloc_bitmap_alloc_full(); /* show all nodes by default */
     nrnodes = hwloc_bitmap_weight(hwloc_topology_get_topology_nodeset(topology));
     nodes = malloc(nrnodes * sizeof(*nodes));
-    if (nodes) {
+    if (nodeset && nodes) {
       struct hwloc_location loc;
       int err;
       loc.type = HWLOC_LOCATION_TYPE_OBJECT;
@@ -593,28 +595,21 @@ hwloc_calc_process_location_info_cb(struct hwloc_calc_location_context_s *lconte
       if (!err) {
         unsigned i;
         if (best_memattr_id != (hwloc_memattr_id_t) -1) {
-          /* only keep the best one for that memattr */
-          int best;
+          /* only keep the best ones for that memattr */
 
           /* won't work if obj is CPU-less: perf from I/O is likely different from perf from CPU objects */
           loc.type = HWLOC_LOCATION_TYPE_CPUSET;
           loc.location.cpuset = obj->cpuset;
-          best = hwloc_utils_get_best_node_in_array_by_memattr(topology, best_memattr_id,
-                                                               nrnodes, nodes, &loc);
-          if (best == -1) {
-            /* no perf info found, report nothing */
+          err = hwloc_utils_get_best_node_in_array_by_memattr(topology, best_memattr_id,
+                                                               nrnodes, nodes, &loc, best_node_flags, nodeset);
+          if (err < -1) {
             if (verbose > 0)
               fprintf(stderr, "Failed to find a best local node for memory attribute.\n");
-            nrnodes = 0;
-          } else {
-            /* only report the best node, but keep the index intact */
-            for(i=0; i<nrnodes; i++)
-              if (i != (unsigned) best)
-                nodes[i] = NULL;
+            /* on error, nodeset is zeroed, and we report nothing below (except if default flag is set) */
           }
         }
         for(i=0; i<nrnodes; i++) {
-          if (!nodes[i])
+          if (!hwloc_bitmap_isset(nodeset, nodes[i]->os_index))
             continue;
           if (show_index_prefix)
 	    snprintf(prefix, sizeof(prefix), "%u.%u: ", current_obj, i);
@@ -626,6 +621,7 @@ hwloc_calc_process_location_info_cb(struct hwloc_calc_location_context_s *lconte
     } else {
       fprintf(stderr, "Failed to allocate array of local NUMA nodes\n");
     }
+    hwloc_bitmap_free(nodeset);
     free(nodes);
   } else {
     hwloc_info_show_single_obj(topology, obj, objs, prefix, verbose);
@@ -957,8 +953,19 @@ main (int argc, char *argv[])
   }
 
   if (best_memattr_str) {
+    char *tmp;
     if (!show_local_memory)
       fprintf(stderr, "--best-memattr is ignored without --local-memory.\n");
+    tmp = strstr(best_memattr_str, ",default");
+    if (tmp) {
+      memmove(tmp, tmp+8, strlen(tmp+8)+1);
+      best_node_flags |= HWLOC_UTILS_BEST_NODE_FLAG_DEFAULT;
+    }
+    tmp = strstr(best_memattr_str, ",strict");
+    if (tmp) {
+      memmove(tmp, tmp+7, strlen(tmp+7)+1);
+      best_node_flags |= HWLOC_UTILS_BEST_NODE_FLAG_STRICT;
+    }
     best_memattr_id = hwloc_utils_parse_memattr_name(topology, best_memattr_str);
     if (best_memattr_id == (hwloc_memattr_id_t) -1) {
       fprintf(stderr, "unrecognized memattr %s\n", best_memattr_str);