Update to use MQ 9.3.0.0 and mq-golang v5.3.0

CHANGELOG.md

@@ -1,14 +1,24 @@
 # Changelog
 Newest updates are at the top of this file.
 
+### Jun 23 2022 (v5.3.0)
+* Update to MQ 9.3.0
+* Update to use v5.3.0 of mq-golang repository
+* Update other vendored dependencies
+* Add filters.hideSvrConnJobname config option (#114)
+* Move objects.showInactiveChannels to new filters section (generate error to indicate move)
+- Move objects.queueSubscriptionSelection to new filters section
+* Enable use of durable subscriptions to reduce impact on MAXHANDS configuration
+  * Add script to explicitly remove durable subs
+
 ### Mar 10 2022 (no new version)
 * Add CP4I helm charts to deploy MQ metric samples application in OCP environment.
 
 ### Mar 02 2022 (no new version)
 * Add dspmqrtj application and demo
 
 ### Feb 28 2022 (no new version)
-* Update to require Go 1.17 
+* Update to require Go 1.17 because of influxdb prereq changes
 
 ### Feb 25 2022 (v5.2.5)
 * Update to MQ 9.2.5

Dockerfile.build

@@ -59,7 +59,7 @@ RUN mkdir -p $GOPATH/src $GOPATH/bin $GOPATH/pkg \
 # Location of the downloadable MQ client package \
 ENV RDURL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist" \
     RDTAR="IBM-MQC-Redist-LinuxX64.tar.gz" \
-    VRMF=9.2.5.0
+    VRMF=9.3.0.0
 
 # Install the MQ client from the Redistributable package. This also contains the
 # header files we need to compile against. Setup the subset of the package

Dockerfile.run

@@ -31,7 +31,7 @@ RUN apt-get update \
 # Location of the downloadable MQ client package \
 ENV RDURL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist" \
     RDTAR="IBM-MQC-Redist-LinuxX64.tar.gz" \
-    VRMF=9.2.5.0
+    VRMF=9.3.0.0
 
 # Install the MQ client from the Redistributable package. This also contains the
 # header files we need to compile against. Setup the subset of the package

README.md

@@ -10,7 +10,7 @@ dashboard and historic reporting.
 ## The dspmqrtj program
 The repository also includes a program which traces the route a message can
 take through the MQ network. It is similar to the `dspmqrte` program that is
-part of the MQ product, but writes the output in JSON format. See 
+part of the MQ product, but writes the output in JSON format. See
 the `dspmqrtj` subdirectory for more information.
 
 ## Health Warning
@@ -30,7 +30,8 @@ file if you wish to reload all of the dependencies by running `go mod vendor`.
 
 You will require the following programs:
 
-* Go compiler. This should be at least version 17.
+* Go compiler. Building the InfluxDB collector requires Go 17 as the minimum compiler level. If you don't want 
+to build that particular collector then older levels can be used.
 
 To build the programs on Linux and MacOS, you may set an environment variable to permit some compile/link flags.
 This is due to security controls in the compiler.
@@ -87,9 +88,10 @@ builds all the collectors and corresponding YAML configuration files into %GOPAT
 ## Queue manager configuration
 When metrics are being collected from the publish/subscribe interface (all platforms except z/OS),
 there are some considerations:
-* MAXHANDS on queue manager: Each subscription uses an object handle. If many queues are being monitored
+* MAXHANDS on queue manager: The default configuration of these collectors uses non-durable subscriptions to get
+information about queue metrics. Each subscription uses an object handle. If many queues are being monitored
 the default MAXHANDS may need to be increased. A warning is printed if the monitor thinks this attribute
-appears too low.
+appears too low. See below for an alternative option.
 * MAXDEPTH on model queues: The model queue used as the basis for publication and reply queues in the
 monitor must have a MAXDEPTH suitable for the expected amount of data. For published metrics, this is
 estimated based on holding one minute's amount of publications; the number of monitored channels is also
@@ -99,6 +101,24 @@ monitor.
 SYSTEM.ADMIN.TOPIC) determines what happens if the subscriber's queue is full. You might prefer to set this to
 NO to avoid filling the system DLQ if the collection program does not read the publications frequently enough.
 
+
+### Using durable subscriptions
+An alternative collection mechanism uses durable subscriptions for the queue metric data. This may avoid needing to increase
+the MAXHANDS attribute on a queue manager. (Queue manager-level metrics are still collected using non-durable subscriptions.)
+
+To set it up, you must provide suitable configuration options. In the
+YAML configuration, these are the attributes (command line or environment variable equivalents exist):
+- `replyQueue` must refer to a local queue (not a model queue)
+- `replyQueue2` must also be set, referring to a different local queue
+- `durableSubPrefix` is a string that is unique across any collectors that might be connected to this queue manager
+
+If you use durable subscriptions, then the named reply queues may continue to receive publications even when the
+collector is not running, so that may induce queue-full reports in the error log or events. The subscriptions can
+be manually removed using the "DELETE SUB()" MQSC command for all subscriptions where the subscription ids begin with the
+`durableSubPrefix` value. The `scripts/cleanDur.sh` program can be used for this deletion. You should also clean
+the subscriptions when the configuration of which data to collect has changed, particularly the `queueSubscriptionSelector`
+option.
+
 ## Monitor configuration
 The monitors always collect all of the available queue manager-wide metrics.
 They can also be configured to collect statistics for specific sets of queues where
@@ -229,7 +249,8 @@ All of the exporters support
 the same configuration options for how to connect to MQ and which objects are monitored. There is
 then an exporter-specific section for additional configuration such as how to contact the back-end
 database.
-The common options are shown in a template in this directory; the exporter-specific options are in individual files in each directory. Combine the two pieces into a single file to get a complete deployable configuration.
+The common options are shown in a template in this directory; the exporter-specific options are in individual 
+files in each directory. Combine the two pieces into a single file to get a complete deployable configuration.
 
 Unlike the command line flags, lists are provided in a more natural format instead of comma-separated
 values in a single string. If an option is provided on both the command line and in the file, it is the file

cmd/mq_aws/config.go

@@ -41,6 +41,7 @@ type mqExporterConfigYaml struct {
 	Global     cf.ConfigYGlobal
 	Connection cf.ConfigYConnection
 	Objects    cf.ConfigYObjects
+	Filters    cf.ConfigYFilters
 	Cloudwatch ConfigYCloudwatch `yaml:"cloudwatch"`
 }
 
@@ -69,7 +70,7 @@ func initConfig() error {
 		if config.cf.ConfigFile != "" {
 			err = cf.ReadConfigFile(config.cf.ConfigFile, &cfy)
 			if err == nil {
-				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects)
+				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects, cfy.Filters)
 				config.ci.Region = cf.CopyParmIfNotSetStr("cloudwatch", "awsregion", cfy.Cloudwatch.Region)
 				config.ci.Namespace = cf.CopyParmIfNotSetStr("cloudwatch", "namespace", cfy.Cloudwatch.Namespace)
 

cmd/mq_coll/config.go

@@ -40,6 +40,7 @@ type mqExporterConfigYaml struct {
 	Global     cf.ConfigYGlobal
 	Connection cf.ConfigYConnection
 	Objects    cf.ConfigYObjects
+	Filters    cf.ConfigYFilters
 	Collectd   ConfigYColl `yaml:"collectd"`
 }
 
@@ -62,7 +63,7 @@ func initConfig() error {
 		if config.cf.ConfigFile != "" {
 			err = cf.ReadConfigFile(config.cf.ConfigFile, &cfy)
 			if err == nil {
-				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects)
+				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects, cfy.Filters)
 				config.interval = cf.CopyParmIfNotSetStr("collectd", "interval", cfy.Collectd.Interval)
 				config.hostname = cf.CopyParmIfNotSetStr("collectd", "hostname", cfy.Collectd.Hostname)
 			}

cmd/mq_influx/config.go

@@ -50,6 +50,7 @@ type mqExporterConfigYaml struct {
 	Global     cf.ConfigYGlobal
 	Connection cf.ConfigYConnection
 	Objects    cf.ConfigYObjects
+	Filters    cf.ConfigYFilters
 	Influx     ConfigYInflux `yaml:"influx"`
 }
 
@@ -80,7 +81,7 @@ func initConfig() error {
 		if config.cf.ConfigFile != "" {
 			err = cf.ReadConfigFile(config.cf.ConfigFile, &cfy)
 			if err == nil {
-				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects)
+				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects, cfy.Filters)
 				config.ci.BucketName = cf.CopyParmIfNotSetStr("influx", "bucketName", cfy.Influx.BucketName)
 				//config.ci.DatabaseName = cf.CopyParmIfNotSetStr("influx", "databaseName", cfy.Influx.DatabaseName)
 				config.ci.DatabaseAddress = cf.CopyParmIfNotSetStr("influx", "databaseAddress", cfy.Influx.DatabaseAddress)

cmd/mq_json/config.go

@@ -37,6 +37,7 @@ type mqExporterConfigYaml struct {
 	Global     cf.ConfigYGlobal
 	Connection cf.ConfigYConnection
 	Objects    cf.ConfigYObjects
+	Filters    cf.ConfigYFilters
 	JSON       ConfigYJson `yaml:"json"`
 }
 
@@ -60,7 +61,7 @@ func initConfig() error {
 		if config.cf.ConfigFile != "" {
 			err = cf.ReadConfigFile(config.cf.ConfigFile, &cfy)
 			if err == nil {
-				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects)
+				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects, cfy.Filters)
 				config.interval = cf.CopyParmIfNotSetStr("json", "interval", cfy.JSON.Interval)
 				config.oneline = cf.CopyParmIfNotSetBool("json", "oneline", cfy.JSON.OneLine)
 			}

cmd/mq_opentsdb/config.go

@@ -43,6 +43,7 @@ type mqExporterConfigYaml struct {
 	Global     cf.ConfigYGlobal
 	Connection cf.ConfigYConnection
 	Objects    cf.ConfigYObjects
+	Filters    cf.ConfigYFilters
 	OpenTSDB   ConfigYOpenTSDB `yaml:"opentsdb"`
 }
 
@@ -69,7 +70,7 @@ func initConfig() error {
 		if config.cf.ConfigFile != "" {
 			err = cf.ReadConfigFile(config.cf.ConfigFile, &cfy)
 			if err == nil {
-				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects)
+				cf.CopyYamlConfig(&config.cf, cfy.Global, cfy.Connection, cfy.Objects, cfy.Filters)
 				config.ci.DatabaseAddress = cf.CopyParmIfNotSetStr("opentsdb", "databaseAddress", cfy.OpenTSDB.DatabaseAddress)
 				config.ci.Interval = cf.CopyParmIfNotSetStr("opentsdb", "interval", cfy.OpenTSDB.Interval)
 				config.ci.MaxErrors = cf.CopyParmIfNotSetInt("opentsdb", "maxErrors", cfy.OpenTSDB.MaxErrors)

cmd/mq_opentsdb/exporter.go

@@ -224,7 +224,7 @@ func Collect() error {
 							series = "queue"
 							usage := ""
 							if usageAttr, ok := mqmetric.GetObjectStatus("", mqmetric.OT_Q).Attributes[mqmetric.ATTR_Q_USAGE].Values[key]; ok {
-								if usageAttr.ValueInt64 == 1 {
+								if usageAttr.ValueInt64 == int64(ibmmq.MQUS_TRANSMISSION) {
 									usage = "XMITQ"
 								} else {
 									usage = "NORMAL"
@@ -302,7 +302,7 @@ func Collect() error {
 					tags["platform"] = platformString
 					usage := ""
 					if usageAttr, ok := mqmetric.GetObjectStatus("", mqmetric.OT_Q).Attributes[mqmetric.ATTR_Q_USAGE].Values[key]; ok {
-						if usageAttr.ValueInt64 == 1 {
+						if usageAttr.ValueInt64 == int64(ibmmq.MQUS_TRANSMISSION) {
 							usage = "XMITQ"
 						} else {
 							usage = "NORMAL"