diff --git a/CHANGES.md b/CHANGELOG.md similarity index 61% rename from CHANGES.md rename to CHANGELOG.md index 8dab244..1958036 100644 --- a/CHANGES.md +++ b/CHANGELOG.md @@ -1,87 +1,95 @@ +# Changelog -08 Jul 2016 -* Initial release +## May 2018 -18 Jul 2016 -* Changed structures so that most applications will not need to use cgo to imbed the MQ C headers - * Go programs will now use int32 where C programs use MQLONG - * Use of message handles, distribution lists require cgo for now -* Package ibmmq now includes the numeric #defines as a Go file, cmqc.go, for easier use -* Removed "src/" prefix from tree in github repo -* Removed need for buffer length parm on Put/Put1 -* Updated comments -* Added MQINQ -* Added MQItoString function for some maps of values to constant names +* Corrected package imports +* Formatted go code with `go fmt` +* Rearranged this file +* Removed logging from golang package `mqmetric` +* Moved samples to a separate repository -25 Jul 2016 -* Added functions to handle basic PCF creation and parsing -* Added a monitor command for exporting MQ V9 queue manager data to Prometheus. See -the [README](cmd/mq_prometheus/README.md) for more details - -04 Aug 2016 -* Added a monitor command for exporting MQ data to InfluxDB. See the [README] -(cmd/mq_influx/README.md) for more details -* Restructured the monitoring code to put common material in the mqmetric -package, called from the Influx and Prometheus monitors. - -12 Aug 2016 -* Added a OpenTSDB monitor. See the [README](cmd/mq_opentsdb/README.md) for -more details. -* Added a Collectd monitor. See the [README](cmd/mq_coll/README.md) for -more details. -* Added MQI MQCNO/MQCSP structures to support client connections and password authentication -with MQCONNX. -* Allow client-mode connections from the monitor programs -* Added Grafana dashboards for the different monitors to show how to query them -* Changed database password mechanism so that "exec" maintains the PID for MQ services +## March 2018 - v1.0.0 -23 Aug 2016 -* Added a collector for Amazon AWS CloudWatch monitoring. See the [README](cmd/mq_aws/README.md) -for more details. +* Added V9.0.5 constant definitions +* Changed #cgo directives for Windows now the compiler supports standard path names +* Added mechanism to set MQ userid and password for Prometheus monitor +* Released v1.0.0 of this repository for use with golang dependency management tools -17 Oct 2016 -* Added some Windows support. An example batch file is included in the mq_influx directory; -changes would be needed to the MQSC script to call it. The other monitor programs can be -supported with similar modifications. -* Added a "getting started" section to this README. +## October 2017 -07 Nov 2016 -* Added a collector that prints metrics in a simple JSON format. -See the [README](cmd/mq_json/README.md) for more details. -* Fixed bug where freespace metrics were showing as non-integer bytes, not percentages +* Added V9.0.4 constant definitions - now generated from original MQ source code +* Added MQSC script to show how to redefine event queues for pub/sub +* Prometheus collector has new parameter to override the first component of the metric name +* Prometheus collector can now process channel-level statistics + +## 18 May 2017 + +* Added the V9.0.3 constant definitions. +* Reinstated 64-bit structure "length" fields in cmqc.go after fixing a bug in the base product C source code generator. + +## 25 Mar 2017 + +* Added the metaPrefix option to the Prometheus monitor. This allows selection of non-default resources such as the MQ Bridge for Salesforce included in MQ 9.0.2. + +## 15 Feb 2017 + +* API BREAKING CHANGE: The MQI verbs have been changed to return a single error indicator instead of two separate values. See mqitest.go for examples of how MQRC/MQCC codes can now be tested and extracted. This change makes the MQI implementation a bit more natural for Go environments. + +## 10 Jan 2017 + +* Added support for the MQCD and MQSCO structures to allow programmable client connectivity, without requiring a CCDT. See the clientconn sample program for an example of using the MQCD. +* Moved sample programs into subdirectory + +## 14 Dec 2016 -14 Dec 2016 * Minor updates to this README for formatting * Removed xxx_CURRENT_LENGTH definitions from cmqc -10 Jan 2017 -* Added support for the MQCD and MQSCO structures to allow programmable client -connectivity, without requiring a CCDT. See the clientconn sample program -for an example of using the MQCD. -* Moved sample programs into subdirectory +## 07 Nov 2016 -15 Feb 2017 -* API BREAKING CHANGE: The MQI verbs have been changed to return a single -error indicator instead of two separate values. See mqitest.go for -examples of how MQRC/MQCC codes can now be tested and extracted. This change -makes the MQI implementation a bit more natural for Go environments. +* Added a collector that prints metrics in a simple JSON format. See the [README](cmd/mq_json/README.md) for more details. +* Fixed bug where freespace metrics were showing as non-integer bytes, not percentages -25 Mar 2017 -* Added the metaPrefix option to the Prometheus monitor. This allows selection of non-default resources such as the MQ Bridge for Salesforce included in MQ 9.0.2. +## 17 Oct 2016 -18 May 2017 -* Added the V9.0.3 constant definitions. -* Reinstated 64-bit structure "length" fields in -cmqc.go after fixing a bug in the base product C source code generator. +* Added some Windows support. An example batch file is included in the mq_influx directory; changes would be needed to the MQSC script to call it. The other monitor programs can be supported with similar modifications. +* Added a "getting started" section to this README. -October 2017 -* Added V9.0.4 constant definitions - now generated from original MQ source code -* Added MQSC script to show how to redefine event queues for pub/sub -* Prometheus collector has new parameter to override the first component of the metric name -* Prometheus collector can now process channel-level statistics +## 23 Aug 2016 -March 2018 -* Added V9.0.5 constant definitions -* Changed #cgo directives for Windows now the compiler supports standard path names -* Added mechanism to set MQ userid and password for Prometheus monitor +* Added a collector for Amazon AWS CloudWatch monitoring. See the [README](cmd/mq_aws/README.md) for more details. + +## 12 Aug 2016 + +* Added a OpenTSDB monitor. See the [README](cmd/mq_opentsdb/README.md) for more details. +* Added a Collectd monitor. See the [README](cmd/mq_coll/README.md) for more details. +* Added MQI MQCNO/MQCSP structures to support client connections and password authentication with MQCONNX. +* Allow client-mode connections from the monitor programs +* Added Grafana dashboards for the different monitors to show how to query them +* Changed database password mechanism so that "exec" maintains the PID for MQ services + +## 04 Aug 2016 + +* Added a monitor command for exporting MQ data to InfluxDB. See the [README](cmd/mq_influx/README.md) for more details +* Restructured the monitoring code to put common material in the mqmetric package, called from the Influx and Prometheus monitors. + +## 25 Jul 2016 + +* Added functions to handle basic PCF creation and parsing +* Added a monitor command for exporting MQ V9 queue manager data to Prometheus. See the [README](cmd/mq_prometheus/README.md) for more details + +## 18 Jul 2016 + +* Changed structures so that most applications will not need to use cgo to imbed the MQ C headers + * Go programs will now use int32 where C programs use MQLONG + * Use of message handles, distribution lists require cgo for now +* Package ibmmq now includes the numeric #defines as a Go file, cmqc.go, for easier use +* Removed "src/" prefix from tree in github repo +* Removed need for buffer length parm on Put/Put1 +* Updated comments +* Added MQINQ +* Added MQItoString function for some maps of values to constant names + +## 08 Jul 2016 +* Initial release \ No newline at end of file diff --git a/README.md b/README.md index 5483afe..e43aa40 100755 --- a/README.md +++ b/README.md @@ -1,126 +1,115 @@ # mq-golang + This repository demonstrates how you can call IBM MQ from applications written in the Go language. > **NOTICE**: Please ensure that you use a dependency management tool such as [dep](https://github.com/golang/dep) or [Glide](http://glide.sh/), and add a specific version dependency. The current content has been marked as version 1.0.0, and a new version with breaking changes will be released soon. By using a dependency manager, you can continue to use the old version if you want to. -The repository also includes programs to export MQ statistics to some monitoring -packages including Prometheus, InfluxDB and OpenTSDB. +This repository previously contained sample programs that exported MQ statistics to some monitoring packages. These have now been moved to a new [GitHub repository called mq-metric-samples](https://github.com/ibm-messaging/mq-metric-samples). -A minimum level of MQ V9 is required to build this package. -The monitoring data published by the queue manager is not available before -that version; the interface also assumes availability of -MQI structures from that level of MQ. +A minimum level of MQ V9 is required to build these packages. +The monitoring data published by the queue manager is not available before that version; the interface also assumes availability of MQI structures from that level of MQ. ## Health Warning -This package is provided as-is with no guarantees of support or updates. There are also no guarantees of compatibility -with any future versions of the package; the API is subject to change based on any feedback. +This package is provided as-is with no guarantees of support or updates. There are also no guarantees of compatibility with any future versions of the package; the API is subject to change based on any feedback. ## MQI Description The ibmmq directory contains a Go package, exposing an MQI-like interface. -The intention is to give an API that is more natural for Go programmers than the common -procedural MQI. For example, fixed length string arrays from the C API such as MQCHAR48 are -represented by the native Go string type. Conversion between these types is handled within the ibmmq -package itself, removing the need for Go programmers to know about it. +The intention is to give an API that is more natural for Go programmers than the common procedural MQI. For example, fixed length string arrays from the C API such as MQCHAR48 are represented by the native Go string type. Conversion between these types is handled within the ibmmq package itself, removing the need for Go programmers to know about it. -A short program in the mqitest directory gives an example of using this interface, to put and get messages -and to subscribe to a topic. +A short program in the mqitest directory gives an example of using this interface, to put and get messages and to subscribe to a topic. -Feedback on the utility of this package, thoughts about whether it should be changed or extended are -welcomed. +Feedback on the utility of this package, thoughts about whether it should be changed or extended are welcomed. ## Using the package -To use code in this repository, you will need to be able to build Go applications, and -have a copy of MQ installed to build against. It uses cgo to access the MQI C structures and definitions. It assumes that MQ has been -installed in the default location on a Linux platform (/opt/mqm) but you can easily change the -cgo directives in the source files if necessary. +To use code in this repository, you will need to be able to build Go applications, and have a copy of MQ installed to build against. It uses cgo to access the MQI C structures and definitions. It assumes that MQ has been installed in the default location on a Linux platform (`/opt/mqm`) but you can easily change the cgo directives in the source files if necessary. -Some Windows capability is also included. This has been tested with Go 1.10 -compiler, which now permits standard Windows paths (eg including spaces) -so the CGO directives can point at the normal MQ install path. +Some Windows capability is also included. This has been tested with Go 1.10 compiler, which now permits standard Windows paths (eg including spaces) so the CGO directives can point at the normal MQ install path. ## Getting started -If you are unfamiliar with Go, the following steps can help create a -working environment with source code in a suitable tree. Initial setup -tends to be platform-specific, but subsequent steps are independent of the -platform. +If you are unfamiliar with Go, the following steps can help create a working environment with source code in a suitable tree. Initial setup tends to be platform-specific, but subsequent steps are independent of the platform. ### Linux -* Install the Go runtime and compiler. On Linux, the packaging may vary -but a typical directory for the code is /usr/lib/golang. -* Create a working directory. For example, mkdir $HOME/gowork +* Install the Go runtime and compiler. On Linux, the packaging may vary but a typical directory for the code is `/usr/lib/golang`. + +* Create a working directory. For example, ```mkdir $HOME/gowork``` + * Set environment variables. Based on the previous lines, - export GOROOT=/usr/lib/golang + ```export GOROOT=/usr/lib/golang``` + + ```export GOPATH=$HOME/gowork``` - export GOPATH=$HOME/gowork +* If using a version of Go from after 2017, you must set environment variables to permit some compile/link flags. This is due to a security fix in the compiler. -* If using a version of Go from after 2017, you must set environment variables -to permit some compile/link flags. This is due to a security fix in the compiler. - export CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" + ```export CGO_LDFLAGS_ALLOW="-Wl,-rpath.*"``` * Install the git client ### Windows -* Install the Go runtime and compiler. On Windows, the -common directory is c:\Go -* Ensure you have a gcc-based compiler, for example from the Cygwin -distribution. I use the mingw variation, to ensure compiled code can -be used on systems without Cygwin installed -* Create a working directory. For example, mkdir c:\Gowork +* Install the Go runtime and compiler. On Windows, the common directory is `c:\Go` +* Ensure you have a gcc-based compiler, for example from the Cygwin distribution. I use the mingw variation, to ensure compiled code can be used on systems without Cygwin installed +* Create a working directory. For example, ```mkdir c:\Gowork``` * Set environment variables. Based on the previous lines, - set GOROOT=c:\Go + ```set GOROOT=c:\Go``` - set GOPATH=c:\Gowork + ```set GOPATH=c:\Gowork``` - set CC=x86_64-w64-mingw32-gcc.exe + ```set CC=x86_64-w64-mingw32-gcc.exe``` -* The CGO_LDFLAGS_ALLOW variable is not needed on Windows +* The `CGO_LDFLAGS_ALLOW` variable is not needed on Windows * Install the git client -* Make sure the MQ include files and libraries are in a path that does -not include spaces or other special characters, as discussed above. +* Make sure the MQ include files and libraries are in a path that does not include spaces or other special characters, as discussed above. ### Common -* Make sure your PATH includes routes to the Go compiler ($GOROOT/bin), -the Git client, and the C compiler. -* Change directory to the workspace you created earlier. (cd $GOPATH) -* Use git to get a copy of the MQ components into a new directory in the -workspace. Use "src" as the destination, to get the directory created -automatically; this path will then be searched by the Go compiler. +* Make sure your PATH includes routes to the Go compiler (`$GOROOT/bin`), the Git client, and the C compiler. +* Change directory to the workspace you created earlier. (`cd $GOPATH`) +* Use git to get a copy of the MQ components into a new directory in the workspace. + + ```git clone https://github.com/ibm-messaging/mq-golang.git src/github.com/ibm-messaging/mq-golang``` + +* Use Go to download prerequisite components for any monitors you are interested in running. The logrus package is required for all of the monitors; but not all of the monitors require further downloads. + + ```go get -u github.com/Sirupsen/logrus``` - git clone https://github.com/ibm-messaging/mq-golang.git src/github.com/ibm-messaging/mq-golang + ```go get -u github.com/prometheus/client_golang/prometheus``` -* Use Go to download prerequisite components for any monitors you are interested -in running. The logrus package is required for all of the monitors; but not -all of the monitors require further downloads. + ```go get -u github.com/influxdata/influxdb/client/v2``` - go get -u github.com/Sirupsen/logrus + ```go get -u github.com/aws/aws-sdk-go/service``` - go get -u github.com/prometheus/client_golang/prometheus +* Compile the `ibmmq` component: - go get -u github.com/influxdata/influxdb/client/v2 + ```go install ./src/github.com/ibm-messaging/mq-golang/ibmmq``` - go get -u github.com/aws/aws-sdk-go/service +* Compile the `mqmetric` component: -* Compile the components you are interested in. For example + ```go install ./src/github.com/ibm-messaging/mq-golang/mqmetric``` - go install ./src/github.com/ibm-messaging/mq-golang/cmd/mq_prometheus +* Use git to get a get a copy of the MQ samples metric clients in a new directory in the workspace. -At this point, you should have a compiled copy of the code in $GOPATH/bin. + ```git clone https://github.com/ibm-messaging/mq-metric-samples.git src/github.com/ibm-messaging/mq-metric-samples``` + +* Compile the sample programs you are interested in. For example: + + ```go install ./src/github.com/ibm-messaging/mq-metric-samples/cmd/mq_prometheus``` + +At this point, you should have a compiled copy of the code in `$GOPATH/bin`. ## Limitations -Not all of the MQI verbs are available through the ibmmq package. This +Not all of the MQI verbs are available through the `ibmmq` package. This implementation concentrates on the core API calls needed to put and get messages. Currently unavailable verbs include: + * MQSET * All of the message property manipulators * MQCB @@ -138,3 +127,7 @@ For feedback and issues relating specifically to this package, please use the [G Contributions to this package can be accepted under the terms of the IBM Contributor License Agreement, found in the [CLA file](CLA.md) of this repository. When submitting a pull request, you must include a statement stating you accept the terms in the CLA. + +## Copyright + +© Copyright IBM Corporation 2016, 2018 \ No newline at end of file diff --git a/cmd/mq_aws/MQ CloudWatch-1471553836402.json b/cmd/mq_aws/MQ CloudWatch-1471553836402.json deleted file mode 100755 index 89df61d..0000000 --- a/cmd/mq_aws/MQ CloudWatch-1471553836402.json +++ /dev/null @@ -1,900 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_MQ_CLOUDWATCH", - "label": "MQ CloudWatch", - "description": "", - "type": "datasource", - "pluginId": "cloudwatch", - "pluginName": "CloudWatch" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "cloudwatch", - "name": "CloudWatch", - "version": "1.0.0" - } - ], - "id": null, - "title": "MQ CloudWatch", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_CLOUDWATCH}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "MQGET: {{object}}", - "dimensions": { - "object": "APP.0", - "qmgr": "QM1" - }, - "metricName": "queue.mqget", - "namespace": "IBM/MQ", - "period": "", - "refId": "A", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQGET: {{object}}", - "dimensions": { - "object": "APP.1", - "qmgr": "QM1" - }, - "metricName": "queue.mqget", - "namespace": "IBM/MQ", - "period": "", - "refId": "B", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQGET: {{object}}", - "dimensions": { - "object": "APP.2", - "qmgr": "QM1" - }, - "metricName": "queue.mqget", - "namespace": "IBM/MQ", - "period": "", - "refId": "C", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQGET: {{object}}", - "dimensions": { - "object": "APP.3", - "qmgr": "QM1" - }, - "metricName": "queue.mqget", - "namespace": "IBM/MQ", - "period": "", - "refId": "D", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQGET: {{object}}", - "dimensions": { - "object": "APP.4", - "qmgr": "QM1" - }, - "metricName": "queue.mqget", - "namespace": "IBM/MQ", - "period": "", - "refId": "E", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQPUT: {{object}}", - "dimensions": { - "object": "APP.0", - "qmgr": "QM1" - }, - "metricName": "queue.mqput_mqput1", - "namespace": "IBM/MQ", - "period": "", - "refId": "F", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQPUT: {{object}}", - "dimensions": { - "object": "APP.1", - "qmgr": "QM1" - }, - "metricName": "queue.mqput_mqput1", - "namespace": "IBM/MQ", - "period": "", - "refId": "G", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQPUT: {{object}}", - "dimensions": { - "object": "APP.2", - "qmgr": "QM1" - }, - "metricName": "queue.mqput_mqput1", - "namespace": "IBM/MQ", - "period": "", - "refId": "H", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQPUT: {{object}}", - "dimensions": { - "object": "APP.3", - "qmgr": "QM1" - }, - "metricName": "queue.mqput_mqput1", - "namespace": "IBM/MQ", - "period": "", - "refId": "I", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "MQPUT: {{object}}", - "dimensions": { - "object": "APP.4", - "qmgr": "QM1" - }, - "metricName": "queue.mqput_mqput1", - "namespace": "IBM/MQ", - "period": "", - "refId": "J", - "region": "us-west-2", - "statistics": [ - "Sum" - ], - "hide": false - }, - { - "alias": "Depth: {{object}}", - "dimensions": { - "object": "APP.0", - "qmgr": "QM1" - }, - "metricName": "queue.queue_depth", - "namespace": "IBM/MQ", - "period": "", - "refId": "K", - "region": "us-west-2", - "statistics": [ - "Average" - ], - "hide": false - }, - { - "alias": "Depth: {{object}}", - "dimensions": { - "object": "APP.1", - "qmgr": "QM1" - }, - "metricName": "queue.queue_depth", - "namespace": "IBM/MQ", - "period": "", - "refId": "L", - "region": "us-west-2", - "statistics": [ - "Average" - ], - "hide": false - }, - { - "alias": "Depth: {{object}}", - "dimensions": { - "object": "APP.2", - "qmgr": "QM1" - }, - "metricName": "queue.queue_depth", - "namespace": "IBM/MQ", - "period": "", - "refId": "M", - "region": "us-west-2", - "statistics": [ - "Average" - ], - "hide": false - }, - { - "alias": "Depth: {{object}}", - "dimensions": { - "object": "APP.3", - "qmgr": "QM1" - }, - "metricName": "queue.queue_depth", - "namespace": "IBM/MQ", - "period": "", - "refId": "N", - "region": "us-west-2", - "statistics": [ - "Average" - ], - "hide": false - }, - { - "alias": "Depth: {{object}}", - "dimensions": { - "object": "APP.4", - "qmgr": "QM1" - }, - "metricName": "queue.queue_depth", - "namespace": "IBM/MQ", - "period": "", - "refId": "O", - "region": "us-west-2", - "statistics": [ - "Average" - ], - "hide": false - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Queue Activity", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "MQ AWS CloudWatch", - "showTitle": true - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_CLOUDWATCH}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "System CPU", - "dimensions": { - "qmgr": "QM1" - }, - "metricName": "qmgr.system_cpu_time_percentage", - "namespace": "IBM/MQ", - "period": "", - "refId": "A", - "region": "us-west-2", - "statistics": [ - "Average" - ] - }, - { - "alias": "User CPU", - "dimensions": { - "qmgr": "QM1" - }, - "metricName": "qmgr.user_cpu_time_percentage", - "namespace": "IBM/MQ", - "period": "", - "refId": "B", - "region": "us-west-2", - "statistics": [ - "Average" - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "title": "Log write latency", - "error": false, - "span": 4, - "editable": true, - "type": "graph", - "isNew": true, - "id": 3, - "targets": [ - { - "refId": "A", - "namespace": "IBM/MQ", - "metricName": "qmgr.log_write_latency_seconds", - "statistics": [ - "Average" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "Latency" - } - ], - "datasource": "${DS_MQ_CLOUDWATCH}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "s" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "grid": { - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": true - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "File system", - "error": false, - "span": 4, - "editable": true, - "type": "graph", - "isNew": true, - "id": 4, - "targets": [ - { - "refId": "A", - "namespace": "IBM/MQ", - "metricName": "qmgr.log_file_system_in_use_bytes", - "statistics": [ - "Average" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "QMgr file system - in use [{{qmgr}}]" - }, - { - "refId": "B", - "namespace": "IBM/MQ", - "metricName": "qmgr.queue_manager_file_system_in_use_bytes", - "statistics": [ - "Average" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "Log file system - in use [{{qmgr}}]" - } - ], - "datasource": "${DS_MQ_CLOUDWATCH}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "bytes" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "grid": { - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": true - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - } - ], - "title": "New row" - }, - { - "title": "New row", - "height": "250px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "Subscriptions", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "isNew": true, - "id": 5, - "targets": [ - { - "refId": "A", - "namespace": "IBM/MQ", - "metricName": "qmgr.non_durable_subscriber_low_water_mark", - "statistics": [ - "Minimum" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "ND Sub - low-water" - }, - { - "refId": "B", - "namespace": "IBM/MQ", - "metricName": "qmgr.non_durable_subscriber_high_water_mark", - "statistics": [ - "Minimum" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "ND Sub - high-water" - }, - { - "refId": "C", - "namespace": "IBM/MQ", - "metricName": "qmgr.create_non_durable_subscription", - "statistics": [ - "Sum" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "ND MQSUB Calls" - } - ], - "datasource": "${DS_MQ_CLOUDWATCH}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "grid": { - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": true - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "Persistent Bytes Put", - "error": false, - "span": 6, - "editable": true, - "type": "singlestat", - "isNew": true, - "id": 6, - "targets": [ - { - "refId": "A", - "namespace": "IBM/MQ", - "metricName": "qmgr.put_persistent_messages_bytes", - "statistics": [ - "Sum" - ], - "dimensions": { - "qmgr": "QM1" - }, - "period": "", - "region": "us-west-2", - "alias": "Persistent Bytes" - } - ], - "links": [], - "datasource": "${DS_MQ_CLOUDWATCH}", - "maxDataPoints": 100, - "interval": null, - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "null", - "op": "=", - "text": "N/A" - } - ], - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "rangeMaps": [ - { - "from": "null", - "to": "null", - "text": "N/A" - } - ], - "mappingType": 1, - "nullPointMode": "connected", - "valueName": "avg", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "", - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - }, - "gauge": { - "show": false, - "minValue": 0, - "maxValue": 100, - "thresholdMarkers": true, - "thresholdLabels": false - } - } - ] - } - ], - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": "1m", - "schemaVersion": 12, - "version": 3, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/cmd/mq_aws/README.md b/cmd/mq_aws/README.md deleted file mode 100755 index ec1a104..0000000 --- a/cmd/mq_aws/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# MQ Exporter for Amazon CloudWatch monitoring - -This directory contains the code for a monitoring solution -that exports queue manager data to a CloudWatch data collection -system. It also contains configuration files to run the monitor program - -The monitor collects metrics published by an MQ V9 queue manager -or the MQ appliance. The monitor program pushes -those metrics into the database, where -they can then be queried directly or used by other packages -such as Grafana. - -You can see data such as disk or CPU usage, queue depths, and MQI call -counts. - -An example Grafana dashboard is included, to show how queries might -be constructed. The data shown is the same as in the corresponding -Prometheus-based dashboard, also in this repository. -To use the dashboard, -create a data source in Grafana called "MQ CloudWatch" that points at your -AWS server, and then import the JSON file. Grafana does not make it as easy -to handle wildcard queries to CloudWatch, so this dashboard explicitly names -the queues to monitor. There may be better solutions using templates, but -that starts to get more complex than I want to show in this example. - -## Building -* This github repository contains both the monitoring program and -the ibmmq package that links to the core MQ application interface. It -also contains the mqmetric package used as a common component for -supporting alternative database collection protocols. - -* You also need access to the AWS Go client interfaces. - - The command `go get -u github.com/aws/aws-sdk-go/service` should pull - down the client code and its dependencies. - -* The error logger package may need to be explicitly downloaded - - On my system, I also had to forcibly download the logger package, - using `go get -u github.com/Sirupsen/logrus`. - -Run `go build -o /mq_aws cmd/mq_aws/*.go` to compile -the program and put it to a specific directory. - -## Configuring MQ -It is convenient to run the monitor program as a queue manager service. -This directory contains an MQSC script to define the service. In fact, the -service definition points at a simple script which sets up any -necessary environment and builds the command line parameters for the -real monitor program. As the last line of the script is "exec", the -process id of the script is inherited by the monitor program, and the -queue manager can then check on the status, and can drive a suitable -`STOP SERVICE` operation during queue manager shutdown. - -Edit the MQSC script and the shell script to point at appropriate directories -where the program exists, and where you want to put stdout/stderr. -Ensure that the ID running the queue manager has permission to access -the programs and output files. - -The monitor always collects all of the available queue manager-wide metrics. -It can also be configured to collect statistics for specific sets of queues. -The sets of queues can be given either directly on the command line with the -`-ibmmq.monitoredQueues` flag, or put into a separate file which is also -named on the command line, with the `ibmmq.monitoredQueuesFile` flag. An -example is included in the startup shell script. - -Note that **for now**, the queue patterns are expanded only at startup -of the monitor program. If you want to change the patterns, or new -queues are defined that match an existing pattern, the monitor must be -restarted with a `STOP SERVICE` and `START SERVICE` pair of commands. - -There are a number of required parameters to configure the service, including -the queue manager name, how to reach a database, and the frequency of reading -the queue manager publications. Look at the mq_aws.sh script or config.go -to see how to provide these parameters. - -For authentication, the -program is expecting to pick up the keys from $HOME/.aws/credentials. That -file is not explicitly referenced in this code; it's used automatically by -the AWS toolkit. You may need to provide a region -name as a command-line option if it is not mentioned in the credentials -file. - -The queue manager will usually generate its publications every 10 seconds. However, the -default interval being used by this monitor program is set to 60 seconds for reading those -publications because of the slower rate that CloudWatch monitoring usually works at, and -to reduce the number of calls to CloudWatch which do contribute to AWS charges. - -## Configuring CloudWatch -No special configuration is required for CloudWatch. You will see the MQ -data appear in the Custom Metrics options, with a default namespace of "IBM/MQ". -There are two sets of metrics in that namespace, one for the queue manager -(with the qmgr filter) and one for the queues (with the object,qmgr filter). - -## Metrics -Once the monitor program has been started, -you will see metrics being available. -console. Two series of metrics are collected, "queue" and "qmgr". All of the queue -manager values are given a tag of the queue manager name; all of the queue-based values -are tagged with both the object and queue manager names. - -The example Grafana dashboard shows how queries can be constructed to extract data -about specific queues or the queue manager. - -More information on the metrics collected through the publish/subscribe -interface can be found in the [MQ KnowledgeCenter] -(https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.mon.doc/mo00013_.htm) -with further description in [an MQDev blog entry] -(https://www.ibm.com/developerworks/community/blogs/messaging/entry/Statistics_published_to_the_system_topic_in_MQ_v9?lang=en) - -The metrics stored in the database are named after the -descriptions that you can see when running the amqsrua sample program, but with some -minor modifications to match a more useful style. diff --git a/cmd/mq_aws/config.go b/cmd/mq_aws/config.go deleted file mode 100755 index f27e6e2..0000000 --- a/cmd/mq_aws/config.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "flag" - - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type mqCloudWatchConfig struct { - qMgrName string - replyQ string - monitoredQueues string - monitoredQueuesFile string - - region string - - namespace string - - cc mqmetric.ConnectionConfig - - interval string - maxErrors int - maxPoints int - - logLevel string -} - -var config mqCloudWatchConfig - -/* -initConfig parses the command line parameters. -*/ -func initConfig() { - - flag.StringVar(&config.qMgrName, "ibmmq.queueManager", "", "Queue Manager name") - flag.StringVar(&config.replyQ, "ibmmq.replyQueue", "SYSTEM.DEFAULT.MODEL.QUEUE", "Reply Queue to collect data") - flag.StringVar(&config.monitoredQueues, "ibmmq.monitoredQueues", "", "Patterns of queues to monitor") - flag.StringVar(&config.monitoredQueuesFile, "ibmmq.monitoredQueuesFile", "", "File with patterns of queues to monitor") - - flag.StringVar(&config.region, "ibmmq.awsregion", "", "AWS Region to connect to") - - flag.StringVar(&config.namespace, "ibmmq.namespace", "IBM/MQ", "Namespace for metrics") - - flag.BoolVar(&config.cc.ClientMode, "ibmmq.client", false, "Connect as MQ client") - - flag.StringVar(&config.interval, "ibmmq.interval", "60", "How many seconds between each collection") - flag.IntVar(&config.maxErrors, "ibmmq.maxErrors", 10000, "Maximum number of errors communicating with server before considered fatal") - flag.IntVar(&config.maxPoints, "ibmmq.maxPoints", 20, "Maximum number of points to include in each write to the server") - - flag.StringVar(&config.logLevel, "log.level", "error", "Log level - debug, info, error") - - flag.Parse() - - if config.monitoredQueuesFile != "" { - config.monitoredQueues = mqmetric.ReadPatterns(config.monitoredQueuesFile) - } -} diff --git a/cmd/mq_aws/exporter.go b/cmd/mq_aws/exporter.go deleted file mode 100755 index 5a0dca5..0000000 --- a/cmd/mq_aws/exporter.go +++ /dev/null @@ -1,174 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -/* -This file pushes collected data to the Amazon CloudWatch service. -The Collect() function is the key operation -invoked at the configured intervals, causing us to read available publications -and update the various data points. -*/ - -import ( - "time" - - log "github.com/Sirupsen/logrus" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type client struct { - sess *session.Session - svc *cloudwatch.CloudWatch -} - -var ( - first = true - errorCount = 0 - c client -) - -/* -Collect is called by the main routine at regular intervals to provide current -data -*/ -func Collect() error { - var err error - var series string - log.Infof("IBM MQ AWS collection started") - - if c.sess == nil { - c.sess, err = session.NewSession() - if err != nil { - log.Fatal("Cannot create session: ", err) - } - c.svc = nil - } - - if c.svc == nil { - if config.region == "" { - c.svc = cloudwatch.New(c.sess, aws.NewConfig()) - } else { - c.svc = cloudwatch.New(c.sess, aws.NewConfig().WithRegion(config.region)) - } - if err != nil { - log.Fatal("Cannot create service: ", err) - } - } - - // Clear out everything we know so far. In particular, replace - // the map of values for each object so the collection starts - // clean. - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - elem.Values = make(map[string]int64) - } - } - } - - // Process all the publications that have arrived - mqmetric.ProcessPublications() - - // Have now processed all of the publications, and all the MQ-owned - // value fields and maps have been updated. - // - // Now need to set all of the real items with the correct values - if first { - // Always ignore the first loop through as there might - // be accumulated stuff from a while ago, and lead to - // a misleading range on graphs. - first = false - } else { - t := time.Now() - bp := newBatchPoints() - - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - for key, value := range elem.Values { - f := mqmetric.Normalise(elem, key, value) - tags := map[string]string{ - "qmgr": config.qMgrName, - } - - series = "qmgr" - if key != mqmetric.QMgrMapKey { - tags["object"] = key - series = "queue" - } - pt, _ := newPoint(series+"."+elem.MetricName, t, float64(f), tags) - bp.addPoint(pt) - - // AWS recommends not sending too many - // data points in a single request. - // So we flush after a configurable set has been - // collected. - if len(bp.Points) >= config.maxPoints { - bp = c.Flush(bp) - } - //log.Debugf("Adding point %v", pt) - } - } - } - } - - c.Flush(bp) - } - - return err - -} - -func (c client) Flush(bp *BatchPoints) *BatchPoints { - // This is where real errors might occur, including the inability to - // contact the server. We will ignore (but log) these errors - // up to a threshold, after which it is considered fatal. - if len(bp.Points) > 0 { - err := c.Put(bp) - if err != nil { - log.Error(err) - errorCount++ - if errorCount >= config.maxErrors { - log.Fatal("Too many errors communicating with server") - } - } else { - errorCount = 0 - } - } - bp = newBatchPoints() - return bp -} - -func (c client) Put(bp *BatchPoints) error { - if len(bp.Points) == 0 { - return nil - } - - log.Infof("Putting %d points", len(bp.Points)) - params := &cloudwatch.PutMetricDataInput{ - MetricData: bp.Points, - Namespace: aws.String(config.namespace), - } - - _, err := c.svc.PutMetricData(params) - return err -} diff --git a/cmd/mq_aws/main.go b/cmd/mq_aws/main.go deleted file mode 100755 index b3b543b..0000000 --- a/cmd/mq_aws/main.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "os" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -func initLog() { - level, err := log.ParseLevel(config.logLevel) - if err != nil { - level = log.InfoLevel - } - log.SetLevel(level) -} - -func main() { - var err error - - initConfig() - if config.qMgrName == "" { - log.Errorln("Must provide a queue manager name to connect to.") - os.Exit(1) - } - d, err := time.ParseDuration(config.interval + "s") - if err != nil { - log.Errorln("Invalid value for interval parameter: ", err) - os.Exit(1) - } - - initLog() - log.Infoln("Starting IBM MQ metrics exporter for AWS CloudWatch monitoring") - - // Connect and open standard queues - err = mqmetric.InitConnection(config.qMgrName, config.replyQ, &config.cc) - if err == nil { - log.Infoln("Connected to queue manager ", config.qMgrName) - defer mqmetric.EndConnection() - } - - // What metrics can the queue manager provide? Find out, and - // subscribe. - if err == nil { - err = mqmetric.DiscoverAndSubscribe(config.monitoredQueues, true, "") - } - - // Go into main loop for sending data to database - if err == nil { - for { - Collect() - time.Sleep(d) - } - - } - - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} diff --git a/cmd/mq_aws/mq_aws.mqsc b/cmd/mq_aws/mq_aws.mqsc deleted file mode 100755 index 9856767..0000000 --- a/cmd/mq_aws/mq_aws.mqsc +++ /dev/null @@ -1,18 +0,0 @@ -* Cleanup any existing service -STOP SERVICE(MQAWS) -DELETE SERVICE(MQAWS) - -* Reset the definition -DEFINE SERVICE(MQAWS) + - CONTROL(QMGR) + - SERVTYPE(SERVER) + - STARTCMD('/usr/local/bin/mqgo/mq_aws.sh') + - STARTARG(+QMNAME+) + - STOPCMD('/usr/bin/kill ' ) + - STOPARG(+MQ_SERVER_PID+) + - STDOUT('/var/mqm/errors/aws.out') + - STDERR('/var/mqm/errors/aws.out') + - DESCR('MQ exporter for AWS') - -* Start it manually now; will be automatically started on future qmgr startup -START SERVICE(MQAWS) diff --git a/cmd/mq_aws/mq_aws.sh b/cmd/mq_aws/mq_aws.sh deleted file mode 100755 index 98af64d..0000000 --- a/cmd/mq_aws/mq_aws.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -# This is used to start the IBM MQ monitoring service for AWS - -# The queue manager name comes in from the service definition as the -# only command line parameter -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of queues to be monitored is given here. -# It is a set of names or patterns ('*' only at the end, to match how MQ works), -# separated by commas. When no queues match a pattern, it is reported but -# is not fatal. -queues="APP.*,MYQ.*" - -# An alternative is to have a file containing the patterns, and named -# via the ibmmq.monitoredQueuesFile option. - -# And other parameters that may be needed -# See config.go for all recognised flags -interval="30" - -ARGS="-ibmmq.queueManager=$qMgr" -ARGS="$ARGS -ibmmq.monitoredQueues=$queues" -ARGS="$ARGS -log.level=error" -ARGS="$ARGS -ibmmq.maxPoints=20" -ARGS="$ARGS -ibmmq.awsregion=us-west-2" - -# Start via "exec" so the pid remains the same. The queue manager can -# then check the existence of the service and use the MQ_SERVER_PID value -# to kill it on shutdown. - -# Credentials are expected to be provided in the $HOME/.aws/credentials file -exec /usr/local/bin/mqgo/mq_aws $ARGS diff --git a/cmd/mq_aws/points.go b/cmd/mq_aws/points.go deleted file mode 100755 index 2bd8a76..0000000 --- a/cmd/mq_aws/points.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "errors" - _ "github.com/Sirupsen/logrus" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "strings" - "time" -) - -func newPoint(metric string, timestamp time.Time, value float64, tags map[string]string) (*cloudwatch.MetricDatum, error) { - var dl []*cloudwatch.Dimension - - if metric == "" { - return nil, errors.New("PointError: Metric can not be empty") - } - - d1 := cloudwatch.Dimension{ - Name: aws.String("qmgr"), - Value: aws.String(tags["qmgr"]), - } - - dl = append(dl, &d1) - - if qName, ok := tags["object"]; ok { - d2 := cloudwatch.Dimension{ - Name: aws.String("object"), - Value: aws.String(qName), - } - dl = append(dl, &d2) - } - - unit := aws.String("None") - if strings.HasSuffix(metric, "_seconds") { - unit = aws.String("Seconds") - } else if strings.HasSuffix(metric, "_bytes") { - unit = aws.String("Bytes") - } - - return &cloudwatch.MetricDatum{ - Dimensions: dl, - MetricName: aws.String(metric), - Timestamp: aws.Time(timestamp), - Unit: unit, - Value: aws.Float64(value), - }, nil -} - -/* -BatchPoints is a set of points collected in one iteration. -*/ -type BatchPoints struct { - Points []*cloudwatch.MetricDatum -} - -func newBatchPoints() *BatchPoints { - return &BatchPoints{} -} - -func (bp *BatchPoints) addPoint(p *cloudwatch.MetricDatum) { - bp.Points = append(bp.Points, p) -} diff --git a/cmd/mq_coll/CollectD Graphite-1470830323096.json b/cmd/mq_coll/CollectD Graphite-1470830323096.json deleted file mode 100755 index 7995a6f..0000000 --- a/cmd/mq_coll/CollectD Graphite-1470830323096.json +++ /dev/null @@ -1,582 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_COLLECTD_GRAPHITE", - "label": "CollectD Graphite", - "description": "", - "type": "datasource", - "pluginId": "graphite", - "pluginName": "Graphite" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "graphite", - "name": "Graphite", - "version": "1.0.0" - } - ], - "id": null, - "title": "CollectD Graphite", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.queue_depth-*)", - "refId": "A" - }, - { - "refId": "B", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.queue_mqput_mqput1-*)" - }, - { - "refId": "C", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.queue_mqget-*)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Queue Activity", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": true, - "title": "MQ Collectd Graphite" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "B", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.system_cpu_time_percentage)" - }, - { - "refId": "A", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.user_cpu_time_percentage)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "alias(averageSeries(collectdlocalhostcollectd.qmgr-QM1.log_write_latency_seconds), 'Latency')" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Log write latency", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.log_file_system_in_use_bytes)" - }, - { - "refId": "B", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.queue_manager_file_system_in_use_bytes)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "File system", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "title": "Subscriptions", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "isNew": true, - "id": 5, - "targets": [ - { - "refId": "A", - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.non_durable_subscriber_low_water_mark)" - }, - { - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.non_durable_subscriber_high_water_mark)", - "refId": "B" - }, - { - "target": "aliasByMetric(collectdlocalhostcollectd.qmgr-QM1.create_non_durable_subscription)", - "refId": "C" - } - ], - "datasource": "${DS_COLLECTD_GRAPHITE}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "grid": { - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": true - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "Persistent Bytes Put", - "error": false, - "span": 6, - "editable": true, - "type": "singlestat", - "isNew": true, - "id": 6, - "targets": [ - { - "refId": "A", - "target": "sumSeries(collectdlocalhostcollectd.qmgr-QM1.put_persistent_messages_bytes)" - } - ], - "links": [], - "datasource": "${DS_COLLECTD_GRAPHITE}", - "maxDataPoints": 100, - "interval": null, - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "null", - "op": "=", - "text": "N/A" - } - ], - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "rangeMaps": [ - { - "from": "null", - "to": "null", - "text": "N/A" - } - ], - "mappingType": 1, - "nullPointMode": "connected", - "valueName": "avg", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "", - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - }, - "gauge": { - "show": false, - "minValue": 0, - "maxValue": 100, - "thresholdMarkers": true, - "thresholdLabels": false - } - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": "10s", - "schemaVersion": 12, - "version": 2, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/cmd/mq_coll/Collectd OpenTSDB-1470820482558.json b/cmd/mq_coll/Collectd OpenTSDB-1470820482558.json deleted file mode 100755 index e7f3728..0000000 --- a/cmd/mq_coll/Collectd OpenTSDB-1470820482558.json +++ /dev/null @@ -1,630 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_COLLECTD_TSDB", - "label": "CollectD TSDB", - "description": "", - "type": "datasource", - "pluginId": "opentsdb", - "pluginName": "OpenTSDB" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "opentsdb", - "name": "OpenTSDB", - "version": "1.0.0" - } - ], - "id": null, - "title": "Collectd OpenTSDB", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_TSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "sum", - "alias": "Depth - APP.0", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.queue_depth.APP_0", - "refId": "A", - "currentFilterKey": "fqdn" - }, - { - "aggregator": "sum", - "alias": "MQPUT - APP.0", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.queue_mqput_mqput1.APP_0", - "refId": "B" - }, - { - "aggregator": "sum", - "alias": "MQGET - APP.0", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.queue_mqget.APP_0", - "refId": "C" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Queue Activity", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": true, - "title": "MQ COLLECTD CAPTURE" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_TSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "System CPU%", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.system_cpu_time_percentage", - "refId": "A" - }, - { - "aggregator": "avg", - "alias": "User CPU%", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.user_cpu_time_percentage", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_TSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "Latency", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.log_write_latency_seconds", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Log write latency", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_TSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "QMgr file system - in use [QM1]", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.queue_manager_file_system_in_use_bytes", - "refId": "B" - }, - { - "aggregator": "avg", - "alias": "Log file system - in use [QM1]", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.log_file_system_in_use_bytes", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "File system", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_COLLECTD_TSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "ND Sub - low water", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.non_durable_subscriber_low_water_mark", - "refId": "B" - }, - { - "aggregator": "avg", - "alias": "ND Sub - high water", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.non_durable_subscriber_high_water_mark", - "refId": "A" - }, - { - "aggregator": "sum", - "alias": "ND MQSUB calls", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.create_non_durable_subscription", - "refId": "C" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Subscriptions", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_COLLECTD_TSDB}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 6, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 6, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "avg", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.QM1.put_persistent_messages_bytes", - "refId": "A" - } - ], - "thresholds": "", - "title": "Persistent Bytes Put", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": "10s", - "schemaVersion": 12, - "version": 4, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/cmd/mq_coll/README.md b/cmd/mq_coll/README.md deleted file mode 100755 index f21e573..0000000 --- a/cmd/mq_coll/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# MQ Exporter for Collectd monitoring - -This directory contains the code for a monitoring solution -that sends queue manager data to collectd system. -It also contains configuration files to run the monitor program - -The monitor collects metrics published by an MQ V9 queue manager -or the MQ appliance. The monitor program prints -these metrics to stdout, in a format that is recognised by -collectd. Once processed by collectd, they may be forwarded to -storage systems, where -they can then be queried directly or used by other packages -such as Grafana. - -You can see data such as disk or CPU usage, queue depths, and MQI call -counts. - -Example Grafana dashboards are included, to show how queries might -be constructed, when collectd is configured to send the data to -an OpenTSDB or Graphite database. To use the dashboard, -create data sources in Grafana called "CollectD TSDB" or "CollectD Graphite" -that point at your database server, and then import the JSON file. - -## Building -* This github repository contains both the monitoring program and -the ibmmq package that links to the core MQ application interface. It -also contains the mqmetric package used as a common component for -supporting alternative database collection protocols. - -* Get the error logger package used by all of these monitors -using `go get -u github.com/Sirupsen/logrus`. - -Run `go build -o /mq_coll cmd/mq_coll/*.go` to compile -the program and put it to a specific directory. - -## Configuring MQ - -No MQ configuration is required, as (unlike the other monitors in this -repository) the program does not run as an MQ Service. - -The monitor always collects all of the available queue manager-wide metrics. -It can also be configured to collect statistics for specific sets of queues. -The sets of queues can be given either directly on the command line with the -`-ibmmq.monitoredQueues` flag, or put into a separate file which is also -named on the command line, with the `ibmmq.monitoredQueuesFile` flag. An -example is included in the startup shell script. - -Note that **for now**, the queue patterns are expanded only at startup -of the monitor program. If you want to change the patterns, or new -queues are defined that match an existing pattern, the monitor must be -restarted. - -## Configuring collectd -There are several steps needed to configure collectd. It will use the -'Exec' interface to manage the MQ collection. -* Edit the mq.conf file to point at the shell script that starts the real -program. The name of the queue manager to be monitored is also given here. -* Edit the shell script mq_coll.sh to set additional parameters for the -program, and ensure the program is being invoked from the correct directory. -* Put mq.conf in the /etc/collectd.d directory. That is read during -startup, telling collectd how to use MQ. It also contains a reference -to the list of metrics generated by the program, which is added to the -global TypesDB configuration. -* Restart collectd to pick up new configuration. For example - -``` - systemctl restart collectd -``` - -## Metrics -Once the monitor program has been started, -you will see metrics being available. The metric names sent to collectd -start with "qmgr" followed by the queue manager name. The next element -is the actual metric (queue_depth, system_cpu etc) followed by the -queue name if appropriate. You can check that the mq_coll program is running -with the ps command. That shows correct configuration of the collectd plugin -and the mq_coll.sh script. - -There is some minor rewriting of object names, replacing "." with "_" to -conform with the collectd Exec interface; the metric names may get rewritten -again by the conversion to the backend storage (re-replacing "_" with "." -perhaps). - -The example Grafana dashboard shows how queries can be constructed to extract data -about specific queues or the queue manager. - -More information on the metrics collected through the publish/subscribe -interface can be found in the [MQ KnowledgeCenter] -(https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.mon.doc/mo00013_.htm) -with further description in [an MQDev blog entry] -(https://www.ibm.com/developerworks/community/blogs/messaging/entry/Statistics_published_to_the_system_topic_in_MQ_v9?lang=en) - -The metrics stored in the database are named after the -descriptions that you can see when running the amqsrua sample program, but with some -minor modifications to match a more useful style. diff --git a/cmd/mq_coll/config.go b/cmd/mq_coll/config.go deleted file mode 100755 index 5be1217..0000000 --- a/cmd/mq_coll/config.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "flag" - "os" - - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type mqTTYConfig struct { - qMgrName string - replyQ string - monitoredQueues string - monitoredQueuesFile string - hostname string - hostlabel string // Used in the output string - - cc mqmetric.ConnectionConfig - - interval string - - logLevel string -} - -var config mqTTYConfig - -/* -initConfig parses the command line parameters. -*/ -func initConfig() { - - flag.StringVar(&config.qMgrName, "ibmmq.queueManager", "", "Queue Manager name") - flag.StringVar(&config.replyQ, "ibmmq.replyQueue", "SYSTEM.DEFAULT.MODEL.QUEUE", "Reply Queue to collect data") - flag.StringVar(&config.monitoredQueues, "ibmmq.monitoredQueues", "", "Patterns of queues to monitor") - flag.StringVar(&config.monitoredQueuesFile, "ibmmq.monitoredQueuesFile", "", "File with patterns of queues to monitor") - - flag.StringVar(&config.interval, "ibmmq.interval", "10", "How many seconds between each collection") - flag.StringVar(&config.hostname, "ibmmq.hostname", "localhost", "Host to connect to") - - flag.BoolVar(&config.cc.ClientMode, "ibmmq.client", false, "Connect as MQ client") - - flag.StringVar(&config.logLevel, "log.level", "error", "Log level - debug, info, error") - - flag.Parse() - - if config.monitoredQueuesFile != "" { - config.monitoredQueues = mqmetric.ReadPatterns(config.monitoredQueuesFile) - } - - // Don't want to use "localhost" as the tag in the metric printing - if config.hostname == "localhost" { - config.hostlabel, _ = os.Hostname() - } else { - config.hostlabel = config.hostname - } -} diff --git a/cmd/mq_coll/exporter.go b/cmd/mq_coll/exporter.go deleted file mode 100755 index df43300..0000000 --- a/cmd/mq_coll/exporter.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -/* -The Collect() function is the key operation -invoked at the configured intervals, causing us to read available publications -and update the various data points. -*/ - -import ( - "fmt" - "strings" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -var ( - first = true - errorCount = 0 -) - -/* -Collect is called by the main routine at regular intervals to provide current -data -*/ -func Collect() error { - var err error - - log.Infof("IBM MQ stdout collector started") - - // Clear out everything we know so far. In particular, replace - // the map of values for each object so the collection starts - // clean. - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - elem.Values = make(map[string]int64) - } - } - } - - // Process all the publications that have arrived - mqmetric.ProcessPublications() - - // Have now processed all of the publications, and all the MQ-owned - // value fields and maps have been updated. - // - // Now need to set all of the real items with the correct values - if first { - // Always ignore the first loop through as there might - // be accumulated stuff from a while ago, and lead to - // a misleading range on graphs. - first = false - } else { - - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - for key, value := range elem.Values { - f := mqmetric.Normalise(elem, key, value) - tags := map[string]string{ - "qmgr": config.qMgrName, - } - - if key != mqmetric.QMgrMapKey { - tags["object"] = key - } - printPoint(elem.MetricName, float32(f), tags) - - } - } - } - } - - } - - return err - -} - -func printPoint(metric string, val float32, tags map[string]string) { - qmgr := tags["qmgr"] - if q, ok := tags["object"]; ok { - if !strings.HasPrefix(metric, "queue") { - metric = "queue_" + metric - } - metric += "-" + fixup(q) - } - fmt.Printf("PUTVAL %s/%s-%s/%s interval=%s N:%f\n", - config.hostlabel, "qmgr", fixup(qmgr), metric, config.interval, val) - return -} - -func fixup(s1 string) string { - s2 := strings.Replace(s1, ".", "_", -1) - return s2 -} diff --git a/cmd/mq_coll/main.go b/cmd/mq_coll/main.go deleted file mode 100755 index bdaea5c..0000000 --- a/cmd/mq_coll/main.go +++ /dev/null @@ -1,86 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "os" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -func initLog() { - level, err := log.ParseLevel(config.logLevel) - if err != nil { - level = log.InfoLevel - } - log.SetLevel(level) - // Since this program prints its data to stdout, need any - // log info to go elsewhere. - log.SetOutput(os.Stderr) -} - -func main() { - var err error - - initConfig() - if config.qMgrName == "" { - log.Errorln("Must provide a queue manager name to connect to.") - os.Exit(1) - } - d, err := time.ParseDuration(config.interval + "s") - if err != nil { - log.Errorln("Invalid value for interval parameter: ", err) - os.Exit(1) - } - - initLog() - log.Infoln("Starting IBM MQ metrics exporter for collectd") - - // Connect and open standard queues - err = mqmetric.InitConnection(config.qMgrName, config.replyQ, &config.cc) - if err == nil { - log.Infoln("Connected to queue manager ", config.qMgrName) - defer mqmetric.EndConnection() - } - - // What metrics can the queue manager provide? Find out, and - // subscribe. - if err == nil { - err = mqmetric.DiscoverAndSubscribe(config.monitoredQueues, true, "") - } - - // Go into main loop for sending data to stdout - // This program runs forever, or at least until killed by - // collectd - if err == nil { - for { - Collect() - time.Sleep(d) - } - - } - - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} diff --git a/cmd/mq_coll/mq.conf b/cmd/mq_coll/mq.conf deleted file mode 100644 index f79934c..0000000 --- a/cmd/mq_coll/mq.conf +++ /dev/null @@ -1,19 +0,0 @@ -# This file is put into the /etc/collectd.d directory. -# -# Edit the program name to point at wherever you have installed it. -# The only parameter to the collection program is the queue manager -# name. -# -# The program is run as the mqm userid to ensure it is suitably -# privileged. It could be run under any other account provided -# the appropriate setmqaut commands are run to allow it to issue -# queries and subscribe to the topics. - -LoadPlugin exec - - Exec "mqm:mqm" "/usr/local/bin/mqgo/mq_coll.sh" "QM1" - - -# We also add a reference to the MQ metric information here. This is -# added to the global types database. -TypesDB "/usr/local/bin/mqgo/mqtypes.db" diff --git a/cmd/mq_coll/mq_coll.sh b/cmd/mq_coll/mq_coll.sh deleted file mode 100755 index fb0ed7d..0000000 --- a/cmd/mq_coll/mq_coll.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -# This is used to start the IBM MQ monitoring service for collectd -# via the 'Exec' interface - -# The queue manager name comes in from the collectd configuration as -# the only command line parameter. -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -# This assumes there is an MQ installation in the default location, even -# if it is not the one associated with the queue manager -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of queues to be monitored is given here. -# It is a set of names or patterns ('*' only at the end, to match how MQ works), -# separated by commas. When no queues match a pattern, it is reported but -# is not fatal. -queues="APP.*,MYQ.*" - -# An alternative is to have a file containing the patterns, and named -# via the ibmmq.monitoredQueuesFile option. - -# And other parameters that may be needed -# See config.go for all recognised flags -interval="5" - -ARGS="-ibmmq.queueManager=$qMgr" -ARGS="$ARGS -ibmmq.monitoredQueues=$queues" -ARGS="$ARGS -ibmmq.interval=$interval" -ARGS="$ARGS -log.level=error" - -# Start via "exec" so the pid remains the same. -# -# The collectd system reads everything sent to stdout -# This program will send error log info to stderr; that is also -# picked up by collectd. -# -# Change this line to match wherever you have installed the MQ monitor program -exec /usr/local/bin/mqgo/mq_coll $ARGS diff --git a/cmd/mq_coll/mqtypes.db b/cmd/mq_coll/mqtypes.db deleted file mode 100755 index 091afff..0000000 --- a/cmd/mq_coll/mqtypes.db +++ /dev/null @@ -1,128 +0,0 @@ -# This file is referenced from the global TypesDB attribute in -# /etc/collectd.conf. It does not need to be copied into the default -# types.db file, but can be named independently. For example -# -# TypesDB "/usr/share/collectd/types.db" "/mqtypes.db" -# -# All of the currently-known metrics generated by MQ are listed here -# as gauges with no specific maximum value. - -alter_durable_subscription value:GAUGE:0:U -commit value:GAUGE:0:U -concurrent_connections_high_water_mark value:GAUGE:0:U -cpu_load_fifteen_minute_average value:GAUGE:0:U -cpu_load_five_minute_average value:GAUGE:0:U -cpu_load_one_minute_average value:GAUGE:0:U -create_durable_subscription value:GAUGE:0:U -create_non_durable_subscription value:GAUGE:0:U -delete_durable_subscription value:GAUGE:0:U -delete_non_durable_subscription value:GAUGE:0:U -durable_subscriber_high_water_mark value:GAUGE:0:U -durable_subscriber_low_water_mark value:GAUGE:0:U -expired_messages value:GAUGE:0:U -failed_browse value:GAUGE:0:U -failed_commit value:GAUGE:0:U -failed_create_alter_resume_subscription value:GAUGE:0:U -failed_mqcb value:GAUGE:0:U -failed_mqclose value:GAUGE:0:U -failed_mqconn_mqconnx value:GAUGE:0:U -failed_mqctl value:GAUGE:0:U -failed_mqget value:GAUGE:0:U -failed_mqinq value:GAUGE:0:U -failed_mqopen value:GAUGE:0:U -failed_mqput value:GAUGE:0:U -failed_mqput1 value:GAUGE:0:U -failed_mqset value:GAUGE:0:U -failed_mqstat value:GAUGE:0:U -failed_mqsubrq value:GAUGE:0:U -failed_topic_mqput_mqput1 value:GAUGE:0:U -got_non_persistent_messages_bytes value:GAUGE:0:U -got_persistent_messages_bytes value:GAUGE:0:U -interval_total_destructive_get value:GAUGE:0:U -interval_total_destructive_get_bytes value:GAUGE:0:U -interval_total_mqput_mqput1 value:GAUGE:0:U -interval_total_mqput_mqput1_bytes value:GAUGE:0:U -interval_total_topic_bytes_put value:GAUGE:0:U -log_file_system_in_use_bytes value:GAUGE:0:U -log_file_system_max_bytes value:GAUGE:0:U -log_in_use_bytes value:GAUGE:0:U -log_logical_written_bytes value:GAUGE:0:U -log_max_bytes value:GAUGE:0:U -log_physical_written_bytes value:GAUGE:0:U -log_write_latency_seconds value:GAUGE:0:U -mqcb value:GAUGE:0:U -mqclose value:GAUGE:0:U -mqconn_mqconnx value:GAUGE:0:U -mqctl value:GAUGE:0:U -mqdisc value:GAUGE:0:U -mq_errors_file_system_free_space_bytes value:GAUGE:0:U -mq_errors_file_system_in_use_bytes value:GAUGE:0:U -mq_fdc_files value:GAUGE:0:U -mqinq value:GAUGE:0:U -mqopen value:GAUGE:0:U -mqset value:GAUGE:0:U -mqstat value:GAUGE:0:U -mqsubrq value:GAUGE:0:U -mq_trace_file_system_free_space_bytes value:GAUGE:0:U -mq_trace_file_system_in_use_bytes value:GAUGE:0:U -non_durable_subscriber_high_water_mark value:GAUGE:0:U -non_durable_subscriber_low_water_mark value:GAUGE:0:U -non_persistent_message_browse value:GAUGE:0:U -non_persistent_message_browse_bytes value:GAUGE:0:U -non_persistent_message_destructive_get value:GAUGE:0:U -non_persistent_message_mqput value:GAUGE:0:U -non_persistent_message_mqput1 value:GAUGE:0:U -non_persistent_topic_mqput_mqput1 value:GAUGE:0:U -persistent_message_browse value:GAUGE:0:U -persistent_message_browse_bytes value:GAUGE:0:U -persistent_message_destructive_get value:GAUGE:0:U -persistent_message_mqput value:GAUGE:0:U -persistent_message_mqput1 value:GAUGE:0:U -persistent_topic_mqput_mqput1 value:GAUGE:0:U -published_to_subscribers_bytes value:GAUGE:0:U -published_to_subscribers_messages value:GAUGE:0:U -purged_queue value:GAUGE:0:U -put_non_persistent_messages_bytes value:GAUGE:0:U -put_persistent_messages_bytes value:GAUGE:0:U -queue_average_queue_time_seconds value:GAUGE:0:U -queue_avoided_bytes value:GAUGE:0:U -queue_avoided_puts value:GAUGE:0:U -queue_depth value:GAUGE:0:U -queue_destructive_mqget_non_persistent_bytes value:GAUGE:0:U -queue_destructive_mqget_non_persistent_messages value:GAUGE:0:U -queue_destructive_mqget_persistent_bytes value:GAUGE:0:U -queue_destructive_mqget_persistent_messages value:GAUGE:0:U -queue_expired_messages value:GAUGE:0:U -queue_lock_contention value:GAUGE:0:U -queue_manager_file_system_free_space_bytes value:GAUGE:0:U -queue_manager_file_system_in_use_bytes value:GAUGE:0:U -queue_mqclose value:GAUGE:0:U -queue_mqget value:GAUGE:0:U -queue_mqget_browse_non_persistent_bytes value:GAUGE:0:U -queue_mqget_browse_non_persistent_messages value:GAUGE:0:U -queue_mqget_browse_persistent_bytes value:GAUGE:0:U -queue_mqget_browse_persistent_messages value:GAUGE:0:U -queue_mqget_bytes value:GAUGE:0:U -queue_mqinq value:GAUGE:0:U -queue_mqopen value:GAUGE:0:U -queue_mqput1_non_persistent_messages value:GAUGE:0:U -queue_mqput1_persistent_messages value:GAUGE:0:U -queue_mqput_bytes value:GAUGE:0:U -queue_mqput_mqput1 value:GAUGE:0:U -queue_mqput_non_persistent_messages value:GAUGE:0:U -queue_mqput_persistent_messages value:GAUGE:0:U -queue_mqset value:GAUGE:0:U -queue_non_persistent_bytes value:GAUGE:0:U -queue_persistent_bytes value:GAUGE:0:U -queue_purged value:GAUGE:0:U -ram_free_percentage value:GAUGE:0:U -ram_total_bytes value:GAUGE:0:U -ram_total_bytes_estimate_for_queue_manager value:GAUGE:0:U -resume_durable_subscription value:GAUGE:0:U -rollback value:GAUGE:0:U -subscription_delete_failure value:GAUGE:0:U -system_cpu_time_estimate_for_queue_manager_percentage value:GAUGE:0:U -system_cpu_time_percentage value:GAUGE:0:U -topic_mqput_mqput1_interval_total value:GAUGE:0:U -user_cpu_time_estimate_for_queue_manager_percentage value:GAUGE:0:U -user_cpu_time_percentage value:GAUGE:0:U diff --git a/cmd/mq_influx/MQ InfluxDB-1470081417682.json b/cmd/mq_influx/MQ InfluxDB-1470081417682.json deleted file mode 100755 index ea3fb68..0000000 --- a/cmd/mq_influx/MQ InfluxDB-1470081417682.json +++ /dev/null @@ -1,1018 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_MQ_INFLUX", - "label": "MQ Influx", - "description": "", - "type": "datasource", - "pluginId": "influxdb", - "pluginName": "InfluxDB" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - } - ], - "id": null, - "title": "MQ InfluxDB", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_INFLUX}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "interval": ">20s", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$col: $tag_object", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "10s" - ], - "type": "time" - }, - { - "params": [ - "object" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "queue", - "policy": "default", - "query": "SELECT sum(\"mqget\") as MQGET FROM \"queue\" WHERE \"queue\" =~ /APP.*/ AND $timeFilter GROUP BY time(10s), \"queue\" fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mqget" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "MQGET" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "object", - "operator": "=~", - "value": "/APP.*/" - } - ] - }, - { - "alias": "$col: $tag_object", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "10s" - ], - "type": "time" - }, - { - "params": [ - "object" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "queue", - "policy": "default", - "query": "SELECT sum(\"mqput_mqput1\") as MQPUT FROM \"queue\" WHERE \"queue\" =~ /APP.*/ AND $timeFilter GROUP BY time(10s), \"queue\" fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mqput_mqput1" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "MQPUT" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "object", - "operator": "=~", - "value": "/APP.*/" - } - ] - }, - { - "alias": "$col: $tag_object", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "10s" - ], - "type": "time" - }, - { - "params": [ - "object" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "queue", - "policy": "default", - "query": "SELECT median(\"queue_depth\") as DEPTH FROM \"queue\" WHERE \"queue\" =~ /APP.*/ AND $timeFilter GROUP BY time(10s), \"queue\" fill(null)", - "rawQuery": false, - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "queue_depth" - ], - "type": "field" - }, - { - "params": [], - "type": "median" - }, - { - "params": [ - "Depth" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "object", - "operator": "=~", - "value": "/APP.*/" - } - ] - } - ], - "timeFrom": "15m", - "timeShift": null, - "title": "Queue Activity", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": true, - "title": "MQ InfluxDB capture" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_INFLUX}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "System CPU%", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "system_cpu_time_percentage" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - }, - { - "alias": "User CPU%", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "user_cpu_time_percentage" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_INFLUX}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Latency", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "log_write_latency_seconds" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Log write latency", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_INFLUX}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "QMgr file system - in use [$tag_qmgr]", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "qmgr" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "queue_manager_file_system_in_use_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - }, - { - "alias": "Log file system - in use [$tag_qmgr]", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "qmgr" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "log_file_system_in_use_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "File system", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_INFLUX}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$col", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "non_durable_subscriber_low_water_mark" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "ND - Low Water" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "non_durable_subscriber_high_water_mark" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "ND - High Water" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "create_non_durable_subscription" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "ND MQSUB calls" - ], - "type": "alias" - } - ] - ], - "tags": [] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Subscriptions", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_MQ_INFLUX}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 6, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 6, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "qmgr", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "put_persistent_messages_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - } - ], - "thresholds": "", - "title": "Persistent Bytes Put", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": "10s", - "schemaVersion": 12, - "version": 21, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/cmd/mq_influx/README.md b/cmd/mq_influx/README.md deleted file mode 100755 index 355b91d..0000000 --- a/cmd/mq_influx/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# MQ Exporter for InfluxDB monitoring - -This directory contains the code for a monitoring solution -that exports queue manager data to an InfluxDB data collection -system. It also contains configuration files to run the monitor program - -The monitor collects metrics published by an MQ V9 queue manager -or the MQ appliance. The monitor program pushes -those metrics into the database, over an HTTP connection, where -they can then be queried directly or used by other packages -such as Grafana. - -You can see data such as disk or CPU usage, queue depths, and MQI call -counts. - -An example Grafana dashboard is included, to show how queries might -be constructed. The data shown is the same as in the corresponding -Prometheus-based dashboard, also in this repository. -To use the dashboard, -create a data source in Grafana called "MQ Influx" that points at your -database server, and then import the JSON file. - -## Building -* This github repository contains both the monitoring program and -the ibmmq package that links to the core MQ application interface. It -also contains the mqmetric package used as a common component for -supporting alternative database collection protocols. - -* You also need access to the InfluxDB Go client interface. - - The command `go get -u github.com/influxdata/influxdb/client/v2` should pull - down the client code and its dependencies. - -* The error logger package may need to be explicitly downloaded - - On my system, I also had to forcibly download the logger package, - using `go get -u github.com/Sirupsen/logrus`. - -Run `go build -o /mq_influx cmd/mq_influx/*.go` to compile -the program and put it to a specific directory. - -## Configuring MQ -It is convenient to run the monitor program as a queue manager service. -This directory contains an MQSC script to define the service. In fact, the -service definition points at a simple script which sets up any -necessary environment and builds the command line parameters for the -real monitor program. As the last line of the script is "exec", the -process id of the script is inherited by the monitor program, and the -queue manager can then check on the status, and can drive a suitable -`STOP SERVICE` operation during queue manager shutdown. - -Edit the MQSC script and the shell script to point at appropriate directories -where the program exists, and where you want to put stdout/stderr. -Ensure that the ID running the queue manager has permission to access -the programs and output files. - -The monitor always collects all of the available queue manager-wide metrics. -It can also be configured to collect statistics for specific sets of queues. -The sets of queues can be given either directly on the command line with the -`-ibmmq.monitoredQueues` flag, or put into a separate file which is also -named on the command line, with the `ibmmq.monitoredQueuesFile` flag. An -example is included in the startup shell script. - -Note that **for now**, the queue patterns are expanded only at startup -of the monitor program. If you want to change the patterns, or new -queues are defined that match an existing pattern, the monitor must be -restarted with a `STOP SERVICE` and `START SERVICE` pair of commands. - -There are a number of required parameters to configure the service, including -the queue manager name, how to reach a database, and the frequency of reading -the queue manager publications. Look at the mq_influx.sh script or config.go -to see how to provide these parameters. - -In particular, if the database requires password authentication, then the password -is not provided as a command-line parameter, or read from the environment. It needs -to be given via a file named on the command line, which is then deleted -after being read. This indirection is done to make it easy to use the -shell "exec" function, keeping the process id of the real program available -to the queue manager's SERVICE definition. - -The queue manager will usually generate its publications every 10 seconds. That is also -the default interval being used in the monitor program to read those publications. - -## Configuring InfluxDB -No special configuration is required for InfluxDB. You may want to create an MQ-specific -database, and suitable userids to access that database. - -## Metrics -Once the monitor program has been started, -you will see metrics being available. -console. Two series of metrics are collected, "queue" and "qmgr". All of the queue -manager values are given a tag of the queue manager name; all of the queue-based values -are tagged with both the object and queue manager names. - -The example Grafana dashboard shows how queries can be constructed to extract data -about specific queues or the queue manager. - -More information on the metrics collected through the publish/subscribe -interface can be found in the [MQ KnowledgeCenter] -(https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.mon.doc/mo00013_.htm) -with further description in [an MQDev blog entry] -(https://www.ibm.com/developerworks/community/blogs/messaging/entry/Statistics_published_to_the_system_topic_in_MQ_v9?lang=en) - -The metrics stored in the Influx database are named after the -descriptions that you can see when running the amqsrua sample program, but with some -minor modifications to match a more useful style. diff --git a/cmd/mq_influx/config.go b/cmd/mq_influx/config.go deleted file mode 100755 index e7c4316..0000000 --- a/cmd/mq_influx/config.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "bufio" - "flag" - "os" - "strings" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type mqInfluxConfig struct { - qMgrName string - replyQ string - monitoredQueues string - monitoredQueuesFile string - - cc mqmetric.ConnectionConfig - - databaseName string - databaseAddress string - userid string - password string - passwordFile string - - interval string - maxErrors int - - logLevel string -} - -var config mqInfluxConfig - -/* -initConfig parses the command line parameters. -*/ -func initConfig() { - - flag.StringVar(&config.qMgrName, "ibmmq.queueManager", "", "Queue Manager name") - flag.StringVar(&config.replyQ, "ibmmq.replyQueue", "SYSTEM.DEFAULT.MODEL.QUEUE", "Reply Queue to collect data") - flag.StringVar(&config.monitoredQueues, "ibmmq.monitoredQueues", "", "Patterns of queues to monitor") - flag.StringVar(&config.monitoredQueuesFile, "ibmmq.monitoredQueuesFile", "", "File with patterns of queues to monitor") - - flag.BoolVar(&config.cc.ClientMode, "ibmmq.client", false, "Connect as MQ client") - - flag.StringVar(&config.databaseName, "ibmmq.databaseName", "", "Name of database") - flag.StringVar(&config.databaseAddress, "ibmmq.databaseAddress", "", "Address of database eg http://example.com:8086") - flag.StringVar(&config.userid, "ibmmq.databaseUserID", "", "UserID to access the database") - flag.StringVar(&config.interval, "ibmmq.interval", "10", "How many seconds between each collection") - flag.StringVar(&config.passwordFile, "ibmmq.pwFile", "", "Where is password help temporarily") - flag.IntVar(&config.maxErrors, "ibmmq.maxErrors", 100, "Maximum number of errors communicating with server before considered fatal") - - flag.StringVar(&config.logLevel, "log.level", "error", "Log level - debug, info, error") - - flag.Parse() - - if config.monitoredQueuesFile != "" { - config.monitoredQueues = mqmetric.ReadPatterns(config.monitoredQueuesFile) - } - - // Read password from a file if there is a userid on the command line - // Delete the file after reading it. - if config.userid != "" { - config.userid = strings.TrimSpace(config.userid) - - f, err := os.Open(config.passwordFile) - if err != nil { - log.Fatalf("Opening file %s: %s", f, err) - } - - defer os.Remove(config.passwordFile) - defer f.Close() - - scanner := bufio.NewScanner(f) - scanner.Scan() - p := scanner.Text() - err = scanner.Err() - if err != nil { - log.Fatalf("Reading file %s: %s", f, err) - } - config.password = strings.TrimSpace(p) - } -} diff --git a/cmd/mq_influx/exporter.go b/cmd/mq_influx/exporter.go deleted file mode 100755 index a4c2868..0000000 --- a/cmd/mq_influx/exporter.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -/* -This file pushes collected data to the InfluxDB. -The Collect() function is the key operation -invoked at the configured intervals, causing us to read available publications -and update the various data points. -*/ - -import ( - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" - client "github.com/influxdata/influxdb/client/v2" -) - -var ( - first = true - errorCount = 0 -) - -/* -Collect is called by the main routine at regular intervals to provide current -data -*/ -func Collect(c client.Client) error { - var err error - var series string - log.Infof("IBMMQ InfluxDB collection started") - - // Clear out everything we know so far. In particular, replace - // the map of values for each object so the collection starts - // clean. - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - elem.Values = make(map[string]int64) - } - } - } - - // Process all the publications that have arrived - mqmetric.ProcessPublications() - - // Have now processed all of the publications, and all the MQ-owned - // value fields and maps have been updated. - // - // Now need to set all of the real items with the correct values - if first { - // Always ignore the first loop through as there might - // be accumulated stuff from a while ago, and lead to - // a misleading range on graphs. - first = false - } else { - t := time.Now() - bp, _ := client.NewBatchPoints(client.BatchPointsConfig{ - Database: config.databaseName, - Precision: "ms", - }) - if err != nil { - // This kind of error is so unlikely, it should be treated as fatal - log.Fatalln("Error creating batch points: ", err) - } - - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - for key, value := range elem.Values { - f := mqmetric.Normalise(elem, key, value) - tags := map[string]string{ - "qmgr": config.qMgrName, - } - - series = "qmgr" - if key != mqmetric.QMgrMapKey { - tags["object"] = key - series = "queue" - } - fields := map[string]interface{}{elem.MetricName: f} - pt, _ := client.NewPoint(series, tags, fields, t) - bp.AddPoint(pt) - log.Debugf("Adding point %v", pt) - } - } - } - } - - // This is where real errors might occur, including the inability to - // contact the database server. We will ignore (but log) these errors - // up to a threshold, after which it is considered fatal. - err = c.Write(bp) - if err != nil { - log.Error(err) - errorCount++ - if errorCount >= config.maxErrors { - log.Fatal("Too many errors communicating with server") - } - } else { - errorCount = 0 - } - } - - return err - -} diff --git a/cmd/mq_influx/main.go b/cmd/mq_influx/main.go deleted file mode 100755 index 9f7c3d6..0000000 --- a/cmd/mq_influx/main.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "os" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -func initLog() { - level, err := log.ParseLevel(config.logLevel) - if err != nil { - level = log.InfoLevel - } - log.SetLevel(level) -} - -func main() { - var err error - var c client.Client - - initConfig() - if config.qMgrName == "" { - log.Errorln("Must provide a queue manager name to connect to.") - os.Exit(1) - } - d, err := time.ParseDuration(config.interval + "s") - if err != nil { - log.Errorln("Invalid value for interval parameter: ", err) - os.Exit(1) - } - - initLog() - log.Infoln("Starting IBM MQ metrics exporter for InfluxDB monitoring") - - // Connect and open standard queues - err = mqmetric.InitConnection(config.qMgrName, config.replyQ, &config.cc) - if err == nil { - log.Infoln("Connected to queue manager ", config.qMgrName) - defer mqmetric.EndConnection() - } - - // What metrics can the queue manager provide? Find out, and - // subscribe. - if err == nil { - err = mqmetric.DiscoverAndSubscribe(config.monitoredQueues, true, "") - } - - // Go into main loop for sending data to database - // Creating the client is not likely to have an error; the error will - // come during the write of the data. - if err == nil { - c, err = client.NewHTTPClient(client.HTTPConfig{ - Addr: config.databaseAddress, - Username: config.userid, - Password: config.password, - }) - - if err != nil { - log.Error(err) - } else { - for { - Collect(c) - time.Sleep(d) - } - } - - } - - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} diff --git a/cmd/mq_influx/mq_influx.bat b/cmd/mq_influx/mq_influx.bat deleted file mode 100644 index 3e61ab6..0000000 --- a/cmd/mq_influx/mq_influx.bat +++ /dev/null @@ -1,31 +0,0 @@ - -set qMgr=V90_W - - -set queues="APP.*,MYQ.*" - - - -# And other parameters that may be needed -# See config.go for all recognised flags -set database=MQDB -set userid=admin -set password=admin -set passwordFile=c:\temp\mqinfluxpw.txt -set svr=http://klein.hursley.ibm.com:8086 -set interval="10" - -set ARGS=-ibmmq.queueManager=%qMgr% -set ARGS=%ARGS% -ibmmq.databaseName=%database% -set ARGS=%ARGS% -ibmmq.databaseAddress=%svr% -set ARGS=%ARGS% -ibmmq.databaseUserID=%userid% -set ARGS=%ARGS% -ibmmq.interval=%interval% -set ARGS=%ARGS% -ibmmq.monitoredQueues=%queues% -set ARGS=%ARGS% -ibmmq.pwFile=%passwordFile% -set ARGS=%ARGS% -log.level=info - - - -del %passwordFile% -echo %password% > %passwordFile% -c:\gowork\bin\mq_influx %ARGS% diff --git a/cmd/mq_influx/mq_influx.mqsc b/cmd/mq_influx/mq_influx.mqsc deleted file mode 100755 index 02c0f27..0000000 --- a/cmd/mq_influx/mq_influx.mqsc +++ /dev/null @@ -1,18 +0,0 @@ -* Cleanup any existing service -STOP SERVICE(MQINFLUX) -DELETE SERVICE(MQINFLUX) - -* Reset the definition -DEFINE SERVICE(MQINFLUX) + - CONTROL(QMGR) + - SERVTYPE(SERVER) + - STARTCMD('/usr/local/bin/mqgo/mq_influx.sh') + - STARTARG(+QMNAME+) + - STOPCMD('/usr/bin/kill ' ) + - STOPARG(+MQ_SERVER_PID+) + - STDOUT('/var/mqm/errors/influx.out') + - STDERR('/var/mqm/errors/influx.out') + - DESCR('MQ exporter for InfluxDB') - -* Start it manually now; will be automatically started on future qmgr startup -START SERVICE(MQINFLUX) diff --git a/cmd/mq_influx/mq_influx.sh b/cmd/mq_influx/mq_influx.sh deleted file mode 100755 index dd1c519..0000000 --- a/cmd/mq_influx/mq_influx.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# This is used to start the IBM MQ monitoring service for InfluxDB - -# The queue manager name comes in from the service definition as the -# only command line parameter -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of queues to be monitored is given here. -# It is a set of names or patterns ('*' only at the end, to match how MQ works), -# separated by commas. When no queues match a pattern, it is reported but -# is not fatal. -queues="APP.*,MYQ.*" - -# An alternative is to have a file containing the patterns, and named -# via the ibmmq.monitoredQueuesFile option. - -# And other parameters that may be needed -# See config.go for all recognised flags -database="MQDB" -userid="admin" -password="admin" # Probably get from an environment variable in reality -passwordFile="/tmp/mqinfluxpw.$$.txt" -svr="http://klein.hursley.ibm.com:8086" -interval="10" - -ARGS="-ibmmq.queueManager=$qMgr" -ARGS="$ARGS -ibmmq.databaseName=$database" -ARGS="$ARGS -ibmmq.databaseAddress=$svr" -ARGS="$ARGS -ibmmq.databaseUserID=$userid" -ARGS="$ARGS -ibmmq.interval=$interval" -ARGS="$ARGS -ibmmq.monitoredQueues=$queues" -ARGS="$ARGS -ibmmq.pwFile=$passwordFile" -ARGS="$ARGS -log.level=error" - -# Start via "exec" so the pid remains the same. The queue manager can -# then check the existence of the service and use the MQ_SERVER_PID value -# to kill it on shutdown. -# Using exec makes it harder to use stdin redirect, hence the use of -# a file to hold a password. The program will delete the file immediately -# after reading it. - -rm -f $passwordFile -umask 077 -echo $password > $passwordFile -exec /usr/local/bin/mqgo/mq_influx $ARGS diff --git a/cmd/mq_json/README.md b/cmd/mq_json/README.md deleted file mode 100755 index 366b15f..0000000 --- a/cmd/mq_json/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# MQ Exporter for JSON-based monitoring - -This directory contains the code for a monitoring solution -that prints queue manager data in JSON format. -It also contains configuration files to run the monitor program - -The monitor collects metrics published by an MQ V9 queue manager -or the MQ appliance. The monitor program prints -these metrics to stdout. - -You can see data such as disk or CPU usage, queue depths, and MQI call -counts. - -## Building -* This github repository contains both the monitoring program and -the ibmmq package that links to the core MQ application interface. It -also contains the mqmetric package used as a common component for -supporting alternative database collection protocols. - -* Get the error logger package used by all of these monitors -using `go get -u github.com/Sirupsen/logrus`. - -Run `go build -o /mq_json cmd/mq_json/*.go` to compile -the program and put it to a specific directory. - -## Configuring MQ -It is convenient to run the monitor program as a queue manager service. -This directory contains an MQSC script to define the service. In fact, the -service definition points at a simple script which sets up any -necessary environment and builds the command line parameters for the -real monitor program. As the last line of the script is "exec", the -process id of the script is inherited by the monitor program, and the -queue manager can then check on the status, and can drive a suitable -`STOP SERVICE` operation during queue manager shutdown. - -Edit the MQSC script to point at appropriate directories -where the program exists, and where you want to put stdout/stderr. -Ensure that the ID running the queue manager has permission to access -the programs and output files. - -Since the output from the monitor is always sent to stdout, you will -probably want to modify the script to pipe the output to a processing -program that works with JSON data, or to a program that automatically -creates and manages multiple log files. - -The monitor always collects all of the available queue manager-wide metrics. -It can also be configured to collect statistics for specific sets of queues. -The sets of queues can be given either directly on the command line with the -`-ibmmq.monitoredQueues` flag, or put into a separate file which is also -named on the command line, with the `ibmmq.monitoredQueuesFile` flag. An -example is included in the startup shell script. - -At each collection interval, a JSON object is printed, consisting of -a timestamp followed by an array of "points" which contain the -metric and the resource it refers to. - -For example, - { - "collectionTime" : { - "timeStamp" : "2016-11-07-T15:00:55Z" - "epoch" : 1478527255 - }, - "points" : [ - { "queueManager" : "QM1", "ramTotalBytes" : 15515735206 }, - { "queueManager" : "QM1", "userCpuTimePercentage" : 1.33 } - ] - } - - -## Metrics -Once the monitor program has been started, -you will see metrics being available. -More information on the metrics collected through the publish/subscribe -interface can be found in the [MQ KnowledgeCenter] -(https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.mon.doc/mo00013_.htm) -with further description in [an MQDev blog entry] -(https://www.ibm.com/developerworks/community/blogs/messaging/entry/Statistics_published_to_the_system_topic_in_MQ_v9?lang=en) - -The metrics printed are named after the -descriptions that you can see when running the amqsrua sample program, but with some -minor modifications to match a more useful style. diff --git a/cmd/mq_json/config.go b/cmd/mq_json/config.go deleted file mode 100755 index 6dd7f08..0000000 --- a/cmd/mq_json/config.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "flag" - - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type mqTTYConfig struct { - qMgrName string - replyQ string - monitoredQueues string - monitoredQueuesFile string - - cc mqmetric.ConnectionConfig - - interval string - - logLevel string -} - -var config mqTTYConfig - -/* -initConfig parses the command line parameters. -*/ -func initConfig() { - - flag.StringVar(&config.qMgrName, "ibmmq.queueManager", "", "Queue Manager name") - flag.StringVar(&config.replyQ, "ibmmq.replyQueue", "SYSTEM.DEFAULT.MODEL.QUEUE", "Reply Queue to collect data") - flag.StringVar(&config.monitoredQueues, "ibmmq.monitoredQueues", "", "Patterns of queues to monitor") - flag.StringVar(&config.monitoredQueuesFile, "ibmmq.monitoredQueuesFile", "", "File with patterns of queues to monitor") - - flag.StringVar(&config.interval, "ibmmq.interval", "10", "How many seconds between each collection") - - flag.BoolVar(&config.cc.ClientMode, "ibmmq.client", false, "Connect as MQ client") - - flag.StringVar(&config.logLevel, "log.level", "error", "Log level - debug, info, error") - - flag.Parse() - - if config.monitoredQueuesFile != "" { - config.monitoredQueues = mqmetric.ReadPatterns(config.monitoredQueuesFile) - } - -} diff --git a/cmd/mq_json/exporter.go b/cmd/mq_json/exporter.go deleted file mode 100755 index ac5e9af..0000000 --- a/cmd/mq_json/exporter.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -/* -The Collect() function is the key operation -invoked at the configured intervals, causing us to read available publications -and update the various data points. -*/ - -import ( - "fmt" - "math" - "strings" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -var ( - first = true - errorCount = 0 -) - -const ( - blankString = " " -) - -/* -Collect is called by the main routine at regular intervals to provide current -data -*/ -func Collect() error { - var err error - - log.Infof("IBM MQ JSON collector started") - - // Clear out everything we know so far. In particular, replace - // the map of values for each object so the collection starts - // clean. - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - elem.Values = make(map[string]int64) - } - } - } - - // Process all the publications that have arrived - mqmetric.ProcessPublications() - - // Have now processed all of the publications, and all the MQ-owned - // value fields and maps have been updated. - // - // Now need to set all of the real items with the correct values - if first { - // Always ignore the first loop through as there might - // be accumulated stuff from a while ago, and lead to - // a misleading range on graphs. - first = false - } else { - firstPoint := true - fmt.Printf("\n{\n") - fmt.Printf("%s\"collectionTime\" : {\n", blankString[0:2]) - t := time.Now() - fmt.Printf("%s\"timeStamp\" : \"%s\",\n", blankString[0:4], t.Format(time.RFC3339)) - fmt.Printf("%s\"epoch\" : %d\n", blankString[0:4], t.Unix()) - fmt.Printf("%s},\n", blankString[0:2]) - - fmt.Printf("%s\"points\" : [\n", blankString[0:2]) - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - for key, value := range elem.Values { - if !firstPoint { - fmt.Printf(",\n") - } else { - firstPoint = false - } - f := mqmetric.Normalise(elem, key, value) - tags := map[string]string{ - "qmgr": config.qMgrName, - } - - if key != mqmetric.QMgrMapKey { - tags["object"] = key - } - printPoint(elem.MetricName, float32(f), tags) - - } - } - } - } - fmt.Printf("\n%s]\n}\n", blankString[0:2]) - - } - - return err - -} - -func printPoint(metric string, val float32, tags map[string]string) { - fmt.Printf("%s{\n", blankString[0:2]) - qmgr := tags["qmgr"] - fmt.Printf("%s\"queueManager\" : \"%s\",\n", blankString[0:4], qmgr) - if q, ok := tags["object"]; ok { - fmt.Printf("%s\"queue\" : \"%s\",\n", blankString[0:4], q) - } - if float64(val) == math.Trunc(float64(val)) { - fmt.Printf("%s\"%s\" : %d\n", blankString[0:4], fixup(metric), int64(val)) - } else { - fmt.Printf("%s\"%s\" : %f\n", blankString[0:4], fixup(metric), val) - } - fmt.Printf("%s}", blankString[0:2]) - return -} - -func fixup(s1 string) string { - // Another reformatting of the metric name - this one converts - // something like queue_avoided_bytes into queueAvoidedBytes - s2 := "" - c := "" - nextCaseUpper := false - - for i := 0; i < len(s1); i++ { - if s1[i] != '_' { - if nextCaseUpper { - c = strings.ToUpper(s1[i : i+1]) - nextCaseUpper = false - } else { - c = strings.ToLower(s1[i : i+1]) - } - s2 += c - } else { - nextCaseUpper = true - } - - } - return s2 -} diff --git a/cmd/mq_json/main.go b/cmd/mq_json/main.go deleted file mode 100755 index 6d86cfc..0000000 --- a/cmd/mq_json/main.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "os" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -func initLog() { - level, err := log.ParseLevel(config.logLevel) - if err != nil { - level = log.InfoLevel - } - log.SetLevel(level) - // Since this program prints its data to stdout, need any - // log info to go elsewhere. - log.SetOutput(os.Stderr) -} - -func main() { - var err error - - initConfig() - if config.qMgrName == "" { - log.Errorln("Must provide a queue manager name to connect to.") - os.Exit(1) - } - d, err := time.ParseDuration(config.interval + "s") - if err != nil { - log.Errorln("Invalid value for interval parameter: ", err) - os.Exit(1) - } - - initLog() - log.Infoln("Starting IBM MQ metrics exporter for JSON") - - // Connect and open standard queues - err = mqmetric.InitConnection(config.qMgrName, config.replyQ, &config.cc) - if err == nil { - log.Infoln("Connected to queue manager ", config.qMgrName) - defer mqmetric.EndConnection() - } - - // What metrics can the queue manager provide? Find out, and - // subscribe. - if err == nil { - err = mqmetric.DiscoverAndSubscribe(config.monitoredQueues, true, "") - } - - // Go into main loop for sending data to stdout - // This program runs forever - if err == nil { - for { - Collect() - time.Sleep(d) - } - - } - - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} diff --git a/cmd/mq_json/mq_json.mqsc b/cmd/mq_json/mq_json.mqsc deleted file mode 100755 index 2f0562a..0000000 --- a/cmd/mq_json/mq_json.mqsc +++ /dev/null @@ -1,18 +0,0 @@ -* Cleanup any existing service -STOP SERVICE(MQJSON) -DELETE SERVICE(MQJSON) - -* Reset the definition -DEFINE SERVICE(MQJSON) + - CONTROL(QMGR) + - SERVTYPE(SERVER) + - STARTCMD('/usr/local/bin/mqgo/mq_json.sh') + - STARTARG(+QMNAME+) + - STOPCMD('/usr/bin/kill ' ) + - STOPARG(+MQ_SERVER_PID+) + - STDOUT('/var/mqm/errors/json.out') + - STDERR('/var/mqm/errors/json.out') + - DESCR('MQ exporter for JSON format') - -* Start it manually now; will be automatically started on future qmgr startup -START SERVICE(MQJSON) diff --git a/cmd/mq_json/mq_json.sh b/cmd/mq_json/mq_json.sh deleted file mode 100755 index 1147645..0000000 --- a/cmd/mq_json/mq_json.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh - -# This is used to start an IBM MQ monitoring service for JSON-formatted output - -# The queue manager name comes in from the MQ service configuration as -# the only command line parameter. -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -# This assumes there is an MQ installation in the default location, even -# if it is not the one associated with the queue manager -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of queues to be monitored is given here. -# It is a set of names or patterns ('*' only at the end, to match how MQ works), -# separated by commas. When no queues match a pattern, it is reported but -# is not fatal. -queues="APP.*,MYQ.*" - -# An alternative is to have a file containing the patterns, and named -# via the ibmmq.monitoredQueuesFile option. - -# And other parameters that may be needed -# See config.go for all recognised flags -interval="5" - -ARGS="-ibmmq.queueManager=$qMgr" -ARGS="$ARGS -ibmmq.monitoredQueues=$queues" -ARGS="$ARGS -ibmmq.interval=$interval" -ARGS="$ARGS -log.level=error" - -# Start via "exec" so the pid remains the same. -# -# This program will send all log info to stderr -# -# Change this line to match wherever you have installed the MQ monitor program -# You probably also want to do something with the stdout from the program, -# such as sending it to a monitoring solution that understands the format. -exec /usr/local/bin/mqgo/mq_json $ARGS diff --git a/cmd/mq_opentsdb/MQ OpenTSDB-1470723491393.json b/cmd/mq_opentsdb/MQ OpenTSDB-1470723491393.json deleted file mode 100755 index 0118f23..0000000 --- a/cmd/mq_opentsdb/MQ OpenTSDB-1470723491393.json +++ /dev/null @@ -1,688 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_MQ_OPENTSDB", - "label": "MQ OpenTSDB", - "description": "", - "type": "datasource", - "pluginId": "opentsdb", - "pluginName": "OpenTSDB" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "opentsdb", - "name": "OpenTSDB", - "version": "1.0.0" - } - ], - "id": null, - "title": "MQ OpenTSDB", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_OPENTSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "sum", - "alias": "MQGET: [[tag_object]]", - "currentFilterGroupBy": false, - "currentFilterKey": "", - "currentFilterType": "literal_or", - "currentFilterValue": "", - "currentTagKey": "object", - "currentTagValue": "APP.0", - "disableDownsampling": true, - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "downsampleInterval": "", - "filters": [ - { - "filter": "APP.*", - "groupBy": true, - "tagk": "object", - "type": "wildcard" - } - ], - "hide": false, - "isCounter": true, - "metric": "queue.mqget", - "refId": "A", - "shouldComputeRate": false - }, - { - "aggregator": "sum", - "alias": "MQPUT: [[tag_object]]", - "currentFilterGroupBy": false, - "currentFilterKey": "", - "currentFilterType": "literal_or", - "currentFilterValue": "", - "currentTagKey": "object", - "currentTagValue": "APP.0", - "disableDownsampling": true, - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "downsampleInterval": "", - "filters": [ - { - "filter": "APP.*", - "groupBy": true, - "tagk": "object", - "type": "wildcard" - } - ], - "hide": false, - "isCounter": true, - "metric": "queue.mqput_mqput1", - "refId": "B", - "shouldComputeRate": false - }, - { - "aggregator": "avg", - "alias": "Depth: [[tag_object]]", - "currentFilterGroupBy": false, - "currentFilterKey": "", - "currentFilterType": "literal_or", - "currentFilterValue": "", - "currentTagKey": "object", - "currentTagValue": "APP.0", - "disableDownsampling": true, - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "downsampleInterval": "", - "filters": [ - { - "filter": "APP.*", - "groupBy": true, - "tagk": "object", - "type": "wildcard" - } - ], - "hide": false, - "isCounter": true, - "metric": "queue.queue_depth", - "refId": "C", - "shouldComputeRate": false - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Queue Activity", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": true, - "title": "MQ OpenTSDB Capture" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_OPENTSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "System CPU%", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.system_cpu_time_percentage", - "refId": "A" - }, - { - "aggregator": "avg", - "alias": "User CPU%", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.user_cpu_time_percentage", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_OPENTSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "Latency", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.log_write_latency_seconds", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Log write latency", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_OPENTSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "QMgr file system - in use [$tag_qmgr]", - "disableDownsampling": true, - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.queue_manager_file_system_in_use_bytes", - "refId": "A" - }, - { - "aggregator": "sum", - "alias": "Log file system - in use [$tag_qmgr]", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.log_file_system_in_use_bytes", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "File system", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_OPENTSDB}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 6, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "alias": "ND Sub - low-water", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.non_durable_subscriber_low_water_mark", - "refId": "B" - }, - { - "aggregator": "avg", - "alias": "ND Sub - high-water", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.non_durable_subscriber_high_water_mark", - "refId": "A" - }, - { - "aggregator": "sum", - "alias": "ND MQSUB calls", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.create_non_durable_subscription", - "refId": "C" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Subscriptions", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_MQ_OPENTSDB}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 7, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 6, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "avg", - "alias": "Persistent bytes PUT", - "downsampleAggregator": "avg", - "downsampleFillPolicy": "none", - "metric": "qmgr.put_persistent_messages_bytes", - "refId": "A" - } - ], - "thresholds": "", - "title": "Persistent Bytes Put", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": "10s", - "schemaVersion": 12, - "version": 10, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/cmd/mq_opentsdb/README.md b/cmd/mq_opentsdb/README.md deleted file mode 100755 index 8db28bd..0000000 --- a/cmd/mq_opentsdb/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# MQ Exporter for OpenTSDB monitoring - -This directory contains the code for a monitoring solution -that exports queue manager data to an OpenTSDB data collection -system. It also contains configuration files to run the monitor program - -The monitor collects metrics published by an MQ V9 queue manager -or the MQ appliance. The monitor program pushes -those metrics into the database, over an HTTP connection, where -they can then be queried directly or used by other packages -such as Grafana. - -You can see data such as disk or CPU usage, queue depths, and MQI call -counts. - -An example Grafana dashboard is included, to show how queries might -be constructed. The data shown is the same as in the corresponding -Prometheus and InfluxDB-based dashboards, also in this repository. -To use the dashboard, -create a data source in Grafana called "MQ OpenTSDB" that points at your -database server, and then import the JSON file. - -## Building -* This github repository contains both the monitoring program and -the ibmmq package that links to the core MQ application interface. It -also contains the mqmetric package used as a common component for -supporting alternative database collection protocols. - -* The error logger package may need to be explicitly downloaded - - On my system, I also had to forcibly download the logger package, - using `go get -u github.com/Sirupsen/logrus`. - -Run `go build -o /mq_opentsdb cmd/mq_opentsdb/*.go` to compile -the program and put it to a specific directory. - -## Configuring MQ -It is convenient to run the monitor program as a queue manager service. -This directory contains an MQSC script to define the service. In fact, the -service definition points at a simple script which sets up any -necessary environment and builds the command line parameters for the -real monitor program. As the last line of the script is "exec", the -process id of the script is inherited by the monitor program, and the -queue manager can then check on the status, and can drive a suitable -`STOP SERVICE` operation during queue manager shutdown. - -Edit the MQSC script and the shell script to point at appropriate directories -where the program exists, and where you want to put stdout/stderr. -Ensure that the ID running the queue manager has permission to access -the programs and output files. - -The monitor always collects all of the available queue manager-wide metrics. -It can also be configured to collect statistics for specific sets of queues. -The sets of queues can be given either directly on the command line with the -`-ibmmq.monitoredQueues` flag, or put into a separate file which is also -named on the command line, with the `ibmmq.monitoredQueuesFile` flag. An -example is included in the startup shell script. - -Note that **for now**, the queue patterns are expanded only at startup -of the monitor program. If you want to change the patterns, or new -queues are defined that match an existing pattern, the monitor must be -restarted with a `STOP SERVICE` and `START SERVICE` pair of commands. - -There are a number of required parameters to configure the service, including -the queue manager name, how to reach a database, and the frequency of reading -the queue manager publications. Look at the mq_opentsdb.sh script or config.go -to see how to provide these parameters. - -In particular, if the database requires password authentication, then the password -is not provided as a command-line parameter, or read from the environment. It needs -to be given to the command via a file; while the shell script has a hardcoded password -which is sent to the real command, you may prefer to have an alternative mechanism to -discover that password. - -The queue manager will usually generate its publications every 10 seconds. That is also -the default interval being used in the monitor program to read those publications. - -## Configuring OpenTSDB -No special configuration is required for the database. - -## Metrics -Once the monitor program has been started, -you will see metrics being available. -console. Two series of metrics are collected, "queue" and "qmgr". All of the queue -manager values are given a tag of the queue manager name; all of the queue-based values -are tagged with both the queue and queue manager names. The queue name is actually -given by the "object" tag; that is to simplify things if the queue manager -ever generates this kind of statistics for other object types such as topics. - -The example Grafana dashboard shows how queries can be constructed to extract data -about specific queues or the queue manager. - -More information on the metrics collected through the publish/subscribe -interface can be found in the [MQ KnowledgeCenter] -(https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.mon.doc/mo00013_.htm) -with further description in [an MQDev blog entry] -(https://www.ibm.com/developerworks/community/blogs/messaging/entry/Statistics_published_to_the_system_topic_in_MQ_v9?lang=en) - -The metrics stored in the database are named after the -descriptions that you can see when running the amqsrua sample program, but with some -minor modifications to match a more useful style. diff --git a/cmd/mq_opentsdb/config.go b/cmd/mq_opentsdb/config.go deleted file mode 100755 index 7cc9001..0000000 --- a/cmd/mq_opentsdb/config.go +++ /dev/null @@ -1,104 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "bufio" - "flag" - "os" - "strings" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type mqOpenTSDBConfig struct { - qMgrName string - replyQ string - monitoredQueues string - monitoredQueuesFile string - - cc mqmetric.ConnectionConfig - - databaseName string - databaseAddress string - userid string - password string - passwordFile string - - interval string - maxErrors int - maxPoints int - - logLevel string -} - -var config mqOpenTSDBConfig - -/* -initConfig parses the command line parameters. -*/ -func initConfig() { - - flag.StringVar(&config.qMgrName, "ibmmq.queueManager", "", "Queue Manager name") - flag.StringVar(&config.replyQ, "ibmmq.replyQueue", "SYSTEM.DEFAULT.MODEL.QUEUE", "Reply Queue to collect data") - flag.StringVar(&config.monitoredQueues, "ibmmq.monitoredQueues", "", "Patterns of queues to monitor") - flag.StringVar(&config.monitoredQueuesFile, "ibmmq.monitoredQueuesFile", "", "File with patterns of queues to monitor") - - flag.BoolVar(&config.cc.ClientMode, "ibmmq.client", false, "Connect as MQ client") - - flag.StringVar(&config.databaseName, "ibmmq.databaseName", "", "Name of database") - flag.StringVar(&config.databaseAddress, "ibmmq.databaseAddress", "", "Address of database eg http://example.com:8086") - flag.StringVar(&config.userid, "ibmmq.databaseUserID", "", "UserID to access the database") - flag.StringVar(&config.passwordFile, "ibmmq.pwFile", "", "Where is password help temporarily") - flag.StringVar(&config.interval, "ibmmq.interval", "10", "How many seconds between each collection") - flag.IntVar(&config.maxErrors, "ibmmq.maxErrors", 10000, "Maximum number of errors communicating with server before considered fatal") - flag.IntVar(&config.maxPoints, "ibmmq.maxPoints", 30, "Maximum number of points to include in each write to the server") - - flag.StringVar(&config.logLevel, "log.level", "error", "Log level - debug, info, error") - - flag.Parse() - - if config.monitoredQueuesFile != "" { - config.monitoredQueues = mqmetric.ReadPatterns(config.monitoredQueuesFile) - } - - // Read password from a file if there is a userid on the command line - // Delete the file after reading it. - if config.userid != "" { - config.userid = strings.TrimSpace(config.userid) - - f, err := os.Open(config.passwordFile) - if err != nil { - log.Fatalf("Opening file %s: %s", f, err) - } - - defer os.Remove(config.passwordFile) - defer f.Close() - - scanner := bufio.NewScanner(f) - scanner.Scan() - p := scanner.Text() - err = scanner.Err() - if err != nil { - log.Fatalf("Reading file %s: %s", f, err) - } - config.password = strings.TrimSpace(p) - } -} diff --git a/cmd/mq_opentsdb/exporter.go b/cmd/mq_opentsdb/exporter.go deleted file mode 100755 index 3df83bc..0000000 --- a/cmd/mq_opentsdb/exporter.go +++ /dev/null @@ -1,207 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -/* -This file pushes collected data to OpenTSDB. -The Collect() function is the key operation -invoked at the configured intervals, causing us to read available publications -and update the various data points. -*/ - -import ( - "bytes" - "io/ioutil" - "net/http" - "net/url" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type client struct { - url *url.URL - httpClient *http.Client - tr *http.Transport -} - -var ( - first = true - errorCount = 0 - c *client -) - -/* -Collect is called by the main routine at regular intervals to provide current -data -*/ -func Collect() error { - var err error - var series string - log.Infof("IBM MQ OpenTSDB collection started") - - if c == nil { - c, err = newClient() - } - - // Clear out everything we know so far. In particular, replace - // the map of values for each object so the collection starts - // clean. - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - elem.Values = make(map[string]int64) - } - } - } - - // Process all the publications that have arrived - mqmetric.ProcessPublications() - - // Have now processed all of the publications, and all the MQ-owned - // value fields and maps have been updated. - // - // Now need to set all of the real items with the correct values - if first { - // Always ignore the first loop through as there might - // be accumulated stuff from a while ago, and lead to - // a misleading range on graphs. - first = false - } else { - t := time.Now().Unix() - bp := newBatchPoints() - - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - for key, value := range elem.Values { - f := mqmetric.Normalise(elem, key, value) - tags := map[string]string{ - "qmgr": config.qMgrName, - } - - series = "qmgr" - if key != mqmetric.QMgrMapKey { - tags["object"] = key - series = "queue" - } - pt, _ := newPoint(series+"."+elem.MetricName, t, float32(f), tags) - bp.addPoint(pt) - - // OpenTSDB recommends not sending too many - // data points in a single request. Large requests - // may require http chunking which is disabled by default - // in the database. So we flush after a configurable set has been - // collected. - if len(bp.Points) >= config.maxPoints { - bp = c.Flush(bp) - } - //log.Debugf("Adding point %v", pt) - } - } - } - } - - c.Flush(bp) - } - - return err - -} - -func (c *client) Flush(bp *BatchPoints) *BatchPoints { - // This is where real errors might occur, including the inability to - // contact the database server. We will ignore (but log) these errors - // up to a threshold, after which it is considered fatal. - if len(bp.Points) > 0 { - _, err := c.Put(bp, "details") - if err != nil { - log.Error(err) - errorCount++ - if errorCount >= config.maxErrors { - log.Fatal("Too many errors communicating with server") - } - } else { - errorCount = 0 - } - } - bp = newBatchPoints() - return bp -} - -func newClient() (*client, error) { - - tr := &http.Transport{} - u, err := url.Parse(config.databaseAddress) - if err != nil { - return nil, err - } - - return &client{ - url: u, - httpClient: &http.Client{ - Timeout: 0, - Transport: tr, - }, - tr: tr, - }, nil -} - -func (c *client) Close() error { - c.tr.CloseIdleConnections() - return nil -} - -func (c *client) Put(bp *BatchPoints, params string) ([]byte, error) { - if len(bp.Points) == 0 { - return nil, nil - } - - data, err := bp.toJSON() - if err != nil { - return nil, err - } - log.Debugf("Serialised points are %s", string(data)) - - u := c.url - u.Path = "api/put" - u.RawQuery = params - - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(data)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - // log.Infof("Request is %v", req) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - log.Infoln("Response body: ", string(body)) - return body, nil -} diff --git a/cmd/mq_opentsdb/main.go b/cmd/mq_opentsdb/main.go deleted file mode 100755 index 3a55f28..0000000 --- a/cmd/mq_opentsdb/main.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "os" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -func initLog() { - level, err := log.ParseLevel(config.logLevel) - if err != nil { - level = log.InfoLevel - } - log.SetLevel(level) -} - -func main() { - var err error - - initConfig() - if config.qMgrName == "" { - log.Errorln("Must provide a queue manager name to connect to.") - os.Exit(1) - } - d, err := time.ParseDuration(config.interval + "s") - if err != nil { - log.Errorln("Invalid value for interval parameter: ", err) - os.Exit(1) - } - - initLog() - log.Infoln("Starting IBM MQ metrics exporter for OpenTSDB monitoring") - - // Connect and open standard queues - err = mqmetric.InitConnection(config.qMgrName, config.replyQ, &config.cc) - if err == nil { - log.Infoln("Connected to queue manager ", config.qMgrName) - defer mqmetric.EndConnection() - } - - // What metrics can the queue manager provide? Find out, and - // subscribe. - if err == nil { - err = mqmetric.DiscoverAndSubscribe(config.monitoredQueues, true, "") - } - - // Go into main loop for sending data to database - if err == nil { - for { - Collect() - time.Sleep(d) - } - - } - - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} diff --git a/cmd/mq_opentsdb/mq_opentsdb.mqsc b/cmd/mq_opentsdb/mq_opentsdb.mqsc deleted file mode 100755 index 4369f4e..0000000 --- a/cmd/mq_opentsdb/mq_opentsdb.mqsc +++ /dev/null @@ -1,18 +0,0 @@ -* Cleanup any existing service -STOP SERVICE(MQOPENTSDB) -DELETE SERVICE(MQOPENTSDB) - -* Reset the definition -DEFINE SERVICE(MQOPENTSDB) + - CONTROL(QMGR) + - SERVTYPE(SERVER) + - STARTCMD('/usr/local/bin/mqgo/mq_opentsdb.sh') + - STARTARG(+QMNAME+) + - STOPCMD('/usr/bin/kill ' ) + - STOPARG(+MQ_SERVER_PID+) + - STDOUT('/var/mqm/errors/opentsdb.out') + - STDERR('/var/mqm/errors/opentsdb.out') + - DESCR('MQ exporter for OpenTSDB') - -* Start it manually now; will be automatically started on future qmgr startup -START SERVICE(MQOPENTSDB) diff --git a/cmd/mq_opentsdb/mq_opentsdb.sh b/cmd/mq_opentsdb/mq_opentsdb.sh deleted file mode 100755 index 4c9eb64..0000000 --- a/cmd/mq_opentsdb/mq_opentsdb.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -# This is used to start the IBM MQ monitoring service for OpenTSDB - -# The queue manager name comes in from the service definition as the -# only command line parameter -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of queues to be monitored is given here. -# It is a set of names or patterns ('*' only at the end, to match how MQ works), -# separated by commas. When no queues match a pattern, it is reported but -# is not fatal. -queues="APP.*,MYQ.*" - -# An alternative is to have a file containing the patterns, and named -# via the ibmmq.monitoredQueuesFile option. - -# And other parameters that may be needed -# See config.go for all recognised flags -userid="admin" -password="admin" # Probably get from an environment variable in reality -passwordFile="/tmp/mqopentsdbpw.$$.txt" -svr="http://klein.hursley.ibm.com:4242" -interval="5" - -ARGS="-ibmmq.queueManager=$qMgr" -ARGS="$ARGS -ibmmq.databaseAddress=$svr" -ARGS="$ARGS -ibmmq.databaseUserID=$userid" -ARGS="$ARGS -ibmmq.interval=$interval" -ARGS="$ARGS -ibmmq.monitoredQueues=$queues" -ARGS="$ARGS -ibmmq.pwFile=$passwordFile" -ARGS="$ARGS -log.level=error" - -# Start via "exec" so the pid remains the same. The queue manager can -# then check the existence of the service and use the MQ_SERVER_PID value -# to kill it on shutdown. -# Using exec makes it harder to use stdin redirect, hence the use of -# a file to hold a password. The program will delete the file immediately -# after reading it. - -rm -f $passwordFile -umask 077 -echo $password > $passwordFile -exec /usr/local/bin/mqgo/mq_opentsdb $ARGS diff --git a/cmd/mq_opentsdb/points.go b/cmd/mq_opentsdb/points.go deleted file mode 100755 index 441796c..0000000 --- a/cmd/mq_opentsdb/points.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "encoding/json" - "errors" - _ "github.com/Sirupsen/logrus" -) - -/* -Point contains the elements needed for a single entry in the -database. -*/ -type Point struct { - // Opentsdb metric name - Metric string `json:"metric"` - - // Timestamp unix time e.g.: time.Now().Unix() - Timestamp int64 `json:"timestamp"` - - Value float32 `json:"value"` - - // Map of tags, example: {"host": "desktop"} - Tags map[string]string `json:"tags"` -} - -func newPoint(metric string, timestamp int64, value float32, tags map[string]string) (*Point, error) { - if metric == "" { - return nil, errors.New("PointError: Metric can not be empty") - } - - return &Point{ - Metric: metric, - Timestamp: timestamp, - Value: value, - Tags: tags, - }, nil -} - -/* -BatchPoints is the set of points collected in one iteration. -*/ -type BatchPoints struct { - Points []*Point `json:""` -} - -func newBatchPoints() *BatchPoints { - return &BatchPoints{} -} - -func (bp *BatchPoints) addPoint(p *Point) { - bp.Points = append(bp.Points, p) -} - -func (bp *BatchPoints) toJSON() ([]byte, error) { - j, err := json.Marshal(bp.Points) - //log.Debug("Points set = ", string(j)) - return j, err -} diff --git a/cmd/mq_prometheus/MQ Prometheus-1470081511600.json b/cmd/mq_prometheus/MQ Prometheus-1470081511600.json deleted file mode 100755 index 81bc7b2..0000000 --- a/cmd/mq_prometheus/MQ Prometheus-1470081511600.json +++ /dev/null @@ -1,634 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_MQ_PROMETHEUS", - "label": "MQ Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "id": null, - "title": "MQ Prometheus", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ibmmq_object_mqget{object=~\"APP.*\"}", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MQGET: {{object}}", - "metric": "", - "refId": "A", - "step": 2 - }, - { - "expr": "ibmmq_object_mqput_mqput1{object=~\"APP.*\"}", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MQPUT: {{object}}", - "metric": "", - "refId": "B", - "step": 2 - }, - { - "expr": "ibmmq_object_queue_depth{object=~\"APP.*\"}", - "intervalFactor": 2, - "legendFormat": "Depth: {{object}}", - "refId": "C", - "step": 2 - } - ], - "timeFrom": "15m", - "timeShift": null, - "title": "Queue Activity", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": true, - "title": "MQ Prometheus Capture" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ibmmq_qmgr_system_cpu_time_percentage", - "intervalFactor": 2, - "legendFormat": "System CPU%", - "refId": "A", - "step": 2 - }, - { - "expr": "ibmmq_qmgr_user_cpu_time_percentage", - "intervalFactor": 2, - "legendFormat": "User CPU%", - "refId": "B", - "step": 2 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ibmmq_qmgr_log_write_latency_seconds", - "intervalFactor": 2, - "legendFormat": "Latency", - "metric": "ibmmq_qmgr_log_write_latency_us", - "refId": "A", - "step": 2 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Log write latency", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_PROMETHEUS}", - "decimals": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ibmmq_qmgr_queue_manager_file_system_in_use_bytes", - "intervalFactor": 2, - "legendFormat": "QMgr file system - in use [{{qmgr}}]", - "metric": "ibmmq_qmgr_queue_manager_file_system_bytes_in_use_MB", - "refId": "A", - "step": 2 - }, - { - "expr": "ibmmq_qmgr_log_file_system_in_use_bytes", - "intervalFactor": 2, - "legendFormat": "Log file system - in use [{{qmgr}}]", - "metric": "ibmmq_qmgr_log_file_system_in_use_bytes", - "refId": "B", - "step": 2 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "File system", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MQ_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ibmmq_qmgr_non_durable_subscriber_low_water_mark", - "intervalFactor": 2, - "legendFormat": "ND Sub - low-water", - "metric": "ibmmq_qmgr_non_durable_subscriber_low_water_mark", - "refId": "A", - "step": 2 - }, - { - "expr": "ibmmq_qmgr_non_durable_subscriber_high_water_mark", - "intervalFactor": 2, - "legendFormat": "ND Sub - high-water", - "metric": "ibmmq_qmgr_non_durable_subscriber_high_water_mark", - "refId": "B", - "step": 2 - }, - { - "expr": "ibmmq_qmgr_create_non_durable_subscription", - "intervalFactor": 2, - "legendFormat": "ND MQSUB calls", - "metric": "ibmmq_qmgr_create_non_durable_subscription", - "refId": "C", - "step": 2 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Subscriptions", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_MQ_PROMETHEUS}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 6, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 6, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum(ibmmq_qmgr_put_persistent_messages_bytes)", - "intervalFactor": 2, - "legendFormat": "Persistent bytes PUT", - "metric": "ibmmq_qmgr_put_persistent_messages_bytes", - "refId": "A", - "step": 20 - } - ], - "thresholds": "", - "title": "Persistent Bytes Put", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": "10s", - "schemaVersion": 12, - "version": 27, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/cmd/mq_prometheus/README.md b/cmd/mq_prometheus/README.md deleted file mode 100755 index c4f4a82..0000000 --- a/cmd/mq_prometheus/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# MQ Exporter for Prometheus monitoring - -This directory contains the code for a monitoring solution -that exports queue manager data to a Prometheus data collection -system. It also contains configuration files to run the monitor program - -The monitor collects metrics published by an MQ V9 queue manager -or the MQ appliance. Prometheus than calls the monitor program -at regular intervals to pull those metrics into its database, where -they can then be queried directly or used by other packages -such as Grafana. - -You can see data such as disk or CPU usage, queue depths, and MQI call -counts. - -An example Grafana dashboard is included, to show how queries might -be constructed. The data shown is the same as in the corresponding -InfluxDB-based dashboard, also in this repository. -To use the dashboard, -create a data source in Grafana called "MQ Prometheus" that points at your -database server, and then import the JSON file. - -There is also a script to start the collector so that it processes -the statistics generated by the MQ Bridge for Salesforce, included in -MQ V9.0.2 - - -## Building -* This github repository contains the monitoring program and -the ibmmq package that links to the core MQ application interface. It -also contains the mqmetric package used as a common component for -supporting alternative database collection protocols. - -* You also need access to the Prometheus Go client interface. - - The command `go get -u github.com/prometheus/client_golang/prometheus` should pull - down the client code and its dependencies. - -* The error logger package may need to be explicitly downloaded - - On my system, I also had to forcibly download the logger package, - using `go get -u github.com/Sirupsen/logrus`. - -Run `go build -o /mq_prometheus cmd/mq_prometheus/*.go` to compile -the program and put it to a specific directory. - -## Configuring MQ -It is convenient to run the monitor program as a queue manager service. - -This directory contains an MQSC script to define the service. In fact, the -service definition points at a simple script which sets up any -necessary environment and builds the command line parameters for the -real monitor program. As the last line of the script is "exec", the -process id of the script is inherited by the monitor program, and the -queue manager can then check on the status, and can drive a suitable -`STOP SERVICE` operation during queue manager shutdown. - -Edit the MQSC script and the shell script to point at appropriate directories -where the program exists, and where you want to put stdout/stderr. -Ensure that the ID running the queue manager has permission to access -the programs and output files. - -The monitor listens for calls from Prometheus on a TCP port. The default -port, reserved for this use in the Prometheus list, is 9157. If you -want to use a different number, then use the `-ibmmq.httpListenPort` -command parameter. - -The monitor always collects all of the available queue manager-wide metrics. -It can also be configured to collect statistics for specific sets of queues. -The sets of queues can be given either directly on the command line with the -`-ibmmq.monitoredQueues` flag, or put into a separate file which is also -named on the command line, with the `ibmmq.monitoredQueuesFile` flag. An -example is included in the startup shell script. - -Note that **for now**, the queue patterns are expanded only at startup -of the monitor program. If you want to change the patterns, or new -queues are defined that match an existing pattern, the monitor must be -restarted with a `STOP SERVICE` and `START SERVICE` pair of commands. - -### Authentication -This monitor can be configured to authenticate to the queue manager, -sending a userid and password. - -The userid is configured using the `-ibmmq.userid` flag. The password can -be set either by using the `-ibmmq.password` flag, or by passing it via stdin. -That allows it to be piped from an external stash file or some other -mechanism. Command line flags for controlling passwords are not recommended! - -## Configuring Prometheus -The Prometheus server has to know how to contact the MQ monitor. The -simplest way is just to add a reference to the monitor in the -server's configuration file. For example, by adding this block -to /etc/prometheus/prometheus.yml. - -``` - # Adding a reference to an MQ monitor. All we have to do is - # name the host and port on which the monitor is listening. - # Port 9157 is the reserved default port for this monitor. - - job_name: 'ibmmq' - scrape_interval: 15s - - static_configs: - - targets: ['hostname.example.com:9157'] -``` - -The server documentation has information on more complex -options, including the ability to pull information on which hosts -should be monitored from a variety of discovery tools. - -## Metrics -Once the monitor program has been started, and Prometheus refreshed to -connect to it, you will see metrics being available in the prometheus -console. All of the metrics are given the jobname prefix of **ibmmq**. - -More information on the metrics collected through the publish/subscribe -interface can be found in the [MQ KnowledgeCenter] -(https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.mon.doc/mo00013_.htm) -with further description in [an MQDev blog entry] -(https://www.ibm.com/developerworks/community/blogs/messaging/entry/Statistics_published_to_the_system_topic_in_MQ_v9?lang=en) - -The metrics shown in the Prometheus console are named after the descriptions -that you can see when running the amqsrua sample program, but with some -minor modifications to match the required style. diff --git a/cmd/mq_prometheus/config.go b/cmd/mq_prometheus/config.go deleted file mode 100755 index 4b4453f..0000000 --- a/cmd/mq_prometheus/config.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "bufio" - "flag" - "fmt" - "os" - - "github.com/ibm-messaging/mq-golang/mqmetric" -) - -type mqExporterConfig struct { - qMgrName string - replyQ string - monitoredQueues string - monitoredQueuesFile string - - // TODO: enable these - monitorChannelStatistics bool - statisticsQueueName string - - metaPrefix string - - cc mqmetric.ConnectionConfig - - httpListenPort string - httpMetricPath string - logLevel string - namespace string -} - -const ( - defaultPort = "9157" // reserved in the prometheus wiki - defaultNamespace = "ibmmq" -) - -var config mqExporterConfig - -/* -initConfig parses the command line parameters. Note that the logging -package requires flag.Parse to be called before we can do things like -info/error logging - -The default IP port for this monitor is registered with prometheus so -does not have to be provided. -*/ -func initConfig() { - - flag.StringVar(&config.qMgrName, "ibmmq.queueManager", "", "Queue Manager name") - flag.StringVar(&config.replyQ, "ibmmq.replyQueue", "SYSTEM.DEFAULT.MODEL.QUEUE", "Reply Queue to collect data") - flag.StringVar(&config.monitoredQueues, "ibmmq.monitoredQueues", "", "Patterns of queues to monitor") - flag.StringVar(&config.monitoredQueuesFile, "ibmmq.monitoredQueuesFile", "", "File with patterns of queues to monitor") - flag.StringVar(&config.metaPrefix, "metaPrefix", "", "Override path to monitoring resource topic") - - flag.BoolVar(&config.cc.ClientMode, "ibmmq.client", false, "Connect as MQ client") - flag.StringVar(&config.cc.UserId, "ibmmq.userid", "", "UserId for MQ connection") - // TODO: Also enable a different mechanism to read the password - flag.StringVar(&config.cc.Password, "ibmmq.password", "", "Password for MQ connection") - - // TODO: turn these on - //flag.BoolVar(&config.monitorChannelStatistics, "ibmmq.monitorChannelStatistics", false, "Whether to collect channel stats") - //flag.StringVar(&config.statisticsQueueName, "ibmmq.statisticsQueueName", "SYSTEM.ADMIN.STATISTICS.QUEUE", "Which queue holds channel stats") - config.monitorChannelStatistics = false - config.statisticsQueueName = "" - - flag.StringVar(&config.httpListenPort, "ibmmq.httpListenPort", defaultPort, "HTTP Listener") - flag.StringVar(&config.httpMetricPath, "ibmmq.httpMetricPath", "/metrics", "Path to exporter metrics") - - flag.StringVar(&config.logLevel, "log.level", "error", "Log level - debug, info, error") - flag.StringVar(&config.namespace, "namespace", defaultNamespace, "Namespace for metrics") - - flag.Parse() - - if config.monitoredQueuesFile != "" { - config.monitoredQueues = mqmetric.ReadPatterns(config.monitoredQueuesFile) - } - - if config.cc.UserId != "" && config.cc.Password == "" { - scanner := bufio.NewScanner(os.Stdin) - fmt.Printf("Enter password: \n") - scanner.Scan() - config.cc.Password = scanner.Text() - } -} diff --git a/cmd/mq_prometheus/exporter.go b/cmd/mq_prometheus/exporter.go deleted file mode 100755 index 2793ecf..0000000 --- a/cmd/mq_prometheus/exporter.go +++ /dev/null @@ -1,220 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -/* -This file provides the main link between the MQ monitoring collection, and -the Prometheus request for data. The Collect() function is the key operation -invoked at the scrape intervals, causing us to read available publications -and update the various Gauges. -*/ - -import ( - "strings" - "sync" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" - "github.com/prometheus/client_golang/prometheus" -) - -type exporter struct { - mutex sync.RWMutex - metrics mqmetric.AllMetrics -} - -func newExporter() *exporter { - return &exporter{ - metrics: mqmetric.Metrics, - } -} - -var ( - first = true - gaugeMap = make(map[string]*prometheus.GaugeVec) -) - -/* -Describe is called by Prometheus on startup of this monitor. It needs to tell -the caller about all of the available metrics. -*/ -func (e *exporter) Describe(ch chan<- *prometheus.Desc) { - - log.Infof("IBMMQ Describe started") - - for _, cl := range e.metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - gaugeMap[makeKey(elem)].Describe(ch) - } - } - } -} - -/* -Collect is called by Prometheus at regular intervals to provide current -data -*/ -func (e *exporter) Collect(ch chan<- prometheus.Metric) { - e.mutex.Lock() // To protect metrics from concurrent collects. - defer e.mutex.Unlock() - - log.Infof("IBMMQ Collect started") - - // Clear out everything we know so far. In particular, replace - // the map of values for each object so the collection starts - // clean. - for _, cl := range e.metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - gaugeMap[makeKey(elem)].Reset() - elem.Values = make(map[string]int64) - } - } - } - - // Deal with all the publications that have arrived - mqmetric.ProcessPublications() - - // Have now processed all of the publications, and all the MQ-owned - // value fields and maps have been updated. - // - // Now need to set all of the real Gauges with the correct values - if first { - // Always ignore the first loop through as there might - // be accumulated stuff from a while ago, and lead to - // a misleading range on graphs. - first = false - } else { - - for _, cl := range e.metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - for key, value := range elem.Values { - f := mqmetric.Normalise(elem, key, value) - g := gaugeMap[makeKey(elem)] - if key == mqmetric.QMgrMapKey { - g.WithLabelValues(config.qMgrName).Set(f) - } else { - g.WithLabelValues(key, config.qMgrName).Set(f) - } - } - } - } - } - } - - // And finally tell Prometheus about the data - for _, cl := range e.metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - gaugeMap[makeKey(elem)].Collect(ch) - } - } - } - -} - -/* -allocateGauges creates a Prometheus gauge for each -resource that we know about. These are stored in a local map keyed -from the resource names. -*/ -func allocateGauges() { - for _, cl := range mqmetric.Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - g := newMqGaugeVec(elem) - key := makeKey(elem) - gaugeMap[key] = g - } - } - } -} - -/* -makeKey uses the 3 parts of a resource's name to build a unique string. -The "/" character cannot be part of a name, so is a convenient way -to build a unique key. If we ever have metrics for other object -types such as topics, then the object type would be used too. -This key is not used outside of this module, so the format can change. -*/ -func makeKey(elem *mqmetric.MonElement) string { - key := elem.Parent.Parent.Name + "/" + - elem.Parent.Name + "/" + - elem.MetricName - return key -} - -/* -newMqGaugeVec returns the structure which will contain the -values and suitable labels. For queues we tag each entry -with both the queue and qmgr name; for the qmgr-wide entries, we -only need the single label. -*/ -func newMqGaugeVec(elem *mqmetric.MonElement) *prometheus.GaugeVec { - queueLabelNames := []string{"object", "qmgr"} - qmgrLabelNames := []string{"qmgr"} - - labels := qmgrLabelNames - prefix := "qmgr_" - - if strings.Contains(elem.Parent.ObjectTopic, "%s") { - labels = queueLabelNames - prefix = "object_" - } - - gaugeVec := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: config.namespace, - Name: prefix + elem.MetricName, - Help: elem.Description, - }, - labels, - ) - - log.Infof("Created gauge for %s", elem.MetricName) - return gaugeVec -} - -/* -newMqGaugeVec returns the structure which will contain the -values and suitable labels. For queues we tag each entry -with both the queue and qmgr name; for the qmgr-wide entries, we -only need the single label. -*/ - -// TODO: Finish this -/* -func newMqGaugeVecChl(elem *ibmmq.Statistic) *prometheus.GaugeVec { - prefix := "channel_" - - gaugeVec := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: config.namespace, - Name: prefix + elem.MetricName, - Help: elem.Description, - }, - labels, - ) - - log.Infof("Created gauge for %s", elem.MetricName) - return gaugeVec -} -*/ diff --git a/cmd/mq_prometheus/main.go b/cmd/mq_prometheus/main.go deleted file mode 100644 index 1922602..0000000 --- a/cmd/mq_prometheus/main.go +++ /dev/null @@ -1,117 +0,0 @@ -package main - -/* - Copyright (c) IBM Corporation 2016 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific - - Contributors: - Mark Taylor - Initial Contribution -*/ - -import ( - "net/http" - "os" - - log "github.com/Sirupsen/logrus" - "github.com/ibm-messaging/mq-golang/mqmetric" - "github.com/prometheus/client_golang/prometheus" -) - -func initLog() { - level, err := log.ParseLevel(config.logLevel) - if err != nil { - level = log.InfoLevel - } - log.SetLevel(level) -} - -func main() { - var err error - - initConfig() - if config.qMgrName == "" { - log.Errorln("Must provide a queue manager name to connect to.") - os.Exit(1) - } - - initLog() - log.Infoln("Starting IBM MQ metrics exporter for Prometheus monitoring") - - // Connect and open standard queues - err = mqmetric.InitConnectionStats(config.qMgrName, config.replyQ, config.statisticsQueueName, &config.cc) - if err == nil { - log.Infoln("Connected to queue manager ", config.qMgrName) - defer mqmetric.EndConnection() - } - - // What metrics can the queue manager provide? Find out, and - // subscribe. - if err == nil { - // Do we need to expand wildcarded queue names - // or use the wildcard as-is in the subscriptions - wildcardResource := true - if config.metaPrefix != "" { - wildcardResource = false - } - err = mqmetric.DiscoverAndSubscribe(config.monitoredQueues, wildcardResource, config.metaPrefix) - } - - // Once everything has been discovered, and the subscriptions - // created, allocate the Prometheus gauges for each resource - if err == nil { - allocateGauges() - } - - // TODO: continue with the channel stat collection - //if err == nil && config.statisticsQueueName != "" { - // mqmetric.InitChlStatistics() - //allocateChlGauges() - //} - - // Go into main loop for handling requests from Prometheus - if err == nil { - exporter := newExporter() - prometheus.MustRegister(exporter) - - http.Handle(config.httpMetricPath, prometheus.Handler()) - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write(landingPage()) - }) - - log.Infoln("Listening on", config.httpListenPort) - log.Fatal(http.ListenAndServe(":"+config.httpListenPort, nil)) - - } - - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} - -/* -landingPage gives a very basic response if someone just connects to our port. -The only link on it jumps to the list of available metrics. -*/ -func landingPage() []byte { - return []byte( - ` -IBM MQ Exporter - -

IBM MQ Exporter

-

Metrics

- - -`) -} diff --git a/cmd/mq_prometheus/mq_prometheus.mqsc b/cmd/mq_prometheus/mq_prometheus.mqsc deleted file mode 100755 index bfba388..0000000 --- a/cmd/mq_prometheus/mq_prometheus.mqsc +++ /dev/null @@ -1,18 +0,0 @@ -* Cleanup any existing service -STOP SERVICE(PROMETHEUS) -DELETE SERVICE(PROMETHEUS) - -* Reset the definition -DEFINE SERVICE(PROMETHEUS) + - CONTROL(QMGR) + - SERVTYPE(SERVER) + - STARTCMD('/usr/local/bin/mqgo/mq_prometheus.sh') + - STARTARG(+QMNAME+) + - STOPCMD('/usr/bin/kill ' ) + - STOPARG(+MQ_SERVER_PID+) + - STDOUT('/var/mqm/errors/prom.out') + - STDERR('/var/mqm/errors/prom.out') + - DESCR('MQ exporter for Prometheus') - -* Start it manually now; will be automatically started on future qmgr startup -START SERVICE(PROMETHEUS) diff --git a/cmd/mq_prometheus/mq_prometheus.sh b/cmd/mq_prometheus/mq_prometheus.sh deleted file mode 100755 index ab694da..0000000 --- a/cmd/mq_prometheus/mq_prometheus.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# This is used to start the IBM MQ monitoring service for Prometheus - -# The queue manager name comes in from the service definition as the -# only command line parameter -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of queues to be monitored is given here. -# It is a set of names or patterns ('*' only at the end, to match how MQ works), -# separated by commas. When no queues match a pattern, it is reported but -# is not fatal. -queues="APP.*,MYQ.*" - -# An alternative is to have a file containing the patterns, and named -# via the ibmmq.monitoredQueuesFile option. - -# See config.go for all recognised flags - -# Start via "exec" so the pid remains the same. The queue manager can -# then check the existence of the service and use the MQ_SERVER_PID value -# to kill it on shutdown. -exec /usr/local/bin/mqgo/mq_prometheus -ibmmq.queueManager=$qMgr -ibmmq.monitoredQueues="$queues" -log.level=error diff --git a/cmd/mq_prometheus/mqsf_prometheus.sh b/cmd/mq_prometheus/mqsf_prometheus.sh deleted file mode 100755 index b3f6d10..0000000 --- a/cmd/mq_prometheus/mqsf_prometheus.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# This is used to start the IBM MQ monitoring service for Prometheus -# to collect data from the MQ Bridge for Salesforce. - -# The queue manager name comes in from the service definition as the -# only command line parameter -qMgr=$1 - -# Set the environment to ensure we pick up libmqm.so etc -. /opt/mqm/bin/setmqenv -m $qMgr -k - -# A list of topics to be monitored is given here. Can use specific topics -# or MQ wildcards such as "#". We are still using a "queues" command line -# parameter, and ought to change it, but this works. -queues="#,/topic/PT1,/topic/PT2,/event/PE2__e,/event/PE1__e" - -# See config.go for all recognised flags - -# Start via "exec" so the pid remains the same. The queue manager can -# then check the existence of the service and use the MQ_SERVER_PID value -# to kill it on shutdown. -# Need to specify an HTTP port that is not currently used. The default 9157 may -# be used by the collector accessing queue manager resource statistics. -exec /usr/local/bin/mqgo/mq_prometheus -ibmmq.queueManager=$qMgr -ibmmq.monitoredQueues="$queues" -log.level=error -metaPrefix="\$SYS/Application/runmqsfb" -ibmmq.httpListenPort=9158 -namespace=ibmmqsf diff --git a/cmd/mq_prometheus/prometheus.yml b/cmd/mq_prometheus/prometheus.yml deleted file mode 100644 index 400c610..0000000 --- a/cmd/mq_prometheus/prometheus.yml +++ /dev/null @@ -1,42 +0,0 @@ -# my global config -global: - scrape_interval: 15s # By default, scrape targets every 15 seconds. - evaluation_interval: 15s # By default, scrape targets every 15 seconds. - # scrape_timeout is set to the global default (10s). - - # Attach these labels to any time series or alerts when communicating with - # external systems (federation, remote storage, Alertmanager). - external_labels: - monitor: 'codelab-monitor' - -# Load and evaluate rules in this file every 'evaluation_interval' seconds. -rule_files: - # - "first.rules" - # - "second.rules" - -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. -scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' - - # Override the global default and scrape targets from this job every 5 seconds. - scrape_interval: 5s - - # metrics_path defaults to '/metrics' - # scheme defaults to 'http'. - - static_configs: - - targets: ['localhost:9090'] - - # Adding a reference to an MQ monitor. All we have to do is - # name the host and port on which the monitor is listening. - # Port 9157 is the reserved default port for this monitor. - - job_name: 'ibmmq' - scrape_interval: 15s - - static_configs: - - targets: ['hostname.example.com:9157'] - # targets: ['hostname.example.com:9157','hostname.example.com:9158'] is - # the syntax if you want to have several prometheus monitors on the same - # box. diff --git a/mqmetric/discover.go b/mqmetric/discover.go index f02de56..2cb0bfb 100755 --- a/mqmetric/discover.go +++ b/mqmetric/discover.go @@ -37,7 +37,6 @@ import ( "os" "strings" - log "github.com/Sirupsen/logrus" "github.com/ibm-messaging/mq-golang/ibmmq" ) @@ -160,7 +159,7 @@ func discoverClasses(metaPrefix string) error { case ibmmq.MQCA_TOPIC_STRING: cl.typesTopic = elem.String[0] default: - log.Errorf("Unknown parameter %d in class discovery", elem.Parameter) + return fmt.Errorf("Unknown parameter %d in class discovery", elem.Parameter) } } Metrics.Classes[classIndex] = cl @@ -176,7 +175,6 @@ func discoverTypes(cl *MonClass) error { var sub ibmmq.MQObject var err error - //log.Infof("Working on class %s", cl.Name) sub, err = subscribe(cl.typesTopic) if err == nil { data, err = getMessage(true) @@ -210,7 +208,7 @@ func discoverTypes(cl *MonClass) error { case ibmmq.MQCA_TOPIC_STRING: ty.elementTopic = elem.String[0] default: - log.Errorf("Unknown parameter %d in type discovery", elem.Parameter) + return fmt.Errorf("Unknown parameter %d in type discovery", elem.Parameter) } } cl.Types[typeIndex] = ty @@ -261,7 +259,7 @@ func discoverElements(ty *MonType) error { case ibmmq.MQCAMO_MONITOR_DESC: elem.Description = e.String[0] default: - log.Errorf("Unknown parameter %d in type discovery", e.Parameter) + return fmt.Errorf("Unknown parameter %d in type discovery", e.Parameter) } } @@ -303,18 +301,6 @@ func discoverStats(metaPrefix string) error { } - for _, cl := range Metrics.Classes { - for _, ty := range cl.Types { - for _, elem := range ty.Elements { - log.Debugf("DUMP Element: Desc = %s ParentType = %s MetaTopic = %s Real Topic = %s Type = %d", - elem.MetricName, - elem.Parent.Name, - elem.Parent.elementTopic, - ty.ObjectTopic, elem.Datatype) - } - } - } - return err } @@ -350,8 +336,7 @@ func discoverQueues(monitoredQueues string) error { if strings.Count(pattern, "*") > 1 || (strings.Count(pattern, "*") == 1 && !strings.HasSuffix(pattern, "*")) { - log.Errorf("Queue pattern '%s' is not valid", pattern) - continue + return fmt.Errorf("Queue pattern '%s' is not valid", pattern) } putmqmd := ibmmq.NewMQMD() @@ -395,7 +380,7 @@ func discoverQueues(monitoredQueues string) error { err = cmdQObj.Put(putmqmd, pmo, buf) if err != nil { - log.Error(err) + return err } // Now get the response @@ -414,9 +399,7 @@ func discoverQueues(monitoredQueues string) error { if err == nil { cfh, offset := ibmmq.ReadPCFHeader(buf) if cfh.CompCode != ibmmq.MQCC_OK { - log.Errorf("PCF command failed with CC %d RC %d", - cfh.CompCode, - cfh.Reason) + return fmt.Errorf("PCF command failed with CC %d RC %d", cfh.CompCode, cfh.Reason) } else { parmAvail := true bytesRead := 0 @@ -431,7 +414,7 @@ func discoverQueues(monitoredQueues string) error { switch elem.Parameter { case ibmmq.MQCACF_Q_NAMES: if len(elem.String) == 0 { - log.Errorf("No queues matching '%s' exist", pattern) + return fmt.Errorf("No queues matching '%s' exist", pattern) } for i := 0; i < len(elem.String); i++ { qList = append(qList, strings.TrimSpace(elem.String[i])) @@ -440,12 +423,10 @@ func discoverQueues(monitoredQueues string) error { } } } else { - log.Error(err) + return err } } - log.Infof("Discovered queues = %v", qList) - return err } @@ -457,7 +438,6 @@ func createSubscriptions() error { var err error var sub ibmmq.MQObject -loop: for _, cl := range Metrics.Classes { for _, ty := range cl.Types { @@ -473,8 +453,7 @@ loop: } if err != nil { - log.Error("Error subscribing: ", err) - break loop + return fmt.Errorf("Error subscribing to %s: %v", ty.ObjectTopic, err) } } } @@ -588,17 +567,8 @@ func ProcessPublications() { elem.Values[objectName] = value } } - } else { - mqreturn := err.(*ibmmq.MQReturn) - // Printing this for 2033 is excessive, even in debug - if mqreturn.MQRC != ibmmq.MQRC_NO_MSG_AVAILABLE { - log.Debugf("getMessage returned %v", err) - } } - } - log.Debugf("Processed %d messages", cnt) - } /* @@ -723,12 +693,12 @@ ReadPatterns is called during the initial configuration step to read a file containing object name patterns if they are not explicitly given on the command line. */ -func ReadPatterns(f string) string { +func ReadPatterns(f string) (string, error) { var s string file, err := os.Open(f) if err != nil { - log.Fatalf("Opening file %s: %s", f, err) + return "", fmt.Errorf("Error Opening file %s: %v", f, err) } defer file.Close() scanner := bufio.NewScanner(file) @@ -739,11 +709,10 @@ func ReadPatterns(f string) string { s += scanner.Text() } if err := scanner.Err(); err != nil { - log.Fatalf("Reading from %s: %s", f, err) + return "", fmt.Errorf("Error Reading from %s: %v", f, err) } - log.Infof("Read patterns from %s: '%s'", f, s) - return s + return s, nil } /* @@ -760,9 +729,6 @@ func Normalise(elem *MonElement, key string, value int64) float64 { f = 0 } - //log.Debugf("Pushing Elem %s [%s] Type %d Value %f", - // elem.MetricName, key, elem.Datatype, f) - // Convert suitable metrics to base units if elem.Datatype == ibmmq.MQIAMO_MONITOR_PERCENT || elem.Datatype == ibmmq.MQIAMO_MONITOR_HUNDREDTHS { diff --git a/mqmetric/mqif.go b/mqmetric/mqif.go index 764fd63..b1b36e4 100644 --- a/mqmetric/mqif.go +++ b/mqmetric/mqif.go @@ -24,7 +24,8 @@ don't need to repeat common setups eg of MQMD or MQSD structures. */ import ( - log "github.com/Sirupsen/logrus" + "fmt" + "github.com/ibm-messaging/mq-golang/ibmmq" ) @@ -95,9 +96,6 @@ func InitConnectionStats(qMgrName string, replyQ string, statsQ string, cc *Conn mqod.ObjectName = "SYSTEM.ADMIN.COMMAND.QUEUE" cmdQObj, err = qMgr.Open(mqod, openOptions) - if err == nil { - log.Infoln("Command queue open ok") - } } @@ -110,7 +108,6 @@ func InitConnectionStats(qMgrName string, replyQ string, statsQ string, cc *Conn statsQObj, err = qMgr.Open(mqod, openOptions) if err == nil { statsQueuesOpened = true - log.Infoln("Stats queue open ok") } } @@ -123,16 +120,14 @@ func InitConnectionStats(qMgrName string, replyQ string, statsQ string, cc *Conn replyQObj, err = qMgr.Open(mqod, openOptions) if err == nil { queuesOpened = true - log.Infoln("Reply queue open ok") } } if err != nil { - log.Errorf("Cannot access qmgr. Error %s", err) + return fmt.Errorf("Cannot access queue manager. Error: %v", err) } return err - } /* @@ -207,14 +202,14 @@ func getMessageWithHObj(wait bool, hObj ibmmq.MQObject) ([]byte, error) { if mqreturn.MQRC == ibmmq.MQRC_Q_MGR_NOT_AVAILABLE || mqreturn.MQRC == ibmmq.MQRC_Q_MGR_NAME_ERROR || mqreturn.MQRC == ibmmq.MQRC_Q_MGR_QUIESCING { - log.Fatal("Queue Manager error: ", err) + return nil, fmt.Errorf("Queue Manager error: %v", err) } if mqreturn.MQCC == ibmmq.MQCC_FAILED && mqreturn.MQRC != ibmmq.MQRC_NO_MSG_AVAILABLE { - log.Error("Get message: ", err) + return nil, fmt.Errorf("Get message error: %v", err) } } - return getBuffer[0:datalen], err + return getBuffer[0:datalen], nil } /* @@ -233,11 +228,10 @@ func subscribe(topic string) (ibmmq.MQObject, error) { mqsd.ObjectString = topic - log.Infof("Subscribing to topic '%s'", topic) subObj, err := qMgr.Sub(mqsd, &replyQObj) if err != nil { - log.Errorf("Error subscribing to topic '%s': %v", topic, err) + return subObj, fmt.Errorf("Error subscribing to topic '%s': %v", topic, err) } - return subObj, err + return subObj, err } diff --git a/cmd/samples/clientconn/clientconn.go b/samples/clientconn/clientconn.go similarity index 100% rename from cmd/samples/clientconn/clientconn.go rename to samples/clientconn/clientconn.go diff --git a/cmd/samples/mqitest/mqitest.go b/samples/mqitest/mqitest.go similarity index 100% rename from cmd/samples/mqitest/mqitest.go rename to samples/mqitest/mqitest.go