Fixing Issue #5259 - Data Collection Issues with Corner Cases

When the Data Template contains more Data Sources than the Graph Template RRDfiles won't update
Cacti · Mar 19, 2023 · 3615778 · 3615778
1 parent 192e501
commit 3615778
Show file tree

Hide file tree

Showing 9 changed files with 97 additions and 62 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,7 @@
 Cacti CHANGELOG
 
 1.2.25
+-issue#5259: When the Data Template contains more Data Sources than the Graph Template RRDfiles won't update
 -issue#5263: Warnings in Cacti log when re-indexing devices
 -issue#5272: Boost continues to loose data when archive tables are missed due to overloaded MariaDB
 -feature: Upgrade billboard.js to version 3.7.4

diff --git a/data_templates.php b/data_templates.php
@@ -329,14 +329,16 @@ function form_save() {
 					array($data_template_id));
 
 				db_execute_prepared('REPLACE INTO poller_data_template_field_mappings
-					SELECT dtr.data_template_id,
-					dif.data_name,
-					GROUP_CONCAT(dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
+					SELECT dtr.data_template_id, dif.data_name,
+					GROUP_CONCAT(DISTINCT dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
 					NOW() AS last_updated
-					FROM data_template_rrd AS dtr
+					FROM graph_templates_item AS gti
+					INNER JOIN data_template_rrd AS dtr
+					gti.task_item_id = dtr.id
 					INNER JOIN data_input_fields AS dif
 					ON dtr.data_input_field_id = dif.id
 					WHERE dtr.local_data_id = 0
+					AND gti.local_graph_id = 0
 					AND dtr.data_template_id = ?
 					GROUP BY dtr.data_template_id, dif.data_name',
 					array($data_template_id));

diff --git a/lib/boost.php b/lib/boost.php
@@ -900,16 +900,20 @@ function boost_process_poller_output($local_data_id, $rrdtool_pipe = '') {
 				$outlen += strlen($output);
 				$vals_in_buffer++;
 			} elseif ($value != '') {
-				/* break out multiple value output to an array */
-				$values = explode(' ', $value);
+				$values = preg_split('/\s+/', $value);
 
 				if (!$multi_vals_set) {
-					$rrd_field_names = array_rekey(db_fetch_assoc_prepared('SELECT
-						data_template_rrd.data_source_name,
-						data_input_fields.data_name
-						FROM (data_template_rrd, data_input_fields)
-						WHERE data_template_rrd.data_input_field_id = data_input_fields.id
-						AND data_template_rrd.local_data_id = ?', array($item['local_data_id'])), 'data_name', 'data_source_name');
+					$rrd_field_names = array_rekey(
+						db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+							FROM graph_templates_item AS gti
+							INNER JOIN data_template_rrd AS dtr
+							ON gti.task_item_id = dtr.id
+							INNER JOIN data_input_fields AS dif
+							ON dtr.data_input_field_id = dif.id
+							WHERE dtr.local_data_id = ?',
+							array($item['local_data_id'])),
+						'data_name', 'data_source_name'
+					);
 
 					$rrd_tmpl = '';
 				}
@@ -929,15 +933,15 @@ function boost_process_poller_output($local_data_id, $rrdtool_pipe = '') {
 									$rrd_tmpl .= ':';
 								}
 
-								$rrd_tmpl  .= $rrd_field_names[$matches[1]];
+								$rrd_tmpl  .= $rrd_field_names[$matches[0]];
 								$first_tmpl = false;
 							}
 
 							if (is_numeric($matches[1]) || ($matches[1] == 'U')) {
 								$output  = ':' . $matches[1];
 								$outbuf .= $output;
 								$outlen += strlen($output);
-							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[2]))) {
+							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[1]))) {
 								$output  = ':' . hexdec($matches[1]);
 								$outbuf .= $output;
 								$outlen += strlen($output);

diff --git a/lib/dsstats.php b/lib/dsstats.php
@@ -689,11 +689,16 @@ function dsstats_poller_output(&$rrd_update_array) {
 			/* make the association between the multi-part name value pairs and the RRDfile internal
 			 * data source names.
 			 */
-			$ds_multi = array_rekey(db_fetch_assoc('SELECT DISTINCT data_name, data_source_name
-				FROM data_template_rrd AS dtr
-				INNER JOIN data_input_fields AS dif
-				ON dif.id = dtr.data_input_field_id
-				WHERE dtr.data_input_field_id != 0'), 'data_name', 'data_source_name');
+			$ds_multi = array_rekey(
+				db_fetch_assoc('SELECT DISTINCT data_name, data_source_name
+					FROM graph_templates_item AS gti
+					INNER JOIN data_template_rrd AS dtr
+					ON gti.task_item_id = dtr.id
+					INNER JOIN data_input_fields AS dif
+					ON dif.id = dtr.data_input_field_id
+					WHERE dtr.data_input_field_id != 0'),
+				'data_name', 'data_source_name'
+			);
 
 			/* required for updating tables */
 			$cache_i      = 1;

diff --git a/lib/import.php b/lib/import.php
@@ -1165,14 +1165,16 @@ function xml_to_data_template($hash, &$xml_array, &$hash_cache, $import_as_new,
 
 		/* push out field mappings for the data collector */
 		db_execute_prepared('REPLACE INTO poller_data_template_field_mappings
-			SELECT dtr.data_template_id,
-			dif.data_name,
-			GROUP_CONCAT(dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
+			SELECT dtr.data_template_id, dif.data_name,
+			GROUP_CONCAT(DISTINCT dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
 			NOW() AS last_updated
-			FROM data_template_rrd AS dtr
+			FROM graph_templates_item AS gti
+			INNER data_template_rrd AS dtr
+			ON gti.task_item_id = dtr.id
 			INNER JOIN data_input_fields AS dif
 			ON dtr.data_input_field_id = dif.id
 			WHERE dtr.local_data_id = 0
+			AND gti.local_graph_id = 0
 			AND dtr.data_template_id = ?
 			GROUP BY dtr.data_template_id, dif.data_name', array($data_template_id));
 	}

diff --git a/lib/poller.php b/lib/poller.php
@@ -620,11 +620,14 @@ function process_poller_output(&$rrdtool_pipe, $remainder = 0) {
 						} else {
 							// Handle data source without a data template
 							$nt_rrd_field_names = array_rekey(
-								db_fetch_assoc_prepared('SELECT dtr.data_source_name, dif.data_name
-									FROM data_template_rrd AS dtr
+								db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+									FROM graph_templates_item AS gti
+									INNER JOIN data_template_rrd AS dtr
+									ON gti.task_item_id = dtr.id
 									INNER JOIN data_input_fields AS dif
 									ON dtr.data_input_field_id=dif.id
-									WHERE dtr.local_data_id = ?', array($item['local_data_id'])),
+									WHERE dtr.local_data_id = ?',
+									array($item['local_data_id'])),
 								'data_name', 'data_source_name'
 							);
 

diff --git a/poller.php b/poller.php
@@ -449,13 +449,16 @@ function sig_handler($signo) {
  * Freshen the field mappings in cases where they
  * may have gotten out of sync
  */
-db_execute('INSERT IGNORE INTO poller_data_template_field_mappings
+db_execute('REPLACE INTO poller_data_template_field_mappings
 	SELECT dtr.data_template_id, dif.data_name,
-	GROUP_CONCAT(dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names, NOW()
-	FROM data_template_rrd AS dtr
+	GROUP_CONCAT(DISTINCT dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names, NOW()
+	FROM graph_templates_item AS gti
+	INNER JOIN data_template_rrd AS dtr
+	ON gti.task_item_id = dtr.id
 	INNER JOIN data_input_fields AS dif
 	ON dtr.data_input_field_id = dif.id
 	WHERE dtr.local_data_id = 0
+	AND gti.local_graph_id = 0
 	GROUP BY dtr.data_template_id, dif.data_name');
 
 while ($poller_runs_completed < $poller_runs) {

diff --git a/poller_boost.php b/poller_boost.php
@@ -186,7 +186,7 @@
 				set_config_option('boost_last_run_time', $last_run_time);
 			}
 
-			if ($rrd_updated > 0) {
+			if ($rrd_updates > 0) {
 				/* cleanup - remove empty arch tables*/
 				$tables = db_fetch_assoc("SELECT table_name AS name
 					FROM information_schema.tables
@@ -845,24 +845,32 @@ function boost_process_local_data_ids($last_id, $child, $rrdtool_pipe) {
 				$vals_in_buffer++;
 			} elseif (strlen($value)) {
 				/* break out multiple value output to an array */
-				$values = explode(' ', $value);
+				$values = preg_split('/\s+/', $value);
 
 				if (!$multi_vals_set) {
-					$rrd_field_names = array_rekey(db_fetch_assoc('SELECT
-						data_template_rrd.data_source_name,
-						data_input_fields.data_name
-						FROM (data_template_rrd,data_input_fields)
-						WHERE data_template_rrd.data_input_field_id=data_input_fields.id
-						AND data_template_rrd.local_data_id=' . $item['local_data_id']), 'data_name', 'data_source_name');
+					$rrd_field_names = array_rekey(
+						db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+							FROM graph_templates_item AS gti
+							INNER JOIN data_template_rrd AS dtr
+							ON gti.task_item_id = dtr.id
+							INNER JOIN data_input_fields AS dif
+							ON dtr.data_input_field_id = dif.id
+							AND dtr.local_data_id = ?',
+							array($item['local_data_id'])),
+						'data_name', 'data_source_name'
+					);
 
 					$rrd_tmpl = '';
 				}
 
 				$first_tmpl = true;
 				$multi_ok   = false;
-				for ($i=0; $i<count($values); $i++) {
-					if (preg_match('/^([a-zA-Z0-9_\.-]+):([eE0-9Uu\+\.-]+)$/', $values[$i], $matches)) {
-						if (isset($rrd_field_names[$matches[1]])) {
+
+				if (cacti_sizeof($values)) {
+					foreach($values as $value) {
+						$matches = explode(':', $value);
+
+						if (isset($rrd_field_names[$matches[0]])) {
 							$multi_ok = true;
 
 							if (trim(read_config_option('path_boost_log')) != '') {
@@ -874,19 +882,19 @@ function boost_process_local_data_ids($last_id, $child, $rrdtool_pipe) {
 									$rrd_tmpl .= ':';
 								}
 
-								$rrd_tmpl  .= $rrd_field_names[$matches[1]];
+								$rrd_tmpl  .= $rrd_field_names[$matches[0]];
 								$first_tmpl = false;
 							}
 
-							if (is_numeric($matches[2]) || ($matches[2] == 'U')) {
-								$tv_tmpl[$rrd_field_names[$matches[1]]] = $matches[2];
-								$buflen += strlen(':' . $matches[2]);
-							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[2]))) {
-								$tval = hexdec($matches[2]);
-								$tv_tmpl[$rrd_field_names[$matches[1]]] = $tval;
+							if (is_numeric($matches[1]) || ($matches[1] == 'U')) {
+								$tv_tmpl[$rrd_field_names[$matches[0]]] = $matches[1];
+								$buflen += strlen(':' . $matches[1]);
+							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[1]))) {
+								$tval = hexdec($matches[1]);
+								$tv_tmpl[$rrd_field_names[$matches[0]]] = $tval;
 								$buflen += strlen(':' . $tval);
 							} else {
-								$tv_tmpl[$rrd_field_names[$matches[1]]] = 'U';
+								$tv_tmpl[$rrd_field_names[$matches[0]]] = 'U';
 								$buflen += 2;
 							}
 						}

diff --git a/poller_realtime.php b/poller_realtime.php
@@ -243,21 +243,28 @@ function process_poller_output_rt($rrdtool_pipe, $poller_id, $interval) {
 			/* single one value output */
 			if ((is_numeric($value)) || ($value == 'U')) {
 				$rrd_update_array[$item['rrd_path']]['times'][$unix_time][$item['rrd_name']] = $value;
-			/* multiple value output */
 			} else {
-				$values = explode(' ', $value);
-
-				$rrd_field_names = array_rekey(db_fetch_assoc_prepared('SELECT
-					data_template_rrd.data_source_name,
-					data_input_fields.data_name
-					FROM (data_template_rrd,data_input_fields)
-					WHERE data_template_rrd.data_input_field_id=data_input_fields.id
-					AND data_template_rrd.local_data_id = ?', array($item['local_data_id'])), 'data_name', 'data_source_name');
-
-				for ($i=0; $i<cacti_count($values); $i++) {
-					if (preg_match('/^([a-zA-Z0-9_\.-]+):([eE0-9\+\.-]+)$/', $values[$i], $matches)) {
-						if (isset($rrd_field_names[$matches[1]])) {
-							$rrd_update_array[$item['rrd_path']]['times'][$unix_time][$rrd_field_names[$matches[1]]] = $matches[2];
+				/* multiple value output */
+				$values = preg_split('/\s+/', $value);
+
+				$rrd_field_names = array_rekey(
+					db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+						FROM graph_templates_item AS gti
+						INNER JOIN data_template_rrd AS dtr
+						ON gti.task_item_id = dtr.id
+						INNER JOIN data_input_fields AS dif
+						ON dtr.data_input_field_id = dif.id
+						AND dtr.local_data_id = ?',
+						array($item['local_data_id'])),
+					'data_name', 'data_source_name'
+				);
+
+				if (cacti_sizeof($values)) {
+					foreach($values as $value) {
+						$matches = explode(':', $value);
+
+						if (isset($rrd_field_names[$matches[0]])) {
+							$rrd_update_array[$item['rrd_path']]['times'][$unix_time][$rrd_field_names[$matches[0]]] = $matches[1];
 						}
 					}
 				}