Porting #5259 - Issues with Data Template and Graphs

When the Data Template contains more Data Sources than the Graph Template RRDfiles won't update
Cacti · Apr 1, 2023 · 4022428 · 4022428
1 parent 868f6d7
commit 4022428
Show file tree

Hide file tree

Showing 9 changed files with 104 additions and 67 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -60,6 +60,7 @@ Cacti CHANGELOG
 1.2.25
 -issue#5254: Created a data source template to specify the snmp port, but it doesn't seem to work properly
 -issue#5258: Warnings occur when installing thold plugin and as such thold will not upgrade correctly
+-issue#5259: When the Data Template contains more Data Sources than the Graph Template RRDfiles won't update
 -issue#5275: Boost stats can display odd values when running
 -issue#5277: Boost should not attempt to start if there are no poller_output_boost records
 -issue#5279: Rebuild poller cache does not work as expected

diff --git a/data_templates.php b/data_templates.php
@@ -336,14 +336,16 @@ function form_save() {
 					array($data_template_id));
 
 				db_execute_prepared('REPLACE INTO poller_data_template_field_mappings
-					SELECT dtr.data_template_id,
-					dif.data_name,
-					GROUP_CONCAT(dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
+					SELECT dtr.data_template_id, dif.data_name,
+					GROUP_CONCAT(DISTINCT dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
 					NOW() AS last_updated
-					FROM data_template_rrd AS dtr
+					FROM graph_templates_item AS gti
+					INNER JOIN data_template_rrd AS dtr
+					gti.task_item_id = dtr.id
 					INNER JOIN data_input_fields AS dif
 					ON dtr.data_input_field_id = dif.id
 					WHERE dtr.local_data_id = 0
+					AND gti.local_graph_id = 0
 					AND dtr.data_template_id = ?
 					GROUP BY dtr.data_template_id, dif.data_name',
 					array($data_template_id));

diff --git a/lib/boost.php b/lib/boost.php
@@ -919,42 +919,49 @@ function boost_process_poller_output($local_data_id, $rrdtool_pipe = null) {
 				$vals_in_buffer++;
 			} elseif ($value != '') {
 				/* break out multiple value output to an array */
-				$values = explode(' ', $value);
+				$values = preg_split('/\s+/', $value);
 
 				if (!$multi_vals_set) {
-					$rrd_field_names = array_rekey(db_fetch_assoc_prepared('SELECT
-						data_template_rrd.data_source_name,
-						data_input_fields.data_name
-						FROM (data_template_rrd, data_input_fields)
-						WHERE data_template_rrd.data_input_field_id = data_input_fields.id
-						AND data_template_rrd.local_data_id = ?', array($item['local_data_id'])), 'data_name', 'data_source_name');
+					$rrd_field_names = array_rekey(
+						db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+							FROM graph_templates_item AS gti
+							INNER JOIN data_template_rrd AS dtr
+							ON gti.task_item_id = dtr.id
+							INNER JOIN data_input_fields AS dif
+							ON dtr.data_input_field_id = dif.id
+							WHERE dtr.local_data_id = ?',
+							array($item['local_data_id'])),
+						'data_name', 'data_source_name'
+					);
 
 					$rrd_tmpl = '';
 				}
 
 				$first_tmpl = true;
 				$multi_ok   = false;
 
-				for ($i=0; $i < cacti_count($values); $i++) {
-					if (preg_match("/^([a-zA-Z0-9_\.-]+):([eE0-9Uu\+\.-]+)$/", $values[$i], $matches)) {
-						if (isset($rrd_field_names[$matches[1]])) {
+				if (cacti_sizeof($values)) {
+					foreach($values as $value) {
+						$matches = explode(':', $value);
+
+						if (isset($rrd_field_names[$matches[0]])) {
 							$multi_ok = true;
 
 							if (!$multi_vals_set) {
 								if (!$first_tmpl) {
 									$rrd_tmpl .= ':';
 								}
 
-								$rrd_tmpl .= $rrd_field_names[$matches[1]];
+								$rrd_tmpl .= $rrd_field_names[$matches[0]];
 								$first_tmpl = false;
 							}
 
-							if (is_numeric($matches[2]) || ($matches[2] == 'U')) {
-								$output  = ':' . $matches[2];
+							if (is_numeric($matches[1]) || ($matches[1] == 'U')) {
+								$output  = ':' . $matches[1];
 								$outbuf .= $output;
 								$outlen += strlen($output);
-							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[2]))) {
-								$output  = ':' . hexdec($matches[2]);
+							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[1]))) {
+								$output  = ':' . hexdec($matches[1]);
 								$outbuf .= $output;
 								$outlen += strlen($output);
 							} else {

diff --git a/lib/dsstats.php b/lib/dsstats.php
@@ -805,11 +805,16 @@ function dsstats_poller_output(&$rrd_update_array) {
 			/* make the association between the multi-part name value pairs and the RRDfile internal
 			 * data source names.
 			 */
-			$ds_multi = array_rekey(db_fetch_assoc('SELECT DISTINCT data_name, data_source_name
-				FROM data_template_rrd AS dtr
-				INNER JOIN data_input_fields AS dif
-				ON dif.id = dtr.data_input_field_id
-				WHERE dtr.data_input_field_id != 0'), 'data_name', 'data_source_name');
+			$ds_multi = array_rekey(
+				db_fetch_assoc('SELECT DISTINCT data_name, data_source_name
+					FROM graph_templates_item AS gti
+					INNER JOIN data_template_rrd AS dtr
+					ON gti.task_item_id = dtr.id
+					INNER JOIN data_input_fields AS dif
+					ON dif.id = dtr.data_input_field_id
+					WHERE dtr.data_input_field_id != 0'),
+				'data_name', 'data_source_name'
+			);
 
 			/* required for updating tables */
 			$cache_i      = 1;

diff --git a/lib/import.php b/lib/import.php
@@ -1186,16 +1186,19 @@ function xml_to_data_template($hash, &$xml_array, &$hash_cache, $import_as_new,
 
 		/* push out field mappings for the data collector */
 		db_execute_prepared('REPLACE INTO poller_data_template_field_mappings
-			SELECT dtr.data_template_id,
-			dif.data_name,
-			GROUP_CONCAT(dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
+			SELECT dtr.data_template_id, dif.data_name,
+			GROUP_CONCAT(DISTINCT dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names,
 			NOW() AS last_updated
-			FROM data_template_rrd AS dtr
+			FROM graph_templates_item AS gti
+			INNER data_template_rrd AS dtr
+			ON gti.task_item_id = dtr.id
 			INNER JOIN data_input_fields AS dif
 			ON dtr.data_input_field_id = dif.id
 			WHERE dtr.local_data_id = 0
+			AND gti.local_graph_id = 0
 			AND dtr.data_template_id = ?
-			GROUP BY dtr.data_template_id, dif.data_name', array($data_template_id));
+			GROUP BY dtr.data_template_id, dif.data_name',
+			array($data_template_id));
 	}
 
 	/* status information that will be presented to the user */

diff --git a/lib/poller.php b/lib/poller.php
@@ -623,11 +623,14 @@ function process_poller_output(&$rrdtool_pipe, $remainder = 0) {
 						} else {
 							// Handle data source without a data template
 							$nt_rrd_field_names = array_rekey(
-								db_fetch_assoc_prepared('SELECT dtr.data_source_name, dif.data_name
-									FROM data_template_rrd AS dtr
+								db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+									FROM graph_templates_item AS gti
+									INNER JOIN data_template_rrd AS dtr
+									ON gti.task_item_id = dtr.id
 									INNER JOIN data_input_fields AS dif
 									ON dtr.data_input_field_id=dif.id
-									WHERE dtr.local_data_id = ?', array($item['local_data_id'])),
+									WHERE dtr.local_data_id = ?',
+									array($item['local_data_id'])),
 								'data_name', 'data_source_name'
 							);
 

diff --git a/poller.php b/poller.php
@@ -462,13 +462,16 @@ function sig_handler($signo) {
  * Freshen the field mappings in cases where they
  * may have gotten out of sync
  */
-db_execute('INSERT IGNORE INTO poller_data_template_field_mappings
+db_execute('REPLACE IGNORE INTO poller_data_template_field_mappings
 	SELECT dtr.data_template_id, dif.data_name,
-	GROUP_CONCAT(dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names, NOW()
-	FROM data_template_rrd AS dtr
+	GROUP_CONCAT(DISTINCT dtr.data_source_name ORDER BY dtr.data_source_name) AS data_source_names, NOW()
+	ROM graph_templates_item AS gti
+	INNER JOIN data_template_rrd AS dtr
+	ON gti.task_item_id = dtr.id
 	INNER JOIN data_input_fields AS dif
 	ON dtr.data_input_field_id = dif.id
 	WHERE dtr.local_data_id = 0
+	AND gti.local_graph_id = 0
 	GROUP BY dtr.data_template_id, dif.data_name');
 
 while ($poller_runs_completed < $poller_runs) {

diff --git a/poller_boost.php b/poller_boost.php
@@ -869,25 +869,32 @@ function boost_process_local_data_ids($last_id, $child, $rrdtool_pipe) {
 				$vals_in_buffer++;
 			} elseif (strlen($value)) {
 				/* break out multiple value output to an array */
-				$values = explode(' ', $value);
+				$values = preg_split('/\s+/', $value);
 
 				if (!$multi_vals_set) {
-					$rrd_field_names = array_rekey(db_fetch_assoc('SELECT
-						data_template_rrd.data_source_name,
-						data_input_fields.data_name
-						FROM (data_template_rrd,data_input_fields)
-						WHERE data_template_rrd.data_input_field_id=data_input_fields.id
-						AND data_template_rrd.local_data_id=' . $item['local_data_id']), 'data_name', 'data_source_name');
+					$rrd_field_names = array_rekey(
+						db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+							FROM graph_templates_item AS gti
+							INNER JOIN data_template_rrd AS dtr
+							ON gti.task_item_id = dtr.id
+							INNER JOIN data_input_fields AS dif
+							ON dtr.data_input_field_id = dif.id
+							AND dtr.local_data_id = ?',
+							array($item['local_data_id'])),
+						'data_name', 'data_source_name'
+					);
 
 					$rrd_tmpl = '';
 				}
 
 				$first_tmpl = true;
 				$multi_ok   = false;
 
-				for ($i=0; $i < count($values); $i++) {
-					if (preg_match('/^([a-zA-Z0-9_\.-]+):([eE0-9Uu\+\.-]+)$/', $values[$i], $matches)) {
-						if (isset($rrd_field_names[$matches[1]])) {
+				if (cacti_sizeof($values)) {
+					foreach($values as $value) {
+						$matches = explode(':', $value);
+
+						if (isset($rrd_field_names[$matches[0]])) {
 							$multi_ok = true;
 
 							if (trim(read_config_option('path_boost_log')) != '') {
@@ -899,19 +906,18 @@ function boost_process_local_data_ids($last_id, $child, $rrdtool_pipe) {
 									$rrd_tmpl .= ':';
 								}
 
-								$rrd_tmpl .= $rrd_field_names[$matches[1]];
+								$rrd_tmpl .= $rrd_field_names[$matches[0]];
 								$first_tmpl = false;
 							}
 
-							if (is_numeric($matches[2]) || ($matches[2] == 'U')) {
-								$tv_tmpl[$rrd_field_names[$matches[1]]] = $matches[2];
-								$buflen += strlen(':' . $matches[2]);
-							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[2]))) {
-								$tval                                   = hexdec($matches[2]);
-								$tv_tmpl[$rrd_field_names[$matches[1]]] = $tval;
+							if (is_numeric($matches[1]) || ($matches[1] == 'U')) {
+								$tv_tmpl[$rrd_field_names[$matches[0]]] = $matches[1];
+								$buflen += strlen(':' . $matches[1]);
+							} elseif ((function_exists('is_hexadecimal')) && (is_hexadecimal($matches[1]))) {
+								$tv_tmpl[$rrd_field_names[$matches[0]]] = hexdec($matches[1]);
 								$buflen += strlen(':' . $tval);
 							} else {
-								$tv_tmpl[$rrd_field_names[$matches[1]]] = 'U';
+								$tv_tmpl[$rrd_field_names[$matches[0]]] = 'U';
 								$buflen += 2;
 							}
 						}

diff --git a/poller_realtime.php b/poller_realtime.php
@@ -258,21 +258,28 @@ function process_poller_output_rt($rrdtool_pipe, $poller_id, $interval) {
 			/* single one value output */
 			if ((is_numeric($value)) || ($value == 'U')) {
 				$rrd_update_array[$item['rrd_path']]['times'][$unix_time][$item['rrd_name']] = $value;
-				/* multiple value output */
 			} else {
-				$values = explode(' ', $value);
-
-				$rrd_field_names = array_rekey(db_fetch_assoc_prepared('SELECT
-					data_template_rrd.data_source_name,
-					data_input_fields.data_name
-					FROM (data_template_rrd,data_input_fields)
-					WHERE data_template_rrd.data_input_field_id=data_input_fields.id
-					AND data_template_rrd.local_data_id = ?', array($item['local_data_id'])), 'data_name', 'data_source_name');
-
-				for ($i=0; $i < cacti_count($values); $i++) {
-					if (preg_match('/^([a-zA-Z0-9_\.-]+):([eE0-9\+\.-]+)$/', $values[$i], $matches)) {
-						if (isset($rrd_field_names[$matches[1]])) {
-							$rrd_update_array[$item['rrd_path']]['times'][$unix_time][$rrd_field_names[$matches[1]]] = $matches[2];
+				/* multiple value output */
+				$values = preg_split('/\s+/', $value);
+
+				$rrd_field_names = array_rekey(
+					db_fetch_assoc_prepared('SELECT DISTINCT dtr.data_source_name, dif.data_name
+						FROM graph_templates_item AS gti
+						INNER JOIN data_template_rrd AS dtr
+						ON gti.task_item_id = dtr.id
+						INNER JOIN data_input_fields AS dif
+						ON dtr.data_input_field_id = dif.id
+						AND dtr.local_data_id = ?',
+						array($item['local_data_id'])),
+					'data_name', 'data_source_name'
+				);
+
+				if (cacti_sizeof($values)) {
+					foreach($values as $value) {
+						$matches = explode(':', $value);
+
+						if (isset($rrd_field_names[$matches[0]])) {
+							$rrd_update_array[$item['rrd_path']]['times'][$unix_time][$rrd_field_names[$matches[0]]] = $matches[1];
 						}
 					}
 				}