From 669818db5d6c0c8d4305475c7904b4bfe44699f9 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 14 Apr 2024 08:16:00 -0400 Subject: [PATCH 1/6] Add note to _load_csv --- andes/routines/tds.py | 1 + 1 file changed, 1 insertion(+) diff --git a/andes/routines/tds.py b/andes/routines/tds.py index c5ca28f50..6b506fead 100644 --- a/andes/routines/tds.py +++ b/andes/routines/tds.py @@ -949,6 +949,7 @@ def _load_csv(self, csv_file): logger.warning("Check if the CSV data file is generated from the test case.") else: logger.info("Output selection detected.") + # NOTE: data has first column as time, so the rest should be `n+m` from `Output` if data.shape[1] - 1 < (len(self.system.Output.xidx + self.system.Output.yidx)): logger.warning("CSV data contains fewer variables than required.") logger.warning("Check if the CSV data file is generated with selected output.") From e4bf4e43c1bad3720f4eb39dbf7925140eea50a6 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 14 Apr 2024 09:01:44 -0400 Subject: [PATCH 2/6] Move CSV data completetion in _load_csv; In run(), move _load_csv() after init() --- andes/routines/tds.py | 54 +++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/andes/routines/tds.py b/andes/routines/tds.py index 6b506fead..3a020f727 100644 --- a/andes/routines/tds.py +++ b/andes/routines/tds.py @@ -237,15 +237,8 @@ def init(self): # discard initialized values and use that from CSV if provided if self.data_csv is not None: - if system.Output.n < 1: - system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] - system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] - else: - xyidx = system.Output.xidx + [yidx+system.dae.n for yidx in system.Output.yidx] - _xy = np.zeros(system.dae.n + system.dae.m) - _xy[xyidx] = self.data_csv[self.k_csv, 1:] - system.dae.x[:] = _xy[:system.dae.n] - system.dae.y[:] = _xy[system.dae.n:] + system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] + system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] system.vars_to_models() # connect to data streaming server @@ -348,14 +341,6 @@ def run(self, no_summary=False, from_csv=None, **kwargs): system.exit_code += 1 return succeed - # load from csv is provided - if from_csv is not None: - self.from_csv = from_csv - else: - self.from_csv = system.options.get("from_csv") - if self.from_csv is not None: - self.data_csv = self._load_csv(self.from_csv) - if no_summary is False and (system.dae.t == 0): self.summary() @@ -365,6 +350,14 @@ def run(self, no_summary=False, from_csv=None, **kwargs): else: # resume simulation self.init_resume() + # load from csv is provided + if from_csv is not None: + self.from_csv = from_csv + else: + self.from_csv = system.options.get("from_csv") + if self.from_csv is not None: + self.data_csv = self._load_csv(self.from_csv) + if system.options.get("init") is True: logger.debug("Initialization only is requested and done") return self.initialized @@ -551,16 +544,8 @@ def _csv_step(self): system = self.system if self.data_csv is not None: - if system.Output.n < 1: - system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] - system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] - else: - xyidx = system.Output.xidx + [yidx+system.dae.n for yidx in system.Output.yidx] - _xy = np.zeros(system.dae.n + system.dae.m) - _xy[xyidx] = self.data_csv[self.k_csv, 1:] - system.dae.x[:] = _xy[:system.dae.n] - system.dae.y[:] = _xy[system.dae.n:] - + system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] + system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] system.vars_to_models() self.converged = True @@ -943,16 +928,25 @@ def _load_csv(self, csv_file): raise ValueError("Data from CSV is not 2-dimensional (time versus variable)") if data.shape[0] < 2: logger.warning("CSV data does not contain more than one time step.") - if data.shape[1] < (self.system.dae.m + self.system.dae.n): - if self.system.Output.n < 1: + + system = self.system + if data.shape[1] < (system.dae.m + system.dae.n): + if system.Output.n < 1: logger.warning("CSV data contains fewer variables than required.") logger.warning("Check if the CSV data file is generated from the test case.") else: logger.info("Output selection detected.") # NOTE: data has first column as time, so the rest should be `n+m` from `Output` - if data.shape[1] - 1 < (len(self.system.Output.xidx + self.system.Output.yidx)): + if data.shape[1] - 1 < (len(system.Output.xidx + system.Output.yidx)): logger.warning("CSV data contains fewer variables than required.") logger.warning("Check if the CSV data file is generated with selected output.") + else: + # reconstruct data with padding zeros + _xy = np.zeros((data.shape[0], system.dae.n + system.dae.m)) + xyidx = system.Output.xidx + [yidx+system.dae.n for yidx in system.Output.yidx] + t = data[:, 0] + _xy[:, xyidx] = data[:, 1:] + data = np.hstack((t[:, np.newaxis], _xy)) # set start and end times from data self.config.t0 = data[0, 0] From febb68ce208f9949e4491dcd40d7f5250dbd2ce8 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 14 Apr 2024 09:07:52 -0400 Subject: [PATCH 3/6] Update release-notes --- docs/source/release-notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index b4defd6bc..8cbe8adec 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -15,6 +15,7 @@ v1.9.3 (2024-04-XX) services that are specified as complex. This will allow generating simplified expressions. - Adjust `BusFreq.Tw.default` to 0.1. +- Add parameter from_csv=None in TDS.run() to allow loading data from CSV files at TDS begining. - Fix `TDS.init()` and `TDS._csv_step()` to fit loading from CSV when `Output` exists. v1.9.2 (2024-03-25) From 67925611f59d70a78d4934cfc3b7194b757318f8 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 16 Apr 2024 17:20:38 -0400 Subject: [PATCH 4/6] Improve _csv_step() by adding a helper function --- andes/routines/tds.py | 50 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/andes/routines/tds.py b/andes/routines/tds.py index 3a020f727..4b2a4c523 100644 --- a/andes/routines/tds.py +++ b/andes/routines/tds.py @@ -237,9 +237,7 @@ def init(self): # discard initialized values and use that from CSV if provided if self.data_csv is not None: - system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] - system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] - system.vars_to_models() + self._csv_output() # connect to data streaming server if system.streaming.dimec is None: @@ -344,13 +342,7 @@ def run(self, no_summary=False, from_csv=None, **kwargs): if no_summary is False and (system.dae.t == 0): self.summary() - # only initializing at t<0 allows to continue when `run` is called again. - if system.dae.t < 0: - self.init() - else: # resume simulation - self.init_resume() - - # load from csv is provided + # load from csv if provided if from_csv is not None: self.from_csv = from_csv else: @@ -358,6 +350,12 @@ def run(self, no_summary=False, from_csv=None, **kwargs): if self.from_csv is not None: self.data_csv = self._load_csv(self.from_csv) + # only initializing at t<0 allows to continue when `run` is called again. + if system.dae.t < 0: + self.init() + else: # resume simulation + self.init_resume() + if system.options.get("init") is True: logger.debug("Initialization only is requested and done") return self.initialized @@ -542,11 +540,7 @@ def _csv_step(self): while the remaining variables are set to zero. """ - system = self.system - if self.data_csv is not None: - system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] - system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] - system.vars_to_models() + self._csv_output() self.converged = True return self.converged @@ -940,13 +934,6 @@ def _load_csv(self, csv_file): if data.shape[1] - 1 < (len(system.Output.xidx + system.Output.yidx)): logger.warning("CSV data contains fewer variables than required.") logger.warning("Check if the CSV data file is generated with selected output.") - else: - # reconstruct data with padding zeros - _xy = np.zeros((data.shape[0], system.dae.n + system.dae.m)) - xyidx = system.Output.xidx + [yidx+system.dae.n for yidx in system.Output.yidx] - t = data[:, 0] - _xy[:, xyidx] = data[:, 1:] - data = np.hstack((t[:, np.newaxis], _xy)) # set start and end times from data self.config.t0 = data[0, 0] @@ -1103,3 +1090,22 @@ def check_criteria(self): res = deltadelta(self.system.dae.x[self.system.SynGen.delta_addr], self.config.ddelta_limit) return res + + def _csv_output(self): + """ + Helper function to fetch data when replaying from CSV file. + + When loading from a partial CSV file, the recorded data is loaded + while the rest of the variables remain to be initial values. + """ + system = self.system + if system.Output.n < 1: + system.dae.x[:] = self.data_csv[self.k_csv, 1:system.dae.n + 1] + system.dae.y[:] = self.data_csv[self.k_csv, system.dae.n + 1:system.dae.n + system.dae.m + 1] + else: + xidx = system.Output.xidx + system.dae.x[xidx] = self.data_csv[self.k_csv, 1:len(xidx) + 1] + yidx = system.Output.yidx + system.dae.y[yidx] = self.data_csv[self.k_csv, len(xidx) + 1:len(xidx) + len(yidx) + 1] + + system.vars_to_models() From e0a703ab8fbd412999cf4b71dfa585135a0e1e41 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 16 Apr 2024 17:38:18 -0400 Subject: [PATCH 5/6] Rename TDS._csv_output() as TDS._csv_data_to_dae() for clarity --- andes/routines/tds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/andes/routines/tds.py b/andes/routines/tds.py index 4b2a4c523..65ee526a6 100644 --- a/andes/routines/tds.py +++ b/andes/routines/tds.py @@ -237,7 +237,7 @@ def init(self): # discard initialized values and use that from CSV if provided if self.data_csv is not None: - self._csv_output() + self._csv_data_to_dae() # connect to data streaming server if system.streaming.dimec is None: @@ -1091,7 +1091,7 @@ def check_criteria(self): self.config.ddelta_limit) return res - def _csv_output(self): + def _csv_data_to_dae(self): """ Helper function to fetch data when replaying from CSV file. From 65e10e020d407eddfdcf124f24dffdd0a33b858d Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 16 Apr 2024 17:42:11 -0400 Subject: [PATCH 6/6] Minor fix --- andes/routines/tds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/andes/routines/tds.py b/andes/routines/tds.py index 65ee526a6..d55da6f8c 100644 --- a/andes/routines/tds.py +++ b/andes/routines/tds.py @@ -540,7 +540,7 @@ def _csv_step(self): while the remaining variables are set to zero. """ - self._csv_output() + self._csv_data_to_dae() self.converged = True return self.converged