fix panic during normlizing invalid data

microsoft · Nov 22, 2023 · 02c7a01 · 02c7a01
1 parent 98f569e
commit 02c7a01
Showing 1 changed file with 9 additions and 6 deletions.
diff --git a/scripts/data_collector/base.py b/scripts/data_collector/base.py
@@ -301,12 +301,15 @@ def _executor(self, file_path: Path):
             na_values={col: symbol_na if col == self._symbol_field_name else default_na for col in columns},
         )
 
-        df = self._normalize_obj.normalize(df)
-        if df is not None and not df.empty:
-            if self._end_date is not None:
-                _mask = pd.to_datetime(df[self._date_field_name]) <= pd.Timestamp(self._end_date)
-                df = df[_mask]
-            df.to_csv(self._target_dir.joinpath(file_path.name), index=False)
+        try:
+            df = self._normalize_obj.normalize(df)
+            if df is not None and not df.empty:
+                if self._end_date is not None:
+                    _mask = pd.to_datetime(df[self._date_field_name]) <= pd.Timestamp(self._end_date)
+                    df = df[_mask]
+                df.to_csv(self._target_dir.joinpath(file_path.name), index=False)
+        except Exception as e:
+            logger.error(f"normalize {file_path.name} failed, error: {e}")
 
     def normalize(self):
         logger.info("normalize data......")