Various fixes for BBG data download

README.md

@@ -115,6 +115,7 @@ individual data providers)
 
 # Release Notes
 
+* 0.1.28 - findatapy (19 Jul 2022)
 * 0.1.27 - findatapy (20 May 2022)
 * 0.1.26 - findatapy (07 Oct 2021)
 * 0.1.25 - findatapy (07 Oct 2021)
@@ -134,6 +135,8 @@ individual data providers)
 
 # Coding log
 
+* 19 Jul 2022
+  * Various fixes for data download
 * 20 May 2022
   * Added more customisation for data vendors
 * 25 Jan 2022

findatapy/market/datavendor.py

@@ -55,7 +55,8 @@ def load_ticker(self, md_request):
     def kill_session(self):
         return
 
-    def construct_vendor_md_request(self, md_request):
+    def construct_vendor_md_request(self, md_request,
+                                    fill_vendors_tickers_only=False):
         """Creates a MarketDataRequest with the vendor tickers
 
         Parameters
@@ -64,21 +65,25 @@ def construct_vendor_md_request(self, md_request):
             contains all the various parameters detailing time series start and
             finish, tickers etc
 
+        fill_vendors_tickers_only : bool
+            Only search for vendors tickers (and ignore fields etc.)
+
         Returns
         -------
         MarketDataRequest
         """
 
-        symbols_vendor = self.translate_to_vendor_ticker(md_request)
-        fields_vendor = self.translate_to_vendor_field(md_request)
-
         md_request_vendor = MarketDataRequest(
             md_request=md_request)
 
-        md_request_vendor.tickers = symbols_vendor
-        md_request_vendor.fields = fields_vendor
+        md_request_vendor.tickers = self.translate_to_vendor_ticker(md_request)
+
+        if not fill_vendors_tickers_only:
+            md_request_vendor.fields = \
+                self.translate_to_vendor_field(md_request)
 
-        md_request_vendor.old_tickers = md_request.tickers
+            md_request_vendor.old_tickers = \
+                md_request.tickers
 
         return md_request_vendor
 

findatapy/market/datavendorbbg.py

@@ -954,8 +954,7 @@ def process_message(self, msg):
             # names=['field', 'ticker'])
             data_frame.index = pd.to_datetime(data_frame.index)
             logger.info("Read: " + ticker + ' ' + str(
-                data_frame.index[0]) + ' - ' + str(data_frame.index[-1]))
-
+                data_frame.index.min()) + ' - ' + str(data_frame.index.max()))
 
         return data_frame
 

findatapy/market/marketdatagenerator.py

@@ -211,6 +211,20 @@ def fetch_market_data(self, md_request):
                     md_request.category, md_request.data_source,
                     md_request.freq, md_request.cut)
 
+            if md_request.pretransformation is not None:
+                df_tickers = ConfigManager().get_instance()\
+                    .get_dataframe_tickers()
+
+                df_tickers = df_tickers[
+                    (df_tickers["category"] == md_request.category) &
+                    (df_tickers["data_source"] == md_request.data_source) &
+                    (df_tickers["freq"] == md_request.freq) &
+                    (df_tickers["cut"] == md_request.cut)]
+
+                if "pretransformation" in df_tickers.columns:
+                    md_request.pretransformation = \
+                        df_tickers["pretransformation"].tolist()
+
         # intraday or tick: only one ticker per cache file
         if md_request.freq in ["intraday", "tick", "second", "hour",
                                 "minute"]:
@@ -559,6 +573,10 @@ def download_daily(self, md_request):
                     md_request_single.vendor_tickers = \
                         md_request.vendor_tickers[i:i + group_size]
 
+                if md_request.pretransformation is not None:
+                    md_request_single.pretransformation = \
+                        md_request.pretransformation[i:i + group_size]
+
                 md_request_list.append(md_request_single)
 
             # Special case where we make smaller calls one after the other

findatapy/market/marketdatarequest.py

@@ -109,6 +109,11 @@ def __init__(self, data_source=None,
                  fred_api_key=data_constants.fred_api_key, 
                  alpha_vantage_api_key=data_constants.alpha_vantage_api_key,
                  eikon_api_key=data_constants.eikon_api_key,
+                 macrobond_client_id=data_constants.macrobond_client_id,
+                 macrobond_client_secret=data_constants.macrobond_client_secret,
+
+                 pretransformation=None,
+
                  push_to_cache=True,
                  overrides={},
                  freeform_md_request={},
@@ -170,6 +175,13 @@ def __init__(self, data_source=None,
                 self.alpha_vantage_api_key = \
                     copy.deepcopy(md_request.alpha_vantage_api_key)
                 self.eikon_api_key = copy.deepcopy(md_request.eikon_api_key)
+                self.macrobond_client_id = \
+                    copy.deepcopy(md_request.macrobond_client_id)
+                self.macrobond_client_secret = \
+                    copy.deepcopy(md_request.macrobond_client_secret)
+
+                self.pretransformation = copy.deepcopy(md_request.pretransformation)
+
                 self.overrides = copy.deepcopy(md_request.overrides)
                 self.push_to_cache = copy.deepcopy(md_request.push_to_cache)
                 self.freeform_md_request = \
@@ -219,6 +231,10 @@ def __init__(self, data_source=None,
             self.fred_api_key = fred_api_key
             self.alpha_vantage_api_key = alpha_vantage_api_key
             self.eikon_api_key = eikon_api_key
+            self.macrobond_client_id = macrobond_client_id
+            self.macrobond_client_secret = macrobond_client_secret
+
+            self.pretransformation = pretransformation
 
             self.overrides = overrides
             self.push_to_cache = push_to_cache
@@ -733,7 +749,34 @@ def eikon_api_key(self):
     @eikon_api_key.setter
     def eikon_api_key(self, eikon_api_key):
         self.__eikon_api_key = eikon_api_key
+
+    @property
+    def macrobond_client_id(self):
+        return self.__macrobond_client_id
 
+    @macrobond_client_id.setter
+    def macrobond_client_id(self, macrobond_client_id):
+        self.__macrobond_client_id = macrobond_client_id
+
+    @property
+    def macrobond_client_secret(self):
+        return self.__macrobond_client_secret
+
+    @macrobond_client_secret.setter
+    def macrobond_client_secret(self, macrobond_client_secret):
+        self.__macrobond_client_secret = macrobond_client_secret
+
+    @property
+    def pretransformation(self):
+        return self.__pretransformation
+
+    @pretransformation.setter
+    def pretransformation(self, pretransformation):
+        if not isinstance(pretransformation, list):
+            pretransformation = [pretransformation]
+
+        self.__pretransformation = pretransformation
+
     @property
     def overrides(self):
         return self.__overrides

findatapy/util/configmanager.py

@@ -88,6 +88,8 @@ def get_instance(cls, data_constants=None):
     @staticmethod
     def populate_time_series_dictionaries(data_constants=None):
 
+        logger = LoggerManager.getLogger(__name__)
+
         if data_constants is None:
             data_constants = DataConstants()
 
@@ -125,6 +127,7 @@ def populate_time_series_dictionaries(data_constants=None):
                 # reader = csv.DictReader(open(tickers_list_file))
                 df = pd.read_csv(tickers_list_file)
                 df = df.dropna(how="all")
+
                 df_tickers.append(df)
 
                 for index, line in df.iterrows():
@@ -138,57 +141,61 @@ def populate_time_series_dictionaries(data_constants=None):
                     for freq in freq_list:
                         tickers = line["tickers"]
                         cut = line["cut"]
-                        vendor_tickers = line["vendor_tickers"]
+                        vendor_tickers = str(line["vendor_tickers"])
                         expiry = None
 
-                        try:
-                            expiry = line["expiry"]
-                        except:
-                            pass
-
-                        if category != "":
-                            # Conversion from library tickers to vendor vendor_tickers
-                            ConfigManager.\
-                            _dict_time_series_tickers_list_library_to_vendor[
-                                category + "." +
-                                data_source + "." +
-                                freq + "." +
-                                cut + "." +
-                                tickers] = vendor_tickers
-
-                            try:
-                                if expiry != "":
-                                    expiry = parse(expiry)
+                        # Skip row where the vendor ticker hasn't been
+                        # specified
+                        if vendor_tickers.strip().lower() != "nan" \
+                            or vendor_tickers.strip() != "" \
+                            or vendor_tickers.strip().lower() != "none":
+
+                            if "expiry" in line.keys():
+                                expiry = line["expiry"]
+
+                            if category != "":
+                                # Conversion from library tickers to vendor vendor_tickers
+                                ConfigManager.\
+                                    _dict_time_series_tickers_list_library_to_vendor[
+                                        category + "." +
+                                        data_source + "." +
+                                        freq + "." +
+                                        cut + "." +
+                                        tickers] = vendor_tickers
+
+                                try:
+                                    if expiry != "":
+                                        expiry = parse(expiry)
+                                    else:
+                                        expiry = None
+                                except:
+                                    pass
+
+                                # Library of tickers by category
+                                key = category + "." + data_source + "." + freq \
+                                      + "." + cut
+
+                                # Conversion from library tickers to library expiry date
+                                ConfigManager._dict_time_series_ticker_expiry_date_library_to_library[
+                                    data_source + "." +
+                                    tickers] = expiry
+
+                                # Conversion from vendor vendor_tickers to library tickers
+                                try:
+                                    ConfigManager._dict_time_series_tickers_list_vendor_to_library[
+                                        key + "." + vendor_tickers] = tickers
+                                except:
+                                    logger.warning(
+                                        "Ticker not specified correctly (is some "
+                                        "of this missing?) " + str(
+                                            key) + "." + str(vendor_tickers))
+
+                                if key in ConfigManager._dict_time_series_category_tickers_library_to_library:
+                                    ConfigManager._dict_time_series_category_tickers_library_to_library[
+                                        key].append(tickers)
                                 else:
-                                    expiry = None
-                            except:
-                                pass
-
-                            # Library of tickers by category
-                            key = category + "." + data_source + "." + freq \
-                                  + "." + cut
-
-                            # Conversion from library tickers to library expiry date
-                            ConfigManager._dict_time_series_ticker_expiry_date_library_to_library[
-                                data_source + "." +
-                                tickers] = expiry
-
-                            # Conversion from vendor vendor_tickers to library tickers
-                            try:
-                                ConfigManager._dict_time_series_tickers_list_vendor_to_library[
-                                    key + "." + vendor_tickers] = tickers
-                            except:
-                                logger.warning(
-                                    "Ticker not specified correctly (is some "
-                                    "of this missing?) " + str(
-                                        key) + "." + str(vendor_tickers))
-
-                            if key in ConfigManager._dict_time_series_category_tickers_library_to_library:
-                                ConfigManager._dict_time_series_category_tickers_library_to_library[
-                                    key].append(tickers)
-                            else:
-                                ConfigManager._dict_time_series_category_tickers_library_to_library[
-                                    key] = [tickers]
+                                    ConfigManager._dict_time_series_category_tickers_library_to_library[
+                                        key] = [tickers]
 
         try:
             df_tickers = pd.concat(df_tickers).sort_values(
@@ -431,11 +438,14 @@ def split_ticker_string(md_request_str):
     @staticmethod
     def smart_group_dataframe_tickers(df,
                                       ret_fields=["category", "data_source",
-                                                  "freq", "cut"]):
+                                                  "freq", "cut"],
+                                      data_constants=None):
         """Groups together a DataFrame of metadata associated with assets, 
         which can be used to create MarketDataRequest
         objects
         """
+        if data_constants is None:
+            data_constants = DataConstants()
 
         if ret_fields is None:
             ret_fields = df.columns.to_list()

findatapy/util/dataconstants.py

@@ -207,11 +207,15 @@ class DataConstants(object):
                           'cal-non-settle-dates': 'CALENDAR_NON_SETTLEMENT_DATES'
     }
 
-    # Depending on the ticker field inclusion of specific keywords, apply a particular BBG override (make sure all lowercase)
+    # Depending on the ticker field inclusion of specific keywords,
+    # apply a particular BBG override (make sure all lowercase)
     bbg_keyword_dict_override = {
-        'RELEASE_STAGE_OVERRIDE' : {'A' : ['gdp', 'advance'],
-                                    'F' : ['gdp', 'final'],
-                                    'P' : ['gdp', 'preliminary']}
+        'RELEASE_STAGE_OVERRIDE' : {'A': ['gdp', 'advance'],
+                                    'F': ['gdp', 'final'],
+                                    'P': ['gdp', 'preliminary'],
+                                    'F': ['cpi', 'final'],
+                                    'P': ['cpi', 'preliminary']
+                                    }
     }
 
     #######  Dukascopy settings
@@ -234,6 +238,10 @@ class DataConstants(object):
     #######  Eikon settings
     eikon_api_key = key_store("Eikon")
 
+    ####### Macrobond settings
+    macrobond_client_id = key_store("Macrobond_client_id")
+    macrobond_client_secret = key_store("Macrobond_client_secret")
+
     #######  Twitter settings (you need to set these up on Twitter)
     TWITTER_APP_KEY             = key_store("Twitter App Key")
     TWITTER_APP_SECRET          = key_store("Twitter App Secret")

setup.py

@@ -5,7 +5,7 @@
 tickers, using configuration files. There is also functionality which is particularly useful for those downloading FX market data."""
 
 setup(name='findatapy',
-      version='0.1.27',
+      version='0.1.28',
       description='Market data library',
       author='Saeed Amen',
       author_email='saeed@cuemacro.com',