diff --git a/README.md b/README.md index 1cff71a..0ea411c 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ individual data providers) # Release Notes +* 0.1.28 - findatapy (19 Jul 2022) * 0.1.27 - findatapy (20 May 2022) * 0.1.26 - findatapy (07 Oct 2021) * 0.1.25 - findatapy (07 Oct 2021) @@ -134,6 +135,8 @@ individual data providers) # Coding log +* 19 Jul 2022 + * Various fixes for data download * 20 May 2022 * Added more customisation for data vendors * 25 Jan 2022 diff --git a/findatapy/market/datavendor.py b/findatapy/market/datavendor.py index f79ec65..3e088f9 100644 --- a/findatapy/market/datavendor.py +++ b/findatapy/market/datavendor.py @@ -55,7 +55,8 @@ def load_ticker(self, md_request): def kill_session(self): return - def construct_vendor_md_request(self, md_request): + def construct_vendor_md_request(self, md_request, + fill_vendors_tickers_only=False): """Creates a MarketDataRequest with the vendor tickers Parameters @@ -64,21 +65,25 @@ def construct_vendor_md_request(self, md_request): contains all the various parameters detailing time series start and finish, tickers etc + fill_vendors_tickers_only : bool + Only search for vendors tickers (and ignore fields etc.) + Returns ------- MarketDataRequest """ - symbols_vendor = self.translate_to_vendor_ticker(md_request) - fields_vendor = self.translate_to_vendor_field(md_request) - md_request_vendor = MarketDataRequest( md_request=md_request) - md_request_vendor.tickers = symbols_vendor - md_request_vendor.fields = fields_vendor + md_request_vendor.tickers = self.translate_to_vendor_ticker(md_request) + + if not fill_vendors_tickers_only: + md_request_vendor.fields = \ + self.translate_to_vendor_field(md_request) - md_request_vendor.old_tickers = md_request.tickers + md_request_vendor.old_tickers = \ + md_request.tickers return md_request_vendor diff --git a/findatapy/market/datavendorbbg.py b/findatapy/market/datavendorbbg.py index 01c91b3..e133f7c 100644 --- a/findatapy/market/datavendorbbg.py +++ b/findatapy/market/datavendorbbg.py @@ -954,8 +954,7 @@ def process_message(self, msg): # names=['field', 'ticker']) data_frame.index = pd.to_datetime(data_frame.index) logger.info("Read: " + ticker + ' ' + str( - data_frame.index[0]) + ' - ' + str(data_frame.index[-1])) - + data_frame.index.min()) + ' - ' + str(data_frame.index.max())) return data_frame diff --git a/findatapy/market/marketdatagenerator.py b/findatapy/market/marketdatagenerator.py index b73f406..cf24642 100644 --- a/findatapy/market/marketdatagenerator.py +++ b/findatapy/market/marketdatagenerator.py @@ -211,6 +211,20 @@ def fetch_market_data(self, md_request): md_request.category, md_request.data_source, md_request.freq, md_request.cut) + if md_request.pretransformation is not None: + df_tickers = ConfigManager().get_instance()\ + .get_dataframe_tickers() + + df_tickers = df_tickers[ + (df_tickers["category"] == md_request.category) & + (df_tickers["data_source"] == md_request.data_source) & + (df_tickers["freq"] == md_request.freq) & + (df_tickers["cut"] == md_request.cut)] + + if "pretransformation" in df_tickers.columns: + md_request.pretransformation = \ + df_tickers["pretransformation"].tolist() + # intraday or tick: only one ticker per cache file if md_request.freq in ["intraday", "tick", "second", "hour", "minute"]: @@ -559,6 +573,10 @@ def download_daily(self, md_request): md_request_single.vendor_tickers = \ md_request.vendor_tickers[i:i + group_size] + if md_request.pretransformation is not None: + md_request_single.pretransformation = \ + md_request.pretransformation[i:i + group_size] + md_request_list.append(md_request_single) # Special case where we make smaller calls one after the other diff --git a/findatapy/market/marketdatarequest.py b/findatapy/market/marketdatarequest.py index 96bf5df..979b767 100644 --- a/findatapy/market/marketdatarequest.py +++ b/findatapy/market/marketdatarequest.py @@ -109,6 +109,11 @@ def __init__(self, data_source=None, fred_api_key=data_constants.fred_api_key, alpha_vantage_api_key=data_constants.alpha_vantage_api_key, eikon_api_key=data_constants.eikon_api_key, + macrobond_client_id=data_constants.macrobond_client_id, + macrobond_client_secret=data_constants.macrobond_client_secret, + + pretransformation=None, + push_to_cache=True, overrides={}, freeform_md_request={}, @@ -170,6 +175,13 @@ def __init__(self, data_source=None, self.alpha_vantage_api_key = \ copy.deepcopy(md_request.alpha_vantage_api_key) self.eikon_api_key = copy.deepcopy(md_request.eikon_api_key) + self.macrobond_client_id = \ + copy.deepcopy(md_request.macrobond_client_id) + self.macrobond_client_secret = \ + copy.deepcopy(md_request.macrobond_client_secret) + + self.pretransformation = copy.deepcopy(md_request.pretransformation) + self.overrides = copy.deepcopy(md_request.overrides) self.push_to_cache = copy.deepcopy(md_request.push_to_cache) self.freeform_md_request = \ @@ -219,6 +231,10 @@ def __init__(self, data_source=None, self.fred_api_key = fred_api_key self.alpha_vantage_api_key = alpha_vantage_api_key self.eikon_api_key = eikon_api_key + self.macrobond_client_id = macrobond_client_id + self.macrobond_client_secret = macrobond_client_secret + + self.pretransformation = pretransformation self.overrides = overrides self.push_to_cache = push_to_cache @@ -733,7 +749,34 @@ def eikon_api_key(self): @eikon_api_key.setter def eikon_api_key(self, eikon_api_key): self.__eikon_api_key = eikon_api_key + + @property + def macrobond_client_id(self): + return self.__macrobond_client_id + @macrobond_client_id.setter + def macrobond_client_id(self, macrobond_client_id): + self.__macrobond_client_id = macrobond_client_id + + @property + def macrobond_client_secret(self): + return self.__macrobond_client_secret + + @macrobond_client_secret.setter + def macrobond_client_secret(self, macrobond_client_secret): + self.__macrobond_client_secret = macrobond_client_secret + + @property + def pretransformation(self): + return self.__pretransformation + + @pretransformation.setter + def pretransformation(self, pretransformation): + if not isinstance(pretransformation, list): + pretransformation = [pretransformation] + + self.__pretransformation = pretransformation + @property def overrides(self): return self.__overrides diff --git a/findatapy/util/configmanager.py b/findatapy/util/configmanager.py index b891e6e..9b3c461 100644 --- a/findatapy/util/configmanager.py +++ b/findatapy/util/configmanager.py @@ -88,6 +88,8 @@ def get_instance(cls, data_constants=None): @staticmethod def populate_time_series_dictionaries(data_constants=None): + logger = LoggerManager.getLogger(__name__) + if data_constants is None: data_constants = DataConstants() @@ -125,6 +127,7 @@ def populate_time_series_dictionaries(data_constants=None): # reader = csv.DictReader(open(tickers_list_file)) df = pd.read_csv(tickers_list_file) df = df.dropna(how="all") + df_tickers.append(df) for index, line in df.iterrows(): @@ -138,57 +141,61 @@ def populate_time_series_dictionaries(data_constants=None): for freq in freq_list: tickers = line["tickers"] cut = line["cut"] - vendor_tickers = line["vendor_tickers"] + vendor_tickers = str(line["vendor_tickers"]) expiry = None - try: - expiry = line["expiry"] - except: - pass - - if category != "": - # Conversion from library tickers to vendor vendor_tickers - ConfigManager.\ - _dict_time_series_tickers_list_library_to_vendor[ - category + "." + - data_source + "." + - freq + "." + - cut + "." + - tickers] = vendor_tickers - - try: - if expiry != "": - expiry = parse(expiry) + # Skip row where the vendor ticker hasn't been + # specified + if vendor_tickers.strip().lower() != "nan" \ + or vendor_tickers.strip() != "" \ + or vendor_tickers.strip().lower() != "none": + + if "expiry" in line.keys(): + expiry = line["expiry"] + + if category != "": + # Conversion from library tickers to vendor vendor_tickers + ConfigManager.\ + _dict_time_series_tickers_list_library_to_vendor[ + category + "." + + data_source + "." + + freq + "." + + cut + "." + + tickers] = vendor_tickers + + try: + if expiry != "": + expiry = parse(expiry) + else: + expiry = None + except: + pass + + # Library of tickers by category + key = category + "." + data_source + "." + freq \ + + "." + cut + + # Conversion from library tickers to library expiry date + ConfigManager._dict_time_series_ticker_expiry_date_library_to_library[ + data_source + "." + + tickers] = expiry + + # Conversion from vendor vendor_tickers to library tickers + try: + ConfigManager._dict_time_series_tickers_list_vendor_to_library[ + key + "." + vendor_tickers] = tickers + except: + logger.warning( + "Ticker not specified correctly (is some " + "of this missing?) " + str( + key) + "." + str(vendor_tickers)) + + if key in ConfigManager._dict_time_series_category_tickers_library_to_library: + ConfigManager._dict_time_series_category_tickers_library_to_library[ + key].append(tickers) else: - expiry = None - except: - pass - - # Library of tickers by category - key = category + "." + data_source + "." + freq \ - + "." + cut - - # Conversion from library tickers to library expiry date - ConfigManager._dict_time_series_ticker_expiry_date_library_to_library[ - data_source + "." + - tickers] = expiry - - # Conversion from vendor vendor_tickers to library tickers - try: - ConfigManager._dict_time_series_tickers_list_vendor_to_library[ - key + "." + vendor_tickers] = tickers - except: - logger.warning( - "Ticker not specified correctly (is some " - "of this missing?) " + str( - key) + "." + str(vendor_tickers)) - - if key in ConfigManager._dict_time_series_category_tickers_library_to_library: - ConfigManager._dict_time_series_category_tickers_library_to_library[ - key].append(tickers) - else: - ConfigManager._dict_time_series_category_tickers_library_to_library[ - key] = [tickers] + ConfigManager._dict_time_series_category_tickers_library_to_library[ + key] = [tickers] try: df_tickers = pd.concat(df_tickers).sort_values( @@ -431,11 +438,14 @@ def split_ticker_string(md_request_str): @staticmethod def smart_group_dataframe_tickers(df, ret_fields=["category", "data_source", - "freq", "cut"]): + "freq", "cut"], + data_constants=None): """Groups together a DataFrame of metadata associated with assets, which can be used to create MarketDataRequest objects """ + if data_constants is None: + data_constants = DataConstants() if ret_fields is None: ret_fields = df.columns.to_list() diff --git a/findatapy/util/dataconstants.py b/findatapy/util/dataconstants.py index b9dceee..e9ba7e6 100644 --- a/findatapy/util/dataconstants.py +++ b/findatapy/util/dataconstants.py @@ -207,11 +207,15 @@ class DataConstants(object): 'cal-non-settle-dates': 'CALENDAR_NON_SETTLEMENT_DATES' } - # Depending on the ticker field inclusion of specific keywords, apply a particular BBG override (make sure all lowercase) + # Depending on the ticker field inclusion of specific keywords, + # apply a particular BBG override (make sure all lowercase) bbg_keyword_dict_override = { - 'RELEASE_STAGE_OVERRIDE' : {'A' : ['gdp', 'advance'], - 'F' : ['gdp', 'final'], - 'P' : ['gdp', 'preliminary']} + 'RELEASE_STAGE_OVERRIDE' : {'A': ['gdp', 'advance'], + 'F': ['gdp', 'final'], + 'P': ['gdp', 'preliminary'], + 'F': ['cpi', 'final'], + 'P': ['cpi', 'preliminary'] + } } ####### Dukascopy settings @@ -234,6 +238,10 @@ class DataConstants(object): ####### Eikon settings eikon_api_key = key_store("Eikon") + ####### Macrobond settings + macrobond_client_id = key_store("Macrobond_client_id") + macrobond_client_secret = key_store("Macrobond_client_secret") + ####### Twitter settings (you need to set these up on Twitter) TWITTER_APP_KEY = key_store("Twitter App Key") TWITTER_APP_SECRET = key_store("Twitter App Secret") diff --git a/setup.py b/setup.py index 8fc7f8b..286aa56 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ tickers, using configuration files. There is also functionality which is particularly useful for those downloading FX market data.""" setup(name='findatapy', - version='0.1.27', + version='0.1.28', description='Market data library', author='Saeed Amen', author_email='saeed@cuemacro.com',