add fetch_cryptocompare_*

pandas_ml_utils/__init__.py

@@ -14,6 +14,7 @@
 from pandas_ml_utils.analysis.selection import feature_selection
 from pandas.core.base import PandasObject
 import pandas as pd
+from pandas_ml_utils.datafetching.fetch_cryptocompare import fetch_cryptocompare_daily, fetch_cryptocompare_hourly
 
 
 # add functions to pandas
@@ -39,6 +40,8 @@
 
 # data fetcher
 setattr(pd, 'fetch_yahoo', fetch_yahoo)
+setattr(pd, 'fetch_cryptocompare_daily', fetch_cryptocompare_daily)
+setattr(pd, 'fetch_cryptocompare_hourly', fetch_cryptocompare_hourly)
 
 __doc__ = """
 The main concept is to extend pandas DataFrame objects such that you can apply any statistical or machine learning

pandas_ml_utils/datafetching/fetch_cryptocompare.py

@@ -1 +1,24 @@
+import datetime
+
+import cachetools
+import pandas as pd
+from pandas_ml_utils.extern.cryptocompare import CURR, LIMIT, TIME, get_historical_price_day, get_historical_price_hour
 # TODO need to be implemented analog fetch yahoo
+
+
+@cachetools.func.ttl_cache(maxsize=1, ttl=10 * 60)
+def fetch_cryptocompare_daily(coin, curr=CURR, limit=LIMIT) -> pd.DataFrame:
+    data = get_historical_price_day(coin, curr, limit)
+    return _data_to_frame(data)
+
+
+@cachetools.func.ttl_cache(maxsize=1, ttl=10 * 60)
+def fetch_cryptocompare_hourly(coin, curr=CURR, limit=LIMIT) -> pd.DataFrame:
+    data = get_historical_price_hour(coin, curr, limit)
+    return _data_to_frame(data)
+
+
+def _data_to_frame(data):
+    df = pd.DataFrame(data)
+    df.index = pd.DatetimeIndex(df[TIME].apply(datetime.datetime.fromtimestamp))
+    return df.drop(TIME)

pandas_ml_utils/extern/cryptocompare.py

@@ -4,6 +4,7 @@
 import logging
 import time
 
+import cachetools
 import requests
 
 _log = logging.getLogger(__name__)
@@ -42,15 +43,16 @@
 ###############################################################################
 
 
+@cachetools.func.ttl_cache(maxsize=1, ttl=10 * 60)
 def query_cryptocompare(url,errorCheck=True):
     try:
         _log.debug(url)
         response = requests.get(url).json()
     except Exception as e:
-        print('Error getting coin information. %s' % str(e))
+        _log.error('Error getting coin information. %s' % str(e))
         return None
     if errorCheck and (response.get('Response') == 'Error'):
-        print('[ERROR] %s' % response.get('Message'))
+        _log.error('[ERROR] %s' % response.get('Message'))
         return None
     return response
 
@@ -102,9 +104,10 @@ def get_historical_price_day(coin, curr=CURR, limit=LIMIT):
 
 
 def get_historical_price_hour(coin, curr=CURR, limit=LIMIT):
+    current_ts = int(time.time()) + 60 * 60 * 24 + 1 # to be on the safe side
     if limit is None or limit > LIMIT:
         _log.info("batch download < now")
-        data = query_cryptocompare(URL_HIST_PRICE_HOUR.format(coin, format_parameter(curr), MAX_LIMIT, MAX_INT))
+        data = query_cryptocompare(URL_HIST_PRICE_HOUR.format(coin, format_parameter(curr), MAX_LIMIT, current_ts))
         batch = data
 
         while True:
@@ -121,7 +124,7 @@ def get_historical_price_hour(coin, curr=CURR, limit=LIMIT):
                     data[DATA] = data[DATA][:limit]
                     return data
     else:
-        return query_cryptocompare(URL_HIST_PRICE_HOUR.format(coin, format_parameter(curr), limit, MAX_INT))
+        return query_cryptocompare(URL_HIST_PRICE_HOUR.format(coin, format_parameter(curr), limit, current_ts))
 
 
 def get_historical_price_minute(coin, curr=CURR, limit=LIMIT):