Merge pull request #51 from Wikia/dev

0.0.7 version with GDrive auth temporary fix and MySQL extension

.githooks/pre-commit

@@ -3,4 +3,4 @@ set -e
 # Run flake8
 flake8 .
 
-python ./.githooks/isort_hooks.py
+python3 ./.githooks/isort_hooks.py

.gitignore

@@ -12,3 +12,4 @@ sroka.egg-info/
 **.csv
 config.ini
 sroka_test_file.py
+build

README.md

@@ -15,7 +15,7 @@ Package providing simple Python access to data in:
 * MySQL
 * neo4j
 
-Sroka library was checked to work for Python **>=3.6**.
+Sroka library was checked to work for Python **3.7, 3.8 and 3.9**.
 
 ## Developers
 

requirements.txt

@@ -17,3 +17,4 @@ requests>=2.20
 retrying>=1.3.3
 urllib3>=1.26.5
 py2neo>=4.2.0
+db-dtypes

setup.py

@@ -8,10 +8,10 @@
 
 setuptools.setup(
     name="sroka",
-    version="0.0.6",
+    version="0.0.7",
     author="Ad Engineering FANDOM",
     author_email="murbanek@fandom.com",
-    description="Package for access GA, GAM, MOAT, Qubole, Athena, S3, Rubicon APIs, BigQuery",
+    description="Package for access GA, GAM, MOAT, Qubole, Athena, S3, Rubicon APIs, BigQuery, MySQL",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/Wikia/sroka",

sroka/api/ga/ga.py

@@ -1,13 +1,13 @@
 import os
+from contextlib import contextmanager
+from typing import Dict
 
 import pandas as pd
-from typing import Dict
 from google.auth.exceptions import RefreshError
 from googleapiclient import discovery
 from googleapiclient.errors import HttpError
 
 import sroka.config.config as config
-from contextlib import contextmanager
 
 
 class GADataNotYetAvailable(Exception):
@@ -165,7 +165,7 @@ def ga_request(input_dict, print_sample_size=False, sampling_level='HIGHER_PRECI
             __print_sample_size(print_sample_size, results)
 
             return df
-    except:
+    except Exception:
         # the error message is displayed by context manager
         # this except clause is here just to keep the old API behavior (an empty DataFrame in case of an error)
         return pd.DataFrame([])

sroka/api/mysql/README.md

@@ -13,12 +13,18 @@ Here are the configuration variables that the `mysql` connector supports:
 
 ## Methods
 
-### `query_mysql(input_query, filename)`
+### `query_mysql(input_query, filename, host=None, port=None, unix_socket=None, user=None, password=None, database=None)`
 
 #### Arguments
 
 * string `input_query` - query to run
 * string `filename` - path to the file in which to store the results (optional, if `filename=None`, results are returned as a [`pandas`](https://pandas.pydata.org/pandas-docs/stable/) [`DataFrame`](https://pandas.pydata.org/pandas-docs/stable/reference/frame.html))
+* string `host` - see description in `Configuration` section
+* string `port` - see description in `Configuration` section
+* string `unix_socket` - see description in `Configuration` section
+* string `user` - see description in `Configuration` section
+* string `password` - see description in `Configuration` section
+* string `database` - see description in `Configuration` section
 
 #### Returns
 
@@ -42,5 +48,13 @@ dataframe = query_mysql("""
     WHERE year='2018' and month='10' and day='07'
     LIMIT 10
 """)
+
+## Queries database with configuration different from config.ini file
+dataframe = query_mysql("""
+    SELECT * FROM table
+    WHERE year='2018' and month='10' and day='07'
+    LIMIT 10
+""", port='1111', database='db_name')
+
 ```
 

sroka/api/mysql/mysql.py

@@ -11,14 +11,31 @@
 @retry(stop_max_attempt_number=1,
        wait_exponential_multiplier=1 * 2,
        wait_exponential_max=1 * 2 * 2)
-def query_mysql(query: str, filename=None):
+def query_mysql(query: str, filename=None,
+                host=None, port=None,
+                unix_socket=None, user=None,
+                password=None, database=None
+                ):
     try:
         options = get_options_from_config()
 
     except NoSectionError:
         print('Missing MySQL section in configuration')
         return pd.DataFrame([])
 
+    if host:
+        options['host'] = host
+    if port:
+        options['port'] = port
+    if unix_socket:
+        options['unix_socket'] = unix_socket
+    if user:
+        options['user'] = user
+    if password:
+        options['password'] = password
+    if database:
+        options['database'] = database
+
     if not validate_options(options):
         return pd.DataFrame([])
 

sroka/config/config.py

@@ -17,7 +17,7 @@ def set_google_credentials(authorized_user_file,
         credentials = Credentials.from_authorized_user_file(authorized_user_file, scopes=[scope])
     except FileNotFoundError:
         flow = InstalledAppFlow.from_client_secrets_file(key_file_location, [scope])
-        credentials = flow.run_console()
+        credentials = flow.run_local_server()
         os.makedirs(os.path.expanduser('~/.cache/'), exist_ok=True)
         with open(authorized_user_file, 'w') as file:
             json.dump({