Added query runner for Google Analytics Data API (GA4) and Google Search Console API (getredash#5868)

* quick and dirty GA4 integration to support limited queries

request pattern:
{
   "propertyId": 123456789,
   "dateRanges": [{ "startDate": "yesterday", "endDate": "yesterday" }],
   "dimensions": [{ "name": "date" }],
   "metrics": [{ "name": "activeUsers" }]
}

* Update Dockerfile

* Update requirements_bundles.txt

related issue: 
getredash#5851
https://stackoverflow.com/questions/73929564/entrypoints-object-has-no-attribute-get-digital-ocean

* updated integration code to support all usecases for runReport endpoint (does not have support for runPivotReport yet)

* added google search console as query runner

* removed info logger

* removed files that were causing merge conflicts

* fixed failing testcases and pre-commit formatting changes

* added testcases

* fixed linting errors in test files

* code optimization

Author: vinodpandey

client/app/assets/images/db-logos/google_analytics4.png

@@ -0,0 +1,179 @@
+import datetime
+import logging
+from base64 import b64decode
+
+import requests
+
+from redash.query_runner import (
+    TYPE_DATE,
+    TYPE_DATETIME,
+    TYPE_FLOAT,
+    TYPE_INTEGER,
+    TYPE_STRING,
+    BaseQueryRunner,
+    register,
+)
+from redash.utils import json_dumps, json_loads
+
+logger = logging.getLogger(__name__)
+
+try:
+    import httplib2
+    from apiclient.discovery import build
+    from oauth2client.service_account import ServiceAccountCredentials
+
+    enabled = True
+except ImportError:
+    enabled = False
+
+types_conv = dict(
+    STRING=TYPE_STRING,
+    INTEGER=TYPE_INTEGER,
+    FLOAT=TYPE_FLOAT,
+    DATE=TYPE_DATE,
+    DATETIME=TYPE_DATETIME,
+)
+
+ga_report_endpoint = "https://analyticsdata.googleapis.com/v1beta/properties/{propertyId}:runReport"
+ga_metadata_endpoint = "https://analyticsdata.googleapis.com/v1beta/properties/{propertyId}/metadata"
+
+
+def format_column_value(column_name, value, columns):
+    column_type = [col for col in columns if col["name"] == column_name][0]["type"]
+
+    if column_type == TYPE_DATE:
+        value = datetime.datetime.strptime(value, "%Y%m%d")
+    elif column_type == TYPE_DATETIME:
+        if len(value) == 10:
+            value = datetime.datetime.strptime(value, "%Y%m%d%H")
+        elif len(value) == 12:
+            value = datetime.datetime.strptime(value, "%Y%m%d%H%M")
+        else:
+            raise Exception("Unknown date/time format in results: '{}'".format(value))
+
+    return value
+
+
+def get_formatted_column_json(column_name):
+    data_type = None
+
+    if column_name == "date":
+        data_type = "DATE"
+    elif column_name == "dateHour":
+        data_type = "DATETIME"
+
+    result = {
+        "name": column_name,
+        "friendly_name": column_name,
+        "type": types_conv.get(data_type, "string"),
+    }
+
+    return result
+
+
+def parse_ga_response(response):
+    columns = []
+
+    for dim_header in response["dimensionHeaders"]:
+        columns.append(get_formatted_column_json(dim_header["name"]))
+
+    for met_header in response["metricHeaders"]:
+        columns.append(get_formatted_column_json(met_header["name"]))
+
+    rows = []
+    for r in response["rows"]:
+        counter = 0
+        d = {}
+        for item in r["dimensionValues"]:
+            column_name = columns[counter]["name"]
+            value = item["value"]
+
+            d[column_name] = format_column_value(column_name, value, columns)
+            counter = counter + 1
+
+        for item in r["metricValues"]:
+            column_name = columns[counter]["name"]
+            value = item["value"]
+
+            d[column_name] = format_column_value(column_name, value, columns)
+            counter = counter + 1
+
+        rows.append(d)
+
+    return {"columns": columns, "rows": rows}
+
+
+class GoogleAnalytics4(BaseQueryRunner):
+    should_annotate_query = False
+
+    @classmethod
+    def type(cls):
+        return "google_analytics4"
+
+    @classmethod
+    def name(cls):
+        return "Google Analytics 4"
+
+    @classmethod
+    def enabled(cls):
+        return enabled
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+                "propertyId": {"type": "number", "title": "Property Id"},
+                "jsonKeyFile": {"type": "string", "title": "JSON Key File"},
+            },
+            "required": ["propertyId", "jsonKeyFile"],
+            "secret": ["jsonKeyFile"],
+        }
+
+    def _get_access_token(self):
+        key = json_loads(b64decode(self.configuration["jsonKeyFile"]))
+
+        scope = ["https://www.googleapis.com/auth/analytics.readonly"]
+        creds = ServiceAccountCredentials.from_json_keyfile_dict(key, scope)
+
+        build("analyticsdata", "v1beta", http=creds.authorize(httplib2.Http()))
+
+        return creds.access_token
+
+    def run_query(self, query, user):
+        access_token = self._get_access_token()
+        params = json_loads(query)
+
+        property_id = self.configuration["propertyId"]
+
+        headers = {"Content-Type": "application/json", "Authorization": f"Bearer {access_token}"}
+
+        url = ga_report_endpoint.replace("{propertyId}", str(property_id))
+        r = requests.post(url, json=params, headers=headers)
+        r.raise_for_status()
+
+        raw_result = r.json()
+
+        data = parse_ga_response(raw_result)
+
+        error = None
+        json_data = json_dumps(data)
+
+        return json_data, error
+
+    def test_connection(self):
+        try:
+            access_token = self._get_access_token()
+            property_id = self.configuration["propertyId"]
+
+            url = ga_metadata_endpoint.replace("{propertyId}", str(property_id))
+
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {access_token}"}
+
+            r = requests.get(url, headers=headers)
+            r.raise_for_status()
+        except Exception as e:
+            raise Exception(e)
+
+
+register(GoogleAnalytics4)

client/app/assets/images/db-logos/google_search_console.png

@@ -0,0 +1,160 @@
+import logging
+from base64 import b64decode
+from datetime import datetime
+
+from redash.query_runner import (
+    TYPE_DATE,
+    TYPE_DATETIME,
+    TYPE_FLOAT,
+    TYPE_INTEGER,
+    TYPE_STRING,
+    BaseSQLQueryRunner,
+    register,
+)
+from redash.utils import json_dumps, json_loads
+
+logger = logging.getLogger(__name__)
+
+try:
+    import httplib2
+    from apiclient.discovery import build
+    from apiclient.errors import HttpError
+    from oauth2client.service_account import ServiceAccountCredentials
+
+    enabled = True
+except ImportError:
+    enabled = False
+
+
+types_conv = dict(
+    STRING=TYPE_STRING,
+    INTEGER=TYPE_INTEGER,
+    FLOAT=TYPE_FLOAT,
+    DATE=TYPE_DATE,
+    DATETIME=TYPE_DATETIME,
+)
+
+
+def parse_ga_response(response, dimensions):
+    columns = []
+
+    for item in dimensions:
+        if item == "date":
+            data_type = "date"
+        else:
+            data_type = "string"
+        columns.append(
+            {
+                "name": item,
+                "friendly_name": item,
+                "type": data_type,
+            }
+        )
+
+    default_items = ["clicks", "impressions", "ctr", "position"]
+    for item in default_items:
+        columns.append({"name": item, "friendly_name": item, "type": "number"})
+
+    rows = []
+    for r in response.get("rows", []):
+        d = {}
+        for k, value in r.items():
+            if k == "keys":
+                for index, val in enumerate(value):
+                    column_name = columns[index]["name"]
+                    column_type = columns[index]["type"]
+                    val = get_formatted_value(column_type, val)
+                    d[column_name] = val
+            else:
+                column_name = k
+                column_type = [col for col in columns if col["name"] == column_name][0]["type"]
+                value = get_formatted_value(column_type, value)
+                d[column_name] = value
+        rows.append(d)
+
+    return {"columns": columns, "rows": rows}
+
+
+def get_formatted_value(column_type, value):
+    if column_type == "number":
+        value = round(value, 2)
+    elif column_type == TYPE_DATE:
+        value = datetime.strptime(value, "%Y-%m-%d")
+    elif column_type == TYPE_DATETIME:
+        if len(value) == 10:
+            value = datetime.strptime(value, "%Y%m%d%H")
+        elif len(value) == 12:
+            value = datetime.strptime(value, "%Y%m%d%H%M")
+        else:
+            raise Exception("Unknown date/time format in results: '{}'".format(value))
+    return value
+
+
+class GoogleSearchConsole(BaseSQLQueryRunner):
+    should_annotate_query = False
+
+    @classmethod
+    def type(cls):
+        return "google_search_console"
+
+    @classmethod
+    def name(cls):
+        return "Google Search Console"
+
+    @classmethod
+    def enabled(cls):
+        return enabled
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+                "siteURL": {"type": "string", "title": "Site URL"},
+                "jsonKeyFile": {"type": "string", "title": "JSON Key File"},
+            },
+            "required": ["jsonKeyFile"],
+            "secret": ["jsonKeyFile"],
+        }
+
+    def __init__(self, configuration):
+        super(GoogleSearchConsole, self).__init__(configuration)
+        self.syntax = "json"
+
+    def _get_search_service(self):
+        scope = ["https://www.googleapis.com/auth/webmasters.readonly"]
+        key = json_loads(b64decode(self.configuration["jsonKeyFile"]))
+        creds = ServiceAccountCredentials.from_json_keyfile_dict(key, scope)
+        return build("searchconsole", "v1", http=creds.authorize(httplib2.Http()))
+
+    def test_connection(self):
+        try:
+            service = self._get_search_service()
+            service.sites().list().execute()
+        except HttpError as e:
+            # Make sure we return a more readable error to the end user
+            raise Exception(e._get_reason())
+
+    def run_query(self, query, user):
+        logger.debug("Search Analytics is about to execute query: %s", query)
+        params = json_loads(query)
+        site_url = self.configuration["siteURL"]
+        api = self._get_search_service()
+
+        if len(params) > 0:
+            try:
+                response = api.searchanalytics().query(siteUrl=site_url, body=params).execute()
+                data = parse_ga_response(response, params["dimensions"])
+                error = None
+                json_data = json_dumps(data)
+            except HttpError as e:
+                # Make sure we return a more readable error to the end user
+                error = e._get_reason()
+                json_data = None
+        else:
+            error = "Wrong query format."
+            json_data = None
+        return json_data, error
+
+
+register(GoogleSearchConsole)

redash/query_runner/google_analytics4.py

@@ -327,6 +327,8 @@ def email_server_is_configured():
     "redash.query_runner.databend",
     "redash.query_runner.nz",
     "redash.query_runner.arango",
+    "redash.query_runner.google_analytics4",
+    "redash.query_runner.google_search_console",
 ]
 
 enabled_query_runners = array_from_string(