From c96866b21fe80b7dc29758d1aba5c771a78adede Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 18 Dec 2023 22:50:17 +0000 Subject: [PATCH 01/26] added sending of logs to code --- requirements.txt | 5 +++- src/main.py | 58 +++++++++++++++++++++++++------------ tests/test_log_dataclass.py | 22 ++++++++++++++ tests/test_resource_path.py | 4 +++ 4 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 tests/test_log_dataclass.py diff --git a/requirements.txt b/requirements.txt index 6430920..bdd7a5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ google-cloud-pubsub -google-cloud \ No newline at end of file +google-cloud +requests +dataclass_json +ndjson \ No newline at end of file diff --git a/src/main.py b/src/main.py index dbfee4c..ee992e1 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,8 @@ import json import time +import os +import requests +import ndjson from dataclasses import dataclass from urllib.parse import urlparse @@ -8,6 +11,34 @@ from google.api_core import retry from google.cloud import pubsub_v1 +REQUESTS_SESSION = requests.Session() + +FIRETAIL_API = os.getenv("FIRETAIL_API", "https://api.logging.eu-west-1.sandbox.firetail.app/aws/lb/bulk") +FIRETAIL_APP_TOKEN = os.environ["FIRETAIL_APP_TOKEN"] +PROJECT_ID = os.environ["PROJECT_ID"] #"gcp-test-395910" +SUBSCRIPTION_ID = os.environ["SUBSCRIPTION_ID"] #"firetail" + + +class FireTailFailedIngest(Exception): + pass + + +class InvalidRequest(Exception): + pass + + +class InvalidRawGCPLog(Exception): + pass + +def ship_logs(logs: list): + if logs == []: + return + response = REQUESTS_SESSION.post( + url=FIRETAIL_API, headers={"x-ft-app-key": FIRETAIL_APP_TOKEN}, data=ndjson.dumps(logs) + ) + if response.status_code != 201: + raise FireTailFailedIngest(response.text) + return response.json() def get_param_from_url(url): return [i.split("=") for i in url.split("?", 1)[-1].split("&")] @@ -82,15 +113,20 @@ class FireTailRequest: method: str httpProtocol: str uri: str + resource: str ip: str headers: dict[str, list[str]] @staticmethod def load_request(log: dict): + uri = log.get("httpRequest", {}).get("requestUrl") + backendPath = log.get("jsonPayload", {}).get("backendRequest", {}).get("path", "/") + resource = get_resource_path(uri, backendPath) return FireTailRequest( method=log.get("httpRequest", {}).get("requestMethod"), httpProtocol=log.get("httpRequest", {}).get("protocol"), - uri=log.get("httpRequest", {}).get("requestUrl"), + uri=uri, + resource=resource, headers={"User-Agent": [log.get("httpRequest", {}).get("userAgent")]}, ip=log.get("httpRequest", {}).get("remoteIp"), ) @@ -136,15 +172,6 @@ def load_log(log: dict): dateCreated=calc_datecreated_time(log["timestamp"]), ) - -def call_firetail(): - pass - - -def reformat_message(message): - return message - - def process_messages(subscriber, subscription_path, max_messages=3): # Wrap the subscriber in a 'with' block to automatically call close() to # close the underlying gRPC channel when done. @@ -163,22 +190,17 @@ def process_messages(subscriber, subscription_path, max_messages=3): continue ack_ids = [] + logs = [] for received_message in response.received_messages: print(f"Received: {received_message.message.data}.") data = json.loads(received_message.message.data.decode()) - print(FireTailLog.load_log(data)) + logs.append(FireTailLog.load_log(data)) ack_ids.append(received_message.ack_id) - + ship_logs(logs) # Acknowledges the received messages so they will not be sent again. subscriber.acknowledge(request={"subscription": subscription_path, "ack_ids": ack_ids}) - print(f"Received and acknowledged {len(response.received_messages)} messages from {subscription_path}.") - - if __name__ == "__main__": - PROJECT_ID = "gcp-test-395910" - SUBSCRIPTION_ID = "firetail" - subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(PROJECT_ID, SUBSCRIPTION_ID) diff --git a/tests/test_log_dataclass.py b/tests/test_log_dataclass.py new file mode 100644 index 0000000..767d1f6 --- /dev/null +++ b/tests/test_log_dataclass.py @@ -0,0 +1,22 @@ +import os +import sys +import json +sys.path.insert(0, ".") +sys.path.insert(1, "src/") +current_dir = os.path.dirname(__file__) + +os.environ['FIRETAIL_APP_TOKEN'] = "fake" +os.environ['PROJECT_ID'] = "fake" +os.environ['SUBSCRIPTION_ID'] = "fake" + +def read_file(file_name): + with open(file_name) as f: + return json.loads(f.read()) + +LOG = os.path.join(current_dir, "examples/2_dynamic_paths.json") +from main import FireTailLog + +def test_log_processed(): + log_file = read_file(LOG) + result = FireTailLog.load_log(log_file) + assert result.to_dict() == {'version': '1.0.0-alpha', 'metadata': {'apiId': '//apigateway.googleapis.com/projects/61377165045/locations/global/apis/firetail-testing-api-gateway', 'apiConfig': '//apigateway.googleapis.com/projects/61377165045/locations/global/apis/firetail-testing-api-gateway/configs/test', 'apiKey': '', 'apiMethod': '1.firetail_testing_api_gateway_022u7tyhkefhv_apigateway_gcp_test_395910_cloud_goog.Pet_id', 'backendRequestDuration': '29ms', 'backendRequestHostname': 'backend-cluster-google.com:443', 'backendRequestPath': '/?pet_id=1&pet2_id=67', 'consumerNumber': '0', 'responseDetails': 'via_upstream', 'logName': 'projects/gcp-test-395910/logs/apigateway.googleapis.com%2Frequests', 'resourceType': 'apigateway.googleapis.com/Gateway', 'requestPayload': False, 'responsePayload': False}, 'request': {'method': 'POST', 'httpProtocol': 'HTTP/1.1', 'uri': 'https://gatewat-s72dnn9.nw.gateway.dev/pets/1/67', 'resource': '/pets/{pet_id}/{pet2_id}', 'ip': '109.77.144.121', 'headers': {'User-Agent': ['PostmanRuntime/7.35.0']}}, 'response': {'statusCode': 405, 'headers': {'Content-Length': ['1613']}}, 'executionTime': 29, 'dateCreated': 1702587460111} \ No newline at end of file diff --git a/tests/test_resource_path.py b/tests/test_resource_path.py index 1eeb4c1..0ea3870 100644 --- a/tests/test_resource_path.py +++ b/tests/test_resource_path.py @@ -1,3 +1,4 @@ +import os import sys sys.path.insert(0, ".") @@ -5,6 +6,9 @@ from main import get_resource_path +os.environ['FIRETAIL_APP_TOKEN'] = "fake" +os.environ['PROJECT_ID'] = "fake" +os.environ['SUBSCRIPTION_ID'] = "fake" def test_resource_path(): result = get_resource_path("https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?item2=37") From b558a3eb2dc1bf1e84729615ba28c7df303a6312 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 18 Dec 2023 22:51:48 +0000 Subject: [PATCH 02/26] update function name --- src/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.py b/src/main.py index ee992e1..50f166c 100644 --- a/src/main.py +++ b/src/main.py @@ -17,7 +17,7 @@ FIRETAIL_APP_TOKEN = os.environ["FIRETAIL_APP_TOKEN"] PROJECT_ID = os.environ["PROJECT_ID"] #"gcp-test-395910" SUBSCRIPTION_ID = os.environ["SUBSCRIPTION_ID"] #"firetail" - +MAX_MESSAGES = os.getenv("MAX_MESSAGES", 10) class FireTailFailedIngest(Exception): pass @@ -172,7 +172,7 @@ def load_log(log: dict): dateCreated=calc_datecreated_time(log["timestamp"]), ) -def process_messages(subscriber, subscription_path, max_messages=3): +def process_bulk_messages(subscriber, subscription_path, max_messages=3): # Wrap the subscriber in a 'with' block to automatically call close() to # close the underlying gRPC channel when done. with subscriber: @@ -204,4 +204,4 @@ def process_messages(subscriber, subscription_path, max_messages=3): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(PROJECT_ID, SUBSCRIPTION_ID) - process_messages(subscriber, subscription_path) + process_bulk_messages(subscriber, subscription_path, MAX_MESSAGES) From 08d71db40049ab8041166cd73268b21d3dc613f4 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Mon, 8 Apr 2024 11:13:31 +0100 Subject: [PATCH 03/26] formatted code --- src/main.py | 66 ++++++++++++++++++++++++++----------- tests/test_log_dataclass.py | 42 ++++++++++++++++++++--- tests/test_resource_path.py | 19 +++++++---- 3 files changed, 96 insertions(+), 31 deletions(-) diff --git a/src/main.py b/src/main.py index 50f166c..75650d1 100644 --- a/src/main.py +++ b/src/main.py @@ -1,11 +1,11 @@ import json -import time import os -import requests -import ndjson +import time from dataclasses import dataclass from urllib.parse import urlparse +import ndjson +import requests from dataclasses_json import dataclass_json from dateutil import parser as date_parser from google.api_core import retry @@ -13,12 +13,15 @@ REQUESTS_SESSION = requests.Session() -FIRETAIL_API = os.getenv("FIRETAIL_API", "https://api.logging.eu-west-1.sandbox.firetail.app/aws/lb/bulk") +FIRETAIL_API = os.getenv( + "FIRETAIL_API", "https://api.logging.eu-west-1.prod.firetail.app/aws/lb/bulk" +) FIRETAIL_APP_TOKEN = os.environ["FIRETAIL_APP_TOKEN"] -PROJECT_ID = os.environ["PROJECT_ID"] #"gcp-test-395910" -SUBSCRIPTION_ID = os.environ["SUBSCRIPTION_ID"] #"firetail" +PROJECT_ID = os.environ["PROJECT_ID"] # "gcp-test-395910" +SUBSCRIPTION_ID = os.environ["SUBSCRIPTION_ID"] # "firetail" MAX_MESSAGES = os.getenv("MAX_MESSAGES", 10) + class FireTailFailedIngest(Exception): pass @@ -30,16 +33,21 @@ class InvalidRequest(Exception): class InvalidRawGCPLog(Exception): pass + def ship_logs(logs: list): if logs == []: return - response = REQUESTS_SESSION.post( - url=FIRETAIL_API, headers={"x-ft-app-key": FIRETAIL_APP_TOKEN}, data=ndjson.dumps(logs) - ) - if response.status_code != 201: - raise FireTailFailedIngest(response.text) + for i in range(0, len(logs), 100): + response = REQUESTS_SESSION.post( + url=FIRETAIL_API, + headers={"x-ft-app-key": FIRETAIL_APP_TOKEN}, + data=ndjson.dumps(logs[i : i + 100]), + ) + if response.status_code != 201: + raise FireTailFailedIngest(response.text) return response.json() + def get_param_from_url(url): return [i.split("=") for i in url.split("?", 1)[-1].split("&")] @@ -91,17 +99,23 @@ def load_metadata(log: dict): apiConfig=log.get("jsonPayload", {}).get("apiConfig"), apiKey=log.get("jsonPayload", {}).get("apiKey"), apiMethod=log.get("jsonPayload", {}).get("apiMethod"), - backendRequestDuration=log.get("jsonPayload", {}).get("backendRequest", {}).get("duration", "0ms"), + backendRequestDuration=log.get("jsonPayload", {}) + .get("backendRequest", {}) + .get("duration", "0ms"), backendRequestHostname=log.get("jsonPayload", {}) .get("backendRequest", {}) .get( "hostname", ), - backendRequestPath=log.get("jsonPayload", {}).get("backendRequest", {}).get("path", "/"), + backendRequestPath=log.get("jsonPayload", {}) + .get("backendRequest", {}) + .get("path", "/"), consumerNumber=log.get("jsonPayload", {}).get("consumerNumber"), responseDetails=log.get("jsonPayload", {}).get("responseDetails"), logName=log.get("logName"), - resourceType=log.get("resource", {}).get("type", "apigateway.googleapis.com/Gateway"), + resourceType=log.get("resource", {}).get( + "type", "apigateway.googleapis.com/Gateway" + ), requestPayload=False, responsePayload=False, ) @@ -120,7 +134,9 @@ class FireTailRequest: @staticmethod def load_request(log: dict): uri = log.get("httpRequest", {}).get("requestUrl") - backendPath = log.get("jsonPayload", {}).get("backendRequest", {}).get("path", "/") + backendPath = ( + log.get("jsonPayload", {}).get("backendRequest", {}).get("path", "/") + ) resource = get_resource_path(uri, backendPath) return FireTailRequest( method=log.get("httpRequest", {}).get("requestMethod"), @@ -142,7 +158,9 @@ class FireTailResponse: def load_response(log: dict): return FireTailResponse( statusCode=log.get("httpRequest", {}).get("status"), - headers={"Content-Length": [log.get("httpRequest", {}).get("responseSize", 0)]}, + headers={ + "Content-Length": [log.get("httpRequest", {}).get("responseSize", 0)] + }, ) @@ -158,7 +176,9 @@ class FireTailLog: @staticmethod def load_log(log: dict): - execution_time = log.get("jsonPayload", {}).get("backendRequest", {}).get("duration", "0ms") + execution_time = ( + log.get("jsonPayload", {}).get("backendRequest", {}).get("duration", "0ms") + ) try: execution_time = int(execution_time.replace("ms", "")) except: @@ -172,6 +192,7 @@ def load_log(log: dict): dateCreated=calc_datecreated_time(log["timestamp"]), ) + def process_bulk_messages(subscriber, subscription_path, max_messages=3): # Wrap the subscriber in a 'with' block to automatically call close() to # close the underlying gRPC channel when done. @@ -179,10 +200,12 @@ def process_bulk_messages(subscriber, subscription_path, max_messages=3): # The subscriber pulls a specific number of messages. The actual # number of messages pulled may be smaller than max_messages. while True: - print("loop") time.sleep(2) response = subscriber.pull( - request={"subscription": subscription_path, "max_messages": max_messages}, + request={ + "subscription": subscription_path, + "max_messages": max_messages, + }, retry=retry.Retry(deadline=300), ) @@ -198,7 +221,10 @@ def process_bulk_messages(subscriber, subscription_path, max_messages=3): ack_ids.append(received_message.ack_id) ship_logs(logs) # Acknowledges the received messages so they will not be sent again. - subscriber.acknowledge(request={"subscription": subscription_path, "ack_ids": ack_ids}) + subscriber.acknowledge( + request={"subscription": subscription_path, "ack_ids": ack_ids} + ) + if __name__ == "__main__": subscriber = pubsub_v1.SubscriberClient() diff --git a/tests/test_log_dataclass.py b/tests/test_log_dataclass.py index 767d1f6..f4445b7 100644 --- a/tests/test_log_dataclass.py +++ b/tests/test_log_dataclass.py @@ -1,22 +1,54 @@ +import json import os import sys -import json + sys.path.insert(0, ".") sys.path.insert(1, "src/") current_dir = os.path.dirname(__file__) -os.environ['FIRETAIL_APP_TOKEN'] = "fake" -os.environ['PROJECT_ID'] = "fake" -os.environ['SUBSCRIPTION_ID'] = "fake" +os.environ["FIRETAIL_APP_TOKEN"] = "fake" +os.environ["PROJECT_ID"] = "fake" +os.environ["SUBSCRIPTION_ID"] = "fake" + def read_file(file_name): with open(file_name) as f: return json.loads(f.read()) + LOG = os.path.join(current_dir, "examples/2_dynamic_paths.json") from main import FireTailLog + def test_log_processed(): log_file = read_file(LOG) result = FireTailLog.load_log(log_file) - assert result.to_dict() == {'version': '1.0.0-alpha', 'metadata': {'apiId': '//apigateway.googleapis.com/projects/61377165045/locations/global/apis/firetail-testing-api-gateway', 'apiConfig': '//apigateway.googleapis.com/projects/61377165045/locations/global/apis/firetail-testing-api-gateway/configs/test', 'apiKey': '', 'apiMethod': '1.firetail_testing_api_gateway_022u7tyhkefhv_apigateway_gcp_test_395910_cloud_goog.Pet_id', 'backendRequestDuration': '29ms', 'backendRequestHostname': 'backend-cluster-google.com:443', 'backendRequestPath': '/?pet_id=1&pet2_id=67', 'consumerNumber': '0', 'responseDetails': 'via_upstream', 'logName': 'projects/gcp-test-395910/logs/apigateway.googleapis.com%2Frequests', 'resourceType': 'apigateway.googleapis.com/Gateway', 'requestPayload': False, 'responsePayload': False}, 'request': {'method': 'POST', 'httpProtocol': 'HTTP/1.1', 'uri': 'https://gatewat-s72dnn9.nw.gateway.dev/pets/1/67', 'resource': '/pets/{pet_id}/{pet2_id}', 'ip': '109.77.144.121', 'headers': {'User-Agent': ['PostmanRuntime/7.35.0']}}, 'response': {'statusCode': 405, 'headers': {'Content-Length': ['1613']}}, 'executionTime': 29, 'dateCreated': 1702587460111} \ No newline at end of file + assert result.to_dict() == { + "version": "1.0.0-alpha", + "metadata": { + "apiId": "//apigateway.googleapis.com/projects/61377165045/locations/global/apis/firetail-testing-api-gateway", + "apiConfig": "//apigateway.googleapis.com/projects/61377165045/locations/global/apis/firetail-testing-api-gateway/configs/test", + "apiKey": "", + "apiMethod": "1.firetail_testing_api_gateway_022u7tyhkefhv_apigateway_gcp_test_395910_cloud_goog.Pet_id", + "backendRequestDuration": "29ms", + "backendRequestHostname": "backend-cluster-google.com:443", + "backendRequestPath": "/?pet_id=1&pet2_id=67", + "consumerNumber": "0", + "responseDetails": "via_upstream", + "logName": "projects/gcp-test-395910/logs/apigateway.googleapis.com%2Frequests", + "resourceType": "apigateway.googleapis.com/Gateway", + "requestPayload": False, + "responsePayload": False, + }, + "request": { + "method": "POST", + "httpProtocol": "HTTP/1.1", + "uri": "https://gatewat-s72dnn9.nw.gateway.dev/pets/1/67", + "resource": "/pets/{pet_id}/{pet2_id}", + "ip": "109.77.144.121", + "headers": {"User-Agent": ["PostmanRuntime/7.35.0"]}, + }, + "response": {"statusCode": 405, "headers": {"Content-Length": ["1613"]}}, + "executionTime": 29, + "dateCreated": 1702587460111, + } diff --git a/tests/test_resource_path.py b/tests/test_resource_path.py index 0ea3870..4578a70 100644 --- a/tests/test_resource_path.py +++ b/tests/test_resource_path.py @@ -6,22 +6,29 @@ from main import get_resource_path -os.environ['FIRETAIL_APP_TOKEN'] = "fake" -os.environ['PROJECT_ID'] = "fake" -os.environ['SUBSCRIPTION_ID'] = "fake" +os.environ["FIRETAIL_APP_TOKEN"] = "fake" +os.environ["PROJECT_ID"] = "fake" +os.environ["SUBSCRIPTION_ID"] = "fake" + def test_resource_path(): - result = get_resource_path("https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?item2=37") + result = get_resource_path( + "https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?item2=37" + ) assert result == "/pets/1/{item2}" def test_resource_path_2(): - result = get_resource_path("https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?hello=1&item2=37") + result = get_resource_path( + "https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?hello=1&item2=37" + ) assert result == "/pets/{hello}/{item2}" def test_resource_path_3(): - result = get_resource_path("https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?hello=1&item2=3") + result = get_resource_path( + "https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/?hello=1&item2=3" + ) assert result == "/pets/{hello}/37" From ccf583980a5c085716689fa3f4262a988205c87c Mon Sep 17 00:00:00 2001 From: riley priddle Date: Tue, 9 Apr 2024 22:18:54 +0100 Subject: [PATCH 04/26] added changes to support pubsub --- .gitignore | 3 +- build_setup/bundle.sh | 2 + build_setup/iac.sh | 39 +++++++++++++++++++ readme.md | 5 +++ requirements.txt | 5 --- setup.sh | 30 +++++++++++++++ src/main.py | 74 ++++++++++++------------------------- src/requirements.txt | 7 ++++ tests/test_resource_path.py | 6 +++ 9 files changed, 114 insertions(+), 57 deletions(-) create mode 100644 build_setup/bundle.sh create mode 100644 build_setup/iac.sh create mode 100644 readme.md delete mode 100644 requirements.txt create mode 100755 setup.sh create mode 100644 src/requirements.txt diff --git a/.gitignore b/.gitignore index ff74963..1f8e139 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ google-cloud-sdk .pytest_cache __pycache__ -venv \ No newline at end of file +venv +*.zip \ No newline at end of file diff --git a/build_setup/bundle.sh b/build_setup/bundle.sh new file mode 100644 index 0000000..e35ebde --- /dev/null +++ b/build_setup/bundle.sh @@ -0,0 +1,2 @@ +#!/bin/bash +zip -j gcp-function.zip src/* requirements.txt diff --git a/build_setup/iac.sh b/build_setup/iac.sh new file mode 100644 index 0000000..56f68d5 --- /dev/null +++ b/build_setup/iac.sh @@ -0,0 +1,39 @@ +CURRENT_PROJECT_ID=$(gcloud info --format="value(config.project)") + +GCP_REGION="europe-west1" +GATEWAY_NAME="gatewat" + +#gcloud pubsub subscriptions create --topic firetail-apigateway-stackdriver-sink firetail-subscription --project=$CURRENT_PROJECT_ID + +FILTERS="resource.type=apigateway.googleapis.com/Gateway AND resource.labels.gateway_id=gatewat AND resource.labels.location=europe-west1" +DESTINATION="pubsub.googleapis.com/projects/$CURRENT_PROJECT_ID/topics/firetail-apigateway-stackdriver-sink" + +gcloud pubsub topics create firetail-apigateway-stackdriver-sink --project=$CURRENT_PROJECT_ID + +gcloud logging sinks create firetail-log-routing $DESTINATION \ + --project=$CURRENT_PROJECT_ID \ + --description="log router for firetail" \ + --log-filter="resource.type=apigateway.googleapis.com/Gateway AND resource.labels.gateway_id=${GATEWAY_NAME} AND resource.labels.location=europe-west1" + +gcloud pubsub topics describe firetail-apigateway-stackdriver-sink --project=$CURRENT_PROJECT_ID + +service_account=$(gcloud logging sinks describe --format='value(writerIdentity)' firetail-log-routing) + +gcloud pubsub topics add-iam-policy-binding projects/${CURRENT_PROJECT_ID}/topics/firetail-apigateway-stackdriver-sink \ + --member=${service_account} --role=roles/pubsub.publisher + +gcloud functions deploy firetail-apigateway-logging-2 \ +--gen2 \ +--runtime="python312" \ +--entry-point="firetail_apigateway_logging" \ +--memory="256MB" \ +--timeout="60s" \ +--project=$CURRENT_PROJECT_ID \ +--region=$GCP_REGION \ +--source="src/" \ +--entry-point="subscribe" \ +--trigger-topic="firetail-apigateway-stackdriver-sink" \ +--set-env-vars FIRETAIL_APP_TOKEN=$FT_TOKEN,FIRETAIL_API="https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk" + + + diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..f04b5eb --- /dev/null +++ b/readme.md @@ -0,0 +1,5 @@ +https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https://github.com/firetail-io/repo + +./run.sh --app-token=<> --gcp_region=<> --firetail-api + + diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index bdd7a5a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -google-cloud-pubsub -google-cloud -requests -dataclass_json -ndjson \ No newline at end of file diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..92f3a68 --- /dev/null +++ b/setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +echo "Which region do you want to connect to?" +echo "1. EU" +echo "2. US" +read -p "Enter your choice (1 or 2): " choice + +case $choice in + 1) + endpoint="https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk" + ;; + 2) + endpoint="https://api.logging.us-east-2.prod.us.firetail.app/gcp/apigw/bulk" + ;; + *) + echo "Invalid choice. Please enter 1 or 2." + exit 1 + ;; +esac + +read -p "Enter APP Token: " token + +read -p "Enter Google location: " location + +gcloud api-gateway gateways list --location=$location + +read -p "Enter Google API Gateway ID: " gateway_id + + +echo "You have chosen to connect to the $location location. $gateway_id $endpoint" \ No newline at end of file diff --git a/src/main.py b/src/main.py index 75650d1..10e89b0 100644 --- a/src/main.py +++ b/src/main.py @@ -1,25 +1,21 @@ +import base64 import json import os import time from dataclasses import dataclass -from urllib.parse import urlparse +from urllib.parse import unquote_plus, urlparse +import functions_framework import ndjson import requests from dataclasses_json import dataclass_json from dateutil import parser as date_parser -from google.api_core import retry -from google.cloud import pubsub_v1 REQUESTS_SESSION = requests.Session() - FIRETAIL_API = os.getenv( "FIRETAIL_API", "https://api.logging.eu-west-1.prod.firetail.app/aws/lb/bulk" ) -FIRETAIL_APP_TOKEN = os.environ["FIRETAIL_APP_TOKEN"] -PROJECT_ID = os.environ["PROJECT_ID"] # "gcp-test-395910" -SUBSCRIPTION_ID = os.environ["SUBSCRIPTION_ID"] # "firetail" -MAX_MESSAGES = os.getenv("MAX_MESSAGES", 10) +FIRETAIL_APP_TOKEN = os.getenv("FIRETAIL_APP_TOKEN") class FireTailFailedIngest(Exception): @@ -91,6 +87,9 @@ class FireTailMetadata: resourceType: str requestPayload: bool responsePayload: bool + gatewayId: str | None = None + location: str | None = None + resourceContainer: str | None = None @staticmethod def load_metadata(log: dict): @@ -107,17 +106,22 @@ def load_metadata(log: dict): .get( "hostname", ), - backendRequestPath=log.get("jsonPayload", {}) - .get("backendRequest", {}) - .get("path", "/"), + backendRequestPath=unquote_plus( + log.get("jsonPayload", {}).get("backendRequest", {}).get("path", "/") + ), consumerNumber=log.get("jsonPayload", {}).get("consumerNumber"), responseDetails=log.get("jsonPayload", {}).get("responseDetails"), - logName=log.get("logName"), + logName=log.get("logName", ""), resourceType=log.get("resource", {}).get( "type", "apigateway.googleapis.com/Gateway" ), requestPayload=False, responsePayload=False, + gatewayId=log.get("resource", {}).get("labels", {}).get("gateway_id"), + location=log.get("resource", {}).get("labels", {}).get("location"), + resourceContainer=log.get("resource", {}) + .get("labels", {}) + .get("resource_container"), ) @@ -133,7 +137,7 @@ class FireTailRequest: @staticmethod def load_request(log: dict): - uri = log.get("httpRequest", {}).get("requestUrl") + uri = unquote_plus(log.get("httpRequest", {}).get("requestUrl")) backendPath = ( log.get("jsonPayload", {}).get("backendRequest", {}).get("path", "/") ) @@ -193,41 +197,9 @@ def load_log(log: dict): ) -def process_bulk_messages(subscriber, subscription_path, max_messages=3): - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - # The subscriber pulls a specific number of messages. The actual - # number of messages pulled may be smaller than max_messages. - while True: - time.sleep(2) - response = subscriber.pull( - request={ - "subscription": subscription_path, - "max_messages": max_messages, - }, - retry=retry.Retry(deadline=300), - ) - - if len(response.received_messages) == 0: - continue - - ack_ids = [] - logs = [] - for received_message in response.received_messages: - print(f"Received: {received_message.message.data}.") - data = json.loads(received_message.message.data.decode()) - logs.append(FireTailLog.load_log(data)) - ack_ids.append(received_message.ack_id) - ship_logs(logs) - # Acknowledges the received messages so they will not be sent again. - subscriber.acknowledge( - request={"subscription": subscription_path, "ack_ids": ack_ids} - ) - - -if __name__ == "__main__": - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(PROJECT_ID, SUBSCRIPTION_ID) - - process_bulk_messages(subscriber, subscription_path, MAX_MESSAGES) +@functions_framework.cloud_event +def subscribe(cloud_event): + log = json.loads( + base64.b64decode(cloud_event.data["message"]["data"]).decode("utf-8") + ) + ship_logs([FireTailLog.load_log(log).to_dict()]) diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..bd53957 --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,7 @@ +google-cloud-pubsub +google-cloud +requests +dataclasses-json +ndjson +functions-framework +python-dateutil \ No newline at end of file diff --git a/tests/test_resource_path.py b/tests/test_resource_path.py index 4578a70..823fe49 100644 --- a/tests/test_resource_path.py +++ b/tests/test_resource_path.py @@ -35,3 +35,9 @@ def test_resource_path_3(): def test_resource_path_4(): result = get_resource_path("https://gatewat-s72dnn9.nw.gateway.dev/pets/1/37", "/") assert result == "/pets/1/37" + +def test_resource_path_same_id(): + result = get_resource_path( + "https://gatewat-s72dnn9.nw.gateway.dev/pets/123/123", "/?pet_id=123&pet2_id=123" + ) + assert result == "/pets/{pet_id}/{pet2_id}" \ No newline at end of file From fd6e4b1a60eef8ffe8c5effe7f66364adcb49ed4 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Wed, 10 Apr 2024 13:47:56 +0100 Subject: [PATCH 05/26] added placeholder for run --- run.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 run.sh diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..cd1cf01 --- /dev/null +++ b/run.sh @@ -0,0 +1 @@ +echo "placeholder" \ No newline at end of file From bf0f043864732892bcdcf727dbf0c2b6452da879 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Wed, 10 Apr 2024 13:50:55 +0100 Subject: [PATCH 06/26] chmod+x run --- run.sh | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 run.sh diff --git a/run.sh b/run.sh old mode 100644 new mode 100755 index cd1cf01..6482d0f --- a/run.sh +++ b/run.sh @@ -1 +1,2 @@ +#!/bin/bash echo "placeholder" \ No newline at end of file From da1d58e45c979e12125ed603d0f3bd5e87529f22 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Wed, 10 Apr 2024 15:18:14 +0100 Subject: [PATCH 07/26] added new run --- run.sh | 193 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 192 insertions(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 6482d0f..66cdf88 100755 --- a/run.sh +++ b/run.sh @@ -1,2 +1,193 @@ #!/bin/bash -echo "placeholder" \ No newline at end of file + +LOG_DESTINATION="${LOG_DESTINATION:-${0}.log}" +FIRETAIL_API="${FIRETAIL_API:-https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk}" +DEFAULT_GCP_FUNCTION_NAME="firetail_logging" +PUBSUB_TOPIC_NAME="firetail-apigateway-stackdriver-sink" + +declare \ + FT_LOGGING_ENDPOINT \ + FT_APP_TOKEN \ + GCP_REGION \ + GCP_GATEWAY_ID \ + GCP_FUNCTION_NAME \ + GCP_PROJECT_ID + +function main() { + check_gcloud_cli + get_gcp_project_id + get_arguments "$@" + create_pubsub_topic + deploy_cloud_function +} + +function check_gcloud_cli() { + if ! command -v gcloud >/dev/null; then + local err_msg="Failed to get gcloud CLI from PATH. Please install and authenticate gcloud CLI" + log ERROR "${err_msg}" + alert_quit "${err_msg}" + fi +} + +function get_gcp_project_id() { + gcp_account="$(gcloud auth list --filter=status:ACTIVE --format="value(account)")" + log INFO "GCP Account: ${gcp_account}" + echo "GCP account: ${gcp_account}" + PS3="Enter the number of the project for Cloud Function deployment: " + select gcp_project_id in $(gcloud projects list --format="value(projectId)"); do + gcloud config set project "${gcp_project_id}" || + alert_quit "Failed to set project ID to ${gcp_project_id}" + GCP_PROJECT_ID="${gcp_project_id}" + log INFO "Cloud Function will deploy to ${gcp_project_id}" + # if invalid response, keep prompting + [[ -n "${response}" ]] && break + done +} + +function get_arguments() { + while true; do + case "$1" in + --help) + show_help + exit + ;; + --ft-logging-endpoint=*) + FT_LOGGING_ENDPOINT="${1#--ft-logging-endpoint=}" + log INFO "FT_LOGGING_ENDPOINT = ${FT_LOGGING_ENDPOINT}" + ;; + --ft--app-token=*) + FT_APP_TOKEN="${1#--ft-app-token=}" + log INFO "FT_APP_TOKEN = ${FT_APP_TOKEN}" + ;; + --gcp-region=*) + GCP_REGION="${1#--gcp-region=}" + log INFO "GCP_REGION = ${GCP_REGION}" + ;; + --gcp-gateway-id=*) + GCP_GATEWAY_ID="${1#--gcp-gateway-id=}" + log INFO "GCP_GATEWAY_ID = ${GCP_GATEWAY_ID}" + ;; + --gcp-function-name=*) + GCP_FUNCTION_NAME="${1#--gcp-function-name=}" + if [[ "${GCP_FUNCTION_NAME}" == "" ]]; then + log WARN "No function name provided, using default: '${DEFAULT_GCP_FUNCTION_NAME}'" + GCP_FUNCTION_NAME="${DEFAULT_GCP_FUNCTION_NAME}" + fi + log INFO "GCP_FUNCTION_NAME = ${GCP_FUNCTION_NAME}" + ;; + "") + break + ;; + *) + alert_quit "unrecognized flag; try '${0} --help' for more information" + ;; + esac + shift + done + check_args_provided +} + +function check_args_provided() { + if [[ -z "${FT_LOGGING_ENDPOINT}" ]]; then + alert_quit "FireTail Region is missing; try '${0} --help' for more information" + fi + if [[ -z "${FT_APP_TOKEN}" ]]; then + alert_quit "FireTail Token is missing; try '${0} --help' for more information" + fi + if [[ -z "${GCP_REGION}" ]]; then + alert_quit "Region for Google Cloud Platform is missing; try '${0} --help' for more information" + fi + if [[ -z "${GCP_GATEWAY_ID}" ]]; then + alert_quit "Gateway ID for Google Cloud Platform is missing; try '${0} --help' for more information" + fi +} + +function create_pubsub_topic() { + destination="pubsub.googleapis.com/projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" + log_filter="resource.type=apigateway.googleapis.com/Gateway AND " + log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " + log_filter="resource.labels.location=${GCP_REGION}" + log_sink_name="firetail-log-routing" + + gcloud services enable pubsub.googleapis.com + + gcloud pubsub subscriptions create \ + --topic firetail-apigateway-stackdriver-sink firetail-subscription \ + --project="${GCP_PROJECT_ID}" + + gcloud pubsub topics create "${PUBSUB_TOPIC_NAME}" --project="${GCP_PROJECT_ID}" + + gcloud logging sinks create "${log_sink_name}" "${destination}" \ + --project="${GCP_PROJECT_ID}" \ + --description="log router for firetail" \ + --log-filter="${log_filter}" + + service_account=$(gcloud logging sinks describe --format='value(writerIdentity)' ${log_sink_name}) + + gcloud pubsub topics add-iam-policy-binding "projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" \ + --member="${service_account}" \ + --role=roles/pubsub.publisher +} + +function deploy_cloud_function() { + { + gcloud services enable cloudbuild.googleapis.com + gcloud services enable cloudfunctions.googleapis.com + } >/dev/null + + if [[ "${GCP_FUNCTION_NAME}" =~ ^firetail_* ]]; then + function_name_to_deploy="${GCP_FUNCTION_NAME}" + else + function_name_to_deploy="firetail_${GCP_FUNCTION_NAME}" + fi + + topic_prefix="${GCP_FUNCTION_NAME}-pubsub-topic-logs-to-firetail" + + if ! gcloud functions deploy "${function_name_to_deploy}" \ + --entry-point="subscribe" \ + --gen2 \ + --memory="256MB" \ + --no-allow-unauthenticated \ + --project="${GCP_PROJECT_ID}" \ + --region="${GCP_REGION}" \ + --runtime="python312" \ + --set-env-vars=FIRETAIL_API="${FIRETAIL_API}" \ + --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ + --source="src/" \ + --timeout="60s" \ + --trigger-topic="${topic_prefix}" \ + --trigger-topic=${PUBSUB_TOPIC_NAME}; then + alert_quit "Failed to create Cloud Function" + fi +} + +function log() ( + log_level="${1}" + timestamp=$(date '+%F %T%z') + IFS=" " + shift + # keep aligned + printf "[%-5s ${timestamp}] ${*}\n" "${log_level}" +) >>"${LOG_DESTINATION}" + +function alert_quit() ( + IFS=" " + echo -e "\e[0;31m${0}: ${*}\e[0m" + exit 1 +) + +# Prints usage +# Output: +# Help usage +function show_help() { + echo "Usage: ${0} --ft-logging-endpoint= --ft-app-token= --gcp-region= --gcp-gateway-id= [--gcp-function-name=]" + echo "Flags require --key=value format" + echo " --ft-logging-endpoint Endpoint for FireTail app" + echo " --ft-app-token Token from target FireTail app" + echo " --gcp-region Region for GCP Cloud Function" + echo " --gcp-gateway-id GCP gateway ID" + echo " --gcp-function-name Cloud Function name and for prefix for services" + echo " --help Show usage" +} + +main "$@" \ No newline at end of file From 0f5a220d36cb6a930e9f486fa3519bcbab9899ff Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:22:53 +0100 Subject: [PATCH 08/26] Delete setup.sh --- setup.sh | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100755 setup.sh diff --git a/setup.sh b/setup.sh deleted file mode 100755 index 92f3a68..0000000 --- a/setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -echo "Which region do you want to connect to?" -echo "1. EU" -echo "2. US" -read -p "Enter your choice (1 or 2): " choice - -case $choice in - 1) - endpoint="https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk" - ;; - 2) - endpoint="https://api.logging.us-east-2.prod.us.firetail.app/gcp/apigw/bulk" - ;; - *) - echo "Invalid choice. Please enter 1 or 2." - exit 1 - ;; -esac - -read -p "Enter APP Token: " token - -read -p "Enter Google location: " location - -gcloud api-gateway gateways list --location=$location - -read -p "Enter Google API Gateway ID: " gateway_id - - -echo "You have chosen to connect to the $location location. $gateway_id $endpoint" \ No newline at end of file From 4502b3a682768fc8bbe2c2b877a7f949513e9780 Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:14:19 +0100 Subject: [PATCH 09/26] Update run.sh --- run.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/run.sh b/run.sh index 66cdf88..ad89f30 100755 --- a/run.sh +++ b/run.sh @@ -3,7 +3,7 @@ LOG_DESTINATION="${LOG_DESTINATION:-${0}.log}" FIRETAIL_API="${FIRETAIL_API:-https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk}" DEFAULT_GCP_FUNCTION_NAME="firetail_logging" -PUBSUB_TOPIC_NAME="firetail-apigateway-stackdriver-sink" +PUBSUB_TOPIC_SUFFIX="pubsub-topic-logs-to-firetail" declare \ FT_LOGGING_ENDPOINT \ @@ -11,7 +11,8 @@ declare \ GCP_REGION \ GCP_GATEWAY_ID \ GCP_FUNCTION_NAME \ - GCP_PROJECT_ID + GCP_PROJECT_ID \ + PUBSUB_TOPIC_NAME function main() { check_gcloud_cli @@ -72,8 +73,10 @@ function get_arguments() { if [[ "${GCP_FUNCTION_NAME}" == "" ]]; then log WARN "No function name provided, using default: '${DEFAULT_GCP_FUNCTION_NAME}'" GCP_FUNCTION_NAME="${DEFAULT_GCP_FUNCTION_NAME}" + PUBSUB_TOPIC_NAME="${GCP_FUNCTION_NAME}-${PUBSUB_TOPIC_SUFFIX}" fi log INFO "GCP_FUNCTION_NAME = ${GCP_FUNCTION_NAME}" + log INFO "PUBSUB_TOPIC_NAME = ${PUBSUB_TOPIC_NAME}" ;; "") break @@ -106,15 +109,11 @@ function create_pubsub_topic() { destination="pubsub.googleapis.com/projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" log_filter="resource.type=apigateway.googleapis.com/Gateway AND " log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " - log_filter="resource.labels.location=${GCP_REGION}" + log_filter+="resource.labels.location=${GCP_REGION}" log_sink_name="firetail-log-routing" gcloud services enable pubsub.googleapis.com - gcloud pubsub subscriptions create \ - --topic firetail-apigateway-stackdriver-sink firetail-subscription \ - --project="${GCP_PROJECT_ID}" - gcloud pubsub topics create "${PUBSUB_TOPIC_NAME}" --project="${GCP_PROJECT_ID}" gcloud logging sinks create "${log_sink_name}" "${destination}" \ @@ -141,8 +140,6 @@ function deploy_cloud_function() { function_name_to_deploy="firetail_${GCP_FUNCTION_NAME}" fi - topic_prefix="${GCP_FUNCTION_NAME}-pubsub-topic-logs-to-firetail" - if ! gcloud functions deploy "${function_name_to_deploy}" \ --entry-point="subscribe" \ --gen2 \ @@ -155,8 +152,7 @@ function deploy_cloud_function() { --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ --source="src/" \ --timeout="60s" \ - --trigger-topic="${topic_prefix}" \ - --trigger-topic=${PUBSUB_TOPIC_NAME}; then + --trigger-topic="${PUBSUB_TOPIC_NAME}"; then alert_quit "Failed to create Cloud Function" fi } @@ -190,4 +186,4 @@ function show_help() { echo " --help Show usage" } -main "$@" \ No newline at end of file +main "$@" From 97a736127deaad7b6af4de32c56c2e5345c89045 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Wed, 10 Apr 2024 19:01:43 +0300 Subject: [PATCH 10/26] fix: Bootstrapping script https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 121 +++++++++++++++++++++++++++------------------------------ 1 file changed, 58 insertions(+), 63 deletions(-) diff --git a/run.sh b/run.sh index 66cdf88..143a9e2 100755 --- a/run.sh +++ b/run.sh @@ -2,49 +2,41 @@ LOG_DESTINATION="${LOG_DESTINATION:-${0}.log}" FIRETAIL_API="${FIRETAIL_API:-https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk}" -DEFAULT_GCP_FUNCTION_NAME="firetail_logging" -PUBSUB_TOPIC_NAME="firetail-apigateway-stackdriver-sink" +# args declare \ FT_LOGGING_ENDPOINT \ FT_APP_TOKEN \ GCP_REGION \ GCP_GATEWAY_ID \ - GCP_FUNCTION_NAME \ - GCP_PROJECT_ID + GCP_PROJECT_ID \ + GCP_RESOURCE_PREFIX + +# derived from args +declare \ + PUBSUB_TOPIC_NAME \ + GCP_FUNCTION_NAME function main() { - check_gcloud_cli - get_gcp_project_id get_arguments "$@" + check_gcloud_cli create_pubsub_topic deploy_cloud_function } function check_gcloud_cli() { if ! command -v gcloud >/dev/null; then - local err_msg="Failed to get gcloud CLI from PATH. Please install and authenticate gcloud CLI" + local err_msg="Failed to get gcloud CLI from PATH." log ERROR "${err_msg}" - alert_quit "${err_msg}" + alert_quit "${err_msg} Please install and authenticate gcloud CLI" fi } -function get_gcp_project_id() { - gcp_account="$(gcloud auth list --filter=status:ACTIVE --format="value(account)")" - log INFO "GCP Account: ${gcp_account}" - echo "GCP account: ${gcp_account}" - PS3="Enter the number of the project for Cloud Function deployment: " - select gcp_project_id in $(gcloud projects list --format="value(projectId)"); do - gcloud config set project "${gcp_project_id}" || - alert_quit "Failed to set project ID to ${gcp_project_id}" - GCP_PROJECT_ID="${gcp_project_id}" - log INFO "Cloud Function will deploy to ${gcp_project_id}" - # if invalid response, keep prompting - [[ -n "${response}" ]] && break - done -} - function get_arguments() { + (($# == 0)) && { + show_help + exit 0 + } while true; do case "$1" in --help) @@ -53,27 +45,27 @@ function get_arguments() { ;; --ft-logging-endpoint=*) FT_LOGGING_ENDPOINT="${1#--ft-logging-endpoint=}" - log INFO "FT_LOGGING_ENDPOINT = ${FT_LOGGING_ENDPOINT}" + log INFO "FT_LOGGING_ENDPOINT is ${FT_LOGGING_ENDPOINT}" ;; - --ft--app-token=*) + --ft-app-token=*) FT_APP_TOKEN="${1#--ft-app-token=}" - log INFO "FT_APP_TOKEN = ${FT_APP_TOKEN}" + log INFO "FT_APP_TOKEN is ${FT_APP_TOKEN}" ;; --gcp-region=*) GCP_REGION="${1#--gcp-region=}" - log INFO "GCP_REGION = ${GCP_REGION}" + log INFO "GCP_REGION is ${GCP_REGION}" ;; --gcp-gateway-id=*) GCP_GATEWAY_ID="${1#--gcp-gateway-id=}" - log INFO "GCP_GATEWAY_ID = ${GCP_GATEWAY_ID}" + log INFO "GCP_GATEWAY_ID is ${GCP_GATEWAY_ID}" ;; - --gcp-function-name=*) - GCP_FUNCTION_NAME="${1#--gcp-function-name=}" - if [[ "${GCP_FUNCTION_NAME}" == "" ]]; then - log WARN "No function name provided, using default: '${DEFAULT_GCP_FUNCTION_NAME}'" - GCP_FUNCTION_NAME="${DEFAULT_GCP_FUNCTION_NAME}" - fi - log INFO "GCP_FUNCTION_NAME = ${GCP_FUNCTION_NAME}" + --gcp-project-id=*) + GCP_PROJECT_ID="${1#--gcp-gateway-id=}" + log INFO "GCP_PROJECT_ID is ${GCP_PROJECT_ID}" + ;; + --gcp-resource-prefix=*) + GCP_RESOURCE_PREFIX="${1#--gcp-resource-prefix=}" + log INFO "GCP_RESOURCE_PREFIX is ${GCP_RESOURCE_PREFIX}" ;; "") break @@ -89,31 +81,41 @@ function get_arguments() { function check_args_provided() { if [[ -z "${FT_LOGGING_ENDPOINT}" ]]; then - alert_quit "FireTail Region is missing; try '${0} --help' for more information" + show_help + alert_quit "--ft-logging-endpoint is missing" fi if [[ -z "${FT_APP_TOKEN}" ]]; then - alert_quit "FireTail Token is missing; try '${0} --help' for more information" + show_help + alert_quit "--ft-app-token is missing" fi if [[ -z "${GCP_REGION}" ]]; then - alert_quit "Region for Google Cloud Platform is missing; try '${0} --help' for more information" + show_help + alert_quit "--gcp-region is missing" fi if [[ -z "${GCP_GATEWAY_ID}" ]]; then - alert_quit "Gateway ID for Google Cloud Platform is missing; try '${0} --help' for more information" + show_help + alert_quit "--gcp-gateway-id is missing" + fi + if [[ -z "${GCP_PROJECT_ID}" ]]; then + show_help + alert_quit "--gcp-project-id is missing" + fi + if [[ -z "${GCP_RESOURCE_PREFIX}" ]]; then + show_help + alert_quit "--gcp-resource-prefix is missing" fi } function create_pubsub_topic() { + PUBSUB_TOPIC_NAME="${GCP_RESOURCE_PREFIX}-pubsub-topic-logs-to-firetail" destination="pubsub.googleapis.com/projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" - log_filter="resource.type=apigateway.googleapis.com/Gateway AND " - log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " - log_filter="resource.labels.location=${GCP_REGION}" log_sink_name="firetail-log-routing" - gcloud services enable pubsub.googleapis.com + log_filter="resource.type=apigateway.googleapis.com/Gateway AND " + log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " + log_filter+="resource.labels.location=${GCP_REGION}" - gcloud pubsub subscriptions create \ - --topic firetail-apigateway-stackdriver-sink firetail-subscription \ - --project="${GCP_PROJECT_ID}" + gcloud services enable pubsub.googleapis.com >/dev/null gcloud pubsub topics create "${PUBSUB_TOPIC_NAME}" --project="${GCP_PROJECT_ID}" @@ -130,20 +132,13 @@ function create_pubsub_topic() { } function deploy_cloud_function() { + GCP_FUNCTION_NAME="${GCP_RESOURCE_PREFIX}-firetail-logging" { gcloud services enable cloudbuild.googleapis.com gcloud services enable cloudfunctions.googleapis.com } >/dev/null - if [[ "${GCP_FUNCTION_NAME}" =~ ^firetail_* ]]; then - function_name_to_deploy="${GCP_FUNCTION_NAME}" - else - function_name_to_deploy="firetail_${GCP_FUNCTION_NAME}" - fi - - topic_prefix="${GCP_FUNCTION_NAME}-pubsub-topic-logs-to-firetail" - - if ! gcloud functions deploy "${function_name_to_deploy}" \ + if ! gcloud functions deploy "${GCP_FUNCTION_NAME}" \ --entry-point="subscribe" \ --gen2 \ --memory="256MB" \ @@ -155,8 +150,7 @@ function deploy_cloud_function() { --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ --source="src/" \ --timeout="60s" \ - --trigger-topic="${topic_prefix}" \ - --trigger-topic=${PUBSUB_TOPIC_NAME}; then + --trigger-topic="${PUBSUB_TOPIC_NAME}"; then alert_quit "Failed to create Cloud Function" fi } @@ -172,7 +166,7 @@ function log() ( function alert_quit() ( IFS=" " - echo -e "\e[0;31m${0}: ${*}\e[0m" + echo -e "\033[0;31m${0}: ${*}\033[0m" exit 1 ) @@ -180,14 +174,15 @@ function alert_quit() ( # Output: # Help usage function show_help() { - echo "Usage: ${0} --ft-logging-endpoint= --ft-app-token= --gcp-region= --gcp-gateway-id= [--gcp-function-name=]" - echo "Flags require --key=value format" + echo "Usage: ${0} --ft-logging-endpoint= --ft-app-token= --gcp-region= --gcp-gateway-id= --gcp-project-id= --gcp-resource-prefix=" + echo -e "\033[1mFlags require --key=value format\033[0m" echo " --ft-logging-endpoint Endpoint for FireTail app" echo " --ft-app-token Token from target FireTail app" - echo " --gcp-region Region for GCP Cloud Function" + echo " --gcp-region Region for GCP cloud function" echo " --gcp-gateway-id GCP gateway ID" - echo " --gcp-function-name Cloud Function name and for prefix for services" + echo " --gcp-project-id GCP project for cloud function and pubsub topic" + echo " --gcp-resource-prefix Prefix for cloud function name and pubsub topic" echo " --help Show usage" } -main "$@" \ No newline at end of file +main "$@" From 100a44435c1f7043bc536195da0bdae9be88a55a Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Wed, 10 Apr 2024 19:24:30 +0300 Subject: [PATCH 11/26] fix: Fail bootstrap function deploy faster https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/run.sh b/run.sh index 143a9e2..8301cc0 100755 --- a/run.sh +++ b/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -o errexit + LOG_DESTINATION="${LOG_DESTINATION:-${0}.log}" FIRETAIL_API="${FIRETAIL_API:-https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk}" @@ -115,30 +117,35 @@ function create_pubsub_topic() { log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " log_filter+="resource.labels.location=${GCP_REGION}" - gcloud services enable pubsub.googleapis.com >/dev/null + gcloud services enable pubsub.googleapis.com || + alert_quit "Failed to enable pubsub.googleapis.com" - gcloud pubsub topics create "${PUBSUB_TOPIC_NAME}" --project="${GCP_PROJECT_ID}" + gcloud pubsub topics create "${PUBSUB_TOPIC_NAME}" --project="${GCP_PROJECT_ID}" || + alert_quit "Failed to create pubsub topic" gcloud logging sinks create "${log_sink_name}" "${destination}" \ --project="${GCP_PROJECT_ID}" \ --description="log router for firetail" \ - --log-filter="${log_filter}" + --log-filter="${log_filter}" || + alert_quit "Failed to create logging sink" service_account=$(gcloud logging sinks describe --format='value(writerIdentity)' ${log_sink_name}) gcloud pubsub topics add-iam-policy-binding "projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" \ --member="${service_account}" \ - --role=roles/pubsub.publisher + --role=roles/pubsub.publisher || + alert_quit "Failed to add IAM policy binding" } function deploy_cloud_function() { GCP_FUNCTION_NAME="${GCP_RESOURCE_PREFIX}-firetail-logging" - { - gcloud services enable cloudbuild.googleapis.com - gcloud services enable cloudfunctions.googleapis.com - } >/dev/null - if ! gcloud functions deploy "${GCP_FUNCTION_NAME}" \ + gcloud services enable cloudbuild.googleapis.com || + alert_quit "Failed to enable cloudbuild.googleapis.com" + gcloud services enable cloudfunctions.googleapis.com || + alert_quit "Failed to enable cloudfunctions.googleapis.com" + + gcloud functions deploy "${GCP_FUNCTION_NAME}" \ --entry-point="subscribe" \ --gen2 \ --memory="256MB" \ @@ -150,9 +157,9 @@ function deploy_cloud_function() { --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ --source="src/" \ --timeout="60s" \ - --trigger-topic="${PUBSUB_TOPIC_NAME}"; then - alert_quit "Failed to create Cloud Function" - fi + --trigger-topic="${PUBSUB_TOPIC_NAME}" || + alert_quit "Failed to deploy Cloud Function" + } function log() ( From 0380ee87699cc9a3b565ad5f565edce24ac66c24 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Wed, 10 Apr 2024 19:32:02 +0300 Subject: [PATCH 12/26] fix: Specify project in all gcloud commands https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/run.sh b/run.sh index 8301cc0..a4ddeba 100755 --- a/run.sh +++ b/run.sh @@ -117,7 +117,7 @@ function create_pubsub_topic() { log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " log_filter+="resource.labels.location=${GCP_REGION}" - gcloud services enable pubsub.googleapis.com || + gcloud services enable pubsub.googleapis.com --project "${GCP_PROJECT_ID}" || alert_quit "Failed to enable pubsub.googleapis.com" gcloud pubsub topics create "${PUBSUB_TOPIC_NAME}" --project="${GCP_PROJECT_ID}" || @@ -129,7 +129,11 @@ function create_pubsub_topic() { --log-filter="${log_filter}" || alert_quit "Failed to create logging sink" - service_account=$(gcloud logging sinks describe --format='value(writerIdentity)' ${log_sink_name}) + service_account=$( + gcloud logging sinks describe ${log_sink_name} \ + --format='value(writerIdentity)' \ + --project "${GCP_PROJECT_ID}" + ) gcloud pubsub topics add-iam-policy-binding "projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" \ --member="${service_account}" \ @@ -140,9 +144,9 @@ function create_pubsub_topic() { function deploy_cloud_function() { GCP_FUNCTION_NAME="${GCP_RESOURCE_PREFIX}-firetail-logging" - gcloud services enable cloudbuild.googleapis.com || + gcloud services enable cloudbuild.googleapis.com --project "${GCP_PROJECT_ID}" || alert_quit "Failed to enable cloudbuild.googleapis.com" - gcloud services enable cloudfunctions.googleapis.com || + gcloud services enable cloudfunctions.googleapis.com --project "${GCP_PROJECT_ID}" || alert_quit "Failed to enable cloudfunctions.googleapis.com" gcloud functions deploy "${GCP_FUNCTION_NAME}" \ From d2fc39c5635b45a75f0aeebc6e4292e09f603c24 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Thu, 11 Apr 2024 10:00:26 +0300 Subject: [PATCH 13/26] feat: Better bad flag handling in bootstrap script https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run.sh b/run.sh index a4ddeba..8bbe000 100755 --- a/run.sh +++ b/run.sh @@ -73,7 +73,8 @@ function get_arguments() { break ;; *) - alert_quit "unrecognized flag; try '${0} --help' for more information" + show_help + alert_quit "unrecognized flag: '${1}'" ;; esac shift From 85b3fb1e062f3aca26e1fb5b87ed6ed4964c2330 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Thu, 11 Apr 2024 10:08:04 +0300 Subject: [PATCH 14/26] fix: Parse project ID correctly in bootstrap script https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 8bbe000..d980724 100755 --- a/run.sh +++ b/run.sh @@ -62,7 +62,7 @@ function get_arguments() { log INFO "GCP_GATEWAY_ID is ${GCP_GATEWAY_ID}" ;; --gcp-project-id=*) - GCP_PROJECT_ID="${1#--gcp-gateway-id=}" + GCP_PROJECT_ID="${1#--gcp-project-id=}" log INFO "GCP_PROJECT_ID is ${GCP_PROJECT_ID}" ;; --gcp-resource-prefix=*) @@ -178,7 +178,7 @@ function log() ( function alert_quit() ( IFS=" " - echo -e "\033[0;31m${0}: ${*}\033[0m" + echo -e "\033[0;1;31m${0}: ${*}\033[0m" exit 1 ) From 1d35407ba3aad603aeec3229a1922b57ca3406a5 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Thu, 11 Apr 2024 11:56:21 +0300 Subject: [PATCH 15/26] feat: Accept 2-part flags w/o equals sign https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/run.sh b/run.sh index d980724..d40b15b 100755 --- a/run.sh +++ b/run.sh @@ -43,39 +43,73 @@ function get_arguments() { case "$1" in --help) show_help - exit + exit 0 ;; --ft-logging-endpoint=*) FT_LOGGING_ENDPOINT="${1#--ft-logging-endpoint=}" log INFO "FT_LOGGING_ENDPOINT is ${FT_LOGGING_ENDPOINT}" ;; + --ft-logging-endpoint) + FT_LOGGING_ENDPOINT="${2}" + shift + log INFO "FT_LOGGING_ENDPOINT is ${FT_LOGGING_ENDPOINT}" + ;; --ft-app-token=*) FT_APP_TOKEN="${1#--ft-app-token=}" log INFO "FT_APP_TOKEN is ${FT_APP_TOKEN}" ;; + --ft-app-token) + FT_APP_TOKEN="${2}" + shift + log INFO "FT_APP_TOKEN is ${FT_APP_TOKEN}" + ;; --gcp-region=*) GCP_REGION="${1#--gcp-region=}" log INFO "GCP_REGION is ${GCP_REGION}" ;; + --gcp-region) + GCP_REGION="${2}" + shift + log INFO "GCP_REGION is ${GCP_REGION}" + ;; --gcp-gateway-id=*) GCP_GATEWAY_ID="${1#--gcp-gateway-id=}" log INFO "GCP_GATEWAY_ID is ${GCP_GATEWAY_ID}" ;; + --gcp-gateway-id) + GCP_GATEWAY_ID="${2}" + shift + log INFO "GCP_GATEWAY_ID is ${GCP_GATEWAY_ID}" + ;; --gcp-project-id=*) GCP_PROJECT_ID="${1#--gcp-project-id=}" log INFO "GCP_PROJECT_ID is ${GCP_PROJECT_ID}" ;; + --gcp-project-id) + GCP_PROJECT_ID="${2}" + shift + log INFO "GCP_PROJECT_ID is ${GCP_PROJECT_ID}" + ;; --gcp-resource-prefix=*) GCP_RESOURCE_PREFIX="${1#--gcp-resource-prefix=}" log INFO "GCP_RESOURCE_PREFIX is ${GCP_RESOURCE_PREFIX}" ;; + --gcp-resource-prefix) + GCP_RESOURCE_PREFIX="${2}" + shift + log INFO "GCP_RESOURCE_PREFIX is ${GCP_RESOURCE_PREFIX}" + ;; "") break ;; - *) + --*) show_help alert_quit "unrecognized flag: '${1}'" ;; + *) + show_help + alert_quit "unrecognized argument: '${1}'" + ;; esac shift done @@ -136,7 +170,8 @@ function create_pubsub_topic() { --project "${GCP_PROJECT_ID}" ) - gcloud pubsub topics add-iam-policy-binding "projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" \ + gcloud pubsub topics add-iam-policy-binding \ + "projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" \ --member="${service_account}" \ --role=roles/pubsub.publisher || alert_quit "Failed to add IAM policy binding" @@ -186,8 +221,11 @@ function alert_quit() ( # Output: # Help usage function show_help() { - echo "Usage: ${0} --ft-logging-endpoint= --ft-app-token= --gcp-region= --gcp-gateway-id= --gcp-project-id= --gcp-resource-prefix=" - echo -e "\033[1mFlags require --key=value format\033[0m" + top_line="Usage: ${0} --ft-logging-endpoint= --ft-app-token= " + top_line+="--gcp-region= --gcp-gateway-id= --gcp-project-id= " + top_line+="--gcp-resource-prefix=" + + echo -e "\n${top_line}\n" echo " --ft-logging-endpoint Endpoint for FireTail app" echo " --ft-app-token Token from target FireTail app" echo " --gcp-region Region for GCP cloud function" @@ -195,6 +233,7 @@ function show_help() { echo " --gcp-project-id GCP project for cloud function and pubsub topic" echo " --gcp-resource-prefix Prefix for cloud function name and pubsub topic" echo " --help Show usage" + echo "" } main "$@" From af3d975b5cb3c35c8010e0390afbc7d197a0ed89 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Thu, 11 Apr 2024 12:42:29 +0300 Subject: [PATCH 16/26] fix: Accept GCP project number and get its ID https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/run.sh b/run.sh index d40b15b..809c9c1 100755 --- a/run.sh +++ b/run.sh @@ -10,18 +10,20 @@ declare \ FT_LOGGING_ENDPOINT \ FT_APP_TOKEN \ GCP_REGION \ + GCP_PROJECT_NUM \ GCP_GATEWAY_ID \ - GCP_PROJECT_ID \ GCP_RESOURCE_PREFIX # derived from args declare \ PUBSUB_TOPIC_NAME \ + GCP_PROJECT_ID \ GCP_FUNCTION_NAME function main() { get_arguments "$@" check_gcloud_cli + set_gcp_project create_pubsub_topic deploy_cloud_function } @@ -34,6 +36,12 @@ function check_gcloud_cli() { fi } +function set_gcp_project(){ + gcloud config set project "${GCP_PROJECT_NUM}" || + alert_quit "Failed to set project ID to ${GCP_PROJECT_NUM}" + GCP_PROJECT_ID=$(gcloud projects describe "${GCP_PROJECT_NUM}" --format='value(projectId)') +} + function get_arguments() { (($# == 0)) && { show_help @@ -81,14 +89,14 @@ function get_arguments() { shift log INFO "GCP_GATEWAY_ID is ${GCP_GATEWAY_ID}" ;; - --gcp-project-id=*) - GCP_PROJECT_ID="${1#--gcp-project-id=}" - log INFO "GCP_PROJECT_ID is ${GCP_PROJECT_ID}" + --gcp-project-num=*) + GCP_PROJECT_NUM="${1#--gcp-project-num=}" + log INFO "GCP_PROJECT_NUM is ${GCP_PROJECT_NUM}" ;; - --gcp-project-id) - GCP_PROJECT_ID="${2}" + --gcp-project-num) + GCP_PROJECT_NUM="${2}" shift - log INFO "GCP_PROJECT_ID is ${GCP_PROJECT_ID}" + log INFO "GCP_PROJECT_NUM is ${GCP_PROJECT_NUM}" ;; --gcp-resource-prefix=*) GCP_RESOURCE_PREFIX="${1#--gcp-resource-prefix=}" @@ -133,9 +141,9 @@ function check_args_provided() { show_help alert_quit "--gcp-gateway-id is missing" fi - if [[ -z "${GCP_PROJECT_ID}" ]]; then + if [[ -z "${GCP_PROJECT_NUM}" ]]; then show_help - alert_quit "--gcp-project-id is missing" + alert_quit "--gcp-project-num is missing" fi if [[ -z "${GCP_RESOURCE_PREFIX}" ]]; then show_help @@ -222,7 +230,7 @@ function alert_quit() ( # Help usage function show_help() { top_line="Usage: ${0} --ft-logging-endpoint= --ft-app-token= " - top_line+="--gcp-region= --gcp-gateway-id= --gcp-project-id= " + top_line+="--gcp-region= --gcp-gateway-id= --gcp-project-num= " top_line+="--gcp-resource-prefix=" echo -e "\n${top_line}\n" @@ -230,7 +238,7 @@ function show_help() { echo " --ft-app-token Token from target FireTail app" echo " --gcp-region Region for GCP cloud function" echo " --gcp-gateway-id GCP gateway ID" - echo " --gcp-project-id GCP project for cloud function and pubsub topic" + echo " --gcp-project-num GCP project number for cloud function and pubsub topic" echo " --gcp-resource-prefix Prefix for cloud function name and pubsub topic" echo " --help Show usage" echo "" From aec2e50de0f98af133ffd080a11cdb58d1ace0a7 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Thu, 11 Apr 2024 12:45:14 +0300 Subject: [PATCH 17/26] docs: Add new command to README https://firetail-io.atlassian.net/browse/FIRE-2349 --- readme.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index f04b5eb..721f490 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,10 @@ -https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https://github.com/firetail-io/repo +# GCP API Gateway Logging Cloud Function -./run.sh --app-token=<> --gcp_region=<> --firetail-api +## Launch in Cloud Shell + + +```sh +./run.sh --ft-logging-endpoint= --ft-app-token= --gcp-region= --gcp-gateway-id= --gcp-project-num= --gcp-resource-prefix= +``` From 7f71b5fd18e676095aa86303a5e059cdcfddddf6 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 11:05:17 +0100 Subject: [PATCH 18/26] made small changes --- run.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/run.sh b/run.sh index 809c9c1..ce8623d 100755 --- a/run.sh +++ b/run.sh @@ -186,7 +186,7 @@ function create_pubsub_topic() { } function deploy_cloud_function() { - GCP_FUNCTION_NAME="${GCP_RESOURCE_PREFIX}-firetail-logging" + GCP_FUNCTION_NAME="${GCP_RESOURCE_PREFIX}-logging-function" gcloud services enable cloudbuild.googleapis.com --project "${GCP_PROJECT_ID}" || alert_quit "Failed to enable cloudbuild.googleapis.com" @@ -234,12 +234,12 @@ function show_help() { top_line+="--gcp-resource-prefix=" echo -e "\n${top_line}\n" - echo " --ft-logging-endpoint Endpoint for FireTail app" - echo " --ft-app-token Token from target FireTail app" - echo " --gcp-region Region for GCP cloud function" + echo " --ft-logging-endpoint FireTail logging endpoint" + echo " --ft-app-token FireTail application token" + echo " --gcp-region Region where the Gateway is deployed" echo " --gcp-gateway-id GCP gateway ID" - echo " --gcp-project-num GCP project number for cloud function and pubsub topic" - echo " --gcp-resource-prefix Prefix for cloud function name and pubsub topic" + echo " --gcp-project-num GCP project number" + echo " --gcp-resource-prefix Prefix added to resources created by this script" echo " --help Show usage" echo "" } From 6da1a3148fd0bc8e66e1c25899977e0c3838db24 Mon Sep 17 00:00:00 2001 From: Greg Sheppard Date: Thu, 11 Apr 2024 13:04:35 +0300 Subject: [PATCH 19/26] fix: Add prefix to log sink in bootstrap script https://firetail-io.atlassian.net/browse/FIRE-2349 --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index ce8623d..4578193 100755 --- a/run.sh +++ b/run.sh @@ -154,7 +154,7 @@ function check_args_provided() { function create_pubsub_topic() { PUBSUB_TOPIC_NAME="${GCP_RESOURCE_PREFIX}-pubsub-topic-logs-to-firetail" destination="pubsub.googleapis.com/projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" - log_sink_name="firetail-log-routing" + log_sink_name="${GCP_RESOURCE_PREFIX}-firetail-log-routing" log_filter="resource.type=apigateway.googleapis.com/Gateway AND " log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " @@ -173,7 +173,7 @@ function create_pubsub_topic() { alert_quit "Failed to create logging sink" service_account=$( - gcloud logging sinks describe ${log_sink_name} \ + gcloud logging sinks describe "${log_sink_name}" \ --format='value(writerIdentity)' \ --project "${GCP_PROJECT_ID}" ) From 9dbbb4ff50153a0cf3f38521f10be701b940d2ac Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 11:12:01 +0100 Subject: [PATCH 20/26] small changes on names --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 4578193..60e1c64 100755 --- a/run.sh +++ b/run.sh @@ -154,7 +154,7 @@ function check_args_provided() { function create_pubsub_topic() { PUBSUB_TOPIC_NAME="${GCP_RESOURCE_PREFIX}-pubsub-topic-logs-to-firetail" destination="pubsub.googleapis.com/projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" - log_sink_name="${GCP_RESOURCE_PREFIX}-firetail-log-routing" + log_sink_name="${GCP_RESOURCE_PREFIX}-log-sink" log_filter="resource.type=apigateway.googleapis.com/Gateway AND " log_filter+="resource.labels.gateway_id=${GCP_GATEWAY_ID} AND " @@ -168,7 +168,7 @@ function create_pubsub_topic() { gcloud logging sinks create "${log_sink_name}" "${destination}" \ --project="${GCP_PROJECT_ID}" \ - --description="log router for firetail" \ + --description="log router for FireTail" \ --log-filter="${log_filter}" || alert_quit "Failed to create logging sink" From bffcc01235372e3d7dc8760abe73fe402baf63bb Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 11:59:00 +0100 Subject: [PATCH 21/26] fixed run script added seperate service account for function --- run.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 60e1c64..8de5a2e 100755 --- a/run.sh +++ b/run.sh @@ -64,7 +64,6 @@ function get_arguments() { ;; --ft-app-token=*) FT_APP_TOKEN="${1#--ft-app-token=}" - log INFO "FT_APP_TOKEN is ${FT_APP_TOKEN}" ;; --ft-app-token) FT_APP_TOKEN="${2}" @@ -181,6 +180,7 @@ function create_pubsub_topic() { gcloud pubsub topics add-iam-policy-binding \ "projects/${GCP_PROJECT_ID}/topics/${PUBSUB_TOPIC_NAME}" \ --member="${service_account}" \ + --project "${GCP_PROJECT_ID}" \ --role=roles/pubsub.publisher || alert_quit "Failed to add IAM policy binding" } @@ -193,10 +193,18 @@ function deploy_cloud_function() { gcloud services enable cloudfunctions.googleapis.com --project "${GCP_PROJECT_ID}" || alert_quit "Failed to enable cloudfunctions.googleapis.com" + REDUCED_PREFIX=$(echo "${GCP_RESOURCE_PREFIX}" | cut -c1-20) + SERVICE_ACCOUNT_NAME=$(echo "${REDUCED_PREFIX}-function" | cut -c1-30) + CLOUD_RUN_SERVICE_ACCOUNT=$(gcloud iam service-accounts create ${SERVICE_ACCOUNT_NAME} \ + --description "Cloud Run Function service account for FireTail Logging" \ + --display-name "FireTail Logging Cloud Run service account" \ + --project "${GCP_PROJECT_ID}") + gcloud functions deploy "${GCP_FUNCTION_NAME}" \ --entry-point="subscribe" \ --gen2 \ --memory="256MB" \ + --run-service-account=$CLOUD_RUN_SERVICE_ACCOUNT \ --no-allow-unauthenticated \ --project="${GCP_PROJECT_ID}" \ --region="${GCP_REGION}" \ From 5b50eb1747a04e5f8d98eba1438e2ba71beec4f0 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 12:03:06 +0100 Subject: [PATCH 22/26] made ingress internal --- run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run.sh b/run.sh index 8de5a2e..5e6047a 100755 --- a/run.sh +++ b/run.sh @@ -208,6 +208,7 @@ function deploy_cloud_function() { --no-allow-unauthenticated \ --project="${GCP_PROJECT_ID}" \ --region="${GCP_REGION}" \ + --ingress internal \ --runtime="python312" \ --set-env-vars=FIRETAIL_API="${FIRETAIL_API}" \ --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ From 7b937a6e4fa10ad14ad5a39e52d4c4dfe88f3817 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 13:08:59 +0100 Subject: [PATCH 23/26] fix run --- run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/run.sh b/run.sh index 5e6047a..8de5a2e 100755 --- a/run.sh +++ b/run.sh @@ -208,7 +208,6 @@ function deploy_cloud_function() { --no-allow-unauthenticated \ --project="${GCP_PROJECT_ID}" \ --region="${GCP_REGION}" \ - --ingress internal \ --runtime="python312" \ --set-env-vars=FIRETAIL_API="${FIRETAIL_API}" \ --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ From 01e709457df32075f0a7b26653e2294481189687 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 13:17:53 +0100 Subject: [PATCH 24/26] fix endpoint --- run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 8de5a2e..860d5f6 100755 --- a/run.sh +++ b/run.sh @@ -3,7 +3,6 @@ set -o errexit LOG_DESTINATION="${LOG_DESTINATION:-${0}.log}" -FIRETAIL_API="${FIRETAIL_API:-https://api.logging.eu-west-1.prod.firetail.app/gcp/apigw/bulk}" # args declare \ @@ -209,7 +208,7 @@ function deploy_cloud_function() { --project="${GCP_PROJECT_ID}" \ --region="${GCP_REGION}" \ --runtime="python312" \ - --set-env-vars=FIRETAIL_API="${FIRETAIL_API}" \ + --set-env-vars=FIRETAIL_API="${FT_LOGGING_ENDPOINT}" \ --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ --source="src/" \ --timeout="60s" \ From dd53cda370077446f89f5367021906b8fc7b8180 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 11 Apr 2024 15:17:05 +0100 Subject: [PATCH 25/26] fixed endpoint --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 860d5f6..af3d75a 100755 --- a/run.sh +++ b/run.sh @@ -208,7 +208,7 @@ function deploy_cloud_function() { --project="${GCP_PROJECT_ID}" \ --region="${GCP_REGION}" \ --runtime="python312" \ - --set-env-vars=FIRETAIL_API="${FT_LOGGING_ENDPOINT}" \ + --set-env-vars=FIRETAIL_API="${FT_LOGGING_ENDPOINT}/gcp/apigw/bulk" \ --set-env-vars=FIRETAIL_APP_TOKEN="${FT_APP_TOKEN}" \ --source="src/" \ --timeout="60s" \ From 6c01022d585d4862c807d27e84e8b17d316b41ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Ru=CC=88ppell?= Date: Tue, 8 Jul 2025 16:23:21 +0300 Subject: [PATCH 26/26] Replace ndjson --- src/main.py | 17 ++++++++++++++++- src/requirements.txt | 1 - 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main.py b/src/main.py index 10e89b0..3b2fa78 100644 --- a/src/main.py +++ b/src/main.py @@ -6,10 +6,11 @@ from urllib.parse import unquote_plus, urlparse import functions_framework -import ndjson import requests from dataclasses_json import dataclass_json from dateutil import parser as date_parser +from typing import Any, Callable + REQUESTS_SESSION = requests.Session() FIRETAIL_API = os.getenv( @@ -17,6 +18,20 @@ ) FIRETAIL_APP_TOKEN = os.getenv("FIRETAIL_APP_TOKEN") +class ndjson: + """A class to handle newline-delimited JSON (NDJSON) format.""" + + @staticmethod + def dumps(data: Any, default: Callable[[object], str] | None = None) -> str: + """Convert data to NDJSON format.""" + + return "\n".join(json.dumps(item, default=default) for item in data) + + @staticmethod + def loads(data: str) -> list[dict[str, object]]: + """Load NDJSON data into a list of dictionaries.""" + return [json.loads(line) for line in data.splitlines() if line.strip()] + class FireTailFailedIngest(Exception): pass diff --git a/src/requirements.txt b/src/requirements.txt index bd53957..abe103b 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -2,6 +2,5 @@ google-cloud-pubsub google-cloud requests dataclasses-json -ndjson functions-framework python-dateutil \ No newline at end of file