diff --git a/.azurePipeline/templatePublishLogsFromPods.yml b/.azurePipeline/templatePublishLogsFromPods.yml index 8ee6df7b..f7bd0d38 100644 --- a/.azurePipeline/templatePublishLogsFromPods.yml +++ b/.azurePipeline/templatePublishLogsFromPods.yml @@ -24,7 +24,7 @@ steps: connectionType: 'Kubernetes Service Connection' kubernetesServiceEndpoint: ${{ parameters.kubernetesServiceEndpoint }} command: 'logs' - arguments: '--namespace ${{ parameters.namespace }} -lrelease=${{ parameters.releaseName }} -ltier=backend --tail=-1 --all-containers=true --prefix=true' + arguments: '--namespace ${{ parameters.namespace }} -lrelease=${{ parameters.releaseName }} -ltier=backend --tail=-1 -c celery-worker --prefix=true' - task: Kubernetes@1 displayName: 'Anonlink Entity Service NGINX Logs' diff --git a/backend/entityservice/__init__.py b/backend/entityservice/__init__.py index 2bc13838..902ae405 100644 --- a/backend/entityservice/__init__.py +++ b/backend/entityservice/__init__.py @@ -38,7 +38,7 @@ # Config could be Config, DevelopmentConfig or ProductionConfig app.config.from_object(config) -logger = structlog.get_logger() +logger = structlog.wrap_logger(app.logger) # Tracer setup (currently just trace all requests) flask_tracer = FlaskTracer(initialize_tracer, True, app) diff --git a/backend/entityservice/async_worker.py b/backend/entityservice/async_worker.py index c395bdcc..cb339df9 100644 --- a/backend/entityservice/async_worker.py +++ b/backend/entityservice/async_worker.py @@ -1,7 +1,7 @@ import logging from celery import Celery -from celery.signals import task_prerun, worker_process_init, worker_process_shutdown +from celery import signals import structlog from entityservice.database.util import init_db_pool, close_db_pool @@ -20,6 +20,7 @@ celery.conf.result_backend_transport_options = Config.CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS celery.conf.worker_prefetch_multiplier = Config.CELERYD_PREFETCH_MULTIPLIER celery.conf.worker_max_tasks_per_child = Config.CELERYD_MAX_TASKS_PER_CHILD + if Config.CELERYD_CONCURRENCY > 0: # If set to 0, let celery choose the default, which is the number of available CPUs on the machine. celery.conf.worker_concurrency = Config.CELERYD_CONCURRENCY @@ -30,7 +31,7 @@ logger = structlog.wrap_logger(logging.getLogger('entityservice.tasks')) -@worker_process_init.connect() +@signals.worker_process_init.connect() def init_worker(**kwargs): db_min_connections = Config.CELERY_DB_MIN_CONNECTIONS db_max_connections = Config.CELERY_DB_MAX_CONNECTIONS @@ -39,16 +40,16 @@ def init_worker(**kwargs): logger.debug("Debug logging enabled") -@worker_process_shutdown.connect() +@signals.worker_process_shutdown.connect() def shutdown_worker(**kwargs): close_db_pool() logger.info("Shutting down a worker process") -@task_prerun.connect() + +@signals.task_prerun.connect() def configure_structlog(sender, body=None, **kwargs): task = kwargs['task'] task.logger = logger.new( - task_id=kwargs['task_id'], task_name=sender.__name__ ) diff --git a/backend/entityservice/database/insertions.py b/backend/entityservice/database/insertions.py index 8159554d..e04e1b25 100644 --- a/backend/entityservice/database/insertions.py +++ b/backend/entityservice/database/insertions.py @@ -277,7 +277,6 @@ def is_dataprovider_allowed_to_upload_and_lock(db, dp_id): RETURNING id, uploaded """ query_response = query_db(db, sql_update, [dp_id]) - print(query_response) length = len(query_response) if length < 1: return False diff --git a/backend/entityservice/logger_setup.py b/backend/entityservice/logger_setup.py index a0a1e3bb..a4927ab0 100644 --- a/backend/entityservice/logger_setup.py +++ b/backend/entityservice/logger_setup.py @@ -1,32 +1,27 @@ import logging.config -import os from pathlib import Path + import structlog -import yaml -from entityservice.errors import InvalidConfiguration +from entityservice.utils import load_yaml_config +from entityservice.settings import Config as config -def setup_logging( - default_path='default_logging.yaml', - env_key='LOG_CFG' -): +def setup_logging(default_path='default_logging.yaml'): """ Setup logging configuration """ - path = os.getenv(env_key, Path(__file__).parent / default_path) - try: - with open(path, 'rt') as f: - config = yaml.safe_load(f) - logging.config.dictConfig(config) - except yaml.YAMLError as e: - raise InvalidConfiguration("Parsing YAML logging config failed") from e - except FileNotFoundError as e: - raise InvalidConfiguration(f"Logging config YAML file '{path}' doesn't exist.") from e + if config.LOG_CONFIG_FILENAME is not None: + path = Path(config.LOG_CONFIG_FILENAME) + else: + path = Path(__file__).parent / default_path + + loggingconfig = load_yaml_config(path) + logging.config.dictConfig(loggingconfig) # Configure Structlog wrapper for client use setup_structlog() - logging.info("Loaded logging config from file") + logging.debug(f"Loaded logging config from file: {path}") def setup_structlog(): @@ -36,7 +31,7 @@ def setup_structlog(): structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.stdlib.PositionalArgumentsFormatter(), - structlog.processors.TimeStamper(fmt='iso'), + #structlog.processors.TimeStamper(fmt='iso'), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, @@ -45,7 +40,7 @@ def setup_structlog(): #structlog.processors.JSONRenderer(), structlog.dev.ConsoleRenderer() ], - context_class=dict, + context_class=structlog.threadlocal.wrap_dict(dict), logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True, diff --git a/backend/entityservice/settings.py b/backend/entityservice/settings.py index ebb2f26c..5c99bf29 100644 --- a/backend/entityservice/settings.py +++ b/backend/entityservice/settings.py @@ -18,6 +18,9 @@ class Config(object): CONNEXION_STRICT_VALIDATION = os.getenv("CONNEXION_STRICT_VALIDATION", "true").lower() == "true" CONNEXION_RESPONSE_VALIDATION = os.getenv("CONNEXION_RESPONSE_VALIDATION", "true").lower() == "true" + LOG_CONFIG_FILENAME = os.getenv("LOG_CFG") + TRACING_CONFIG_FILENAME = os.getenv("TRACE_CFG") + LOGFILE = os.getenv("LOGFILE") LOG_HTTP_HEADER_FIELDS = os.getenv("LOG_HTTP_HEADER_FIELDS") @@ -88,9 +91,6 @@ class Config(object): BIN_FILENAME_FMT = "raw-clks/{}.bin" SIMILARITY_SCORES_FILENAME_FMT = "similarity-scores/{}.bin" - TRACING_HOST = os.getenv("TRACING_HOST", "jaeger") - TRACING_PORT = os.getenv("TRACING_PORT", "5775") - # Encoding size (in bytes) MIN_ENCODING_SIZE = int(os.getenv('MIN_ENCODING_SIZE', '1')) MAX_ENCODING_SIZE = int(os.getenv('MAX_ENCODING_SIZE', '1024')) diff --git a/backend/entityservice/tests/test_serialization.py b/backend/entityservice/tests/test_serialization.py index 8dc0e0ef..7d37b904 100644 --- a/backend/entityservice/tests/test_serialization.py +++ b/backend/entityservice/tests/test_serialization.py @@ -8,11 +8,7 @@ import anonlink from entityservice.serialization import deserialize_bytes, generate_scores -from entityservice.tests.util import serialize_bytes - - -def random_bytes(l=1024): - return random.getrandbits(l).to_bytes(l // 8, 'big') +from entityservice.tests.util import serialize_bytes, generate_bytes class SerializationTest(unittest.TestCase): @@ -27,7 +23,7 @@ def test_bitarray(self): self.assertEqual(banew, ba) def test_random_bytes(self): - rb = random_bytes(2048) + rb = generate_bytes(2048//8) srb = serialize_bytes(rb) dsrb = deserialize_bytes(srb) self.assertEqual(dsrb, rb) diff --git a/backend/entityservice/tests/test_utils.py b/backend/entityservice/tests/test_utils.py new file mode 100644 index 00000000..665f7d1d --- /dev/null +++ b/backend/entityservice/tests/test_utils.py @@ -0,0 +1,64 @@ +import textwrap + +import pytest + +from entityservice.errors import InvalidConfiguration +from entityservice.utils import load_yaml_config +from entityservice.tests.util import generate_bytes, temp_file_containing + + +class TestYamlLoader: + + def test_empty(self): + with temp_file_containing(b'') as fp: + filename = fp.name + assert None == load_yaml_config(filename) + + def test_list(self): + with temp_file_containing(b'[1,2,3]') as fp: + filename = fp.name + assert [1,2,3] == load_yaml_config(filename) + + def test_missing_file(self): + filename = 'unlikely a valid file' + with pytest.raises(InvalidConfiguration): + load_yaml_config(filename) + + def test_random_bytes(self): + with temp_file_containing(generate_bytes(128)) as fp: + filename = fp.name + with pytest.raises(InvalidConfiguration): + load_yaml_config(filename) + + def test_valid_yaml(self): + yamldata = textwrap.dedent(""" + api: + number: 42 + ingress: + enabled: true + host: example.com + """) + self._check_valid_yaml(yamldata) + + def _check_valid_yaml(self, yamldata:str): + with temp_file_containing(yamldata.encode()) as fp: + filename = fp.name + loaded = load_yaml_config(filename) + assert 'api' in loaded + assert 'number' in loaded['api'] + assert loaded['api']['number'] == 42 + assert loaded['api']['ingress']['enabled'] + return loaded + + def test_valid_yaml_with_comments(self): + yamldata = textwrap.dedent(""" + ## Api is a thing + api: + number: 42 + ingress: + enabled: true + # host: example.com + """) + loaded = self._check_valid_yaml(yamldata) + assert 'host' not in loaded['api']['ingress'] + diff --git a/backend/entityservice/tests/util.py b/backend/entityservice/tests/util.py index afb14724..8d9a3e52 100644 --- a/backend/entityservice/tests/util.py +++ b/backend/entityservice/tests/util.py @@ -5,6 +5,7 @@ import os import random import time +import tempfile from contextlib import contextmanager from enum import IntEnum @@ -14,6 +15,14 @@ from entityservice.tests.config import url +@contextmanager +def temp_file_containing(data): + with tempfile.NamedTemporaryFile('wb') as fp: + fp.write(data) + fp.seek(0) + yield fp + + def serialize_bytes(hash_bytes): """ Serialize bloomfilter bytes @@ -142,10 +151,8 @@ def create_project_no_data(requests, @contextmanager def temporary_blank_project(requests, result_type='groups'): - # Code to acquire resource, e.g.: project = create_project_no_data(requests, result_type) yield project - # Release project resource delete_project(requests, project) diff --git a/backend/entityservice/tracing.py b/backend/entityservice/tracing.py index 6abd6392..1a437ba6 100644 --- a/backend/entityservice/tracing.py +++ b/backend/entityservice/tracing.py @@ -7,34 +7,27 @@ from opentracing_instrumentation import get_current_span, span_in_context from entityservice.settings import Config as config +from entityservice.utils import load_yaml_config +DEFAULT_TRACER_CONFIG = {'sampler': {'type': 'const', 'param': 1}} -def initialize_tracer(service_name='anonlink'): - jaeger_config = jaeger_client.Config( - config={ - 'sampler': {'type': 'const', 'param': 1}, - 'local_agent': { - 'reporting_host': config.TRACING_HOST, - 'reporting_port': config.TRACING_PORT, - } - }, - service_name=service_name) +def get_tracer_config(service_name): + if config.TRACING_CONFIG_FILENAME is not None: + tracing_config = load_yaml_config(config.TRACING_CONFIG_FILENAME) + else: + tracing_config = DEFAULT_TRACER_CONFIG + return jaeger_client.Config(config=tracing_config, service_name=service_name) + + +def initialize_tracer(service_name='api'): + jaeger_config = get_tracer_config(service_name) # Note this call also sets opentracing.tracer return jaeger_config.initialize_tracer() def create_tracer(service_name='worker'): - jaeger_config = jaeger_client.Config( - config={ - 'sampler': {'type': 'const', 'param': 1}, - 'local_agent': { - 'reporting_host': config.TRACING_HOST, - 'reporting_port': config.TRACING_PORT, - } - }, - service_name=service_name) - + jaeger_config = get_tracer_config(service_name) return jaeger_config.new_tracer() diff --git a/backend/entityservice/utils.py b/backend/entityservice/utils.py index ac5c0ae7..e06f22e5 100644 --- a/backend/entityservice/utils.py +++ b/backend/entityservice/utils.py @@ -8,15 +8,36 @@ import binascii import bitmath -from flask import request from connexion import ProblemException +from flask import request from structlog import get_logger +import yaml +from entityservice.errors import InvalidConfiguration from entityservice.database import DBConn, get_number_parties_uploaded, get_number_parties_ready, get_project_column logger = get_logger() +def load_yaml_config(filename): + """ + Load a yaml file as a Python object. + + :param filename: a Path or String object + :raises InvalidConfiguration if the file isn't found or the yaml isn't valid. + :return: Python representation of yaml file's contents (usually Dict), or None if empty. + """ + try: + with open(filename, 'rt') as f: + return yaml.safe_load(f) + except UnicodeDecodeError as e: + raise InvalidConfiguration(f"YAML file '{filename}' appears corrupt") from e + except yaml.YAMLError as e: + raise InvalidConfiguration(f"Parsing YAML config from '{filename}' failed") from e + except FileNotFoundError as e: + raise InvalidConfiguration(f"Logging config YAML file '{filename}' doesn't exist.") from e + + def fmt_bytes(num_bytes): """ Displays an integer number of bytes in a human friendly form. diff --git a/deployment/entity-service/templates/api-deployment.yaml b/deployment/entity-service/templates/api-deployment.yaml index 601e4e98..0a5f7d42 100644 --- a/deployment/entity-service/templates/api-deployment.yaml +++ b/deployment/entity-service/templates/api-deployment.yaml @@ -6,6 +6,10 @@ metadata: {{- include "es.release_labels" . | indent 4 }} component: "{{ .Values.api.name }}" tier: frontend + {{- if .Values.api.deploymentAnnotations }} + annotations: +{{ toYaml .Values.api.deploymentAnnotations | indent 4 }} + {{- end }} spec: replicas: {{ required "api.replicaCount must be provided." .Values.api.replicaCount }} selector: @@ -95,20 +99,13 @@ spec: initialDelaySeconds: 60 periodSeconds: 60 timeoutSeconds: 5 - {{- if .Values.loggingCfg }} volumeMounts: - name: config-volume mountPath: /var/config - {{- end }} - {{- if .Values.loggingCfg }} volumes: - name: config-volume configMap: - name: {{ template "es.fullname" . }}-logging - items: - - key: "loggingCfg" - path: "logging_file.yaml" - {{- end }} + name: {{ template "es.fullname" . }}-monitoring-config {{- if .Values.api.pullSecret }} imagePullSecrets: - name: {{ .Values.api.pullSecret }} diff --git a/deployment/entity-service/templates/configmap.yaml b/deployment/entity-service/templates/configmap.yaml index 157394aa..7a0c4306 100644 --- a/deployment/entity-service/templates/configmap.yaml +++ b/deployment/entity-service/templates/configmap.yaml @@ -66,7 +66,9 @@ data: MIN_ENCODING_SIZE: "8" MAX_ENCODING_SIZE: "1024" - {{- if .Values.loggingCfg }} - # Not from the setting.py file. - LOG_CFG: "/var/config/logging_file.yaml" - {{- end }} \ No newline at end of file + LOG_CFG: "/var/config/loggingCfg" + TRACE_CFG: "/var/config/tracingCfg" + +{{- if .Values.anonlink.config }} +{{ toYaml .Values.anonlink.config | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/deployment/entity-service/templates/highmemory-worker-deployment.yaml b/deployment/entity-service/templates/highmemory-worker-deployment.yaml index f91bcbd8..13b23dd1 100644 --- a/deployment/entity-service/templates/highmemory-worker-deployment.yaml +++ b/deployment/entity-service/templates/highmemory-worker-deployment.yaml @@ -6,6 +6,10 @@ metadata: component: {{ list (required "workers.name must be provided." .Values.workers.name) "highmemory" | join "-" | quote }} tier: backend name: {{ .Release.Name }}-highmemory-worker + {{- if .Values.workers.deploymentAnnotations }} + annotations: +{{ toYaml .Values.workers.deploymentAnnotations | indent 4 }} + {{- end }} spec: replicas: {{ required "workers.highmemory.replicaCount must be provided." .Values.workers.highmemory.replicaCount }} selector: @@ -69,24 +73,17 @@ spec: - "fair" - "-Q" - "celery,compute,highmemory" - {{- if .Values.loggingCfg }} volumeMounts: - name: config-volume mountPath: /var/config - {{- end }} args: {{- range $key, $value := .Values.workers.extraArgs }} - --{{ $key }}={{ $value }} {{- end }} - {{- if .Values.loggingCfg }} volumes: - name: config-volume configMap: - name: {{ template "es.fullname" . }}-logging - items: - - key: loggingCfg - path: "logging_file.yaml" - {{- end }} + name: {{ template "es.fullname" . }}-monitoring-config {{- if .Values.api.pullSecret }} imagePullSecrets: - name: {{ .Values.api.pullSecret }} diff --git a/deployment/entity-service/templates/init-db-job.yaml b/deployment/entity-service/templates/init-db-job.yaml index a65e2898..8f2ecfad 100644 --- a/deployment/entity-service/templates/init-db-job.yaml +++ b/deployment/entity-service/templates/init-db-job.yaml @@ -42,6 +42,13 @@ spec: - "-m" - "flask" - "initdb" + volumeMounts: + - name: config-volume + mountPath: /var/config + volumes: + - name: config-volume + configMap: + name: {{ template "es.fullname" . }}-monitoring-config restartPolicy: Never {{- if .Values.api.pullSecret }} imagePullSecrets: diff --git a/deployment/entity-service/templates/logging-configmap.yaml b/deployment/entity-service/templates/monitoring-configmap.yaml similarity index 58% rename from deployment/entity-service/templates/logging-configmap.yaml rename to deployment/entity-service/templates/monitoring-configmap.yaml index 8a2863f6..c5e68a4b 100644 --- a/deployment/entity-service/templates/logging-configmap.yaml +++ b/deployment/entity-service/templates/monitoring-configmap.yaml @@ -1,11 +1,11 @@ -{{- if .Values.loggingCfg }} apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "es.fullname" . }}-logging + name: {{ template "es.fullname" . }}-monitoring-config labels: {{- include "es.release_labels" . | indent 4 }} data: + tracingCfg: |- +{{ .Values.tracingConfig | indent 4 }} loggingCfg: |- {{ .Values.loggingCfg | indent 4 }} -{{- end }} \ No newline at end of file diff --git a/deployment/entity-service/templates/worker-deployment.yaml b/deployment/entity-service/templates/worker-deployment.yaml index d0337a60..e7cc7d2a 100644 --- a/deployment/entity-service/templates/worker-deployment.yaml +++ b/deployment/entity-service/templates/worker-deployment.yaml @@ -6,6 +6,10 @@ metadata: component: {{ required "workers.name must be provided." .Values.workers.name | quote }} tier: backend name: {{ .Release.Name }}-worker + {{- if .Values.workers.deploymentAnnotations }} + annotations: +{{ toYaml .Values.workers.deploymentAnnotations | indent 4 }} + {{- end }} spec: replicas: {{ required "workers.replicaCount must be provided." .Values.workers.replicaCount }} selector: @@ -69,24 +73,17 @@ spec: - "fair" - "-Q" - "celery,compute" - {{- if .Values.loggingCfg }} volumeMounts: - name: config-volume mountPath: /var/config - {{- end }} args: {{- range $key, $value := .Values.workers.extraArgs }} - --{{ $key }}={{ $value }} {{- end }} - {{- if .Values.loggingCfg }} volumes: - name: config-volume configMap: - name: {{ template "es.fullname" . }}-logging - items: - - key: loggingCfg - path: "logging_file.yaml" - {{- end }} + name: {{ template "es.fullname" . }}-monitoring-config {{- if .Values.api.pullSecret }} imagePullSecrets: - name: {{ .Values.api.pullSecret }} diff --git a/deployment/entity-service/values.yaml b/deployment/entity-service/values.yaml index 6e217571..3cfd2f1b 100644 --- a/deployment/entity-service/values.yaml +++ b/deployment/entity-service/values.yaml @@ -3,6 +3,12 @@ rbac: ## See issue #88 create: false +anonlink: + + ## Set arbitrary environment variables for the API and Workers. + config: {} + + api: ## Deployment component name @@ -22,6 +28,11 @@ api: podAnnotations: {} # iam.amazonaws.com/role: linkage + ## Annotations added to the api Deployment + deploymentAnnotations: # {} + # This annotation enables jaeger injection for open tracing + "sidecar.jaegertracing.io/inject": "true" + ## Settings for the nginx proxy www: @@ -155,6 +166,10 @@ workers: podAnnotations: {} + deploymentAnnotations: # {} + # This annotation enables jaeger injection for open tracing + "sidecar.jaegertracing.io/inject": "true" + #strategy: "" ## Additional Entity Service Worker container arguments @@ -357,28 +372,90 @@ provision: postgresql: true redis: true -## Custom logging file used to override the default settings. Will be used by the workers and the flask container. + +## Tracing config used by jaeger-client-python +## https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/config.py +tracingConfig: |- + logging: true + metrics: true + sampler: + type: const + param: 1 + +## Custom logging file used to override the default settings. Will be used by the workers and the api container. ## Example of logging configuration: -#loggingCfg: |- -# version: 1 -# disable_existing_loggers: False -# formatters: -# simple: -# format: "%(message)s" -# file: -# format: "%(asctime)-15s %(name)-12s %(levelname)-8s: %(message)s" -# filters: -# stderr_filter: -# (): entityservice.logger_setup.StdErrFilter -# stdout_filter: -# (): entityservice.logger_setup.StdOutFilter -# handlers: -# stdout: -# class: logging.StreamHandler -# level: DEBUG -# formatter: simple -# filters: [stdout_filter] -# stream: ext://sys.stdout -# root: -# level: INFO -# handlers: [stdout] +loggingCfg: |- + version: 1 + disable_existing_loggers: False + formatters: + simple: + format: "%(message)s" + file: + format: "%(asctime)-15s %(name)-12s %(levelname)-8s: %(message)s" + filters: + stderr_filter: + (): entityservice.logger_setup.StdErrFilter + stdout_filter: + (): entityservice.logger_setup.StdOutFilter + + handlers: + stdout: + class: logging.StreamHandler + level: DEBUG + formatter: simple + filters: [stdout_filter] + stream: ext://sys.stdout + + stderr: + class: logging.StreamHandler + level: ERROR + formatter: simple + filters: [stderr_filter] + stream: ext://sys.stderr + + info_file_handler: + class: logging.handlers.RotatingFileHandler + level: INFO + formatter: file + filename: info.log + maxBytes: 10485760 # 10MB + backupCount: 20 + encoding: utf8 + + error_file_handler: + class: logging.handlers.RotatingFileHandler + level: ERROR + formatter: file + filename: errors.log + maxBytes: 10485760 # 10MB + backupCount: 20 + encoding: utf8 + + loggers: + entityservice: + level: INFO + + entityservice.database.util: + level: WARNING + + entityservice.cache: + level: WARNING + + entityservice.utils: + level: INFO + + celery: + level: INFO + + jaeger_tracing: + level: WARNING + propagate: no + + werkzeug: + level: WARNING + propagate: no + + root: + level: INFO + handlers: [stdout, stderr, info_file_handler, error_file_handler] + diff --git a/docs/logging.rst b/docs/logging.rst index 0c8e7e22..2107490e 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -16,8 +16,10 @@ The following named loggers are used: The following environment variables affect logging: * `LOG_CFG` - sets the path to a logging configuration file. There are two examples: + - `entityservice/default_logging.yaml` - `entityservice/verbose_logging.yaml` + * `DEBUG` - sets the logging level to debug for all application code. * `LOGFILE` - directs the log output to this file instead of stdout. * `LOG_HTTP_HEADER_FIELDS` - HTTP headers to include in the application logs. @@ -63,5 +65,4 @@ With `DEBUG` enabled there are a lot of logs from the backend and workers:: Tracing ------- -* `TRACING_HOST` -* `TRACING_PORT` +`TRACING_CFG` overrides the path to an open tracing configuration file.