diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 2d6a9d06e2542..f919cf0483ac3 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -40,6 +40,7 @@ def get_long_description(): "avro-gen3==0.5.0", "avro-python3>=1.8.2", "python-dateutil", + "stackprinter", } kafka_common = { diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py index 2a29b6bd32366..3a0b1a879525a 100644 --- a/metadata-ingestion/src/datahub/entrypoints.py +++ b/metadata-ingestion/src/datahub/entrypoints.py @@ -4,6 +4,7 @@ import sys import click +import stackprinter from pydantic import ValidationError import datahub as datahub_package @@ -25,12 +26,14 @@ ) logging.basicConfig(format=BASE_LOGGING_FORMAT) +MAX_CONTENT_WIDTH = 120 + @click.group( context_settings=dict( # Avoid truncation of help text. # See https://github.com/pallets/click/issues/486. - max_content_width=120, + max_content_width=MAX_CONTENT_WIDTH, ) ) @click.option("--debug/--no-debug", default=False) @@ -66,6 +69,7 @@ def version() -> None: ) def ingest(config: str) -> None: """Ingest metadata into DataHub.""" + logger.debug("DataHub CLI version: %s", datahub_package.nice_version_name()) config_file = pathlib.Path(config) pipeline_config = load_config_file(config_file) @@ -77,7 +81,9 @@ def ingest(config: str) -> None: click.echo(e, err=True) sys.exit(1) + logger.info("Starting metadata ingestion") pipeline.run() + logger.info("Finished metadata ingestion") ret = pipeline.pretty_print_summary() sys.exit(ret) @@ -96,3 +102,13 @@ def main(**kwargs): except click.ClickException as error: error.show() sys.exit(1) + except Exception as exc: + logger.error( + stackprinter.format( + exc, + line_wrap=MAX_CONTENT_WIDTH, + truncate_vals=10 * MAX_CONTENT_WIDTH, + suppressed_paths=[r"lib/python.*/site-packages/click/"], + ) + ) + sys.exit(1)