Skip to content

add test samples tracking #38502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kristapratico merged 3 commits into from
Nov 15, 2024
Merged

add test samples tracking #38502

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cb3199f
add test samples tracking
kristapratico Nov 13, 2024
336ffd7
add samples tracking to csv/html/md reports
kristapratico Nov 13, 2024
7e9a3d4
fix for catastrophic pipeline failure
kristapratico Nov 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion scripts/repo_health_status_report/output_health_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ class TestsPipelineResult(typing.TypedDict, total=False):
link: str
result: DEVOPS_BUILD_STATUS
tests: CheckStatus
samples: CheckStatus


class CIPipelineResult(typing.TypedDict, total=False):
Expand Down Expand Up @@ -152,6 +153,7 @@ class LibraryStatus(typing.TypedDict, total=False):
sphinx: Status
sdk_owned: bool
tests: Status
samples: Status
ci: Status


Expand Down Expand Up @@ -224,6 +226,18 @@ def skip_package(package_name: str) -> bool:
)


def samples_enabled(package_path: pathlib.Path) -> bool:
tests_yaml = package_path.parent / "tests.yml"
if not tests_yaml.exists():
return False
with open(tests_yaml, "r") as file:
parameters = file.read()

if "TestSamples=.*/true" in parameters:
return True
return False


def get_dataplane(
include_sdk_owned: bool = True,
) -> dict[ServiceDirectory, dict[LibraryName, LibraryStatus]]:
Expand Down Expand Up @@ -336,6 +350,7 @@ def record_all_pipeline(
{
"result": status,
"tests": CheckStatus(status=status),
"samples": CheckStatus(status=status),
}
)
)
Expand Down Expand Up @@ -367,7 +382,7 @@ def record_all_library(details: LibraryStatus, status: CHECK_STATUS) -> None:
details["sphinx"] = Status(status=status, link=None)
details["ci"] = Status(status=status, link=None)
details["tests"] = Status(status=status, link=None)

details["samples"] = Status(status=status, link=None)

def get_ci_result(service: str, pipeline_id: int | None, pipelines: dict[ServiceDirectory, PipelineResults]) -> None:
if not pipeline_id:
Expand All @@ -393,6 +408,9 @@ def get_ci_result(service: str, pipeline_id: int | None, pipelines: dict[Service
pipelines[service]["ci"].update({"result": result["result"]})
build_id = result["id"]
timeline_response = httpx.get(get_build_timeline_url(build_id), headers=AUTH_HEADERS)
if timeline_response.status_code != 200:
record_all_pipeline("tests", pipelines[service], "UNKNOWN")
return
timeline_result = json.loads(timeline_response.text)

for task in timeline_result["records"]:
Expand Down Expand Up @@ -432,11 +450,16 @@ def get_tests_result(service: str, pipeline_id: int | None, pipelines: dict[Serv
pipelines[service]["tests"].update({"result": result["result"]})
build_id = result["id"]
timeline_response = httpx.get(get_build_timeline_url(build_id), headers=AUTH_HEADERS)
if timeline_response.status_code != 200:
record_all_pipeline("tests", pipelines[service], "UNKNOWN")
return
timeline_result = json.loads(timeline_response.text)

for task in timeline_result["records"]:
if "Run Tests" in task["name"]:
record_test_result(task, "tests", pipelines[service]["tests"])
if "Test Samples" in task["name"]:
record_test_result(task, "samples", pipelines[service]["tests"])


def get_tests_weekly_result(service: str, pipeline_id: int | None, pipelines: dict[ServiceDirectory, PipelineResults]) -> None:
Expand All @@ -459,6 +482,9 @@ def get_tests_weekly_result(service: str, pipeline_id: int | None, pipelines: di
pipelines[service]["tests_weekly"].update({"result": result["result"]})
build_id = result["id"]
timeline_response = httpx.get(get_build_timeline_url(build_id), headers=AUTH_HEADERS)
if timeline_response.status_code != 200:
record_all_pipeline("tests", pipelines[service], "UNKNOWN")
return
timeline_result = json.loads(timeline_response.text)

for task in timeline_result["records"]:
Expand Down Expand Up @@ -506,6 +532,25 @@ def report_test_result(
library_details[test_type] = Status(status="UNKNOWN", link=pipeline[test_type].get("link"))


def report_samples_result(
check: typing.Literal["samples"],
pipeline: PipelineResults,
library_details: LibraryStatus,
) -> None:
enabled = samples_enabled(library_details["path"])
if not enabled:
library_details[check] = Status(status="DISABLED", link=None)
return

ci_check = pipeline["tests"][check]["status"]
if ci_check == "succeeded":
library_details[check] = Status(status="PASS", link=pipeline["tests"]["link"])
elif ci_check == "failed":
library_details[check] = Status(status="FAIL", link=pipeline["tests"]["link"])
else:
library_details[check] = Status(status="UNKNOWN", link=pipeline["tests"].get("link"))


def report_check_result(
check: CheckTypes,
pipeline: PipelineResults,
Expand Down Expand Up @@ -558,6 +603,7 @@ def report_status(
details["type_check_samples"] = (
"ENABLED" if is_check_enabled(str(details["path"]), "type_check_samples") else "DISABLED"
)
report_samples_result("samples", pipelines[service_directory], details)
details["sdk_owned"] = details["path"].name in SDK_TEAM_OWNED
report_test_result("tests", pipelines[service_directory], details)
report_test_result("ci", pipelines[service_directory], details)
Expand Down Expand Up @@ -687,6 +733,7 @@ def write_to_csv(libraries: dict[ServiceDirectory, dict[LibraryName, LibraryStat
"Sphinx",
"Tests - CI",
"Tests - Live",
"Tests - Samples",
"SLA - Questions",
"SLA - Bugs",
"Total customer-reported issues",
Expand All @@ -696,6 +743,7 @@ def write_to_csv(libraries: dict[ServiceDirectory, dict[LibraryName, LibraryStat
"Sphinx_link",
"Tests - CI_link",
"Tests - Live_link",
"Tests - Samples_link",
"SLA - Questions_link",
"SLA - Bugs_link",
"Total customer-reported issues_link",
Expand All @@ -718,6 +766,7 @@ def write_to_csv(libraries: dict[ServiceDirectory, dict[LibraryName, LibraryStat
details["sphinx"]["status"],
details["ci"]["status"],
details["tests"]["status"],
details["samples"]["status"],
details.get("sla", {}).get("question", {}).get("num", 0),
details.get("sla", {}).get("bug", {}).get("num", 0),
details.get("customer_issues", {}).get("num", 0),
Expand All @@ -727,6 +776,7 @@ def write_to_csv(libraries: dict[ServiceDirectory, dict[LibraryName, LibraryStat
details["sphinx"].get("link", ""),
details["ci"].get("link", ""),
details["tests"].get("link", ""),
details["samples"].get("link", ""),
details.get("sla", {}).get("question", {}).get("link", ""),
details.get("sla", {}).get("bug", {}).get("link", ""),
details.get("customer_issues", {}).get("link", ""),
Expand All @@ -751,6 +801,7 @@ def write_to_markdown(libraries: dict[ServiceDirectory, dict[LibraryName, Librar
"Sphinx",
"Tests - CI",
"Tests - Live",
"Tests - Samples",
"SLA - Questions / Bugs",
"Total customer-reported issues",
]
Expand Down Expand Up @@ -795,6 +846,8 @@ def write_to_markdown(libraries: dict[ServiceDirectory, dict[LibraryName, Librar
+ (f" ([link]({details['ci']['link']}))" if details["ci"]["link"] is not None else ""),
details["tests"]["status"]
+ (f" ([link]({details['tests']['link']}))" if details["tests"]["link"] is not None else ""),
details["samples"]["status"]
+ (f" ([link]({details['samples']['link']}))" if details["samples"]["link"] is not None else ""),
sla_str,
str(details.get("customer_issues", {}).get("num", 0))
+ (
Expand Down