Use --query argument when calling summary tool

erichaagdev · erichaagdev · commit fcb4e2000dba · 2025-03-28T16:25:58.000-05:00
Calling the summary tool with the --query argument will allow all build
scan data to be fetched at once. This is possible when both build scans
are published during script execution. When that's not the case, the
--query argument is not used.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -61,7 +61,7 @@ val isDevelopmentRelease = !hasProperty("finalRelease")
 val releaseVersion = releaseVersion()
 val releaseNotes = releaseNotes()
 val distributionVersion = distributionVersion()
-val buildScanSummaryVersion = "1.0.4-2024.1"
+val buildScanSummaryVersion = "1.0.4-2024.1" // todo update to 1.0.5-2024.1 once released
 
 allprojects {
     version = releaseVersion.get()
diff --git a/components/scripts/gradle/01-validate-incremental-building.sh b/components/scripts/gradle/01-validate-incremental-building.sh
@@ -62,7 +62,7 @@ execute() {
   rename_project_dir "build_${project_name}" "second-build_${project_name}"
 
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -109,7 +109,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -139,8 +139,8 @@ print_gradle_command() {
     info "./gradlew --no-build-cache -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false $*$(print_extra_args)"
 }
 
-fetch_build_cache_metrics() {
-  process_build_scan_data_online
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
 }
 
 # Overrides summary.sh#print_performance_characteristics
diff --git a/components/scripts/gradle/02-validate-local-build-caching-same-location.sh b/components/scripts/gradle/02-validate-local-build-caching-same-location.sh
@@ -63,7 +63,7 @@ execute() {
   rename_project_dir "build_${project_name}" "second-build_${project_name}"
 
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -111,7 +111,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -144,8 +144,8 @@ print_gradle_command() {
   info "./gradlew --build-cache -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
 }
 
-fetch_build_cache_metrics() {
-  process_build_scan_data_online
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
 }
 
 print_introduction() {
diff --git a/components/scripts/gradle/03-validate-local-build-caching-different-locations.sh b/components/scripts/gradle/03-validate-local-build-caching-different-locations.sh
@@ -64,7 +64,7 @@ execute() {
   execute_second_build
 
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -115,7 +115,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -149,8 +149,8 @@ print_gradle_command() {
   info "./gradlew --build-cache -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
 }
 
-fetch_build_cache_metrics() {
-  process_build_scan_data_online
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
 }
 
 print_introduction() {
diff --git a/components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh b/components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh
@@ -52,7 +52,7 @@ main() {
 }
 
 execute() {
-  fetch_build_scans
+  process_build_scan_data
   make_experiment_dir
 
   print_bl
@@ -91,7 +91,7 @@ wizard_execute() {
   explain_command_to_repeat_experiment_after_collecting_parameters
 
   print_bl
-  fetch_build_scans
+  process_build_scan_data
   make_experiment_dir
 
   print_bl
@@ -116,8 +116,8 @@ validate_required_args() {
   fi
 }
 
-fetch_build_scans() {
-  fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}"
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_VERBOSE" "$(query_none)"
 }
 
 # Overrides summary.sh#print_experiment_specific_summary_info
diff --git a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh
@@ -66,7 +66,7 @@ execute() {
   execute_build
 
   print_bl
-  process_build_scan_data_online
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -130,7 +130,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  process_build_scan_data_online
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -224,6 +224,10 @@ execute_build() {
   invoke_gradle 1 "${args[@]}"
 }
 
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_none)"
+}
+
 # Overrides summary.sh#print_experiment_specific_summary_info
 print_experiment_specific_summary_info() {
   summary_row "Custom value mapping file:" "${mapping_file:-<none>}"
diff --git a/components/scripts/lib/build-scan-online.sh b/components/scripts/lib/build-scan-online.sh
@@ -1,11 +1,37 @@
 #!/usr/bin/env bash
 
-# This is a helper function for the common pattern of reading Build Scan metadata
-# from the build-scans.csv file, then retrieving build metrics using the
-# Develocity API.
+readonly LOGGING_BRIEF='brief_logging'
+readonly LOGGING_VERBOSE='verbose_logging'
+
+# Main entrypoint for processing data online using the Build Scan summary tool.
+# All scripts should call this function to fetch Build Scan data used in the
+# experiment summary.
+#
+# USAGE: fetch_build_scans_and_build_time_metrics <logging_level> <query>
+#
+# <logging_level> should be set using either LOGGING_BRIEF or LOGGING_VERBOSE
+#                 constant, e.g., "$LOGGING_BRIEF"
+# <query>         should be set using either query_none or query_run_id
+#                 function, e.g., "$(query_none)"
 process_build_scan_data_online() {
+  local logging_level="$1"
+  local query="$2"
+
+  # Always call since it will only read if the metadata file exists
   read_build_scan_metadata
-  fetch_build_scans_and_build_time_metrics 'brief_logging' "${build_scan_urls[@]}"
+
+  local build_scan_data
+  build_scan_data="$(fetch_build_scan_data "${@}")"
+  parse_build_scans_and_build_time_metrics "${build_scan_data}"
+}
+
+query_none() {
+  # Intentionally no-op so query definitions can appear consistent
+  true
+}
+
+query_run_id() {
+  echo "value:\"Experiment run id=$RUN_ID\""
 }
 
 read_build_scan_metadata() {
@@ -46,38 +72,25 @@ is_build_scan_metadata_missing() {
   return 0
 }
 
+# Used by CI / Local experiments to fetch data for the first CI build.
 fetch_single_build_scan() {
   local build_scan_url="$1"
 
   local build_scan_data
-  build_scan_data="$(fetch_build_scan_data 'verbose_logging' "${build_scan_url}")"
+  build_scan_data="$(fetch_build_scan_data "$LOGGING_VERBOSE" "$(query_none)")"
 
   parse_single_build_scan "${build_scan_data}"
 }
 
-# The value of logging_level should be either 'brief_logging' or
-# 'verbose_logging'
-fetch_build_scans_and_build_time_metrics() {
-  local logging_level="$1"
-  shift
-  local build_scan_urls=("$@")
-
-  if [[ "${logging_level}" == 'verbose_logging' ]]; then
-    info "Fetching build scan data"
-  fi
-
-  local build_scan_data
-  build_scan_data="$(fetch_build_scan_data "${logging_level}" "${build_scan_urls[@]}")"
-
-  parse_build_scans_and_build_time_metrics "${build_scan_data}"
-}
-
-# Note: Callers of this function require stdout to be clean. No logging can be
-#       done inside this function.
+# WARNING: Experiment scripts should not call this function directly and instead
+#          use the process_build_scan_data_online or fetch_single_build_scan
+#          function.
+#
+# WARNING: Callers of this function require stdout to be clean. No logging can
+#          be done inside this function.
 fetch_build_scan_data() {
   local logging_level="$1"
-  shift
-  local build_scan_urls=("$@")
+  local query="$2"
 
   if [[ "${debug_mode}" == "on" ]]; then
     args+=("--debug")
@@ -91,14 +104,18 @@ fetch_build_scan_data() {
     args+=("--network-settings-file" "${SCRIPT_DIR}/network.settings")
   fi
 
-  if [[ "${logging_level}" == "brief_logging" ]]; then
+  if [[ "${logging_level}" == "${LOGGING_BRIEF}" ]]; then
     args+=("--brief-logging")
   fi
 
   if [[ "${fail_if_not_fully_cacheable}" == "on" ]]; then
     args+=("--build-scan-availability-wait-timeout" "60")
   fi
 
+  if [[ -n "${query}" ]]; then
+    args+=("--query" "$query")
+  fi
+
   for run_num in "${!build_scan_urls[@]}"; do
     args+=( "${run_num},${build_scan_urls[run_num]}" )
   done
diff --git a/components/scripts/maven/01-validate-local-build-caching-same-location.sh b/components/scripts/maven/01-validate-local-build-caching-same-location.sh
@@ -62,7 +62,7 @@ execute() {
   rename_project_dir "build_${project_name}" "second-build_${project_name}"
 
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -110,7 +110,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -149,8 +149,8 @@ print_maven_command() {
    info "./mvnw -Dscan -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
  }
 
-fetch_build_cache_metrics() {
-  process_build_scan_data_online
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
 }
 
 print_introduction() {
diff --git a/components/scripts/maven/02-validate-local-build-caching-different-locations.sh b/components/scripts/maven/02-validate-local-build-caching-different-locations.sh
@@ -62,7 +62,7 @@ execute() {
   execute_second_build
 
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -113,7 +113,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  fetch_build_cache_metrics
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -153,8 +153,8 @@ print_maven_command() {
   info "./mvnw -Dscan -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
 }
 
-fetch_build_cache_metrics() {
-  process_build_scan_data_online
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
 }
 
 print_introduction() {
diff --git a/components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh b/components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh
@@ -50,7 +50,7 @@ main() {
 }
 
 execute() {
-  fetch_build_scans
+  process_build_scan_data
   make_experiment_dir
 
   print_bl
@@ -89,7 +89,7 @@ wizard_execute() {
   explain_command_to_repeat_experiment_after_collecting_parameters
 
   print_bl
-  fetch_build_scans
+  process_build_scan_data
   make_experiment_dir
 
   print_bl
@@ -114,8 +114,8 @@ validate_required_args() {
   fi
 }
 
-fetch_build_scans() {
-  fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}"
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_VERBOSE" "$(query_none)"
 }
 
 # Overrides summary.sh#print_experiment_specific_summary_info
diff --git a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh
@@ -63,7 +63,7 @@ execute() {
   execute_build
 
   print_bl
-  process_build_scan_data_online
+  process_build_scan_data
 
   print_bl
   print_summary
@@ -127,7 +127,7 @@ wizard_execute() {
   print_bl
   explain_measure_build_results
   print_bl
-  process_build_scan_data_online
+  process_build_scan_data
   print_bl
   explain_and_print_summary
 }
@@ -219,6 +219,10 @@ execute_build() {
   invoke_maven 1 "${args[@]}"
 }
 
+process_build_scan_data() {
+  process_build_scan_data_online "$LOGGING_BRIEF" "$(query_none)"
+}
+
 # Overrides summary.sh#print_experiment_specific_summary_info
 print_experiment_specific_summary_info() {
   summary_row "Custom value mapping file:" "${mapping_file:-<none>}"
diff --git a/release/changes.md b/release/changes.md
@@ -3,6 +3,7 @@
 
 - [NEW] Support Develocity and Gradle Enterprise remote build cache connectors in the Gradle CI/Local experiment  
 - [NEW] Better handling of remote build cache misconfigurations
+- [NEW] Fewer calls are made to the Develocity API when possible, resulting in faster and more stable execution 
 - [FIX] Scripts do not wait long enough for build scans to become available when `--fail-if-not-fully-cacheable` is used
 - [FIX] Successful exit code returned when performance characteristics are unknown and `--fail-if-not-fully-cacheable` is used
 - [FIX] Gradle experiments do not disable background Build Scan publication
