chore: move build_engine scripts into this repo

shorebird/ci/internal/generate_manifest.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# This script outputs an artifact_manifest.yaml mapping
+# a shorebird engine revision to a flutter engine revision.
+# Usage:
+#  ./generate_manifest.sh <flutter_engine_revision> > artifact_manifest.yaml
+
+set -e
+
+# NOTE: If you edit this file you also may need to edit the global list
+# of all known artifacts in the artifact_proxy's config.dart
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: ./generate_manifest.sh <flutter_engine_revision>"
+  exit 1
+fi
+
+FLUTTER_ENGINE_REVISION=$1
+
+cat <<EOF
+# This file is generated by shorebird/packages/artifact_proxy/tool/generate_manifest.sh
+# Do not edit this file manually.
+
+flutter_engine_revision: $FLUTTER_ENGINE_REVISION
+storage_bucket: download.shorebird.dev
+artifact_overrides:
+  # Android release artifacts
+  # artifacts.zip includes flutter.jar, libflutter.so, etc.
+  # symbols.zip includes symbols for libflutter.so
+  # darwin-x64.zip gen_snapshot for darwin-x64
+  # windows-x64.zip gen_snapshot for windows-x64
+  # linux-x64.zip gen_snapshot for linux-x64
+  - flutter_infra_release/flutter/\$engine/android-arm64-release/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/android-arm64-release/darwin-x64.zip
+  - flutter_infra_release/flutter/\$engine/android-arm64-release/linux-x64.zip
+  - flutter_infra_release/flutter/\$engine/android-arm64-release/symbols.zip
+  - flutter_infra_release/flutter/\$engine/android-arm64-release/windows-x64.zip
+
+  - flutter_infra_release/flutter/\$engine/android-arm-release/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/android-arm-release/darwin-x64.zip
+  - flutter_infra_release/flutter/\$engine/android-arm-release/linux-x64.zip
+  - flutter_infra_release/flutter/\$engine/android-arm-release/symbols.zip
+  - flutter_infra_release/flutter/\$engine/android-arm-release/windows-x64.zip
+
+  - flutter_infra_release/flutter/\$engine/android-x64-release/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/android-x64-release/darwin-x64.zip
+  - flutter_infra_release/flutter/\$engine/android-x64-release/linux-x64.zip
+  - flutter_infra_release/flutter/\$engine/android-x64-release/symbols.zip
+  - flutter_infra_release/flutter/\$engine/android-x64-release/windows-x64.zip
+
+  # engine_stamp.json
+  - flutter_infra_release/flutter/\$engine/engine_stamp.json
+
+  # Dart SDK
+  - flutter_infra_release/flutter/\$engine/dart-sdk-darwin-arm64.zip
+  - flutter_infra_release/flutter/\$engine/dart-sdk-darwin-x64.zip
+  - flutter_infra_release/flutter/\$engine/dart-sdk-linux-x64.zip
+  - flutter_infra_release/flutter/\$engine/dart-sdk-windows-x64.zip
+
+  # embedding release
+  - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-\$engine/flutter_embedding_release-1.0.0-\$engine.pom
+  - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-\$engine/flutter_embedding_release-1.0.0-\$engine.jar
+  # arm64_v8a release
+  - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-\$engine/arm64_v8a_release-1.0.0-\$engine.pom
+  - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-\$engine/arm64_v8a_release-1.0.0-\$engine.jar
+  # armeabi_v7a release
+  - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-\$engine/armeabi_v7a_release-1.0.0-\$engine.pom
+  - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-\$engine/armeabi_v7a_release-1.0.0-\$engine.jar
+  # x86_64 release
+  - download.flutter.io/io/flutter/x86_64_release/1.0.0-\$engine/x86_64_release-1.0.0-\$engine.pom
+  - download.flutter.io/io/flutter/x86_64_release/1.0.0-\$engine/x86_64_release-1.0.0-\$engine.jar
+
+  # Common release artifacts
+  - flutter_infra_release/flutter/\$engine/flutter_patched_sdk_product.zip
+
+  # iOS release artifacts
+  # Includes unified Flutter.framework for device and simulator (debug)
+  - flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/ios-release/Flutter.dSYM.zip
+
+  # Linux release artifacts
+  - flutter_infra_release/flutter/\$engine/linux-x64/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/linux-x64-release/linux-x64-flutter-gtk.zip
+
+  # macOS release artifacts
+  - flutter_infra_release/flutter/\$engine/darwin-x64-release/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/darwin-x64-release/framework.zip
+  - flutter_infra_release/flutter/\$engine/darwin-x64-release/gen_snapshot.zip
+
+  # Windows release artifacts
+  - flutter_infra_release/flutter/\$engine/windows-x64/artifacts.zip
+  - flutter_infra_release/flutter/\$engine/windows-x64-release/windows-x64-flutter.zip
+EOF

shorebird/ci/internal/linux_build.sh

@@ -0,0 +1,105 @@
+#!/bin/bash -e
+
+# Usage:
+# ./linux_build.sh engine_path
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 engine_path"
+    exit 1
+fi
+
+# Convert to an absolute path so we don't need to worry about cd'ing back to
+# the root directory between commands.
+ENGINE_ROOT=$(realpath $1)
+
+ENGINE_SRC=$ENGINE_ROOT/src
+ENGINE_OUT=$ENGINE_SRC/out
+UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater
+HOST_ARCH='linux-x64'
+
+# Build the Rust library.
+cd $UPDATER_SRC/library
+
+# 24 is Flutter's current minimum supported version,
+# see https://docs.flutter.dev/reference/supported-platforms
+# previous iterations of cargo-ndk required the version to be passed as
+# -p <version>, but that no longer seems needed.
+# We always use the hermetic NDK from the engine repo.
+ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \
+cargo ndk \
+    --target armv7-linux-androideabi \
+    --target aarch64-linux-android \
+    --target i686-linux-android \
+    --target x86_64-linux-android \
+    build --release
+
+cargo build --release --target x86_64-unknown-linux-gnu
+
+# Build the patch tool.
+cd $UPDATER_SRC/patch
+cargo build --release
+
+# Compile the engine using the steps here:
+# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux
+cd $ENGINE_SRC
+
+NINJA="ninja"
+GN=./flutter/tools/gn
+# We could probably use our own prebuilt dart SDK, by modifying the gn files.
+GN_ARGS="--no-rbe --no-enable-unittests"
+
+# We could use Linux to generate all of our Android binaries, but we don't yet.
+# https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/linux_android_aot_engine.json#L40
+
+# Build the default and gen_snapshot targets.
+#
+# Linux doesn't seem to use "archive_gen_snapshot" as a target name yet.
+# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247
+ANDROID_TARGETS="default gen_snapshot"
+
+# Android arm64 release
+$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release
+$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS
+
+# Android arm32 release
+$GN $GN_ARGS --runtime-mode=release --android
+$NINJA -C out/android_release $ANDROID_TARGETS
+
+# Android x64 release
+$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release
+$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS
+
+# Build Dart and Flutter
+$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk
+# build Dart and the linux shell and flutter_patched_sdk_product.zip
+$NINJA -C out/host_release dart_sdk flutter/shell/platform/linux:flutter_gtk flutter/build/archives:flutter_patched_sdk
+# We want to build flutter/tools/font_subset, but that doesn't work with
+# --no-prebuilt-dart-sdk.
+# https://github.com/flutter/flutter/issues/164531
+
+# Build debug Linux artifacts
+# These are output to the `linux-x64` directory in host_debug, and are used
+# by `flutter build linux --release`.
+$GN $GN_ARGS --no-prebuilt-dart-sdk
+$NINJA -C ./out/host_debug flutter/build/archives:artifacts
+
+# Shorebird AOT Tools (Linker)
+mkdir -p $ENGINE_OUT/host_release/aot_tools
+
+# Dart kernel (.dill) files are not stable and can change with the version of Dart, so we
+# can't use this machine's `dart`.  Here we're using the version of Dart that this
+# version of the engine depends on, which should be the same version that
+# `flutter` ends up depending on.
+DART=$ENGINE_OUT/host_release/dart-sdk/bin/dart
+AOT_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/dart/pkg/aot_tools
+# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113
+(cd $AOT_TOOLS_PKG; $DART pub get)
+# This should be built as part of Dart and then pulled down as part of the engine build.
+# https://github.com/shorebirdtech/_build_engine/issues/88
+$DART compile kernel $AOT_TOOLS_PKG/bin/aot_tools.dart -o $ENGINE_OUT/host_release/aot_tools/aot-tools.dill
+
+mkdir -p $ENGINE_OUT/host_release/updater_tools
+UPDATER_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/updater/updater_tools
+# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113
+
+# We could also build the `patch` tool for Linux here.

shorebird/ci/internal/linux_setup.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -e
+
+# Usage:
+# ./linux_setup.sh
+
+# Per https://github.com/flutter/flutter/wiki/Setting-up-the-Engine-development-environment
+# Subset of ./flutter/build/install-build-deps-linux-desktop.sh
+sudo apt -y install libfreetype6-dev pkg-config
+
+# This assumes rust is installed, but could also install rust/cargo.
+
+# Need NDK from https://developer.android.com/ndk/downloads
+# The NDK version should match that of DEPS, e.g.
+# https://github.com/flutter/flutter/blame/b45fa18946ecc2d9b4009952c636ba7e2ffbb787/DEPS#L615
+# Example:
+# curl -O https://dl.google.com/android/repository/android-ndk-r27d-linux.zip
+# unzip android-ndk-r27d-linux.zip
+# On the GHA runners we set this in .github/workflows/build_engine.yaml
+# env:
+#   NDK_HOME: /home/gha/bin/android-ndk-r27d
+
+# We require an old version of cargo-ndk to support the old NDK Flutter
+# engine currently uses.
+cargo install cargo-ndk
+rustup target add \
+   aarch64-linux-android \
+   armv7-linux-androideabi \
+   x86_64-linux-android \
+   i686-linux-android \
+   x86_64-unknown-linux-gnu

shorebird/ci/internal/linux_test_build.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -e
+
+# The path to the Flutter engine.
+# Convert to an absolute path so we don't need to worry about cd'ing back to
+# the root directory between commands.
+ENGINE_ROOT=$(realpath $1)
+ENGINE_SRC=$ENGINE_ROOT/src
+
+cd $ENGINE_SRC
+
+UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater
+(cd $UPDATER_SRC &&
+  ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \
+    cargo ndk \
+    --target armv7-linux-androideabi \
+    --target aarch64-linux-android \
+    --target i686-linux-android \
+    --target x86_64-linux-android \
+    build --release &&
+  cargo build --release --target x86_64-unknown-linux-gnu
+)
+
+# Generate an unoptimized debug build of the engine (expected by the test script).
+./flutter/tools/gn --unoptimized --no-rbe
+ninja -C out/host_debug_unopt
+
+# Generate an unoptimized android debug build for java engine tests
+./flutter/tools/gn --android --unoptimized --no-rbe
+ninja -C out/android_debug_unopt

shorebird/ci/internal/linux_upload.sh

@@ -0,0 +1,138 @@
+#!/bin/bash -e
+
+# Usage:
+# ./linux_upload.sh engine_path git_hash
+
+# Convert to an absolute path so we don't need to worry about cd'ing back to
+# the root directory between commands.
+ENGINE_ROOT=$(realpath $1)
+ENGINE_HASH=$2
+
+STORAGE_BUCKET="download.shorebird.dev"
+SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH
+
+ENGINE_SRC=$ENGINE_ROOT/src
+ENGINE_OUT=$ENGINE_SRC/out
+ENGINE_FLUTTER=$ENGINE_SRC/flutter
+# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/)
+FLUTTER_ROOT=$(dirname $ENGINE_ROOT)
+
+cd $FLUTTER_ROOT
+
+# Compute the content-aware hash for the Dart SDK.
+# This allows Flutter checkouts that haven't changed engine content to share
+# the same pre-built Dart SDK, even if they have different git commit SHAs.
+CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh)
+
+# We do not generate a manifest file, we assume another builder did that.
+# TODO(bryanoltman): should we generate a manifest file as part of an upload
+#   script, or should it be done once all build and uploads have completed?
+#   See https://github.com/shorebirdtech/build_engine/issues/25
+
+# TODO(eseidel): This should not be in shell, it's too complicated/repetitive.
+
+HOST_ARCH='linux-x64'
+
+INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH"
+MAVEN_VER="1.0.0-$ENGINE_HASH"
+MAVEN_ROOT="gs://$STORAGE_BUCKET/download.flutter.io/io/flutter"
+
+# Dart SDK
+# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip
+# We also upload to the content-aware hash path to support local development branches.
+CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH"
+
+HOST_RELEASE=$ENGINE_OUT/host_release
+DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip
+(
+    cd $HOST_RELEASE;
+    zip -r $DART_ZIP_FILE dart-sdk
+)
+ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE
+gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST
+# Also upload to content-aware hash path
+gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE
+
+# Android Arm64 release Flutter artifacts
+ARCH_OUT=$ENGINE_OUT/android_release_arm64
+ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release
+ZIPS_DEST=$INFRA_ROOT/android-arm64-release
+gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip
+gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip
+gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip
+# Android Arm64 release Maven artifacts
+ARCH_PATH=$ARCH_OUT/arm64_v8a_release
+MAVEN_PATH=$MAVEN_ROOT/arm64_v8a_release/$MAVEN_VER/arm64_v8a_release-$MAVEN_VER
+gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom
+gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar
+gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml
+
+# Android Arm32 release Flutter artifacts
+ARCH_OUT=$ENGINE_OUT/android_release
+ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release
+ZIPS_DEST=$INFRA_ROOT/android-arm-release
+gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip
+gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip
+gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip
+# Android Arm32 release Maven artifacts
+ARCH_PATH=$ARCH_OUT/armeabi_v7a_release
+MAVEN_PATH=$MAVEN_ROOT/armeabi_v7a_release/$MAVEN_VER/armeabi_v7a_release-$MAVEN_VER
+gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom
+gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar
+gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml
+
+# Not sure which flutter_embedding_release files to use? 32 or 64 bit?
+# It does not seem to contain the libflutter.so file, but does seem to
+# differ between the two build dirs.
+ARCH_OUT=$ENGINE_OUT/android_release
+ARCH_PATH=$ARCH_OUT/flutter_embedding_release
+MAVEN_PATH=$MAVEN_ROOT/flutter_embedding_release/$MAVEN_VER/flutter_embedding_release-$MAVEN_VER
+gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom
+gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar
+gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml
+
+# Android x64 release Flutter artifacts
+ARCH_OUT=$ENGINE_OUT/android_release_x64
+ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release
+ZIPS_DEST=$INFRA_ROOT/android-x64-release
+gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip
+gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip
+gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip
+# Android x64 release Maven artifacts
+ARCH_PATH=$ARCH_OUT/x86_64_release
+MAVEN_PATH=$MAVEN_ROOT/x86_64_release/$MAVEN_VER/x86_64_release-$MAVEN_VER
+gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom
+gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar
+gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml
+
+# Shorebird AOT Tools (Linker)
+gsutil cp $ENGINE_OUT/host_release/aot_tools/aot-tools.dill $SHOREBIRD_ROOT/aot-tools.dill
+
+# Common Product-mode artifacts
+ARCH_OUT=$ENGINE_OUT/host_release
+ZIPS_OUT=$ARCH_OUT/zip_archives
+ZIPS_DEST=$INFRA_ROOT
+gsutil cp $ZIPS_OUT/flutter_patched_sdk_product.zip $ZIPS_DEST/flutter_patched_sdk_product.zip
+
+# We want to build flutter/tools/font_subset, but that doesn't work with
+# --no-prebuilt-dart-sdk.
+# https://github.com/flutter/flutter/issues/164531
+# Linux x64 host_release font_subset (ConstFinder)
+# ARCH_OUT=$ENGINE_OUT/host_release
+# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH
+# ZIPS_DEST=$INFRA_ROOT/linux-x64-release
+# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip
+
+# Linux Desktop Support
+ARCH_OUT=$ENGINE_OUT/host_release
+ZIPS_OUT=$ARCH_OUT/zip_archives/linux-x64-release
+ZIPS_DEST=$INFRA_ROOT/linux-x64-release
+gsutil cp $ZIPS_OUT/linux-x64-flutter-gtk.zip $ZIPS_DEST/linux-x64-flutter-gtk.zip
+
+ARCH_OUT=$ENGINE_OUT/host_debug
+ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH
+ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH
+gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip
+
+# We could upload patch if we built it here.
+# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip