From 323d267a8cb5c4774661164be4aef2b3f9c3240f Mon Sep 17 00:00:00 2001 From: Dennis Kempin Date: Wed, 22 Jun 2022 21:13:09 +0000 Subject: [PATCH] infra: Add build_windows recipe Refactors the crosvm recipe API a little to provide 3 different environments for builds: Just the source, building with containers and building on the host. For building on the host, we will install rustup-init via CIPD and then use rustup to install the required rust version. BUG=b:233914170 TEST=./recipes.py run build_windows Change-Id: I12ef22f286af584edeb02beed4d231565b698099 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3718900 Reviewed-by: Vikram Auradkar Tested-by: kokoro --- infra/README.recipes.md | 99 +++-- infra/cipd/README.md | 9 + infra/cipd/package_protoc.sh | 27 ++ infra/cipd/package_rustup-init.sh | 27 ++ infra/recipe_modules/crosvm/__init__.py | 2 + infra/recipe_modules/crosvm/api.py | 189 ++++++-- .../basic.json | 6 +- ..._context.py => container_build_context.py} | 4 +- .../basic_linux.json | 406 +++++++++++++++++ .../basic_windows.json | 417 ++++++++++++++++++ .../crosvm/examples/host_build_context.py | 21 + .../{prepare_source.py => source_context.py} | 3 +- .../build_default.json | 85 ---- infra/recipes/build_chromeos.py | 3 - infra/recipes/build_linux.py | 2 +- .../recipes/build_windows.expected/build.json | 68 +++ infra/recipes/build_windows.py | 51 +++ infra/recipes/health_check.py | 2 +- infra/recipes/push_to_github.py | 3 +- infra/recipes/update_chromeos_merges.py | 2 +- 20 files changed, 1251 insertions(+), 175 deletions(-) create mode 100644 infra/cipd/README.md create mode 100755 infra/cipd/package_protoc.sh create mode 100755 infra/cipd/package_rustup-init.sh rename infra/recipe_modules/crosvm/examples/{build_context.expected => container_build_context.expected}/basic.json (99%) rename infra/recipe_modules/crosvm/examples/{build_context.py => container_build_context.py} (72%) create mode 100644 infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_linux.json create mode 100644 infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_windows.json create mode 100644 infra/recipe_modules/crosvm/examples/host_build_context.py rename infra/recipe_modules/crosvm/examples/{prepare_source.py => source_context.py} (95%) create mode 100644 infra/recipes/build_windows.expected/build.json create mode 100644 infra/recipes/build_windows.py diff --git a/infra/README.recipes.md b/infra/README.recipes.md index d7f9a66101..0a555009a2 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md @@ -8,8 +8,10 @@ **[Recipes](#Recipes)** * [build_chromeos](#recipes-build_chromeos) (Python3 ✅) * [build_linux](#recipes-build_linux) (Python3 ✅) - * [crosvm:examples/build_context](#recipes-crosvm_examples_build_context) (Python3 ✅) - * [crosvm:examples/prepare_source](#recipes-crosvm_examples_prepare_source) (Python3 ✅) + * [build_windows](#recipes-build_windows) (Python3 ✅) + * [crosvm:examples/container_build_context](#recipes-crosvm_examples_container_build_context) (Python3 ✅) + * [crosvm:examples/host_build_context](#recipes-crosvm_examples_host_build_context) (Python3 ✅) + * [crosvm:examples/source_context](#recipes-crosvm_examples_source_context) (Python3 ✅) * [health_check](#recipes-health_check) (Python3 ✅) * [push_to_github](#recipes-push_to_github) (Python3 ✅) * [update_chromeos_merges](#recipes-update_chromeos_merges) (Python3 ✅) @@ -17,43 +19,66 @@ ### *recipe_modules* / [crosvm](/infra/recipe_modules/crosvm) -[DEPS](/infra/recipe_modules/crosvm/__init__.py#7): [depot\_tools/bot\_update][depot_tools/recipe_modules/bot_update], [depot\_tools/gclient][depot_tools/recipe_modules/gclient], [depot\_tools/git][depot_tools/recipe_modules/git], [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/step][recipe_engine/recipe_modules/step] +[DEPS](/infra/recipe_modules/crosvm/__init__.py#7): [depot\_tools/bot\_update][depot_tools/recipe_modules/bot_update], [depot\_tools/gclient][depot_tools/recipe_modules/gclient], [depot\_tools/git][depot_tools/recipe_modules/git], [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/cipd][recipe_engine/recipe_modules/cipd], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step] PYTHON_VERSION_COMPATIBILITY: PY3 -#### **class [CrosvmApi](/infra/recipe_modules/crosvm/api.py#10)([RecipeApi][recipe_engine/wkt/RecipeApi]):** +#### **class [CrosvmApi](/infra/recipe_modules/crosvm/api.py#11)([RecipeApi][recipe_engine/wkt/RecipeApi]):** Crosvm specific functionality shared between recipes. -— **def [build\_context](/infra/recipe_modules/crosvm/api.py#28)(self, source=True, container=True):** +  **@property**
— **def [builder\_dir](/infra/recipe_modules/crosvm/api.py#43)(self):** -Prepares everything needed to build crosvm on the revision that needs to be verified. +  **@property**
— **def [cache\_dir](/infra/recipe_modules/crosvm/api.py#39)(self):** -This updates the cwd to the crosvm source directory, ensures the revision to be tested -is checked out and the dev container is ready. +  **@property**
— **def [cargo\_home](/infra/recipe_modules/crosvm/api.py#24)(self):** + +CARGO_HOME is cached between runs. + +  **@property**
— **def [cargo\_target\_dir](/infra/recipe_modules/crosvm/api.py#29)(self):** + +CARGO_TARGET_DIR is cleaned up between runs + +— **def [container\_build\_context](/infra/recipe_modules/crosvm/api.py#56)(self):** + +Prepares source and system to build crosvm via dev container. Usage: - with api.crosvm.build_context(): + with api.crosvm.container_build_context(): api.crosvm.step_in_container("build crosvm", ["cargo build"]) -  **@property**
— **def [builder\_dir](/infra/recipe_modules/crosvm/api.py#17)(self):** +  **@contextlib.contextmanager**
— **def [host\_build\_context](/infra/recipe_modules/crosvm/api.py#68)(self):** -The builder directory is cached between builds to improve build times. +Prepares source and system to build crosvm directly on the host. -— **def [prepare\_container](/infra/recipe_modules/crosvm/api.py#71)(self):** +This will install the required rust version via rustup. However no further dependencies +are installed. -— **def [prepare\_git](/infra/recipe_modules/crosvm/api.py#85)(self):** +Usage: + with api.crosvm.host_build_context(): + api.step("build crosvm", ["cargo build"]) -— **def [prepare\_source](/infra/recipe_modules/crosvm/api.py#46)(self):** +  **@property**
— **def [local\_bin](/infra/recipe_modules/crosvm/api.py#34)(self):** -Prepares the local crosvm source for testing in `self.source_dir` +Directory used to install local tools required by the build. -CI jobs will check out the revision to be tested, try jobs will check out the gerrit -change to be tested. +— **def [prepare\_git](/infra/recipe_modules/crosvm/api.py#111)(self):** -  **@property**
— **def [source\_dir](/infra/recipe_modules/crosvm/api.py#13)(self):** +  **@property**
— **def [rustup\_home](/infra/recipe_modules/crosvm/api.py#19)(self):** -— **def [step\_in\_container](/infra/recipe_modules/crosvm/api.py#99)(self, step_name, command):** +RUSTUP_HOME is cached between runs. + +— **def [source\_context](/infra/recipe_modules/crosvm/api.py#47)(self):** + +Updates the source to the revision to be tested and drops into the source directory. + +Use when no build commands are needed. + +  **@property**
— **def [source\_dir](/infra/recipe_modules/crosvm/api.py#14)(self):** + +Where the crosvm source will be checked out. + +— **def [step\_in\_container](/infra/recipe_modules/crosvm/api.py#97)(self, step_name, command):** Runs a luci step inside the crosvm dev container. ## Recipes @@ -64,13 +89,13 @@ Runs a luci step inside the crosvm dev container. PYTHON_VERSION_COMPATIBILITY: PY3 -— **def [BuildAndTest](/infra/recipes/build_chromeos.py#79)(api, board):** +— **def [BuildAndTest](/infra/recipes/build_chromeos.py#76)(api, board):** -— **def [CleanUp](/infra/recipes/build_chromeos.py#92)(api):** +— **def [CleanUp](/infra/recipes/build_chromeos.py#89)(api):** -— **def [PrepareBuild](/infra/recipes/build_chromeos.py#64)(api):** +— **def [PrepareBuild](/infra/recipes/build_chromeos.py#61)(api):** -— **def [RunSteps](/infra/recipes/build_chromeos.py#97)(api, properties):** +— **def [RunSteps](/infra/recipes/build_chromeos.py#94)(api, properties):** — **def [SetupSource](/infra/recipes/build_chromeos.py#27)(api, workspace):** ### *recipes* / [build\_linux](/infra/recipes/build_linux.py) @@ -84,20 +109,34 @@ PYTHON_VERSION_COMPATIBILITY: PY3 — **def [get\_test\_args](/infra/recipes/build_linux.py#22)(api, test_arch):** Returns architecture specific arguments for ./tools/run_tests -### *recipes* / [crosvm:examples/build\_context](/infra/recipe_modules/crosvm/examples/build_context.py) +### *recipes* / [build\_windows](/infra/recipes/build_windows.py) -[DEPS](/infra/recipe_modules/crosvm/examples/build_context.py#7): [crosvm](#recipe_modules-crosvm) +[DEPS](/infra/recipes/build_windows.py#9): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] PYTHON_VERSION_COMPATIBILITY: PY3 -— **def [RunSteps](/infra/recipe_modules/crosvm/examples/build_context.py#12)(api):** -### *recipes* / [crosvm:examples/prepare\_source](/infra/recipe_modules/crosvm/examples/prepare_source.py) +— **def [RunSteps](/infra/recipes/build_windows.py#18)(api):** +### *recipes* / [crosvm:examples/container\_build\_context](/infra/recipe_modules/crosvm/examples/container_build_context.py) -[DEPS](/infra/recipe_modules/crosvm/examples/prepare_source.py#12): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket] +[DEPS](/infra/recipe_modules/crosvm/examples/container_build_context.py#7): [crosvm](#recipe_modules-crosvm) PYTHON_VERSION_COMPATIBILITY: PY3 -— **def [RunSteps](/infra/recipe_modules/crosvm/examples/prepare_source.py#18)(api):** +— **def [RunSteps](/infra/recipe_modules/crosvm/examples/container_build_context.py#12)(api):** +### *recipes* / [crosvm:examples/host\_build\_context](/infra/recipe_modules/crosvm/examples/host_build_context.py) + +[DEPS](/infra/recipe_modules/crosvm/examples/host_build_context.py#7): [crosvm](#recipe_modules-crosvm), [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +PYTHON_VERSION_COMPATIBILITY: PY3 + +— **def [RunSteps](/infra/recipe_modules/crosvm/examples/host_build_context.py#14)(api):** +### *recipes* / [crosvm:examples/source\_context](/infra/recipe_modules/crosvm/examples/source_context.py) + +[DEPS](/infra/recipe_modules/crosvm/examples/source_context.py#12): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket] + +PYTHON_VERSION_COMPATIBILITY: PY3 + +— **def [RunSteps](/infra/recipe_modules/crosvm/examples/source_context.py#18)(api):** ### *recipes* / [health\_check](/infra/recipes/health_check.py) [DEPS](/infra/recipes/health_check.py#9): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] @@ -125,9 +164,11 @@ PYTHON_VERSION_COMPATIBILITY: PY3 [depot_tools/recipe_modules/gclient]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-gclient [depot_tools/recipe_modules/git]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-git [recipe_engine/recipe_modules/buildbucket]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-buildbucket +[recipe_engine/recipe_modules/cipd]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-cipd [recipe_engine/recipe_modules/context]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-context [recipe_engine/recipe_modules/file]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-file [recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-path +[recipe_engine/recipe_modules/platform]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-platform [recipe_engine/recipe_modules/properties]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-properties [recipe_engine/recipe_modules/raw_io]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-raw_io [recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-step diff --git a/infra/cipd/README.md b/infra/cipd/README.md new file mode 100644 index 0000000000..5eabcde1fe --- /dev/null +++ b/infra/cipd/README.md @@ -0,0 +1,9 @@ +# Crosvm CIPD Packages + +This directory contains definitions and scripts to build CIPD packages used by crosvm luci builders. + +You must be authenticated with luci-auth and have the necessary privileges to upload new packages. + +Package information is available at: + +https://chrome-infra-packages.appspot.com/p/crosvm diff --git a/infra/cipd/package_protoc.sh b/infra/cipd/package_protoc.sh new file mode 100755 index 0000000000..bbb7f31a98 --- /dev/null +++ b/infra/cipd/package_protoc.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +BUILD_DIR=$(mktemp -d) +cd "$BUILD_DIR" || exit 1 + +CIPD_ARGS=( + -pkg-var "description:Protoc compiler" + -install-mode copy + -ref latest +) + +PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.1/protoc-21.1" + +cd $(mktemp -d) +mkdir pkg +wget -q "$PROTOC_URL-win64.zip" -O "protoc.zip" +unzip -p "protoc.zip" "bin/protoc.exe" > "pkg/protoc.exe" +cipd create -in "pkg" -name "crosvm/protoc/windows-amd64" "${CIPD_ARGS[@]}" + +cd $(mktemp -d) +mkdir pkg +wget -q "$PROTOC_URL-linux-x86_64.zip" -O "protoc.zip" +unzip -p "protoc.zip" "bin/protoc" > "pkg/protoc" +cipd create -in "pkg" -name "crosvm/protoc/linux-amd64" "${CIPD_ARGS[@]}" diff --git a/infra/cipd/package_rustup-init.sh b/infra/cipd/package_rustup-init.sh new file mode 100755 index 0000000000..f327026f27 --- /dev/null +++ b/infra/cipd/package_rustup-init.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +set -ex + +BUILD_DIR=$(mktemp -d) +cd "$BUILD_DIR" || exit 1 + +CIPD_ARGS=( + -pkg-var "description:rustup-init to set up rustup.rs" + -install-mode copy + -ref latest +) + +RUSTUP_WIN="https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe" +RUSTUP_LINUX="https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" + +cd $(mktemp -d) +wget "$RUSTUP_WIN" -O "rustup-init.exe" +cipd create -in "." -name "crosvm/rustup-init/windows-amd64" "${CIPD_ARGS[@]}" + +cd $(mktemp -d) +wget "$RUSTUP_LINUX" -O "rustup-init" +chmod +x "rustup-init" +cipd create -in "." -name "crosvm/rustup-init/linux-amd64" "${CIPD_ARGS[@]}" + diff --git a/infra/recipe_modules/crosvm/__init__.py b/infra/recipe_modules/crosvm/__init__.py index 75f32d1997..e3ccc5da4d 100644 --- a/infra/recipe_modules/crosvm/__init__.py +++ b/infra/recipe_modules/crosvm/__init__.py @@ -9,8 +9,10 @@ DEPS = [ "depot_tools/gclient", "depot_tools/git", "recipe_engine/buildbucket", + "recipe_engine/cipd", "recipe_engine/context", "recipe_engine/file", "recipe_engine/path", + "recipe_engine/platform", "recipe_engine/step", ] diff --git a/infra/recipe_modules/crosvm/api.py b/infra/recipe_modules/crosvm/api.py index 712f85ecc1..a03781d0bc 100644 --- a/infra/recipe_modules/crosvm/api.py +++ b/infra/recipe_modules/crosvm/api.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import contextlib from recipe_engine import recipe_api CROSVM_REPO_URL = "https://chromium.googlesource.com/crosvm/crosvm" @@ -12,44 +13,160 @@ class CrosvmApi(recipe_api.RecipeApi): @property def source_dir(self): + "Where the crosvm source will be checked out." return self.builder_dir.join("crosvm") + @property + def rustup_home(self): + "RUSTUP_HOME is cached between runs." + return self.cache_dir.join("rustup") + + @property + def cargo_home(self): + "CARGO_HOME is cached between runs." + return self.cache_dir.join("cargo_home") + + @property + def cargo_target_dir(self): + "CARGO_TARGET_DIR is cleaned up between runs" + return self.m.path["cleanup"].join("cargo_target") + + @property + def local_bin(self): + "Directory used to install local tools required by the build." + return self.cache_dir.join("local_bin") + + @property + def cache_dir(self): + return self.m.path["cache"].join("crosvm_api") + @property def builder_dir(self): - "The builder directory is cached between builds to improve build times." return self.m.path["cache"].join("builder") - def __set_git_config(self, prop, value): - self.m.step( - "Set git config: %s" % prop, - ["git", "config", "--global", prop, value], - ) - - def build_context(self, source=True, container=True): + def source_context(self): """ - Prepares everything needed to build crosvm on the revision that needs to be verified. + Updates the source to the revision to be tested and drops into the source directory. - This updates the cwd to the crosvm source directory, ensures the revision to be tested - is checked out and the dev container is ready. - - Usage: - with api.crosvm.build_context(): - api.crosvm.step_in_container("build crosvm", ["cargo build"]) + Use when no build commands are needed. """ - self.prepare_git() - if source: - self.prepare_source() - if container: - self.prepare_container() + self.__prepare_source() return self.m.context(cwd=self.source_dir) - def prepare_source(self): + def container_build_context(self): + """ + Prepares source and system to build crosvm via dev container. + + Usage: + with api.crosvm.container_build_context(): + api.crosvm.step_in_container("build crosvm", ["cargo build"]) + """ + self.__prepare_source() + self.__prepare_container() + return self.m.context(cwd=self.source_dir) + + @contextlib.contextmanager + def host_build_context(self): + """ + Prepares source and system to build crosvm directly on the host. + + This will install the required rust version via rustup. However no further dependencies + are installed. + + Usage: + with api.crosvm.host_build_context(): + api.step("build crosvm", ["cargo build"]) + """ + self.__prepare_source() + env = { + "RUSTUP_HOME": str(self.rustup_home), + "CARGO_HOME": str(self.cargo_home), + "CARGO_TARGET_DIR": str(self.cargo_target_dir), + } + env_prefixes = { + "PATH": [ + self.cargo_home.join("bin"), + self.local_bin, + ], + } + with self.m.context(env=env, env_prefixes=env_prefixes, cwd=self.source_dir): + self.__prepare_rust() + self.__prepare_host_depdendencies() + yield + + def step_in_container(self, step_name, command): + """ + Runs a luci step inside the crosvm dev container. + """ + self.m.step( + step_name, + [ + "vpython3", + self.source_dir.join("tools/dev_container"), + "--verbose", + ] + + command, + ) + + def prepare_git(self): + with self.m.step.nest("Prepare git"): + with self.m.context(cwd=self.m.path["start_dir"]): + name = self.m.git.config_get("user.name") + email = self.m.git.config_get("user.email") + if not name or not email: + self.__set_git_config("user.name", "Crosvm Bot") + self.__set_git_config( + "user.email", "crosvm-bot@crosvm-infra.iam.gserviceaccount.com" + ) + # Use gcloud for authentication, which will make sure we are interacting with gerrit + # using the Luci configured identity. + if not self.m.platform.is_win: + self.__set_git_config("credential.helper", "gcloud.sh") + + def __prepare_rust(self): + """ + Prepares the rust toolchain via rustup. + + Installs rustup-init via CIPD, which is then used to install the rust toolchain version + required by the crosvm sources. + + Note: You want to run this after prepare_source to ensure the correct version is installed. + """ + with self.m.step.nest("Prepare rust"): + rustup_init = self.m.cipd.ensure_tool("crosvm/rustup-init/${platform}", "latest") + + self.m.step("Install rustup", [rustup_init, "-y", "--default-toolchain", "none"]) + + if self.m.platform.is_win: + self.m.step( + "Set rustup default host", + ["rustup", "set", "default-host", "x86_64-pc-windows-gnu"], + ) + + # Rustup installs a rustc wrapper that will download and use the version specified by + # crosvm in the rust-toolchain file. + self.m.step("Ensure toolchain is installed", ["rustc", "--version"]) + + def __prepare_host_depdendencies(self): + """ + Installs additional dependencies of crosvm host-side builds. This is mainly used for + builds on windows where the dev container is not available. + """ + with self.m.step.nest("Prepare host dependencies"): + self.m.file.ensure_directory("Ensure local_bin exists", self.local_bin) + + ensure_file = self.m.cipd.EnsureFile() + ensure_file.add_package("crosvm/protoc/${platform}", "latest") + self.m.cipd.ensure(self.local_bin, ensure_file) + + def __prepare_source(self): """ Prepares the local crosvm source for testing in `self.source_dir` CI jobs will check out the revision to be tested, try jobs will check out the gerrit change to be tested. """ + self.prepare_git() with self.m.step.nest("Prepare source"): self.m.file.ensure_directory("Ensure builder_dir exists", self.builder_dir) with self.m.context(cwd=self.builder_dir): @@ -68,7 +185,7 @@ class CrosvmApi(recipe_api.RecipeApi): with self.m.context(cwd=self.source_dir): self.m.step("Sync Submodules", ["git", "submodule", "update", "--init"]) - def prepare_container(self): + def __prepare_container(self): with self.m.step.nest("Prepare dev_container"): with self.m.context(cwd=self.source_dir): self.m.step( @@ -82,30 +199,8 @@ class CrosvmApi(recipe_api.RecipeApi): ) self.m.crosvm.step_in_container("Ensure dev container exists", ["true"]) - def prepare_git(self): - with self.m.step.nest("Prepare git"): - with self.m.context(cwd=self.m.path["start_dir"]): - name = self.m.git.config_get("user.name") - email = self.m.git.config_get("user.email") - if not name or not email: - self.__set_git_config("user.name", "Crosvm Bot") - self.__set_git_config( - "user.email", "crosvm-bot@crosvm-infra.iam.gserviceaccount.com" - ) - # Use gcloud for authentication, which will make sure we are interacting with gerrit - # using the Luci configured identity. - self.__set_git_config("credential.helper", "gcloud.sh") - - def step_in_container(self, step_name, command): - """ - Runs a luci step inside the crosvm dev container. - """ + def __set_git_config(self, prop, value): self.m.step( - step_name, - [ - "vpython3", - self.source_dir.join("tools/dev_container"), - "--verbose", - ] - + command, + "Set git config: %s" % prop, + ["git", "config", "--global", prop, value], ) diff --git a/infra/recipe_modules/crosvm/examples/build_context.expected/basic.json b/infra/recipe_modules/crosvm/examples/container_build_context.expected/basic.json similarity index 99% rename from infra/recipe_modules/crosvm/examples/build_context.expected/basic.json rename to infra/recipe_modules/crosvm/examples/container_build_context.expected/basic.json index 9376e4c2f0..0156a63092 100644 --- a/infra/recipe_modules/crosvm/examples/build_context.expected/basic.json +++ b/infra/recipe_modules/crosvm/examples/container_build_context.expected/basic.json @@ -212,11 +212,11 @@ "vpython3", "[CACHE]/builder/crosvm/tools/dev_container", "--verbose", - "echo", - "success" + "cargo", + "build" ], "cwd": "[CACHE]/builder/crosvm", - "name": "Foo" + "name": "Build" }, { "name": "$result" diff --git a/infra/recipe_modules/crosvm/examples/build_context.py b/infra/recipe_modules/crosvm/examples/container_build_context.py similarity index 72% rename from infra/recipe_modules/crosvm/examples/build_context.py rename to infra/recipe_modules/crosvm/examples/container_build_context.py index aa16697b4d..313d2fc739 100644 --- a/infra/recipe_modules/crosvm/examples/build_context.py +++ b/infra/recipe_modules/crosvm/examples/container_build_context.py @@ -10,8 +10,8 @@ DEPS = [ def RunSteps(api): - with api.crosvm.build_context(): - api.crosvm.step_in_container("Foo", ["echo", "success"]) + with api.crosvm.container_build_context(): + api.crosvm.step_in_container("Build", ["cargo", "build"]) def GenTests(api): diff --git a/infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_linux.json b/infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_linux.json new file mode 100644 index 0000000000..68b65087d3 --- /dev/null +++ b/infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_linux.json @@ -0,0 +1,406 @@ +[ + { + "cmd": [], + "name": "Prepare git" + }, + { + "cmd": [ + "git", + "config", + "--get", + "user.name" + ], + "infra_step": true, + "name": "Prepare git.git config user.name", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--get", + "user.email" + ], + "infra_step": true, + "name": "Prepare git.git config user.email", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--global", + "user.name", + "Crosvm Bot" + ], + "name": "Prepare git.Set git config: user.name", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--global", + "user.email", + "crosvm-bot@crosvm-infra.iam.gserviceaccount.com" + ], + "name": "Prepare git.Set git config: user.email", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--global", + "credential.helper", + "gcloud.sh" + ], + "name": "Prepare git.Set git config: credential.helper", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Prepare source" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Prepare source.Ensure builder_dir exists", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'crosvm', 'url': 'https://chromium.googlesource.com/crosvm/crosvm'}]", + "--revision_mapping_file", + "{\"got_revision\": \"crosvm\"}", + "--git-cache-dir", + "[CACHE]/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "crosvm@HEAD", + "--gerrit_no_reset" + ], + "cwd": "[CACHE]/builder", + "env": { + "DEPOT_TOOLS_COLLECT_METRICS": "0", + "GIT_HTTP_LOW_SPEED_LIMIT": "102400", + "GIT_HTTP_LOW_SPEED_TIME": "1800" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "Prepare source.bot_update", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"crosvm\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"crosvm\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/crosvm.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"2e31dd442a44af9e1e51e96aeca18017ddb9922c\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"crosvm\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"2e31dd442a44af9e1e51e96aeca18017ddb9922c\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/main@{#119844}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"crosvm\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"crosvm\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/crosvm.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"2e31dd442a44af9e1e51e96aeca18017ddb9922c\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"2e31dd442a44af9e1e51e96aeca18017ddb9922c\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/main@{#119844}\"@@@" + ] + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init" + ], + "cwd": "[CACHE]/builder/crosvm", + "name": "Prepare source.Sync Submodules", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Prepare rust" + }, + { + "cmd": [], + "name": "Prepare rust.install crosvm/rustup-init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cipd_tool/crosvm/rustup-init/latest" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "infra_step": true, + "name": "Prepare rust.install crosvm/rustup-init.ensure package directory", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[START_DIR]/cipd_tool/crosvm/rustup-init/latest", + "-ensure-file", + "crosvm/rustup-init/${platform} latest", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "infra_step": true, + "name": "Prepare rust.install crosvm/rustup-init.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"crosvm/rustup-init/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/cipd_tool/crosvm/rustup-init/latest/rustup-init", + "-y", + "--default-toolchain", + "none" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "name": "Prepare rust.Install rustup", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "rustc", + "--version" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "name": "Prepare rust.Ensure toolchain is installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Prepare host dependencies" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/crosvm_api/local_bin" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "infra_step": true, + "name": "Prepare host dependencies.Ensure local_bin exists", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/crosvm_api/local_bin", + "-ensure-file", + "crosvm/protoc/${platform} latest", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "name": "Prepare host dependencies.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"crosvm/protoc/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "cargo", + "build" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "name": "Build" + }, + { + "name": "$result" + } +] \ No newline at end of file diff --git a/infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_windows.json b/infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_windows.json new file mode 100644 index 0000000000..5e5e05351c --- /dev/null +++ b/infra/recipe_modules/crosvm/examples/host_build_context.expected/basic_windows.json @@ -0,0 +1,417 @@ +[ + { + "cmd": [], + "name": "Prepare git" + }, + { + "cmd": [ + "git", + "config", + "--get", + "user.name" + ], + "infra_step": true, + "name": "Prepare git.git config user.name", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--get", + "user.email" + ], + "infra_step": true, + "name": "Prepare git.git config user.email", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--global", + "user.name", + "Crosvm Bot" + ], + "name": "Prepare git.Set git config: user.name", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "git", + "config", + "--global", + "user.email", + "crosvm-bot@crosvm-infra.iam.gserviceaccount.com" + ], + "name": "Prepare git.Set git config: user.email", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Prepare source" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]\\builder" + ], + "infra_step": true, + "name": "Prepare source.Ensure builder_dir exists", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]\\resources\\bot_update.py", + "--spec-path", + "cache_dir = '[CACHE]\\\\git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'crosvm', 'url': 'https://chromium.googlesource.com/crosvm/crosvm'}]", + "--revision_mapping_file", + "{\"got_revision\": \"crosvm\"}", + "--git-cache-dir", + "[CACHE]\\git", + "--cleanup-dir", + "[CLEANUP]\\bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "crosvm@HEAD", + "--gerrit_no_reset" + ], + "cwd": "[CACHE]\\builder", + "env": { + "DEPOT_TOOLS_COLLECT_METRICS": "0", + "GIT_HTTP_LOW_SPEED_LIMIT": "102400", + "GIT_HTTP_LOW_SPEED_TIME": "1800" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "Prepare source.bot_update", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"crosvm\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"crosvm\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/crosvm.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"2e31dd442a44af9e1e51e96aeca18017ddb9922c\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"crosvm\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"2e31dd442a44af9e1e51e96aeca18017ddb9922c\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/main@{#119844}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"crosvm\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"crosvm\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/crosvm.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"2e31dd442a44af9e1e51e96aeca18017ddb9922c\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"2e31dd442a44af9e1e51e96aeca18017ddb9922c\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/main@{#119844}\"@@@" + ] + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "name": "Prepare source.Sync Submodules", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Prepare rust" + }, + { + "cmd": [], + "name": "Prepare rust.install crosvm/rustup-init", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]\\cipd_tool\\crosvm\\rustup-init\\latest" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "infra_step": true, + "name": "Prepare rust.install crosvm/rustup-init.ensure package directory", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "cipd.bat", + "ensure", + "-root", + "[START_DIR]\\cipd_tool\\crosvm\\rustup-init\\latest", + "-ensure-file", + "crosvm/rustup-init/${platform} latest", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "infra_step": true, + "name": "Prepare rust.install crosvm/rustup-init.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"crosvm/rustup-init/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[START_DIR]\\cipd_tool\\crosvm\\rustup-init\\latest\\rustup-init", + "-y", + "--default-toolchain", + "none" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "name": "Prepare rust.Install rustup", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "rustup", + "set", + "default-host", + "x86_64-pc-windows-gnu" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "name": "Prepare rust.Set rustup default host", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "rustc", + "--version" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "name": "Prepare rust.Ensure toolchain is installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Prepare host dependencies" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]\\crosvm_api\\local_bin" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "infra_step": true, + "name": "Prepare host dependencies.Ensure local_bin exists", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "cipd.bat", + "ensure", + "-root", + "[CACHE]\\crosvm_api\\local_bin", + "-ensure-file", + "crosvm/protoc/${platform} latest", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "name": "Prepare host dependencies.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"crosvm/protoc/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "cargo", + "build" + ], + "cwd": "[CACHE]\\builder\\crosvm", + "env": { + "CARGO_HOME": "[CACHE]\\crosvm_api\\cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]\\cargo_target", + "RUSTUP_HOME": "[CACHE]\\crosvm_api\\rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]\\crosvm_api\\cargo_home\\bin", + "[CACHE]\\crosvm_api\\local_bin" + ] + }, + "name": "Build" + }, + { + "name": "$result" + } +] \ No newline at end of file diff --git a/infra/recipe_modules/crosvm/examples/host_build_context.py b/infra/recipe_modules/crosvm/examples/host_build_context.py new file mode 100644 index 0000000000..3f11c76a38 --- /dev/null +++ b/infra/recipe_modules/crosvm/examples/host_build_context.py @@ -0,0 +1,21 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +PYTHON_VERSION_COMPATIBILITY = "PY3" + +DEPS = [ + "crosvm", + "recipe_engine/platform", + "recipe_engine/step", +] + + +def RunSteps(api): + with api.crosvm.host_build_context(): + api.step("Build", ["cargo", "build"]) + + +def GenTests(api): + yield api.test("basic_linux") + api.platform("linux", 64) + yield api.test("basic_windows") + api.platform("win", 64) diff --git a/infra/recipe_modules/crosvm/examples/prepare_source.py b/infra/recipe_modules/crosvm/examples/source_context.py similarity index 95% rename from infra/recipe_modules/crosvm/examples/prepare_source.py rename to infra/recipe_modules/crosvm/examples/source_context.py index 297e0481fa..f71a092be8 100644 --- a/infra/recipe_modules/crosvm/examples/prepare_source.py +++ b/infra/recipe_modules/crosvm/examples/source_context.py @@ -16,7 +16,8 @@ DEPS = [ def RunSteps(api): - api.crosvm.prepare_source() + with api.crosvm.source_context(): + pass def GenTests(api): diff --git a/infra/recipes/build_chromeos.expected/build_default.json b/infra/recipes/build_chromeos.expected/build_default.json index 20bd4c540d..736a0d4dd3 100644 --- a/infra/recipes/build_chromeos.expected/build_default.json +++ b/infra/recipes/build_chromeos.expected/build_default.json @@ -14,91 +14,6 @@ "infra_step": true, "name": "Ensure workspace exists" }, - { - "cmd": [], - "name": "Prepare git" - }, - { - "cmd": [ - "git", - "config", - "--get", - "user.name" - ], - "env": { - "DEPOT_TOOLS_UPDATE": "0" - }, - "infra_step": true, - "name": "Prepare git.git config user.name", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { - "cmd": [ - "git", - "config", - "--get", - "user.email" - ], - "env": { - "DEPOT_TOOLS_UPDATE": "0" - }, - "infra_step": true, - "name": "Prepare git.git config user.email", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { - "cmd": [ - "git", - "config", - "--global", - "user.name", - "Crosvm Bot" - ], - "env": { - "DEPOT_TOOLS_UPDATE": "0" - }, - "name": "Prepare git.Set git config: user.name", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { - "cmd": [ - "git", - "config", - "--global", - "user.email", - "crosvm-bot@crosvm-infra.iam.gserviceaccount.com" - ], - "env": { - "DEPOT_TOOLS_UPDATE": "0" - }, - "name": "Prepare git.Set git config: user.email", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, - { - "cmd": [ - "git", - "config", - "--global", - "credential.helper", - "gcloud.sh" - ], - "cwd": "[CLEANUP]/workspace", - "env": { - "DEPOT_TOOLS_UPDATE": "0" - }, - "name": "Prepare git.Set git config: credential.helper", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@" - ] - }, { "cmd": [ "RECIPE_REPO[depot_tools]/repo", diff --git a/infra/recipes/build_chromeos.py b/infra/recipes/build_chromeos.py index 3eb8f63cf6..d8a6dd528a 100644 --- a/infra/recipes/build_chromeos.py +++ b/infra/recipes/build_chromeos.py @@ -30,9 +30,6 @@ def SetupSource(api, workspace): upstream_url = "https://chromium.googlesource.com/crosvm/crosvm" revision = gitilies.id or "HEAD" - # Initialize git user name to make repo happy. - api.crosvm.prepare_git() - # Init and sync the ChromeOS checkout api.step( "Init repo", diff --git a/infra/recipes/build_linux.py b/infra/recipes/build_linux.py index 5121356ade..6fc466ddb9 100644 --- a/infra/recipes/build_linux.py +++ b/infra/recipes/build_linux.py @@ -33,7 +33,7 @@ def get_test_args(api, test_arch): def RunSteps(api, properties): - with api.crosvm.build_context(): + with api.crosvm.container_build_context(): api.crosvm.step_in_container( "Build crosvm tests", [ diff --git a/infra/recipes/build_windows.expected/build.json b/infra/recipes/build_windows.expected/build.json new file mode 100644 index 0000000000..a2ae9efa8f --- /dev/null +++ b/infra/recipes/build_windows.expected/build.json @@ -0,0 +1,68 @@ +[ + { + "cmd": [ + "vpython3", + "./tools/run_tests", + "--verbose", + "--target=host", + "--build-only" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "luci_context": { + "realm": { + "name": "crosvm/crosvm:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Build crosvm tests" + }, + { + "cmd": [ + "vpython3", + "./tools/run_tests", + "--target=host" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CARGO_HOME": "[CACHE]/crosvm_api/cargo_home", + "CARGO_TARGET_DIR": "[CLEANUP]/cargo_target", + "RUSTUP_HOME": "[CACHE]/crosvm_api/rustup" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/crosvm_api/cargo_home/bin", + "[CACHE]/crosvm_api/local_bin" + ] + }, + "luci_context": { + "realm": { + "name": "crosvm/crosvm:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Run crosvm tests" + } +] \ No newline at end of file diff --git a/infra/recipes/build_windows.py b/infra/recipes/build_windows.py new file mode 100644 index 0000000000..f6afa86e9d --- /dev/null +++ b/infra/recipes/build_windows.py @@ -0,0 +1,51 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.post_process import Filter + +PYTHON_VERSION_COMPATIBILITY = "PY3" + +DEPS = [ + "crosvm", + "recipe_engine/buildbucket", + "recipe_engine/context", + "recipe_engine/properties", + "recipe_engine/step", +] + + +def RunSteps(api): + # Note: The recipe does work on linux as well, if the required dependencies have been installed + # on the host via ./tools/install-deps. + # This allows the build to be tested via `./recipe.py run build_windows` + with api.crosvm.host_build_context(): + api.step( + "Build crosvm tests", + [ + "vpython3", + "./tools/run_tests", + "--verbose", + "--target=host", + "--build-only", + ], + ) + api.step( + "Run crosvm tests", + [ + "vpython3", + "./tools/run_tests", + "--target=host", + ], + ) + + +def GenTests(api): + filter_steps = Filter("Build crosvm tests", "Run crosvm tests") + yield ( + api.test( + "build", + api.buildbucket.ci_build(project="crosvm/crosvm"), + ) + + api.post_process(filter_steps) + ) diff --git a/infra/recipes/health_check.py b/infra/recipes/health_check.py index e388cf01f5..ff29080c08 100644 --- a/infra/recipes/health_check.py +++ b/infra/recipes/health_check.py @@ -16,7 +16,7 @@ DEPS = [ def RunSteps(api): - with api.crosvm.build_context(): + with api.crosvm.container_build_context(): api.step( "Self-test dev-container", [ diff --git a/infra/recipes/push_to_github.py b/infra/recipes/push_to_github.py index 73319b9d20..087fe1efc9 100644 --- a/infra/recipes/push_to_github.py +++ b/infra/recipes/push_to_github.py @@ -18,8 +18,7 @@ DEPS = [ def RunSteps(api): - api.crosvm.prepare_source() - with api.context(cwd=api.crosvm.source_dir): + with api.crosvm.source_context(): # Execute push in a bash script so there is no chance of leaking the github token via luci # logs. api.step("Pushing to github", ["bash", api.resource("push_to_github.sh")]) diff --git a/infra/recipes/update_chromeos_merges.py b/infra/recipes/update_chromeos_merges.py index 04a5f767a9..e56c5e934d 100644 --- a/infra/recipes/update_chromeos_merges.py +++ b/infra/recipes/update_chromeos_merges.py @@ -14,7 +14,7 @@ DEPS = [ def RunSteps(api): - with api.crosvm.build_context(container=False): + with api.crosvm.source_context(): api.step( "Update Merges", [