Merge with upstream 2022-06-03

eac74eb8 broker_ipc: add crate
b803f10f base: add unix no op impl of enable_high_res_timers.
59cfdaaf tools: Add tools/cl script as a helper for uploading to gerrit

9cf56a869e..eac74eb836

BUG=232318124
BUG=b:221088786
BUG=b:232318124

Change-Id: I67d45fbc646dad453ee1b1a240fb3cda741afd9d

Cargo.toml

@@ -47,6 +47,7 @@ members = [
     "argh_helpers",
     "base",
     "bit_field",
+    "broker_ipc",
     "cros_async",
     "crosvm-fuzz",
     "crosvm_control",
@@ -154,6 +155,7 @@ assertions = { path = "common/assertions" }
 audio_streams = "*"
 base = "*"
 bit_field = { path = "bit_field" }
+broker_ipc = { path = "broker_ipc" }
 cfg-if = "1.0.0"
 crosvm_plugin = { path = "crosvm_plugin", optional = true }
 data_model = "*"

base/src/lib.rs

@@ -126,6 +126,7 @@ pub use crate::descriptor::{
 };
 
 pub use platform::getpid;
+pub use platform::platform_timer_resolution::enable_high_res_timers;
 pub use platform::{get_filesystem_type, open_file};
 pub use platform::{number_of_logical_cores, pagesize, round_up_to_page_size};
 pub use platform::{FileReadWriteAtVolatile, FileReadWriteVolatile, FileSetLen, FileSync};

base/src/sys/unix/mod.rs

@@ -30,6 +30,7 @@ mod mmap;
 pub mod net;
 mod netlink;
 mod notifiers;
+pub mod platform_timer_resolution;
 mod poll;
 mod priority;
 pub mod rand;

base/src/sys/unix/platform_timer_resolution.rs

@@ -0,0 +1,14 @@
+// Copyright 2022 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use crate::{EnabledHighResTimer, Result};
+
+/// Noop struct on unix.
+/// On windows, restores the platform timer resolution to its original value on Drop.
+pub struct UnixSetTimerResolution {}
+impl EnabledHighResTimer for UnixSetTimerResolution {}
+
+pub fn enable_high_res_timers() -> Result<Box<dyn EnabledHighResTimer>> {
+    Ok(Box::new(UnixSetTimerResolution {}))
+}

broker_ipc/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "broker_ipc"
+authors = ["The Chromium OS Authors"]
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0.32"
+base = { path = "../base" }
+serde = { version = "1", features = [ "derive" ] }
+metrics = { path = "../metrics" }
+

broker_ipc/src/generic.rs

@@ -0,0 +1,32 @@
+// Copyright 2022 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+//! Generic implementation of product specific functions that are called on child process
+//! initialization.
+
+use crate::{log_file_from_path, CommonChildStartupArgs};
+use base::Tube;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Serialize, Deserialize)]
+pub struct ProductAttributes {}
+
+impl CommonChildStartupArgs {
+    pub fn new(syslog_path: Option<PathBuf>, metrics_tube: Option<Tube>) -> anyhow::Result<Self> {
+        Ok(Self {
+            product_attrs: ProductAttributes {},
+            metrics_tube,
+            syslog_file: log_file_from_path(syslog_path)?,
+        })
+    }
+}
+
+pub(crate) fn init_child_crash_reporting(_attrs: &ProductAttributes) {
+    // Do nothing. Crash reporting is implemented by a specific product.
+}
+
+pub(crate) fn product_child_setup(_attrs: &ProductAttributes) -> anyhow::Result<()> {
+    Ok(())
+}

broker_ipc/src/lib.rs

@@ -0,0 +1,78 @@
+// Copyright 2022 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+//! Contains shared code between the broker & its children, specifically any IPC messages or common
+//! bootstrapping code.
+
+use anyhow::Context;
+use base::{enable_high_res_timers, syslog, FromRawDescriptor, IntoRawDescriptor, SafeDescriptor};
+use base::{EnabledHighResTimer, Tube};
+use serde::{Deserialize, Serialize};
+use std::fs::{File, OpenOptions};
+
+mod generic;
+use generic as product;
+
+use product::{init_child_crash_reporting, product_child_setup, ProductAttributes};
+
+use std::path::PathBuf;
+
+/// Arguments that are common to all devices & helper processes.
+#[derive(Serialize, Deserialize)]
+pub struct CommonChildStartupArgs {
+    syslog_file: Option<SafeDescriptor>,
+    metrics_tube: Option<Tube>,
+    product_attrs: ProductAttributes,
+}
+
+pub struct ChildLifecycleCleanup {
+    _timer_resolution: Box<dyn EnabledHighResTimer>,
+}
+
+/// Initializes crash reporting, metrics, logging, and product specific features
+/// for a process.
+///
+/// Returns a value that should be dropped when the process exits.
+pub fn common_child_setup(args: CommonChildStartupArgs) -> anyhow::Result<ChildLifecycleCleanup> {
+    // Crash reporting should start as early as possible, in case other startup tasks fail.
+    init_child_crash_reporting(&args.product_attrs);
+
+    let mut cfg = syslog::LogConfig::default();
+    if let Some(log_file_descriptor) = args.syslog_file {
+        // Safe because we are taking ownership of a SafeDescriptor.
+        let log_file =
+            unsafe { File::from_raw_descriptor(log_file_descriptor.into_raw_descriptor()) };
+        cfg.pipe = Some(Box::new(log_file));
+        cfg.stderr = false;
+    } else {
+        cfg.stderr = true;
+    }
+    syslog::init_with(cfg)?;
+
+    // Initialize anything product specific.
+    product_child_setup(&args.product_attrs)?;
+
+    if let Some(metrics_tube) = args.metrics_tube {
+        metrics::initialize(metrics_tube);
+    }
+
+    let timer_resolution = enable_high_res_timers().context("failed to enable high res timer")?;
+
+    Ok(ChildLifecycleCleanup {
+        _timer_resolution: timer_resolution,
+    })
+}
+
+pub(crate) fn log_file_from_path(path: Option<PathBuf>) -> anyhow::Result<Option<SafeDescriptor>> {
+    Ok(match path {
+        Some(path) => Some(SafeDescriptor::from(
+            OpenOptions::new()
+                .append(true)
+                .create(true)
+                .open(path.as_path())
+                .context(format!("failed to open log file {}", path.display()))?,
+        )),
+        None => None,
+    })
+}

tools/cl

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from impl.common import confirm, run_commands, cmd, CROSVM_ROOT
+import sys
+
+USAGE = """\
+./tools/cl [upload|rebase|status]
+
+Upload changes to the upstream crosvm gerrit.
+
+Multiple projects have their own downstream repository of crosvm and tooling
+to upload to those.
+
+This tool allows developers to send commits to the upstream gerrit review site
+of crosvm and helps rebase changes if needed.
+
+You need to be on a local branch tracking a remote one. `repo start` does this
+for AOSP and chromiumos, or you can do this yourself:
+
+    $ git checkout -b mybranch --track origin/main
+
+Then to upload commits you have made:
+
+    [mybranch] $ ./tools/cl upload
+
+If you are tracking a different branch (e.g. aosp/main or cros/chromeos), the upload may
+fail if your commits do not apply cleanly. This tool can help rebase the changes, it will
+create a new branch tracking origin/main and cherry-picks your commits.
+
+    [mybranch] $ ./tools/cl rebase
+    [mybranch-upstream] ... resolve conflicts
+    [mybranch-upstream] $ git add .
+    [mybranch-upstream] $ git cherry-pick --continue
+    [mybranch-upstream] $ ./tools/cl upload
+
+"""
+
+GERRIT_URL = "https://chromium-review.googlesource.com"
+CROSVM_URL = "https://chromium.googlesource.com/chromiumos/platform/crosvm"
+
+git = cmd("git")
+curl = cmd("curl --silent --fail")
+chmod = cmd("chmod")
+
+
+def get_upstream(branch: str = ""):
+    try:
+        return git(f"rev-parse --abbrev-ref --symbolic-full-name {branch}@{{u}}").stdout()
+    except:
+        return None
+
+
+def list_local_changes(branch: str = ""):
+    upstream = get_upstream(branch)
+    if not upstream:
+        return []
+    for line in git(f"log --oneline --first-parent {upstream}..{branch or 'HEAD'}").lines():
+        yield line.split(" ", 1)
+
+
+def list_local_branches():
+    return git("for-each-ref --format=%(refname:short) refs/heads").lines()
+
+
+def get_active_upstream():
+    upstream = get_upstream()
+    if not upstream:
+        raise Exception("You are not tracking an upstream branch.")
+    parts = upstream.split("/")
+    if len(parts) != 2:
+        raise Exception(f"Your upstream branch '{upstream}' is not remote.")
+    return (parts[0], parts[1])
+
+
+def prerequisites():
+    print("This tool is experimental and a work in progress, please use carefully.")
+    print()
+
+    if not git("remote get-url origin").success():
+        print("Setting up origin")
+        git("remote add origin", CROSVM_URL).fg()
+    if git("remote get-url origin").stdout() != CROSVM_URL:
+        print("Your remote 'origin' does not point to the main crosvm repository.")
+        if confirm(f"Do you want to fix it?"):
+            git("remote set-url origin", CROSVM_URL).fg()
+        else:
+            sys.exit(1)
+
+    # Install gerrit commit hook
+    hook_path = CROSVM_ROOT / ".git/hooks/commit-msg"
+    if not hook_path.exists():
+        hook_path.parent.mkdir(exist_ok=True)
+        curl(f"{GERRIT_URL}/tools/hooks/commit-msg").write_to(hook_path)
+        chmod("+x", hook_path).fg()
+
+
+def status():
+    """
+    Lists all branches and their local commits.
+    """
+    for branch in list_local_branches():
+        print("Branch", branch, "tracking", get_upstream(branch))
+        changes = [*list_local_changes(branch)]
+        for sha, title in changes:
+            print(" ", title)
+        if not changes:
+            print("  No changes")
+        print()
+
+
+def rebase():
+    """
+    Rebases changes from the current branch onto origin/main.
+
+    Will create a new branch called 'current-branch'-upstream tracking origin/main. Changes from
+    the current branch will then be rebased into the -upstream branch.
+    """
+    prerequisites()
+    branch_name = git("branch --show-current").stdout()
+    upstream_branch_name = branch_name + "-upstream"
+
+    print(f"Checking out '{upstream_branch_name}'")
+    rev = git("rev-parse", upstream_branch_name).stdout(check=False)
+    if rev:
+        print(f"Leaving behind previous revision of {upstream_branch_name}: {rev}")
+    git("checkout -B", upstream_branch_name, "origin/main").fg(quiet=True)
+
+    print(f"Cherry-picking changes from {branch_name}")
+    git(f"cherry-pick {branch_name}@{{u}}..{branch_name}").fg()
+
+
+def upload():
+    """
+    Uploads changes to the crosvm main branch.
+    """
+    prerequisites()
+
+    remote, branch = get_active_upstream()
+    changes = [*list_local_changes()]
+    if not changes:
+        print("No changes to upload")
+        return
+
+    print("Uploading to origin/main:")
+    for sha, title in changes:
+        print(" ", sha, title)
+    print()
+
+    if (remote, branch) != ("origin", "main"):
+        print(f"WARNING! Your changes are based on {remote}/{branch}, not origin/main.")
+        print("If gerrit rejects your changes, try `./tools/cl rebase -h`.")
+        print()
+        if not confirm("Upload anyway?"):
+            return
+        print()
+
+    git("push", remote, f"HEAD:refs/for/{branch}").fg()
+
+
+if __name__ == "__main__":
+    run_commands(upload, rebase, status, usage=USAGE)

tools/impl/common.py

@@ -191,6 +191,9 @@ class Command(object):
                 raise subprocess.CalledProcessError(result.returncode, str(self), result.stdout)
         return result.returncode
 
+    def success(self):
+        return self.fg(check=False, quiet=True) == 0
+
     def stdout(self, check: bool = True):
         """
         Runs a program and returns stdout. Stderr is still directed to the user.
@@ -467,7 +470,11 @@ def run_main(main_fn: Callable[..., Any]):
     run_commands(default_fn=main_fn)
 
 
-def run_commands(*functions: Callable[..., Any], default_fn: Optional[Callable[..., Any]] = None):
+def run_commands(
+    *functions: Callable[..., Any],
+    default_fn: Optional[Callable[..., Any]] = None,
+    usage: Optional[str] = None,
+):
     """
     Allow the user to call the provided functions with command line arguments translated to
     function arguments via argh: https://pythonhosted.org/argh
@@ -480,7 +487,7 @@ def run_commands(*functions: Callable[..., Any], default_fn: Optional[Callable[.
         sys.exit(1)
     try:
         # Add global verbose arguments
-        parser = argparse.ArgumentParser()
+        parser = argparse.ArgumentParser(usage=usage)
         add_verbose_args(parser)
 
         # Add provided commands to parser. Do not use sub-commands if we just got one function.
@@ -547,6 +554,16 @@ def find_scripts(path: Path, shebang: str):
             yield file
 
 
+def confirm(message: str, default=False):
+    print(message, "[y/N]" if default == False else "[Y/n]")
+    response = sys.stdin.readline().strip()
+    if response in ("y", "Y"):
+        return True
+    if response in ("n", "N"):
+        return False
+    return default
+
+
 if __name__ == "__main__":
     import doctest