diff --git a/Cargo.lock b/Cargo.lock
index c6c63bc31..fd19f2eae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1031,6 +1031,7 @@ dependencies = [
  "config",
  "criterion",
  "digest",
+ "either",
  "esl01-renderdag",
  "git2",
  "hex",
diff --git a/Cargo.toml b/Cargo.toml
index 31c7b772e..1579eae11 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,6 +37,7 @@ criterion = "0.5.1"
 crossterm = { version = "0.26", default-features = false }
 digest = "0.10.7"
 dirs = "5.0.1"
+either = "1.9.0"
 esl01-renderdag = "0.3.0"
 glob = "0.3.1"
 git2 = "0.17.2"
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index 3cf929337..764fe37f5 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -26,6 +26,7 @@ bytes.workspace = true
 chrono.workspace = true
 config.workspace = true
 digest.workspace = true
+either.workspace = true
 git2.workspace = true
 hex.workspace = true
 itertools.workspace = true
diff --git a/lib/src/revset.rs b/lib/src/revset.rs
index 8bed26cbe..ed634485f 100644
--- a/lib/src/revset.rs
+++ b/lib/src/revset.rs
@@ -14,7 +14,7 @@
 
 #![allow(missing_docs)]
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::convert::Infallible;
 use std::ops::Range;
 use std::path::Path;
@@ -23,6 +23,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::{error, fmt};
 
+use either::Either;
 use itertools::Itertools;
 use once_cell::sync::Lazy;
 use pest::iterators::{Pair, Pairs};
@@ -231,6 +232,16 @@ impl StringPattern {
             StringPattern::Substring(needle) => haystack.contains(needle),
         }
     }
+
+    /// Returns a literal string if this pattern is of that kind.
+    ///
+    /// This can be used to optimize map lookup by exact key.
+    pub fn as_literal(&self) -> Option<&str> {
+        match self {
+            StringPattern::Literal(literal) => Some(literal),
+            StringPattern::Substring(_) => None,
+        }
+    }
 }
 
 /// Symbol or function to be resolved to `CommitId`s.
@@ -1784,6 +1795,21 @@ pub fn walk_revs<'index>(
         .evaluate(repo)
 }
 
+fn filter_map_values_by_key_pattern<'a: 'b, 'b, V>(
+    map: &'a BTreeMap<String, V>,
+    pattern: &'b StringPattern,
+) -> impl Iterator<Item = &'a V> + 'b {
+    if let Some(key) = pattern.as_literal() {
+        Either::Left(map.get(key).into_iter())
+    } else {
+        Either::Right(
+            map.iter()
+                .filter(|(key, _)| pattern.matches(key))
+                .map(|(_, value)| value),
+        )
+    }
+}
+
 fn resolve_git_ref(repo: &dyn Repo, symbol: &str) -> Option<Vec<CommitId>> {
     let view = repo.view();
     // TODO: We should remove `refs/remotes` from this list once we have a better
@@ -2023,30 +2049,25 @@ fn resolve_commit_ref(
         RevsetCommitRef::Symbol(symbol) => symbol_resolver.resolve_symbol(symbol),
         RevsetCommitRef::VisibleHeads => Ok(repo.view().heads().iter().cloned().collect_vec()),
         RevsetCommitRef::Branches(pattern) => {
-            let mut commit_ids = vec![];
-            for (branch_name, branch_target) in repo.view().branches() {
-                if !pattern.matches(branch_name) {
-                    continue;
-                }
-                commit_ids.extend(branch_target.local_target.added_ids().cloned());
-            }
+            let view = repo.view();
+            let commit_ids = filter_map_values_by_key_pattern(view.branches(), pattern)
+                .flat_map(|branch_target| branch_target.local_target.added_ids())
+                .cloned()
+                .collect();
             Ok(commit_ids)
         }
         RevsetCommitRef::RemoteBranches {
             branch_pattern,
             remote_pattern,
         } => {
-            let mut commit_ids = vec![];
-            for (branch_name, branch_target) in repo.view().branches() {
-                if !branch_pattern.matches(branch_name) {
-                    continue;
-                }
-                for (remote_name, remote_target) in branch_target.remote_targets.iter() {
-                    if remote_pattern.matches(remote_name) {
-                        commit_ids.extend(remote_target.added_ids().cloned());
-                    }
-                }
-            }
+            let view = repo.view();
+            let commit_ids = filter_map_values_by_key_pattern(view.branches(), branch_pattern)
+                .flat_map(|branch_target| {
+                    filter_map_values_by_key_pattern(&branch_target.remote_targets, remote_pattern)
+                })
+                .flat_map(|remote_target| remote_target.added_ids())
+                .cloned()
+                .collect();
             Ok(commit_ids)
         }
         RevsetCommitRef::Tags => {