diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml index 04b48c93ff..c1f7e79d58 100644 --- a/crates/language/Cargo.toml +++ b/crates/language/Cargo.toml @@ -72,6 +72,8 @@ ctor.workspace = true env_logger.workspace = true indoc.workspace = true rand.workspace = true +unindent.workspace = true + tree-sitter-embedded-template = "*" tree-sitter-html = "*" tree-sitter-javascript = "*" @@ -81,4 +83,3 @@ tree-sitter-rust = "*" tree-sitter-python = "*" tree-sitter-typescript = "*" tree-sitter-ruby = "*" -unindent.workspace = true diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index ebfa38c148..03c3eff1a9 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -1,3 +1,6 @@ +#[cfg(test)] +mod syntax_map_tests; + use crate::{Grammar, InjectionConfig, Language, LanguageRegistry}; use collections::HashMap; use futures::FutureExt; @@ -1220,7 +1223,7 @@ fn get_injections( } } -fn splice_included_ranges( +pub(crate) fn splice_included_ranges( mut ranges: Vec, changed_ranges: &[Range], new_ranges: &[tree_sitter::Range], @@ -1616,1175 +1619,3 @@ impl ToTreeSitterPoint for Point { Point::new(point.row as u32, point.column as u32) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::LanguageConfig; - use rand::rngs::StdRng; - use std::env; - use text::Buffer; - use unindent::Unindent as _; - use util::test::marked_text_ranges; - - #[test] - fn test_splice_included_ranges() { - let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)]; - - let new_ranges = splice_included_ranges( - ranges.clone(), - &[54..56, 58..68], - &[ts_range(50..54), ts_range(59..67)], - ); - assert_eq!( - new_ranges, - &[ - ts_range(20..30), - ts_range(50..54), - ts_range(59..67), - ts_range(80..90), - ] - ); - - let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]); - assert_eq!( - new_ranges, - &[ts_range(20..30), ts_range(50..60), ts_range(80..90)] - ); - - let new_ranges = - splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]); - assert_eq!( - new_ranges, - &[ - ts_range(0..2), - ts_range(20..30), - ts_range(50..60), - ts_range(70..75), - ts_range(80..90) - ] - ); - - let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]); - assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]); - - fn ts_range(range: Range) -> tree_sitter::Range { - tree_sitter::Range { - start_byte: range.start, - start_point: tree_sitter::Point { - row: 0, - column: range.start, - }, - end_byte: range.end, - end_point: tree_sitter::Point { - row: 0, - column: range.end, - }, - } - } - } - - #[gpui::test] - fn test_syntax_map_layers_for_range() { - let registry = Arc::new(LanguageRegistry::test()); - let language = Arc::new(rust_lang()); - registry.add(language.clone()); - - let mut buffer = Buffer::new( - 0, - 0, - r#" - fn a() { - assert_eq!( - b(vec![C {}]), - vec![d.e], - ); - println!("{}", f(|_| true)); - } - "# - .unindent(), - ); - - let mut syntax_map = SyntaxMap::new(); - syntax_map.set_language_registry(registry.clone()); - syntax_map.reparse(language.clone(), &buffer); - - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(2, 0)..Point::new(2, 0), - &[ - "...(function_item ... (block (expression_statement (macro_invocation...", - "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", - ], - ); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(2, 14)..Point::new(2, 16), - &[ - "...(function_item ...", - "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", - "...(array_expression (struct_expression ...", - ], - ); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(3, 14)..Point::new(3, 16), - &[ - "...(function_item ...", - "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", - "...(array_expression (field_expression ...", - ], - ); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(5, 12)..Point::new(5, 16), - &[ - "...(function_item ...", - "...(call_expression ... (arguments (closure_expression ...", - ], - ); - - // Replace a vec! macro invocation with a plain slice, removing a syntactic layer. - let macro_name_range = range_for_text(&buffer, "vec!"); - buffer.edit([(macro_name_range, "&")]); - syntax_map.interpolate(&buffer); - syntax_map.reparse(language.clone(), &buffer); - - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(2, 14)..Point::new(2, 16), - &[ - "...(function_item ...", - "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...", - ], - ); - - // Put the vec! macro back, adding back the syntactic layer. - buffer.undo(); - syntax_map.interpolate(&buffer); - syntax_map.reparse(language.clone(), &buffer); - - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(2, 14)..Point::new(2, 16), - &[ - "...(function_item ...", - "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", - "...(array_expression (struct_expression ...", - ], - ); - } - - #[gpui::test] - fn test_dynamic_language_injection() { - let registry = Arc::new(LanguageRegistry::test()); - let markdown = Arc::new(markdown_lang()); - registry.add(markdown.clone()); - registry.add(Arc::new(rust_lang())); - registry.add(Arc::new(ruby_lang())); - - let mut buffer = Buffer::new( - 0, - 0, - r#" - This is a code block: - - ```rs - fn foo() {} - ``` - "# - .unindent(), - ); - - let mut syntax_map = SyntaxMap::new(); - syntax_map.set_language_registry(registry.clone()); - syntax_map.reparse(markdown.clone(), &buffer); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(3, 0)..Point::new(3, 0), - &[ - "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...", - "...(function_item name: (identifier) parameters: (parameters) body: (block)...", - ], - ); - - // Replace Rust with Ruby in code block. - let macro_name_range = range_for_text(&buffer, "rs"); - buffer.edit([(macro_name_range, "ruby")]); - syntax_map.interpolate(&buffer); - syntax_map.reparse(markdown.clone(), &buffer); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(3, 0)..Point::new(3, 0), - &[ - "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...", - "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...", - ], - ); - - // Replace Ruby with a language that hasn't been loaded yet. - let macro_name_range = range_for_text(&buffer, "ruby"); - buffer.edit([(macro_name_range, "html")]); - syntax_map.interpolate(&buffer); - syntax_map.reparse(markdown.clone(), &buffer); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(3, 0)..Point::new(3, 0), - &[ - "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..." - ], - ); - assert!(syntax_map.contains_unknown_injections()); - - registry.add(Arc::new(html_lang())); - syntax_map.reparse(markdown.clone(), &buffer); - assert_layers_for_range( - &syntax_map, - &buffer, - Point::new(3, 0)..Point::new(3, 0), - &[ - "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...", - "(fragment (text))", - ], - ); - assert!(!syntax_map.contains_unknown_injections()); - } - - #[gpui::test] - fn test_typing_multiple_new_injections() { - let (buffer, syntax_map) = test_edit_sequence( - "Rust", - &[ - "fn a() { dbg }", - "fn a() { dbg«!» }", - "fn a() { dbg!«()» }", - "fn a() { dbg!(«b») }", - "fn a() { dbg!(b«.») }", - "fn a() { dbg!(b.«c») }", - "fn a() { dbg!(b.c«()») }", - "fn a() { dbg!(b.c(«vec»)) }", - "fn a() { dbg!(b.c(vec«!»)) }", - "fn a() { dbg!(b.c(vec!«[]»)) }", - "fn a() { dbg!(b.c(vec![«d»])) }", - "fn a() { dbg!(b.c(vec![d«.»])) }", - "fn a() { dbg!(b.c(vec![d.«e»])) }", - ], - ); - - assert_capture_ranges( - &syntax_map, - &buffer, - &["field"], - "fn a() { dbg!(b.«c»(vec![d.«e»])) }", - ); - } - - #[gpui::test] - fn test_pasting_new_injection_line_between_others() { - let (buffer, syntax_map) = test_edit_sequence( - "Rust", - &[ - " - fn a() { - b!(B {}); - c!(C {}); - d!(D {}); - e!(E {}); - f!(F {}); - g!(G {}); - } - ", - " - fn a() { - b!(B {}); - c!(C {}); - d!(D {}); - « h!(H {}); - » e!(E {}); - f!(F {}); - g!(G {}); - } - ", - ], - ); - - assert_capture_ranges( - &syntax_map, - &buffer, - &["struct"], - " - fn a() { - b!(«B {}»); - c!(«C {}»); - d!(«D {}»); - h!(«H {}»); - e!(«E {}»); - f!(«F {}»); - g!(«G {}»); - } - ", - ); - } - - #[gpui::test] - fn test_joining_injections_with_child_injections() { - let (buffer, syntax_map) = test_edit_sequence( - "Rust", - &[ - " - fn a() { - b!( - c![one.two.three], - d![four.five.six], - ); - e!( - f![seven.eight], - ); - } - ", - " - fn a() { - b!( - c![one.two.three], - d![four.five.six], - ˇ f![seven.eight], - ); - } - ", - ], - ); - - assert_capture_ranges( - &syntax_map, - &buffer, - &["field"], - " - fn a() { - b!( - c![one.«two».«three»], - d![four.«five».«six»], - f![seven.«eight»], - ); - } - ", - ); - } - - #[gpui::test] - fn test_editing_edges_of_injection() { - test_edit_sequence( - "Rust", - &[ - " - fn a() { - b!(c!()) - } - ", - " - fn a() { - «d»!(c!()) - } - ", - " - fn a() { - «e»d!(c!()) - } - ", - " - fn a() { - ed!«[»c!()«]» - } - ", - ], - ); - } - - #[gpui::test] - fn test_edits_preceding_and_intersecting_injection() { - test_edit_sequence( - "Rust", - &[ - // - "const aaaaaaaaaaaa: B = c!(d(e.f));", - "const aˇa: B = c!(d(eˇ));", - ], - ); - } - - #[gpui::test] - fn test_non_local_changes_create_injections() { - test_edit_sequence( - "Rust", - &[ - " - // a! { - static B: C = d; - // } - ", - " - ˇa! { - static B: C = d; - ˇ} - ", - ], - ); - } - - #[gpui::test] - fn test_creating_many_injections_in_one_edit() { - test_edit_sequence( - "Rust", - &[ - " - fn a() { - one(Two::three(3)); - four(Five::six(6)); - seven(Eight::nine(9)); - } - ", - " - fn a() { - one«!»(Two::three(3)); - four«!»(Five::six(6)); - seven«!»(Eight::nine(9)); - } - ", - " - fn a() { - one!(Two::three«!»(3)); - four!(Five::six«!»(6)); - seven!(Eight::nine«!»(9)); - } - ", - ], - ); - } - - #[gpui::test] - fn test_editing_across_injection_boundary() { - test_edit_sequence( - "Rust", - &[ - " - fn one() { - two(); - three!( - three.four, - five.six, - ); - } - ", - " - fn one() { - two(); - th«irty_five![» - three.four, - five.six, - « seven.eight, - ];» - } - ", - ], - ); - } - - #[gpui::test] - fn test_removing_injection_by_replacing_across_boundary() { - test_edit_sequence( - "Rust", - &[ - " - fn one() { - two!( - three.four, - ); - } - ", - " - fn one() { - t«en - .eleven( - twelve, - » - three.four, - ); - } - ", - ], - ); - } - - #[gpui::test] - fn test_combined_injections() { - let (buffer, syntax_map) = test_edit_sequence( - "ERB", - &[ - " - - <% if @one %> -
- <% else %> -
- <% end %> -
- - ", - " - - <% if @one %> -
- ˇ else ˇ -
- <% end %> -
- - ", - " - - <% if @one «;» end %> -
- - ", - ], - ); - - assert_capture_ranges( - &syntax_map, - &buffer, - &["tag", "ivar"], - " - <«body»> - <% if «@one» ; end %> - - - ", - ); - } - - #[gpui::test] - fn test_combined_injections_empty_ranges() { - test_edit_sequence( - "ERB", - &[ - " - <% if @one %> - <% else %> - <% end %> - ", - " - <% if @one %> - ˇ<% end %> - ", - ], - ); - } - - #[gpui::test] - fn test_combined_injections_edit_edges_of_ranges() { - let (buffer, syntax_map) = test_edit_sequence( - "ERB", - &[ - " - <%= one @two %> - <%= three @four %> - ", - " - <%= one @two %ˇ - <%= three @four %> - ", - " - <%= one @two %«>» - <%= three @four %> - ", - ], - ); - - assert_capture_ranges( - &syntax_map, - &buffer, - &["tag", "ivar"], - " - <%= one «@two» %> - <%= three «@four» %> - ", - ); - } - - #[gpui::test] - fn test_combined_injections_splitting_some_injections() { - let (_buffer, _syntax_map) = test_edit_sequence( - "ERB", - &[ - r#" - <%A if b(:c) %> - d - <% end %> - eee - <% f %> - "#, - r#" - <%« AAAAAAA %> - hhhhhhh - <%=» if b(:c) %> - d - <% end %> - eee - <% f %> - "#, - ], - ); - } - - #[gpui::test] - fn test_combined_injections_inside_injections() { - let (_buffer, _syntax_map) = test_edit_sequence( - "Markdown", - &[ - r#" - here is some ERB code: - - ```erb -
    - <% people.each do |person| %> -
  • <%= person.name %>
    • - <% end %> -
- ``` - "#, - r#" - here is some ERB code: - - ```erb -
    - <% people«2».each do |person| %> -
  • <%= person.name %>
    • - <% end %> -
- ``` - "#, - ], - ); - } - - #[gpui::test(iterations = 50)] - fn test_random_syntax_map_edits(mut rng: StdRng) { - let operations = env::var("OPERATIONS") - .map(|i| i.parse().expect("invalid `OPERATIONS` variable")) - .unwrap_or(10); - - let text = r#" - fn test_something() { - let vec = vec![5, 1, 3, 8]; - assert_eq!( - vec - .into_iter() - .map(|i| i * 2) - .collect::>(), - vec![ - 5 * 2, 1 * 2, 3 * 2, 8 * 2 - ], - ); - } - "# - .unindent() - .repeat(2); - - let registry = Arc::new(LanguageRegistry::test()); - let language = Arc::new(rust_lang()); - registry.add(language.clone()); - let mut buffer = Buffer::new(0, 0, text); - - let mut syntax_map = SyntaxMap::new(); - syntax_map.set_language_registry(registry.clone()); - syntax_map.reparse(language.clone(), &buffer); - - let mut reference_syntax_map = SyntaxMap::new(); - reference_syntax_map.set_language_registry(registry.clone()); - - log::info!("initial text:\n{}", buffer.text()); - - for _ in 0..operations { - let prev_buffer = buffer.snapshot(); - let prev_syntax_map = syntax_map.snapshot(); - - buffer.randomly_edit(&mut rng, 3); - log::info!("text:\n{}", buffer.text()); - - syntax_map.interpolate(&buffer); - check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer); - - syntax_map.reparse(language.clone(), &buffer); - - reference_syntax_map.clear(); - reference_syntax_map.reparse(language.clone(), &buffer); - } - - for i in 0..operations { - let i = operations - i - 1; - buffer.undo(); - log::info!("undoing operation {}", i); - log::info!("text:\n{}", buffer.text()); - - syntax_map.interpolate(&buffer); - syntax_map.reparse(language.clone(), &buffer); - - reference_syntax_map.clear(); - reference_syntax_map.reparse(language.clone(), &buffer); - assert_eq!( - syntax_map.layers(&buffer).len(), - reference_syntax_map.layers(&buffer).len(), - "wrong number of layers after undoing edit {i}" - ); - } - - let layers = syntax_map.layers(&buffer); - let reference_layers = reference_syntax_map.layers(&buffer); - for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter()) - { - assert_eq!( - edited_layer.node().to_sexp(), - reference_layer.node().to_sexp() - ); - assert_eq!(edited_layer.node().range(), reference_layer.node().range()); - } - } - - #[gpui::test(iterations = 50)] - fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) { - let operations = env::var("OPERATIONS") - .map(|i| i.parse().expect("invalid `OPERATIONS` variable")) - .unwrap_or(10); - - let text = r#" -
- <% if one?(:two) %> -

- <%= yield :five %> -

- <% elsif Six.seven(8) %> -

- <%= yield :five %> -

- <% else %> - Ok - <% end %> -
- "# - .unindent() - .repeat(8); - - let registry = Arc::new(LanguageRegistry::test()); - let language = Arc::new(erb_lang()); - registry.add(language.clone()); - registry.add(Arc::new(ruby_lang())); - registry.add(Arc::new(html_lang())); - let mut buffer = Buffer::new(0, 0, text); - - let mut syntax_map = SyntaxMap::new(); - syntax_map.set_language_registry(registry.clone()); - syntax_map.reparse(language.clone(), &buffer); - - let mut reference_syntax_map = SyntaxMap::new(); - reference_syntax_map.set_language_registry(registry.clone()); - - log::info!("initial text:\n{}", buffer.text()); - - for _ in 0..operations { - let prev_buffer = buffer.snapshot(); - let prev_syntax_map = syntax_map.snapshot(); - - buffer.randomly_edit(&mut rng, 3); - log::info!("text:\n{}", buffer.text()); - - syntax_map.interpolate(&buffer); - check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer); - - syntax_map.reparse(language.clone(), &buffer); - - reference_syntax_map.clear(); - reference_syntax_map.reparse(language.clone(), &buffer); - } - - for i in 0..operations { - let i = operations - i - 1; - buffer.undo(); - log::info!("undoing operation {}", i); - log::info!("text:\n{}", buffer.text()); - - syntax_map.interpolate(&buffer); - syntax_map.reparse(language.clone(), &buffer); - - reference_syntax_map.clear(); - reference_syntax_map.reparse(language.clone(), &buffer); - assert_eq!( - syntax_map.layers(&buffer).len(), - reference_syntax_map.layers(&buffer).len(), - "wrong number of layers after undoing edit {i}" - ); - } - - let layers = syntax_map.layers(&buffer); - let reference_layers = reference_syntax_map.layers(&buffer); - for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter()) - { - assert_eq!( - edited_layer.node().to_sexp(), - reference_layer.node().to_sexp() - ); - assert_eq!(edited_layer.node().range(), reference_layer.node().range()); - } - } - - fn check_interpolation( - old_syntax_map: &SyntaxSnapshot, - new_syntax_map: &SyntaxSnapshot, - old_buffer: &BufferSnapshot, - new_buffer: &BufferSnapshot, - ) { - let edits = new_buffer - .edits_since::(&old_buffer.version()) - .collect::>(); - - for (old_layer, new_layer) in old_syntax_map - .layers - .iter() - .zip(new_syntax_map.layers.iter()) - { - assert_eq!(old_layer.range, new_layer.range); - let Some(old_tree) = old_layer.content.tree() else { continue }; - let Some(new_tree) = new_layer.content.tree() else { continue }; - let old_start_byte = old_layer.range.start.to_offset(old_buffer); - let new_start_byte = new_layer.range.start.to_offset(new_buffer); - let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point(); - let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point(); - let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point); - let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point); - check_node_edits( - old_layer.depth, - &old_layer.range, - old_node, - new_node, - old_buffer, - new_buffer, - &edits, - ); - } - - fn check_node_edits( - depth: usize, - range: &Range, - old_node: Node, - new_node: Node, - old_buffer: &BufferSnapshot, - new_buffer: &BufferSnapshot, - edits: &[text::Edit], - ) { - assert_eq!(old_node.kind(), new_node.kind()); - - let old_range = old_node.byte_range(); - let new_range = new_node.byte_range(); - - let is_edited = edits - .iter() - .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start); - if is_edited { - assert!( - new_node.has_changes(), - concat!( - "failed to mark node as edited.\n", - "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n", - "node kind: {}, old node range: {:?}, new node range: {:?}", - ), - depth, - range.to_offset(old_buffer), - range.to_offset(new_buffer), - new_node.kind(), - old_range, - new_range, - ); - } - - if !new_node.has_changes() { - assert_eq!( - old_buffer - .text_for_range(old_range.clone()) - .collect::(), - new_buffer - .text_for_range(new_range.clone()) - .collect::(), - concat!( - "mismatched text for node\n", - "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n", - "node kind: {}, old node range:{:?}, new node range:{:?}", - ), - depth, - range.to_offset(old_buffer), - range.to_offset(new_buffer), - new_node.kind(), - old_range, - new_range, - ); - } - - for i in 0..new_node.child_count() { - check_node_edits( - depth, - range, - old_node.child(i).unwrap(), - new_node.child(i).unwrap(), - old_buffer, - new_buffer, - edits, - ) - } - } - } - - fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) { - let registry = Arc::new(LanguageRegistry::test()); - registry.add(Arc::new(rust_lang())); - registry.add(Arc::new(ruby_lang())); - registry.add(Arc::new(html_lang())); - registry.add(Arc::new(erb_lang())); - registry.add(Arc::new(markdown_lang())); - let language = registry - .language_for_name(language_name) - .now_or_never() - .unwrap() - .unwrap(); - let mut buffer = Buffer::new(0, 0, Default::default()); - - let mut mutated_syntax_map = SyntaxMap::new(); - mutated_syntax_map.set_language_registry(registry.clone()); - mutated_syntax_map.reparse(language.clone(), &buffer); - - for (i, marked_string) in steps.into_iter().enumerate() { - buffer.edit_via_marked_text(&marked_string.unindent()); - - // Reparse the syntax map - mutated_syntax_map.interpolate(&buffer); - mutated_syntax_map.reparse(language.clone(), &buffer); - - // Create a second syntax map from scratch - let mut reference_syntax_map = SyntaxMap::new(); - reference_syntax_map.set_language_registry(registry.clone()); - reference_syntax_map.reparse(language.clone(), &buffer); - - // Compare the mutated syntax map to the new syntax map - let mutated_layers = mutated_syntax_map.layers(&buffer); - let reference_layers = reference_syntax_map.layers(&buffer); - assert_eq!( - mutated_layers.len(), - reference_layers.len(), - "wrong number of layers at step {i}" - ); - for (edited_layer, reference_layer) in - mutated_layers.into_iter().zip(reference_layers.into_iter()) - { - assert_eq!( - edited_layer.node().to_sexp(), - reference_layer.node().to_sexp(), - "different layer at step {i}" - ); - assert_eq!( - edited_layer.node().range(), - reference_layer.node().range(), - "different layer at step {i}" - ); - } - } - - (buffer, mutated_syntax_map) - } - - fn html_lang() -> Language { - Language::new( - LanguageConfig { - name: "HTML".into(), - path_suffixes: vec!["html".to_string()], - ..Default::default() - }, - Some(tree_sitter_html::language()), - ) - .with_highlights_query( - r#" - (tag_name) @tag - (erroneous_end_tag_name) @tag - (attribute_name) @property - "#, - ) - .unwrap() - } - - fn ruby_lang() -> Language { - Language::new( - LanguageConfig { - name: "Ruby".into(), - path_suffixes: vec!["rb".to_string()], - ..Default::default() - }, - Some(tree_sitter_ruby::language()), - ) - .with_highlights_query( - r#" - ["if" "do" "else" "end"] @keyword - (instance_variable) @ivar - "#, - ) - .unwrap() - } - - fn erb_lang() -> Language { - Language::new( - LanguageConfig { - name: "ERB".into(), - path_suffixes: vec!["erb".to_string()], - ..Default::default() - }, - Some(tree_sitter_embedded_template::language()), - ) - .with_highlights_query( - r#" - ["<%" "%>"] @keyword - "#, - ) - .unwrap() - .with_injection_query( - r#" - ((code) @content - (#set! "language" "ruby") - (#set! "combined")) - - ((content) @content - (#set! "language" "html") - (#set! "combined")) - "#, - ) - .unwrap() - } - - fn rust_lang() -> Language { - Language::new( - LanguageConfig { - name: "Rust".into(), - path_suffixes: vec!["rs".to_string()], - ..Default::default() - }, - Some(tree_sitter_rust::language()), - ) - .with_highlights_query( - r#" - (field_identifier) @field - (struct_expression) @struct - "#, - ) - .unwrap() - .with_injection_query( - r#" - (macro_invocation - (token_tree) @content - (#set! "language" "rust")) - "#, - ) - .unwrap() - } - - fn markdown_lang() -> Language { - Language::new( - LanguageConfig { - name: "Markdown".into(), - path_suffixes: vec!["md".into()], - ..Default::default() - }, - Some(tree_sitter_markdown::language()), - ) - .with_injection_query( - r#" - (fenced_code_block - (info_string - (language) @language) - (code_fence_content) @content) - "#, - ) - .unwrap() - } - - fn range_for_text(buffer: &Buffer, text: &str) -> Range { - let start = buffer.as_rope().to_string().find(text).unwrap(); - start..start + text.len() - } - - fn assert_layers_for_range( - syntax_map: &SyntaxMap, - buffer: &BufferSnapshot, - range: Range, - expected_layers: &[&str], - ) { - let layers = syntax_map - .layers_for_range(range, &buffer) - .collect::>(); - assert_eq!( - layers.len(), - expected_layers.len(), - "wrong number of layers" - ); - for (i, (layer, expected_s_exp)) in layers.iter().zip(expected_layers.iter()).enumerate() { - let actual_s_exp = layer.node().to_sexp(); - assert!( - string_contains_sequence( - &actual_s_exp, - &expected_s_exp.split("...").collect::>() - ), - "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}", - ); - } - } - - fn assert_capture_ranges( - syntax_map: &SyntaxMap, - buffer: &BufferSnapshot, - highlight_query_capture_names: &[&str], - marked_string: &str, - ) { - let mut actual_ranges = Vec::>::new(); - let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| { - grammar.highlights_query.as_ref() - }); - let queries = captures - .grammars() - .iter() - .map(|grammar| grammar.highlights_query.as_ref().unwrap()) - .collect::>(); - for capture in captures { - let name = &queries[capture.grammar_index].capture_names()[capture.index as usize]; - if highlight_query_capture_names.contains(&name.as_str()) { - actual_ranges.push(capture.node.byte_range()); - } - } - - let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false); - assert_eq!(text, buffer.text()); - assert_eq!(actual_ranges, expected_ranges); - } - - pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool { - let mut last_part_end = 0; - for part in parts { - if let Some(start_ix) = text[last_part_end..].find(part) { - last_part_end = start_ix + part.len(); - } else { - return false; - } - } - true - } -} diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs new file mode 100644 index 0000000000..f8d63b2a4f --- /dev/null +++ b/crates/language/src/syntax_map/syntax_map_tests.rs @@ -0,0 +1,1171 @@ +use super::*; +use crate::LanguageConfig; +use rand::rngs::StdRng; +use std::{env, ops::Range, sync::Arc}; +use text::Buffer; +use tree_sitter::Node; +use unindent::Unindent as _; +use util::test::marked_text_ranges; + +#[test] +fn test_splice_included_ranges() { + let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)]; + + let new_ranges = splice_included_ranges( + ranges.clone(), + &[54..56, 58..68], + &[ts_range(50..54), ts_range(59..67)], + ); + assert_eq!( + new_ranges, + &[ + ts_range(20..30), + ts_range(50..54), + ts_range(59..67), + ts_range(80..90), + ] + ); + + let new_ranges = splice_included_ranges(ranges.clone(), &[70..71, 91..100], &[]); + assert_eq!( + new_ranges, + &[ts_range(20..30), ts_range(50..60), ts_range(80..90)] + ); + + let new_ranges = + splice_included_ranges(ranges.clone(), &[], &[ts_range(0..2), ts_range(70..75)]); + assert_eq!( + new_ranges, + &[ + ts_range(0..2), + ts_range(20..30), + ts_range(50..60), + ts_range(70..75), + ts_range(80..90) + ] + ); + + let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]); + assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]); + + fn ts_range(range: Range) -> tree_sitter::Range { + tree_sitter::Range { + start_byte: range.start, + start_point: tree_sitter::Point { + row: 0, + column: range.start, + }, + end_byte: range.end, + end_point: tree_sitter::Point { + row: 0, + column: range.end, + }, + } + } +} + +#[gpui::test] +fn test_syntax_map_layers_for_range() { + let registry = Arc::new(LanguageRegistry::test()); + let language = Arc::new(rust_lang()); + registry.add(language.clone()); + + let mut buffer = Buffer::new( + 0, + 0, + r#" + fn a() { + assert_eq!( + b(vec![C {}]), + vec![d.e], + ); + println!("{}", f(|_| true)); + } + "# + .unindent(), + ); + + let mut syntax_map = SyntaxMap::new(); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(language.clone(), &buffer); + + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 0)..Point::new(2, 0), + &[ + "...(function_item ... (block (expression_statement (macro_invocation...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + ], + ); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 14)..Point::new(2, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + "...(array_expression (struct_expression ...", + ], + ); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(3, 14)..Point::new(3, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + "...(array_expression (field_expression ...", + ], + ); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(5, 12)..Point::new(5, 16), + &[ + "...(function_item ...", + "...(call_expression ... (arguments (closure_expression ...", + ], + ); + + // Replace a vec! macro invocation with a plain slice, removing a syntactic layer. + let macro_name_range = range_for_text(&buffer, "vec!"); + buffer.edit([(macro_name_range, "&")]); + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 14)..Point::new(2, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (reference_expression value: (array_expression...", + ], + ); + + // Put the vec! macro back, adding back the syntactic layer. + buffer.undo(); + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(2, 14)..Point::new(2, 16), + &[ + "...(function_item ...", + "...(tuple_expression (call_expression ... arguments: (arguments (macro_invocation...", + "...(array_expression (struct_expression ...", + ], + ); +} + +#[gpui::test] +fn test_dynamic_language_injection() { + let registry = Arc::new(LanguageRegistry::test()); + let markdown = Arc::new(markdown_lang()); + registry.add(markdown.clone()); + registry.add(Arc::new(rust_lang())); + registry.add(Arc::new(ruby_lang())); + + let mut buffer = Buffer::new( + 0, + 0, + r#" + This is a code block: + + ```rs + fn foo() {} + ``` + "# + .unindent(), + ); + + let mut syntax_map = SyntaxMap::new(); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(markdown.clone(), &buffer); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(3, 0)..Point::new(3, 0), + &[ + "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...", + "...(function_item name: (identifier) parameters: (parameters) body: (block)...", + ], + ); + + // Replace Rust with Ruby in code block. + let macro_name_range = range_for_text(&buffer, "rs"); + buffer.edit([(macro_name_range, "ruby")]); + syntax_map.interpolate(&buffer); + syntax_map.reparse(markdown.clone(), &buffer); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(3, 0)..Point::new(3, 0), + &[ + "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...", + "...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...", + ], + ); + + // Replace Ruby with a language that hasn't been loaded yet. + let macro_name_range = range_for_text(&buffer, "ruby"); + buffer.edit([(macro_name_range, "html")]); + syntax_map.interpolate(&buffer); + syntax_map.reparse(markdown.clone(), &buffer); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(3, 0)..Point::new(3, 0), + &[ + "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..." + ], + ); + assert!(syntax_map.contains_unknown_injections()); + + registry.add(Arc::new(html_lang())); + syntax_map.reparse(markdown.clone(), &buffer); + assert_layers_for_range( + &syntax_map, + &buffer, + Point::new(3, 0)..Point::new(3, 0), + &[ + "...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...", + "(fragment (text))", + ], + ); + assert!(!syntax_map.contains_unknown_injections()); +} + +#[gpui::test] +fn test_typing_multiple_new_injections() { + let (buffer, syntax_map) = test_edit_sequence( + "Rust", + &[ + "fn a() { dbg }", + "fn a() { dbg«!» }", + "fn a() { dbg!«()» }", + "fn a() { dbg!(«b») }", + "fn a() { dbg!(b«.») }", + "fn a() { dbg!(b.«c») }", + "fn a() { dbg!(b.c«()») }", + "fn a() { dbg!(b.c(«vec»)) }", + "fn a() { dbg!(b.c(vec«!»)) }", + "fn a() { dbg!(b.c(vec!«[]»)) }", + "fn a() { dbg!(b.c(vec![«d»])) }", + "fn a() { dbg!(b.c(vec![d«.»])) }", + "fn a() { dbg!(b.c(vec![d.«e»])) }", + ], + ); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["field"], + "fn a() { dbg!(b.«c»(vec![d.«e»])) }", + ); +} + +#[gpui::test] +fn test_pasting_new_injection_line_between_others() { + let (buffer, syntax_map) = test_edit_sequence( + "Rust", + &[ + " + fn a() { + b!(B {}); + c!(C {}); + d!(D {}); + e!(E {}); + f!(F {}); + g!(G {}); + } + ", + " + fn a() { + b!(B {}); + c!(C {}); + d!(D {}); + « h!(H {}); + » e!(E {}); + f!(F {}); + g!(G {}); + } + ", + ], + ); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["struct"], + " + fn a() { + b!(«B {}»); + c!(«C {}»); + d!(«D {}»); + h!(«H {}»); + e!(«E {}»); + f!(«F {}»); + g!(«G {}»); + } + ", + ); +} + +#[gpui::test] +fn test_joining_injections_with_child_injections() { + let (buffer, syntax_map) = test_edit_sequence( + "Rust", + &[ + " + fn a() { + b!( + c![one.two.three], + d![four.five.six], + ); + e!( + f![seven.eight], + ); + } + ", + " + fn a() { + b!( + c![one.two.three], + d![four.five.six], + ˇ f![seven.eight], + ); + } + ", + ], + ); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["field"], + " + fn a() { + b!( + c![one.«two».«three»], + d![four.«five».«six»], + f![seven.«eight»], + ); + } + ", + ); +} + +#[gpui::test] +fn test_editing_edges_of_injection() { + test_edit_sequence( + "Rust", + &[ + " + fn a() { + b!(c!()) + } + ", + " + fn a() { + «d»!(c!()) + } + ", + " + fn a() { + «e»d!(c!()) + } + ", + " + fn a() { + ed!«[»c!()«]» + } + ", + ], + ); +} + +#[gpui::test] +fn test_edits_preceding_and_intersecting_injection() { + test_edit_sequence( + "Rust", + &[ + // + "const aaaaaaaaaaaa: B = c!(d(e.f));", + "const aˇa: B = c!(d(eˇ));", + ], + ); +} + +#[gpui::test] +fn test_non_local_changes_create_injections() { + test_edit_sequence( + "Rust", + &[ + " + // a! { + static B: C = d; + // } + ", + " + ˇa! { + static B: C = d; + ˇ} + ", + ], + ); +} + +#[gpui::test] +fn test_creating_many_injections_in_one_edit() { + test_edit_sequence( + "Rust", + &[ + " + fn a() { + one(Two::three(3)); + four(Five::six(6)); + seven(Eight::nine(9)); + } + ", + " + fn a() { + one«!»(Two::three(3)); + four«!»(Five::six(6)); + seven«!»(Eight::nine(9)); + } + ", + " + fn a() { + one!(Two::three«!»(3)); + four!(Five::six«!»(6)); + seven!(Eight::nine«!»(9)); + } + ", + ], + ); +} + +#[gpui::test] +fn test_editing_across_injection_boundary() { + test_edit_sequence( + "Rust", + &[ + " + fn one() { + two(); + three!( + three.four, + five.six, + ); + } + ", + " + fn one() { + two(); + th«irty_five![» + three.four, + five.six, + « seven.eight, + ];» + } + ", + ], + ); +} + +#[gpui::test] +fn test_removing_injection_by_replacing_across_boundary() { + test_edit_sequence( + "Rust", + &[ + " + fn one() { + two!( + three.four, + ); + } + ", + " + fn one() { + t«en + .eleven( + twelve, + » + three.four, + ); + } + ", + ], + ); +} + +#[gpui::test] +fn test_combined_injections() { + let (buffer, syntax_map) = test_edit_sequence( + "ERB", + &[ + " + + <% if @one %> +
+ <% else %> +
+ <% end %> +
+ + ", + " + + <% if @one %> +
+ ˇ else ˇ +
+ <% end %> +
+ + ", + " + + <% if @one «;» end %> +
+ + ", + ], + ); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["tag", "ivar"], + " + <«body»> + <% if «@one» ; end %> + + + ", + ); +} + +#[gpui::test] +fn test_combined_injections_empty_ranges() { + test_edit_sequence( + "ERB", + &[ + " + <% if @one %> + <% else %> + <% end %> + ", + " + <% if @one %> + ˇ<% end %> + ", + ], + ); +} + +#[gpui::test] +fn test_combined_injections_edit_edges_of_ranges() { + let (buffer, syntax_map) = test_edit_sequence( + "ERB", + &[ + " + <%= one @two %> + <%= three @four %> + ", + " + <%= one @two %ˇ + <%= three @four %> + ", + " + <%= one @two %«>» + <%= three @four %> + ", + ], + ); + + assert_capture_ranges( + &syntax_map, + &buffer, + &["tag", "ivar"], + " + <%= one «@two» %> + <%= three «@four» %> + ", + ); +} + +#[gpui::test] +fn test_combined_injections_splitting_some_injections() { + let (_buffer, _syntax_map) = test_edit_sequence( + "ERB", + &[ + r#" + <%A if b(:c) %> + d + <% end %> + eee + <% f %> + "#, + r#" + <%« AAAAAAA %> + hhhhhhh + <%=» if b(:c) %> + d + <% end %> + eee + <% f %> + "#, + ], + ); +} + +#[gpui::test] +fn test_combined_injections_inside_injections() { + let (_buffer, _syntax_map) = test_edit_sequence( + "Markdown", + &[ + r#" + here is some ERB code: + + ```erb +
    + <% people.each do |person| %> +
  • <%= person.name %>
    • + <% end %> +
+ ``` + "#, + r#" + here is some ERB code: + + ```erb +
    + <% people«2».each do |person| %> +
  • <%= person.name %>
    • + <% end %> +
+ ``` + "#, + ], + ); +} + +#[gpui::test(iterations = 50)] +fn test_random_syntax_map_edits(mut rng: StdRng) { + let operations = env::var("OPERATIONS") + .map(|i| i.parse().expect("invalid `OPERATIONS` variable")) + .unwrap_or(10); + + let text = r#" + fn test_something() { + let vec = vec![5, 1, 3, 8]; + assert_eq!( + vec + .into_iter() + .map(|i| i * 2) + .collect::>(), + vec![ + 5 * 2, 1 * 2, 3 * 2, 8 * 2 + ], + ); + } + "# + .unindent() + .repeat(2); + + let registry = Arc::new(LanguageRegistry::test()); + let language = Arc::new(rust_lang()); + registry.add(language.clone()); + let mut buffer = Buffer::new(0, 0, text); + + let mut syntax_map = SyntaxMap::new(); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(language.clone(), &buffer); + + let mut reference_syntax_map = SyntaxMap::new(); + reference_syntax_map.set_language_registry(registry.clone()); + + log::info!("initial text:\n{}", buffer.text()); + + for _ in 0..operations { + let prev_buffer = buffer.snapshot(); + let prev_syntax_map = syntax_map.snapshot(); + + buffer.randomly_edit(&mut rng, 3); + log::info!("text:\n{}", buffer.text()); + + syntax_map.interpolate(&buffer); + check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer); + + syntax_map.reparse(language.clone(), &buffer); + + reference_syntax_map.clear(); + reference_syntax_map.reparse(language.clone(), &buffer); + } + + for i in 0..operations { + let i = operations - i - 1; + buffer.undo(); + log::info!("undoing operation {}", i); + log::info!("text:\n{}", buffer.text()); + + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + reference_syntax_map.clear(); + reference_syntax_map.reparse(language.clone(), &buffer); + assert_eq!( + syntax_map.layers(&buffer).len(), + reference_syntax_map.layers(&buffer).len(), + "wrong number of layers after undoing edit {i}" + ); + } + + let layers = syntax_map.layers(&buffer); + let reference_layers = reference_syntax_map.layers(&buffer); + for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter()) { + assert_eq!( + edited_layer.node().to_sexp(), + reference_layer.node().to_sexp() + ); + assert_eq!(edited_layer.node().range(), reference_layer.node().range()); + } +} + +#[gpui::test(iterations = 50)] +fn test_random_syntax_map_edits_with_combined_injections(mut rng: StdRng) { + let operations = env::var("OPERATIONS") + .map(|i| i.parse().expect("invalid `OPERATIONS` variable")) + .unwrap_or(10); + + let text = r#" +
+ <% if one?(:two) %> +

+ <%= yield :five %> +

+ <% elsif Six.seven(8) %> +

+ <%= yield :five %> +

+ <% else %> + Ok + <% end %> +
+ "# + .unindent() + .repeat(8); + + let registry = Arc::new(LanguageRegistry::test()); + let language = Arc::new(erb_lang()); + registry.add(language.clone()); + registry.add(Arc::new(ruby_lang())); + registry.add(Arc::new(html_lang())); + let mut buffer = Buffer::new(0, 0, text); + + let mut syntax_map = SyntaxMap::new(); + syntax_map.set_language_registry(registry.clone()); + syntax_map.reparse(language.clone(), &buffer); + + let mut reference_syntax_map = SyntaxMap::new(); + reference_syntax_map.set_language_registry(registry.clone()); + + log::info!("initial text:\n{}", buffer.text()); + + for _ in 0..operations { + let prev_buffer = buffer.snapshot(); + let prev_syntax_map = syntax_map.snapshot(); + + buffer.randomly_edit(&mut rng, 3); + log::info!("text:\n{}", buffer.text()); + + syntax_map.interpolate(&buffer); + check_interpolation(&prev_syntax_map, &syntax_map, &prev_buffer, &buffer); + + syntax_map.reparse(language.clone(), &buffer); + + reference_syntax_map.clear(); + reference_syntax_map.reparse(language.clone(), &buffer); + } + + for i in 0..operations { + let i = operations - i - 1; + buffer.undo(); + log::info!("undoing operation {}", i); + log::info!("text:\n{}", buffer.text()); + + syntax_map.interpolate(&buffer); + syntax_map.reparse(language.clone(), &buffer); + + reference_syntax_map.clear(); + reference_syntax_map.reparse(language.clone(), &buffer); + assert_eq!( + syntax_map.layers(&buffer).len(), + reference_syntax_map.layers(&buffer).len(), + "wrong number of layers after undoing edit {i}" + ); + } + + let layers = syntax_map.layers(&buffer); + let reference_layers = reference_syntax_map.layers(&buffer); + for (edited_layer, reference_layer) in layers.into_iter().zip(reference_layers.into_iter()) { + assert_eq!( + edited_layer.node().to_sexp(), + reference_layer.node().to_sexp() + ); + assert_eq!(edited_layer.node().range(), reference_layer.node().range()); + } +} + +fn check_interpolation( + old_syntax_map: &SyntaxSnapshot, + new_syntax_map: &SyntaxSnapshot, + old_buffer: &BufferSnapshot, + new_buffer: &BufferSnapshot, +) { + let edits = new_buffer + .edits_since::(&old_buffer.version()) + .collect::>(); + + for (old_layer, new_layer) in old_syntax_map + .layers + .iter() + .zip(new_syntax_map.layers.iter()) + { + assert_eq!(old_layer.range, new_layer.range); + let Some(old_tree) = old_layer.content.tree() else { continue }; + let Some(new_tree) = new_layer.content.tree() else { continue }; + let old_start_byte = old_layer.range.start.to_offset(old_buffer); + let new_start_byte = new_layer.range.start.to_offset(new_buffer); + let old_start_point = old_layer.range.start.to_point(old_buffer).to_ts_point(); + let new_start_point = new_layer.range.start.to_point(new_buffer).to_ts_point(); + let old_node = old_tree.root_node_with_offset(old_start_byte, old_start_point); + let new_node = new_tree.root_node_with_offset(new_start_byte, new_start_point); + check_node_edits( + old_layer.depth, + &old_layer.range, + old_node, + new_node, + old_buffer, + new_buffer, + &edits, + ); + } + + fn check_node_edits( + depth: usize, + range: &Range, + old_node: Node, + new_node: Node, + old_buffer: &BufferSnapshot, + new_buffer: &BufferSnapshot, + edits: &[text::Edit], + ) { + assert_eq!(old_node.kind(), new_node.kind()); + + let old_range = old_node.byte_range(); + let new_range = new_node.byte_range(); + + let is_edited = edits + .iter() + .any(|edit| edit.new.start < new_range.end && edit.new.end > new_range.start); + if is_edited { + assert!( + new_node.has_changes(), + concat!( + "failed to mark node as edited.\n", + "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n", + "node kind: {}, old node range: {:?}, new node range: {:?}", + ), + depth, + range.to_offset(old_buffer), + range.to_offset(new_buffer), + new_node.kind(), + old_range, + new_range, + ); + } + + if !new_node.has_changes() { + assert_eq!( + old_buffer + .text_for_range(old_range.clone()) + .collect::(), + new_buffer + .text_for_range(new_range.clone()) + .collect::(), + concat!( + "mismatched text for node\n", + "layer depth: {}, old layer range: {:?}, new layer range: {:?},\n", + "node kind: {}, old node range:{:?}, new node range:{:?}", + ), + depth, + range.to_offset(old_buffer), + range.to_offset(new_buffer), + new_node.kind(), + old_range, + new_range, + ); + } + + for i in 0..new_node.child_count() { + check_node_edits( + depth, + range, + old_node.child(i).unwrap(), + new_node.child(i).unwrap(), + old_buffer, + new_buffer, + edits, + ) + } + } +} + +fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) { + let registry = Arc::new(LanguageRegistry::test()); + registry.add(Arc::new(rust_lang())); + registry.add(Arc::new(ruby_lang())); + registry.add(Arc::new(html_lang())); + registry.add(Arc::new(erb_lang())); + registry.add(Arc::new(markdown_lang())); + let language = registry + .language_for_name(language_name) + .now_or_never() + .unwrap() + .unwrap(); + let mut buffer = Buffer::new(0, 0, Default::default()); + + let mut mutated_syntax_map = SyntaxMap::new(); + mutated_syntax_map.set_language_registry(registry.clone()); + mutated_syntax_map.reparse(language.clone(), &buffer); + + for (i, marked_string) in steps.into_iter().enumerate() { + buffer.edit_via_marked_text(&marked_string.unindent()); + + // Reparse the syntax map + mutated_syntax_map.interpolate(&buffer); + mutated_syntax_map.reparse(language.clone(), &buffer); + + // Create a second syntax map from scratch + let mut reference_syntax_map = SyntaxMap::new(); + reference_syntax_map.set_language_registry(registry.clone()); + reference_syntax_map.reparse(language.clone(), &buffer); + + // Compare the mutated syntax map to the new syntax map + let mutated_layers = mutated_syntax_map.layers(&buffer); + let reference_layers = reference_syntax_map.layers(&buffer); + assert_eq!( + mutated_layers.len(), + reference_layers.len(), + "wrong number of layers at step {i}" + ); + for (edited_layer, reference_layer) in + mutated_layers.into_iter().zip(reference_layers.into_iter()) + { + assert_eq!( + edited_layer.node().to_sexp(), + reference_layer.node().to_sexp(), + "different layer at step {i}" + ); + assert_eq!( + edited_layer.node().range(), + reference_layer.node().range(), + "different layer at step {i}" + ); + } + } + + (buffer, mutated_syntax_map) +} + +fn html_lang() -> Language { + Language::new( + LanguageConfig { + name: "HTML".into(), + path_suffixes: vec!["html".to_string()], + ..Default::default() + }, + Some(tree_sitter_html::language()), + ) + .with_highlights_query( + r#" + (tag_name) @tag + (erroneous_end_tag_name) @tag + (attribute_name) @property + "#, + ) + .unwrap() +} + +fn ruby_lang() -> Language { + Language::new( + LanguageConfig { + name: "Ruby".into(), + path_suffixes: vec!["rb".to_string()], + ..Default::default() + }, + Some(tree_sitter_ruby::language()), + ) + .with_highlights_query( + r#" + ["if" "do" "else" "end"] @keyword + (instance_variable) @ivar + "#, + ) + .unwrap() +} + +fn erb_lang() -> Language { + Language::new( + LanguageConfig { + name: "ERB".into(), + path_suffixes: vec!["erb".to_string()], + ..Default::default() + }, + Some(tree_sitter_embedded_template::language()), + ) + .with_highlights_query( + r#" + ["<%" "%>"] @keyword + "#, + ) + .unwrap() + .with_injection_query( + r#" + ( + (code) @content + (#set! "language" "ruby") + (#set! "combined") + ) + + ( + (content) @content + (#set! "language" "html") + (#set! "combined") + ) + "#, + ) + .unwrap() +} + +fn rust_lang() -> Language { + Language::new( + LanguageConfig { + name: "Rust".into(), + path_suffixes: vec!["rs".to_string()], + ..Default::default() + }, + Some(tree_sitter_rust::language()), + ) + .with_highlights_query( + r#" + (field_identifier) @field + (struct_expression) @struct + "#, + ) + .unwrap() + .with_injection_query( + r#" + (macro_invocation + (token_tree) @content + (#set! "language" "rust")) + "#, + ) + .unwrap() +} + +fn markdown_lang() -> Language { + Language::new( + LanguageConfig { + name: "Markdown".into(), + path_suffixes: vec!["md".into()], + ..Default::default() + }, + Some(tree_sitter_markdown::language()), + ) + .with_injection_query( + r#" + (fenced_code_block + (info_string + (language) @language) + (code_fence_content) @content) + "#, + ) + .unwrap() +} + +fn range_for_text(buffer: &Buffer, text: &str) -> Range { + let start = buffer.as_rope().to_string().find(text).unwrap(); + start..start + text.len() +} + +fn assert_layers_for_range( + syntax_map: &SyntaxMap, + buffer: &BufferSnapshot, + range: Range, + expected_layers: &[&str], +) { + let layers = syntax_map + .layers_for_range(range, &buffer) + .collect::>(); + assert_eq!( + layers.len(), + expected_layers.len(), + "wrong number of layers" + ); + for (i, (layer, expected_s_exp)) in layers.iter().zip(expected_layers.iter()).enumerate() { + let actual_s_exp = layer.node().to_sexp(); + assert!( + string_contains_sequence( + &actual_s_exp, + &expected_s_exp.split("...").collect::>() + ), + "layer {i}:\n\nexpected: {expected_s_exp}\nactual: {actual_s_exp}", + ); + } +} + +fn assert_capture_ranges( + syntax_map: &SyntaxMap, + buffer: &BufferSnapshot, + highlight_query_capture_names: &[&str], + marked_string: &str, +) { + let mut actual_ranges = Vec::>::new(); + let captures = syntax_map.captures(0..buffer.len(), buffer, |grammar| { + grammar.highlights_query.as_ref() + }); + let queries = captures + .grammars() + .iter() + .map(|grammar| grammar.highlights_query.as_ref().unwrap()) + .collect::>(); + for capture in captures { + let name = &queries[capture.grammar_index].capture_names()[capture.index as usize]; + if highlight_query_capture_names.contains(&name.as_str()) { + actual_ranges.push(capture.node.byte_range()); + } + } + + let (text, expected_ranges) = marked_text_ranges(&marked_string.unindent(), false); + assert_eq!(text, buffer.text()); + assert_eq!(actual_ranges, expected_ranges); +} + +pub fn string_contains_sequence(text: &str, parts: &[&str]) -> bool { + let mut last_part_end = 0; + for part in parts { + if let Some(start_ix) = text[last_part_end..].find(part) { + last_part_end = start_ix + part.len(); + } else { + return false; + } + } + true +}