From 8ca3ab56b56c11dd085ec4f97cdbec24e7d8e98a Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sun, 5 Jul 2020 16:45:07 +0000 Subject: [PATCH] update the mdbook documentation, introducing a diagram --- book/src/SUMMARY.md | 1 + book/src/plumbing.md | 9 +- book/src/plumbing/database.md | 54 ++---- book/src/plumbing/diagram.md | 58 +++++++ book/src/plumbing/query_groups.md | 186 +++++++-------------- components/salsa-macros/src/query_group.rs | 10 +- 6 files changed, 142 insertions(+), 176 deletions(-) create mode 100644 book/src/plumbing/diagram.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 86329d55..547fa37c 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -8,6 +8,7 @@ - [On-demand (Lazy) inputs](./common_patterns/on_demand_inputs.md) - [YouTube videos](./videos.md) - [Plumbing](./plumbing.md) + - [Diagram](./plumbing/diagram.md) - [Query groups](./plumbing/query_groups.md) - [Database](./plumbing/database.md) - [RFCs](./rfcs.md) diff --git a/book/src/plumbing.md b/book/src/plumbing.md index 29bce6f3..5d4b1a72 100644 --- a/book/src/plumbing.md +++ b/book/src/plumbing.md @@ -1,17 +1,20 @@ # Plumbing -**Last updated:** 2020-06-24 - This chapter documents the code that salsa generates and its "inner workings". We refer to this as the "plumbing". This page walks through the ["Hello, World!"] example and explains the code that it generates. Please take it with a grain of salt: while we make an effort to keep this documentation up to date, this sort of thing can fall out of date -easily. +easily. See the page history below for major updates. ["Hello, World!"]: https://github.com/salsa-rs/salsa/blob/master/examples/hello_world/main.rs If you'd like to see for yourself, you can set the environment variable `SALSA_DUMP` to 1 while the procedural macro runs, and it will dump the full output to stdout. I recommend piping the output through rustfmt. + +## History + +* 2020-07-05: Updated to take [RFC 6](rfcs/RFC0006-Dynamic-Databases.md) into account. +* 2020-06-24: Initial version. \ No newline at end of file diff --git a/book/src/plumbing/database.md b/book/src/plumbing/database.md index 10a68bbd..3ea7598c 100644 --- a/book/src/plumbing/database.md +++ b/book/src/plumbing/database.md @@ -15,13 +15,9 @@ structs (like `HelloWorldStorage`) and generates the following items: each query group. Note: these are the structs full of hashmaps etc that are generaetd by the query group procdural macro, not the `HelloWorldStorage` struct itself. -* a struct `__SalsaDatabaseKey` that wraps an enum `__SalsaDatabaseKeyKind`. The - enum contains one variant per query group, and in each variant contains the - group key. This can be used to identify any query in the database. * an impl of `HasQueryGroup` for each query group `G` * an impl of `salsa::plumbing::DatabaseStorageTypes` for the database struct * an impl of `salsa::plumbing::DatabaseOps` for the database struct -* an impl of `salsa::plumbing::DatabaseKey` for the database struct `DB` ## Key constraint: we do not know the names of individual queries @@ -44,30 +40,11 @@ struct __SalsaDatabaseStorage { } ``` -## The database key struct / enum and the `DatabaseKey` impl +We also generate a `Default` impl for `__SalsaDatabaseStorage`. It invokes +a `new` method on each group storage with the unique index assigned to that group. +This invokes the [inherent `new` method generated by the `#[salsa::query_group]` macro][new]. -The `__SalsaDatabaseKey` and `__SalsaDatabaseKeyKind` types create a **database -key**, which uniquely identifies any query in the database. It builds on the -**group keys** created by the query groups, which uniquely identify a query -within a given query group. - -```rust -struct __SalsaDatabaseKey { - kind: __SalsaDatabaseKeyKind -} - -enum __SalsaDatabaseKeyKind { - HelloWorld( - >::GroupKey - ) -} -``` - -We also generate an impl of `DatabaseKey`: - -```rust,ignore -{{#include ../../../components/salsa-macros/src/database_storage.rs:DatabaseKey}} -``` +[new]: query_groups.md#group-storage ## The `HasQueryGroup` impl @@ -78,23 +55,14 @@ within the greater database. The impl is generated here: {{#include ../../../components/salsa-macros/src/database_storage.rs:HasQueryGroup}} ``` -and so for our example it would look something like +The `HasQueryGroup` impl combines with [the blanket impl] from the +`#[salsa::query_group]` macro so that the database can implement the query group +trait (e.g., the `HelloWorld` trait) but without knowing all the names of the +query methods and the like. -```rust -impl salsa::plumbing::HasQueryGroup for DatabaseStruct { - fn group_storage(&self) -> &HelloWorldStorage::GroupStorage { - &self.hello_world - } +[the blanket impl]: query_groups.md#impl-of-the-hello-world-trait - fn database_key(group_key: HelloWorldStorage::GroupKey) -> __SalsaDatabaseKey { - __SalsaDatabaseKey { - kind: __SalsaDatabaseKeyKind::HelloWorld(group_key) - } - } -} -``` - -## Other impls +## The `DatabaseStorageTypes` impl Then there are a variety of other impls, like this one for `DatabaseStorageTypes`: @@ -102,6 +70,8 @@ Then there are a variety of other impls, like this one for `DatabaseStorageTypes {{#include ../../../components/salsa-macros/src/database_storage.rs:DatabaseStorageTypes}} ``` +## The `DatabaseOps` impl + Or this one for `DatabaseOps`, which defines the for-each method to invoke an operation on every kind of query in the database. It ultimately delegates to the `for_each` methods for the groups: diff --git a/book/src/plumbing/diagram.md b/book/src/plumbing/diagram.md new file mode 100644 index 00000000..15050897 --- /dev/null +++ b/book/src/plumbing/diagram.md @@ -0,0 +1,58 @@ +# Diagram + +Based on the hello world example: + +```rust,ignore +{{#include ../../../examples/hello_world/main.rs:trait}} +``` + +```rust,ignore +{{#include ../../../examples/hello_world/main.rs:database}} +``` + +```mermaid +graph LR + classDef diagramNode text-align:left; + subgraph query group + HelloWorldTrait["trait HelloWorld: Database + HasQueryGroup(HelloWorldStroage)"] + HelloWorldImpl["impl(DB) HelloWorld for DB
where DB: HasQueryGroup(HelloWorldStorage)"] + click HelloWorldImpl "http:query_groups.html#impl-of-the-hello-world-trait" "more info" + HelloWorldStorage["struct HelloWorldStorage"] + click HelloWorldStorage "http:query_groups.html#the-group-struct-and-querygroup-trait" "more info" + QueryGroupImpl["impl QueryGroup for HelloWorldStorage
  type DynDb = dyn HelloWorld
  type Storage = HelloWorldGroupStorage__;"] + click QueryGroupImpl "http:query_groups.html#the-group-struct-and-querygroup-trait" "more info" + HelloWorldGroupStorage["struct HelloWorldGroupStorage__"] + click HelloWorldGroupStorage "http:query_groups.html#group-storage" "more info" + subgraph for each query... + LengthQuery[struct LengthQuery] + LengthQueryImpl["impl Query for LengthQuery
  type Key = ()
  type Value = usize
  type Storage = salsa::DerivedStorage(Self)
  type QueryGroup = HelloWorldStorage"] + LengthQueryFunctionImpl["impl QueryFunction for LengthQuery
  fn execute(db: &dyn HelloWorld, key: ()) -> usize"] + click LengthQuery "http:query_groups.html#for-each-query-a-query-struct" "more info" + click LengthQueryImpl "http:query_groups.html#for-each-query-a-query-struct" "more info" + click LengthQueryFunctionImpl "http:query_groups.html#for-each-query-a-query-struct" "more info" + end + class HelloWorldTrait,HelloWorldImpl,HelloWorldStorage,QueryGroupImpl,HelloWorldGroupStorage diagramNode; + class LengthQuery,LengthQueryImpl,LengthQueryFunctionImpl diagramNode; + end + subgraph database + DatabaseStruct["struct Database { .. storage: Storage(Self) .. }"] + subgraph for each group... + HasQueryGroup["impl plumbing::HasQueryGroup(HelloWorldStorage) for DatabaseStruct"] + click HasQueryGroup "http:database.html#the-hasquerygroup-impl" "more info" + end + DatabaseStorageTypes["impl plumbing::DatabaseStorageTypes for DatabaseStruct
  type DatabaseStorage = __SalsaDatabaseStorage"] + click DatabaseStorageTypes "http:database.html#the-databasestoragetypes-impl" "more info" + DatabaseStorage["struct __SalsaDatabaseStorage"] + click DatabaseStorage "http:database.html#the-database-storage-struct" "more info" + DatabaseOps["impl plumbing::DatabaseOps for DatabaseStruct"] + click DatabaseOps "http:database.html#the-databaseops-impl" "more info" + class DatabaseStruct,DatabaseStorage,DatabaseStorageTypes,DatabaseOps,HasQueryGroup diagramNode; + end + subgraph salsa crate + DerivedStorage["DerivedStorage"] + class DerivedStorage diagramNode; + end + LengthQueryImpl --> DerivedStorage; + DatabaseStruct --> HelloWorldImpl + HasQueryGroup --> HelloWorldImpl +``` \ No newline at end of file diff --git a/book/src/plumbing/query_groups.md b/book/src/plumbing/query_groups.md index cca95234..fdbbad85 100644 --- a/book/src/plumbing/query_groups.md +++ b/book/src/plumbing/query_groups.md @@ -6,39 +6,39 @@ When you define a query group trait: {{#include ../../../examples/hello_world/main.rs:trait}} ``` -the `salsa::query_group` macro generates a number of things: +the `salsa::query_group` macro generates a number of things, shown in the sample +generated code below (details in the sections to come). -* a copy of the `HelloWorld` trait, minus the salsa annotations, and lightly edited -* a "group struct" named `HelloWorldStorage` that represents the group; this struct implements `plumbing::QueryGroup` - * somewhat confusingly, this struct doesn't actually contain the storage itself, but rather has an associated type that leads to the "true" storage struct -* an impl of the `HelloWorld` trait, for any database type -* for each query, a "query struct" named after the query; these structs implement `plumbing::Query` and sometimes other plumbing traits -* a group key, an enum that can identify any query within the group and store its key -* the associated storage struct, which contains the actual hashmaps that store the data for all queries in the group - -Note that there are a number of structs and types (e.g., the group descriptor and associated storage struct) that represent things which don't have "public" +Note that there are a number of structs and types (e.g., the group descriptor names. We currently generate mangled names with `__` afterwards, but those names are not meant to be exposed to the user (ideally we'd use hygiene to enforce this). -So the generated code looks something like this. We'll go into more detail on -each part in the following sections. - ```rust,ignore -// First, a copy of the trait, though sometimes with some extra -// methods (e.g., `set_input_string`) -trait HelloWorld: salsa::Database { +// First, a copy of the trait, though with extra supertraits and +// sometimes with some extra methods (e.g., `set_input_string`) +trait HelloWorld: + salsa::Database + + salsa::plumbing::HasQueryGroup +{ fn input_string(&self, key: ()) -> Arc; fn set_input_string(&mut self, key: (), value: Arc); fn length(&self, key: ()) -> usize; } -// Next, the group struct +// Next, the "query group struct", whose name was given by the +// user. This struct implements the `QueryGroup` trait which +// defines a few associated types common to the entire group. struct HelloWorldStorage { } -impl salsa::plumbing::QueryGroup for HelloWorldStorage { ... } +impl salsa::plumbing::QueryGroup for HelloWorldStorage { + type DynDb = dyn HelloWorld; + type GroupStorage = HelloWorldGroupStorage__; +} -// Next, the impl of the trait +// Next, a blanket impl of the `HelloWorld` trait. This impl +// works for any database `DB` that implements the +// appropriate `HasQueryGroup`. impl HelloWorld for DB where DB: salsa::Database, @@ -47,41 +47,29 @@ where ... } -// Next, a series of query structs and query impls -struct InputQuery { } -unsafe impl salsa::Query for InputQuery -where - DB: HelloWorld, - DB: salsa::plumbing::HasQueryGroup<#group_struct>, - DB: salsa::Database, -{ +// Next, for each query, a "query struct" that represents it. +// The query struct has inherent methods like `in_db` and +// implements the `Query` trait, which defines various +// details about the query (e.g., its key, value, etc). +pub struct InputQuery { } +impl InputQuery { /* definition for `in_db`, etc */ } +impl salsa::Query for InputQuery { + /* associated types */ +} + +// Same as above, but for the derived query `length`. +// For derived queries, we also implement `QueryFunction` +// which defines how to execute the query. +pub struct LengthQuery { } +impl salsa::Query for LengthQuery { ... } -struct LengthQuery { } -unsafe impl salsa::Query for LengthQuery -where - DB: HelloWorld, - DB: salsa::plumbing::HasQueryGroup<#group_struct>, - DB: salsa::Database, -{ +impl salsa::QueryFunction for LengthQuery { ... } -// For derived queries, those include implementations -// of additional traits like `QueryFunction` -unsafe impl salsa::QueryFunction for LengthQuery -where - DB: HelloWorld, - DB: salsa::plumbing::HasQueryGroup<#group_struct>, - DB: salsa::Database, -{ - ... -} - -// The group key -enum HelloWorldGroupKey__ { .. } - -// The group storage +// Finally, the group storage, which contains the actual +// hashmaps and other data used to implement the queries. struct HelloWorldGroupStorage__ { .. } ``` @@ -98,10 +86,9 @@ storage" struct: ```rust,ignore struct HelloWorldStorage { } -impl salsa::plumbing::QueryGroup for HelloWorldStorage { +impl salsa::plumbing::QueryGroup for HelloWorldStorage { + type DynDb = dyn HelloWorld; type GroupStorage = HelloWorldGroupStorage__; // generated struct - type GroupKey = HelloWorldGroupKey__; - type GroupData = ((), Arc, (), usize); } ``` @@ -164,52 +151,7 @@ a bunch of metadata about the query (and repeats, for convenience, some of the data about the group that the query is in): ```rust,ignore -unsafe impl salsa::Query for LengthQuery -where - DB: HelloWorld, - DB: salsa::plumbing::HasQueryGroup<#group_struct>, - DB: salsa::Database, -{ - // A tuple of the types of the function parameters trom trait. - type Key = ((), ); - - // The return value of the function in the trait. - type Value = Arc; - - // The "query storage" references a type from within salsa - // that stores the actual query data and defines the - // logic for accessing and revalidating it. - // - // It is generic over the query type which lets it - // customize itself to the keys/value of this particular - // query. - type Storage = salsa::derived::DerivedStorage< - DB, - LengthQuery, - salsa::plumbing::MemoizedStorage, - >; - - // Types from the query group, repeated for convenience. - type Group = HelloWorldStorage; - type GroupStorage = HelloWorldGroupStorage__; - type GroupKey = HelloWorldGroupKey__; - - // Given the storage for the entire group, extract - // the storage for just this query. Described when - // we talk about group storage. - fn query_storage( - group_storage: &HelloWorldGroupStorage__, - ) -> &std::sync::Arc { - &group_storage.length - } - - // Given the key for this query, construct the "group key" - // that situates it within the group. Described when - // we talk about group key. - fn group_key(key: Self::Key) -> Self::GroupKey { - HelloWorldGroupKey__::length(key) - } -} +{{#include ../../../components/salsa-macros/src/query_group.rs:Query_impl}} ``` Depending on the kind of query, we may also generate other impls, such as an @@ -217,27 +159,10 @@ impl of `salsa::plumbing::QueryFunction`, which defines the methods for executing the body of a query. This impl would then include a call to the user's actual function. -## Group key - -The "query key" is the inputs to the query, and identifies a particular query -instace: in our example, it is a value of type `()` (so there is only one -instance of the query), but typically it's some other type. The "group key" then -broadens that to include the identifier of the query within the group. So instead -of just `()` the group key would encode (e.g.) `Length(())` (the "length" query -applied to the `()` key). It is represented as an enum, which we generate, -with one variant per query: - ```rust,ignore -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -enum HelloWorldGroupKey__ { - input(()), - length(()), -} +{{#include ../../../components/salsa-macros/src/query_group.rs:QueryFunction_impl}} ``` -The `Query` trait that we saw earlier includes a method `group_key` for wrapping -the key for some individual query into the group key. - ## Group storage The "group storage" is the actual struct that contains all the hashtables and @@ -246,22 +171,25 @@ so forth for each query. The types of these are ultimately defined by the final database type: ```rust,ignore -struct HelloWorldGroupStorage__ { - input: >::Storage, - length: >::Storage, +struct HelloWorldGroupStorage__ { + input: ::Storage, } ``` -We also generate some impls: first is an impl of `Default` and the second is a -method `for_each_query` that simply iterates over each field and invokes a -method on it. This method is called by some of the code we generate for the -database in order to implement debugging methods that "sweep" over all the -queries. +We also generate some inherent methods. First, a `new` method that takes +the group index as a parameter and passes it along to each of the query +storage `new` methods: ```rust,ignore -impl HelloWorldGroupStorage__ { - fn for_each_query(&self, db: &DB, method: &mut dyn FnMut(...)) { - ... - } -} +{{#include ../../../components/salsa-macros/src/query_group.rs:group_storage_new}} ``` + +And then various methods that will dispatch from a `DatabaseKeyIndex` that +corresponds to this query group into the appropriate query within the group. +Each has a similar structure of matching on the query index and then delegating +to some method defined by the query storage: + +```rust,ignore +{{#include ../../../components/salsa-macros/src/query_group.rs:group_storage_methods}} +``` \ No newline at end of file diff --git a/components/salsa-macros/src/query_group.rs b/components/salsa-macros/src/query_group.rs index 77c35598..28c9173f 100644 --- a/components/salsa-macros/src/query_group.rs +++ b/components/salsa-macros/src/query_group.rs @@ -395,8 +395,7 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream } } - // Unsafe proof obligation: that our key/value are a part - // of the `GroupData`. + // ANCHOR:Query_impl impl salsa::Query for #qt { type Key = (#(#keys),*); @@ -416,6 +415,7 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream &group_storage.#fn_name } } + // ANCHOR_END:Query_impl }); // Implement the QueryFunction trait for queries which need it. @@ -447,6 +447,7 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream }; output.extend(quote_spanned! {span=> + // ANCHOR:QueryFunction_impl impl salsa::plumbing::QueryFunction for #qt { fn execute(db: &Self::DynDb, #key_pattern: ::Key) @@ -456,6 +457,7 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream #recover } + // ANCHOR_END:QueryFunction_impl }); } } @@ -495,6 +497,7 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream #storage_fields } + // ANCHOR:group_storage_new impl #group_storage { #trait_vis fn new(group_index: u16) -> Self { #group_storage { @@ -505,7 +508,9 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream } } } + // ANCHOR_END:group_storage_new + // ANCHOR:group_storage_methods impl #group_storage { #trait_vis fn fmt_index( &self, @@ -539,6 +544,7 @@ pub(crate) fn query_group(args: TokenStream, input: TokenStream) -> TokenStream #for_each_ops } } + // ANCHOR_END:group_storage_methods }); if std::env::var("SALSA_DUMP").is_ok() {