Switchable tokenizer (#776)

* [REFACTORING]Rename whitespace_tokenizer to tag_tokenizer for
registration

Name representing its purpose is preferred.

* Add lindera-tantivy to plume-model's dependencies

* Install lindera-tantivy

* Add SearchTokenizerConfig struct

* Add search tokenizers to config option

* Use CONFIG for tokenizers

* Use enum to hold tokenizer config instead of initializing on config phase

* Use guard instead of duplicate default values

* Use as_deref() instead of guard

* Move SearchTokenizer from plume-models to plume-models::search::tokenizer

* Rename SearchTokenizer to TokenizerKind

* Define SearchTokenierConfig::determine_tokenizer()

* Use determine_tokenizer in SearchTokenizerConfig::init()

* Pass tokenizer config to Searcher methods

* Add LowerCase filter to Lindera tokenizer

* Add test for Lindera tokenizer

* Define SEARCH_LANG env to specify tokenizers set

* Run cargo fmt

* Make Lindera tokenizer optional

* Fix typos

plume-models/src/users.rs

@@ -1026,6 +1026,7 @@ impl NewUser {
 pub(crate) mod tests {
     use super::*;
     use crate::{
+        config::CONFIG,
         instance::{tests as instance_tests, Instance},
         search::tests::get_searcher,
         tests::{db, rockets},
@@ -1122,7 +1123,9 @@ pub(crate) mod tests {
             let inserted = fill_database(conn);
 
             assert!(User::get(conn, inserted[0].id).is_ok());
-            inserted[0].delete(conn, &get_searcher()).unwrap();
+            inserted[0]
+                .delete(conn, &get_searcher(&CONFIG.search_tokenizers))
+                .unwrap();
             assert!(User::get(conn, inserted[0].id).is_err());
             Ok(())
         });