Switchable tokenizer (#776)

* [REFACTORING]Rename whitespace_tokenizer to tag_tokenizer for
registration

Name representing its purpose is preferred.

* Add lindera-tantivy to plume-model's dependencies

* Install lindera-tantivy

* Add SearchTokenizerConfig struct

* Add search tokenizers to config option

* Use CONFIG for tokenizers

* Use enum to hold tokenizer config instead of initializing on config phase

* Use guard instead of duplicate default values

* Use as_deref() instead of guard

* Move SearchTokenizer from plume-models to plume-models::search::tokenizer

* Rename SearchTokenizer to TokenizerKind

* Define SearchTokenierConfig::determine_tokenizer()

* Use determine_tokenizer in SearchTokenizerConfig::init()

* Pass tokenizer config to Searcher methods

* Add LowerCase filter to Lindera tokenizer

* Add test for Lindera tokenizer

* Define SEARCH_LANG env to specify tokenizers set

* Run cargo fmt

* Make Lindera tokenizer optional

* Fix typos

plume-cli/Cargo.toml

@@ -23,3 +23,4 @@ path = "../plume-models"
 [features]
 postgres = ["plume-models/postgres", "diesel/postgres"]
 sqlite = ["plume-models/sqlite", "diesel/sqlite"]
+search-lindera = ["plume-models/search-lindera"]

plume-cli/src/search.rs

@@ -82,7 +82,7 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
         }
     };
     if can_do || force {
-        let searcher = Searcher::create(&path).unwrap();
+        let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap();
         refill(args, conn, Some(searcher));
     } else {
         eprintln!(
@@ -98,7 +98,8 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
         Some(path) => Path::new(path).join("search_index"),
         None => Path::new(&CONFIG.search_index).to_path_buf(),
     };
-    let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
+    let searcher =
+        searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap());
 
     searcher.fill(conn).expect("Couldn't import post");
     println!("Commiting result");