Switchable tokenizer (#776)

* [REFACTORING]Rename whitespace_tokenizer to tag_tokenizer for
registration

Name representing its purpose is preferred.

* Add lindera-tantivy to plume-model's dependencies

* Install lindera-tantivy

* Add SearchTokenizerConfig struct

* Add search tokenizers to config option

* Use CONFIG for tokenizers

* Use enum to hold tokenizer config instead of initializing on config phase

* Use guard instead of duplicate default values

* Use as_deref() instead of guard

* Move SearchTokenizer from plume-models to plume-models::search::tokenizer

* Rename SearchTokenizer to TokenizerKind

* Define SearchTokenierConfig::determine_tokenizer()

* Use determine_tokenizer in SearchTokenizerConfig::init()

* Pass tokenizer config to Searcher methods

* Add LowerCase filter to Lindera tokenizer

* Add test for Lindera tokenizer

* Define SEARCH_LANG env to specify tokenizers set

* Run cargo fmt

* Make Lindera tokenizer optional

* Fix typos

plume-models/src/search/tokenizer.rs

@@ -1,5 +1,34 @@
+#[cfg(feature = "search-lindera")]
+use lindera_tantivy::tokenizer::LinderaTokenizer;
 use std::str::CharIndices;
-use tantivy::tokenizer::{BoxTokenStream, Token, TokenStream, Tokenizer};
+use tantivy::tokenizer::*;
+
+#[derive(Clone, Copy)]
+pub enum TokenizerKind {
+    Simple,
+    Ngram,
+    Whitespace,
+    #[cfg(feature = "search-lindera")]
+    Lindera,
+}
+
+impl From<TokenizerKind> for TextAnalyzer {
+    fn from(tokenizer: TokenizerKind) -> TextAnalyzer {
+        use TokenizerKind::*;
+
+        match tokenizer {
+            Simple => TextAnalyzer::from(SimpleTokenizer)
+                .filter(RemoveLongFilter::limit(40))
+                .filter(LowerCaser),
+            Ngram => TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser),
+            Whitespace => TextAnalyzer::from(WhitespaceTokenizer).filter(LowerCaser),
+            #[cfg(feature = "search-lindera")]
+            Lindera => {
+                TextAnalyzer::from(LinderaTokenizer::new("decompose", "")).filter(LowerCaser)
+            }
+        }
+    }
+}
 
 /// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
 /// but not splitting on punctuation