Upgrade Tantivy to v0.12.0 (#771)

* Upgrade Tantivy to 0.12.0

* Follow Tantivy Tokenizer's new type definition

* Wrap tokenizers with TextAnalyzer to use filter methods

* Replace async IndexWriter::garbage_collect_files with sync functions

* Update Cargo.toml

plume-models/src/search/searcher.rs

@@ -71,13 +71,15 @@ impl Searcher {
     }
 
     pub fn create(path: &dyn AsRef<Path>) -> Result<Self> {
-        let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
+        let whitespace_tokenizer =
+            TextAnalyzer::from(tokenizer::WhitespaceTokenizer).filter(LowerCaser);
 
-        let content_tokenizer = SimpleTokenizer
+        let content_tokenizer = TextAnalyzer::from(SimpleTokenizer)
             .filter(RemoveLongFilter::limit(40))
             .filter(LowerCaser);
 
-        let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
+        let property_tokenizer =
+            TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser);
 
         let schema = Self::schema();
 
@@ -110,15 +112,17 @@ impl Searcher {
     }
 
     pub fn open(path: &dyn AsRef<Path>) -> Result<Self> {
-        let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
+        let whitespace_tokenizer =
+            TextAnalyzer::from(tokenizer::WhitespaceTokenizer).filter(LowerCaser);
 
-        let content_tokenizer = SimpleTokenizer
+        let content_tokenizer = TextAnalyzer::from(SimpleTokenizer)
             .filter(RemoveLongFilter::limit(40))
             .filter(LowerCaser);
 
-        let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
+        let property_tokenizer =
+            TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser);
 
-        let index =
+        let mut index =
             Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
                 .map_err(|_| SearcherError::IndexOpeningError)?;
 
@@ -128,12 +132,27 @@ impl Searcher {
             tokenizer_manager.register("content_tokenizer", content_tokenizer);
             tokenizer_manager.register("property_tokenizer", property_tokenizer);
         } //to please the borrow checker
-        let mut writer = index
+        let writer = index
             .writer(50_000_000)
             .map_err(|_| SearcherError::WriteLockAcquisitionError)?;
-        writer
-            .garbage_collect_files()
+
+        // Since Tantivy v0.12.0, IndexWriter::garbage_collect_files() returns Future.
+        // To avoid conflict with Plume async project, we don't introduce async now.
+        // After async is introduced to Plume, we can use garbage_collect_files() again.
+        // Algorithm stolen from Tantivy's SegmentUpdater::list_files()
+        use std::collections::HashSet;
+        use std::path::PathBuf;
+        let mut files: HashSet<PathBuf> = index
+            .list_all_segment_metas()
+            .into_iter()
+            .flat_map(|segment_meta| segment_meta.list_files())
+            .collect();
+        files.insert(Path::new("meta.json").to_path_buf());
+        index
+            .directory_mut()
+            .garbage_collect(|| files)
             .map_err(|_| SearcherError::IndexEditionError)?;
+
         Ok(Self {
             writer: Mutex::new(Some(writer)),
             reader: index

plume-models/src/search/tokenizer.rs

@@ -1,5 +1,5 @@
 use std::str::CharIndices;
-use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
+use tantivy::tokenizer::{BoxTokenStream, Token, TokenStream, Tokenizer};
 
 /// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
 /// but not splitting on punctuation
@@ -12,15 +12,13 @@ pub struct WhitespaceTokenStream<'a> {
     token: Token,
 }
 
-impl<'a> Tokenizer<'a> for WhitespaceTokenizer {
-    type TokenStreamImpl = WhitespaceTokenStream<'a>;
-
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
-        WhitespaceTokenStream {
+impl Tokenizer for WhitespaceTokenizer {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(WhitespaceTokenStream {
             text,
             chars: text.char_indices(),
             token: Token::default(),
-        }
+        })
     }
 }
 impl<'a> WhitespaceTokenStream<'a> {