From c5d03d300b689fa09b75a82dad1b835cca51bb35 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 18 Jul 2020 09:40:47 +0900 Subject: [PATCH 1/6] Cause IndexInvalidDataError when search index is invalid --- plume-models/src/search/searcher.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index a1f4c813..2e509a0d 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -5,10 +5,11 @@ use crate::{ use chrono::Datelike; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use itertools::Itertools; -use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex}; +use std::{cmp, fs::create_dir_all, io, path::Path, sync::Mutex}; use tantivy::{ collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter, ReloadPolicy, Term, + TantivyError, }; use whatlang::{detect as detect_lang, Lang}; @@ -18,6 +19,7 @@ pub enum SearcherError { WriteLockAcquisitionError, IndexOpeningError, IndexEditionError, + InvalidIndexDataError, } pub struct Searcher { @@ -135,7 +137,19 @@ impl Searcher { .reader_builder() .reload_policy(ReloadPolicy::Manual) .try_into() - .map_err(|_| SearcherError::IndexCreationError)?, + .map_err(|e| { + if let TantivyError::IOError(err) = e { + let err: io::Error = err.into(); + if err.kind() == io::ErrorKind::InvalidData { + // Search index was created in older Tantivy format. + SearcherError::InvalidIndexDataError + } else { + SearcherError::IndexCreationError + } + } else { + SearcherError::IndexCreationError + } + })?, index, }) } From 50753b3a6590be65c62e0407e328ac33b655e709 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 18 Jul 2020 09:40:58 +0900 Subject: [PATCH 2/6] Recreate search index if its format is outdated --- src/main.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index e52fde7b..9478e807 100644 --- a/src/main.rs +++ b/src/main.rs @@ -98,8 +98,15 @@ Then try to restart Plume. } let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); // we want a fast exit here, so + let mut open_searcher = + UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); + if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { + UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers) + .expect("main: recreating search index error. Try backing up search index, removing it and running `plm search init`"); + open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); + } #[allow(clippy::match_wild_err_arm)] - let searcher = match UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers) { + let searcher = match open_searcher { Err(Error::Search(e)) => match e { SearcherError::WriteLockAcquisitionError => panic!( r#" From 5fc827c1c9489dd70db9c5471869f05e1f77ec93 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 18 Jul 2020 21:00:58 +0900 Subject: [PATCH 3/6] Re-initialize search index when recreating is failed --- src/main.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/main.rs diff --git a/src/main.rs b/src/main.rs old mode 100644 new mode 100755 index 9478e807..cbbc36d8 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ extern crate serde_json; #[macro_use] extern crate validator_derive; +use chrono::Utc; use clap::App; use diesel::r2d2::ConnectionManager; use plume_models::{ @@ -21,6 +22,8 @@ use plume_models::{ }; use rocket_csrf::CsrfFairingBuilder; use scheduled_thread_pool::ScheduledThreadPool; +use std::fs; +use std::path::Path; use std::process::exit; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -102,7 +105,25 @@ Then try to restart Plume. UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers) - .expect("main: recreating search index error. Try backing up search index, removing it and running `plm search init`"); + .or_else(|_| { + let current_path = Path::new(&CONFIG.search_index); + let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); + let backup_path = Path::new(&backup_path); + fs::rename(current_path, backup_path) + .expect("main: error on backing up search index directory for recreating"); + UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).and_then( + |searcher| { + if fs::remove_dir_all(backup_path).is_err() { + eprintln!( + "error on removing backup directory: {}. it remains", + backup_path.display() + ); + } + Ok(searcher) + }, + ) + }) + .expect("main: error on recreating search index in new index format"); open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); } #[allow(clippy::match_wild_err_arm)] From 484659fde255eebe82e046d1b716c2e191190361 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 18 Jul 2020 22:52:39 +0900 Subject: [PATCH 4/6] Run cargo fmt --- plume-models/plume.db-journal | Bin 0 -> 512 bytes plume-models/plume_tests.sqlite-journal | Bin 0 -> 512 bytes plume-models/src/search/searcher.rs | 3 +-- 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 plume-models/plume.db-journal create mode 100644 plume-models/plume_tests.sqlite-journal diff --git a/plume-models/plume.db-journal b/plume-models/plume.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..2a40935dd94b2dfbdf6d5bdb1b613cc126a25a58 GIT binary patch literal 512 bcmZQzK!6zQ2ksCa6Oa->NQ| Date: Sun, 19 Jul 2020 07:16:34 +0900 Subject: [PATCH 5/6] Add action user can take to error message --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index cbbc36d8..50116288 100755 --- a/src/main.rs +++ b/src/main.rs @@ -123,7 +123,7 @@ Then try to restart Plume. }, ) }) - .expect("main: error on recreating search index in new index format"); + .expect("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); } #[allow(clippy::match_wild_err_arm)] From 5e30bede40855878c2a4a6ff601f815440341b31 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sun, 19 Jul 2020 08:24:17 +0900 Subject: [PATCH 6/6] Don't care about needless return value of closures --- src/main.rs | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/main.rs b/src/main.rs index 50116288..4ae2f88f 100755 --- a/src/main.rs +++ b/src/main.rs @@ -104,26 +104,23 @@ Then try to restart Plume. let mut open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { - UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers) - .or_else(|_| { - let current_path = Path::new(&CONFIG.search_index); - let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); - let backup_path = Path::new(&backup_path); - fs::rename(current_path, backup_path) - .expect("main: error on backing up search index directory for recreating"); - UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).and_then( - |searcher| { - if fs::remove_dir_all(backup_path).is_err() { - eprintln!( - "error on removing backup directory: {}. it remains", - backup_path.display() - ); - } - Ok(searcher) - }, - ) - }) - .expect("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); + if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_err() { + let current_path = Path::new(&CONFIG.search_index); + let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); + let backup_path = Path::new(&backup_path); + fs::rename(current_path, backup_path) + .expect("main: error on backing up search index directory for recreating"); + if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_ok() { + if fs::remove_dir_all(backup_path).is_err() { + eprintln!( + "error on removing backup directory: {}. it remains", + backup_path.display() + ); + } + } else { + panic!("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); + } + } open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); } #[allow(clippy::match_wild_err_arm)]