Add a search engine into Plume (#324)

* Add search engine to the model

Add a Tantivy based search engine to the model
Implement most required functions for it

* Implement indexing and plm subcommands

Implement indexation on insert, update and delete
Modify func args to get the indexer where required
Add subcommand to initialize, refill and unlock search db

* Move to a new threadpool engine allowing scheduling

* Autocommit search index every half an hour

* Implement front part of search

Add default fields for search
Add new routes and templates for search and result
Implement FromFormValue for Page to reuse it on search result pagination
Add optional query parameters to paginate template's macro
Update to newer rocket_csrf, don't get csrf token on GET forms

* Handle process termination to release lock

Handle process termination
Add tests to search

* Add proper support for advanced search

Add an advanced search form to /search, in template and route
Modify Tantivy schema, add new tokenizer for some properties
Create new String query parser
Create Tantivy query AST from our own

* Split search.rs, add comment and tests

Split search.rs into multiple submodules
Add comments and tests for Query
Make user@domain be treated as one could assume

src/main.rs

@@ -6,6 +6,7 @@ extern crate atom_syndication;
 extern crate canapi;
 extern crate chrono;
 extern crate colored;
+extern crate ctrlc;
 extern crate diesel;
 extern crate dotenv;
 extern crate failure;
@@ -13,6 +14,7 @@ extern crate gettextrs;
 extern crate guid_create;
 extern crate heck;
 extern crate multipart;
+extern crate num_cpus;
 extern crate plume_api;
 extern crate plume_common;
 extern crate plume_models;
@@ -22,6 +24,7 @@ extern crate rocket_contrib;
 extern crate rocket_csrf;
 extern crate rocket_i18n;
 extern crate rpassword;
+extern crate scheduled_thread_pool;
 extern crate serde;
 #[macro_use]
 extern crate serde_derive;
@@ -32,20 +35,24 @@ extern crate validator;
 #[macro_use]
 extern crate validator_derive;
 extern crate webfinger;
-extern crate workerpool;
 
 use diesel::r2d2::ConnectionManager;
 use rocket::State;
 use rocket_contrib::Template;
 use rocket_csrf::CsrfFairingBuilder;
-use plume_models::{DATABASE_URL, Connection, db_conn::DbPool};
-use workerpool::{Pool, thunk::ThunkWorker};
+use plume_models::{DATABASE_URL, Connection,
+    db_conn::DbPool, search::Searcher as UnmanagedSearcher};
+use scheduled_thread_pool::ScheduledThreadPool;
+use std::process::exit;
+use std::sync::Arc;
+use std::time::Duration;
 
 mod api;
 mod inbox;
 mod routes;
 
-type Worker<'a> = State<'a, Pool<ThunkWorker<()>>>;
+type Worker<'a> = State<'a, ScheduledThreadPool>;
+type Searcher<'a> = State<'a, Arc<UnmanagedSearcher>>;
 
 /// Initializes a database pool.
 fn init_pool() -> Option<DbPool> {
@@ -56,7 +63,21 @@ fn init_pool() -> Option<DbPool> {
 }
 
 fn main() {
-    let pool = init_pool().expect("main: database pool initialization error");
+
+    let dbpool = init_pool().expect("main: database pool initialization error");
+    let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
+    let searcher = Arc::new(UnmanagedSearcher::open(&"search_index").unwrap());
+
+    let commiter = searcher.clone();
+    workpool.execute_with_fixed_delay(Duration::from_secs(5), Duration::from_secs(60*30), move || commiter.commit());
+
+    let search_unlocker = searcher.clone();
+    ctrlc::set_handler(move || {
+        search_unlocker.drop_writer();
+        exit(0);
+    }).expect("Error setting Ctrl-c handler");
+
+
     rocket::ignite()
         .mount("/", routes![
             routes::blogs::paginated_details,
@@ -119,6 +140,9 @@ fn main() {
             routes::reshares::create,
             routes::reshares::create_auth,
 
+            routes::search::index,
+            routes::search::query,
+
             routes::session::new,
             routes::session::new_message,
             routes::session::create,
@@ -167,8 +191,9 @@ fn main() {
             routes::errors::not_found,
             routes::errors::server_error
         ])
-        .manage(pool)
-        .manage(Pool::<ThunkWorker<()>>::new(4))
+        .manage(dbpool)
+        .manage(workpool)
+        .manage(searcher)
         .attach(Template::custom(|engines| {
             rocket_i18n::tera(&mut engines.tera);
         }))