Add a search engine into Plume (#324)

* Add search engine to the model

Add a Tantivy based search engine to the model
Implement most required functions for it

* Implement indexing and plm subcommands

Implement indexation on insert, update and delete
Modify func args to get the indexer where required
Add subcommand to initialize, refill and unlock search db

* Move to a new threadpool engine allowing scheduling

* Autocommit search index every half an hour

* Implement front part of search

Add default fields for search
Add new routes and templates for search and result
Implement FromFormValue for Page to reuse it on search result pagination
Add optional query parameters to paginate template's macro
Update to newer rocket_csrf, don't get csrf token on GET forms

* Handle process termination to release lock

Handle process termination
Add tests to search

* Add proper support for advanced search

Add an advanced search form to /search, in template and route
Modify Tantivy schema, add new tokenizer for some properties
Create new String query parser
Create Tantivy query AST from our own

* Split search.rs, add comment and tests

Split search.rs into multiple submodules
Add comments and tests for Query
Make user@domain be treated as one could assume

plume-cli/src/main.rs

@@ -11,6 +11,7 @@ use plume_models::{DATABASE_URL, Connection as Conn};
 
 mod instance;
 mod users;
+mod search;
 
 fn main() {
     let mut app = App::new("Plume CLI")
@@ -18,7 +19,8 @@ fn main() {
         .version(env!("CARGO_PKG_VERSION"))
         .about("Collection of tools to manage your Plume instance.")
         .subcommand(instance::command())
-        .subcommand(users::command());
+        .subcommand(users::command())
+        .subcommand(search::command());
     let matches = app.clone().get_matches();
 
     dotenv::dotenv().ok();
@@ -27,6 +29,7 @@ fn main() {
     match matches.subcommand() {
         ("instance", Some(args)) => instance::run(args, &conn.expect("Couldn't connect to the database.")),
         ("users", Some(args)) => users::run(args, &conn.expect("Couldn't connect to the database.")),
+        ("search", Some(args)) => search::run(args, &conn.expect("Couldn't connect to the database.")),
         _ => app.print_help().expect("Couldn't print help")
     };
 }

plume-cli/src/search.rs

@@ -0,0 +1,111 @@
+use clap::{Arg, ArgMatches, App, SubCommand};
+use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
+
+use std::fs::{read_dir, remove_file};
+use std::io::ErrorKind;
+use std::path::Path;
+use plume_models::{
+    Connection,
+    posts::Post,
+    schema::posts,
+    search::Searcher,
+};
+
+pub fn command<'a, 'b>() -> App<'a, 'b> {
+    SubCommand::with_name("search")
+        .about("Manage search index")
+        .subcommand(SubCommand::with_name("init")
+            .arg(Arg::with_name("path")
+                .short("p")
+                .long("path")
+                .takes_value(true)
+                .required(true)
+                .help("Path to Plume's working directory"))
+            .arg(Arg::with_name("force")
+                .short("f")
+                .long("force")
+                .help("Ignore already using directory")
+            ).about("Initialize Plume's internal search engine"))
+        .subcommand(SubCommand::with_name("refill")
+            .arg(Arg::with_name("path")
+                .short("p")
+                .long("path")
+                .takes_value(true)
+                .required(true)
+                .help("Path to Plume's working directory")
+            ).about("Regenerate Plume's search index"))
+        .subcommand(SubCommand::with_name("unlock")
+            .arg(Arg::with_name("path")
+                .short("p")
+                .long("path")
+                .takes_value(true)
+                .required(true)
+                .help("Path to Plume's working directory")
+            ).about("Release lock on search directory"))
+}
+
+pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) {
+    let conn = conn;
+    match args.subcommand() {
+        ("init", Some(x)) => init(x, conn),
+        ("refill", Some(x)) => refill(x, conn, None),
+        ("unlock", Some(x)) => unlock(x),
+        _ => println!("Unknown subcommand"),
+    }
+}
+
+fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
+    let path = args.value_of("path").unwrap();
+    let force = args.is_present("force");
+    let path = Path::new(path).join("search_index");
+
+    let can_do = match read_dir(path.clone()) { // try to read the directory specified
+        Ok(mut contents) => {
+            if contents.next().is_none()  {
+                true
+            } else {
+                false
+            }
+        },
+        Err(e) => if e.kind() == ErrorKind::NotFound {
+            true
+        } else {
+            panic!("Error while initialising search index : {}", e);
+        }
+    };
+    if can_do || force {
+        let searcher = Searcher::create(&path).unwrap();
+        refill(args, conn, Some(searcher));
+    } else {
+        eprintln!("Can't create new index, {} exist and is not empty", path.to_str().unwrap());
+    }
+}
+
+fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searcher>) {
+    let path = args.value_of("path").unwrap();
+    let path = Path::new(path).join("search_index");
+    let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
+
+    let posts = posts::table
+        .filter(posts::published.eq(true))
+        .load::<Post>(conn)
+        .expect("Post::get_recents: loading error");
+
+    let len = posts.len();
+    for (i,post) in posts.iter().enumerate() {
+        println!("Importing {}/{} : {}", i+1, len, post.title);
+        searcher.update_document(conn, &post);
+    }
+    println!("Commiting result");
+    searcher.commit();
+}
+
+
+fn unlock<'a>(args: &ArgMatches<'a>) {
+    let path = args.value_of("path").unwrap();
+    let path = Path::new(path).join("search_index/.tantivy-indexer.lock");
+
+    remove_file(path).unwrap();
+}
+
+