Add a search engine into Plume (#324)

* Add search engine to the model Add a Tantivy based search engine to the model Implement most required functions for it * Implement indexing and plm subcommands Implement indexation on insert, update and delete Modify func args to get the indexer where required Add subcommand to initialize, refill and unlock search db * Move to a new threadpool engine allowing scheduling * Autocommit search index every half an hour * Implement front part of search Add default fields for search Add new routes and templates for search and result Implement FromFormValue for Page to reuse it on search result pagination Add optional query parameters to paginate template's macro Update to newer rocket_csrf, don't get csrf token on GET forms * Handle process termination to release lock Handle process termination Add tests to search * Add proper support for advanced search Add an advanced search form to /search, in template and route Modify Tantivy schema, add new tokenizer for some properties Create new String query parser Create Tantivy query AST from our own * Split search.rs, add comment and tests Split search.rs into multiple submodules Add comments and tests for Query Make user@domain be treated as one could assume
2018-12-02 17:37:51 +01:00
parent 9714bafded
commit 449641d158
29 changed files with 1613 additions and 143 deletions
@@ -0,0 +1,167 @@
+mod searcher;
+mod query;
+mod tokenizer;
+pub use self::searcher::*;
+pub use self::query::PlumeQuery as Query;
+
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::{Query, Searcher};
+    use std::env::temp_dir;
+    use diesel::Connection;
+
+    use plume_common::activity_pub::inbox::Deletable;
+    use plume_common::utils::random_hex;
+    use blogs::tests::fill_database;
+    use posts::{NewPost, Post};
+    use post_authors::*;
+    use safe_string::SafeString;
+    use tests::db;
+
+
+    pub(crate) fn get_searcher() -> Searcher {
+        let dir = temp_dir().join("plume-test");
+        if dir.exists() {
+            Searcher::open(&dir)
+        } else {
+            Searcher::create(&dir)
+        }.unwrap()
+    }
+
+    #[test]
+    fn get_first_token() {
+        let vector = vec![
+            ("+\"my token\" other", ("+\"my token\"", " other")),
+            ("-\"my token\" other", ("-\"my token\"", " other")),
+            (" \"my token\" other", ("\"my token\"", " other")),
+            ("\"my token\" other", ("\"my token\"", " other")),
+            ("+my token other", ("+my", " token other")),
+            ("-my token other", ("-my", " token other")),
+            (" my token other", ("my", " token other")),
+            ("my token other", ("my", " token other")),
+            ("+\"my token other", ("+\"my token other", "")),
+            ("-\"my token other", ("-\"my token other", "")),
+            (" \"my token other", ("\"my token other", "")),
+            ("\"my token other", ("\"my token other", "")),
+        ];
+        for (source, res) in vector {
+            assert_eq!(Query::get_first_token(source), res);
+        }
+    }
+
+    #[test]
+    fn from_str() {
+        let vector = vec![
+            ("", ""),
+            ("a query", "a query"),
+            ("\"a query\"", "\"a query\""),
+            ("+a -\"query\"", "+a -query"),
+            ("title:\"something\" a query", "a query title:something"),
+            ("-title:\"something\" a query", "a query -title:something"),
+            ("author:user@domain", "author:user@domain"),
+            ("-author:@user@domain", "-author:user@domain"),
+            ("before:2017-11-05 before:2018-01-01", "before:2017-11-05"),
+            ("after:2017-11-05 after:2018-01-01", "after:2018-01-01"),
+        ];
+        for (source, res) in vector {
+            assert_eq!(&Query::from_str(source).to_string(), res);
+            assert_eq!(Query::new().parse_query(source).to_string(), res);
+        }
+    }
+
+    #[test]
+    fn setters() {
+        let vector = vec![
+            ("something", "title:something"),
+            ("+something", "+title:something"),
+            ("-something", "-title:something"),
+            ("+\"something\"", "+title:something"),
+            ("+some thing", "+title:\"some thing\""),
+        ];
+        for (source, res) in vector {
+            assert_eq!(&Query::new().title(source, None).to_string(), res);
+        }
+
+        let vector = vec![
+            ("something", "author:something"),
+            ("+something", "+author:something"),
+            ("-something", "-author:something"),
+            ("+\"something\"", "+author:something"),
+            ("+@someone@somewhere", "+author:someone@somewhere"),
+        ];
+        for (source, res) in vector {
+            assert_eq!(&Query::new().author(source, None).to_string(), res);
+        }
+    }
+
+    #[test]
+    fn open() {
+        {get_searcher()};//make sure $tmp/plume-test-tantivy exist
+
+        let dir = temp_dir().join("plume-test");
+        Searcher::open(&dir).unwrap();
+    }
+
+    #[test]
+    fn create() {
+        let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
+
+        assert!(Searcher::open(&dir).is_err());
+        {Searcher::create(&dir).unwrap();}
+        Searcher::open(&dir).unwrap();//verify it's well created
+    }
+
+    #[test]
+    fn search() {
+        let conn = &db();
+        conn.test_transaction::<_, (), _>(|| {
+            let searcher = get_searcher();
+            let blog = &fill_database(conn)[0];
+            let author = &blog.list_authors(conn)[0];
+
+            let title = random_hex()[..8].to_owned();
+
+            let mut post = Post::insert(conn, NewPost {
+                blog_id: blog.id,
+                slug: title.clone(),
+                title: title.clone(),
+                content: SafeString::new(""),
+                published: true,
+                license: "CC-BY-SA".to_owned(),
+                ap_url: "".to_owned(),
+                creation_date: None,
+                subtitle: "".to_owned(),
+                source: "".to_owned(),
+                cover_id: None,
+            }, &searcher);
+            PostAuthor::insert(conn, NewPostAuthor {
+                post_id: post.id,
+                author_id: author.id,
+            });
+
+            searcher.commit();
+            assert_eq!(searcher.search_document(conn, Query::from_str(&title), (0,1))[0].id, post.id);
+
+            let newtitle = random_hex()[..8].to_owned();
+            post.title = newtitle.clone();
+            post.update(conn, &searcher);
+            searcher.commit();
+            assert_eq!(searcher.search_document(conn, Query::from_str(&newtitle), (0,1))[0].id, post.id);
+            assert!(searcher.search_document(conn, Query::from_str(&title), (0,1)).is_empty());
+
+            post.delete(&(conn, &searcher));
+            searcher.commit();
+            assert!(searcher.search_document(conn, Query::from_str(&newtitle), (0,1)).is_empty());
+
+            Ok(())
+        });
+    }
+
+    #[test]
+    fn drop_writer() {
+        let searcher = get_searcher();
+        searcher.drop_writer();
+        get_searcher();
+    }
+}