aitzol/Plume
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Run 'cargo fmt' to format code (#489)

Browse Source
This commit is contained in:
Atul Bhosale
2019-03-20 22:26:17 +05:30
committed by Baptiste Gelez
parent 732f514da7
commit b945d1f602
58 changed files with 3160 additions and 2195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plume-models/src/search/mod.rs
+54 -32
View File
@@ -1,33 +1,32 @@
mod searcher;
mod query;
mod searcher;
mod tokenizer;
pub use self::searcher::*;
pub use self::query::PlumeQuery as Query;
pub use self::searcher::*;
#[cfg(test)]
pub(crate) mod tests {
use super::{Query, Searcher};
use diesel::Connection;
use std::env::temp_dir;
use std::str::FromStr;
use diesel::Connection;
use blogs::tests::fill_database;
use plume_common::activity_pub::inbox::Deletable;
use plume_common::utils::random_hex;
use blogs::tests::fill_database;
use posts::{NewPost, Post};
use post_authors::*;
use posts::{NewPost, Post};
use safe_string::SafeString;
use tests::db;
pub(crate) fn get_searcher() -> Searcher {
let dir = temp_dir().join("plume-test");
if dir.exists() {
Searcher::open(&dir)
} else {
Searcher::create(&dir)
}.unwrap()
}
.unwrap()
}
#[test]
@@ -98,7 +97,9 @@ pub(crate) mod tests {
#[test]
fn open() {
{get_searcher()};//make sure $tmp/plume-test-tantivy exist
{
get_searcher()
}; //make sure $tmp/plume-test-tantivy exist
let dir = temp_dir().join("plume-test");
Searcher::open(&dir).unwrap();
@@ -109,8 +110,10 @@ pub(crate) mod tests {
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
assert!(Searcher::open(&dir).is_err());
{Searcher::create(&dir).unwrap();}
Searcher::open(&dir).unwrap();//verify it's well created
{
Searcher::create(&dir).unwrap();
}
Searcher::open(&dir).unwrap(); //verify it's well created
}
#[test]
@@ -123,37 +126,56 @@ pub(crate) mod tests {
let title = random_hex()[..8].to_owned();
let mut post = Post::insert(conn, NewPost {
blog_id: blog.id,
slug: title.clone(),
title: title.clone(),
content: SafeString::new(""),
published: true,
license: "CC-BY-SA".to_owned(),
ap_url: "".to_owned(),
creation_date: None,
subtitle: "".to_owned(),
source: "".to_owned(),
cover_id: None,
}, &searcher).unwrap();
PostAuthor::insert(conn, NewPostAuthor {
post_id: post.id,
author_id: author.id,
}).unwrap();
let mut post = Post::insert(
conn,
NewPost {
blog_id: blog.id,
slug: title.clone(),
title: title.clone(),
content: SafeString::new(""),
published: true,
license: "CC-BY-SA".to_owned(),
ap_url: "".to_owned(),
creation_date: None,
subtitle: "".to_owned(),
source: "".to_owned(),
cover_id: None,
},
&searcher,
)
.unwrap();
PostAuthor::insert(
conn,
NewPostAuthor {
post_id: post.id,
author_id: author.id,
},
)
.unwrap();
searcher.commit();
assert_eq!(searcher.search_document(conn, Query::from_str(&title).unwrap(), (0,1))[0].id, post.id);
assert_eq!(
searcher.search_document(conn, Query::from_str(&title).unwrap(), (0, 1))[0].id,
post.id
);
let newtitle = random_hex()[..8].to_owned();
post.title = newtitle.clone();
post.update(conn, &searcher).unwrap();
searcher.commit();
assert_eq!(searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0,1))[0].id, post.id);
assert!(searcher.search_document(conn, Query::from_str(&title).unwrap(), (0,1)).is_empty());
assert_eq!(
searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))[0].id,
post.id
);
assert!(searcher
.search_document(conn, Query::from_str(&title).unwrap(), (0, 1))
.is_empty());
post.delete(&(conn, &searcher)).unwrap();
searcher.commit();
assert!(searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0,1)).is_empty());
assert!(searcher
.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))
.is_empty());
Ok(())
});
plume-models/src/search/query.rs
+92 -52
View File
@@ -1,8 +1,7 @@
use chrono::{Datelike, naive::NaiveDate, offset::Utc};
use tantivy::{query::*, schema::*, Term};
use std::{cmp,ops::Bound};
use chrono::{naive::NaiveDate, offset::Utc, Datelike};
use search::searcher::Searcher;
use std::{cmp, ops::Bound};
use tantivy::{query::*, schema::*, Term};
//Generate functions for advanced search
macro_rules! gen_func {
@@ -142,13 +141,11 @@ pub struct PlumeQuery {
}
impl PlumeQuery {
/// Create a new empty Query
pub fn new() -> Self {
Default::default()
}
/// Parse a query string into this Query
pub fn parse_query(&mut self, query: &str) -> &mut Self {
self.from_str_req(&query.trim())
@@ -160,9 +157,11 @@ impl PlumeQuery {
gen_to_query!(self, result; normal: title, subtitle, content, tag;
oneoff: instance, author, blog, lang, license);
for (occur, token) in self.text { // text entries need to be added as multiple Terms
for (occur, token) in self.text {
// text entries need to be added as multiple Terms
match occur {
Occur::Must => { // a Must mean this must be in one of title subtitle or content, not in all 3
Occur::Must => {
// a Must mean this must be in one of title subtitle or content, not in all 3
let subresult = vec![
(Occur::Should, Self::token_to_query(&token, "title")),
(Occur::Should, Self::token_to_query(&token, "subtitle")),
@@ -170,20 +169,26 @@ impl PlumeQuery {
];
result.push((Occur::Must, Box::new(BooleanQuery::from(subresult))));
},
}
occur => {
result.push((occur, Self::token_to_query(&token, "title")));
result.push((occur, Self::token_to_query(&token, "subtitle")));
result.push((occur, Self::token_to_query(&token, "content")));
},
}
}
}
if self.before.is_some() || self.after.is_some() { // if at least one range bound is provided
let after = self.after.unwrap_or_else(|| i64::from(NaiveDate::from_ymd(2000, 1, 1).num_days_from_ce()));
let before = self.before.unwrap_or_else(|| i64::from(Utc::today().num_days_from_ce()));
if self.before.is_some() || self.after.is_some() {
// if at least one range bound is provided
let after = self
.after
.unwrap_or_else(|| i64::from(NaiveDate::from_ymd(2000, 1, 1).num_days_from_ce()));
let before = self
.before
.unwrap_or_else(|| i64::from(Utc::today().num_days_from_ce()));
let field = Searcher::schema().get_field("creation_date").unwrap();
let range = RangeQuery::new_i64_bounds(field, Bound::Included(after), Bound::Included(before));
let range =
RangeQuery::new_i64_bounds(field, Bound::Included(after), Bound::Included(before));
result.push((Occur::Must, Box::new(range)));
}
@@ -195,14 +200,18 @@ impl PlumeQuery {
// documents newer than the provided date will be ignored
pub fn before<D: Datelike>(&mut self, date: &D) -> &mut Self {
let before = self.before.unwrap_or_else(|| i64::from(Utc::today().num_days_from_ce()));
let before = self
.before
.unwrap_or_else(|| i64::from(Utc::today().num_days_from_ce()));
self.before = Some(cmp::min(before, i64::from(date.num_days_from_ce())));
self
}
// documents older than the provided date will be ignored
pub fn after<D: Datelike>(&mut self, date: &D) -> &mut Self {
let after = self.after.unwrap_or_else(|| i64::from(NaiveDate::from_ymd(2000, 1, 1).num_days_from_ce()));
let after = self
.after
.unwrap_or_else(|| i64::from(NaiveDate::from_ymd(2000, 1, 1).num_days_from_ce()));
self.after = Some(cmp::max(after, i64::from(date.num_days_from_ce())));
self
}
@@ -212,18 +221,22 @@ impl PlumeQuery {
query = query.trim();
if query.is_empty() {
("", "")
} else if query.get(0..1).map(|v| v=="\"").unwrap_or(false) {
if let Some(index) = query[1..].find('"') {
query.split_at(index+2)
} else {
(query, "")
}
} else if query.get(0..2).map(|v| v=="+\"" || v=="-\"").unwrap_or(false) {
if let Some(index) = query[2..].find('"') {
query.split_at(index+3)
} else {
(query, "")
}
} else if query.get(0..1).map(|v| v == "\"").unwrap_or(false) {
if let Some(index) = query[1..].find('"') {
query.split_at(index + 2)
} else {
(query, "")
}
} else if query
.get(0..2)
.map(|v| v == "+\"" || v == "-\"")
.unwrap_or(false)
{
if let Some(index) = query[2..].find('"') {
query.split_at(index + 3)
} else {
(query, "")
}
} else if let Some(index) = query.find(' ') {
query.split_at(index)
} else {
@@ -247,13 +260,13 @@ impl PlumeQuery {
fn from_str_req(&mut self, mut query: &str) -> &mut Self {
query = query.trim_left();
if query.is_empty() {
return self
return self;
}
let occur = if query.get(0..1).map(|v| v=="+").unwrap_or(false) {
let occur = if query.get(0..1).map(|v| v == "+").unwrap_or(false) {
query = &query[1..];
Occur::Must
} else if query.get(0..1).map(|v| v=="-").unwrap_or(false) {
} else if query.get(0..1).map(|v| v == "-").unwrap_or(false) {
query = &query[1..];
Occur::MustNot
} else {
@@ -270,31 +283,59 @@ impl PlumeQuery {
let token = token.to_lowercase();
let token = token.as_str();
let field = Searcher::schema().get_field(field_name).unwrap();
if token.contains('@') && (field_name=="author" || field_name=="blog") {
if token.contains('@') && (field_name == "author" || field_name == "blog") {
let pos = token.find('@').unwrap();
let user_term = Term::from_field_text(field, &token[..pos]);
let instance_term = Term::from_field_text(Searcher::schema().get_field("instance").unwrap(), &token[pos+1..]);
let instance_term = Term::from_field_text(
Searcher::schema().get_field("instance").unwrap(),
&token[pos + 1..],
);
Box::new(BooleanQuery::from(vec![
(Occur::Must, Box::new(TermQuery::new(user_term, if field_name=="author" { IndexRecordOption::Basic }
else { IndexRecordOption::WithFreqsAndPositions }
)) as Box<dyn Query + 'static>),
(Occur::Must, Box::new(TermQuery::new(instance_term, IndexRecordOption::Basic))),
(
Occur::Must,
Box::new(TermQuery::new(
user_term,
if field_name == "author" {
IndexRecordOption::Basic
} else {
IndexRecordOption::WithFreqsAndPositions
},
)) as Box<dyn Query + 'static>,
),
(
Occur::Must,
Box::new(TermQuery::new(instance_term, IndexRecordOption::Basic)),
),
]))
} else if token.contains(' ') { // phrase query
} else if token.contains(' ') {
// phrase query
match field_name {
"instance" | "author" | "tag" => // phrase query are not available on these fields, treat it as multiple Term queries
Box::new(BooleanQuery::from(token.split_whitespace()
.map(|token| {
let term = Term::from_field_text(field, token);
(Occur::Should, Box::new(TermQuery::new(term, IndexRecordOption::Basic))
as Box<dyn Query + 'static>)
})
.collect::<Vec<_>>())),
_ => Box::new(PhraseQuery::new(token.split_whitespace()
.map(|token| Term::from_field_text(field, token))
.collect()))
"instance" | "author" | "tag" =>
// phrase query are not available on these fields, treat it as multiple Term queries
{
Box::new(BooleanQuery::from(
token
.split_whitespace()
.map(|token| {
let term = Term::from_field_text(field, token);
(
Occur::Should,
Box::new(TermQuery::new(term, IndexRecordOption::Basic))
as Box<dyn Query + 'static>,
)
})
.collect::<Vec<_>>(),
))
}
_ => Box::new(PhraseQuery::new(
token
.split_whitespace()
.map(|token| Term::from_field_text(field, token))
.collect(),
)),
}
} else { // Term Query
} else {
// Term Query
let term = Term::from_field_text(field, token);
let index_option = match field_name {
"instance" | "author" | "tag" => IndexRecordOption::Basic,
@@ -306,7 +347,6 @@ impl PlumeQuery {
}
impl std::str::FromStr for PlumeQuery {
type Err = !;
/// Create a new Query from &str
@@ -340,7 +380,7 @@ impl ToString for PlumeQuery {
instance, author, blog, lang, license;
date: before, after);
result.pop();// remove trailing ' '
result.pop(); // remove trailing ' '
result
}
}
plume-models/src/search/searcher.rs
+54 -37
View File
@@ -5,15 +5,14 @@ use Connection;
use chrono::Datelike;
use itertools::Itertools;
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use tantivy::{
collector::TopDocs, directory::MmapDirectory,
schema::*, tokenizer::*, Index, IndexWriter, Term
collector::TopDocs, directory::MmapDirectory, schema::*, tokenizer::*, Index, IndexWriter, Term,
};
use whatlang::{detect as detect_lang, Lang};
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use search::query::PlumeQuery;
use super::tokenizer;
use search::query::PlumeQuery;
use Result;
#[derive(Debug)]
@@ -31,20 +30,23 @@ pub struct Searcher {
impl Searcher {
pub fn schema() -> Schema {
let tag_indexing = TextOptions::default()
.set_indexing_options(TextFieldIndexing::default()
.set_tokenizer("whitespace_tokenizer")
.set_index_option(IndexRecordOption::Basic));
let tag_indexing = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer("whitespace_tokenizer")
.set_index_option(IndexRecordOption::Basic),
);
let content_indexing = TextOptions::default()
.set_indexing_options(TextFieldIndexing::default()
.set_tokenizer("content_tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions));
let content_indexing = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer("content_tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
);
let property_indexing = TextOptions::default()
.set_indexing_options(TextFieldIndexing::default()
.set_tokenizer("property_tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions));
let property_indexing = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer("property_tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
);
let mut schema_builder = SchemaBuilder::default();
@@ -66,56 +68,65 @@ impl Searcher {
schema_builder.build()
}
pub fn create(path: &AsRef<Path>) -> Result<Self> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer
.filter(LowerCaser);
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false)
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
let schema = Self::schema();
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
let index = Index::create(MmapDirectory::open(path).map_err(|_| SearcherError::IndexCreationError)?, schema).map_err(|_| SearcherError::IndexCreationError)?;
let index = Index::create(
MmapDirectory::open(path).map_err(|_| SearcherError::IndexCreationError)?,
schema,
)
.map_err(|_| SearcherError::IndexCreationError)?;
{
let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer);
}//to please the borrow checker
} //to please the borrow checker
Ok(Self {
writer: Mutex::new(Some(index.writer(50_000_000).map_err(|_| SearcherError::WriteLockAcquisitionError)?)),
index
writer: Mutex::new(Some(
index
.writer(50_000_000)
.map_err(|_| SearcherError::WriteLockAcquisitionError)?,
)),
index,
})
}
pub fn open(path: &AsRef<Path>) -> Result<Self> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer
.filter(LowerCaser);
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false)
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
let index = Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?).map_err(|_| SearcherError::IndexOpeningError)?;
let index =
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
.map_err(|_| SearcherError::IndexOpeningError)?;
{
let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer);
}//to please the borrow checker
let mut writer = index.writer(50_000_000).map_err(|_| SearcherError::WriteLockAcquisitionError)?;
writer.garbage_collect_files().map_err(|_| SearcherError::IndexEditionError)?;
} //to please the borrow checker
let mut writer = index
.writer(50_000_000)
.map_err(|_| SearcherError::WriteLockAcquisitionError)?;
writer
.garbage_collect_files()
.map_err(|_| SearcherError::IndexEditionError)?;
Ok(Self {
writer: Mutex::new(Some(writer)),
index,
@@ -173,18 +184,24 @@ impl Searcher {
self.add_document(conn, post)
}
pub fn search_document(&self, conn: &Connection, query: PlumeQuery, (min, max): (i32, i32)) -> Vec<Post>{
pub fn search_document(
&self,
conn: &Connection,
query: PlumeQuery,
(min, max): (i32, i32),
) -> Vec<Post> {
let schema = self.index.schema();
let post_id = schema.get_field("post_id").unwrap();
let collector = TopDocs::with_limit(cmp::max(1,max) as usize);
let collector = TopDocs::with_limit(cmp::max(1, max) as usize);
let searcher = self.index.searcher();
let res = searcher.search(&query.into_query(), &collector).unwrap();
res.get(min as usize..).unwrap_or(&[])
res.get(min as usize..)
.unwrap_or(&[])
.iter()
.filter_map(|(_,doc_add)| {
.filter_map(|(_, doc_add)| {
let doc = searcher.doc(*doc_add).ok()?;
let id = doc.get_first(post_id)?;
Post::get(conn, id.i64_value() as i32).ok()