Compare commits

...

7 Commits

Author SHA1 Message Date
Ebrahim 6405ba57ee add test comment 😅 2020-06-23 08:51:31 +04:30
Gelez d99b42582d Add Persian to the list of languages (#782) 2020-06-17 18:33:58 +02:00
KITAITI Makoto 92a386277b Switchable tokenizer (#776)
* [REFACTORING]Rename whitespace_tokenizer to tag_tokenizer for
registration

Name representing its purpose is preferred.

* Add lindera-tantivy to plume-model's dependencies

* Install lindera-tantivy

* Add SearchTokenizerConfig struct

* Add search tokenizers to config option

* Use CONFIG for tokenizers

* Use enum to hold tokenizer config instead of initializing on config phase

* Use guard instead of duplicate default values

* Use as_deref() instead of guard

* Move SearchTokenizer from plume-models to plume-models::search::tokenizer

* Rename SearchTokenizer to TokenizerKind

* Define SearchTokenierConfig::determine_tokenizer()

* Use determine_tokenizer in SearchTokenizerConfig::init()

* Pass tokenizer config to Searcher methods

* Add LowerCase filter to Lindera tokenizer

* Add test for Lindera tokenizer

* Define SEARCH_LANG env to specify tokenizers set

* Run cargo fmt

* Make Lindera tokenizer optional

* Fix typos
2020-06-17 16:57:28 +02:00
Gelez 297d9fcf40 Don't show boosts and likes for "all" and "local" in timelines (#781)
Fixes #711
2020-06-15 19:50:28 +02:00
KITAITI Makoto ef70cb93e6 Upgrade Tantivy to v0.12.0 (#771)
* Upgrade Tantivy to 0.12.0

* Follow Tantivy Tokenizer's new type definition

* Wrap tokenizers with TextAnalyzer to use filter methods

* Replace async IndexWriter::garbage_collect_files with sync functions

* Update Cargo.toml
2020-05-20 13:31:45 +02:00
Daniel Watkins efb76a3c17 remove dependency on runtime-fmt (#773)
Per the issue, "runtime-fmt uses perma-unstable rust APIs and is
therefore susceptible to breakage".

This replaces the two calls to rt_format! with .replace() and drops the
dependency.

Fixes #769
2020-05-18 20:18:07 +02:00
KITAITI Makoto 197f0d7ecd Add test for hash including ZWSP (#772)
* Add test for hash including ZWSP

* Run cargo fmt
2020-05-17 13:53:31 +02:00
18 changed files with 876 additions and 322 deletions
Generated
+649 -245
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -24,7 +24,6 @@ rocket = "0.4.2"
rocket_contrib = { version = "0.4.2", features = ["json"] } rocket_contrib = { version = "0.4.2", features = ["json"] }
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" } rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
rpassword = "4.0" rpassword = "4.0"
runtime-fmt = "0.4.0"
scheduled-thread-pool = "0.2.2" scheduled-thread-pool = "0.2.2"
serde = "1.0" serde = "1.0"
serde_json = "1.0" serde_json = "1.0"
@@ -79,6 +78,7 @@ postgres = ["plume-models/postgres", "diesel/postgres"]
sqlite = ["plume-models/sqlite", "diesel/sqlite"] sqlite = ["plume-models/sqlite", "diesel/sqlite"]
debug-mailer = [] debug-mailer = []
test = [] test = []
search-lindera = ["plume-models/search-lindera"]
[workspace] [workspace]
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"] members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]
+1
View File
@@ -23,3 +23,4 @@ path = "../plume-models"
[features] [features]
postgres = ["plume-models/postgres", "diesel/postgres"] postgres = ["plume-models/postgres", "diesel/postgres"]
sqlite = ["plume-models/sqlite", "diesel/sqlite"] sqlite = ["plume-models/sqlite", "diesel/sqlite"]
search-lindera = ["plume-models/search-lindera"]
+3 -2
View File
@@ -82,7 +82,7 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
} }
}; };
if can_do || force { if can_do || force {
let searcher = Searcher::create(&path).unwrap(); let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap();
refill(args, conn, Some(searcher)); refill(args, conn, Some(searcher));
} else { } else {
eprintln!( eprintln!(
@@ -98,7 +98,8 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
Some(path) => Path::new(path).join("search_index"), Some(path) => Path::new(path).join("search_index"),
None => Path::new(&CONFIG.search_index).to_path_buf(), None => Path::new(&CONFIG.search_index).to_path_buf(),
}; };
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap()); let searcher =
searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap());
searcher.fill(conn).expect("Couldn't import post"); searcher.fill(conn).expect("Couldn't import post");
println!("Commiting result"); println!("Commiting result");
+1
View File
@@ -442,6 +442,7 @@ mod tests {
("not_a#hashtag", vec![]), ("not_a#hashtag", vec![]),
("#نرم‌افزار_آزاد", vec!["نرم‌افزار_آزاد"]), ("#نرم‌افزار_آزاد", vec!["نرم‌افزار_آزاد"]),
("[#hash in link](https://example.org/)", vec![]), ("[#hash in link](https://example.org/)", vec![]),
("#zwsp\u{200b}inhash", vec!["zwsp"]),
]; ];
for (md, mentions) in tests { for (md, mentions) in tests {
+4
View File
@@ -19,6 +19,7 @@ init_i18n!(
en, en,
eo, eo,
es, es,
fa,
fr, fr,
gl, gl,
hi, hi,
@@ -45,6 +46,9 @@ lazy_static! {
let lang = js! { return navigator.language }.into_string().unwrap(); let lang = js! { return navigator.language }.into_string().unwrap();
let lang = lang.splitn(2, '-').next().unwrap_or("en"); let lang = lang.splitn(2, '-').next().unwrap_or("en");
// Force a language (Add .env setting)
//let lang = "fa"; ???
let english_position = catalogs let english_position = catalogs
.iter() .iter()
.position(|(language_code, _)| *language_code == "en") .position(|(language_code, _)| *language_code == "en")
+3 -1
View File
@@ -22,7 +22,7 @@ scheduled-thread-pool = "0.2.2"
serde = "1.0" serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
serde_json = "1.0" serde_json = "1.0"
tantivy = "0.10.1" tantivy = "0.12.0"
url = "2.1" url = "2.1"
walkdir = "2.2" walkdir = "2.2"
webfinger = "0.4.1" webfinger = "0.4.1"
@@ -30,6 +30,7 @@ whatlang = "0.7.1"
shrinkwraprs = "0.2.1" shrinkwraprs = "0.2.1"
diesel-derive-newtype = "0.1.2" diesel-derive-newtype = "0.1.2"
glob = "0.3.0" glob = "0.3.0"
lindera-tantivy = { version = "0.1.2", optional = true }
[dependencies.chrono] [dependencies.chrono]
features = ["serde"] features = ["serde"]
@@ -54,3 +55,4 @@ diesel_migrations = "1.3.0"
[features] [features]
postgres = ["diesel/postgres", "plume-macro/postgres" ] postgres = ["diesel/postgres", "plume-macro/postgres" ]
sqlite = ["diesel/sqlite", "plume-macro/sqlite" ] sqlite = ["diesel/sqlite", "plume-macro/sqlite" ]
search-lindera = ["lindera-tantivy"]
+5 -2
View File
@@ -498,6 +498,7 @@ pub(crate) mod tests {
use super::*; use super::*;
use crate::{ use crate::{
blog_authors::*, blog_authors::*,
config::CONFIG,
instance::tests as instance_tests, instance::tests as instance_tests,
medias::NewMedia, medias::NewMedia,
search::tests::get_searcher, search::tests::get_searcher,
@@ -767,7 +768,9 @@ pub(crate) mod tests {
conn.test_transaction::<_, (), _>(|| { conn.test_transaction::<_, (), _>(|| {
let (_, blogs) = fill_database(conn); let (_, blogs) = fill_database(conn);
blogs[0].delete(conn, &get_searcher()).unwrap(); blogs[0]
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
.unwrap();
assert!(Blog::get(conn, blogs[0].id).is_err()); assert!(Blog::get(conn, blogs[0].id).is_err());
Ok(()) Ok(())
}) })
@@ -777,7 +780,7 @@ pub(crate) mod tests {
fn delete_via_user() { fn delete_via_user() {
let conn = &db(); let conn = &db();
conn.test_transaction::<_, (), _>(|| { conn.test_transaction::<_, (), _>(|| {
let searcher = get_searcher(); let searcher = get_searcher(&CONFIG.search_tokenizers);
let (user, _) = fill_database(conn); let (user, _) = fill_database(conn);
let b1 = Blog::insert( let b1 = Blog::insert(
+53
View File
@@ -1,3 +1,4 @@
use crate::search::TokenizerKind as SearchTokenizer;
use rocket::config::Limits; use rocket::config::Limits;
use rocket::Config as RocketConfig; use rocket::Config as RocketConfig;
use std::env::{self, var}; use std::env::{self, var};
@@ -14,6 +15,7 @@ pub struct Config {
pub db_max_size: Option<u32>, pub db_max_size: Option<u32>,
pub db_min_idle: Option<u32>, pub db_min_idle: Option<u32>,
pub search_index: String, pub search_index: String,
pub search_tokenizers: SearchTokenizerConfig,
pub rocket: Result<RocketConfig, RocketError>, pub rocket: Result<RocketConfig, RocketError>,
pub logo: LogoConfig, pub logo: LogoConfig,
pub default_theme: String, pub default_theme: String,
@@ -188,6 +190,56 @@ impl Default for LogoConfig {
} }
} }
pub struct SearchTokenizerConfig {
pub tag_tokenizer: SearchTokenizer,
pub content_tokenizer: SearchTokenizer,
pub property_tokenizer: SearchTokenizer,
}
impl SearchTokenizerConfig {
pub fn init() -> Self {
use SearchTokenizer::*;
match var("SEARCH_LANG").ok().as_deref() {
Some("ja") => {
#[cfg(not(feature = "search-lindera"))]
panic!("You need build Plume with search-lindera feature, or execute it with SEARCH_TAG_TOKENIZER=ngram and SEARCH_CONTENT_TOKENIZER=ngram to enable Japanese search feature");
#[cfg(feature = "search-lindera")]
Self {
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Lindera),
content_tokenizer: Self::determine_tokenizer(
"SEARCH_CONTENT_TOKENIZER",
Lindera,
),
property_tokenizer: Ngram,
}
}
_ => Self {
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Whitespace),
content_tokenizer: Self::determine_tokenizer("SEARCH_CONTENT_TOKENIZER", Simple),
property_tokenizer: Ngram,
},
}
}
fn determine_tokenizer(var_name: &str, default: SearchTokenizer) -> SearchTokenizer {
use SearchTokenizer::*;
match var(var_name).ok().as_deref() {
Some("simple") => Simple,
Some("ngram") => Ngram,
Some("whitespace") => Whitespace,
Some("lindera") => {
#[cfg(not(feature = "search-lindera"))]
panic!("You need build Plume with search-lindera feature to use Lindera tokenizer");
#[cfg(feature = "search-lindera")]
Lindera
}
_ => default,
}
}
}
lazy_static! { lazy_static! {
pub static ref CONFIG: Config = Config { pub static ref CONFIG: Config = Config {
base_url: var("BASE_URL").unwrap_or_else(|_| format!( base_url: var("BASE_URL").unwrap_or_else(|_| format!(
@@ -209,6 +261,7 @@ lazy_static! {
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
database_url: var("DATABASE_URL").unwrap_or_else(|_| format!("{}.sqlite", DB_NAME)), database_url: var("DATABASE_URL").unwrap_or_else(|_| format!("{}.sqlite", DB_NAME)),
search_index: var("SEARCH_INDEX").unwrap_or_else(|_| "search_index".to_owned()), search_index: var("SEARCH_INDEX").unwrap_or_else(|_| "search_index".to_owned()),
search_tokenizers: SearchTokenizerConfig::init(),
rocket: get_rocket_config(), rocket: get_rocket_config(),
logo: LogoConfig::default(), logo: LogoConfig::default(),
default_theme: var("DEFAULT_THEME").unwrap_or_else(|_| "default-light".to_owned()), default_theme: var("DEFAULT_THEME").unwrap_or_else(|_| "default-light".to_owned()),
+1 -1
View File
@@ -320,7 +320,7 @@ mod tests {
pub fn rockets() -> super::PlumeRocket { pub fn rockets() -> super::PlumeRocket {
super::PlumeRocket { super::PlumeRocket {
conn: db_conn::DbConn((*DB_POOL).get().unwrap()), conn: db_conn::DbConn((*DB_POOL).get().unwrap()),
searcher: Arc::new(search::tests::get_searcher()), searcher: Arc::new(search::tests::get_searcher(&CONFIG.search_tokenizers)),
worker: Arc::new(ScheduledThreadPool::new(2)), worker: Arc::new(ScheduledThreadPool::new(2)),
user: None, user: None,
} }
+74 -12
View File
@@ -3,10 +3,11 @@ mod searcher;
mod tokenizer; mod tokenizer;
pub use self::query::PlumeQuery as Query; pub use self::query::PlumeQuery as Query;
pub use self::searcher::*; pub use self::searcher::*;
pub use self::tokenizer::TokenizerKind;
#[cfg(test)] #[cfg(test)]
pub(crate) mod tests { pub(crate) mod tests {
use super::{Query, Searcher}; use super::{Query, Searcher, TokenizerKind};
use diesel::Connection; use diesel::Connection;
use plume_common::utils::random_hex; use plume_common::utils::random_hex;
use std::env::temp_dir; use std::env::temp_dir;
@@ -14,18 +15,20 @@ pub(crate) mod tests {
use crate::{ use crate::{
blogs::tests::fill_database, blogs::tests::fill_database,
config::SearchTokenizerConfig,
post_authors::*, post_authors::*,
posts::{NewPost, Post}, posts::{NewPost, Post},
safe_string::SafeString, safe_string::SafeString,
tests::db, tests::db,
CONFIG,
}; };
pub(crate) fn get_searcher() -> Searcher { pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher {
let dir = temp_dir().join(&format!("plume-test-{}", random_hex())); let dir = temp_dir().join(&format!("plume-test-{}", random_hex()));
if dir.exists() { if dir.exists() {
Searcher::open(&dir) Searcher::open(&dir, tokenizers)
} else { } else {
Searcher::create(&dir) Searcher::create(&dir, tokenizers)
} }
.unwrap() .unwrap()
} }
@@ -100,27 +103,27 @@ pub(crate) mod tests {
fn open() { fn open() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex())); let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
{ {
Searcher::create(&dir).unwrap(); Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
} }
Searcher::open(&dir).unwrap(); Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap();
} }
#[test] #[test]
fn create() { fn create() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex())); let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
assert!(Searcher::open(&dir).is_err()); assert!(Searcher::open(&dir, &CONFIG.search_tokenizers).is_err());
{ {
Searcher::create(&dir).unwrap(); Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
} }
Searcher::open(&dir).unwrap(); //verify it's well created Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created
} }
#[test] #[test]
fn search() { fn search() {
let conn = &db(); let conn = &db();
conn.test_transaction::<_, (), _>(|| { conn.test_transaction::<_, (), _>(|| {
let searcher = get_searcher(); let searcher = get_searcher(&CONFIG.search_tokenizers);
let blog = &fill_database(conn).1[0]; let blog = &fill_database(conn).1[0];
let author = &blog.list_authors(conn).unwrap()[0]; let author = &blog.list_authors(conn).unwrap()[0];
@@ -180,10 +183,69 @@ pub(crate) mod tests {
}); });
} }
#[cfg(feature = "search-lindera")]
#[test]
fn search_japanese() {
let conn = &db();
conn.test_transaction::<_, (), _>(|| {
let tokenizers = SearchTokenizerConfig {
tag_tokenizer: TokenizerKind::Lindera,
content_tokenizer: TokenizerKind::Lindera,
property_tokenizer: TokenizerKind::Ngram,
};
let searcher = get_searcher(&tokenizers);
let blog = &fill_database(conn).1[0];
let title = random_hex()[..8].to_owned();
let post = Post::insert(
conn,
NewPost {
blog_id: blog.id,
slug: title.clone(),
title: title.clone(),
content: SafeString::new("ブログエンジンPlumeです。"),
published: true,
license: "CC-BY-SA".to_owned(),
ap_url: "".to_owned(),
creation_date: None,
subtitle: "".to_owned(),
source: "".to_owned(),
cover_id: None,
},
&searcher,
)
.unwrap();
searcher.commit();
assert_eq!(
searcher.search_document(conn, Query::from_str("ブログエンジン").unwrap(), (0, 1))
[0]
.id,
post.id
);
assert_eq!(
searcher.search_document(conn, Query::from_str("Plume").unwrap(), (0, 1))[0].id,
post.id
);
assert_eq!(
searcher.search_document(conn, Query::from_str("です").unwrap(), (0, 1))[0].id,
post.id
);
assert_eq!(
searcher.search_document(conn, Query::from_str("").unwrap(), (0, 1))[0].id,
post.id
);
Ok(())
});
}
#[test] #[test]
fn drop_writer() { fn drop_writer() {
let searcher = get_searcher(); let searcher = get_searcher(&CONFIG.search_tokenizers);
searcher.drop_writer(); searcher.drop_writer();
get_searcher(); get_searcher(&CONFIG.search_tokenizers);
} }
} }
+32 -37
View File
@@ -1,18 +1,14 @@
use crate::{ use crate::{
instance::Instance, config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts,
posts::Post, search::query::PlumeQuery, tags::Tag, Connection, Result,
schema::posts,
search::{query::PlumeQuery, tokenizer},
tags::Tag,
Connection, Result,
}; };
use chrono::Datelike; use chrono::Datelike;
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use itertools::Itertools; use itertools::Itertools;
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex}; use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use tantivy::{ use tantivy::{
collector::TopDocs, directory::MmapDirectory, schema::*, tokenizer::*, Index, IndexReader, collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter,
IndexWriter, ReloadPolicy, Term, ReloadPolicy, Term,
}; };
use whatlang::{detect as detect_lang, Lang}; use whatlang::{detect as detect_lang, Lang};
@@ -34,7 +30,7 @@ impl Searcher {
pub fn schema() -> Schema { pub fn schema() -> Schema {
let tag_indexing = TextOptions::default().set_indexing_options( let tag_indexing = TextOptions::default().set_indexing_options(
TextFieldIndexing::default() TextFieldIndexing::default()
.set_tokenizer("whitespace_tokenizer") .set_tokenizer("tag_tokenizer")
.set_index_option(IndexRecordOption::Basic), .set_index_option(IndexRecordOption::Basic),
); );
@@ -70,15 +66,7 @@ impl Searcher {
schema_builder.build() schema_builder.build()
} }
pub fn create(path: &dyn AsRef<Path>) -> Result<Self> { pub fn create(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
let schema = Self::schema(); let schema = Self::schema();
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?; create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
@@ -90,9 +78,9 @@ impl Searcher {
{ {
let tokenizer_manager = index.tokenizers(); let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer); tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer); tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer); tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
} //to please the borrow checker } //to please the borrow checker
Ok(Self { Ok(Self {
writer: Mutex::new(Some( writer: Mutex::new(Some(
@@ -109,31 +97,38 @@ impl Searcher {
}) })
} }
pub fn open(path: &dyn AsRef<Path>) -> Result<Self> { pub fn open(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser); let mut index =
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
let index =
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?) Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
.map_err(|_| SearcherError::IndexOpeningError)?; .map_err(|_| SearcherError::IndexOpeningError)?;
{ {
let tokenizer_manager = index.tokenizers(); let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer); tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer); tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer); tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
} //to please the borrow checker } //to please the borrow checker
let mut writer = index let writer = index
.writer(50_000_000) .writer(50_000_000)
.map_err(|_| SearcherError::WriteLockAcquisitionError)?; .map_err(|_| SearcherError::WriteLockAcquisitionError)?;
writer
.garbage_collect_files() // Since Tantivy v0.12.0, IndexWriter::garbage_collect_files() returns Future.
// To avoid conflict with Plume async project, we don't introduce async now.
// After async is introduced to Plume, we can use garbage_collect_files() again.
// Algorithm stolen from Tantivy's SegmentUpdater::list_files()
use std::collections::HashSet;
use std::path::PathBuf;
let mut files: HashSet<PathBuf> = index
.list_all_segment_metas()
.into_iter()
.flat_map(|segment_meta| segment_meta.list_files())
.collect();
files.insert(Path::new("meta.json").to_path_buf());
index
.directory_mut()
.garbage_collect(|| files)
.map_err(|_| SearcherError::IndexEditionError)?; .map_err(|_| SearcherError::IndexEditionError)?;
Ok(Self { Ok(Self {
writer: Mutex::new(Some(writer)), writer: Mutex::new(Some(writer)),
reader: index reader: index
+34 -7
View File
@@ -1,5 +1,34 @@
#[cfg(feature = "search-lindera")]
use lindera_tantivy::tokenizer::LinderaTokenizer;
use std::str::CharIndices; use std::str::CharIndices;
use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; use tantivy::tokenizer::*;
#[derive(Clone, Copy)]
pub enum TokenizerKind {
Simple,
Ngram,
Whitespace,
#[cfg(feature = "search-lindera")]
Lindera,
}
impl From<TokenizerKind> for TextAnalyzer {
fn from(tokenizer: TokenizerKind) -> TextAnalyzer {
use TokenizerKind::*;
match tokenizer {
Simple => TextAnalyzer::from(SimpleTokenizer)
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser),
Ngram => TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser),
Whitespace => TextAnalyzer::from(WhitespaceTokenizer).filter(LowerCaser),
#[cfg(feature = "search-lindera")]
Lindera => {
TextAnalyzer::from(LinderaTokenizer::new("decompose", "")).filter(LowerCaser)
}
}
}
}
/// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer, /// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
/// but not splitting on punctuation /// but not splitting on punctuation
@@ -12,15 +41,13 @@ pub struct WhitespaceTokenStream<'a> {
token: Token, token: Token,
} }
impl<'a> Tokenizer<'a> for WhitespaceTokenizer { impl Tokenizer for WhitespaceTokenizer {
type TokenStreamImpl = WhitespaceTokenStream<'a>; fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
BoxTokenStream::from(WhitespaceTokenStream {
fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
WhitespaceTokenStream {
text, text,
chars: text.char_indices(), chars: text.char_indices(),
token: Token::default(), token: Token::default(),
} })
} }
} }
impl<'a> WhitespaceTokenStream<'a> { impl<'a> WhitespaceTokenStream<'a> {
+2 -2
View File
@@ -406,8 +406,8 @@ impl Bool {
} }
} }
Bool::HasCover => Ok(post.cover_id.is_some()), Bool::HasCover => Ok(post.cover_id.is_some()),
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local()), Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local() && kind == Kind::Original),
Bool::All => Ok(true), Bool::All => Ok(kind == Kind::Original),
} }
} }
} }
+4 -1
View File
@@ -1026,6 +1026,7 @@ impl NewUser {
pub(crate) mod tests { pub(crate) mod tests {
use super::*; use super::*;
use crate::{ use crate::{
config::CONFIG,
instance::{tests as instance_tests, Instance}, instance::{tests as instance_tests, Instance},
search::tests::get_searcher, search::tests::get_searcher,
tests::{db, rockets}, tests::{db, rockets},
@@ -1122,7 +1123,9 @@ pub(crate) mod tests {
let inserted = fill_database(conn); let inserted = fill_database(conn);
assert!(User::get(conn, inserted[0].id).is_ok()); assert!(User::get(conn, inserted[0].id).is_ok());
inserted[0].delete(conn, &get_searcher()).unwrap(); inserted[0]
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
.unwrap();
assert!(User::get(conn, inserted[0].id).is_err()); assert!(User::get(conn, inserted[0].id).is_err());
Ok(()) Ok(())
}); });
+3 -4
View File
@@ -6,8 +6,6 @@ extern crate gettext_macros;
#[macro_use] #[macro_use]
extern crate rocket; extern crate rocket;
#[macro_use] #[macro_use]
extern crate runtime_fmt;
#[macro_use]
extern crate serde_json; extern crate serde_json;
#[macro_use] #[macro_use]
extern crate validator_derive; extern crate validator_derive;
@@ -28,7 +26,8 @@ use std::sync::{Arc, Mutex};
use std::time::Duration; use std::time::Duration;
init_i18n!( init_i18n!(
"plume", ar, bg, ca, cs, de, en, eo, es, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr, sk, sv "plume", ar, bg, ca, cs, de, en, eo, es, fa, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr,
sk, sv
); );
mod api; mod api;
@@ -100,7 +99,7 @@ Then try to restart Plume.
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
// we want a fast exit here, so // we want a fast exit here, so
#[allow(clippy::match_wild_err_arm)] #[allow(clippy::match_wild_err_arm)]
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index) { let searcher = match UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers) {
Err(Error::Search(e)) => match e { Err(Error::Search(e)) => match e {
SearcherError::WriteLockAcquisitionError => panic!( SearcherError::WriteLockAcquisitionError => panic!(
r#" r#"
+1 -1
View File
@@ -643,7 +643,7 @@ pub fn remote_interact_post(
.and_then(|blog| Post::find_by_slug(&rockets.conn, &slug, blog.id))?; .and_then(|blog| Post::find_by_slug(&rockets.conn, &slug, blog.id))?;
if let Some(uri) = User::fetch_remote_interact_uri(&remote.remote) if let Some(uri) = User::fetch_remote_interact_uri(&remote.remote)
.ok() .ok()
.and_then(|uri| rt_format!(uri, uri = target.ap_url).ok()) .map(|uri| uri.replace("{uri}", &target.ap_url))
{ {
Ok(Redirect::to(uri).into()) Ok(Redirect::to(uri).into())
} else { } else {
+5 -6
View File
@@ -201,15 +201,14 @@ pub fn follow_not_connected(
if let Some(uri) = User::fetch_remote_interact_uri(&remote_form) if let Some(uri) = User::fetch_remote_interact_uri(&remote_form)
.ok() .ok()
.and_then(|uri| { .and_then(|uri| {
rt_format!( Some(uri.replace(
uri, "{uri}",
uri = format!( &format!(
"{}@{}", "{}@{}",
target.fqn, target.fqn,
target.get_instance(&rockets.conn).ok()?.public_domain target.get_instance(&rockets.conn).ok()?.public_domain
) ),
) ))
.ok()
}) })
{ {
Ok(Redirect::to(uri).into()) Ok(Redirect::to(uri).into())