Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6405ba57ee | |||
| d99b42582d | |||
| 92a386277b | |||
| 297d9fcf40 | |||
| ef70cb93e6 | |||
| efb76a3c17 | |||
| 197f0d7ecd |
Generated
+649
-245
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -24,7 +24,6 @@ rocket = "0.4.2"
|
||||
rocket_contrib = { version = "0.4.2", features = ["json"] }
|
||||
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
|
||||
rpassword = "4.0"
|
||||
runtime-fmt = "0.4.0"
|
||||
scheduled-thread-pool = "0.2.2"
|
||||
serde = "1.0"
|
||||
serde_json = "1.0"
|
||||
@@ -79,6 +78,7 @@ postgres = ["plume-models/postgres", "diesel/postgres"]
|
||||
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
|
||||
debug-mailer = []
|
||||
test = []
|
||||
search-lindera = ["plume-models/search-lindera"]
|
||||
|
||||
[workspace]
|
||||
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]
|
||||
|
||||
@@ -23,3 +23,4 @@ path = "../plume-models"
|
||||
[features]
|
||||
postgres = ["plume-models/postgres", "diesel/postgres"]
|
||||
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
|
||||
search-lindera = ["plume-models/search-lindera"]
|
||||
|
||||
@@ -82,7 +82,7 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
|
||||
}
|
||||
};
|
||||
if can_do || force {
|
||||
let searcher = Searcher::create(&path).unwrap();
|
||||
let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap();
|
||||
refill(args, conn, Some(searcher));
|
||||
} else {
|
||||
eprintln!(
|
||||
@@ -98,7 +98,8 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
|
||||
Some(path) => Path::new(path).join("search_index"),
|
||||
None => Path::new(&CONFIG.search_index).to_path_buf(),
|
||||
};
|
||||
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
|
||||
let searcher =
|
||||
searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap());
|
||||
|
||||
searcher.fill(conn).expect("Couldn't import post");
|
||||
println!("Commiting result");
|
||||
|
||||
@@ -442,6 +442,7 @@ mod tests {
|
||||
("not_a#hashtag", vec![]),
|
||||
("#نرمافزار_آزاد", vec!["نرمافزار_آزاد"]),
|
||||
("[#hash in link](https://example.org/)", vec![]),
|
||||
("#zwsp\u{200b}inhash", vec!["zwsp"]),
|
||||
];
|
||||
|
||||
for (md, mentions) in tests {
|
||||
|
||||
@@ -19,6 +19,7 @@ init_i18n!(
|
||||
en,
|
||||
eo,
|
||||
es,
|
||||
fa,
|
||||
fr,
|
||||
gl,
|
||||
hi,
|
||||
@@ -45,6 +46,9 @@ lazy_static! {
|
||||
let lang = js! { return navigator.language }.into_string().unwrap();
|
||||
let lang = lang.splitn(2, '-').next().unwrap_or("en");
|
||||
|
||||
// Force a language (Add .env setting)
|
||||
//let lang = "fa"; ???
|
||||
|
||||
let english_position = catalogs
|
||||
.iter()
|
||||
.position(|(language_code, _)| *language_code == "en")
|
||||
|
||||
@@ -22,7 +22,7 @@ scheduled-thread-pool = "0.2.2"
|
||||
serde = "1.0"
|
||||
serde_derive = "1.0"
|
||||
serde_json = "1.0"
|
||||
tantivy = "0.10.1"
|
||||
tantivy = "0.12.0"
|
||||
url = "2.1"
|
||||
walkdir = "2.2"
|
||||
webfinger = "0.4.1"
|
||||
@@ -30,6 +30,7 @@ whatlang = "0.7.1"
|
||||
shrinkwraprs = "0.2.1"
|
||||
diesel-derive-newtype = "0.1.2"
|
||||
glob = "0.3.0"
|
||||
lindera-tantivy = { version = "0.1.2", optional = true }
|
||||
|
||||
[dependencies.chrono]
|
||||
features = ["serde"]
|
||||
@@ -54,3 +55,4 @@ diesel_migrations = "1.3.0"
|
||||
[features]
|
||||
postgres = ["diesel/postgres", "plume-macro/postgres" ]
|
||||
sqlite = ["diesel/sqlite", "plume-macro/sqlite" ]
|
||||
search-lindera = ["lindera-tantivy"]
|
||||
|
||||
@@ -498,6 +498,7 @@ pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
blog_authors::*,
|
||||
config::CONFIG,
|
||||
instance::tests as instance_tests,
|
||||
medias::NewMedia,
|
||||
search::tests::get_searcher,
|
||||
@@ -767,7 +768,9 @@ pub(crate) mod tests {
|
||||
conn.test_transaction::<_, (), _>(|| {
|
||||
let (_, blogs) = fill_database(conn);
|
||||
|
||||
blogs[0].delete(conn, &get_searcher()).unwrap();
|
||||
blogs[0]
|
||||
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
|
||||
.unwrap();
|
||||
assert!(Blog::get(conn, blogs[0].id).is_err());
|
||||
Ok(())
|
||||
})
|
||||
@@ -777,7 +780,7 @@ pub(crate) mod tests {
|
||||
fn delete_via_user() {
|
||||
let conn = &db();
|
||||
conn.test_transaction::<_, (), _>(|| {
|
||||
let searcher = get_searcher();
|
||||
let searcher = get_searcher(&CONFIG.search_tokenizers);
|
||||
let (user, _) = fill_database(conn);
|
||||
|
||||
let b1 = Blog::insert(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::search::TokenizerKind as SearchTokenizer;
|
||||
use rocket::config::Limits;
|
||||
use rocket::Config as RocketConfig;
|
||||
use std::env::{self, var};
|
||||
@@ -14,6 +15,7 @@ pub struct Config {
|
||||
pub db_max_size: Option<u32>,
|
||||
pub db_min_idle: Option<u32>,
|
||||
pub search_index: String,
|
||||
pub search_tokenizers: SearchTokenizerConfig,
|
||||
pub rocket: Result<RocketConfig, RocketError>,
|
||||
pub logo: LogoConfig,
|
||||
pub default_theme: String,
|
||||
@@ -188,6 +190,56 @@ impl Default for LogoConfig {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SearchTokenizerConfig {
|
||||
pub tag_tokenizer: SearchTokenizer,
|
||||
pub content_tokenizer: SearchTokenizer,
|
||||
pub property_tokenizer: SearchTokenizer,
|
||||
}
|
||||
|
||||
impl SearchTokenizerConfig {
|
||||
pub fn init() -> Self {
|
||||
use SearchTokenizer::*;
|
||||
|
||||
match var("SEARCH_LANG").ok().as_deref() {
|
||||
Some("ja") => {
|
||||
#[cfg(not(feature = "search-lindera"))]
|
||||
panic!("You need build Plume with search-lindera feature, or execute it with SEARCH_TAG_TOKENIZER=ngram and SEARCH_CONTENT_TOKENIZER=ngram to enable Japanese search feature");
|
||||
#[cfg(feature = "search-lindera")]
|
||||
Self {
|
||||
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Lindera),
|
||||
content_tokenizer: Self::determine_tokenizer(
|
||||
"SEARCH_CONTENT_TOKENIZER",
|
||||
Lindera,
|
||||
),
|
||||
property_tokenizer: Ngram,
|
||||
}
|
||||
}
|
||||
_ => Self {
|
||||
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Whitespace),
|
||||
content_tokenizer: Self::determine_tokenizer("SEARCH_CONTENT_TOKENIZER", Simple),
|
||||
property_tokenizer: Ngram,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn determine_tokenizer(var_name: &str, default: SearchTokenizer) -> SearchTokenizer {
|
||||
use SearchTokenizer::*;
|
||||
|
||||
match var(var_name).ok().as_deref() {
|
||||
Some("simple") => Simple,
|
||||
Some("ngram") => Ngram,
|
||||
Some("whitespace") => Whitespace,
|
||||
Some("lindera") => {
|
||||
#[cfg(not(feature = "search-lindera"))]
|
||||
panic!("You need build Plume with search-lindera feature to use Lindera tokenizer");
|
||||
#[cfg(feature = "search-lindera")]
|
||||
Lindera
|
||||
}
|
||||
_ => default,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
pub static ref CONFIG: Config = Config {
|
||||
base_url: var("BASE_URL").unwrap_or_else(|_| format!(
|
||||
@@ -209,6 +261,7 @@ lazy_static! {
|
||||
#[cfg(feature = "sqlite")]
|
||||
database_url: var("DATABASE_URL").unwrap_or_else(|_| format!("{}.sqlite", DB_NAME)),
|
||||
search_index: var("SEARCH_INDEX").unwrap_or_else(|_| "search_index".to_owned()),
|
||||
search_tokenizers: SearchTokenizerConfig::init(),
|
||||
rocket: get_rocket_config(),
|
||||
logo: LogoConfig::default(),
|
||||
default_theme: var("DEFAULT_THEME").unwrap_or_else(|_| "default-light".to_owned()),
|
||||
|
||||
@@ -320,7 +320,7 @@ mod tests {
|
||||
pub fn rockets() -> super::PlumeRocket {
|
||||
super::PlumeRocket {
|
||||
conn: db_conn::DbConn((*DB_POOL).get().unwrap()),
|
||||
searcher: Arc::new(search::tests::get_searcher()),
|
||||
searcher: Arc::new(search::tests::get_searcher(&CONFIG.search_tokenizers)),
|
||||
worker: Arc::new(ScheduledThreadPool::new(2)),
|
||||
user: None,
|
||||
}
|
||||
|
||||
@@ -3,10 +3,11 @@ mod searcher;
|
||||
mod tokenizer;
|
||||
pub use self::query::PlumeQuery as Query;
|
||||
pub use self::searcher::*;
|
||||
pub use self::tokenizer::TokenizerKind;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::{Query, Searcher};
|
||||
use super::{Query, Searcher, TokenizerKind};
|
||||
use diesel::Connection;
|
||||
use plume_common::utils::random_hex;
|
||||
use std::env::temp_dir;
|
||||
@@ -14,18 +15,20 @@ pub(crate) mod tests {
|
||||
|
||||
use crate::{
|
||||
blogs::tests::fill_database,
|
||||
config::SearchTokenizerConfig,
|
||||
post_authors::*,
|
||||
posts::{NewPost, Post},
|
||||
safe_string::SafeString,
|
||||
tests::db,
|
||||
CONFIG,
|
||||
};
|
||||
|
||||
pub(crate) fn get_searcher() -> Searcher {
|
||||
pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher {
|
||||
let dir = temp_dir().join(&format!("plume-test-{}", random_hex()));
|
||||
if dir.exists() {
|
||||
Searcher::open(&dir)
|
||||
Searcher::open(&dir, tokenizers)
|
||||
} else {
|
||||
Searcher::create(&dir)
|
||||
Searcher::create(&dir, tokenizers)
|
||||
}
|
||||
.unwrap()
|
||||
}
|
||||
@@ -100,27 +103,27 @@ pub(crate) mod tests {
|
||||
fn open() {
|
||||
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
|
||||
{
|
||||
Searcher::create(&dir).unwrap();
|
||||
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
|
||||
}
|
||||
Searcher::open(&dir).unwrap();
|
||||
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create() {
|
||||
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
|
||||
|
||||
assert!(Searcher::open(&dir).is_err());
|
||||
assert!(Searcher::open(&dir, &CONFIG.search_tokenizers).is_err());
|
||||
{
|
||||
Searcher::create(&dir).unwrap();
|
||||
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
|
||||
}
|
||||
Searcher::open(&dir).unwrap(); //verify it's well created
|
||||
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search() {
|
||||
let conn = &db();
|
||||
conn.test_transaction::<_, (), _>(|| {
|
||||
let searcher = get_searcher();
|
||||
let searcher = get_searcher(&CONFIG.search_tokenizers);
|
||||
let blog = &fill_database(conn).1[0];
|
||||
let author = &blog.list_authors(conn).unwrap()[0];
|
||||
|
||||
@@ -180,10 +183,69 @@ pub(crate) mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "search-lindera")]
|
||||
#[test]
|
||||
fn search_japanese() {
|
||||
let conn = &db();
|
||||
conn.test_transaction::<_, (), _>(|| {
|
||||
let tokenizers = SearchTokenizerConfig {
|
||||
tag_tokenizer: TokenizerKind::Lindera,
|
||||
content_tokenizer: TokenizerKind::Lindera,
|
||||
property_tokenizer: TokenizerKind::Ngram,
|
||||
};
|
||||
let searcher = get_searcher(&tokenizers);
|
||||
let blog = &fill_database(conn).1[0];
|
||||
|
||||
let title = random_hex()[..8].to_owned();
|
||||
|
||||
let post = Post::insert(
|
||||
conn,
|
||||
NewPost {
|
||||
blog_id: blog.id,
|
||||
slug: title.clone(),
|
||||
title: title.clone(),
|
||||
content: SafeString::new("ブログエンジンPlumeです。"),
|
||||
published: true,
|
||||
license: "CC-BY-SA".to_owned(),
|
||||
ap_url: "".to_owned(),
|
||||
creation_date: None,
|
||||
subtitle: "".to_owned(),
|
||||
source: "".to_owned(),
|
||||
cover_id: None,
|
||||
},
|
||||
&searcher,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
searcher.commit();
|
||||
|
||||
assert_eq!(
|
||||
searcher.search_document(conn, Query::from_str("ブログエンジン").unwrap(), (0, 1))
|
||||
[0]
|
||||
.id,
|
||||
post.id
|
||||
);
|
||||
assert_eq!(
|
||||
searcher.search_document(conn, Query::from_str("Plume").unwrap(), (0, 1))[0].id,
|
||||
post.id
|
||||
);
|
||||
assert_eq!(
|
||||
searcher.search_document(conn, Query::from_str("です").unwrap(), (0, 1))[0].id,
|
||||
post.id
|
||||
);
|
||||
assert_eq!(
|
||||
searcher.search_document(conn, Query::from_str("。").unwrap(), (0, 1))[0].id,
|
||||
post.id
|
||||
);
|
||||
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_writer() {
|
||||
let searcher = get_searcher();
|
||||
let searcher = get_searcher(&CONFIG.search_tokenizers);
|
||||
searcher.drop_writer();
|
||||
get_searcher();
|
||||
get_searcher(&CONFIG.search_tokenizers);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,14 @@
|
||||
use crate::{
|
||||
instance::Instance,
|
||||
posts::Post,
|
||||
schema::posts,
|
||||
search::{query::PlumeQuery, tokenizer},
|
||||
tags::Tag,
|
||||
Connection, Result,
|
||||
config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts,
|
||||
search::query::PlumeQuery, tags::Tag, Connection, Result,
|
||||
};
|
||||
use chrono::Datelike;
|
||||
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
|
||||
use itertools::Itertools;
|
||||
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
|
||||
use tantivy::{
|
||||
collector::TopDocs, directory::MmapDirectory, schema::*, tokenizer::*, Index, IndexReader,
|
||||
IndexWriter, ReloadPolicy, Term,
|
||||
collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter,
|
||||
ReloadPolicy, Term,
|
||||
};
|
||||
use whatlang::{detect as detect_lang, Lang};
|
||||
|
||||
@@ -34,7 +30,7 @@ impl Searcher {
|
||||
pub fn schema() -> Schema {
|
||||
let tag_indexing = TextOptions::default().set_indexing_options(
|
||||
TextFieldIndexing::default()
|
||||
.set_tokenizer("whitespace_tokenizer")
|
||||
.set_tokenizer("tag_tokenizer")
|
||||
.set_index_option(IndexRecordOption::Basic),
|
||||
);
|
||||
|
||||
@@ -70,15 +66,7 @@ impl Searcher {
|
||||
schema_builder.build()
|
||||
}
|
||||
|
||||
pub fn create(path: &dyn AsRef<Path>) -> Result<Self> {
|
||||
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
|
||||
|
||||
let content_tokenizer = SimpleTokenizer
|
||||
.filter(RemoveLongFilter::limit(40))
|
||||
.filter(LowerCaser);
|
||||
|
||||
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
|
||||
|
||||
pub fn create(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
|
||||
let schema = Self::schema();
|
||||
|
||||
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
|
||||
@@ -90,9 +78,9 @@ impl Searcher {
|
||||
|
||||
{
|
||||
let tokenizer_manager = index.tokenizers();
|
||||
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
|
||||
tokenizer_manager.register("content_tokenizer", content_tokenizer);
|
||||
tokenizer_manager.register("property_tokenizer", property_tokenizer);
|
||||
tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
|
||||
tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
|
||||
tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
|
||||
} //to please the borrow checker
|
||||
Ok(Self {
|
||||
writer: Mutex::new(Some(
|
||||
@@ -109,31 +97,38 @@ impl Searcher {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open(path: &dyn AsRef<Path>) -> Result<Self> {
|
||||
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
|
||||
|
||||
let content_tokenizer = SimpleTokenizer
|
||||
.filter(RemoveLongFilter::limit(40))
|
||||
.filter(LowerCaser);
|
||||
|
||||
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
|
||||
|
||||
let index =
|
||||
pub fn open(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
|
||||
let mut index =
|
||||
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
|
||||
.map_err(|_| SearcherError::IndexOpeningError)?;
|
||||
|
||||
{
|
||||
let tokenizer_manager = index.tokenizers();
|
||||
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
|
||||
tokenizer_manager.register("content_tokenizer", content_tokenizer);
|
||||
tokenizer_manager.register("property_tokenizer", property_tokenizer);
|
||||
tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
|
||||
tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
|
||||
tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
|
||||
} //to please the borrow checker
|
||||
let mut writer = index
|
||||
let writer = index
|
||||
.writer(50_000_000)
|
||||
.map_err(|_| SearcherError::WriteLockAcquisitionError)?;
|
||||
writer
|
||||
.garbage_collect_files()
|
||||
|
||||
// Since Tantivy v0.12.0, IndexWriter::garbage_collect_files() returns Future.
|
||||
// To avoid conflict with Plume async project, we don't introduce async now.
|
||||
// After async is introduced to Plume, we can use garbage_collect_files() again.
|
||||
// Algorithm stolen from Tantivy's SegmentUpdater::list_files()
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
let mut files: HashSet<PathBuf> = index
|
||||
.list_all_segment_metas()
|
||||
.into_iter()
|
||||
.flat_map(|segment_meta| segment_meta.list_files())
|
||||
.collect();
|
||||
files.insert(Path::new("meta.json").to_path_buf());
|
||||
index
|
||||
.directory_mut()
|
||||
.garbage_collect(|| files)
|
||||
.map_err(|_| SearcherError::IndexEditionError)?;
|
||||
|
||||
Ok(Self {
|
||||
writer: Mutex::new(Some(writer)),
|
||||
reader: index
|
||||
|
||||
@@ -1,5 +1,34 @@
|
||||
#[cfg(feature = "search-lindera")]
|
||||
use lindera_tantivy::tokenizer::LinderaTokenizer;
|
||||
use std::str::CharIndices;
|
||||
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
|
||||
use tantivy::tokenizer::*;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum TokenizerKind {
|
||||
Simple,
|
||||
Ngram,
|
||||
Whitespace,
|
||||
#[cfg(feature = "search-lindera")]
|
||||
Lindera,
|
||||
}
|
||||
|
||||
impl From<TokenizerKind> for TextAnalyzer {
|
||||
fn from(tokenizer: TokenizerKind) -> TextAnalyzer {
|
||||
use TokenizerKind::*;
|
||||
|
||||
match tokenizer {
|
||||
Simple => TextAnalyzer::from(SimpleTokenizer)
|
||||
.filter(RemoveLongFilter::limit(40))
|
||||
.filter(LowerCaser),
|
||||
Ngram => TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser),
|
||||
Whitespace => TextAnalyzer::from(WhitespaceTokenizer).filter(LowerCaser),
|
||||
#[cfg(feature = "search-lindera")]
|
||||
Lindera => {
|
||||
TextAnalyzer::from(LinderaTokenizer::new("decompose", "")).filter(LowerCaser)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
|
||||
/// but not splitting on punctuation
|
||||
@@ -12,15 +41,13 @@ pub struct WhitespaceTokenStream<'a> {
|
||||
token: Token,
|
||||
}
|
||||
|
||||
impl<'a> Tokenizer<'a> for WhitespaceTokenizer {
|
||||
type TokenStreamImpl = WhitespaceTokenStream<'a>;
|
||||
|
||||
fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
|
||||
WhitespaceTokenStream {
|
||||
impl Tokenizer for WhitespaceTokenizer {
|
||||
fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
|
||||
BoxTokenStream::from(WhitespaceTokenStream {
|
||||
text,
|
||||
chars: text.char_indices(),
|
||||
token: Token::default(),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
impl<'a> WhitespaceTokenStream<'a> {
|
||||
|
||||
@@ -406,8 +406,8 @@ impl Bool {
|
||||
}
|
||||
}
|
||||
Bool::HasCover => Ok(post.cover_id.is_some()),
|
||||
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local()),
|
||||
Bool::All => Ok(true),
|
||||
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local() && kind == Kind::Original),
|
||||
Bool::All => Ok(kind == Kind::Original),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1026,6 +1026,7 @@ impl NewUser {
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
config::CONFIG,
|
||||
instance::{tests as instance_tests, Instance},
|
||||
search::tests::get_searcher,
|
||||
tests::{db, rockets},
|
||||
@@ -1122,7 +1123,9 @@ pub(crate) mod tests {
|
||||
let inserted = fill_database(conn);
|
||||
|
||||
assert!(User::get(conn, inserted[0].id).is_ok());
|
||||
inserted[0].delete(conn, &get_searcher()).unwrap();
|
||||
inserted[0]
|
||||
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
|
||||
.unwrap();
|
||||
assert!(User::get(conn, inserted[0].id).is_err());
|
||||
Ok(())
|
||||
});
|
||||
|
||||
+3
-4
@@ -6,8 +6,6 @@ extern crate gettext_macros;
|
||||
#[macro_use]
|
||||
extern crate rocket;
|
||||
#[macro_use]
|
||||
extern crate runtime_fmt;
|
||||
#[macro_use]
|
||||
extern crate serde_json;
|
||||
#[macro_use]
|
||||
extern crate validator_derive;
|
||||
@@ -28,7 +26,8 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
init_i18n!(
|
||||
"plume", ar, bg, ca, cs, de, en, eo, es, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr, sk, sv
|
||||
"plume", ar, bg, ca, cs, de, en, eo, es, fa, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr,
|
||||
sk, sv
|
||||
);
|
||||
|
||||
mod api;
|
||||
@@ -100,7 +99,7 @@ Then try to restart Plume.
|
||||
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
|
||||
// we want a fast exit here, so
|
||||
#[allow(clippy::match_wild_err_arm)]
|
||||
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index) {
|
||||
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers) {
|
||||
Err(Error::Search(e)) => match e {
|
||||
SearcherError::WriteLockAcquisitionError => panic!(
|
||||
r#"
|
||||
|
||||
+1
-1
@@ -643,7 +643,7 @@ pub fn remote_interact_post(
|
||||
.and_then(|blog| Post::find_by_slug(&rockets.conn, &slug, blog.id))?;
|
||||
if let Some(uri) = User::fetch_remote_interact_uri(&remote.remote)
|
||||
.ok()
|
||||
.and_then(|uri| rt_format!(uri, uri = target.ap_url).ok())
|
||||
.map(|uri| uri.replace("{uri}", &target.ap_url))
|
||||
{
|
||||
Ok(Redirect::to(uri).into())
|
||||
} else {
|
||||
|
||||
+5
-6
@@ -201,15 +201,14 @@ pub fn follow_not_connected(
|
||||
if let Some(uri) = User::fetch_remote_interact_uri(&remote_form)
|
||||
.ok()
|
||||
.and_then(|uri| {
|
||||
rt_format!(
|
||||
uri,
|
||||
uri = format!(
|
||||
Some(uri.replace(
|
||||
"{uri}",
|
||||
&format!(
|
||||
"{}@{}",
|
||||
target.fqn,
|
||||
target.get_instance(&rockets.conn).ok()?.public_domain
|
||||
)
|
||||
)
|
||||
.ok()
|
||||
),
|
||||
))
|
||||
})
|
||||
{
|
||||
Ok(Redirect::to(uri).into())
|
||||
|
||||
Reference in New Issue
Block a user