Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d99b42582d | |||
| 92a386277b | |||
| 297d9fcf40 | |||
| ef70cb93e6 | |||
| efb76a3c17 | |||
| 197f0d7ecd |
Generated
+649
-245
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -24,7 +24,6 @@ rocket = "0.4.2"
|
|||||||
rocket_contrib = { version = "0.4.2", features = ["json"] }
|
rocket_contrib = { version = "0.4.2", features = ["json"] }
|
||||||
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
|
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
|
||||||
rpassword = "4.0"
|
rpassword = "4.0"
|
||||||
runtime-fmt = "0.4.0"
|
|
||||||
scheduled-thread-pool = "0.2.2"
|
scheduled-thread-pool = "0.2.2"
|
||||||
serde = "1.0"
|
serde = "1.0"
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
@@ -79,6 +78,7 @@ postgres = ["plume-models/postgres", "diesel/postgres"]
|
|||||||
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
|
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
|
||||||
debug-mailer = []
|
debug-mailer = []
|
||||||
test = []
|
test = []
|
||||||
|
search-lindera = ["plume-models/search-lindera"]
|
||||||
|
|
||||||
[workspace]
|
[workspace]
|
||||||
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]
|
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]
|
||||||
|
|||||||
@@ -23,3 +23,4 @@ path = "../plume-models"
|
|||||||
[features]
|
[features]
|
||||||
postgres = ["plume-models/postgres", "diesel/postgres"]
|
postgres = ["plume-models/postgres", "diesel/postgres"]
|
||||||
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
|
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
|
||||||
|
search-lindera = ["plume-models/search-lindera"]
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
if can_do || force {
|
if can_do || force {
|
||||||
let searcher = Searcher::create(&path).unwrap();
|
let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap();
|
||||||
refill(args, conn, Some(searcher));
|
refill(args, conn, Some(searcher));
|
||||||
} else {
|
} else {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
@@ -98,7 +98,8 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
|
|||||||
Some(path) => Path::new(path).join("search_index"),
|
Some(path) => Path::new(path).join("search_index"),
|
||||||
None => Path::new(&CONFIG.search_index).to_path_buf(),
|
None => Path::new(&CONFIG.search_index).to_path_buf(),
|
||||||
};
|
};
|
||||||
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
|
let searcher =
|
||||||
|
searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap());
|
||||||
|
|
||||||
searcher.fill(conn).expect("Couldn't import post");
|
searcher.fill(conn).expect("Couldn't import post");
|
||||||
println!("Commiting result");
|
println!("Commiting result");
|
||||||
|
|||||||
@@ -442,6 +442,7 @@ mod tests {
|
|||||||
("not_a#hashtag", vec![]),
|
("not_a#hashtag", vec![]),
|
||||||
("#نرمافزار_آزاد", vec!["نرمافزار_آزاد"]),
|
("#نرمافزار_آزاد", vec!["نرمافزار_آزاد"]),
|
||||||
("[#hash in link](https://example.org/)", vec![]),
|
("[#hash in link](https://example.org/)", vec![]),
|
||||||
|
("#zwsp\u{200b}inhash", vec!["zwsp"]),
|
||||||
];
|
];
|
||||||
|
|
||||||
for (md, mentions) in tests {
|
for (md, mentions) in tests {
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ init_i18n!(
|
|||||||
en,
|
en,
|
||||||
eo,
|
eo,
|
||||||
es,
|
es,
|
||||||
|
fa,
|
||||||
fr,
|
fr,
|
||||||
gl,
|
gl,
|
||||||
hi,
|
hi,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ scheduled-thread-pool = "0.2.2"
|
|||||||
serde = "1.0"
|
serde = "1.0"
|
||||||
serde_derive = "1.0"
|
serde_derive = "1.0"
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
tantivy = "0.10.1"
|
tantivy = "0.12.0"
|
||||||
url = "2.1"
|
url = "2.1"
|
||||||
walkdir = "2.2"
|
walkdir = "2.2"
|
||||||
webfinger = "0.4.1"
|
webfinger = "0.4.1"
|
||||||
@@ -30,6 +30,7 @@ whatlang = "0.7.1"
|
|||||||
shrinkwraprs = "0.2.1"
|
shrinkwraprs = "0.2.1"
|
||||||
diesel-derive-newtype = "0.1.2"
|
diesel-derive-newtype = "0.1.2"
|
||||||
glob = "0.3.0"
|
glob = "0.3.0"
|
||||||
|
lindera-tantivy = { version = "0.1.2", optional = true }
|
||||||
|
|
||||||
[dependencies.chrono]
|
[dependencies.chrono]
|
||||||
features = ["serde"]
|
features = ["serde"]
|
||||||
@@ -54,3 +55,4 @@ diesel_migrations = "1.3.0"
|
|||||||
[features]
|
[features]
|
||||||
postgres = ["diesel/postgres", "plume-macro/postgres" ]
|
postgres = ["diesel/postgres", "plume-macro/postgres" ]
|
||||||
sqlite = ["diesel/sqlite", "plume-macro/sqlite" ]
|
sqlite = ["diesel/sqlite", "plume-macro/sqlite" ]
|
||||||
|
search-lindera = ["lindera-tantivy"]
|
||||||
|
|||||||
@@ -498,6 +498,7 @@ pub(crate) mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::{
|
use crate::{
|
||||||
blog_authors::*,
|
blog_authors::*,
|
||||||
|
config::CONFIG,
|
||||||
instance::tests as instance_tests,
|
instance::tests as instance_tests,
|
||||||
medias::NewMedia,
|
medias::NewMedia,
|
||||||
search::tests::get_searcher,
|
search::tests::get_searcher,
|
||||||
@@ -767,7 +768,9 @@ pub(crate) mod tests {
|
|||||||
conn.test_transaction::<_, (), _>(|| {
|
conn.test_transaction::<_, (), _>(|| {
|
||||||
let (_, blogs) = fill_database(conn);
|
let (_, blogs) = fill_database(conn);
|
||||||
|
|
||||||
blogs[0].delete(conn, &get_searcher()).unwrap();
|
blogs[0]
|
||||||
|
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
|
||||||
|
.unwrap();
|
||||||
assert!(Blog::get(conn, blogs[0].id).is_err());
|
assert!(Blog::get(conn, blogs[0].id).is_err());
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
@@ -777,7 +780,7 @@ pub(crate) mod tests {
|
|||||||
fn delete_via_user() {
|
fn delete_via_user() {
|
||||||
let conn = &db();
|
let conn = &db();
|
||||||
conn.test_transaction::<_, (), _>(|| {
|
conn.test_transaction::<_, (), _>(|| {
|
||||||
let searcher = get_searcher();
|
let searcher = get_searcher(&CONFIG.search_tokenizers);
|
||||||
let (user, _) = fill_database(conn);
|
let (user, _) = fill_database(conn);
|
||||||
|
|
||||||
let b1 = Blog::insert(
|
let b1 = Blog::insert(
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use crate::search::TokenizerKind as SearchTokenizer;
|
||||||
use rocket::config::Limits;
|
use rocket::config::Limits;
|
||||||
use rocket::Config as RocketConfig;
|
use rocket::Config as RocketConfig;
|
||||||
use std::env::{self, var};
|
use std::env::{self, var};
|
||||||
@@ -14,6 +15,7 @@ pub struct Config {
|
|||||||
pub db_max_size: Option<u32>,
|
pub db_max_size: Option<u32>,
|
||||||
pub db_min_idle: Option<u32>,
|
pub db_min_idle: Option<u32>,
|
||||||
pub search_index: String,
|
pub search_index: String,
|
||||||
|
pub search_tokenizers: SearchTokenizerConfig,
|
||||||
pub rocket: Result<RocketConfig, RocketError>,
|
pub rocket: Result<RocketConfig, RocketError>,
|
||||||
pub logo: LogoConfig,
|
pub logo: LogoConfig,
|
||||||
pub default_theme: String,
|
pub default_theme: String,
|
||||||
@@ -188,6 +190,56 @@ impl Default for LogoConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct SearchTokenizerConfig {
|
||||||
|
pub tag_tokenizer: SearchTokenizer,
|
||||||
|
pub content_tokenizer: SearchTokenizer,
|
||||||
|
pub property_tokenizer: SearchTokenizer,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SearchTokenizerConfig {
|
||||||
|
pub fn init() -> Self {
|
||||||
|
use SearchTokenizer::*;
|
||||||
|
|
||||||
|
match var("SEARCH_LANG").ok().as_deref() {
|
||||||
|
Some("ja") => {
|
||||||
|
#[cfg(not(feature = "search-lindera"))]
|
||||||
|
panic!("You need build Plume with search-lindera feature, or execute it with SEARCH_TAG_TOKENIZER=ngram and SEARCH_CONTENT_TOKENIZER=ngram to enable Japanese search feature");
|
||||||
|
#[cfg(feature = "search-lindera")]
|
||||||
|
Self {
|
||||||
|
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Lindera),
|
||||||
|
content_tokenizer: Self::determine_tokenizer(
|
||||||
|
"SEARCH_CONTENT_TOKENIZER",
|
||||||
|
Lindera,
|
||||||
|
),
|
||||||
|
property_tokenizer: Ngram,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => Self {
|
||||||
|
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Whitespace),
|
||||||
|
content_tokenizer: Self::determine_tokenizer("SEARCH_CONTENT_TOKENIZER", Simple),
|
||||||
|
property_tokenizer: Ngram,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn determine_tokenizer(var_name: &str, default: SearchTokenizer) -> SearchTokenizer {
|
||||||
|
use SearchTokenizer::*;
|
||||||
|
|
||||||
|
match var(var_name).ok().as_deref() {
|
||||||
|
Some("simple") => Simple,
|
||||||
|
Some("ngram") => Ngram,
|
||||||
|
Some("whitespace") => Whitespace,
|
||||||
|
Some("lindera") => {
|
||||||
|
#[cfg(not(feature = "search-lindera"))]
|
||||||
|
panic!("You need build Plume with search-lindera feature to use Lindera tokenizer");
|
||||||
|
#[cfg(feature = "search-lindera")]
|
||||||
|
Lindera
|
||||||
|
}
|
||||||
|
_ => default,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
pub static ref CONFIG: Config = Config {
|
pub static ref CONFIG: Config = Config {
|
||||||
base_url: var("BASE_URL").unwrap_or_else(|_| format!(
|
base_url: var("BASE_URL").unwrap_or_else(|_| format!(
|
||||||
@@ -209,6 +261,7 @@ lazy_static! {
|
|||||||
#[cfg(feature = "sqlite")]
|
#[cfg(feature = "sqlite")]
|
||||||
database_url: var("DATABASE_URL").unwrap_or_else(|_| format!("{}.sqlite", DB_NAME)),
|
database_url: var("DATABASE_URL").unwrap_or_else(|_| format!("{}.sqlite", DB_NAME)),
|
||||||
search_index: var("SEARCH_INDEX").unwrap_or_else(|_| "search_index".to_owned()),
|
search_index: var("SEARCH_INDEX").unwrap_or_else(|_| "search_index".to_owned()),
|
||||||
|
search_tokenizers: SearchTokenizerConfig::init(),
|
||||||
rocket: get_rocket_config(),
|
rocket: get_rocket_config(),
|
||||||
logo: LogoConfig::default(),
|
logo: LogoConfig::default(),
|
||||||
default_theme: var("DEFAULT_THEME").unwrap_or_else(|_| "default-light".to_owned()),
|
default_theme: var("DEFAULT_THEME").unwrap_or_else(|_| "default-light".to_owned()),
|
||||||
|
|||||||
@@ -320,7 +320,7 @@ mod tests {
|
|||||||
pub fn rockets() -> super::PlumeRocket {
|
pub fn rockets() -> super::PlumeRocket {
|
||||||
super::PlumeRocket {
|
super::PlumeRocket {
|
||||||
conn: db_conn::DbConn((*DB_POOL).get().unwrap()),
|
conn: db_conn::DbConn((*DB_POOL).get().unwrap()),
|
||||||
searcher: Arc::new(search::tests::get_searcher()),
|
searcher: Arc::new(search::tests::get_searcher(&CONFIG.search_tokenizers)),
|
||||||
worker: Arc::new(ScheduledThreadPool::new(2)),
|
worker: Arc::new(ScheduledThreadPool::new(2)),
|
||||||
user: None,
|
user: None,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,11 @@ mod searcher;
|
|||||||
mod tokenizer;
|
mod tokenizer;
|
||||||
pub use self::query::PlumeQuery as Query;
|
pub use self::query::PlumeQuery as Query;
|
||||||
pub use self::searcher::*;
|
pub use self::searcher::*;
|
||||||
|
pub use self::tokenizer::TokenizerKind;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) mod tests {
|
pub(crate) mod tests {
|
||||||
use super::{Query, Searcher};
|
use super::{Query, Searcher, TokenizerKind};
|
||||||
use diesel::Connection;
|
use diesel::Connection;
|
||||||
use plume_common::utils::random_hex;
|
use plume_common::utils::random_hex;
|
||||||
use std::env::temp_dir;
|
use std::env::temp_dir;
|
||||||
@@ -14,18 +15,20 @@ pub(crate) mod tests {
|
|||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
blogs::tests::fill_database,
|
blogs::tests::fill_database,
|
||||||
|
config::SearchTokenizerConfig,
|
||||||
post_authors::*,
|
post_authors::*,
|
||||||
posts::{NewPost, Post},
|
posts::{NewPost, Post},
|
||||||
safe_string::SafeString,
|
safe_string::SafeString,
|
||||||
tests::db,
|
tests::db,
|
||||||
|
CONFIG,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) fn get_searcher() -> Searcher {
|
pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher {
|
||||||
let dir = temp_dir().join(&format!("plume-test-{}", random_hex()));
|
let dir = temp_dir().join(&format!("plume-test-{}", random_hex()));
|
||||||
if dir.exists() {
|
if dir.exists() {
|
||||||
Searcher::open(&dir)
|
Searcher::open(&dir, tokenizers)
|
||||||
} else {
|
} else {
|
||||||
Searcher::create(&dir)
|
Searcher::create(&dir, tokenizers)
|
||||||
}
|
}
|
||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
@@ -100,27 +103,27 @@ pub(crate) mod tests {
|
|||||||
fn open() {
|
fn open() {
|
||||||
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
|
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
|
||||||
{
|
{
|
||||||
Searcher::create(&dir).unwrap();
|
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
|
||||||
}
|
}
|
||||||
Searcher::open(&dir).unwrap();
|
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn create() {
|
fn create() {
|
||||||
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
|
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
|
||||||
|
|
||||||
assert!(Searcher::open(&dir).is_err());
|
assert!(Searcher::open(&dir, &CONFIG.search_tokenizers).is_err());
|
||||||
{
|
{
|
||||||
Searcher::create(&dir).unwrap();
|
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
|
||||||
}
|
}
|
||||||
Searcher::open(&dir).unwrap(); //verify it's well created
|
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn search() {
|
fn search() {
|
||||||
let conn = &db();
|
let conn = &db();
|
||||||
conn.test_transaction::<_, (), _>(|| {
|
conn.test_transaction::<_, (), _>(|| {
|
||||||
let searcher = get_searcher();
|
let searcher = get_searcher(&CONFIG.search_tokenizers);
|
||||||
let blog = &fill_database(conn).1[0];
|
let blog = &fill_database(conn).1[0];
|
||||||
let author = &blog.list_authors(conn).unwrap()[0];
|
let author = &blog.list_authors(conn).unwrap()[0];
|
||||||
|
|
||||||
@@ -180,10 +183,69 @@ pub(crate) mod tests {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "search-lindera")]
|
||||||
|
#[test]
|
||||||
|
fn search_japanese() {
|
||||||
|
let conn = &db();
|
||||||
|
conn.test_transaction::<_, (), _>(|| {
|
||||||
|
let tokenizers = SearchTokenizerConfig {
|
||||||
|
tag_tokenizer: TokenizerKind::Lindera,
|
||||||
|
content_tokenizer: TokenizerKind::Lindera,
|
||||||
|
property_tokenizer: TokenizerKind::Ngram,
|
||||||
|
};
|
||||||
|
let searcher = get_searcher(&tokenizers);
|
||||||
|
let blog = &fill_database(conn).1[0];
|
||||||
|
|
||||||
|
let title = random_hex()[..8].to_owned();
|
||||||
|
|
||||||
|
let post = Post::insert(
|
||||||
|
conn,
|
||||||
|
NewPost {
|
||||||
|
blog_id: blog.id,
|
||||||
|
slug: title.clone(),
|
||||||
|
title: title.clone(),
|
||||||
|
content: SafeString::new("ブログエンジンPlumeです。"),
|
||||||
|
published: true,
|
||||||
|
license: "CC-BY-SA".to_owned(),
|
||||||
|
ap_url: "".to_owned(),
|
||||||
|
creation_date: None,
|
||||||
|
subtitle: "".to_owned(),
|
||||||
|
source: "".to_owned(),
|
||||||
|
cover_id: None,
|
||||||
|
},
|
||||||
|
&searcher,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
searcher.commit();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
searcher.search_document(conn, Query::from_str("ブログエンジン").unwrap(), (0, 1))
|
||||||
|
[0]
|
||||||
|
.id,
|
||||||
|
post.id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
searcher.search_document(conn, Query::from_str("Plume").unwrap(), (0, 1))[0].id,
|
||||||
|
post.id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
searcher.search_document(conn, Query::from_str("です").unwrap(), (0, 1))[0].id,
|
||||||
|
post.id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
searcher.search_document(conn, Query::from_str("。").unwrap(), (0, 1))[0].id,
|
||||||
|
post.id
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn drop_writer() {
|
fn drop_writer() {
|
||||||
let searcher = get_searcher();
|
let searcher = get_searcher(&CONFIG.search_tokenizers);
|
||||||
searcher.drop_writer();
|
searcher.drop_writer();
|
||||||
get_searcher();
|
get_searcher(&CONFIG.search_tokenizers);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,18 +1,14 @@
|
|||||||
use crate::{
|
use crate::{
|
||||||
instance::Instance,
|
config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts,
|
||||||
posts::Post,
|
search::query::PlumeQuery, tags::Tag, Connection, Result,
|
||||||
schema::posts,
|
|
||||||
search::{query::PlumeQuery, tokenizer},
|
|
||||||
tags::Tag,
|
|
||||||
Connection, Result,
|
|
||||||
};
|
};
|
||||||
use chrono::Datelike;
|
use chrono::Datelike;
|
||||||
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
|
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
|
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
|
||||||
use tantivy::{
|
use tantivy::{
|
||||||
collector::TopDocs, directory::MmapDirectory, schema::*, tokenizer::*, Index, IndexReader,
|
collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter,
|
||||||
IndexWriter, ReloadPolicy, Term,
|
ReloadPolicy, Term,
|
||||||
};
|
};
|
||||||
use whatlang::{detect as detect_lang, Lang};
|
use whatlang::{detect as detect_lang, Lang};
|
||||||
|
|
||||||
@@ -34,7 +30,7 @@ impl Searcher {
|
|||||||
pub fn schema() -> Schema {
|
pub fn schema() -> Schema {
|
||||||
let tag_indexing = TextOptions::default().set_indexing_options(
|
let tag_indexing = TextOptions::default().set_indexing_options(
|
||||||
TextFieldIndexing::default()
|
TextFieldIndexing::default()
|
||||||
.set_tokenizer("whitespace_tokenizer")
|
.set_tokenizer("tag_tokenizer")
|
||||||
.set_index_option(IndexRecordOption::Basic),
|
.set_index_option(IndexRecordOption::Basic),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -70,15 +66,7 @@ impl Searcher {
|
|||||||
schema_builder.build()
|
schema_builder.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn create(path: &dyn AsRef<Path>) -> Result<Self> {
|
pub fn create(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
|
||||||
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
|
|
||||||
|
|
||||||
let content_tokenizer = SimpleTokenizer
|
|
||||||
.filter(RemoveLongFilter::limit(40))
|
|
||||||
.filter(LowerCaser);
|
|
||||||
|
|
||||||
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
|
|
||||||
|
|
||||||
let schema = Self::schema();
|
let schema = Self::schema();
|
||||||
|
|
||||||
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
|
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
|
||||||
@@ -90,9 +78,9 @@ impl Searcher {
|
|||||||
|
|
||||||
{
|
{
|
||||||
let tokenizer_manager = index.tokenizers();
|
let tokenizer_manager = index.tokenizers();
|
||||||
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
|
tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
|
||||||
tokenizer_manager.register("content_tokenizer", content_tokenizer);
|
tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
|
||||||
tokenizer_manager.register("property_tokenizer", property_tokenizer);
|
tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
|
||||||
} //to please the borrow checker
|
} //to please the borrow checker
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
writer: Mutex::new(Some(
|
writer: Mutex::new(Some(
|
||||||
@@ -109,31 +97,38 @@ impl Searcher {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn open(path: &dyn AsRef<Path>) -> Result<Self> {
|
pub fn open(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
|
||||||
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
|
let mut index =
|
||||||
|
|
||||||
let content_tokenizer = SimpleTokenizer
|
|
||||||
.filter(RemoveLongFilter::limit(40))
|
|
||||||
.filter(LowerCaser);
|
|
||||||
|
|
||||||
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
|
|
||||||
|
|
||||||
let index =
|
|
||||||
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
|
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
|
||||||
.map_err(|_| SearcherError::IndexOpeningError)?;
|
.map_err(|_| SearcherError::IndexOpeningError)?;
|
||||||
|
|
||||||
{
|
{
|
||||||
let tokenizer_manager = index.tokenizers();
|
let tokenizer_manager = index.tokenizers();
|
||||||
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
|
tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
|
||||||
tokenizer_manager.register("content_tokenizer", content_tokenizer);
|
tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
|
||||||
tokenizer_manager.register("property_tokenizer", property_tokenizer);
|
tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
|
||||||
} //to please the borrow checker
|
} //to please the borrow checker
|
||||||
let mut writer = index
|
let writer = index
|
||||||
.writer(50_000_000)
|
.writer(50_000_000)
|
||||||
.map_err(|_| SearcherError::WriteLockAcquisitionError)?;
|
.map_err(|_| SearcherError::WriteLockAcquisitionError)?;
|
||||||
writer
|
|
||||||
.garbage_collect_files()
|
// Since Tantivy v0.12.0, IndexWriter::garbage_collect_files() returns Future.
|
||||||
|
// To avoid conflict with Plume async project, we don't introduce async now.
|
||||||
|
// After async is introduced to Plume, we can use garbage_collect_files() again.
|
||||||
|
// Algorithm stolen from Tantivy's SegmentUpdater::list_files()
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
let mut files: HashSet<PathBuf> = index
|
||||||
|
.list_all_segment_metas()
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|segment_meta| segment_meta.list_files())
|
||||||
|
.collect();
|
||||||
|
files.insert(Path::new("meta.json").to_path_buf());
|
||||||
|
index
|
||||||
|
.directory_mut()
|
||||||
|
.garbage_collect(|| files)
|
||||||
.map_err(|_| SearcherError::IndexEditionError)?;
|
.map_err(|_| SearcherError::IndexEditionError)?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
writer: Mutex::new(Some(writer)),
|
writer: Mutex::new(Some(writer)),
|
||||||
reader: index
|
reader: index
|
||||||
|
|||||||
@@ -1,5 +1,34 @@
|
|||||||
|
#[cfg(feature = "search-lindera")]
|
||||||
|
use lindera_tantivy::tokenizer::LinderaTokenizer;
|
||||||
use std::str::CharIndices;
|
use std::str::CharIndices;
|
||||||
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
|
use tantivy::tokenizer::*;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub enum TokenizerKind {
|
||||||
|
Simple,
|
||||||
|
Ngram,
|
||||||
|
Whitespace,
|
||||||
|
#[cfg(feature = "search-lindera")]
|
||||||
|
Lindera,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<TokenizerKind> for TextAnalyzer {
|
||||||
|
fn from(tokenizer: TokenizerKind) -> TextAnalyzer {
|
||||||
|
use TokenizerKind::*;
|
||||||
|
|
||||||
|
match tokenizer {
|
||||||
|
Simple => TextAnalyzer::from(SimpleTokenizer)
|
||||||
|
.filter(RemoveLongFilter::limit(40))
|
||||||
|
.filter(LowerCaser),
|
||||||
|
Ngram => TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser),
|
||||||
|
Whitespace => TextAnalyzer::from(WhitespaceTokenizer).filter(LowerCaser),
|
||||||
|
#[cfg(feature = "search-lindera")]
|
||||||
|
Lindera => {
|
||||||
|
TextAnalyzer::from(LinderaTokenizer::new("decompose", "")).filter(LowerCaser)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
|
/// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
|
||||||
/// but not splitting on punctuation
|
/// but not splitting on punctuation
|
||||||
@@ -12,15 +41,13 @@ pub struct WhitespaceTokenStream<'a> {
|
|||||||
token: Token,
|
token: Token,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> Tokenizer<'a> for WhitespaceTokenizer {
|
impl Tokenizer for WhitespaceTokenizer {
|
||||||
type TokenStreamImpl = WhitespaceTokenStream<'a>;
|
fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
|
||||||
|
BoxTokenStream::from(WhitespaceTokenStream {
|
||||||
fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
|
|
||||||
WhitespaceTokenStream {
|
|
||||||
text,
|
text,
|
||||||
chars: text.char_indices(),
|
chars: text.char_indices(),
|
||||||
token: Token::default(),
|
token: Token::default(),
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
impl<'a> WhitespaceTokenStream<'a> {
|
impl<'a> WhitespaceTokenStream<'a> {
|
||||||
|
|||||||
@@ -406,8 +406,8 @@ impl Bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Bool::HasCover => Ok(post.cover_id.is_some()),
|
Bool::HasCover => Ok(post.cover_id.is_some()),
|
||||||
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local()),
|
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local() && kind == Kind::Original),
|
||||||
Bool::All => Ok(true),
|
Bool::All => Ok(kind == Kind::Original),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1026,6 +1026,7 @@ impl NewUser {
|
|||||||
pub(crate) mod tests {
|
pub(crate) mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::{
|
use crate::{
|
||||||
|
config::CONFIG,
|
||||||
instance::{tests as instance_tests, Instance},
|
instance::{tests as instance_tests, Instance},
|
||||||
search::tests::get_searcher,
|
search::tests::get_searcher,
|
||||||
tests::{db, rockets},
|
tests::{db, rockets},
|
||||||
@@ -1122,7 +1123,9 @@ pub(crate) mod tests {
|
|||||||
let inserted = fill_database(conn);
|
let inserted = fill_database(conn);
|
||||||
|
|
||||||
assert!(User::get(conn, inserted[0].id).is_ok());
|
assert!(User::get(conn, inserted[0].id).is_ok());
|
||||||
inserted[0].delete(conn, &get_searcher()).unwrap();
|
inserted[0]
|
||||||
|
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
|
||||||
|
.unwrap();
|
||||||
assert!(User::get(conn, inserted[0].id).is_err());
|
assert!(User::get(conn, inserted[0].id).is_err());
|
||||||
Ok(())
|
Ok(())
|
||||||
});
|
});
|
||||||
|
|||||||
+3
-4
@@ -6,8 +6,6 @@ extern crate gettext_macros;
|
|||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate rocket;
|
extern crate rocket;
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate runtime_fmt;
|
|
||||||
#[macro_use]
|
|
||||||
extern crate serde_json;
|
extern crate serde_json;
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate validator_derive;
|
extern crate validator_derive;
|
||||||
@@ -28,7 +26,8 @@ use std::sync::{Arc, Mutex};
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
init_i18n!(
|
init_i18n!(
|
||||||
"plume", ar, bg, ca, cs, de, en, eo, es, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr, sk, sv
|
"plume", ar, bg, ca, cs, de, en, eo, es, fa, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr,
|
||||||
|
sk, sv
|
||||||
);
|
);
|
||||||
|
|
||||||
mod api;
|
mod api;
|
||||||
@@ -100,7 +99,7 @@ Then try to restart Plume.
|
|||||||
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
|
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
|
||||||
// we want a fast exit here, so
|
// we want a fast exit here, so
|
||||||
#[allow(clippy::match_wild_err_arm)]
|
#[allow(clippy::match_wild_err_arm)]
|
||||||
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index) {
|
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers) {
|
||||||
Err(Error::Search(e)) => match e {
|
Err(Error::Search(e)) => match e {
|
||||||
SearcherError::WriteLockAcquisitionError => panic!(
|
SearcherError::WriteLockAcquisitionError => panic!(
|
||||||
r#"
|
r#"
|
||||||
|
|||||||
+1
-1
@@ -643,7 +643,7 @@ pub fn remote_interact_post(
|
|||||||
.and_then(|blog| Post::find_by_slug(&rockets.conn, &slug, blog.id))?;
|
.and_then(|blog| Post::find_by_slug(&rockets.conn, &slug, blog.id))?;
|
||||||
if let Some(uri) = User::fetch_remote_interact_uri(&remote.remote)
|
if let Some(uri) = User::fetch_remote_interact_uri(&remote.remote)
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|uri| rt_format!(uri, uri = target.ap_url).ok())
|
.map(|uri| uri.replace("{uri}", &target.ap_url))
|
||||||
{
|
{
|
||||||
Ok(Redirect::to(uri).into())
|
Ok(Redirect::to(uri).into())
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
+5
-6
@@ -201,15 +201,14 @@ pub fn follow_not_connected(
|
|||||||
if let Some(uri) = User::fetch_remote_interact_uri(&remote_form)
|
if let Some(uri) = User::fetch_remote_interact_uri(&remote_form)
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|uri| {
|
.and_then(|uri| {
|
||||||
rt_format!(
|
Some(uri.replace(
|
||||||
uri,
|
"{uri}",
|
||||||
uri = format!(
|
&format!(
|
||||||
"{}@{}",
|
"{}@{}",
|
||||||
target.fqn,
|
target.fqn,
|
||||||
target.get_instance(&rockets.conn).ok()?.public_domain
|
target.get_instance(&rockets.conn).ok()?.public_domain
|
||||||
)
|
),
|
||||||
)
|
))
|
||||||
.ok()
|
|
||||||
})
|
})
|
||||||
{
|
{
|
||||||
Ok(Redirect::to(uri).into())
|
Ok(Redirect::to(uri).into())
|
||||||
|
|||||||
Reference in New Issue
Block a user