Compare commits

..

1 Commits

Author SHA1 Message Date
Christopher James Halse Rogers adb5dac1d2 snap: Fix build process for Tier 2 platforms
Unfortunately this only actually *fixes* the build for arm64;
it seems that we hit various bugs in Rust on ppc64 and armhf.
2020-01-24 09:03:02 +02:00
33 changed files with 1588 additions and 2206 deletions
-1
View File
@@ -226,7 +226,6 @@ jobs:
steps:
- restore_env:
cache: none
- run: cargo build
- run: crowdin upload -b master
workflows:
Generated
+1350 -1721
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -24,6 +24,7 @@ rocket = "0.4.2"
rocket_contrib = { version = "0.4.2", features = ["json"] }
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
rpassword = "4.0"
runtime-fmt = "0.4.0"
scheduled-thread-pool = "0.2.2"
serde = "1.0"
serde_json = "1.0"
@@ -78,7 +79,6 @@ postgres = ["plume-models/postgres", "diesel/postgres"]
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
debug-mailer = []
test = []
search-lindera = ["plume-models/search-lindera"]
[workspace]
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]
+88 -22
View File
@@ -190,28 +190,7 @@ p.error {
background: $gray;
text-overflow: ellipsis;
footer.authors {
div {
float: left;
margin-right: 0.25em;
}
.likes { color: $red; }
.reshares { color: $primary; }
span.likes, span.resahres {
font-family: "Route159",serif;
font-size: 1em;
}
svg.feather {
width: 0.85em;
height: 0.85em;
}
}
text-overflow: ellipsis;
> * {
margin: 20px;
@@ -490,6 +469,93 @@ figure {
/// Small screens
@media screen and (max-width: 600px) {
@keyframes menuOpening {
from {
transform: scaleX(0);
transform-origin: left;
opacity: 0;
}
to {
transform: scaleX(1);
transform-origin: left;
opacity: 1;
}
}
body > header {
flex-direction: column;
nav#menu {
display: inline-flex;
z-index: 21;
}
#content {
display: none;
appearance: none;
text-align: center;
z-index: 20;
}
}
body > header:focus-within #content, #content.show {
position: fixed;
display: flex;
flex-direction: column;
justify-content: flex-start;
top: 0;
left: 0;
width: 100%;
height: 100%;
box-sizing: border-box;
animation: 0.2s menuOpening;
&::before {
content: "";
position: absolute;
transform: skewX(-10deg);
top: 0;
left: -20%;
width: 100%;
height: 100%;
z-index: -10;
background: $primary;
}
> nav {
flex-direction: column;
align-items: flex-start;
a {
display: flex;
flex-direction: row;
align-items: center;
margin: 0;
padding: 1rem 1.5rem;
color: $background;
font-size: 1.4em;
font-weight: 300;
&.title { font-size: 1.8em; }
> *:first-child { width: 3rem; }
> img:first-child { height: 3rem; }
> *:last-child { margin-left: 1rem; }
> nav hr {
display: block;
margin: 0;
width: 100%;
border: solid $background 0.1rem;
}
.mobile-label { display: initial; }
}
}
}
main .article-meta {
> *, .comments {
margin: 0 5%;
-90
View File
@@ -101,96 +101,6 @@ body > header {
}
}
/// Small screens
@media screen and (max-width: 600px) {
@keyframes menuOpening {
from {
transform: scaleX(0);
transform-origin: left;
opacity: 0;
}
to {
transform: scaleX(1);
transform-origin: left;
opacity: 1;
}
}
body > header {
flex-direction: column;
nav#menu {
display: inline-flex;
z-index: 21;
}
#content {
display: none;
appearance: none;
text-align: center;
z-index: 20;
}
}
body > header:focus-within #content, #content.show {
position: fixed;
display: flex;
flex-direction: column;
justify-content: flex-start;
top: 0;
left: 0;
width: 100%;
height: 100%;
box-sizing: border-box;
animation: 0.2s menuOpening;
&::before {
content: "";
position: absolute;
transform: skewX(-10deg);
top: 0;
left: -20%;
width: 100%;
height: 100%;
z-index: -10;
background: $primary;
}
> nav {
flex-direction: column;
align-items: flex-start;
a {
display: flex;
flex-direction: row;
align-items: center;
margin: 0;
padding: 1rem 1.5rem;
color: $background;
font-size: 1.4em;
font-weight: 300;
&.title { font-size: 1.8em; }
> *:first-child { width: 3rem; }
> img:first-child { height: 3rem; }
> *:last-child { margin-left: 1rem; }
> nav hr {
display: block;
margin: 0;
width: 100%;
border: solid $background 0.1rem;
}
.mobile-label { display: initial; }
}
}
}
}
/* Only enable label animations on large screens */
@media screen and (min-width: 600px) {
header nav a {
-1
View File
@@ -23,4 +23,3 @@ path = "../plume-models"
[features]
postgres = ["plume-models/postgres", "diesel/postgres"]
sqlite = ["plume-models/sqlite", "diesel/sqlite"]
search-lindera = ["plume-models/search-lindera"]
+2 -3
View File
@@ -82,7 +82,7 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
}
};
if can_do || force {
let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap();
let searcher = Searcher::create(&path).unwrap();
refill(args, conn, Some(searcher));
} else {
eprintln!(
@@ -98,8 +98,7 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
Some(path) => Path::new(path).join("search_index"),
None => Path::new(&CONFIG.search_index).to_path_buf(),
};
let searcher =
searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap());
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
searcher.fill(conn).expect("Couldn't import post");
println!("Commiting result");
-1
View File
@@ -22,7 +22,6 @@ serde_json = "1.0"
shrinkwraprs = "0.2.1"
syntect = "3.3"
tokio = "0.1.22"
regex-syntax = { version = "0.6.17", default-features = false, features = ["unicode-perl"] }
[dependencies.chrono]
features = ["serde"]
+6 -25
View File
@@ -1,7 +1,6 @@
use heck::CamelCase;
use openssl::rand::rand_bytes;
use pulldown_cmark::{html, Event, Options, Parser, Tag};
use regex_syntax::is_word_character;
use rocket::{
http::uri::Uri,
response::{Flash, Redirect},
@@ -201,12 +200,6 @@ fn process_image<'a, 'b>(
}
}
#[derive(Default, Debug)]
struct DocumentContext {
in_code: bool,
in_link: bool,
}
/// Returns (HTML, mentions, hashtags)
pub fn md_to_html<'a>(
md: &str,
@@ -230,21 +223,13 @@ pub fn md_to_html<'a>(
.map(|evt| process_image(evt, inline, &media_processor))
// Ignore headings, images, and tables if inline = true
.scan((vec![], inline), inline_tags)
.scan(&mut DocumentContext::default(), |ctx, evt| match evt {
.scan(false, |in_code, evt| match evt {
Event::Start(Tag::CodeBlock(_)) | Event::Start(Tag::Code) => {
ctx.in_code = true;
*in_code = true;
Some((vec![evt], vec![], vec![]))
}
Event::End(Tag::CodeBlock(_)) | Event::End(Tag::Code) => {
ctx.in_code = false;
Some((vec![evt], vec![], vec![]))
}
Event::Start(Tag::Link(_, _)) => {
ctx.in_link = true;
Some((vec![evt], vec![], vec![]))
}
Event::End(Tag::Link(_, _)) => {
ctx.in_link = false;
*in_code = false;
Some((vec![evt], vec![], vec![]))
}
Event::Text(txt) => {
@@ -284,7 +269,7 @@ pub fn md_to_html<'a>(
}
}
State::Hashtag => {
let char_matches = c == '-' || is_word_character(c);
let char_matches = c.is_alphanumeric() || "-_".contains(c);
if char_matches && (n < (txt.chars().count() - 1)) {
text_acc.push(c);
(events, State::Hashtag, text_acc, n + 1, mentions, hashtags)
@@ -315,7 +300,7 @@ pub fn md_to_html<'a>(
}
}
State::Ready => {
if !ctx.in_code && !ctx.in_link && c == '@' {
if !*in_code && c == '@' {
events.push(Event::Text(text_acc.into()));
(
events,
@@ -325,7 +310,7 @@ pub fn md_to_html<'a>(
mentions,
hashtags,
)
} else if !ctx.in_code && !ctx.in_link && c == '#' {
} else if !*in_code && c == '#' {
events.push(Event::Text(text_acc.into()));
(
events,
@@ -414,7 +399,6 @@ mod tests {
("not_a@mention", vec![]),
("`@helo`", vec![]),
("```\n@hello\n```", vec![]),
("[@atmark in link](https://example.org/)", vec![]),
];
for (md, mentions) in tests {
@@ -440,9 +424,6 @@ mod tests {
("with some punctuation #test!", vec!["test"]),
(" #spaces ", vec!["spaces"]),
("not_a#hashtag", vec![]),
("#نرم‌افزار_آزاد", vec!["نرم‌افزار_آزاد"]),
("[#hash in link](https://example.org/)", vec![]),
("#zwsp\u{200b}inhash", vec!["zwsp"]),
];
for (md, mentions) in tests {
-4
View File
@@ -19,7 +19,6 @@ init_i18n!(
en,
eo,
es,
fa,
fr,
gl,
hi,
@@ -46,9 +45,6 @@ lazy_static! {
let lang = js! { return navigator.language }.into_string().unwrap();
let lang = lang.splitn(2, '-').next().unwrap_or("en");
// Force a language (Add .env setting)
//let lang = "fa"; ???
let english_position = catalogs
.iter()
.position(|(language_code, _)| *language_code == "en")
+1 -3
View File
@@ -22,7 +22,7 @@ scheduled-thread-pool = "0.2.2"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
tantivy = "0.12.0"
tantivy = "0.10.1"
url = "2.1"
walkdir = "2.2"
webfinger = "0.4.1"
@@ -30,7 +30,6 @@ whatlang = "0.7.1"
shrinkwraprs = "0.2.1"
diesel-derive-newtype = "0.1.2"
glob = "0.3.0"
lindera-tantivy = { version = "0.1.2", optional = true }
[dependencies.chrono]
features = ["serde"]
@@ -55,4 +54,3 @@ diesel_migrations = "1.3.0"
[features]
postgres = ["diesel/postgres", "plume-macro/postgres" ]
sqlite = ["diesel/sqlite", "plume-macro/sqlite" ]
search-lindera = ["lindera-tantivy"]
+2 -5
View File
@@ -498,7 +498,6 @@ pub(crate) mod tests {
use super::*;
use crate::{
blog_authors::*,
config::CONFIG,
instance::tests as instance_tests,
medias::NewMedia,
search::tests::get_searcher,
@@ -768,9 +767,7 @@ pub(crate) mod tests {
conn.test_transaction::<_, (), _>(|| {
let (_, blogs) = fill_database(conn);
blogs[0]
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
.unwrap();
blogs[0].delete(conn, &get_searcher()).unwrap();
assert!(Blog::get(conn, blogs[0].id).is_err());
Ok(())
})
@@ -780,7 +777,7 @@ pub(crate) mod tests {
fn delete_via_user() {
let conn = &db();
conn.test_transaction::<_, (), _>(|| {
let searcher = get_searcher(&CONFIG.search_tokenizers);
let searcher = get_searcher();
let (user, _) = fill_database(conn);
let b1 = Blog::insert(
-63
View File
@@ -1,4 +1,3 @@
use crate::search::TokenizerKind as SearchTokenizer;
use rocket::config::Limits;
use rocket::Config as RocketConfig;
use std::env::{self, var};
@@ -12,10 +11,7 @@ pub struct Config {
pub base_url: String,
pub database_url: String,
pub db_name: &'static str,
pub db_max_size: Option<u32>,
pub db_min_idle: Option<u32>,
pub search_index: String,
pub search_tokenizers: SearchTokenizerConfig,
pub rocket: Result<RocketConfig, RocketError>,
pub logo: LogoConfig,
pub default_theme: String,
@@ -190,56 +186,6 @@ impl Default for LogoConfig {
}
}
pub struct SearchTokenizerConfig {
pub tag_tokenizer: SearchTokenizer,
pub content_tokenizer: SearchTokenizer,
pub property_tokenizer: SearchTokenizer,
}
impl SearchTokenizerConfig {
pub fn init() -> Self {
use SearchTokenizer::*;
match var("SEARCH_LANG").ok().as_deref() {
Some("ja") => {
#[cfg(not(feature = "search-lindera"))]
panic!("You need build Plume with search-lindera feature, or execute it with SEARCH_TAG_TOKENIZER=ngram and SEARCH_CONTENT_TOKENIZER=ngram to enable Japanese search feature");
#[cfg(feature = "search-lindera")]
Self {
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Lindera),
content_tokenizer: Self::determine_tokenizer(
"SEARCH_CONTENT_TOKENIZER",
Lindera,
),
property_tokenizer: Ngram,
}
}
_ => Self {
tag_tokenizer: Self::determine_tokenizer("SEARCH_TAG_TOKENIZER", Whitespace),
content_tokenizer: Self::determine_tokenizer("SEARCH_CONTENT_TOKENIZER", Simple),
property_tokenizer: Ngram,
},
}
}
fn determine_tokenizer(var_name: &str, default: SearchTokenizer) -> SearchTokenizer {
use SearchTokenizer::*;
match var(var_name).ok().as_deref() {
Some("simple") => Simple,
Some("ngram") => Ngram,
Some("whitespace") => Whitespace,
Some("lindera") => {
#[cfg(not(feature = "search-lindera"))]
panic!("You need build Plume with search-lindera feature to use Lindera tokenizer");
#[cfg(feature = "search-lindera")]
Lindera
}
_ => default,
}
}
}
lazy_static! {
pub static ref CONFIG: Config = Config {
base_url: var("BASE_URL").unwrap_or_else(|_| format!(
@@ -247,21 +193,12 @@ lazy_static! {
var("ROCKET_PORT").unwrap_or_else(|_| "7878".to_owned())
)),
db_name: DB_NAME,
db_max_size: var("DB_MAX_SIZE").map_or(None, |s| Some(
s.parse::<u32>()
.expect("Couldn't parse DB_MAX_SIZE into u32")
)),
db_min_idle: var("DB_MIN_IDLE").map_or(None, |s| Some(
s.parse::<u32>()
.expect("Couldn't parse DB_MIN_IDLE into u32")
)),
#[cfg(feature = "postgres")]
database_url: var("DATABASE_URL")
.unwrap_or_else(|_| format!("postgres://plume:plume@localhost/{}", DB_NAME)),
#[cfg(feature = "sqlite")]
database_url: var("DATABASE_URL").unwrap_or_else(|_| format!("{}.sqlite", DB_NAME)),
search_index: var("SEARCH_INDEX").unwrap_or_else(|_| "search_index".to_owned()),
search_tokenizers: SearchTokenizerConfig::init(),
rocket: get_rocket_config(),
logo: LogoConfig::default(),
default_theme: var("DEFAULT_THEME").unwrap_or_else(|_| "default-light".to_owned()),
+1 -1
View File
@@ -320,7 +320,7 @@ mod tests {
pub fn rockets() -> super::PlumeRocket {
super::PlumeRocket {
conn: db_conn::DbConn((*DB_POOL).get().unwrap()),
searcher: Arc::new(search::tests::get_searcher(&CONFIG.search_tokenizers)),
searcher: Arc::new(search::tests::get_searcher()),
worker: Arc::new(ScheduledThreadPool::new(2)),
user: None,
}
+12 -74
View File
@@ -3,11 +3,10 @@ mod searcher;
mod tokenizer;
pub use self::query::PlumeQuery as Query;
pub use self::searcher::*;
pub use self::tokenizer::TokenizerKind;
#[cfg(test)]
pub(crate) mod tests {
use super::{Query, Searcher, TokenizerKind};
use super::{Query, Searcher};
use diesel::Connection;
use plume_common::utils::random_hex;
use std::env::temp_dir;
@@ -15,20 +14,18 @@ pub(crate) mod tests {
use crate::{
blogs::tests::fill_database,
config::SearchTokenizerConfig,
post_authors::*,
posts::{NewPost, Post},
safe_string::SafeString,
tests::db,
CONFIG,
};
pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher {
pub(crate) fn get_searcher() -> Searcher {
let dir = temp_dir().join(&format!("plume-test-{}", random_hex()));
if dir.exists() {
Searcher::open(&dir, tokenizers)
Searcher::open(&dir)
} else {
Searcher::create(&dir, tokenizers)
Searcher::create(&dir)
}
.unwrap()
}
@@ -103,27 +100,27 @@ pub(crate) mod tests {
fn open() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
{
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
Searcher::create(&dir).unwrap();
}
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap();
Searcher::open(&dir).unwrap();
}
#[test]
fn create() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
assert!(Searcher::open(&dir, &CONFIG.search_tokenizers).is_err());
assert!(Searcher::open(&dir).is_err());
{
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
Searcher::create(&dir).unwrap();
}
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created
Searcher::open(&dir).unwrap(); //verify it's well created
}
#[test]
fn search() {
let conn = &db();
conn.test_transaction::<_, (), _>(|| {
let searcher = get_searcher(&CONFIG.search_tokenizers);
let searcher = get_searcher();
let blog = &fill_database(conn).1[0];
let author = &blog.list_authors(conn).unwrap()[0];
@@ -183,69 +180,10 @@ pub(crate) mod tests {
});
}
#[cfg(feature = "search-lindera")]
#[test]
fn search_japanese() {
let conn = &db();
conn.test_transaction::<_, (), _>(|| {
let tokenizers = SearchTokenizerConfig {
tag_tokenizer: TokenizerKind::Lindera,
content_tokenizer: TokenizerKind::Lindera,
property_tokenizer: TokenizerKind::Ngram,
};
let searcher = get_searcher(&tokenizers);
let blog = &fill_database(conn).1[0];
let title = random_hex()[..8].to_owned();
let post = Post::insert(
conn,
NewPost {
blog_id: blog.id,
slug: title.clone(),
title: title.clone(),
content: SafeString::new("ブログエンジンPlumeです。"),
published: true,
license: "CC-BY-SA".to_owned(),
ap_url: "".to_owned(),
creation_date: None,
subtitle: "".to_owned(),
source: "".to_owned(),
cover_id: None,
},
&searcher,
)
.unwrap();
searcher.commit();
assert_eq!(
searcher.search_document(conn, Query::from_str("ブログエンジン").unwrap(), (0, 1))
[0]
.id,
post.id
);
assert_eq!(
searcher.search_document(conn, Query::from_str("Plume").unwrap(), (0, 1))[0].id,
post.id
);
assert_eq!(
searcher.search_document(conn, Query::from_str("です").unwrap(), (0, 1))[0].id,
post.id
);
assert_eq!(
searcher.search_document(conn, Query::from_str("").unwrap(), (0, 1))[0].id,
post.id
);
Ok(())
});
}
#[test]
fn drop_writer() {
let searcher = get_searcher(&CONFIG.search_tokenizers);
let searcher = get_searcher();
searcher.drop_writer();
get_searcher(&CONFIG.search_tokenizers);
get_searcher();
}
}
+37 -32
View File
@@ -1,14 +1,18 @@
use crate::{
config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts,
search::query::PlumeQuery, tags::Tag, Connection, Result,
instance::Instance,
posts::Post,
schema::posts,
search::{query::PlumeQuery, tokenizer},
tags::Tag,
Connection, Result,
};
use chrono::Datelike;
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use itertools::Itertools;
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use tantivy::{
collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter,
ReloadPolicy, Term,
collector::TopDocs, directory::MmapDirectory, schema::*, tokenizer::*, Index, IndexReader,
IndexWriter, ReloadPolicy, Term,
};
use whatlang::{detect as detect_lang, Lang};
@@ -30,7 +34,7 @@ impl Searcher {
pub fn schema() -> Schema {
let tag_indexing = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer("tag_tokenizer")
.set_tokenizer("whitespace_tokenizer")
.set_index_option(IndexRecordOption::Basic),
);
@@ -66,7 +70,15 @@ impl Searcher {
schema_builder.build()
}
pub fn create(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
pub fn create(path: &dyn AsRef<Path>) -> Result<Self> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
let schema = Self::schema();
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
@@ -78,9 +90,9 @@ impl Searcher {
{
let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer);
} //to please the borrow checker
Ok(Self {
writer: Mutex::new(Some(
@@ -97,38 +109,31 @@ impl Searcher {
})
}
pub fn open(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
let mut index =
pub fn open(path: &dyn AsRef<Path>) -> Result<Self> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false).filter(LowerCaser);
let index =
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
.map_err(|_| SearcherError::IndexOpeningError)?;
{
let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("tag_tokenizer", tokenizers.tag_tokenizer);
tokenizer_manager.register("content_tokenizer", tokenizers.content_tokenizer);
tokenizer_manager.register("property_tokenizer", tokenizers.property_tokenizer);
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer);
} //to please the borrow checker
let writer = index
let mut writer = index
.writer(50_000_000)
.map_err(|_| SearcherError::WriteLockAcquisitionError)?;
// Since Tantivy v0.12.0, IndexWriter::garbage_collect_files() returns Future.
// To avoid conflict with Plume async project, we don't introduce async now.
// After async is introduced to Plume, we can use garbage_collect_files() again.
// Algorithm stolen from Tantivy's SegmentUpdater::list_files()
use std::collections::HashSet;
use std::path::PathBuf;
let mut files: HashSet<PathBuf> = index
.list_all_segment_metas()
.into_iter()
.flat_map(|segment_meta| segment_meta.list_files())
.collect();
files.insert(Path::new("meta.json").to_path_buf());
index
.directory_mut()
.garbage_collect(|| files)
writer
.garbage_collect_files()
.map_err(|_| SearcherError::IndexEditionError)?;
Ok(Self {
writer: Mutex::new(Some(writer)),
reader: index
+7 -34
View File
@@ -1,34 +1,5 @@
#[cfg(feature = "search-lindera")]
use lindera_tantivy::tokenizer::LinderaTokenizer;
use std::str::CharIndices;
use tantivy::tokenizer::*;
#[derive(Clone, Copy)]
pub enum TokenizerKind {
Simple,
Ngram,
Whitespace,
#[cfg(feature = "search-lindera")]
Lindera,
}
impl From<TokenizerKind> for TextAnalyzer {
fn from(tokenizer: TokenizerKind) -> TextAnalyzer {
use TokenizerKind::*;
match tokenizer {
Simple => TextAnalyzer::from(SimpleTokenizer)
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser),
Ngram => TextAnalyzer::from(NgramTokenizer::new(2, 8, false)).filter(LowerCaser),
Whitespace => TextAnalyzer::from(WhitespaceTokenizer).filter(LowerCaser),
#[cfg(feature = "search-lindera")]
Lindera => {
TextAnalyzer::from(LinderaTokenizer::new("decompose", "")).filter(LowerCaser)
}
}
}
}
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
/// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
/// but not splitting on punctuation
@@ -41,13 +12,15 @@ pub struct WhitespaceTokenStream<'a> {
token: Token,
}
impl Tokenizer for WhitespaceTokenizer {
fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
BoxTokenStream::from(WhitespaceTokenStream {
impl<'a> Tokenizer<'a> for WhitespaceTokenizer {
type TokenStreamImpl = WhitespaceTokenStream<'a>;
fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
WhitespaceTokenStream {
text,
chars: text.char_indices(),
token: Token::default(),
})
}
}
}
impl<'a> WhitespaceTokenStream<'a> {
+2 -2
View File
@@ -406,8 +406,8 @@ impl Bool {
}
}
Bool::HasCover => Ok(post.cover_id.is_some()),
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local() && kind == Kind::Original),
Bool::All => Ok(kind == Kind::Original),
Bool::Local => Ok(post.get_blog(&rocket.conn)?.is_local()),
Bool::All => Ok(true),
}
}
}
+2 -6
View File
@@ -54,7 +54,6 @@ pub enum Role {
}
#[derive(Queryable, Identifiable, Clone, Debug, AsChangeset)]
#[changeset_options(treat_none_as_null = "true")]
pub struct User {
pub id: i32,
pub username: String,
@@ -760,7 +759,7 @@ impl User {
mime_type: None,
href: None,
template: Some(format!(
"https://{}/remote_interact?target={{uri}}",
"https://{}/remote_interact?{{uri}}",
self.get_instance(conn)?.public_domain
)),
},
@@ -1026,7 +1025,6 @@ impl NewUser {
pub(crate) mod tests {
use super::*;
use crate::{
config::CONFIG,
instance::{tests as instance_tests, Instance},
search::tests::get_searcher,
tests::{db, rockets},
@@ -1123,9 +1121,7 @@ pub(crate) mod tests {
let inserted = fill_database(conn);
assert!(User::get(conn, inserted[0].id).is_ok());
inserted[0]
.delete(conn, &get_searcher(&CONFIG.search_tokenizers))
.unwrap();
inserted[0].delete(conn, &get_searcher()).unwrap();
assert!(User::get(conn, inserted[0].id).is_err());
Ok(())
});
+1 -1
View File
@@ -10,7 +10,7 @@ msgstr ""
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"
"X-Crowdin-Project: plume\n"
"X-Crowdin-Language: fr\n"
"X-Crowdin-File: /master/po/plume-front/plume-front.pot\n"
+1 -1
View File
@@ -10,7 +10,7 @@ msgstr ""
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"
"X-Crowdin-Project: plume\n"
"X-Crowdin-Language: fr\n"
"X-Crowdin-File: /master/po/plume/plume.pot\n"
+2 -2
View File
@@ -24,9 +24,9 @@ if [ $ARCH == "aarch64" -o $ARCH == "armv71" ] ; then
apt-get install -y --no-install-recommends build-essential subversion ninja-build cmake
mkdir -p /scratch/src
cd /scratch/src
svn co http://llvm.org/svn/llvm-project/llvm/tags/RELEASE_900/final/ llvm
svn co http://llvm.org/svn/llvm-project/llvm/tags/RELEASE_800/final/ llvm
cd /scratch/src/llvm/tools
svn co http://llvm.org/svn/llvm-project/lld/tags/RELEASE_900/final/ lld
svn co http://llvm.org/svn/llvm-project/lld/tags/RELEASE_800/final/ lld
mkdir -p /scratch/build/arm
cd /scratch/build/arm
if [ "$ARCH" == "aarch64" ] ; then
+5 -6
View File
@@ -31,21 +31,20 @@ parts:
- pkg-config
- libsqlite3-dev
- gettext
- libclang-8-dev
- on arm64,armhf,ppc64el,s390x:
- lld-8
- libclang-9-dev
- lld-9
override-build: |
snapcraftctl set-version $(git describe --tags)
export PATH=$PATH:$HOME/.cargo/bin
rustup install stable
rustup install --force --profile=minimal stable
cargo +stable install --force cargo-web
# Only Tier 1 Rust platforms get rust-lld
# On the others (arm64, armhf, powerpc64, s390x) fall back to using
# the system LLD we've installed earlier.
case ${SNAPCRAFT_ARCH_TRIPLET} in \
aarch64-linux-gnu|arm-linux-gnueabihf|powerpc64-linux-gnu|s390x-linux-gnu) \
RUSTFLAGS="-C linker=lld-8" cargo web deploy -p plume-front --release \
aarch64-linux-gnu|arm-linux-gnueabihf|powerpc64le-linux-gnu|s390x-linux-gnu) \
RUSTFLAGS="-C linker=lld-9" cargo web deploy -p plume-front --release \
;; \
*) \
cargo web deploy -p plume-front --release \
+7 -9
View File
@@ -6,6 +6,8 @@ extern crate gettext_macros;
#[macro_use]
extern crate rocket;
#[macro_use]
extern crate runtime_fmt;
#[macro_use]
extern crate serde_json;
#[macro_use]
extern crate validator_derive;
@@ -26,8 +28,7 @@ use std::sync::{Arc, Mutex};
use std::time::Duration;
init_i18n!(
"plume", ar, bg, ca, cs, de, en, eo, es, fa, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr,
sk, sv
"plume", ar, bg, ca, cs, de, en, eo, es, fr, gl, hi, hr, it, ja, nb, pl, pt, ro, ru, sr, sk, sv
);
mod api;
@@ -54,13 +55,10 @@ fn init_pool() -> Option<DbPool> {
}
let manager = ConnectionManager::<Connection>::new(CONFIG.database_url.as_str());
let mut builder = DbPool::builder()
let pool = DbPool::builder()
.connection_customizer(Box::new(PragmaForeignKey))
.min_idle(CONFIG.db_min_idle);
if let Some(max_size) = CONFIG.db_max_size {
builder = builder.max_size(max_size);
};
let pool = builder.build(manager).ok()?;
.build(manager)
.ok()?;
Instance::cache_local(&pool.get().unwrap());
Some(pool)
}
@@ -99,7 +97,7 @@ Then try to restart Plume.
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
// we want a fast exit here, so
#[allow(clippy::match_wild_err_arm)]
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers) {
let searcher = match UnmanagedSearcher::open(&CONFIG.search_index) {
Err(Error::Search(e)) => match e {
SearcherError::WriteLockAcquisitionError => panic!(
r#"
+15 -7
View File
@@ -1,4 +1,5 @@
use activitypub::collection::{OrderedCollection, OrderedCollectionPage};
use atom_syndication::{Entry, FeedBuilder};
use diesel::SaveChangesDsl;
use rocket::{
http::ContentType,
@@ -360,13 +361,20 @@ pub fn outbox_page(
pub fn atom_feed(name: String, rockets: PlumeRocket) -> Option<Content<String>> {
let blog = Blog::find_by_fqn(&rockets, &name).ok()?;
let conn = &*rockets.conn;
let entries = Post::get_recents_for_blog(&*conn, &blog, 15).ok()?;
let uri = Instance::get_local()
.ok()?
.compute_box("~", &name, "atom.xml");
let title = &blog.title;
let default_updated = &blog.creation_date;
let feed = super::build_atom_feed(entries, &uri, title, default_updated, conn);
let feed = FeedBuilder::default()
.title(blog.title.clone())
.id(Instance::get_local()
.ok()?
.compute_box("~", &name, "atom.xml"))
.entries(
Post::get_recents_for_blog(&*conn, &blog, 15)
.ok()?
.into_iter()
.map(|p| super::post_to_atom(p, &*conn))
.collect::<Vec<Entry>>(),
)
.build()
.ok()?;
Some(Content(
ContentType::new("application", "atom+xml"),
feed.to_string(),
Executable → Regular
+5 -42
View File
@@ -1,9 +1,6 @@
#![warn(clippy::too_many_arguments)]
use crate::template_utils::Ructe;
use atom_syndication::{
ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, LinkBuilder, Person, PersonBuilder,
};
use chrono::naive::NaiveDateTime;
use atom_syndication::{ContentBuilder, Entry, EntryBuilder, LinkBuilder, Person, PersonBuilder};
use plume_models::{posts::Post, Connection, CONFIG, ITEMS_PER_PAGE};
use rocket::{
http::{
@@ -118,46 +115,13 @@ pub struct RemoteForm {
pub remote: String,
}
pub fn build_atom_feed(
entries: Vec<Post>,
uri: &str,
title: &str,
default_updated: &NaiveDateTime,
conn: &Connection,
) -> Feed {
let updated = if entries.is_empty() {
default_updated
} else {
&entries[0].creation_date
};
FeedBuilder::default()
.title(title)
.id(uri)
.updated(updated.format("%Y-%m-%dT%H:%M:%SZ").to_string())
.entries(
entries
.into_iter()
.map(|p| post_to_atom(p, conn))
.collect::<Vec<Entry>>(),
)
.links(vec![LinkBuilder::default()
.href(uri)
.rel("self")
.mime_type("application/atom+xml".to_string())
.build()
.expect("Atom feed: link error")])
.build()
.expect("user::atom_feed: Error building Atom feed")
}
fn post_to_atom(post: Post, conn: &Connection) -> Entry {
let formatted_creation_date = post.creation_date.format("%Y-%m-%dT%H:%M:%SZ").to_string();
pub fn post_to_atom(post: Post, conn: &Connection) -> Entry {
EntryBuilder::default()
.title(format!("<![CDATA[{}]]>", post.title))
.content(
ContentBuilder::default()
.value(format!("<![CDATA[{}]]>", *post.content.get()))
.src(post.ap_url.clone())
.content_type("html".to_string())
.build()
.expect("Atom feed: content error"),
@@ -177,9 +141,8 @@ fn post_to_atom(post: Post, conn: &Connection) -> Entry {
)
// Using RFC 4287 format, see https://tools.ietf.org/html/rfc4287#section-3.3 for dates
// eg: 2003-12-13T18:30:02Z (Z is here because there is no timezone support with the NaiveDateTime crate)
.published(formatted_creation_date.clone())
.updated(formatted_creation_date)
.id(post.ap_url.clone())
.published(post.creation_date.format("%Y-%m-%dT%H:%M:%SZ").to_string())
.id(post.id.to_string())
.links(vec![LinkBuilder::default()
.href(post.ap_url)
.build()
+1 -1
View File
@@ -643,7 +643,7 @@ pub fn remote_interact_post(
.and_then(|blog| Post::find_by_slug(&rockets.conn, &slug, blog.id))?;
if let Some(uri) = User::fetch_remote_interact_uri(&remote.remote)
.ok()
.map(|uri| uri.replace("{uri}", &target.ap_url))
.and_then(|uri| rt_format!(uri, uri = target.ap_url).ok())
{
Ok(Redirect::to(uri).into())
} else {
+3 -3
View File
@@ -3,7 +3,7 @@ use lettre::Transport;
use rocket::http::ext::IntoOwned;
use rocket::{
http::{uri::Uri, Cookie, Cookies, SameSite},
request::LenientForm,
request::{Form, LenientForm},
response::{Flash, Redirect},
State,
};
@@ -159,7 +159,7 @@ pub struct ResetForm {
#[post("/password-reset", data = "<form>")]
pub fn password_reset_request(
mail: State<'_, Arc<Mutex<Mailer>>>,
form: LenientForm<ResetForm>,
form: Form<ResetForm>,
rockets: PlumeRocket,
) -> Ructe {
if User::find_by_email(&*rockets.conn, &form.email).is_ok() {
@@ -216,7 +216,7 @@ fn passwords_match(form: &NewPasswordForm) -> Result<(), ValidationError> {
#[post("/password-reset/<token>", data = "<form>")]
pub fn password_reset(
token: String,
form: LenientForm<NewPasswordForm>,
form: Form<NewPasswordForm>,
rockets: PlumeRocket,
) -> Result<Flash<Redirect>, Ructe> {
form.validate()
+22 -16
View File
@@ -2,6 +2,7 @@ use activitypub::{
activity::Create,
collection::{OrderedCollection, OrderedCollectionPage},
};
use atom_syndication::{Entry, FeedBuilder};
use diesel::SaveChangesDsl;
use rocket::{
http::{ContentType, Cookies},
@@ -201,14 +202,15 @@ pub fn follow_not_connected(
if let Some(uri) = User::fetch_remote_interact_uri(&remote_form)
.ok()
.and_then(|uri| {
Some(uri.replace(
"{uri}",
&format!(
rt_format!(
uri,
uri = format!(
"{}@{}",
target.fqn,
target.get_instance(&rockets.conn).ok()?.public_domain
),
))
)
)
.ok()
})
{
Ok(Redirect::to(uri).into())
@@ -397,10 +399,7 @@ pub fn update(
)
.0,
);
user.preferred_theme = form
.theme
.clone()
.and_then(|t| if &t == "" { None } else { Some(t) });
user.preferred_theme = form.theme.clone();
user.hide_custom_css = form.hide_custom_css;
let _: User = user.save_changes(&*conn).map_err(Error::from)?;
@@ -617,13 +616,20 @@ pub fn ap_followers(
pub fn atom_feed(name: String, rockets: PlumeRocket) -> Option<Content<String>> {
let conn = &*rockets.conn;
let author = User::find_by_fqn(&rockets, &name).ok()?;
let entries = Post::get_recents_for_author(conn, &author, 15).ok()?;
let uri = Instance::get_local()
.ok()?
.compute_box("@", &name, "atom.xml");
let title = &author.display_name;
let default_updated = &author.creation_date;
let feed = super::build_atom_feed(entries, &uri, title, default_updated, conn);
let feed = FeedBuilder::default()
.title(author.display_name.clone())
.id(Instance::get_local()
.unwrap()
.compute_box("@", &name, "atom.xml"))
.entries(
Post::get_recents_for_author(conn, &author, 15)
.ok()?
.into_iter()
.map(|p| super::post_to_atom(p, conn))
.collect::<Vec<Entry>>(),
)
.build()
.expect("user::atom_feed: Error building Atom feed");
Some(Content(
ContentType::new("application", "atom+xml"),
feed.to_string(),
+2 -2
View File
@@ -32,9 +32,9 @@
<form method="post" action="@uri!(instance::delete_email_blocklist)">
<header>
@if emails.is_empty() {
<p class="center" >@i18n!(ctx.1, "There are no blocked emails on your instance")</p>
} else {
<input type="submit" class="destructive" value='@i18n!(ctx.1, "Delete selected emails")'>
} else {
<p class="center" >@i18n!(ctx.1, "There are no blocked emails on your instance")</p>
}
</header>
<div class="list">
+10 -23
View File
@@ -17,30 +17,17 @@
<p class="p-summary" dir="auto">@article.subtitle</p>
</main>
<footer class="authors">
<div>
@Html(i18n!(ctx.1, "By {0}"; format!(
"<a class=\"p-author h-card\" href=\"{}\">{}</a>",
uri!(user::details: name = &article.get_authors(ctx.0).unwrap_or_default()[0].fqn),
escape(&article.get_authors(ctx.0).unwrap_or_default()[0].name())
)))
@if article.published {
<span class="dt-published" datetime="@article.creation_date.format("%F %T")">@article.creation_date.format("%B %e, %Y")</span>
}
<a href="@uri!(blogs::details: name = &article.get_blog(ctx.0).unwrap().fqn, page = _)">@article.get_blog(ctx.0).unwrap().title</a>
</div>
@Html(i18n!(ctx.1, "By {0}"; format!(
"<a class=\"p-author h-card\" href=\"{}\">{}</a>",
uri!(user::details: name = &article.get_authors(ctx.0).unwrap_or_default()[0].fqn),
escape(&article.get_authors(ctx.0).unwrap_or_default()[0].name())
)))
@if article.published {
<span class="dt-published" datetime="@article.creation_date.format("%F %T")">@article.creation_date.format("%B %e, %Y")</span>
}
<a href="@uri!(blogs::details: name = &article.get_blog(ctx.0).unwrap().fqn, page = _)">@article.get_blog(ctx.0).unwrap().title</a>
@if !article.published {
<div>⋅ @i18n!(ctx.1, "Draft")</div>
} else {
<div>
<span class="likes" aria-label="@i18n!(ctx.1, "One like", "{0} likes"; article.count_likes(ctx.0).unwrap_or_default())" title="@i18n!(ctx.1, "One like", "{0} likes"; article.count_likes(ctx.0).unwrap_or_default())">
@icon!("heart") @article.count_likes(ctx.0).unwrap_or_default()
</span>
<span class="reshares" aria-label="@i18n!(ctx.1, "One like", "{0} boost"; article.count_reshares(ctx.0).unwrap_or_default())" title="@i18n!(ctx.1, "One boost", "{0} boosts"; article.count_reshares(ctx.0).unwrap_or_default())">
@icon!("repeat") @article.count_reshares(ctx.0).unwrap_or_default()
</span>
</div>
⋅ @i18n!(ctx.1, "Draft")
}
</footer>
</div>
+3 -3
View File
@@ -40,7 +40,7 @@
<div class="article-info" dir="auto">
<span class="author">
@Html(i18n!(ctx.1, "Written by {0}"; format!("<a href=\"{}\">{}</a>",
escape(&uri!(user::details: name = &author.fqn).to_string()),
uri!(user::details: name = &author.fqn),
escape(&author.name()))))
</span>
&mdash;
@@ -103,8 +103,8 @@
</section>
} else {
<p class="center">@Html(i18n!(ctx.1, "{0}Log in{1}, or {2}use your Fediverse account{3} to interact with this article";
format!("<a href='{}'>", escape(&uri!(session::new: m = _).to_string())), "</a>",
format!("<a href='{}'>", escape(&uri!(posts::remote_interact: blog_name = &blog.fqn, slug = &article.slug).to_string())), "</a>"
format!("<a href='{}'>", uri!(session::new: m = _)), "</a>",
format!("<a href='{}'>", uri!(posts::remote_interact: blog_name = &blog.fqn, slug = &article.slug)), "</a>"
))
</p>
<section class="actions">
-1
View File
@@ -25,7 +25,6 @@
.default(login_form.password)
.error(&login_errs)
.set_prop("minlength", 1)
.input_type("password")
.html(ctx.1))
<input type="submit" value="@i18n!(ctx.1, "Log in")" />
</form>