Compare commits

..

1 Commits

Author SHA1 Message Date
Trinity Pointard 6ef8ace025 attempt to do non anonymous ldap connect 2021-02-21 15:34:44 +01:00
22 changed files with 146 additions and 344 deletions
-2
View File
@@ -24,8 +24,6 @@
- Percent-encode URI for remote_interact (#866, #857)
- Menu animation not opening on iOS (#876, #897)
- Make actors subscribe to channel once (#913)
- Upsert posts and media instead of trying to insert and fail (#912)
## [[0.6.0]] - 2020-12-29
Generated
+3 -5
View File
@@ -1,7 +1,5 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "activitypub"
version = "0.1.6"
@@ -2039,13 +2037,13 @@ checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a"
[[package]]
name = "lexical-core"
version = "0.7.5"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374"
checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
dependencies = [
"arrayvec",
"bitflags 1.2.1",
"cfg-if 1.0.0",
"cfg-if 0.1.10",
"ryu",
"static_assertions",
]
-1
View File
@@ -40,7 +40,6 @@ WORKDIR /app
COPY --from=builder /app /app
COPY --from=builder /usr/local/cargo/bin/plm /bin/
COPY --from=builder /usr/local/cargo/bin/plume /bin/
COPY --from=builder /usr/local/cargo/bin/remove-dup-images /bin/
CMD ["plume"]
@@ -1 +0,0 @@
DROP INDEX medias_index_file_path;
@@ -1 +0,0 @@
CREATE INDEX medias_index_file_path ON medias (file_path);
@@ -1 +0,0 @@
DROP INDEX medias_index_file_path;
@@ -1 +0,0 @@
CREATE INDEX medias_index_file_path ON medias (file_path);
-4
View File
@@ -8,10 +8,6 @@ edition = "2018"
name = "plm"
path = "src/main.rs"
[[bin]]
name = "remove-dup-images"
path = "src/remove-dup-images.rs"
[dependencies]
clap = "2.33"
dotenv = "0.14"
-149
View File
@@ -1,149 +0,0 @@
use diesel::{
BoolExpressionMethods, Connection, ExpressionMethods, JoinOnDsl, NullableExpressionMethods,
QueryDsl, RunQueryDsl,
};
use plume_models::{
blogs::Blog, instance::Instance, medias::Media, posts::Post, Connection as Conn, CONFIG,
};
use std::collections::hash_map::{DefaultHasher, HashMap};
use std::fs::File;
use std::hash::Hasher;
use std::io::{BufReader, Read};
use std::path::Path;
fn main() {
match dotenv::dotenv() {
Ok(path) => eprintln!("Configuration read from {}", path.display()),
Err(ref e) if e.not_found() => eprintln!("no .env was found"),
e => e.map(|_| ()).unwrap(),
}
let conn = Conn::establish(CONFIG.database_url.as_str()).expect("extablish connection");
Instance::cache_local(&conn);
let covers = get_remote_post_covers(&conn);
let remote_media_hashes = calculate_remote_media_hashes(covers);
eprintln!("remote medias: {:?}", remote_media_hashes);
let orphan_medias = get_orphan_medias(&conn);
eprintln!("{:?} orphan media(s)", orphan_medias.len());
for media in orphan_medias {
match calculate_file_hash(&Path::new(&media.file_path)) {
Some(hash) => {
match remote_media_hashes.get(&hash) {
Some(file_path) => {
eprintln!(
"File already referred. Removes only medias record. {}",
&file_path
);
// Remove medias record
diesel::delete(&media)
.execute(&conn)
.expect("Delete medias record");
}
None => {
eprintln!("Removes {}", &media.file_path);
// Remove file and medias record
media.delete(&conn).expect("Delete media record and file");
}
}
}
None => {
eprintln!(
"File doesn't exist. Removes medias record. medias.id: {}, path: {}",
&media.id, &media.file_path
);
diesel::delete(&media)
.execute(&conn)
.expect("Delete medias record");
}
}
}
}
fn get_remote_post_covers(conn: &Conn) -> Vec<Media> {
use plume_models::schema::blogs;
use plume_models::schema::posts;
let remote_instances = Instance::get_remotes(&conn).expect("get remote instances");
let remote_instance_ids = remote_instances.iter().map(|instance| instance.id);
let remote_blogs = blogs::table
.filter(blogs::instance_id.eq_any(remote_instance_ids))
.load::<Blog>(conn)
.expect("remote blogs");
let remote_blog_ids = remote_blogs.iter().map(|blog| blog.id);
let remote_posts = posts::table
.filter(posts::blog_id.eq_any(remote_blog_ids))
.load::<Post>(conn)
.expect("remote posts");
remote_posts
.iter()
.filter_map(|post| post.cover_id)
.map(|cover_id| Media::get(conn, cover_id).expect("Media"))
.collect()
}
fn calculate_remote_media_hashes(medias: Vec<Media>) -> HashMap<u64, String> {
let mut media_hashes = HashMap::new();
for media in medias.iter() {
if let Some(hash) = calculate_file_hash(Path::new(&media.file_path)) {
let _ = media_hashes.insert(hash, media.file_path.clone());
}
}
media_hashes
}
fn calculate_file_hash(path: &Path) -> Option<u64> {
if !path.exists() {
return None;
}
let file = File::open(path).expect("open file");
let mut reader = BufReader::new(file);
let mut hasher = DefaultHasher::new();
let mut buffer = [0; 2048];
while let Ok(n) = reader.read(&mut buffer) {
hasher.write(&buffer);
if n == 0 {
break;
}
}
Some(hasher.finish())
}
fn get_orphan_medias(conn: &Conn) -> Vec<Media> {
use plume_models::schema::{self, medias};
use plume_models::schema::{blogs::dsl::blogs, posts::dsl::posts, users::dsl::users};
let query = medias::table
.select((
medias::id,
medias::file_path,
medias::alt_text,
medias::is_remote,
medias::remote_url,
medias::sensitive,
medias::content_warning,
medias::owner_id,
))
.left_outer_join(users.on(schema::users::avatar_id.eq(medias::id.nullable())))
.left_outer_join(
blogs.on(schema::blogs::icon_id
.eq(medias::id.nullable())
.or(schema::blogs::banner_id.eq(medias::id.nullable()))),
)
.left_outer_join(posts.on(schema::posts::cover_id.eq(medias::id.nullable())))
.filter(
schema::users::avatar_id.is_null().and(
schema::blogs::icon_id.is_null().and(
schema::blogs::banner_id.is_null().and(
schema::posts::cover_id
.is_null()
.and(medias::is_remote.eq(false)),
),
),
),
);
eprintln!(
"query for orphan medias: {}",
diesel::debug_query::<_, _>(&query)
);
query.load::<Media>(conn).expect("Load orphan medias")
}
+3 -1
View File
@@ -255,7 +255,9 @@ impl FromId<DbConn> for Comment {
.and_then(|m| {
let author = &Post::get(conn, comm.post_id)?.get_authors(conn)?[0];
let not_author = m.link_props.href_string()? != author.ap_url.clone();
Mention::from_activity(conn, &m, comm.id, false, not_author)
Ok(Mention::from_activity(
conn, &m, comm.id, false, not_author,
)?)
})
.ok();
}
+15 -3
View File
@@ -164,8 +164,11 @@ impl Default for LogoConfig {
};
let mut custom_icons = env::vars()
.filter_map(|(var, val)| {
var.strip_prefix("PLUME_LOGO_")
.map(|size| (size.to_owned(), val))
if let Some(size) = var.strip_prefix("PLUME_LOGO_") {
Some((size.to_owned(), val))
} else {
None
}
})
.filter_map(|(var, val)| var.parse::<u64>().ok().map(|var| (var, val)))
.map(|(dim, src)| Icon {
@@ -251,6 +254,7 @@ pub struct LdapConfig {
pub tls: bool,
pub user_name_attr: String,
pub mail_attr: String,
pub user: Option<(String, String)>,
}
fn get_ldap_config() -> Option<LdapConfig> {
@@ -266,16 +270,24 @@ fn get_ldap_config() -> Option<LdapConfig> {
};
let user_name_attr = var("LDAP_USER_NAME_ATTR").unwrap_or_else(|_| "cn".to_owned());
let mail_attr = var("LDAP_USER_MAIL_ATTR").unwrap_or_else(|_| "mail".to_owned());
let user = var("LDAP_USER").ok();
let password = var("LDAP_PASSWORD").ok();
let user = match (user, password) {
(Some(user), Some(password)) => Some((user, password)),
(None, None) => None,
_ => panic!("Invalid LDAP configuration both or neither of LDAP_USER and LDAP_PASSWORD must be set")
};
Some(LdapConfig {
addr,
base_dn,
tls,
user_name_attr,
mail_attr,
user
})
}
(None, None) => None,
(_, _) => {
_ => {
panic!("Invalid LDAP configuration : both LDAP_ADDR and LDAP_BASE_DN must be set")
}
}
+1 -2
View File
@@ -285,8 +285,7 @@ impl List {
.select(list_elems::word)
.load::<Option<String>>(conn)
.map_err(Error::from)
// .map(|r| r.into_iter().filter_map(|o| o).collect::<Vec<String>>())
.map(|r| r.into_iter().flatten().collect::<Vec<String>>())
.map(|r| r.into_iter().filter_map(|o| o).collect::<Vec<String>>())
}
pub fn clear(&self, conn: &Connection) -> Result<()> {
+30 -63
View File
@@ -12,14 +12,14 @@ use plume_common::{
};
use std::{
fs::{self, DirBuilder},
path::{self, Path, PathBuf},
path::{Path, PathBuf},
};
use tracing::warn;
use url::Url;
const REMOTE_MEDIA_DIRECTORY: &str = "remote";
#[derive(Clone, Identifiable, Queryable, AsChangeset)]
#[derive(Clone, Identifiable, Queryable)]
pub struct Media {
pub id: i32,
pub file_path: String,
@@ -65,7 +65,6 @@ impl MediaCategory {
impl Media {
insert!(medias, NewMedia);
get!(medias);
find_by!(medias, find_by_file_path, file_path as &str);
pub fn for_user(conn: &Connection, owner: i32) -> Result<Vec<Media>> {
medias::table
@@ -156,11 +155,12 @@ impl Media {
if self.is_remote {
Ok(self.remote_url.clone().unwrap_or_default())
} else {
let file_path = self.file_path.replace(path::MAIN_SEPARATOR, "/");
let p = Path::new(&self.file_path);
let filename: String = p.file_name().unwrap().to_str().unwrap().to_owned();
Ok(ap_url(&format!(
"{}/{}",
"{}/static/media/{}",
Instance::get_local()?.public_domain,
&file_path
&filename
)))
}
}
@@ -224,65 +224,32 @@ impl Media {
.copy_to(&mut dest)
.ok()?;
Media::find_by_file_path(conn, &path.to_str()?)
.and_then(|mut media| {
let mut updated = false;
let alt_text = image.object_props.content_string().ok()?;
let sensitive = image.object_props.summary_string().is_ok();
let content_warning = image.object_props.summary_string().ok();
if media.alt_text != alt_text {
media.alt_text = alt_text;
updated = true;
}
if media.is_remote {
media.is_remote = false;
updated = true;
}
if media.remote_url.is_some() {
media.remote_url = None;
updated = true;
}
if media.sensitive != sensitive {
media.sensitive = sensitive;
updated = true;
}
if media.content_warning != content_warning {
media.content_warning = content_warning;
updated = true;
}
if updated {
diesel::update(&media).set(&media).execute(&**conn)?;
}
Ok(media)
})
.or_else(|_| {
Media::insert(
// TODO: upsert
Media::insert(
conn,
NewMedia {
file_path: path.to_str()?.to_string(),
alt_text: image.object_props.content_string().ok()?,
is_remote: false,
remote_url: None,
sensitive: image.object_props.summary_string().is_ok(),
content_warning: image.object_props.summary_string().ok(),
owner_id: User::from_id(
conn,
NewMedia {
file_path: path.to_str()?.to_string(),
alt_text: image.object_props.content_string().ok()?,
is_remote: false,
remote_url: None,
sensitive: image.object_props.summary_string().is_ok(),
content_warning: image.object_props.summary_string().ok(),
owner_id: User::from_id(
conn,
image
.object_props
.attributed_to_link_vec::<Id>()
.ok()?
.into_iter()
.next()?
.as_ref(),
None,
CONFIG.proxy(),
)
.map_err(|(_, e)| e)?
.id,
},
image
.object_props
.attributed_to_link_vec::<Id>()
.ok()?
.into_iter()
.next()?
.as_ref(),
None,
CONFIG.proxy(),
)
})
.map_err(|(_, e)| e)?
.id,
},
)
}
pub fn get_media_processor<'a>(conn: &'a Connection, user: Vec<&User>) -> MediaProcessor<'a> {
+39 -84
View File
@@ -15,7 +15,7 @@ use heck::KebabCase;
use once_cell::sync::Lazy;
use plume_common::{
activity_pub::{
inbox::{AsActor, AsObject, FromId},
inbox::{AsObject, FromId},
Hashtag, Id, IntoId, Licensed, Source, PUBLIC_VISIBILITY,
},
utils::md_to_html,
@@ -94,10 +94,7 @@ impl Post {
let post = Self::get(conn, self.id)?;
// TODO: Call publish_published() when newly published
if post.published {
let blog = post.get_blog(conn);
if blog.is_ok() && blog.unwrap().is_local() {
self.publish_updated();
}
self.publish_updated();
}
Ok(post)
}
@@ -446,7 +443,13 @@ impl Post {
m,
)
})
.filter_map(|(id, m)| id.map(|id| (m, id)))
.filter_map(|(id, m)| {
if let Some(id) = id {
Some((m, id))
} else {
None
}
})
.collect::<Vec<_>>();
let old_mentions = Mention::list_for_post(&conn, self.id)?;
@@ -641,85 +644,37 @@ impl FromId<DbConn> for Post {
.and_then(|img| Media::from_activity(conn, &img).ok().map(|m| m.id));
let title = article.object_props.name_string()?;
let ap_url = article
.object_props
.url_string()
.or_else(|_| article.object_props.id_string())?;
let post = Post::from_db(conn, &ap_url)
.and_then(|mut post| {
let mut updated = false;
// TODO: upsert
let post = Post::insert(
conn,
NewPost {
blog_id: blog?.id,
slug: title.to_kebab_case(),
title,
content: SafeString::new(&article.object_props.content_string()?),
published: true,
license,
// FIXME: This is wrong: with this logic, we may use the display URL as the AP ID. We need two different fields
ap_url: article
.object_props
.url_string()
.or_else(|_| article.object_props.id_string())?,
creation_date: Some(article.object_props.published_utctime()?.naive_utc()),
subtitle: article.object_props.summary_string()?,
source: article.ap_object_props.source_object::<Source>()?.content,
cover_id: cover,
},
)?;
let slug = title.to_kebab_case();
let content = SafeString::new(&article.object_props.content_string()?);
let subtitle = article.object_props.summary_string()?;
let source = article.ap_object_props.source_object::<Source>()?.content;
if post.slug != slug {
post.slug = slug;
updated = true;
}
if post.title != title {
post.title = title.clone();
updated = true;
}
if post.content != content {
post.content = content;
updated = true;
}
if post.license != license {
post.license = license.clone();
updated = true;
}
if post.subtitle != subtitle {
post.subtitle = subtitle;
updated = true;
}
if post.source != source {
post.source = source;
updated = true;
}
if post.cover_id != cover {
post.cover_id = cover;
updated = true;
}
if updated {
post.update(conn)?;
}
Ok(post)
})
.or_else(|_| {
Post::insert(
conn,
NewPost {
blog_id: blog?.id,
slug: title.to_kebab_case(),
title,
content: SafeString::new(&article.object_props.content_string()?),
published: true,
license,
// FIXME: This is wrong: with this logic, we may use the display URL as the AP ID. We need two different fields
ap_url,
creation_date: Some(article.object_props.published_utctime()?.naive_utc()),
subtitle: article.object_props.summary_string()?,
source: article.ap_object_props.source_object::<Source>()?.content,
cover_id: cover,
},
)
.and_then(|post| {
for author in authors {
PostAuthor::insert(
conn,
NewPostAuthor {
post_id: post.id,
author_id: author.id,
},
)?;
}
Ok(post)
})
})?;
for author in authors {
PostAuthor::insert(
conn,
NewPostAuthor {
post_id: post.id,
author_id: author.id,
},
)?;
}
// save mentions and tags
let mut hashtags = md_to_html(&post.source, None, false, None)
+11 -9
View File
@@ -17,23 +17,25 @@ pub struct RemoteFetchActor {
impl RemoteFetchActor {
pub fn init(conn: DbPool) {
let actor = ACTOR_SYS
ACTOR_SYS
.actor_of_args::<RemoteFetchActor, _>("remote-fetch", conn)
.expect("Failed to initialize remote fetch actor");
USER_CHAN.tell(
Subscribe {
actor: Box::new(actor),
topic: "*".into(),
},
None,
)
}
}
impl Actor for RemoteFetchActor {
type Msg = UserEvent;
fn pre_start(&mut self, ctx: &Context<Self::Msg>) {
USER_CHAN.tell(
Subscribe {
actor: Box::new(ctx.myself()),
topic: "*".into(),
},
None,
)
}
fn recv(&mut self, _ctx: &Context<Self::Msg>, msg: Self::Msg, _sender: Sender) {
use UserEvent::*;
+1 -1
View File
@@ -156,7 +156,7 @@ impl<'de> Deserialize<'de> for SafeString {
where
D: Deserializer<'de>,
{
deserializer.deserialize_string(SafeStringVisitor)
Ok(deserializer.deserialize_string(SafeStringVisitor)?)
}
}
+11 -9
View File
@@ -13,23 +13,25 @@ pub struct SearchActor {
impl SearchActor {
pub fn init(searcher: Arc<Searcher>, conn: DbPool) {
let actor = ACTOR_SYS
ACTOR_SYS
.actor_of_args::<SearchActor, _>("search", (searcher, conn))
.expect("Failed to initialize searcher actor");
POST_CHAN.tell(
Subscribe {
actor: Box::new(actor),
topic: "*".into(),
},
None,
)
}
}
impl Actor for SearchActor {
type Msg = PostEvent;
fn pre_start(&mut self, ctx: &Context<Self::Msg>) {
POST_CHAN.tell(
Subscribe {
actor: Box::new(ctx.myself()),
topic: "*".into(),
},
None,
)
}
fn recv(&mut self, _ctx: &Context<Self::Msg>, msg: Self::Msg, _sender: Sender) {
use PostEvent::*;
+3 -2
View File
@@ -601,10 +601,11 @@ fn parse_l<'a, 'b>(stream: &'b [Token<'a>]) -> QueryResult<(&'b [Token<'a>], Lis
}
fn parse_m<'a, 'b>(mut stream: &'b [Token<'a>]) -> QueryResult<(&'b [Token<'a>], Vec<&'a str>)> {
let mut res: Vec<&str> = vec![match stream.get(0)? {
let mut res: Vec<&str> = Vec::new();
res.push(match stream.get(0)? {
Token::Word(_, _, w) => w,
t => return t.get_error(Token::Word(0, 0, "any word")),
}];
});
stream = &stream[1..];
while let Token::Comma(_) = stream[0] {
res.push(match stream.get(1)? {
+25 -1
View File
@@ -293,6 +293,21 @@ impl User {
bcrypt::hash(pass, 10).map_err(Error::from)
}
fn ldap_preconn(ldap_conn: &mut LdapConn) -> Result<()> {
let ldap = CONFIG.ldap.as_ref().unwrap();
if let Some((user, password)) = ldap.user.as_ref() {
let bind = ldap_conn
.simple_bind(user, password)
.map_err(|_| Error::NotFound)?;
if bind.success().is_err() {
return Err(Error::NotFound);
}
}
Ok(())
}
fn ldap_register(conn: &Connection, name: &str, password: &str) -> Result<User> {
if CONFIG.ldap.is_none() {
return Err(Error::NotFound);
@@ -300,6 +315,9 @@ impl User {
let ldap = CONFIG.ldap.as_ref().unwrap();
let mut ldap_conn = LdapConn::new(&ldap.addr).map_err(|_| Error::NotFound)?;
User::ldap_preconn(&mut ldap_conn)?;
let ldap_name = format!("{}={},{}", ldap.user_name_attr, name, ldap.base_dn);
let bind = ldap_conn
.simple_bind(&ldap_name, password)
@@ -346,6 +364,9 @@ impl User {
} else {
return false;
};
if User::ldap_preconn(&mut conn).is_err() {
return false;
}
let name = format!(
"{}={},{}",
ldap.user_name_attr, &self.username, ldap.base_dn
@@ -486,7 +507,10 @@ impl User {
.filter_map(|j| serde_json::from_value(j.clone()).ok())
.collect::<Vec<T>>();
let next = json.get("next").map(|x| x.as_str().unwrap().to_owned());
let next = match json.get("next") {
Some(x) => Some(x.as_str().unwrap().to_owned()),
None => None,
};
Ok((items, next))
}
pub fn fetch_outbox<T: Activity>(&self) -> Result<Vec<T>> {
+1 -1
View File
@@ -1 +1 @@
nightly-2021-03-25
nightly-2021-01-15
+1 -1
View File
@@ -25,7 +25,7 @@ parts:
plume:
plugin: rust
source: .
rust-revision: nightly-2021-03-25
rust-revision: nightly-2020-01-15
build-packages:
- libssl-dev
- pkg-config
+2 -2
View File
@@ -348,7 +348,7 @@ pub fn update(
#[get("/~/<name>/outbox")]
pub fn outbox(name: String, conn: DbConn) -> Option<ActivityStream<OrderedCollection>> {
let blog = Blog::find_by_fqn(&conn, &name).ok()?;
blog.outbox(&conn).ok()
Some(blog.outbox(&conn).ok()?)
}
#[allow(unused_variables)]
#[get("/~/<name>/outbox?<page>")]
@@ -358,7 +358,7 @@ pub fn outbox_page(
conn: DbConn,
) -> Option<ActivityStream<OrderedCollectionPage>> {
let blog = Blog::find_by_fqn(&conn, &name).ok()?;
blog.outbox_page(&conn, page.limits()).ok()
Some(blog.outbox_page(&conn, page.limits()).ok()?)
}
#[get("/~/<name>/atom.xml")]
pub fn atom_feed(name: String, conn: DbConn) -> Option<Content<String>> {