Merge pull request 'Upsert posts and media when fetching remote posts' (#912) from upsert into main

Reviewed-on: https://git.joinplu.me/Plume/Plume/pulls/912
This commit is contained in:
KitaitiMakoto 2021-03-27 19:27:59 +00:00
commit 16953ea907
5 changed files with 149 additions and 62 deletions

View File

@ -25,6 +25,7 @@
- Percent-encode URI for remote_interact (#866, #857) - Percent-encode URI for remote_interact (#866, #857)
- Menu animation not opening on iOS (#876, #897) - Menu animation not opening on iOS (#876, #897)
- Make actors subscribe to channel once (#913) - Make actors subscribe to channel once (#913)
- Upsert posts and media instead of trying to insert and fail (#912)
## [[0.6.0]] - 2020-12-29 ## [[0.6.0]] - 2020-12-29

View File

@ -0,0 +1 @@
DROP INDEX medias_index_file_path;

View File

@ -0,0 +1 @@
CREATE INDEX medias_index_file_path ON medias (file_path);

View File

@ -12,14 +12,14 @@ use plume_common::{
}; };
use std::{ use std::{
fs::{self, DirBuilder}, fs::{self, DirBuilder},
path::{Path, PathBuf}, path::{self, Path, PathBuf},
}; };
use tracing::warn; use tracing::warn;
use url::Url; use url::Url;
const REMOTE_MEDIA_DIRECTORY: &str = "remote"; const REMOTE_MEDIA_DIRECTORY: &str = "remote";
#[derive(Clone, Identifiable, Queryable)] #[derive(Clone, Identifiable, Queryable, AsChangeset)]
pub struct Media { pub struct Media {
pub id: i32, pub id: i32,
pub file_path: String, pub file_path: String,
@ -65,6 +65,7 @@ impl MediaCategory {
impl Media { impl Media {
insert!(medias, NewMedia); insert!(medias, NewMedia);
get!(medias); get!(medias);
find_by!(medias, find_by_file_path, file_path as &str);
pub fn for_user(conn: &Connection, owner: i32) -> Result<Vec<Media>> { pub fn for_user(conn: &Connection, owner: i32) -> Result<Vec<Media>> {
medias::table medias::table
@ -155,12 +156,11 @@ impl Media {
if self.is_remote { if self.is_remote {
Ok(self.remote_url.clone().unwrap_or_default()) Ok(self.remote_url.clone().unwrap_or_default())
} else { } else {
let p = Path::new(&self.file_path); let file_path = self.file_path.replace(path::MAIN_SEPARATOR, "/");
let filename: String = p.file_name().unwrap().to_str().unwrap().to_owned();
Ok(ap_url(&format!( Ok(ap_url(&format!(
"{}/static/media/{}", "{}/{}",
Instance::get_local()?.public_domain, Instance::get_local()?.public_domain,
&filename &file_path
))) )))
} }
} }
@ -224,7 +224,39 @@ impl Media {
.copy_to(&mut dest) .copy_to(&mut dest)
.ok()?; .ok()?;
// TODO: upsert Media::find_by_file_path(conn, &path.to_str()?)
.and_then(|mut media| {
let mut updated = false;
let alt_text = image.object_props.content_string().ok()?;
let sensitive = image.object_props.summary_string().is_ok();
let content_warning = image.object_props.summary_string().ok();
if media.alt_text != alt_text {
media.alt_text = alt_text;
updated = true;
}
if media.is_remote {
media.is_remote = false;
updated = true;
}
if media.remote_url.is_some() {
media.remote_url = None;
updated = true;
}
if media.sensitive != sensitive {
media.sensitive = sensitive;
updated = true;
}
if media.content_warning != content_warning {
media.content_warning = content_warning;
updated = true;
}
if updated {
diesel::update(&media).set(&media).execute(&**conn)?;
}
Ok(media)
})
.or_else(|_| {
Media::insert( Media::insert(
conn, conn,
NewMedia { NewMedia {
@ -250,6 +282,7 @@ impl Media {
.id, .id,
}, },
) )
})
} }
pub fn get_media_processor<'a>(conn: &'a Connection, user: Vec<&User>) -> MediaProcessor<'a> { pub fn get_media_processor<'a>(conn: &'a Connection, user: Vec<&User>) -> MediaProcessor<'a> {

View File

@ -15,7 +15,7 @@ use heck::KebabCase;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use plume_common::{ use plume_common::{
activity_pub::{ activity_pub::{
inbox::{AsObject, FromId}, inbox::{AsActor, AsObject, FromId},
Hashtag, Id, IntoId, Licensed, Source, PUBLIC_VISIBILITY, Hashtag, Id, IntoId, Licensed, Source, PUBLIC_VISIBILITY,
}, },
utils::md_to_html, utils::md_to_html,
@ -94,8 +94,11 @@ impl Post {
let post = Self::get(conn, self.id)?; let post = Self::get(conn, self.id)?;
// TODO: Call publish_published() when newly published // TODO: Call publish_published() when newly published
if post.published { if post.published {
let blog = post.get_blog(conn);
if blog.is_ok() && blog.unwrap().is_local() {
self.publish_updated(); self.publish_updated();
} }
}
Ok(post) Ok(post)
} }
@ -638,8 +641,55 @@ impl FromId<DbConn> for Post {
.and_then(|img| Media::from_activity(conn, &img).ok().map(|m| m.id)); .and_then(|img| Media::from_activity(conn, &img).ok().map(|m| m.id));
let title = article.object_props.name_string()?; let title = article.object_props.name_string()?;
// TODO: upsert let ap_url = article
let post = Post::insert( .object_props
.url_string()
.or_else(|_| article.object_props.id_string())?;
let post = Post::from_db(conn, &ap_url)
.and_then(|mut post| {
let mut updated = false;
let slug = title.to_kebab_case();
let content = SafeString::new(&article.object_props.content_string()?);
let subtitle = article.object_props.summary_string()?;
let source = article.ap_object_props.source_object::<Source>()?.content;
if post.slug != slug {
post.slug = slug;
updated = true;
}
if post.title != title {
post.title = title.clone();
updated = true;
}
if post.content != content {
post.content = content;
updated = true;
}
if post.license != license {
post.license = license.clone();
updated = true;
}
if post.subtitle != subtitle {
post.subtitle = subtitle;
updated = true;
}
if post.source != source {
post.source = source;
updated = true;
}
if post.cover_id != cover {
post.cover_id = cover;
updated = true;
}
if updated {
post.update(conn)?;
}
Ok(post)
})
.or_else(|_| {
Post::insert(
conn, conn,
NewPost { NewPost {
blog_id: blog?.id, blog_id: blog?.id,
@ -649,17 +699,14 @@ impl FromId<DbConn> for Post {
published: true, published: true,
license, license,
// FIXME: This is wrong: with this logic, we may use the display URL as the AP ID. We need two different fields // FIXME: This is wrong: with this logic, we may use the display URL as the AP ID. We need two different fields
ap_url: article ap_url,
.object_props
.url_string()
.or_else(|_| article.object_props.id_string())?,
creation_date: Some(article.object_props.published_utctime()?.naive_utc()), creation_date: Some(article.object_props.published_utctime()?.naive_utc()),
subtitle: article.object_props.summary_string()?, subtitle: article.object_props.summary_string()?,
source: article.ap_object_props.source_object::<Source>()?.content, source: article.ap_object_props.source_object::<Source>()?.content,
cover_id: cover, cover_id: cover,
}, },
)?; )
.and_then(|post| {
for author in authors { for author in authors {
PostAuthor::insert( PostAuthor::insert(
conn, conn,
@ -670,6 +717,10 @@ impl FromId<DbConn> for Post {
)?; )?;
} }
Ok(post)
})
})?;
// save mentions and tags // save mentions and tags
let mut hashtags = md_to_html(&post.source, None, false, None) let mut hashtags = md_to_html(&post.source, None, false, None)
.2 .2