From 175055cf9d49c77cf7a738f369eea1e4818c9714 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 24 Feb 2021 00:34:23 +0900 Subject: [PATCH 1/9] Add migration to add index to medias.file_path % diesel migration generate medias_index_file_path --- .../postgres/2021-02-23-153402_medias_index_file_path/down.sql | 1 + .../postgres/2021-02-23-153402_medias_index_file_path/up.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql create mode 100644 migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql diff --git a/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql b/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql new file mode 100644 index 00000000..291a97c5 --- /dev/null +++ b/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql b/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql new file mode 100644 index 00000000..33cf74b5 --- /dev/null +++ b/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql @@ -0,0 +1 @@ +-- Your SQL goes here \ No newline at end of file From f90d7ddee32e7c376855b24ce0b7eab657640df6 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 24 Feb 2021 00:49:50 +0900 Subject: [PATCH 2/9] Add SQL to add/drop index to/from medias.file_path --- .../postgres/2021-02-23-153402_medias_index_file_path/down.sql | 2 +- .../postgres/2021-02-23-153402_medias_index_file_path/up.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql b/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql index 291a97c5..9604e5d0 100644 --- a/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql +++ b/migrations/postgres/2021-02-23-153402_medias_index_file_path/down.sql @@ -1 +1 @@ --- This file should undo anything in `up.sql` \ No newline at end of file +DROP INDEX medias_index_file_path; diff --git a/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql b/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql index 33cf74b5..d26ad494 100644 --- a/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql +++ b/migrations/postgres/2021-02-23-153402_medias_index_file_path/up.sql @@ -1 +1 @@ --- Your SQL goes here \ No newline at end of file +CREATE INDEX medias_index_file_path ON medias (file_path); From 462c5a1d424e15eee43644a8b2c6a2f5ae6d0eda Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 24 Feb 2021 01:04:31 +0900 Subject: [PATCH 3/9] Define Media::find_by_file_path() --- plume-models/src/medias.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index e0fffb62..1882b8fe 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -65,6 +65,7 @@ impl MediaCategory { impl Media { insert!(medias, NewMedia); get!(medias); + find_by!(medias, find_by_file_path, file_path as &str); pub fn for_user(conn: &Connection, owner: i32) -> Result> { medias::table From ea1f4d48d5bb7df7a9b6514fa05fc3b56fa15571 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 24 Feb 2021 01:05:19 +0900 Subject: [PATCH 4/9] Upsert Media in from_activity() --- plume-models/src/medias.rs | 83 ++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index 1882b8fe..ba477657 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -19,7 +19,7 @@ use url::Url; const REMOTE_MEDIA_DIRECTORY: &str = "remote"; -#[derive(Clone, Identifiable, Queryable)] +#[derive(Clone, Identifiable, Queryable, AsChangeset)] pub struct Media { pub id: i32, pub file_path: String, @@ -225,32 +225,65 @@ impl Media { .copy_to(&mut dest) .ok()?; - // TODO: upsert - Media::insert( - conn, - NewMedia { - file_path: path.to_str()?.to_string(), - alt_text: image.object_props.content_string().ok()?, - is_remote: false, - remote_url: None, - sensitive: image.object_props.summary_string().is_ok(), - content_warning: image.object_props.summary_string().ok(), - owner_id: User::from_id( + Media::find_by_file_path(conn, &path.to_str()?) + .and_then(|mut media| { + let mut updated = false; + + let alt_text = image.object_props.content_string().ok()?; + let sensitive = image.object_props.summary_string().is_ok(); + let content_warning = image.object_props.summary_string().ok(); + if media.alt_text != alt_text { + media.alt_text = alt_text; + updated = true; + } + if media.is_remote { + media.is_remote = false; + updated = true; + } + if media.remote_url.is_some() { + media.remote_url = None; + updated = true; + } + if media.sensitive != sensitive { + media.sensitive = sensitive; + updated = true; + } + if media.content_warning != content_warning { + media.content_warning = content_warning; + updated = true; + } + if updated { + diesel::update(&media).set(&media).execute(&**conn)?; + } + Ok(media) + }) + .or_else(|_| { + Media::insert( conn, - image - .object_props - .attributed_to_link_vec::() - .ok()? - .into_iter() - .next()? - .as_ref(), - None, - CONFIG.proxy(), + NewMedia { + file_path: path.to_str()?.to_string(), + alt_text: image.object_props.content_string().ok()?, + is_remote: false, + remote_url: None, + sensitive: image.object_props.summary_string().is_ok(), + content_warning: image.object_props.summary_string().ok(), + owner_id: User::from_id( + conn, + image + .object_props + .attributed_to_link_vec::() + .ok()? + .into_iter() + .next()? + .as_ref(), + None, + CONFIG.proxy(), + ) + .map_err(|(_, e)| e)? + .id, + }, ) - .map_err(|(_, e)| e)? - .id, - }, - ) + }) } pub fn get_media_processor<'a>(conn: &'a Connection, user: Vec<&User>) -> MediaProcessor<'a> { From 589c159eb98b45c4d54693f17f266cf4a234dfc2 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 24 Feb 2021 02:26:04 +0900 Subject: [PATCH 5/9] Call Post::publish_updated() only for local posts --- plume-models/src/posts.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index f34d6cd0..bace3945 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -15,7 +15,7 @@ use heck::KebabCase; use once_cell::sync::Lazy; use plume_common::{ activity_pub::{ - inbox::{AsObject, FromId}, + inbox::{AsActor, AsObject, FromId}, Hashtag, Id, IntoId, Licensed, Source, PUBLIC_VISIBILITY, }, utils::md_to_html, @@ -94,7 +94,10 @@ impl Post { let post = Self::get(conn, self.id)?; // TODO: Call publish_published() when newly published if post.published { - self.publish_updated(); + let blog = post.get_blog(conn); + if blog.is_ok() && blog.unwrap().is_local() { + self.publish_updated(); + } } Ok(post) } From 33221d386ef6ed33599980581016e39242d905fa Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 24 Feb 2021 02:36:39 +0900 Subject: [PATCH 6/9] Upsert Post in from_activity() --- plume-models/src/posts.rs | 87 +++++++++++++++++++++++++++++---------- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index bace3945..de2d123e 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -641,27 +641,72 @@ impl FromId for Post { .and_then(|img| Media::from_activity(conn, &img).ok().map(|m| m.id)); let title = article.object_props.name_string()?; - // TODO: upsert - let post = Post::insert( - conn, - NewPost { - blog_id: blog?.id, - slug: title.to_kebab_case(), - title, - content: SafeString::new(&article.object_props.content_string()?), - published: true, - license, - // FIXME: This is wrong: with this logic, we may use the display URL as the AP ID. We need two different fields - ap_url: article - .object_props - .url_string() - .or_else(|_| article.object_props.id_string())?, - creation_date: Some(article.object_props.published_utctime()?.naive_utc()), - subtitle: article.object_props.summary_string()?, - source: article.ap_object_props.source_object::()?.content, - cover_id: cover, - }, - )?; + let ap_url = article + .object_props + .url_string() + .or_else(|_| article.object_props.id_string())?; + let post = Post::from_db(conn, &ap_url) + .and_then(|mut post| { + let mut updated = false; + + let slug = title.to_kebab_case(); + let content = SafeString::new(&article.object_props.content_string()?); + let subtitle = article.object_props.summary_string()?; + let source = article.ap_object_props.source_object::()?.content; + if post.slug != slug { + post.slug = slug; + updated = true; + } + if post.title != title { + post.title = title.clone(); + updated = true; + } + if post.content != content { + post.content = content; + updated = true; + } + if post.license != license { + post.license = license.clone(); + updated = true; + } + if post.subtitle != subtitle { + post.subtitle = subtitle; + updated = true; + } + if post.source != source { + post.source = source; + updated = true; + } + if post.cover_id != cover { + post.cover_id = cover; + updated = true; + } + + if updated { + post.update(conn)?; + } + + Ok(post) + }) + .or_else(|_| { + Post::insert( + conn, + NewPost { + blog_id: blog?.id, + slug: title.to_kebab_case(), + title, + content: SafeString::new(&article.object_props.content_string()?), + published: true, + license, + // FIXME: This is wrong: with this logic, we may use the display URL as the AP ID. We need two different fields + ap_url, + creation_date: Some(article.object_props.published_utctime()?.naive_utc()), + subtitle: article.object_props.summary_string()?, + source: article.ap_object_props.source_object::()?.content, + cover_id: cover, + }, + ) + })?; for author in authors { PostAuthor::insert( From 702aa11ecf9bebfd619428edea13ec3aaeb7fa87 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 25 Feb 2021 11:01:47 +0900 Subject: [PATCH 7/9] Don't drop path components of image URI --- plume-models/src/medias.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index ba477657..db5a86ab 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -12,7 +12,7 @@ use plume_common::{ }; use std::{ fs::{self, DirBuilder}, - path::{Path, PathBuf}, + path::{self, Path, PathBuf}, }; use tracing::warn; use url::Url; @@ -156,12 +156,11 @@ impl Media { if self.is_remote { Ok(self.remote_url.clone().unwrap_or_default()) } else { - let p = Path::new(&self.file_path); - let filename: String = p.file_name().unwrap().to_str().unwrap().to_owned(); + let file_path = self.file_path.replace(path::MAIN_SEPARATOR, "/"); Ok(ap_url(&format!( - "{}/static/media/{}", + "{}/{}", Instance::get_local()?.public_domain, - &filename + &file_path ))) } } From ebb0b45299cd3d419d8ffed9b7663f371ba93ddb Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 25 Feb 2021 11:03:32 +0900 Subject: [PATCH 8/9] Create blog_authors only when needed --- plume-models/src/posts.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index de2d123e..21470cb2 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -706,17 +706,20 @@ impl FromId for Post { cover_id: cover, }, ) - })?; + .and_then(|post| { + for author in authors { + PostAuthor::insert( + conn, + NewPostAuthor { + post_id: post.id, + author_id: author.id, + }, + )?; + } - for author in authors { - PostAuthor::insert( - conn, - NewPostAuthor { - post_id: post.id, - author_id: author.id, - }, - )?; - } + Ok(post) + }) + })?; // save mentions and tags let mut hashtags = md_to_html(&post.source, None, false, None) From d3c2dc828629b9691e142b3f3c13281a7f10af4c Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sun, 28 Mar 2021 03:14:02 +0900 Subject: [PATCH 9/9] Add changelog on upsert --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2658a87..2d17ccb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Percent-encode URI for remote_interact (#866, #857) - Menu animation not opening on iOS (#876, #897) - Make actors subscribe to channel once (#913) +- Upsert posts and media instead of trying to insert and fail (#912) ## [[0.6.0]] - 2020-12-29