Merge pull request 'Fix #905 Prevent remote image duplication' (#907) from dup-remote-image into main

Reviewed-on: https://git.joinplu.me/Plume/Plume/pulls/907

plume-models/src/medias.rs

@@ -10,7 +10,14 @@ use plume_common::{
     activity_pub::{inbox::FromId, Id},
     utils::MediaProcessor,
 };
-use std::{fs, path::Path};
+use std::{
+    fs::{self, DirBuilder},
+    path::{Path, PathBuf},
+};
+use tracing::warn;
+use url::Url;
+
+const REMOTE_MEDIA_DIRECTORY: &str = "remote";
 
 #[derive(Clone, Identifiable, Queryable)]
 pub struct Media {
@@ -198,18 +205,14 @@ impl Media {
     // TODO: merge with save_remote?
     pub fn from_activity(conn: &DbConn, image: &Image) -> Result<Media> {
         let remote_url = image.object_props.url_string().ok()?;
-        let ext = remote_url
+        let path = determine_mirror_file_path(&remote_url);
-            .rsplit('.')
+        let parent = path.parent()?;
-            .next()
+        if !parent.is_dir() {
-            .map(ToOwned::to_owned)
+            DirBuilder::new().recursive(true).create(parent)?;
-            .unwrap_or_else(|| String::from("png"));
+        }
-        let path = Path::new(&super::CONFIG.media_directory).join(format!(
-            "{}.{}",
-            GUID::rand().to_string(),
-            ext
-        ));
 
         let mut dest = fs::File::create(path.clone()).ok()?;
+        // TODO: conditional GET
         if let Some(proxy) = CONFIG.proxy() {
             reqwest::ClientBuilder::new().proxy(proxy.clone()).build()?
         } else {
@@ -221,6 +224,7 @@ impl Media {
         .copy_to(&mut dest)
         .ok()?;
 
+        // TODO: upsert
         Media::insert(
             conn,
             NewMedia {
@@ -262,6 +266,33 @@ impl Media {
     }
 }
 
+fn determine_mirror_file_path(url: &str) -> PathBuf {
+    let mut file_path = Path::new(&super::CONFIG.media_directory).join(REMOTE_MEDIA_DIRECTORY);
+    Url::parse(url)
+        .map(|url| {
+            if !url.has_host() {
+                return;
+            }
+            file_path.push(url.host_str().unwrap());
+            for segment in url.path_segments().expect("FIXME") {
+                file_path.push(segment);
+            }
+            // TODO: handle query
+            // HINT: Use characters which must be percent-encoded in path as separator between path and query
+            // HINT: handle extension
+        })
+        .unwrap_or_else(|err| {
+            warn!("Failed to parse url: {} {}", &url, err);
+            let ext = url
+                .rsplit('.')
+                .next()
+                .map(ToOwned::to_owned)
+                .unwrap_or_else(|| String::from("png"));
+            file_path.push(format!("{}.{}", GUID::rand().to_string(), ext));
+        });
+    file_path
+}
+
 #[cfg(test)]
 pub(crate) mod tests {
     use super::*;

plume-models/src/posts.rs

@@ -644,6 +644,7 @@ impl FromId<DbConn> for Post {
             .and_then(|img| Media::from_activity(conn, &img).ok().map(|m| m.id));
 
         let title = article.object_props.name_string()?;
+        // TODO: upsert
         let post = Post::insert(
             conn,
             NewPost {