Add support for generic timeline (#525)

* Begin adding support for timeline

* fix some bugs with parser

* fmt

* add error reporting for parser

* add tests for timeline query parser

* add rejection tests for parse

* begin adding support for lists

also run migration before compiling, so schema.rs is up to date

* add sqlite migration

* end adding lists

still miss tests and query integration

* cargo fmt

* try to add some tests

* Add some constraint to db, and fix list test

and refactor other tests to use begin_transaction

* add more tests for lists

* add support for lists in query executor

* add keywords for including/excluding boosts and likes

* cargo fmt

* add function to list lists used by query

will make it easier to warn users when creating timeline with unknown lists

* add lang support

* add timeline creation error message when using unexisting lists

* Update .po files

* WIP: interface for timelines

* don't use diesel for migrations

not sure how it passed the ci on the other branch

* add some tests for timeline

add an int representing the order of timelines (first one will be on
top, second just under...)
use first() instead of limit(1).get().into_iter().nth(0)
remove migrations from build artifacts as they are now compiled in

* cargo fmt

* remove timeline order

* fix tests

* add tests for timeline creation failure

* cargo fmt

* add tests for timelines

* add test for matching direct lists and keywords

* add test for language filtering

* Add a more complex test for Timeline::matches, and fix TQ::matches for TQ::Or

* Make the main crate compile + FMT

* Use the new timeline system

- Replace the old "feed" system with timelines
- Display all timelines someone can access on their home page (either their personal ones, or instance timelines)
- Remove functions that were used to get user/local/federated feed
- Add new posts to timelines
- Create a default timeline called "My feed" for everyone, and "Local feed"/"Federated feed" with timelines

@fdb-hiroshima I don't know if that's how you pictured it? If you imagined it differently I can of course make changes.

I hope I didn't forgot anything…

* Cargo fmt

* Try to fix the migration

* Fix tests

* Fix the test (for real this time ?)

* Fix the tests ? + fmt

* Use Kind::Like and Kind::Reshare when needed

* Forgot to run cargo fmt once again

* revert translations

* fix reviewed stuff

* reduce code duplication by macros

* cargo fmt

src/api/posts.rs

@@ -7,7 +7,7 @@ use plume_api::posts::*;
 use plume_common::{activity_pub::broadcast, utils::md_to_html};
 use plume_models::{
     blogs::Blog, db_conn::DbConn, instance::Instance, medias::Media, mentions::*, post_authors::*,
-    posts::*, safe_string::SafeString, tags::*, users::User, Error, PlumeRocket,
+    posts::*, safe_string::SafeString, tags::*, timeline::*, users::User, Error, PlumeRocket,
 };
 
 #[get("/posts/<id>")]
@@ -204,6 +204,8 @@ pub fn create(
         worker.execute(move || broadcast(&author, act, dest));
     }
 
+    Timeline::add_to_all_timelines(&rockets, &post, Kind::Original)?;
+
     Ok(Json(PostData {
         authors: post.get_authors(conn)?.into_iter().map(|a| a.fqn).collect(),
         creation_date: post.creation_date.format("%Y-%m-%d").to_string(),

src/main.rs

@@ -193,9 +193,6 @@ Then try to restart Plume
                 routes::comments::delete,
                 routes::comments::activity_pub,
                 routes::instance::index,
-                routes::instance::local,
-                routes::instance::feed,
-                routes::instance::federated,
                 routes::instance::admin,
                 routes::instance::admin_mod,
                 routes::instance::admin_instances,
@@ -243,6 +240,12 @@ Then try to restart Plume
                 routes::plume_static_files,
                 routes::static_files,
                 routes::tags::tag,
+                routes::timelines::details,
+                routes::timelines::new,
+                routes::timelines::create,
+                routes::timelines::edit,
+                routes::timelines::update,
+                routes::timelines::delete,
                 routes::user::me,
                 routes::user::details,
                 routes::user::dashboard,

src/routes/instance.rs

@@ -20,6 +20,7 @@ use plume_models::{
     posts::Post,
     safe_string::SafeString,
     search::Searcher,
+    timeline::Timeline,
     users::{Role, User},
     Connection, Error, PlumeRocket, CONFIG,
 };
@@ -30,64 +31,23 @@ use template_utils::{IntoContext, Ructe};
 pub fn index(rockets: PlumeRocket) -> Result<Ructe, ErrorPage> {
     let conn = &*rockets.conn;
     let inst = Instance::get_local()?;
-    let federated = Post::get_recents_page(conn, Page::default().limits())?;
-    let local = Post::get_instance_page(conn, inst.id, Page::default().limits())?;
-    let user_feed = rockets.user.clone().and_then(|user| {
-        let followed = user.get_followed(conn).ok()?;
-        let mut in_feed = followed.into_iter().map(|u| u.id).collect::<Vec<i32>>();
-        in_feed.push(user.id);
-        Post::user_feed_page(conn, in_feed, Page::default().limits()).ok()
-    });
+    let timelines = Timeline::list_all_for_user(&conn, rockets.user.clone().map(|u| u.id))?
+        .into_iter()
+        .filter_map(|t| {
+            if let Ok(latest) = t.get_latest(&conn, 12) {
+                Some((t, latest))
+            } else {
+                None
+            }
+        })
+        .collect();
 
     Ok(render!(instance::index(
         &rockets.to_context(),
         inst,
         User::count_local(conn)?,
         Post::count_local(conn)?,
-        local,
-        federated,
-        user_feed
-    )))
-}
-
-#[get("/local?<page>")]
-pub fn local(page: Option<Page>, rockets: PlumeRocket) -> Result<Ructe, ErrorPage> {
-    let page = page.unwrap_or_default();
-    let instance = Instance::get_local()?;
-    let articles = Post::get_instance_page(&*rockets.conn, instance.id, page.limits())?;
-    Ok(render!(instance::local(
-        &rockets.to_context(),
-        instance,
-        articles,
-        page.0,
-        Page::total(Post::count_local(&*rockets.conn)? as i32)
-    )))
-}
-
-#[get("/feed?<page>")]
-pub fn feed(user: User, page: Option<Page>, rockets: PlumeRocket) -> Result<Ructe, ErrorPage> {
-    let page = page.unwrap_or_default();
-    let followed = user.get_followed(&*rockets.conn)?;
-    let mut in_feed = followed.into_iter().map(|u| u.id).collect::<Vec<i32>>();
-    in_feed.push(user.id);
-    let articles = Post::user_feed_page(&*rockets.conn, in_feed, page.limits())?;
-    Ok(render!(instance::feed(
-        &rockets.to_context(),
-        articles,
-        page.0,
-        Page::total(Post::count_local(&*rockets.conn)? as i32)
-    )))
-}
-
-#[get("/federated?<page>")]
-pub fn federated(page: Option<Page>, rockets: PlumeRocket) -> Result<Ructe, ErrorPage> {
-    let page = page.unwrap_or_default();
-    let articles = Post::get_recents_page(&*rockets.conn, page.limits())?;
-    Ok(render!(instance::federated(
-        &rockets.to_context(),
-        articles,
-        page.0,
-        Page::total(Post::count_local(&*rockets.conn)? as i32)
+        timelines
     )))
 }
 

src/routes/likes.rs

@@ -4,7 +4,7 @@ use rocket_i18n::I18n;
 use plume_common::activity_pub::broadcast;
 use plume_common::utils;
 use plume_models::{
-    blogs::Blog, inbox::inbox, likes, posts::Post, users::User, Error, PlumeRocket,
+    blogs::Blog, inbox::inbox, likes, posts::Post, timeline::*, users::User, Error, PlumeRocket,
 };
 use routes::errors::ErrorPage;
 
@@ -23,6 +23,8 @@ pub fn create(
         let like = likes::Like::insert(&*conn, likes::NewLike::new(&post, &user))?;
         like.notify(&*conn)?;
 
+        Timeline::add_to_all_timelines(&rockets, &post, Kind::Like(&user))?;
+
         let dest = User::one_by_instance(&*conn)?;
         let act = like.to_activity(&*conn)?;
         rockets.worker.execute(move || broadcast(&user, act, dest));

src/routes/mod.rs

@@ -162,6 +162,7 @@ pub mod reshares;
 pub mod search;
 pub mod session;
 pub mod tags;
+pub mod timelines;
 pub mod user;
 pub mod well_known;
 

src/routes/posts.rs

@@ -23,6 +23,7 @@ use plume_models::{
     posts::*,
     safe_string::SafeString,
     tags::*,
+    timeline::*,
     users::User,
     Error, PlumeRocket,
 };
@@ -339,6 +340,8 @@ pub fn update(
                         .expect("post::update: act error");
                     let dest = User::one_by_instance(&*conn).expect("post::update: dest error");
                     rockets.worker.execute(move || broadcast(&user, act, dest));
+
+                    Timeline::add_to_all_timelines(&rockets, &post, Kind::Original).ok();
                 } else {
                     let act = post
                         .update_activity(&*conn)
@@ -529,8 +532,10 @@ pub fn create(
                 .create_activity(&*conn)
                 .expect("posts::create: activity error");
             let dest = User::one_by_instance(&*conn).expect("posts::create: dest error");
-            let worker = rockets.worker;
+            let worker = &rockets.worker;
             worker.execute(move || broadcast(&user, act, dest));
+
+            Timeline::add_to_all_timelines(&rockets, &post, Kind::Original)?;
         }
 
         Ok(Flash::success(

src/routes/reshares.rs

@@ -4,7 +4,8 @@ use rocket_i18n::I18n;
 use plume_common::activity_pub::broadcast;
 use plume_common::utils;
 use plume_models::{
-    blogs::Blog, inbox::inbox, posts::Post, reshares::*, users::User, Error, PlumeRocket,
+    blogs::Blog, inbox::inbox, posts::Post, reshares::*, timeline::*, users::User, Error,
+    PlumeRocket,
 };
 use routes::errors::ErrorPage;
 
@@ -23,6 +24,8 @@ pub fn create(
         let reshare = Reshare::insert(&*conn, NewReshare::new(&post, &user))?;
         reshare.notify(&*conn)?;
 
+        Timeline::add_to_all_timelines(&rockets, &post, Kind::Reshare(&user))?;
+
         let dest = User::one_by_instance(&*conn)?;
         let act = reshare.to_activity(&*conn)?;
         rockets.worker.execute(move || broadcast(&user, act, dest));

src/routes/timelines.rs

@@ -0,0 +1,51 @@
+#![allow(dead_code)]
+
+use crate::{routes::errors::ErrorPage, template_utils::Ructe};
+use plume_models::{timeline::*, PlumeRocket};
+use rocket::response::Redirect;
+use routes::Page;
+use template_utils::IntoContext;
+
+#[get("/timeline/<id>?<page>")]
+pub fn details(id: i32, rockets: PlumeRocket, page: Option<Page>) -> Result<Ructe, ErrorPage> {
+    let page = page.unwrap_or_default();
+    let all_tl = Timeline::list_all_for_user(&rockets.conn, rockets.user.clone().map(|u| u.id))?;
+    let tl = Timeline::get(&rockets.conn, id)?;
+    let posts = tl.get_page(&rockets.conn, page.limits())?;
+    let total_posts = tl.count_posts(&rockets.conn)?;
+    Ok(render!(timelines::details(
+        &rockets.to_context(),
+        tl,
+        posts,
+        all_tl,
+        page.0,
+        Page::total(total_posts as i32)
+    )))
+}
+
+// TODO
+
+#[get("/timeline/new")]
+pub fn new() -> Result<Ructe, ErrorPage> {
+    unimplemented!()
+}
+
+#[post("/timeline/new")]
+pub fn create() -> Result<Redirect, Ructe> {
+    unimplemented!()
+}
+
+#[get("/timeline/<_id>/edit")]
+pub fn edit(_id: i32) -> Result<Ructe, ErrorPage> {
+    unimplemented!()
+}
+
+#[post("/timeline/<_id>/edit")]
+pub fn update(_id: i32) -> Result<Redirect, Ructe> {
+    unimplemented!()
+}
+
+#[post("/timeline/<_id>/delete")]
+pub fn delete(_id: i32) -> Result<Redirect, ErrorPage> {
+    unimplemented!()
+}

src/template_utils.rs

@@ -108,6 +108,15 @@ pub fn translate_notification(ctx: BaseContext, notif: Notification) -> String {
     }
 }
 
+pub fn i18n_timeline_name(cat: &Catalog, tl: &str) -> String {
+    match tl {
+        "Your feed" => i18n!(cat, "Your feed"),
+        "Local feed" => i18n!(cat, "Local feed"),
+        "Federated feed" => i18n!(cat, "Federated feed"),
+        n => n.to_string(),
+    }
+}
+
 pub enum Size {
     Small,
     Medium,
@@ -143,11 +152,11 @@ pub fn avatar(
     ))
 }
 
-pub fn tabs(links: &[(&str, String, bool)]) -> Html<String> {
+pub fn tabs(links: &[(impl AsRef<str>, String, bool)]) -> Html<String> {
     let mut res = String::from(r#"<div class="tabs">"#);
     for (url, title, selected) in links {
         res.push_str(r#"<a dir="auto" href=""#);
-        res.push_str(url);
+        res.push_str(url.as_ref());
         if *selected {
             res.push_str(r#"" class="selected">"#);
         } else {
@@ -183,7 +192,7 @@ pub fn paginate_param(
                 r#"<a href="?{}page={}">{}</a>"#,
                 param,
                 page - 1,
-                catalog.gettext("Previous page")
+                i18n!(catalog, "Previous page")
             )
             .as_str(),
         );
@@ -194,7 +203,7 @@ pub fn paginate_param(
                 r#"<a href="?{}page={}">{}</a>"#,
                 param,
                 page + 1,
-                catalog.gettext("Next page")
+                i18n!(catalog, "Next page")
             )
             .as_str(),
         );