import migrations and don't require diesel_cli for admins (#555)

* import migrations via macro

* panic on database not to the latest migration

* add subcommand to plm

* create migration that run tantivy index creation

* remove diesel_cli from places it was

* use our migration system for tests

* create table __diesel_schema_migrations if needed

plume-models/Cargo.toml

@@ -12,6 +12,7 @@ guid-create = "0.1"
 heck = "0.3.0"
 itertools = "0.8.0"
 lazy_static = "*"
+migrations_internals= "1.4.0"
 openssl = "0.10.15"
 rocket = "0.4.0"
 rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
@@ -39,9 +40,12 @@ path = "../plume-api"
 [dependencies.plume-common]
 path = "../plume-common"
 
+[dependencies.plume-macro]
+path = "../plume-macro"
+
 [dev-dependencies]
 diesel_migrations = "1.3.0"
 
 [features]
-postgres = ["diesel/postgres"]
-sqlite = ["diesel/sqlite"]
+postgres = ["diesel/postgres", "plume-macro/postgres"]
+sqlite = ["diesel/sqlite", "plume-macro/sqlite"]

plume-models/src/lib.rs

@@ -1,6 +1,7 @@
 #![feature(try_trait)]
 #![feature(never_type)]
 #![feature(custom_attribute)]
+#![feature(proc_macro_hygiene)]
 
 extern crate activitypub;
 extern crate ammonia;
@@ -14,9 +15,12 @@ extern crate heck;
 extern crate itertools;
 #[macro_use]
 extern crate lazy_static;
+extern crate migrations_internals;
 extern crate openssl;
 extern crate plume_api;
 extern crate plume_common;
+#[macro_use]
+extern crate plume_macro;
 extern crate reqwest;
 extern crate rocket;
 extern crate rocket_i18n;
@@ -32,10 +36,6 @@ extern crate url;
 extern crate webfinger;
 extern crate whatlang;
 
-#[cfg(test)]
-#[macro_use]
-extern crate diesel_migrations;
-
 use plume_common::activity_pub::inbox::InboxError;
 
 #[cfg(not(any(feature = "sqlite", feature = "postgres")))]
@@ -302,18 +302,15 @@ mod tests {
     use diesel::r2d2::ConnectionManager;
     #[cfg(feature = "sqlite")]
     use diesel::{dsl::sql_query, RunQueryDsl};
+    use migrations::IMPORTED_MIGRATIONS;
+    use plume_common::utils::random_hex;
     use scheduled_thread_pool::ScheduledThreadPool;
     use search;
+    use std::env::temp_dir;
     use std::sync::Arc;
     use Connection as Conn;
     use CONFIG;
 
-    #[cfg(feature = "sqlite")]
-    embed_migrations!("../migrations/sqlite");
-
-    #[cfg(feature = "postgres")]
-    embed_migrations!("../migrations/postgres");
-
     #[macro_export]
     macro_rules! part_eq {
         ( $x:expr, $y:expr, [$( $var:ident ),*] ) => {
@@ -335,7 +332,10 @@ mod tests {
                 .connection_customizer(Box::new(db_conn::PragmaForeignKey))
                 .build(ConnectionManager::<Conn>::new(CONFIG.database_url.as_str()))
                 .unwrap();
-            embedded_migrations::run(&*pool.get().unwrap()).expect("Migrations error");
+            let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
+            IMPORTED_MIGRATIONS
+                .run_pending_migrations(&pool.get().unwrap(), &dir)
+                .expect("Migrations error");
             pool
         };
     }
@@ -365,6 +365,7 @@ pub mod instance;
 pub mod likes;
 pub mod medias;
 pub mod mentions;
+pub mod migrations;
 pub mod notifications;
 pub mod plume_rocket;
 pub mod post_authors;

plume-models/src/migrations.rs

@@ -0,0 +1,122 @@
+use Connection;
+use Error;
+use Result;
+
+use diesel::connection::{Connection as Conn, SimpleConnection};
+use migrations_internals::{setup_database, MigrationConnection};
+
+use std::path::Path;
+
+#[allow(dead_code)] //variants might not be constructed if not required by current migrations
+enum Action {
+    Sql(&'static str),
+    Function(&'static Fn(&Connection, &Path) -> Result<()>),
+}
+
+impl Action {
+    fn run(&self, conn: &Connection, path: &Path) -> Result<()> {
+        match self {
+            Action::Sql(sql) => conn.batch_execute(sql).map_err(Error::from),
+            Action::Function(f) => f(conn, path),
+        }
+    }
+}
+
+struct ComplexMigration {
+    name: &'static str,
+    up: &'static [Action],
+    down: &'static [Action],
+}
+
+impl ComplexMigration {
+    fn run(&self, conn: &Connection, path: &Path) -> Result<()> {
+        println!("Running migration {}", self.name);
+        for step in self.up {
+            step.run(conn, path)?
+        }
+        Ok(())
+    }
+
+    fn revert(&self, conn: &Connection, path: &Path) -> Result<()> {
+        println!("Reverting migration {}", self.name);
+        for step in self.down {
+            step.run(conn, path)?
+        }
+        Ok(())
+    }
+}
+
+pub struct ImportedMigrations(&'static [ComplexMigration]);
+
+impl ImportedMigrations {
+    pub fn run_pending_migrations(&self, conn: &Connection, path: &Path) -> Result<()> {
+        use diesel::dsl::sql;
+        use diesel::sql_types::Bool;
+        use diesel::{select, RunQueryDsl};
+        #[cfg(feature = "postgres")]
+        let schema_exists: bool = select(sql::<Bool>(
+            "EXISTS \
+             (SELECT 1 \
+             FROM information_schema.tables \
+             WHERE table_name = '__diesel_schema_migrations')",
+        ))
+        .get_result(conn)?;
+        #[cfg(feature = "sqlite")]
+        let schema_exists: bool = select(sql::<Bool>(
+            "EXISTS \
+             (SELECT 1 \
+             FROM sqlite_master \
+             WHERE type = 'table' \
+             AND name = '__diesel_schema_migrations')",
+        ))
+        .get_result(conn)?;
+
+        if !schema_exists {
+            setup_database(conn)?;
+        }
+
+        let latest_migration = conn.latest_run_migration_version()?;
+        let latest_id = if let Some(migration) = latest_migration {
+            self.0
+                .binary_search_by_key(&migration.as_str(), |mig| mig.name)
+                .map(|id| id + 1)
+                .map_err(|_| Error::NotFound)?
+        } else {
+            0
+        };
+
+        let to_run = &self.0[latest_id..];
+        for migration in to_run {
+            conn.transaction(|| {
+                migration.run(conn, path)?;
+                conn.insert_new_migration(migration.name)
+                    .map_err(Error::from)
+            })?;
+        }
+        Ok(())
+    }
+
+    pub fn is_pending(&self, conn: &Connection) -> Result<bool> {
+        let latest_migration = conn.latest_run_migration_version()?;
+        if let Some(migration) = latest_migration {
+            Ok(self.0.last().expect("no migrations found").name != migration)
+        } else {
+            Ok(true)
+        }
+    }
+
+    pub fn rerun_last_migration(&self, conn: &Connection, path: &Path) -> Result<()> {
+        let latest_migration = conn.latest_run_migration_version()?;
+        let id = latest_migration
+            .and_then(|m| self.0.binary_search_by_key(&m.as_str(), |m| m.name).ok())?;
+        let migration = &self.0[id];
+        conn.transaction(|| {
+            migration.revert(conn, path)?;
+            migration.run(conn, path)
+        })
+    }
+}
+
+pub const IMPORTED_MIGRATIONS: ImportedMigrations = {
+    import_migrations! {}
+};

plume-models/src/search/searcher.rs

@@ -1,9 +1,11 @@
 use instance::Instance;
 use posts::Post;
+use schema::posts;
 use tags::Tag;
 use Connection;
 
 use chrono::Datelike;
+use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
 use itertools::Itertools;
 use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
 use tantivy::{
@@ -222,6 +224,16 @@ impl Searcher {
             .collect()
     }
 
+    pub fn fill(&self, conn: &Connection) -> Result<()> {
+        for post in posts::table
+            .filter(posts::published.eq(true))
+            .load::<Post>(conn)?
+        {
+            self.update_document(conn, &post)?
+        }
+        Ok(())
+    }
+
     pub fn commit(&self) {
         let mut writer = self.writer.lock().unwrap();
         writer.as_mut().unwrap().commit().unwrap();

plume-models/tests/lib.rs

@@ -1,22 +1,21 @@
 extern crate diesel;
-#[macro_use]
-extern crate diesel_migrations;
-
+extern crate plume_common;
 extern crate plume_models;
 
 use diesel::Connection;
+use plume_common::utils::random_hex;
+use plume_models::migrations::IMPORTED_MIGRATIONS;
 use plume_models::{Connection as Conn, CONFIG};
 
-#[cfg(feature = "sqlite")]
-embed_migrations!("../migrations/sqlite");
-
-#[cfg(feature = "postgres")]
-embed_migrations!("../migrations/postgres");
+use std::env::temp_dir;
 
 fn db() -> Conn {
     let conn =
         Conn::establish(CONFIG.database_url.as_str()).expect("Couldn't connect to the database");
-    embedded_migrations::run(&conn).expect("Couldn't run migrations");
+    let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
+    IMPORTED_MIGRATIONS
+        .run_pending_migrations(&conn, &dir)
+        .expect("Couldn't run migrations");
     conn
 }