From b1d7b3969d731ea830e51ae30f5af5a4aa5ba183 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 6 Jan 2021 23:01:00 +0900 Subject: [PATCH 01/39] Add Riker to dependencies --- Cargo.lock | 288 +++++++++++++++++++++++++++++++++------- Cargo.toml | 1 + plume-models/Cargo.toml | 1 + 3 files changed, 244 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c38aa831..76fac74e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,7 @@ dependencies = [ "activitystreams-derive", "activitystreams-traits", "activitystreams-types", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", ] @@ -32,7 +32,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670ef03168e704b0cae242e7a5d8b40506772b339687e01a3496fc4afe2e8542" dependencies = [ "failure", - "serde", + "serde 1.0.118", "serde_json", ] @@ -46,7 +46,7 @@ dependencies = [ "activitystreams-traits", "chrono", "mime 0.3.16", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", ] @@ -126,6 +126,9 @@ name = "ahash" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] [[package]] name = "aho-corasick" @@ -168,6 +171,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "arc-swap" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" + [[package]] name = "array_tool" version = "1.0.3" @@ -344,7 +353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" dependencies = [ "byteorder 1.3.4", - "serde", + "serde 1.0.118", ] [[package]] @@ -516,8 +525,8 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ "libc", "num-integer", - "num-traits", - "serde", + "num-traits 0.2.14", + "serde 1.0.118", "time", "winapi 0.3.9", ] @@ -555,6 +564,44 @@ dependencies = [ "memchr", ] +[[package]] +name = "config" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" +dependencies = [ + "lazy_static", + "nom 5.1.2", + "rust-ini", + "serde 1.0.118", + "serde-hjson", + "serde_json", + "toml 0.5.8", + "yaml-rust", +] + +[[package]] +name = "const-random" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f590d95d011aa80b063ffe3253422ed5aa462af4e9867d43ce8337562bac77c4" +dependencies = [ + "const-random-macro", + "proc-macro-hack 0.5.19", +] + +[[package]] +name = "const-random-macro" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615f6e27d000a2bffbc7f2f6a8669179378fa27ee4d0a509e985dfc0a7defb40" +dependencies = [ + "getrandom 0.2.1", + "lazy_static", + "proc-macro-hack 0.5.19", + "tiny-keccak", +] + [[package]] name = "const_fn" version = "0.4.4" @@ -607,7 +654,7 @@ dependencies = [ "idna 0.1.5", "log 0.4.11", "publicsuffix", - "serde", + "serde 1.0.118", "serde_json", "time", "try_from", @@ -789,6 +836,17 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" +[[package]] +name = "dashmap" +version = "3.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" +dependencies = [ + "ahash", + "cfg-if 0.1.10", + "num_cpus", +] + [[package]] name = "data-encoding" version = "2.1.2" @@ -1370,6 +1428,17 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + [[package]] name = "gettext" version = "0.3.0" @@ -1943,7 +2012,7 @@ dependencies = [ "log 0.4.11", "native-tls", "nom 4.2.3", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", ] @@ -2010,7 +2079,7 @@ dependencies = [ "lindera-dictionary", "lindera-ipadic", "lindera-ipadic-builder", - "serde", + "serde 1.0.118", "serde_json", ] @@ -2023,7 +2092,7 @@ dependencies = [ "bincode", "byteorder 1.3.4", "encoding", - "serde", + "serde 1.0.118", "yada", ] @@ -2091,9 +2160,13 @@ dependencies = [ [[package]] name = "linked-hash-map" -version = "0.5.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +checksum = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" +dependencies = [ + "serde 0.8.23", + "serde_test", +] [[package]] name = "lock_api" @@ -2152,7 +2225,7 @@ dependencies = [ "log 0.4.11", "phf", "phf_codegen", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", "string_cache", @@ -2504,7 +2577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg 1.0.1", - "num-traits", + "num-traits 0.2.14", ] [[package]] @@ -2515,7 +2588,16 @@ checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ "autocfg 1.0.1", "num-integer", - "num-traits", + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.14", ] [[package]] @@ -2826,7 +2908,7 @@ dependencies = [ "chrono", "indexmap", "line-wrap", - "serde", + "serde 1.0.118", "xml-rs", ] @@ -2854,6 +2936,7 @@ dependencies = [ "plume-api", "plume-common", "plume-models", + "riker", "rocket", "rocket_contrib", "rocket_csrf", @@ -2862,7 +2945,7 @@ dependencies = [ "rsass", "ructe", "scheduled-thread-pool", - "serde", + "serde 1.0.118", "serde_json", "shrinkwraprs 0.2.3", "tracing", @@ -2876,7 +2959,7 @@ dependencies = [ name = "plume-api" version = "0.6.1-dev" dependencies = [ - "serde", + "serde 1.0.118", "serde_derive", ] @@ -2909,7 +2992,7 @@ dependencies = [ "regex-syntax 0.6.21", "reqwest 0.9.24", "rocket", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", "shrinkwraprs 0.3.0", @@ -2926,7 +3009,7 @@ dependencies = [ "gettext-macros", "gettext-utils", "lazy_static", - "serde", + "serde 1.0.118", "serde_json", "stdweb", "stdweb-internal-runtime", @@ -2966,10 +3049,11 @@ dependencies = [ "plume-common", "plume-macro", "reqwest 0.9.24", + "riker", "rocket", "rocket_i18n", "scheduled-thread-pool", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", "shrinkwraprs 0.2.3", @@ -3220,7 +3304,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.15", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -3268,7 +3352,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.15", ] [[package]] @@ -3445,7 +3529,7 @@ dependencies = [ "mime 0.3.16", "mime_guess 2.0.3", "native-tls", - "serde", + "serde 1.0.118", "serde_json", "serde_urlencoded 0.5.5", "socks", @@ -3484,7 +3568,7 @@ dependencies = [ "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.0", - "serde", + "serde 1.0.118", "serde_urlencoded 0.7.0", "tokio 0.2.24", "tokio-tls", @@ -3495,6 +3579,38 @@ dependencies = [ "winreg 0.7.0", ] +[[package]] +name = "riker" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abff93ece5a5d3d7f2c54dfba7550657a644c9dc0a871c7ddf8c31381971c41b" +dependencies = [ + "chrono", + "config", + "dashmap", + "futures 0.3.8", + "num_cpus", + "pin-utils", + "rand 0.7.3", + "regex", + "riker-macros", + "slog", + "slog-scope", + "slog-stdlog", + "uuid 0.8.1", +] + +[[package]] +name = "riker-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2a8e8f71c9e7980a596c39c7e3537ea8563054526e15712a610ac97a02dba15" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "ring" version = "0.13.5" @@ -3523,7 +3639,7 @@ dependencies = [ "rocket_http", "state", "time", - "toml", + "toml 0.4.10", "version_check 0.9.2", "yansi", ] @@ -3552,7 +3668,7 @@ dependencies = [ "log 0.4.11", "notify", "rocket", - "serde", + "serde 1.0.118", "serde_json", ] @@ -3564,7 +3680,7 @@ dependencies = [ "data-encoding", "ring", "rocket", - "serde", + "serde 1.0.118", "time", ] @@ -3623,7 +3739,7 @@ dependencies = [ "lazy_static", "nom 4.2.3", "num-rational", - "num-traits", + "num-traits 0.2.14", "rand 0.6.5", ] @@ -3640,13 +3756,19 @@ dependencies = [ "nom 5.1.2", ] +[[package]] +name = "rust-ini" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" + [[package]] name = "rust-stemmers" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" dependencies = [ - "serde", + "serde 1.0.118", "serde_derive", ] @@ -3749,6 +3871,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "serde" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" + [[package]] name = "serde" version = "1.0.118" @@ -3758,6 +3886,19 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-hjson" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" +dependencies = [ + "lazy_static", + "linked-hash-map", + "num-traits 0.1.43", + "regex", + "serde 0.8.23", +] + [[package]] name = "serde_derive" version = "1.0.118" @@ -3777,7 +3918,16 @@ checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" dependencies = [ "itoa", "ryu", - "serde", + "serde 1.0.118", +] + +[[package]] +name = "serde_test" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" +dependencies = [ + "serde 0.8.23", ] [[package]] @@ -3788,7 +3938,7 @@ checksum = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" dependencies = [ "dtoa", "itoa", - "serde", + "serde 1.0.118", "url 1.7.2", ] @@ -3801,7 +3951,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde", + "serde 1.0.118", ] [[package]] @@ -3878,6 +4028,34 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-scope" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c44c89dd8b0ae4537d1ae318353eaf7840b4869c536e31c41e963d1ea523ee6" +dependencies = [ + "arc-swap", + "lazy_static", + "slog", +] + +[[package]] +name = "slog-stdlog" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8228ab7302adbf4fcb37e66f3cda78003feb521e7fd9e3847ec117a7784d0f5a" +dependencies = [ + "log 0.4.11", + "slog", + "slog-scope", +] + [[package]] name = "smallvec" version = "0.6.13" @@ -3948,7 +4126,7 @@ checksum = "a68c0ce28cf7400ed022e18da3c4591e14e1df02c70e93573cc59921b3923aeb" dependencies = [ "discard", "rustc_version", - "serde", + "serde 1.0.118", "serde_json", "stdweb-derive", "stdweb-internal-macros", @@ -3964,7 +4142,7 @@ checksum = "0e21ebd9179de08f2300a65454268a17ea3de204627458588c84319c4def3930" dependencies = [ "proc-macro2 0.4.30", "quote 0.6.13", - "serde", + "serde 1.0.118", "serde_derive", "syn 0.15.44", ] @@ -3978,7 +4156,7 @@ dependencies = [ "base-x", "proc-macro2 0.4.30", "quote 0.6.13", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", "sha1", @@ -4010,7 +4188,7 @@ dependencies = [ "new_debug_unreachable", "phf_shared", "precomputed-hash", - "serde", + "serde 1.0.118", "string_cache_codegen", "string_cache_shared", ] @@ -4154,7 +4332,7 @@ dependencies = [ "onig", "plist", "regex-syntax 0.6.21", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", "walkdir", @@ -4194,7 +4372,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "serde", + "serde 1.0.118", "serde_json", "smallvec 1.5.1", "snap", @@ -4322,6 +4500,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.1.0" @@ -4605,7 +4792,16 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" dependencies = [ - "serde", + "serde 1.0.118", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde 1.0.118", ] [[package]] @@ -4674,7 +4870,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" dependencies = [ - "serde", + "serde 1.0.118", "tracing-core", ] @@ -4689,7 +4885,7 @@ dependencies = [ "lazy_static", "matchers", "regex", - "serde", + "serde 1.0.118", "serde_json", "sharded-slab", "smallvec 1.5.1", @@ -4875,7 +5071,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" dependencies = [ "rand 0.7.3", - "serde", + "serde 1.0.118", ] [[package]] @@ -4887,7 +5083,7 @@ dependencies = [ "idna 0.1.5", "lazy_static", "regex", - "serde", + "serde 1.0.118", "serde_derive", "serde_json", "url 1.7.2", @@ -4989,7 +5185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" dependencies = [ "cfg-if 1.0.0", - "serde", + "serde 1.0.118", "serde_json", "wasm-bindgen-macro", ] @@ -5067,7 +5263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec24b1b0700d4b466d280228ed0f62274eedeaa80206820f071fdc8ed787b664" dependencies = [ "reqwest 0.9.24", - "serde", + "serde 1.0.118", "serde_derive", ] diff --git a/Cargo.toml b/Cargo.toml index cf8d5ee2..85cebb03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ validator_derive = "0.8" webfinger = "0.4.1" tracing = "0.1.22" tracing-subscriber = "0.2.15" +riker = "0.4.2" [[bin]] name = "plume" diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 7b1ef7c5..077897e1 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -33,6 +33,7 @@ diesel-derive-newtype = "0.1.2" glob = "0.3.0" lindera-tantivy = { version = "0.7.1", optional = true } tracing = "0.1.22" +riker = "0.4.2" [dependencies.chrono] features = ["serde"] From 43f32d30cb400eaa514425a895148c0d45c7ca1f Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 6 Jan 2021 23:05:22 +0900 Subject: [PATCH 02/39] Add once_cell to dependdencies --- Cargo.lock | 1 + plume-models/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 76fac74e..6a22928a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3044,6 +3044,7 @@ dependencies = [ "ldap3", "lindera-tantivy", "migrations_internals", + "once_cell", "openssl", "plume-api", "plume-common", diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 077897e1..ec97f278 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -34,6 +34,7 @@ glob = "0.3.0" lindera-tantivy = { version = "0.7.1", optional = true } tracing = "0.1.22" riker = "0.4.2" +once_cell = "1.5.2" [dependencies.chrono] features = ["serde"] From 09d9164a1c522d3b38a41164125c99e8807cb52a Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 6 Jan 2021 23:25:41 +0900 Subject: [PATCH 03/39] Extract Searcher::open_or_recreate() from main() --- plume-models/src/search/searcher.rs | 62 ++++++++++++++++++++++++++++- src/main.rs | 61 ++-------------------------- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index 34a45b0b..2aa5cf2b 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -1,15 +1,17 @@ use crate::{ config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts, - search::query::PlumeQuery, tags::Tag, Connection, Result, + search::query::PlumeQuery, tags::Tag, Connection, Error, Result, CONFIG, }; -use chrono::Datelike; +use chrono::{Datelike, Utc}; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use itertools::Itertools; +use std::fs; use std::{cmp, fs::create_dir_all, io, path::Path, sync::Mutex}; use tantivy::{ collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter, ReloadPolicy, TantivyError, Term, }; +use tracing::warn; use whatlang::{detect as detect_lang, Lang}; #[derive(Debug)] @@ -67,6 +69,62 @@ impl Searcher { schema_builder.build() } + pub fn open_or_recreate() -> Self { + let mut open_searcher = Self::open(&CONFIG.search_index, &CONFIG.search_tokenizers); + if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { + if Self::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_err() { + let current_path = Path::new(&CONFIG.search_index); + let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); + let backup_path = Path::new(&backup_path); + fs::rename(current_path, backup_path) + .expect("main: error on backing up search index directory for recreating"); + if Self::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_ok() { + if fs::remove_dir_all(backup_path).is_err() { + warn!( + "error on removing backup directory: {}. it remains", + backup_path.display() + ); + } + } else { + panic!("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); + } + } + open_searcher = Self::open(&CONFIG.search_index, &CONFIG.search_tokenizers); + } + #[allow(clippy::match_wild_err_arm)] + let searcher = match open_searcher { + Err(Error::Search(e)) => match e { + SearcherError::WriteLockAcquisitionError => panic!( + r#" +Your search index is locked. Plume can't start. To fix this issue +make sure no other Plume instance is started, and run: + + plm search unlock + +Then try to restart Plume. +"# + ), + SearcherError::IndexOpeningError => panic!( + r#" +Plume was unable to open the search index. If you created the index +before, make sure to run Plume in the same directory it was created in, or +to set SEARCH_INDEX accordingly. If you did not yet create the search +index, run this command: + + plm search init + +Then try to restart Plume +"# + ), + e => Err(e).unwrap(), + }, + Err(_) => panic!("Unexpected error while opening search index"), + Ok(s) => s, + }; + + searcher + } + pub fn create(path: &dyn AsRef, tokenizers: &SearchTokenizerConfig) -> Result { let schema = Self::schema(); diff --git a/src/main.rs b/src/main.rs index cc7561cd..6c548637 100755 --- a/src/main.rs +++ b/src/main.rs @@ -10,20 +10,17 @@ extern crate serde_json; #[macro_use] extern crate validator_derive; -use chrono::Utc; use clap::App; use diesel::r2d2::ConnectionManager; use plume_models::{ db_conn::{DbPool, PragmaForeignKey}, instance::Instance, migrations::IMPORTED_MIGRATIONS, - search::{Searcher as UnmanagedSearcher, SearcherError}, - Connection, Error, CONFIG, + search::Searcher as UnmanagedSearcher, + Connection, CONFIG, }; use rocket_csrf::CsrfFairingBuilder; use scheduled_thread_pool::ScheduledThreadPool; -use std::fs; -use std::path::Path; use std::process::exit; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -103,59 +100,7 @@ Then try to restart Plume. } let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); // we want a fast exit here, so - let mut open_searcher = - UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); - if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { - if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_err() { - let current_path = Path::new(&CONFIG.search_index); - let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); - let backup_path = Path::new(&backup_path); - fs::rename(current_path, backup_path) - .expect("main: error on backing up search index directory for recreating"); - if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_ok() { - if fs::remove_dir_all(backup_path).is_err() { - warn!( - "error on removing backup directory: {}. it remains", - backup_path.display() - ); - } - } else { - panic!("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); - } - } - open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); - } - #[allow(clippy::match_wild_err_arm)] - let searcher = match open_searcher { - Err(Error::Search(e)) => match e { - SearcherError::WriteLockAcquisitionError => panic!( - r#" -Your search index is locked. Plume can't start. To fix this issue -make sure no other Plume instance is started, and run: - - plm search unlock - -Then try to restart Plume. -"# - ), - SearcherError::IndexOpeningError => panic!( - r#" -Plume was unable to open the search index. If you created the index -before, make sure to run Plume in the same directory it was created in, or -to set SEARCH_INDEX accordingly. If you did not yet create the search -index, run this command: - - plm search init - -Then try to restart Plume -"# - ), - e => Err(e).unwrap(), - }, - Err(_) => panic!("Unexpected error while opening search index"), - Ok(s) => Arc::new(s), - }; - + let searcher = Arc::new(UnmanagedSearcher::open_or_recreate()); let commiter = searcher.clone(); workpool.execute_with_fixed_delay( Duration::from_secs(5), From 996dc309f7cfdd2eb83b8e3ff5f00a19ca215eb3 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 6 Jan 2021 23:40:56 +0900 Subject: [PATCH 04/39] Make Searcher::open_or_create() accept path and tokenizers --- plume-models/src/search/searcher.rs | 17 ++++++++--------- src/main.rs | 5 ++++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index 2aa5cf2b..61608bea 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -1,6 +1,6 @@ use crate::{ config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts, - search::query::PlumeQuery, tags::Tag, Connection, Error, Result, CONFIG, + search::query::PlumeQuery, tags::Tag, Connection, Error, Result, }; use chrono::{Datelike, Utc}; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; @@ -69,16 +69,15 @@ impl Searcher { schema_builder.build() } - pub fn open_or_recreate() -> Self { - let mut open_searcher = Self::open(&CONFIG.search_index, &CONFIG.search_tokenizers); + pub fn open_or_recreate(path: &dyn AsRef, tokenizers: &SearchTokenizerConfig) -> Self { + let mut open_searcher = Self::open(path, tokenizers); if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { - if Self::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_err() { - let current_path = Path::new(&CONFIG.search_index); - let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); + if Self::create(path, tokenizers).is_err() { + let backup_path = format!("{}.{}", path.as_ref().display(), Utc::now().timestamp()); let backup_path = Path::new(&backup_path); - fs::rename(current_path, backup_path) + fs::rename(path, backup_path) .expect("main: error on backing up search index directory for recreating"); - if Self::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_ok() { + if Self::create(path, tokenizers).is_ok() { if fs::remove_dir_all(backup_path).is_err() { warn!( "error on removing backup directory: {}. it remains", @@ -89,7 +88,7 @@ impl Searcher { panic!("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); } } - open_searcher = Self::open(&CONFIG.search_index, &CONFIG.search_tokenizers); + open_searcher = Self::open(path, tokenizers); } #[allow(clippy::match_wild_err_arm)] let searcher = match open_searcher { diff --git a/src/main.rs b/src/main.rs index 6c548637..e6654ada 100755 --- a/src/main.rs +++ b/src/main.rs @@ -100,7 +100,10 @@ Then try to restart Plume. } let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); // we want a fast exit here, so - let searcher = Arc::new(UnmanagedSearcher::open_or_recreate()); + let searcher = Arc::new(UnmanagedSearcher::open_or_recreate( + &CONFIG.search_index, + &CONFIG.search_tokenizers, + )); let commiter = searcher.clone(); workpool.execute_with_fixed_delay( Duration::from_secs(5), From 578768d7df25ce96d8ff2a16adce04f2990ab105 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 6 Jan 2021 23:43:23 +0900 Subject: [PATCH 05/39] Remove unnecessary attribute to suppress clippy --- plume-models/src/search/searcher.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index 61608bea..7ea01c59 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -90,8 +90,8 @@ impl Searcher { } open_searcher = Self::open(path, tokenizers); } - #[allow(clippy::match_wild_err_arm)] let searcher = match open_searcher { + Ok(s) => s, Err(Error::Search(e)) => match e { SearcherError::WriteLockAcquisitionError => panic!( r#" @@ -117,8 +117,7 @@ Then try to restart Plume ), e => Err(e).unwrap(), }, - Err(_) => panic!("Unexpected error while opening search index"), - Ok(s) => s, + _ => panic!("Unexpected error while opening search index"), }; searcher From cfe097edf487c5559dca5025df2fbfb36017fe0d Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 03:39:49 +0900 Subject: [PATCH 06/39] Define PostEvent --- plume-models/src/posts.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 4e368f58..2a5c97f2 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -23,7 +23,7 @@ use std::collections::HashSet; pub type LicensedArticle = CustomObject; -#[derive(Queryable, Identifiable, Clone, AsChangeset)] +#[derive(Queryable, Identifiable, Clone, AsChangeset, Debug)] #[changeset_options(treat_none_as_null = "true")] pub struct Post { pub id: i32, @@ -800,6 +800,25 @@ impl IntoId for Post { } } +#[derive(Clone, Debug)] +pub enum PostEvent { + PostPublished(Post), + PostUpdated(Post), + PostDeleted(Post), +} + +impl From for Post { + fn from(event: PostEvent) -> Self { + use PostEvent::*; + + match event { + PostPublished(post) => post, + PostUpdated(post) => post, + PostDeleted(post) => post, + } + } +} + #[cfg(test)] mod tests { use super::*; From 0a148721de3e3be5e94213aad06e65bc4bb06c7c Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 04:46:35 +0900 Subject: [PATCH 07/39] Define SearchActor --- plume-models/src/search/actor.rs | 58 ++++++++++++++++++++++++++++++++ plume-models/src/search/mod.rs | 1 + 2 files changed, 59 insertions(+) create mode 100644 plume-models/src/search/actor.rs diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs new file mode 100644 index 00000000..9a939f93 --- /dev/null +++ b/plume-models/src/search/actor.rs @@ -0,0 +1,58 @@ +use super::Searcher; +use crate::{db_conn::DbPool, posts::PostEvent, POST_CHAN}; +use riker::actors::{Actor, ActorFactoryArgs, Context, Sender, Subscribe, Tell}; +use std::sync::Arc; +use tracing::error; + +pub struct SearchActor { + searcher: Arc, + conn: DbPool, +} + +impl Actor for SearchActor { + type Msg = PostEvent; + + fn pre_start(&mut self, ctx: &Context) { + &POST_CHAN.tell( + Subscribe { + actor: Box::new(ctx.myself()), + topic: "*".into(), + }, + None, + ); + } + + fn recv(&mut self, _ctx: &Context, msg: Self::Msg, _sender: Sender) { + use PostEvent::*; + + match msg { + PostPublished(post) => { + let conn = self.conn.get(); + if conn.is_ok() { + self.searcher + .add_document(&conn.unwrap(), &post) + .unwrap_or_else(|e| error!("{:?}", e)); + } else { + error!("Failed to get database connection"); + } + } + PostUpdated(post) => { + let conn = self.conn.get(); + if conn.is_ok() { + self.searcher + .update_document(&conn.unwrap(), &post) + .unwrap_or_else(|e| error!("{:?}", e)); + } else { + error!("Failed to get database connection"); + } + } + PostDeleted(post) => self.searcher.delete_document(&post), + } + } +} + +impl ActorFactoryArgs<(Arc, DbPool)> for SearchActor { + fn create_args((searcher, conn): (Arc, DbPool)) -> Self { + Self { searcher, conn } + } +} diff --git a/plume-models/src/search/mod.rs b/plume-models/src/search/mod.rs index 83b9bf62..33de8d49 100644 --- a/plume-models/src/search/mod.rs +++ b/plume-models/src/search/mod.rs @@ -1,3 +1,4 @@ +pub mod actor; mod query; mod searcher; mod tokenizer; From d4d32bcc1118c00602cbd18330fdaee5c77489b9 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 04:56:07 +0900 Subject: [PATCH 08/39] Define ACTOR_SYS and POST_CHAN --- plume-models/src/lib.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index be7cf4b6..5a751877 100755 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -17,7 +17,10 @@ extern crate serde_json; #[macro_use] extern crate tantivy; +use once_cell::sync::Lazy; use plume_common::activity_pub::inbox::InboxError; +use posts::PostEvent; +use riker::actors::{channel, ActorSystem, ChannelRef, SystemBuilder}; #[cfg(not(any(feature = "sqlite", feature = "postgres")))] compile_error!("Either feature \"sqlite\" or \"postgres\" must be enabled for this crate."); @@ -30,6 +33,16 @@ pub type Connection = diesel::SqliteConnection; #[cfg(all(not(feature = "sqlite"), feature = "postgres"))] pub type Connection = diesel::PgConnection; +pub(crate) static ACTOR_SYS: Lazy = Lazy::new(|| { + SystemBuilder::new() + .name("plume") + .create() + .expect("Failed to create actor system") +}); + +pub(crate) static POST_CHAN: Lazy> = + Lazy::new(|| channel("post_events", &*ACTOR_SYS).expect("Failed to create post channel")); + /// All the possible errors that can be encoutered in this crate #[derive(Debug)] pub enum Error { From 1bfed0294f0783a5a36deb8e6e799e5e3abdd1c4 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 04:56:25 +0900 Subject: [PATCH 09/39] Define SearchActor::init() --- plume-models/src/search/actor.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index 9a939f93..881d0284 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -1,6 +1,6 @@ use super::Searcher; -use crate::{db_conn::DbPool, posts::PostEvent, POST_CHAN}; -use riker::actors::{Actor, ActorFactoryArgs, Context, Sender, Subscribe, Tell}; +use crate::{db_conn::DbPool, posts::PostEvent, ACTOR_SYS, POST_CHAN}; +use riker::actors::{Actor, ActorFactoryArgs, ActorRefFactory, Context, Sender, Subscribe, Tell}; use std::sync::Arc; use tracing::error; @@ -9,6 +9,12 @@ pub struct SearchActor { conn: DbPool, } +impl SearchActor { + pub fn init(searcher: Arc, conn: DbPool) { + ACTOR_SYS.actor_of_args::("search", (searcher, conn)).expect("Failed to initialize searcher actor"); + } +} + impl Actor for SearchActor { type Msg = PostEvent; From e18be6ec95415f2320d96022e58eade142f1a478 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 04:56:35 +0900 Subject: [PATCH 10/39] Initialize SearchActor at start --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index e6654ada..60466437 100755 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ use plume_models::{ db_conn::{DbPool, PragmaForeignKey}, instance::Instance, migrations::IMPORTED_MIGRATIONS, - search::Searcher as UnmanagedSearcher, + search::{actor::SearchActor, Searcher as UnmanagedSearcher}, Connection, CONFIG, }; use rocket_csrf::CsrfFairingBuilder; @@ -104,6 +104,7 @@ Then try to restart Plume. &CONFIG.search_index, &CONFIG.search_tokenizers, )); + SearchActor::init(searcher.clone(), dbpool.clone()); let commiter = searcher.clone(); workpool.execute_with_fixed_delay( Duration::from_secs(5), From 16624fd7424fac8facab3f1d8abe500427dd895d Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 12:39:51 +0900 Subject: [PATCH 11/39] Run cargo fmt --- plume-models/src/search/actor.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index 881d0284..99769f60 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -11,7 +11,9 @@ pub struct SearchActor { impl SearchActor { pub fn init(searcher: Arc, conn: DbPool) { - ACTOR_SYS.actor_of_args::("search", (searcher, conn)).expect("Failed to initialize searcher actor"); + ACTOR_SYS + .actor_of_args::("search", (searcher, conn)) + .expect("Failed to initialize searcher actor"); } } From a733ece26e34403cd962b8ac109d2cb06748f330 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 16:38:28 +0900 Subject: [PATCH 12/39] Suppress clippy --- plume-models/src/search/actor.rs | 34 +++++++++++++++++------------ plume-models/src/search/searcher.rs | 6 ++--- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index 99769f60..e8b2f012 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -21,13 +21,13 @@ impl Actor for SearchActor { type Msg = PostEvent; fn pre_start(&mut self, ctx: &Context) { - &POST_CHAN.tell( + POST_CHAN.tell( Subscribe { actor: Box::new(ctx.myself()), topic: "*".into(), }, None, - ); + ) } fn recv(&mut self, _ctx: &Context, msg: Self::Msg, _sender: Sender) { @@ -36,22 +36,28 @@ impl Actor for SearchActor { match msg { PostPublished(post) => { let conn = self.conn.get(); - if conn.is_ok() { - self.searcher - .add_document(&conn.unwrap(), &post) - .unwrap_or_else(|e| error!("{:?}", e)); - } else { - error!("Failed to get database connection"); + match conn { + Ok(_) => { + self.searcher + .add_document(&conn.unwrap(), &post) + .unwrap_or_else(|e| error!("{:?}", e)); + }, + _ => { + error!("Failed to get database connection"); + } } } PostUpdated(post) => { let conn = self.conn.get(); - if conn.is_ok() { - self.searcher - .update_document(&conn.unwrap(), &post) - .unwrap_or_else(|e| error!("{:?}", e)); - } else { - error!("Failed to get database connection"); + match conn { + Ok(_) => { + self.searcher + .update_document(&conn.unwrap(), &post) + .unwrap_or_else(|e| error!("{:?}", e)); + }, + _ => { + error!("Failed to get database connection"); + } } } PostDeleted(post) => self.searcher.delete_document(&post), diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index 7ea01c59..d4ebb642 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -90,7 +90,7 @@ impl Searcher { } open_searcher = Self::open(path, tokenizers); } - let searcher = match open_searcher { + match open_searcher { Ok(s) => s, Err(Error::Search(e)) => match e { SearcherError::WriteLockAcquisitionError => panic!( @@ -118,9 +118,7 @@ Then try to restart Plume e => Err(e).unwrap(), }, _ => panic!("Unexpected error while opening search index"), - }; - - searcher + } } pub fn create(path: &dyn AsRef, tokenizers: &SearchTokenizerConfig) -> Result { From ccbf2124cbee2d7ba402bd0b5f6af67e2f083d56 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 16:46:43 +0900 Subject: [PATCH 13/39] Run cargo fmt --- plume-models/src/search/actor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index e8b2f012..bc70011d 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -41,7 +41,7 @@ impl Actor for SearchActor { self.searcher .add_document(&conn.unwrap(), &post) .unwrap_or_else(|e| error!("{:?}", e)); - }, + } _ => { error!("Failed to get database connection"); } @@ -54,7 +54,7 @@ impl Actor for SearchActor { self.searcher .update_document(&conn.unwrap(), &post) .unwrap_or_else(|e| error!("{:?}", e)); - }, + } _ => { error!("Failed to get database connection"); } From 6a2ba2260d638777ccf201d89624f0766021735d Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:16:51 +0900 Subject: [PATCH 14/39] Send event to channel when post published --- plume-models/src/posts.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 2a5c97f2..6c1b37b8 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -1,7 +1,7 @@ use crate::{ ap_url, blogs::Blog, instance::Instance, medias::Media, mentions::Mention, post_authors::*, safe_string::SafeString, schema::posts, search::Searcher, tags::*, timeline::*, users::User, - Connection, Error, PlumeRocket, Result, CONFIG, + Connection, Error, PlumeRocket, PostEvent::*, Result, CONFIG, POST_CHAN, }; use activitypub::{ activity::{Create, Delete, Update}, @@ -19,6 +19,7 @@ use plume_common::{ }, utils::md_to_html, }; +use riker::actors::{Publish, Tell}; use std::collections::HashSet; pub type LicensedArticle = CustomObject; @@ -62,7 +63,7 @@ impl Post { find_by!(posts, find_by_ap_url, ap_url as &str); last!(posts); - pub fn insert(conn: &Connection, new: NewPost, searcher: &Searcher) -> Result { + pub fn insert(conn: &Connection, new: NewPost, _searcher: &Searcher) -> Result { diesel::insert_into(posts::table) .values(new) .execute(conn)?; @@ -77,7 +78,10 @@ impl Post { let _: Post = post.save_changes(conn)?; } - searcher.add_document(conn, &post)?; + if post.published { + post.publish_published(); + } + Ok(post) } @@ -545,6 +549,16 @@ impl Post { .set_to_link_vec(vec![Id::new(PUBLIC_VISIBILITY)])?; Ok(act) } + + fn publish_published(&self) { + POST_CHAN.tell( + Publish { + msg: PostPublished(self.clone()), + topic: "post.published".into(), + }, + None, + ) + } } impl FromId for Post { From 6ad335960934dd95141ef55e610d9006c2aa529b Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:20:55 +0900 Subject: [PATCH 15/39] Remove searcher from arguments of Post::insert() --- plume-models/src/posts.rs | 5 +---- src/api/posts.rs | 2 -- src/routes/posts.rs | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 6c1b37b8..576e1a1b 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -63,7 +63,7 @@ impl Post { find_by!(posts, find_by_ap_url, ap_url as &str); last!(posts); - pub fn insert(conn: &Connection, new: NewPost, _searcher: &Searcher) -> Result { + pub fn insert(conn: &Connection, new: NewPost) -> Result { diesel::insert_into(posts::table) .values(new) .execute(conn)?; @@ -571,7 +571,6 @@ impl FromId for Post { fn from_activity(c: &PlumeRocket, article: LicensedArticle) -> Result { let conn = &*c.conn; - let searcher = &c.searcher; let license = article.custom_props.license_string().unwrap_or_default(); let article = article.object; @@ -619,7 +618,6 @@ impl FromId for Post { source: article.ap_object_props.source_object::()?.content, cover_id: cover, }, - searcher, )?; for author in authors { @@ -864,7 +862,6 @@ mod tests { source: "Hello".into(), cover_id: None, }, - &r.searcher, ) .unwrap(); PostAuthor::insert( diff --git a/src/api/posts.rs b/src/api/posts.rs index 284746ad..ad483a12 100644 --- a/src/api/posts.rs +++ b/src/api/posts.rs @@ -105,7 +105,6 @@ pub fn create( rockets: PlumeRocket, ) -> Api { let conn = &*rockets.conn; - let search = &rockets.searcher; let worker = &rockets.worker; let author = User::get(conn, auth.0.user_id)?; @@ -155,7 +154,6 @@ pub fn create( source: payload.source.clone(), cover_id: payload.cover_id, }, - search, )?; PostAuthor::insert( diff --git a/src/routes/posts.rs b/src/routes/posts.rs index b04fa4fc..b1e408f4 100644 --- a/src/routes/posts.rs +++ b/src/routes/posts.rs @@ -481,7 +481,6 @@ pub fn create( source: form.content.clone(), cover_id: form.cover, }, - &rockets.searcher, ) .expect("post::create: post save error"); From 69963689f90f9e34b77b495e8d291fc41b8029d9 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:23:31 +0900 Subject: [PATCH 16/39] Update pot --- po/plume/plume.pot | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/po/plume/plume.pot b/po/plume/plume.pot index f5144446..1a5ce938 100644 --- a/po/plume/plume.pot +++ b/po/plume/plume.pot @@ -192,27 +192,27 @@ msgstr "" msgid "Your article has been updated." msgstr "" -# src/routes/posts.rs:555 +# src/routes/posts.rs:554 msgid "Your article has been saved." msgstr "" -# src/routes/posts.rs:562 +# src/routes/posts.rs:561 msgid "New article" msgstr "" -# src/routes/posts.rs:599 +# src/routes/posts.rs:598 msgid "You are not allowed to delete this article." msgstr "" -# src/routes/posts.rs:624 +# src/routes/posts.rs:623 msgid "Your article has been deleted." msgstr "" -# src/routes/posts.rs:629 +# src/routes/posts.rs:628 msgid "It looks like the article you tried to delete doesn't exist. Maybe it is already gone?" msgstr "" -# src/routes/posts.rs:669 +# src/routes/posts.rs:668 msgid "Couldn't obtain enough information about your account. Please make sure your username is correct." msgstr "" From aae2073146bf20fdd7de652d5ba0212f501f778e Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:34:11 +0900 Subject: [PATCH 17/39] Send event to channel when post updated --- plume-models/src/posts.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 576e1a1b..9045839d 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -85,10 +85,11 @@ impl Post { Ok(post) } - pub fn update(&self, conn: &Connection, searcher: &Searcher) -> Result { + pub fn update(&self, conn: &Connection, _searcher: &Searcher) -> Result { diesel::update(self).set(self).execute(conn)?; let post = Self::get(conn, self.id)?; - searcher.update_document(conn, &post)?; + // TODO: Call publish_published() when newly published + self.publish_updated(); Ok(post) } @@ -559,6 +560,16 @@ impl Post { None, ) } + + fn publish_updated(&self) { + POST_CHAN.tell( + Publish { + msg: PostUpdated(self.clone()), + topic: "post.updated".into(), + }, + None, + ) + } } impl FromId for Post { From 864379fb81ae547a698d63bfabc20973defdec7b Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:38:50 +0900 Subject: [PATCH 18/39] Remove searcher from arguments of Post::update() --- plume-models/src/posts.rs | 5 ++--- src/routes/posts.rs | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 9045839d..a71b8a49 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -85,7 +85,7 @@ impl Post { Ok(post) } - pub fn update(&self, conn: &Connection, _searcher: &Searcher) -> Result { + pub fn update(&self, conn: &Connection) -> Result { diesel::update(self).set(self).execute(conn)?; let post = Self::get(conn, self.id)?; // TODO: Call publish_published() when newly published @@ -750,7 +750,6 @@ impl AsObject for PostUpdate { fn activity(self, c: &PlumeRocket, actor: User, _id: &str) -> Result<()> { let conn = &*c.conn; - let searcher = &c.searcher; let mut post = Post::from_id(c, &self.ap_url, None, CONFIG.proxy()).map_err(|(_, e)| e)?; if !post.is_author(conn, actor.id)? { @@ -812,7 +811,7 @@ impl AsObject for PostUpdate { post.update_hashtags(conn, hashtags)?; } - post.update(conn, searcher)?; + post.update(conn)?; Ok(()) } } diff --git a/src/routes/posts.rs b/src/routes/posts.rs index b1e408f4..f76100b1 100644 --- a/src/routes/posts.rs +++ b/src/routes/posts.rs @@ -298,8 +298,7 @@ pub fn update( post.source = form.content.clone(); post.license = form.license.clone(); post.cover_id = form.cover; - post.update(&*conn, &rockets.searcher) - .expect("post::update: update error"); + post.update(&*conn).expect("post::update: update error"); if post.published { post.update_mentions( From 82524c9dca8e86436e451608a6dc18de007bd33d Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:38:56 +0900 Subject: [PATCH 19/39] Update pot --- po/plume/plume.pot | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/po/plume/plume.pot b/po/plume/plume.pot index 1a5ce938..ec4c73a2 100644 --- a/po/plume/plume.pot +++ b/po/plume/plume.pot @@ -188,31 +188,31 @@ msgstr "" msgid "You are not allowed to publish on this blog." msgstr "" -# src/routes/posts.rs:364 +# src/routes/posts.rs:363 msgid "Your article has been updated." msgstr "" -# src/routes/posts.rs:554 +# src/routes/posts.rs:553 msgid "Your article has been saved." msgstr "" -# src/routes/posts.rs:561 +# src/routes/posts.rs:560 msgid "New article" msgstr "" -# src/routes/posts.rs:598 +# src/routes/posts.rs:597 msgid "You are not allowed to delete this article." msgstr "" -# src/routes/posts.rs:623 +# src/routes/posts.rs:622 msgid "Your article has been deleted." msgstr "" -# src/routes/posts.rs:628 +# src/routes/posts.rs:627 msgid "It looks like the article you tried to delete doesn't exist. Maybe it is already gone?" msgstr "" -# src/routes/posts.rs:668 +# src/routes/posts.rs:667 msgid "Couldn't obtain enough information about your account. Please make sure your username is correct." msgstr "" From 2a8cc5f3ba0f6d4cd29ee39d12c379b075c80af4 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:41:48 +0900 Subject: [PATCH 20/39] Send event to channel when post deleted --- plume-models/src/posts.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index a71b8a49..1aafa693 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -93,12 +93,12 @@ impl Post { Ok(post) } - pub fn delete(&self, conn: &Connection, searcher: &Searcher) -> Result<()> { + pub fn delete(&self, conn: &Connection, _searcher: &Searcher) -> Result<()> { for m in Mention::list_for_post(&conn, self.id)? { m.delete(conn)?; } diesel::delete(self).execute(conn)?; - searcher.delete_document(self); + self.publish_deleted(); Ok(()) } @@ -570,6 +570,16 @@ impl Post { None, ) } + + fn publish_deleted(&self) { + POST_CHAN.tell( + Publish { + msg: PostDeleted(self.clone()), + topic: "post.deleted".into(), + }, + None, + ) + } } impl FromId for Post { From fc8ee1c3bc808bffde53ffd0793233c7681edb6c Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:51:46 +0900 Subject: [PATCH 21/39] Remove searcher from arguments of Post::delete() and dependented methods --- plume-models/src/blogs.rs | 6 +++--- plume-models/src/posts.rs | 8 ++++---- plume-models/src/users.rs | 18 ++++++++---------- src/api/posts.rs | 2 +- src/routes/blogs.rs | 3 +-- src/routes/instance.rs | 13 +++---------- src/routes/user.rs | 2 +- 7 files changed, 21 insertions(+), 31 deletions(-) diff --git a/plume-models/src/blogs.rs b/plume-models/src/blogs.rs index 8df4a9dc..e524c282 100644 --- a/plume-models/src/blogs.rs +++ b/plume-models/src/blogs.rs @@ -1,6 +1,6 @@ use crate::{ ap_url, instance::*, medias::Media, posts::Post, safe_string::SafeString, schema::blogs, - search::Searcher, users::User, Connection, Error, PlumeRocket, Result, CONFIG, ITEMS_PER_PAGE, + users::User, Connection, Error, PlumeRocket, Result, CONFIG, ITEMS_PER_PAGE, }; use activitypub::{ actor::Group, @@ -317,9 +317,9 @@ impl Blog { .and_then(|c| c.url().ok()) } - pub fn delete(&self, conn: &Connection, searcher: &Searcher) -> Result<()> { + pub fn delete(&self, conn: &Connection) -> Result<()> { for post in Post::get_for_blog(conn, &self)? { - post.delete(conn, searcher)?; + post.delete(conn)?; } diesel::delete(self) .execute(conn) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 1aafa693..6fe25254 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -1,7 +1,7 @@ use crate::{ ap_url, blogs::Blog, instance::Instance, medias::Media, mentions::Mention, post_authors::*, - safe_string::SafeString, schema::posts, search::Searcher, tags::*, timeline::*, users::User, - Connection, Error, PlumeRocket, PostEvent::*, Result, CONFIG, POST_CHAN, + safe_string::SafeString, schema::posts, tags::*, timeline::*, users::User, Connection, Error, + PlumeRocket, PostEvent::*, Result, CONFIG, POST_CHAN, }; use activitypub::{ activity::{Create, Delete, Update}, @@ -93,7 +93,7 @@ impl Post { Ok(post) } - pub fn delete(&self, conn: &Connection, _searcher: &Searcher) -> Result<()> { + pub fn delete(&self, conn: &Connection) -> Result<()> { for m in Mention::list_for_post(&conn, self.id)? { m.delete(conn)?; } @@ -703,7 +703,7 @@ impl AsObject for Post { .into_iter() .any(|a| actor.id == a.id); if can_delete { - self.delete(&c.conn, &c.searcher).map(|_| ()) + self.delete(&c.conn).map(|_| ()) } else { Err(Error::Unauthorized) } diff --git a/plume-models/src/users.rs b/plume-models/src/users.rs index 753774f8..d3b73684 100644 --- a/plume-models/src/users.rs +++ b/plume-models/src/users.rs @@ -1,8 +1,8 @@ use crate::{ ap_url, blocklisted_emails::BlocklistedEmail, blogs::Blog, db_conn::DbConn, follows::Follow, instance::*, medias::Media, notifications::Notification, post_authors::PostAuthor, posts::Post, - safe_string::SafeString, schema::users, search::Searcher, timeline::Timeline, Connection, - Error, PlumeRocket, Result, CONFIG, ITEMS_PER_PAGE, + safe_string::SafeString, schema::users, timeline::Timeline, Connection, Error, PlumeRocket, + Result, CONFIG, ITEMS_PER_PAGE, }; use activitypub::{ activity::Delete, @@ -129,14 +129,14 @@ impl User { .map_err(Error::from) } - pub fn delete(&self, conn: &Connection, searcher: &Searcher) -> Result<()> { + pub fn delete(&self, conn: &Connection) -> Result<()> { use crate::schema::post_authors; for blog in Blog::find_for_author(conn, self)? .iter() .filter(|b| b.count_authors(conn).map(|c| c <= 1).unwrap_or(false)) { - blog.delete(conn, searcher)?; + blog.delete(conn)?; } // delete the posts if they is the only author let all_their_posts_ids: Vec = post_authors::table @@ -156,7 +156,7 @@ impl User { .unwrap_or(&0) > &0; if !has_other_authors { - Post::get(conn, post_id)?.delete(conn, searcher)?; + Post::get(conn, post_id)?.delete(conn)?; } } @@ -1037,7 +1037,7 @@ impl AsObject for User { fn activity(self, c: &PlumeRocket, actor: User, _id: &str) -> Result<()> { if self.id == actor.id { - self.delete(&c.conn, &c.searcher).map(|_| ()) + self.delete(&c.conn).map(|_| ()) } else { Err(Error::Unauthorized) } @@ -1227,9 +1227,7 @@ pub(crate) mod tests { let inserted = fill_database(conn); assert!(User::get(conn, inserted[0].id).is_ok()); - inserted[0] - .delete(conn, &get_searcher(&CONFIG.search_tokenizers)) - .unwrap(); + inserted[0].delete(conn).unwrap(); assert!(User::get(conn, inserted[0].id).is_err()); Ok(()) }); @@ -1319,7 +1317,7 @@ pub(crate) mod tests { let users = fill_database(conn); let ap_repr = users[0].to_activity(conn).unwrap(); - users[0].delete(conn, &*r.searcher).unwrap(); + users[0].delete(conn).unwrap(); let user = User::from_activity(&r, ap_repr).unwrap(); assert_eq!(user.username, users[0].username); diff --git a/src/api/posts.rs b/src/api/posts.rs index ad483a12..fca56108 100644 --- a/src/api/posts.rs +++ b/src/api/posts.rs @@ -230,7 +230,7 @@ pub fn delete(auth: Authorization, rockets: PlumeRocket, id: i32) - let author = User::get(&*rockets.conn, auth.0.user_id)?; if let Ok(post) = Post::get(&*rockets.conn, id) { if post.is_author(&*rockets.conn, author.id).unwrap_or(false) { - post.delete(&*rockets.conn, &rockets.searcher)?; + post.delete(&*rockets.conn)?; } } Ok(Json(())) diff --git a/src/routes/blogs.rs b/src/routes/blogs.rs index 0c08c536..8696e82d 100644 --- a/src/routes/blogs.rs +++ b/src/routes/blogs.rs @@ -153,8 +153,7 @@ pub fn delete(name: String, rockets: PlumeRocket) -> RespondOrRedirect { .and_then(|u| u.is_author_in(&*conn, &blog).ok()) .unwrap_or(false) { - blog.delete(&conn, &rockets.searcher) - .expect("blog::expect: deletion error"); + blog.delete(&conn).expect("blog::expect: deletion error"); Flash::success( Redirect::to(uri!(super::instance::index)), i18n!(rockets.intl.catalog, "Your blog was deleted."), diff --git a/src/routes/instance.rs b/src/routes/instance.rs index 83bc46b6..bec1b092 100644 --- a/src/routes/instance.rs +++ b/src/routes/instance.rs @@ -21,7 +21,6 @@ use plume_models::{ instance::*, posts::Post, safe_string::SafeString, - search::Searcher, timeline::Timeline, users::{Role, User}, Connection, Error, PlumeRocket, CONFIG, @@ -331,7 +330,6 @@ pub fn edit_users( } let conn = &rockets.conn; - let searcher = &*rockets.searcher; let worker = &*rockets.worker; match form.action { UserActions::Admin => { @@ -351,7 +349,7 @@ pub fn edit_users( } UserActions::Ban => { for u in form.ids.clone() { - ban(u, conn, searcher, worker)?; + ban(u, conn, worker)?; } } } @@ -362,14 +360,9 @@ pub fn edit_users( )) } -fn ban( - id: i32, - conn: &Connection, - searcher: &Searcher, - worker: &ScheduledThreadPool, -) -> Result<(), ErrorPage> { +fn ban(id: i32, conn: &Connection, worker: &ScheduledThreadPool) -> Result<(), ErrorPage> { let u = User::get(&*conn, id)?; - u.delete(&*conn, searcher)?; + u.delete(&*conn)?; if Instance::get_local() .map(|i| u.instance_id == i.id) .unwrap_or(false) diff --git a/src/routes/user.rs b/src/routes/user.rs index f64fd42c..2a9f2080 100644 --- a/src/routes/user.rs +++ b/src/routes/user.rs @@ -421,7 +421,7 @@ pub fn delete( ) -> Result, ErrorPage> { let account = User::find_by_fqn(&rockets, &name)?; if user.id == account.id { - account.delete(&*rockets.conn, &rockets.searcher)?; + account.delete(&*rockets.conn)?; let target = User::one_by_instance(&*rockets.conn)?; let delete_act = account.delete_activity(&*rockets.conn)?; From cd5895d560829141b19b8ea7c5175ec04ff97eb7 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Thu, 7 Jan 2021 22:51:51 +0900 Subject: [PATCH 22/39] Update pot --- po/plume/plume.pot | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/po/plume/plume.pot b/po/plume/plume.pot index ec4c73a2..8d82c5b3 100644 --- a/po/plume/plume.pot +++ b/po/plume/plume.pot @@ -72,27 +72,27 @@ msgstr "" msgid "Your blog was successfully created!" msgstr "" -# src/routes/blogs.rs:160 +# src/routes/blogs.rs:159 msgid "Your blog was deleted." msgstr "" -# src/routes/blogs.rs:168 +# src/routes/blogs.rs:167 msgid "You are not allowed to delete this blog." msgstr "" -# src/routes/blogs.rs:219 +# src/routes/blogs.rs:218 msgid "You are not allowed to edit this blog." msgstr "" -# src/routes/blogs.rs:275 +# src/routes/blogs.rs:274 msgid "You can't use this media as a blog icon." msgstr "" -# src/routes/blogs.rs:293 +# src/routes/blogs.rs:292 msgid "You can't use this media as a blog banner." msgstr "" -# src/routes/blogs.rs:327 +# src/routes/blogs.rs:326 msgid "Your blog information have been updated." msgstr "" @@ -104,39 +104,39 @@ msgstr "" msgid "Your comment has been deleted." msgstr "" -# src/routes/instance.rs:119 +# src/routes/instance.rs:118 msgid "Instance settings have been saved." msgstr "" -# src/routes/instance.rs:151 +# src/routes/instance.rs:150 msgid "{} has been unblocked." msgstr "" -# src/routes/instance.rs:153 +# src/routes/instance.rs:152 msgid "{} has been blocked." msgstr "" -# src/routes/instance.rs:202 +# src/routes/instance.rs:201 msgid "Blocks deleted" msgstr "" -# src/routes/instance.rs:217 +# src/routes/instance.rs:216 msgid "Email already blocked" msgstr "" -# src/routes/instance.rs:222 +# src/routes/instance.rs:221 msgid "Email Blocked" msgstr "" -# src/routes/instance.rs:313 +# src/routes/instance.rs:312 msgid "You can't change your own rights." msgstr "" -# src/routes/instance.rs:324 +# src/routes/instance.rs:323 msgid "You are not allowed to take this action." msgstr "" -# src/routes/instance.rs:361 +# src/routes/instance.rs:359 msgid "Done." msgstr "" From a0cd9dd6bd3c948ba3bfadb987783c8b6031f9e3 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 9 Jan 2021 01:08:31 +0900 Subject: [PATCH 23/39] Fix tests to follow API changes --- plume-models/src/blogs.rs | 13 ++++--------- plume-models/src/inbox.rs | 1 - plume-models/src/lib.rs | 4 ++++ plume-models/src/posts.rs | 2 +- plume-models/src/search/mod.rs | 27 ++++++++++++++------------- plume-models/src/timeline/mod.rs | 9 --------- plume-models/src/users.rs | 2 -- 7 files changed, 23 insertions(+), 35 deletions(-) diff --git a/plume-models/src/blogs.rs b/plume-models/src/blogs.rs index e524c282..cfc0624b 100644 --- a/plume-models/src/blogs.rs +++ b/plume-models/src/blogs.rs @@ -497,10 +497,8 @@ pub(crate) mod tests { use super::*; use crate::{ blog_authors::*, - config::CONFIG, instance::tests as instance_tests, medias::NewMedia, - search::tests::get_searcher, tests::{db, rockets}, users::tests as usersTests, Connection as Conn, @@ -767,9 +765,7 @@ pub(crate) mod tests { conn.test_transaction::<_, (), _>(|| { let (_, blogs) = fill_database(conn); - blogs[0] - .delete(conn, &get_searcher(&CONFIG.search_tokenizers)) - .unwrap(); + blogs[0].delete(conn).unwrap(); assert!(Blog::get(conn, blogs[0].id).is_err()); Ok(()) }) @@ -779,7 +775,6 @@ pub(crate) mod tests { fn delete_via_user() { let conn = &db(); conn.test_transaction::<_, (), _>(|| { - let searcher = get_searcher(&CONFIG.search_tokenizers); let (user, _) = fill_database(conn); let b1 = Blog::insert( @@ -836,10 +831,10 @@ pub(crate) mod tests { ) .unwrap(); - user[0].delete(conn, &searcher).unwrap(); + user[0].delete(conn).unwrap(); assert!(Blog::get(conn, blog[0].id).is_ok()); assert!(Blog::get(conn, blog[1].id).is_err()); - user[1].delete(conn, &searcher).unwrap(); + user[1].delete(conn).unwrap(); assert!(Blog::get(conn, blog[0].id).is_err()); Ok(()) }) @@ -886,7 +881,7 @@ pub(crate) mod tests { let _: Blog = blogs[0].save_changes(conn).unwrap(); let ap_repr = blogs[0].to_activity(conn).unwrap(); - blogs[0].delete(conn, &*r.searcher).unwrap(); + blogs[0].delete(conn).unwrap(); let blog = Blog::from_activity(&r, ap_repr).unwrap(); assert_eq!(blog.actor_id, blogs[0].actor_id); diff --git a/plume-models/src/inbox.rs b/plume-models/src/inbox.rs index 72acba2a..751eb550 100644 --- a/plume-models/src/inbox.rs +++ b/plume-models/src/inbox.rs @@ -97,7 +97,6 @@ pub(crate) mod tests { source: String::new(), cover_id: None, }, - &rockets.searcher, ) .unwrap(); diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index 5a751877..64881fba 100755 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -316,6 +316,10 @@ mod tests { db_conn::DbConn((*DB_POOL).get().unwrap()) } + pub(crate) fn pool() -> &'static db_conn::DbPool { + &*DB_POOL + } + lazy_static! { static ref DB_POOL: db_conn::DbPool = { let pool = db_conn::DbPool::builder() diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 6fe25254..4a9d1c5b 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -893,7 +893,7 @@ mod tests { ) .unwrap(); let create = post.create_activity(conn).unwrap(); - post.delete(conn, &r.searcher).unwrap(); + post.delete(conn).unwrap(); match inbox(&r, serde_json::to_value(create).unwrap()).unwrap() { InboxResult::Post(p) => { diff --git a/plume-models/src/search/mod.rs b/plume-models/src/search/mod.rs index 33de8d49..abaf4a91 100644 --- a/plume-models/src/search/mod.rs +++ b/plume-models/src/search/mod.rs @@ -8,21 +8,21 @@ pub use self::tokenizer::TokenizerKind; #[cfg(test)] pub(crate) mod tests { - use super::{Query, Searcher, TokenizerKind}; - use diesel::Connection; - use plume_common::utils::random_hex; - use std::env::temp_dir; - use std::str::FromStr; - + use super::{actor::SearchActor, Query, Searcher, TokenizerKind}; use crate::{ blogs::tests::fill_database, config::SearchTokenizerConfig, post_authors::*, posts::{NewPost, Post}, safe_string::SafeString, - tests::db, + tests::pool, CONFIG, }; + use diesel::Connection; + use plume_common::utils::random_hex; + use std::env::temp_dir; + use std::str::FromStr; + use std::sync::Arc; pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher { let dir = temp_dir().join(&format!("plume-test-{}", random_hex())); @@ -122,9 +122,12 @@ pub(crate) mod tests { #[test] fn search() { - let conn = &db(); + let pool = pool(); + let conn = &pool.get().unwrap(); + conn.test_transaction::<_, (), _>(|| { - let searcher = get_searcher(&CONFIG.search_tokenizers); + let searcher = Arc::new(get_searcher(&CONFIG.search_tokenizers)); + SearchActor::init(searcher.clone(), pool.clone()); let blog = &fill_database(conn).1[0]; let author = &blog.list_authors(conn).unwrap()[0]; @@ -145,7 +148,6 @@ pub(crate) mod tests { source: "".to_owned(), cover_id: None, }, - &searcher, ) .unwrap(); PostAuthor::insert( @@ -165,7 +167,7 @@ pub(crate) mod tests { let newtitle = random_hex()[..8].to_owned(); post.title = newtitle.clone(); - post.update(conn, &searcher).unwrap(); + post.update(conn).unwrap(); searcher.commit(); assert_eq!( searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))[0].id, @@ -175,7 +177,7 @@ pub(crate) mod tests { .search_document(conn, Query::from_str(&title).unwrap(), (0, 1)) .is_empty()); - post.delete(conn, &searcher).unwrap(); + post.delete(conn).unwrap(); searcher.commit(); assert!(searcher .search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1)) @@ -214,7 +216,6 @@ pub(crate) mod tests { source: "".to_owned(), cover_id: None, }, - &searcher, ) .unwrap(); diff --git a/plume-models/src/timeline/mod.rs b/plume-models/src/timeline/mod.rs index 426c228b..ecfb2b52 100644 --- a/plume-models/src/timeline/mod.rs +++ b/plume-models/src/timeline/mod.rs @@ -408,7 +408,6 @@ mod tests { source: "you must say GNU/Linux, not Linux!!!".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); assert!(gnu_tl.matches(r, &gnu_post, Kind::Original).unwrap()); @@ -428,7 +427,6 @@ mod tests { source: "so is Microsoft".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); assert!(!gnu_tl.matches(r, &non_free_post, Kind::Original).unwrap()); @@ -481,7 +479,6 @@ mod tests { subtitle: "".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); assert!(my_tl.matches(r, &post, Kind::Original).unwrap()); // matches because of "blog in fav_blogs" (and there is no cover) @@ -503,7 +500,6 @@ mod tests { subtitle: "".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); assert!(!my_tl.matches(r, &post, Kind::Like(&users[1])).unwrap()); @@ -549,7 +545,6 @@ mod tests { source: "you must say GNU/Linux, not Linux!!!".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); @@ -568,7 +563,6 @@ mod tests { source: "so is Microsoft".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); @@ -608,7 +602,6 @@ mod tests { source: "you must say GNU/Linux, not Linux!!!".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); gnu_post @@ -745,7 +738,6 @@ mod tests { source: "you must say GNU/Linux, not Linux!!!".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); gnu_post.update_tags(conn, vec![Tag::build_activity("free".to_owned()).unwrap()]).unwrap(); @@ -779,7 +771,6 @@ mod tests { source: "you must say GNU/Linux, not Linux!!!".to_string(), cover_id: None, }, - &r.searcher, ) .unwrap(); diff --git a/plume-models/src/users.rs b/plume-models/src/users.rs index d3b73684..c4729acc 100644 --- a/plume-models/src/users.rs +++ b/plume-models/src/users.rs @@ -1130,9 +1130,7 @@ impl NewUser { pub(crate) mod tests { use super::*; use crate::{ - config::CONFIG, instance::{tests as instance_tests, Instance}, - search::tests::get_searcher, tests::{db, rockets}, Connection as Conn, }; From 5570494a372339b20d81b915399ad5bdfd88077d Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 9 Jan 2021 01:10:53 +0900 Subject: [PATCH 24/39] Update pot --- po/plume-front/cy.po | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/po/plume-front/cy.po b/po/plume-front/cy.po index 09f8c8e2..4c722366 100644 --- a/po/plume-front/cy.po +++ b/po/plume-front/cy.po @@ -10,12 +10,17 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Plural-Forms: nplurals=6; plural=(n == 0) ? 0 : ((n == 1) ? 1 : ((n == 2) ? 2 : ((n == 3) ? 3 : ((n == 6) ? 4 : 5))));\n" +"Plural-Forms: nplurals=6; plural=(n == 0) ? 0 : ((n == 1) ? 1 : ((n == 2) ? " +"2 : ((n == 3) ? 3 : ((n == 6) ? 4 : 5))));\n" "X-Generator: crowdin.com\n" "X-Crowdin-Project: plume\n" "X-Crowdin-Language: cy\n" "X-Crowdin-File: /master/po/plume-front/plume-front.pot\n" +# plume-front/src/editor.rs:189 +msgid "Do you want to load the local autosave last edited at {}?" +msgstr "" + # plume-front/src/editor.rs:114 msgid "Open the rich text editor" msgstr "" @@ -24,8 +29,8 @@ msgstr "" msgid "Title" msgstr "" -# plume-front/src/editor.rs:147 -msgid "Subtitle or summary" +# plume-front/src/editor.rs:319 +msgid "Subtitle, or summary" msgstr "" # plume-front/src/editor.rs:154 @@ -55,4 +60,3 @@ msgstr "" # plume-front/src/editor.rs:259 msgid "Publish" msgstr "" - From a46b7189371e57d3f459956fb5cc16aa095d1bd3 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 9 Jan 2021 14:31:55 +0900 Subject: [PATCH 25/39] Ignore search::tests::search --- plume-models/src/search/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plume-models/src/search/mod.rs b/plume-models/src/search/mod.rs index abaf4a91..93ed109f 100644 --- a/plume-models/src/search/mod.rs +++ b/plume-models/src/search/mod.rs @@ -120,7 +120,16 @@ pub(crate) mod tests { Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created } + /* + This assetions fails but I don't know why. + `test_transaction()` doesn't commit the transaction and + searcher working on the other thread cannot retrieve the result. + But, even when we use real `transaction()`, the searcher cannot retrieve. + I don't know why, but in the Plume process, it works. + It failes only when testing. + */ #[test] + #[ignore] fn search() { let pool = pool(); let conn = &pool.get().unwrap(); From 68d79bfa9c9911c21b1dbec11892a586725c48d2 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 9 Jan 2021 17:57:14 +0900 Subject: [PATCH 26/39] Publish PostUpdated only when the post is published --- plume-models/src/posts.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 4a9d1c5b..2f95a81a 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -89,7 +89,9 @@ impl Post { diesel::update(self).set(self).execute(conn)?; let post = Self::get(conn, self.id)?; // TODO: Call publish_published() when newly published - self.publish_updated(); + if post.published { + self.publish_updated(); + } Ok(post) } From 31418b10798bf79127ec6feebad6997d1df8b4e8 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 9 Jan 2021 18:07:46 +0900 Subject: [PATCH 27/39] Fix search::tests::search to use searcher directly instead of actor --- plume-models/src/lib.rs | 4 ---- plume-models/src/search/mod.rs | 26 +++++++------------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index 64881fba..5a751877 100755 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -316,10 +316,6 @@ mod tests { db_conn::DbConn((*DB_POOL).get().unwrap()) } - pub(crate) fn pool() -> &'static db_conn::DbPool { - &*DB_POOL - } - lazy_static! { static ref DB_POOL: db_conn::DbPool = { let pool = db_conn::DbPool::builder() diff --git a/plume-models/src/search/mod.rs b/plume-models/src/search/mod.rs index 93ed109f..917f7778 100644 --- a/plume-models/src/search/mod.rs +++ b/plume-models/src/search/mod.rs @@ -8,21 +8,20 @@ pub use self::tokenizer::TokenizerKind; #[cfg(test)] pub(crate) mod tests { - use super::{actor::SearchActor, Query, Searcher, TokenizerKind}; + use super::{Query, Searcher}; use crate::{ blogs::tests::fill_database, config::SearchTokenizerConfig, post_authors::*, posts::{NewPost, Post}, safe_string::SafeString, - tests::pool, + tests::db, CONFIG, }; use diesel::Connection; use plume_common::utils::random_hex; use std::env::temp_dir; use std::str::FromStr; - use std::sync::Arc; pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher { let dir = temp_dir().join(&format!("plume-test-{}", random_hex())); @@ -120,23 +119,11 @@ pub(crate) mod tests { Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created } - /* - This assetions fails but I don't know why. - `test_transaction()` doesn't commit the transaction and - searcher working on the other thread cannot retrieve the result. - But, even when we use real `transaction()`, the searcher cannot retrieve. - I don't know why, but in the Plume process, it works. - It failes only when testing. - */ #[test] - #[ignore] fn search() { - let pool = pool(); - let conn = &pool.get().unwrap(); - + let conn = &db(); conn.test_transaction::<_, (), _>(|| { - let searcher = Arc::new(get_searcher(&CONFIG.search_tokenizers)); - SearchActor::init(searcher.clone(), pool.clone()); + let searcher = get_searcher(&CONFIG.search_tokenizers); let blog = &fill_database(conn).1[0]; let author = &blog.list_authors(conn).unwrap()[0]; @@ -167,7 +154,7 @@ pub(crate) mod tests { }, ) .unwrap(); - + searcher.add_document(&conn, &post).unwrap(); searcher.commit(); assert_eq!( searcher.search_document(conn, Query::from_str(&title).unwrap(), (0, 1))[0].id, @@ -177,6 +164,7 @@ pub(crate) mod tests { let newtitle = random_hex()[..8].to_owned(); post.title = newtitle.clone(); post.update(conn).unwrap(); + searcher.update_document(conn, &post).unwrap(); searcher.commit(); assert_eq!( searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))[0].id, @@ -186,7 +174,7 @@ pub(crate) mod tests { .search_document(conn, Query::from_str(&title).unwrap(), (0, 1)) .is_empty()); - post.delete(conn).unwrap(); + searcher.delete_document(&post); searcher.commit(); assert!(searcher .search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1)) From 1d042e1bb22a3621d46d07527e316f502753a392 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Mon, 11 Jan 2021 12:02:18 +0900 Subject: [PATCH 28/39] Add methods to clear data of instances and users table for testing --- plume-models/src/instance.rs | 4 ++++ plume-models/src/users.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/plume-models/src/instance.rs b/plume-models/src/instance.rs index a2114915..9e25a6b8 100644 --- a/plume-models/src/instance.rs +++ b/plume-models/src/instance.rs @@ -306,6 +306,10 @@ pub(crate) mod tests { res } + pub(crate) fn clear_instances(conn: &Conn) { + diesel::delete(crate::schema::instances::table).execute(conn).unwrap(); + } + #[test] fn local_instance() { let conn = &db(); diff --git a/plume-models/src/users.rs b/plume-models/src/users.rs index c4729acc..2b751e45 100644 --- a/plume-models/src/users.rs +++ b/plume-models/src/users.rs @@ -1171,6 +1171,10 @@ pub(crate) mod tests { vec![admin, user, other] } + pub(crate) fn clear_users(conn: &Conn) { + diesel::delete(crate::schema::users::table).execute(conn).unwrap(); + } + #[test] fn find_by() { let r = rockets(); From 47e67d9b64b4915dcd48be0f99f6d2b52822c2c5 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Mon, 11 Jan 2021 12:03:27 +0900 Subject: [PATCH 29/39] Add methods to clear data of instances and users table for testing --- plume-models/src/instance.rs | 4 +++- plume-models/src/users.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plume-models/src/instance.rs b/plume-models/src/instance.rs index 9e25a6b8..4ae62f51 100644 --- a/plume-models/src/instance.rs +++ b/plume-models/src/instance.rs @@ -307,7 +307,9 @@ pub(crate) mod tests { } pub(crate) fn clear_instances(conn: &Conn) { - diesel::delete(crate::schema::instances::table).execute(conn).unwrap(); + diesel::delete(crate::schema::instances::table) + .execute(conn) + .unwrap(); } #[test] diff --git a/plume-models/src/users.rs b/plume-models/src/users.rs index 2b751e45..07a3051d 100644 --- a/plume-models/src/users.rs +++ b/plume-models/src/users.rs @@ -1172,7 +1172,9 @@ pub(crate) mod tests { } pub(crate) fn clear_users(conn: &Conn) { - diesel::delete(crate::schema::users::table).execute(conn).unwrap(); + diesel::delete(crate::schema::users::table) + .execute(conn) + .unwrap(); } #[test] From 2483303138af6192147286fe5fc2ac82c17cc0e3 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Mon, 11 Jan 2021 12:05:46 +0900 Subject: [PATCH 30/39] SearchActor waits for transaction commit --- plume-models/src/search/actor.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index bc70011d..49852a95 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -2,6 +2,8 @@ use super::Searcher; use crate::{db_conn::DbPool, posts::PostEvent, ACTOR_SYS, POST_CHAN}; use riker::actors::{Actor, ActorFactoryArgs, ActorRefFactory, Context, Sender, Subscribe, Tell}; use std::sync::Arc; +use std::thread::sleep; +use std::time::Duration; use tracing::error; pub struct SearchActor { @@ -33,6 +35,9 @@ impl Actor for SearchActor { fn recv(&mut self, _ctx: &Context, msg: Self::Msg, _sender: Sender) { use PostEvent::*; + // Wait for transaction commited + sleep(Duration::from_millis(500)); + match msg { PostPublished(post) => { let conn = self.conn.get(); From 42fb0cf8251e29fe45271dce9dad6ab6459a4cf1 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Mon, 11 Jan 2021 12:06:05 +0900 Subject: [PATCH 31/39] Use pattern match instead of unwrap() --- plume-models/src/search/actor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index 49852a95..b8724f2b 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -42,9 +42,9 @@ impl Actor for SearchActor { PostPublished(post) => { let conn = self.conn.get(); match conn { - Ok(_) => { + Ok(conn) => { self.searcher - .add_document(&conn.unwrap(), &post) + .add_document(&conn, &post) .unwrap_or_else(|e| error!("{:?}", e)); } _ => { From 6c1dd6608d46d7a77133a785824b35ba0e8f1c2a Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Mon, 11 Jan 2021 12:06:22 +0900 Subject: [PATCH 32/39] Add test for search actor --- plume-models/src/instance.rs | 6 -- plume-models/src/search/actor.rs | 137 +++++++++++++++++++++++++++++++ plume-models/src/users.rs | 6 -- 3 files changed, 137 insertions(+), 12 deletions(-) diff --git a/plume-models/src/instance.rs b/plume-models/src/instance.rs index 4ae62f51..a2114915 100644 --- a/plume-models/src/instance.rs +++ b/plume-models/src/instance.rs @@ -306,12 +306,6 @@ pub(crate) mod tests { res } - pub(crate) fn clear_instances(conn: &Conn) { - diesel::delete(crate::schema::instances::table) - .execute(conn) - .unwrap(); - } - #[test] fn local_instance() { let conn = &db(); diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index b8724f2b..6d99fda0 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -75,3 +75,140 @@ impl ActorFactoryArgs<(Arc, DbPool)> for SearchActor { Self { searcher, conn } } } + +#[cfg(test)] +mod tests { + use crate::diesel::Connection; + use crate::diesel::RunQueryDsl; + use crate::{ + blog_authors::{BlogAuthor, NewBlogAuthor}, + blogs::{Blog, NewBlog}, + db_conn::{DbPool, PragmaForeignKey}, + instance::{Instance, NewInstance}, + post_authors::{NewPostAuthor, PostAuthor}, + posts::{NewPost, Post}, + safe_string::SafeString, + search::{actor::SearchActor, tests::get_searcher, Query}, + users::{NewUser, User}, + Connection as Conn, CONFIG, + }; + use diesel::r2d2::ConnectionManager; + use plume_common::utils::random_hex; + use std::str::FromStr; + use std::sync::Arc; + use std::thread::sleep; + use std::time::Duration; + + #[test] + fn post_updated() { + // Need to commit so that searcher on another thread retrieve records. + // So, build DbPool instead of using DB_POOL for testing. + let manager = ConnectionManager::::new(CONFIG.database_url.as_str()); + let db_pool = DbPool::builder() + .connection_customizer(Box::new(PragmaForeignKey)) + .build(manager) + .unwrap(); + + let searcher = Arc::new(get_searcher(&CONFIG.search_tokenizers)); + SearchActor::init(searcher.clone(), db_pool.clone()); + let conn = db_pool.clone().get().unwrap(); + + let title = random_hex()[..8].to_owned(); + let mut post_id = 0; + let (instance, user, blog, post) = &conn + .transaction::<(Instance, User, Blog, Post), diesel::result::Error, _>(|| { + let (instance, user, blog) = fill_database(&conn); + let author = &blog.list_authors(&conn).unwrap()[0]; + + let post = Post::insert( + &conn, + NewPost { + blog_id: blog.id, + slug: title.clone(), + title: title.clone(), + content: SafeString::new(""), + published: true, + license: "CC-BY-SA".to_owned(), + ap_url: "".to_owned(), + creation_date: None, + subtitle: "".to_owned(), + source: "".to_owned(), + cover_id: None, + }, + ) + .unwrap(); + PostAuthor::insert( + &conn, + NewPostAuthor { + post_id: post.id, + author_id: author.id, + }, + ) + .unwrap(); + post_id = post.id; + + Ok((instance, user, blog, post)) + }) + .unwrap(); + + // Wait for searcher on another thread add document asynchronously + sleep(Duration::from_millis(700)); + searcher.commit(); + assert_eq!( + searcher.search_document(&conn, Query::from_str(&title).unwrap(), (0, 1))[0].id, + post_id + ); + // TODO: Make sure records are deleted even when assertion failed + post.delete(&conn).unwrap(); + blog.delete(&conn).unwrap(); + user.delete(&conn).unwrap(); + diesel::delete(instance).execute(&conn).unwrap(); + } + + fn fill_database(conn: &Conn) -> (Instance, User, Blog) { + conn.transaction::<(Instance, User, Blog), diesel::result::Error, _>(|| { + let instance = Instance::insert( + conn, + NewInstance { + default_license: "CC-0-BY-SA".to_string(), + local: true, + long_description: SafeString::new("Good morning"), + long_description_html: "

Good morning

".to_string(), + short_description: SafeString::new("Hello"), + short_description_html: "

Hello

".to_string(), + name: random_hex().to_string(), + open_registrations: true, + public_domain: random_hex().to_string(), + }, + ) + .unwrap(); + let mut user = NewUser::default(); + user.instance_id = instance.id; + user.username = random_hex().to_string(); + user.ap_url = random_hex().to_string(); + user.inbox_url = random_hex().to_string(); + user.outbox_url = random_hex().to_string(); + user.followers_endpoint = random_hex().to_string(); + let user = User::insert(conn, user).unwrap(); + let mut blog = NewBlog::default(); + blog.instance_id = instance.id; + blog.actor_id = random_hex().to_string(); + blog.ap_url = random_hex().to_string(); + blog.inbox_url = random_hex().to_string(); + blog.outbox_url = random_hex().to_string(); + let blog = Blog::insert(conn, blog).unwrap(); + BlogAuthor::insert( + conn, + NewBlogAuthor { + blog_id: blog.id, + author_id: user.id, + is_owner: true, + }, + ) + .unwrap(); + + Ok((instance, user, blog)) + }) + .unwrap() + } +} diff --git a/plume-models/src/users.rs b/plume-models/src/users.rs index 07a3051d..c4729acc 100644 --- a/plume-models/src/users.rs +++ b/plume-models/src/users.rs @@ -1171,12 +1171,6 @@ pub(crate) mod tests { vec![admin, user, other] } - pub(crate) fn clear_users(conn: &Conn) { - diesel::delete(crate::schema::users::table) - .execute(conn) - .unwrap(); - } - #[test] fn find_by() { let r = rockets(); From 6dd730030df51438fd970e38194fa0cba4f58ab9 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Tue, 12 Jan 2021 09:27:55 +0900 Subject: [PATCH 33/39] Wrap Post in PostEvent with Arc See https://github.com/riker-rs/riker/issues/153 --- plume-models/src/posts.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 2f95a81a..fbb03261 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -21,6 +21,7 @@ use plume_common::{ }; use riker::actors::{Publish, Tell}; use std::collections::HashSet; +use std::sync::Arc; pub type LicensedArticle = CustomObject; @@ -556,7 +557,7 @@ impl Post { fn publish_published(&self) { POST_CHAN.tell( Publish { - msg: PostPublished(self.clone()), + msg: PostPublished(Arc::new(self.clone())), topic: "post.published".into(), }, None, @@ -566,7 +567,7 @@ impl Post { fn publish_updated(&self) { POST_CHAN.tell( Publish { - msg: PostUpdated(self.clone()), + msg: PostUpdated(Arc::new(self.clone())), topic: "post.updated".into(), }, None, @@ -576,7 +577,7 @@ impl Post { fn publish_deleted(&self) { POST_CHAN.tell( Publish { - msg: PostDeleted(self.clone()), + msg: PostDeleted(Arc::new(self.clone())), topic: "post.deleted".into(), }, None, @@ -836,12 +837,12 @@ impl IntoId for Post { #[derive(Clone, Debug)] pub enum PostEvent { - PostPublished(Post), - PostUpdated(Post), - PostDeleted(Post), + PostPublished(Arc), + PostUpdated(Arc), + PostDeleted(Arc), } -impl From for Post { +impl From for Arc { fn from(event: PostEvent) -> Self { use PostEvent::*; From c3c8f6804b6ae1ff21c4939336addda9fcc37f85 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Wed, 13 Jan 2021 21:58:25 +0900 Subject: [PATCH 34/39] Remove transaction from test search::tests::search for CI --- plume-models/src/search/actor.rs | 65 ++++++++++++++------------------ 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/plume-models/src/search/actor.rs b/plume-models/src/search/actor.rs index 6d99fda0..a0b0d22e 100644 --- a/plume-models/src/search/actor.rs +++ b/plume-models/src/search/actor.rs @@ -114,42 +114,35 @@ mod tests { let conn = db_pool.clone().get().unwrap(); let title = random_hex()[..8].to_owned(); - let mut post_id = 0; - let (instance, user, blog, post) = &conn - .transaction::<(Instance, User, Blog, Post), diesel::result::Error, _>(|| { - let (instance, user, blog) = fill_database(&conn); - let author = &blog.list_authors(&conn).unwrap()[0]; + let (instance, user, blog) = fill_database(&conn); + let author = &blog.list_authors(&conn).unwrap()[0]; - let post = Post::insert( - &conn, - NewPost { - blog_id: blog.id, - slug: title.clone(), - title: title.clone(), - content: SafeString::new(""), - published: true, - license: "CC-BY-SA".to_owned(), - ap_url: "".to_owned(), - creation_date: None, - subtitle: "".to_owned(), - source: "".to_owned(), - cover_id: None, - }, - ) - .unwrap(); - PostAuthor::insert( - &conn, - NewPostAuthor { - post_id: post.id, - author_id: author.id, - }, - ) - .unwrap(); - post_id = post.id; - - Ok((instance, user, blog, post)) - }) - .unwrap(); + let post = Post::insert( + &conn, + NewPost { + blog_id: blog.id, + slug: title.clone(), + title: title.clone(), + content: SafeString::new(""), + published: true, + license: "CC-BY-SA".to_owned(), + ap_url: "".to_owned(), + creation_date: None, + subtitle: "".to_owned(), + source: "".to_owned(), + cover_id: None, + }, + ) + .unwrap(); + PostAuthor::insert( + &conn, + NewPostAuthor { + post_id: post.id, + author_id: author.id, + }, + ) + .unwrap(); + let post_id = post.id; // Wait for searcher on another thread add document asynchronously sleep(Duration::from_millis(700)); @@ -162,7 +155,7 @@ mod tests { post.delete(&conn).unwrap(); blog.delete(&conn).unwrap(); user.delete(&conn).unwrap(); - diesel::delete(instance).execute(&conn).unwrap(); + diesel::delete(&instance).execute(&conn).unwrap(); } fn fill_database(conn: &Conn) -> (Instance, User, Blog) { From 06962514fec14d78a5ec99eb7ba0d71113ebe734 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 16 Jan 2021 15:58:54 +0900 Subject: [PATCH 35/39] Run database migration on CI --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4bc7386..492a9341 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,6 +141,9 @@ jobs: steps: - restore_env: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run_migration: + cmd: | + cargo run -p plume-cli -- migration run - run_with_coverage: cmd: | cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j" From e817941cc53df90ea8ba36db34b39953ab9378fd Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 16 Jan 2021 16:44:20 +0900 Subject: [PATCH 36/39] Update pot --- po/plume-front/plume-front.pot | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/po/plume-front/plume-front.pot b/po/plume-front/plume-front.pot index 28028a63..5a9d490d 100644 --- a/po/plume-front/plume-front.pot +++ b/po/plume-front/plume-front.pot @@ -12,46 +12,46 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n" -# plume-front/src/editor.rs:189 +# plume-front/src/editor.rs:188 msgid "Do you want to load the local autosave last edited at {}?" msgstr "" -# plume-front/src/editor.rs:282 +# plume-front/src/editor.rs:281 msgid "Open the rich text editor" msgstr "" -# plume-front/src/editor.rs:315 +# plume-front/src/editor.rs:314 msgid "Title" msgstr "" -# plume-front/src/editor.rs:319 +# plume-front/src/editor.rs:318 msgid "Subtitle, or summary" msgstr "" -# plume-front/src/editor.rs:326 +# plume-front/src/editor.rs:325 msgid "Write your article here. Markdown is supported." msgstr "" -# plume-front/src/editor.rs:337 +# plume-front/src/editor.rs:336 msgid "Around {} characters left" msgstr "" -# plume-front/src/editor.rs:414 +# plume-front/src/editor.rs:413 msgid "Tags" msgstr "" -# plume-front/src/editor.rs:415 +# plume-front/src/editor.rs:414 msgid "License" msgstr "" -# plume-front/src/editor.rs:418 +# plume-front/src/editor.rs:417 msgid "Cover" msgstr "" -# plume-front/src/editor.rs:438 +# plume-front/src/editor.rs:437 msgid "This is a draft" msgstr "" -# plume-front/src/editor.rs:445 +# plume-front/src/editor.rs:444 msgid "Publish" msgstr "" From 51483490986f4f171f7b9e86d8c93b150ab67c94 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 16 Jan 2021 16:44:39 +0900 Subject: [PATCH 37/39] Add changlog about actor and searcher --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11dbbeb3..c220aa80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Allow `dir` attributes for LtoR text in RtoL document (#860) - More translation languages (#862) - Proxy support (#829) +- Riker a actor system library (#870) ### Changed @@ -16,6 +17,7 @@ - Use tracing crate (#868) - Update Rust version to nightly-2021-01-15 (#878) - Upgrade Tantivy to 0.13.3 and lindera-tantivy to 0.7.1 (#878) +- Run searcher on actor system (#870) ### Fixed From ac507d1d42d7fa3ea4cb5762dbde1e34be006730 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 16 Jan 2021 16:50:57 +0900 Subject: [PATCH 38/39] Fix CI command --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 492a9341..88339178 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,11 +141,9 @@ jobs: steps: - restore_env: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - - run_migration: - cmd: | - cargo run -p plume-cli -- migration run - run_with_coverage: cmd: | + cargo run -p plume-cli -- migration run cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j" for i in 36 4 2 1 1; do $cmd $i && break From 02277b1acef8f1d9747f5faea521d097151d0a02 Mon Sep 17 00:00:00 2001 From: Kitaiti Makoto Date: Sat, 16 Jan 2021 17:31:05 +0900 Subject: [PATCH 39/39] Specify features on running migration on CI --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 88339178..c7aae109 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,7 +143,7 @@ jobs: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - run_with_coverage: cmd: | - cargo run -p plume-cli -- migration run + cargo run -p plume-cli --no-default-features --features=${FEATURES} -- migration run cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j" for i in 36 4 2 1 1; do $cmd $i && break