aitzol/Plume
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a search engine into Plume (#324)

Browse Source 
* Add search engine to the model

Add a Tantivy based search engine to the model
Implement most required functions for it

* Implement indexing and plm subcommands

Implement indexation on insert, update and delete
Modify func args to get the indexer where required
Add subcommand to initialize, refill and unlock search db

* Move to a new threadpool engine allowing scheduling

* Autocommit search index every half an hour

* Implement front part of search

Add default fields for search
Add new routes and templates for search and result
Implement FromFormValue for Page to reuse it on search result pagination
Add optional query parameters to paginate template's macro
Update to newer rocket_csrf, don't get csrf token on GET forms

* Handle process termination to release lock

Handle process termination
Add tests to search

* Add proper support for advanced search

Add an advanced search form to /search, in template and route
Modify Tantivy schema, add new tokenizer for some properties
Create new String query parser
Create Tantivy query AST from our own

* Split search.rs, add comment and tests

Split search.rs into multiple submodules
Add comments and tests for Query
Make user@domain be treated as one could assume
This commit is contained in:
fdb-hiroshima 2018-12-02 17:37:51 +01:00 committed by GitHub
parent 9714bafded
commit 449641d158
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1613 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -14,3 +14,4 @@ docker-compose.yml
*.sqlite
*.sqlite3
*.swp
search_index

377
Cargo.lock generated
View File

@ -102,6 +102,11 @@ dependencies = [
"nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ascii"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "atom_syndication"
version = "0.6.0"
@ -112,6 +117,16 @@ dependencies = [
"quick-xml 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "atomicwrites"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "atty"
version = "0.2.11"
@ -175,6 +190,14 @@ dependencies = [
"safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "base64"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bcrypt"
version = "0.2.0"
@ -187,6 +210,19 @@ dependencies = [
"rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bit-set"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bit-vec"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "0.7.0"
@ -202,6 +238,14 @@ name = "bitflags"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitpacking"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "block-buffer"
version = "0.3.3"
@ -276,6 +320,11 @@ name = "cc"
version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "census"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "cfg-if"
version = "0.1.5"
@ -333,6 +382,18 @@ dependencies = [
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "combine"
version = "3.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ascii 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "conv"
version = "0.3.3"
@ -377,6 +438,38 @@ dependencies = [
"build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-channel 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-deque 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-epoch 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-channel"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-epoch 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-deque"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-epoch 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-deque"
version = "0.6.1"
@ -399,11 +492,37 @@ dependencies = [
"scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-epoch"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-utils"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "crossbeam-utils"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crunchy"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "csrf"
version = "0.3.0"
@ -417,6 +536,15 @@ dependencies = [
"rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ctrlc"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "custom_derive"
version = "0.1.7"
@ -553,6 +681,11 @@ dependencies = [
"regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "downcast"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "dtoa"
version = "0.4.3"
@ -563,6 +696,11 @@ name = "either"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "either"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encoding_rs"
version = "0.8.10"
@ -587,6 +725,16 @@ dependencies = [
"backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fail"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "failure"
version = "0.1.2"
@ -658,6 +806,25 @@ dependencies = [
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fst"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fst-regex"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"fst 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
@ -821,6 +988,11 @@ dependencies = [
"syn 0.13.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "htmlescape"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "http"
version = "0.1.13"
@ -984,6 +1156,14 @@ dependencies = [
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "itertools"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "itoa"
version = "0.3.4"
@ -1026,6 +1206,14 @@ name = "lazycell"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "levenshtein_automata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"fst 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libc"
version = "0.2.43"
@ -1134,6 +1322,15 @@ dependencies = [
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "memmap"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "memoffset"
version = "0.2.1"
@ -1296,6 +1493,18 @@ dependencies = [
"unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "nix"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "nodrop"
version = "0.1.12"
@ -1374,6 +1583,15 @@ dependencies = [
"vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "owned-read"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rental 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "owning_ref"
version = "0.3.3"
@ -1382,6 +1600,14 @@ dependencies = [
"stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "owning_ref"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "parking_lot"
version = "0.6.4"
@ -1516,6 +1742,7 @@ dependencies = [
"canapi 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"colored 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"ctrlc 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"diesel 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"dotenv 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1523,15 +1750,17 @@ dependencies = [
"guid-create 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"multipart 0.15.3 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"plume-api 0.1.0",
"plume-common 0.2.0",
"plume-models 0.2.0",
"rocket 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)",
"rocket_codegen 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)",
"rocket_contrib 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)",
"rocket_csrf 0.1.0 (git+https://github.com/fdb-hiroshima/rocket_csrf?rev=2805ce5dbae4a6441208484426440885a5640a6a)",
"rocket_csrf 0.1.0 (git+https://github.com/fdb-hiroshima/rocket_csrf?rev=0dfb822d5cbf65a5eee698099368b7c0f4c61fa4)",
"rocket_i18n 0.1.1 (git+https://github.com/BaptisteGelez/rocket_i18n?rev=75a3bfd7b847324c078a355a7f101f8241a9f59b)",
"rpassword 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"scheduled-thread-pool 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1540,7 +1769,6 @@ dependencies = [
"validator 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"validator_derive 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"webfinger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"workerpool 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1601,6 +1829,7 @@ dependencies = [
"diesel_migrations 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"guid-create 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)",
"plume-api 0.1.0",
@ -1610,8 +1839,10 @@ dependencies = [
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
"tantivy 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"webfinger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"whatlang 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1809,6 +2040,11 @@ dependencies = [
"utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex-syntax"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "regex-syntax"
version = "0.5.6"
@ -1841,6 +2077,25 @@ dependencies = [
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rental"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rental-impl 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rental-impl"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "reqwest"
version = "0.9.2"
@ -1936,7 +2191,7 @@ dependencies = [
[[package]]
name = "rocket_csrf"
version = "0.1.0"
source = "git+https://github.com/fdb-hiroshima/rocket_csrf?rev=2805ce5dbae4a6441208484426440885a5640a6a#2805ce5dbae4a6441208484426440885a5640a6a"
source = "git+https://github.com/fdb-hiroshima/rocket_csrf?rev=0dfb822d5cbf65a5eee698099368b7c0f4c61fa4#0dfb822d5cbf65a5eee698099368b7c0f4c61fa4"
dependencies = [
"csrf 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"data-encoding 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1995,6 +2250,15 @@ dependencies = [
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rust-stemmers"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rustc-demangle"
version = "0.1.9"
@ -2204,6 +2468,15 @@ dependencies = [
"unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "snap"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "stable_deref_trait"
version = "1.1.1"
@ -2329,6 +2602,50 @@ name = "take"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tantivy"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"atomicwrites 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bitpacking 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"census 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"combine 3.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-channel 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"downcast 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
"fail 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"fst 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"fst-regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"owned-read 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"rust-stemmers 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
"snap 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"uuid 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempdir"
version = "0.3.7"
@ -2796,6 +3113,7 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -2887,6 +3205,14 @@ dependencies = [
"serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "whatlang"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi"
version = "0.2.8"
@ -2924,14 +3250,6 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "workerpool"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ws2_32-sys"
version = "0.2.1"
@ -2959,17 +3277,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum array_tool 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271"
"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee"
"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
"checksum ascii 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5fc969a8ce2c9c0c4b0429bb8431544f6658283c8326ba5ff8c762b75369335"
"checksum atom_syndication 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0a9a7ab83635ff7a3b04856f4ad95324dccc9b947ab1e790fc5c769ee6d6f60c"
"checksum atomicwrites 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3420b33cdefd3feb223dddc23739fc05cc034eb0f2be792c763e3d89e1eb6e3"
"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
"checksum backtrace 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "150ae7828afa7afb6d474f909d64072d21de1f3365b6e8ad8029bf7b1c6350a0"
"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a"
"checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0"
"checksum base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "621fc7ecb8008f86d7fb9b95356cd692ce9514b80a86d85b397f32a22da7b9e2"
"checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9"
"checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643"
"checksum bcrypt 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a1512813db09170b44a00870b58421876d797b77b085c5205a24db90905f758"
"checksum bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f1efcc46c18245a69c38fcc5cc650f16d3a59d034f3106e9ed63748f695730a"
"checksum bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4440d5cb623bb7390ae27fec0bb6c61111969860f8e3ae198bfa0663645e67cf"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
"checksum bitpacking 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "75c04b83d2b444a22c6a30f4d068597efbe468fe56f068e042e627ded2fb21e7"
"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab"
"checksum block-cipher-trait 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "370424437b9459f3dfd68428ed9376ddfe03d8b70ede29cc533b3557df186ab4"
"checksum blowfish 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95ede07672d9f4144c578439aa352604ec5c67a80c940fe8d382ddbeeeb3c6d8"
@ -2980,21 +3304,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62"
"checksum canapi 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff3e02a04f44b531d851d2db62f95aabf65d033a6724767a4bed9732563e9bc4"
"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16"
"checksum census 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e5c044df9888597e4e96610c916ce9d58c653b67c01b5eac5b7abd7405f4fee4"
"checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3"
"checksum chomp 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9f74ad218e66339b11fd23f693fb8f1d621e80ba6ac218297be26073365d163d"
"checksum chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878"
"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum colored 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc0a60679001b62fb628c4da80e574b9645ab4646056d7c9018885efffe45533"
"checksum combine 3.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "db733c5d0f4f52e78d4417959cadf0eecc7476e7f9ece05677912571a4af34e2"
"checksum conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299"
"checksum cookie 0.11.0-dev (git+https://github.com/alexcrichton/cookie-rs?rev=f191ca50)" = "<none>"
"checksum core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "286e0b41c3a20da26536c6000a280585d519fd07b3956b43aed8a79e9edce980"
"checksum core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "716c271e8613ace48344f723b60b900a93150271e5be206212d052bbc0883efa"
"checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
"checksum crossbeam 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d7408247b1b87f480890f28b670c5f8d9a8a4274833433fe74dc0dfd46d33650"
"checksum crossbeam-channel 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b85741761b7f160bc5e7e0c14986ef685b7f8bf9b7ad081c60c604bb4649827"
"checksum crossbeam-deque 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7792c4a9b5a4222f654e3728a3dd945aacc24d2c3a1a096ed265d80e4929cb9a"
"checksum crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3486aefc4c0487b9cb52372c97df0a48b8c249514af1ee99703bf70d2f2ceda1"
"checksum crossbeam-epoch 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "30fecfcac6abfef8771151f8be4abc9e4edc112c2bcb233314cafde2680536e9"
"checksum crossbeam-epoch 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2449aaa4ec7ef96e5fb24db16024b935df718e9ae1cec0a1e68feeca2efca7b8"
"checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
"checksum crossbeam-utils 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c55913cc2799171a550e307918c0a360e8c16004820291bf3b638969b4a01816"
"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda"
"checksum csrf 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "38f2ee2a7e76740d81de006e61eff53206c56448a30d8017b4ac97b5486682bd"
"checksum ctrlc 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "630391922b1b893692c6334369ff528dcc3a9d8061ccf4c803aa8f83cb13db5e"
"checksum custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9"
"checksum data-encoding 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67df0571a74bf0d97fb8b2ed22abdd9a48475c96bd327db968b7d9cace99655e"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
@ -3011,11 +3344,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum diesel_migrations 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b42c35d1ce9e8d57a3e7001b4127f2bc1b073a89708bb7019f5be27c991c28"
"checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90"
"checksum dotenv 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d0a1279c96732bc6800ce6337b6a614697b0e74ae058dc03c62ebeb78b4d86"
"checksum downcast 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6c6fe31318b6ef21166c8e839e680238eb16f875849d597544eead7ec882eed3"
"checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd"
"checksum either 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a39bffec1e2015c5d8a6773cb0cf48d0d758c842398f624c34969071f5499ea7"
"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0"
"checksum encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)" = "065f4d0c826fdaef059ac45487169d918558e3cf86c9d89f6e81cf52369126e5"
"checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3"
"checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02"
"checksum fail 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd2e1a22c616c8c8c96b6e07c243014551f3ba77291d24c22e0bfea6830c0b4e"
"checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9"
"checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426"
"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
@ -3025,6 +3361,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
"checksum fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "c4bbbf71584aeed076100b5665ac14e3d85eeb31fdbb45fbd41ef9a682b5ec05"
"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
"checksum fst 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9b0408ab57c1bf7c634b2ac6a165d14f642dc3335a43203090a7f8c78b54577b"
"checksum fst-regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "87aca1d91eed3c128132cee31d291fd4e8492df0b742a5b1453857a4c7cedd88"
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
"checksum futf 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b"
@ -3043,6 +3381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82"
"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
"checksum html5ever 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b04478cf718862650a0bf66acaf8f2f8c906fbc703f35c916c1f4211b069a364"
"checksum htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
"checksum http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "24f58e8c2d8e886055c3ead7b28793e1455270b5fb39650984c224bc538ba581"
"checksum httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83"
"checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e"
@ -3057,6 +3396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0"
"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
"checksum isatty 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e31a8281fc93ec9693494da65fbf28c0c2aa60a2eaec25dc58e2f31952e95edc"
"checksum itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0d47946d458e94a1b7bcabbf6521ea7c037062c81f534615abcad76e84d4970d"
"checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c"
"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
@ -3064,6 +3404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7"
"checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0"
"checksum levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73a004f877f468548d8d0ac4977456a249d8fabbdb8416c36db163dfc8f2e8ca"
"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d"
"checksum libflate 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "21138fc6669f438ed7ae3559d5789a5f0ba32f28c1f0608d1e452b0bb06ee936"
"checksum libsqlite3-sys 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d3711dfd91a1081d2458ad2d06ea30a8755256e74038be2ad927d94e1c955ca8"
@ -3077,6 +3418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum markup5ever 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfedc97d5a503e96816d10fedcd5b42f760b2e525ce2f7ec71f6a41780548475"
"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
"checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b"
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
"checksum migrations_internals 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8cf7c8c4f83fa9f47440c0b4af99973502de55e6e7b875f693bd263e03f93e7e"
"checksum migrations_macros 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79f12499ef7353bdeca2d081bc61edd8351dac09a33af845952009b5a3d68c1a"
@ -3092,6 +3434,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b0a7bd714e83db15676d31caf968ad7318e9cc35f93c85a90231c8f22867549"
"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
"checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4"
"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17"
"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
"checksum notify 4.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "873ecfd8c174964ae30f401329d140142312c8e5590719cf1199d5f1717d8078"
"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea"
@ -3101,7 +3444,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5e2e79eede055813a3ac52fb3915caf8e1c9da2dec1587871aec9f6f7b48508d"
"checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
"checksum openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)" = "409d77eeb492a1aebd6eb322b2ee72ff7c7496b4434d98b3bf8be038755de65e"
"checksum owned-read 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05d57fab18d627fc4dffbd78d4a25a5b5b5211fda724231f001bee4cef1b2d3b"
"checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37"
"checksum owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13"
"checksum parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5"
"checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c"
"checksum pear 0.1.0 (git+http://github.com/SergioBenitez/Pear?rev=b475140)" = "<none>"
@ -3140,21 +3485,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384"
"checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341"
"checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957"
"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7"
"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d"
"checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a"
"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5"
"checksum rental 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ca24bf9b98e3df0bb359f1bbb8ef993a0093d8432500c5eaf3ae724f30b5f754"
"checksum rental-impl 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a269533a9b93bbaa4848260e51b64564cc445d46185979f31974ec703374803a"
"checksum reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1d68c7bf0b1dc3860b80c6d31d05808bf54cdc1bfc70a4680893791becd083ae"
"checksum ring 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe642b9dd1ba0038d78c4a3999d1ee56178b4d415c1e1fbaba83b06dce012f0"
"checksum rocket 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)" = "<none>"
"checksum rocket_codegen 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)" = "<none>"
"checksum rocket_codegen_next 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)" = "<none>"
"checksum rocket_contrib 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)" = "<none>"
"checksum rocket_csrf 0.1.0 (git+https://github.com/fdb-hiroshima/rocket_csrf?rev=2805ce5dbae4a6441208484426440885a5640a6a)" = "<none>"
"checksum rocket_csrf 0.1.0 (git+https://github.com/fdb-hiroshima/rocket_csrf?rev=0dfb822d5cbf65a5eee698099368b7c0f4c61fa4)" = "<none>"
"checksum rocket_http 0.4.0-dev (git+https://github.com/SergioBenitez/Rocket?rev=55459db7732b9a240826a5c120c650f87e3372ce)" = "<none>"
"checksum rocket_i18n 0.1.1 (git+https://github.com/BaptisteGelez/rocket_i18n?rev=75a3bfd7b847324c078a355a7f101f8241a9f59b)" = "<none>"
"checksum rpassword 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d127299b02abda51634f14025aec43ae87a7aa7a95202b6a868ec852607d1451"
"checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a"
"checksum rust-stemmers 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fbf06149ec391025664a5634200ced1afb489f0f3f8a140d515ebc0eb04b4bc0"
"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395"
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
@ -3183,6 +3532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum slug 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373"
"checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013"
"checksum smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "153ffa32fd170e9944f7e0838edf824a754ec4c1fc64746fcc9fe1f8fa602e5d"
"checksum snap 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "95d697d63d44ad8b78b8d235bf85b34022a78af292c8918527c5f0cffdde7f43"
"checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8"
"checksum state 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028"
"checksum string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00caf261d6f90f588f8450b8e1230fa0d5be49ee6140fdfbcb55335aff350970"
@ -3198,6 +3548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
"checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7"
"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5"
"checksum tantivy 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "34fab04422b020c9e6e4b5f4a2eb5d6727ce89d244a9f96434347956c8d9dad6"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "55c1195ef8513f3273d55ff59fe5da6940287a0d7a98331254397f464833675b"
"checksum tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9de21546595a0873061940d994bbbc5c35f024ae4fd61ec5c5b159115684f508"
@ -3260,12 +3611,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum want 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a05d9d966753fa4b5c8db73fcab5eed4549cfe0e1e4e66911e5564a0085c35d1"
"checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3"
"checksum webfinger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edc8f298f29f04bf5b6a85d7d448de4f16b7d45807d0a3ec422efcfbf1960519"
"checksum whatlang 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9d6e6c33992562189a3c9a073525c818e8a8b984771e87e126107be7913b3c2"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum workerpool 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4f49756646617bde19ff95b370cfa5c0f7ead17a90c90d7cb62dc31dfaa8c625"
"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
"checksum yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d60c3b48c9cdec42fb06b3b84b5b087405e1fa1c644a1af3930e4dfafe93de48"

9
Cargo.toml
View File

@ -12,7 +12,9 @@ failure = "0.1"
gettext-rs = "0.4"
guid-create = "0.1"
heck = "0.3.0"
num_cpus = "1.0"
rpassword = "2.0"
scheduled-thread-pool = "0.2.0"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
@ -21,7 +23,6 @@ tera = "0.11"
validator = "0.7"
validator_derive = "0.7"
webfinger = "0.3.1"
workerpool = "1.1"
[[bin]]
name = "plume"
@ -31,6 +32,10 @@ path = "src/main.rs"
features = ["serde"]
version = "0.4"
[dependencies.ctrlc]
features = ["termination"]
version = "3.1.1"
[dependencies.diesel]
features = ["r2d2", "chrono"]
version = "*"
@ -64,7 +69,7 @@ rev = "55459db7732b9a240826a5c120c650f87e3372ce"
[dependencies.rocket_csrf]
git = "https://github.com/fdb-hiroshima/rocket_csrf"
rev = "2805ce5dbae4a6441208484426440885a5640a6a"
rev = "0dfb822d5cbf65a5eee698099368b7c0f4c61fa4"
[dependencies.rocket_i18n]
git = "https://github.com/BaptisteGelez/rocket_i18n"

5
plume-cli/src/main.rs
View File

@ -11,6 +11,7 @@ use plume_models::{DATABASE_URL, Connection as Conn};
mod instance;
mod users;
mod search;
fn main() {
let mut app = App::new("Plume CLI")
@ -18,7 +19,8 @@ fn main() {
.version(env!("CARGO_PKG_VERSION"))
.about("Collection of tools to manage your Plume instance.")
.subcommand(instance::command())
.subcommand(users::command());
.subcommand(users::command())
.subcommand(search::command());
let matches = app.clone().get_matches();
dotenv::dotenv().ok();
@ -27,6 +29,7 @@ fn main() {
match matches.subcommand() {
("instance", Some(args)) => instance::run(args, &conn.expect("Couldn't connect to the database.")),
("users", Some(args)) => users::run(args, &conn.expect("Couldn't connect to the database.")),
("search", Some(args)) => search::run(args, &conn.expect("Couldn't connect to the database.")),
_ => app.print_help().expect("Couldn't print help")
};
}

111
plume-cli/src/search.rs Normal file
View File

@ -0,0 +1,111 @@
use clap::{Arg, ArgMatches, App, SubCommand};
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use std::fs::{read_dir, remove_file};
use std::io::ErrorKind;
use std::path::Path;
use plume_models::{
Connection,
posts::Post,
schema::posts,
search::Searcher,
};
pub fn command<'a, 'b>() -> App<'a, 'b> {
SubCommand::with_name("search")
.about("Manage search index")
.subcommand(SubCommand::with_name("init")
.arg(Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(true)
.help("Path to Plume's working directory"))
.arg(Arg::with_name("force")
.short("f")
.long("force")
.help("Ignore already using directory")
).about("Initialize Plume's internal search engine"))
.subcommand(SubCommand::with_name("refill")
.arg(Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(true)
.help("Path to Plume's working directory")
).about("Regenerate Plume's search index"))
.subcommand(SubCommand::with_name("unlock")
.arg(Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(true)
.help("Path to Plume's working directory")
).about("Release lock on search directory"))
}
pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let conn = conn;
match args.subcommand() {
("init", Some(x)) => init(x, conn),
("refill", Some(x)) => refill(x, conn, None),
("unlock", Some(x)) => unlock(x),
_ => println!("Unknown subcommand"),
}
}
fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let path = args.value_of("path").unwrap();
let force = args.is_present("force");
let path = Path::new(path).join("search_index");
let can_do = match read_dir(path.clone()) { // try to read the directory specified
Ok(mut contents) => {
if contents.next().is_none() {
true
} else {
false
}
},
Err(e) => if e.kind() == ErrorKind::NotFound {
true
} else {
panic!("Error while initialising search index : {}", e);
}
};
if can_do || force {
let searcher = Searcher::create(&path).unwrap();
refill(args, conn, Some(searcher));
} else {
eprintln!("Can't create new index, {} exist and is not empty", path.to_str().unwrap());
}
}
fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searcher>) {
let path = args.value_of("path").unwrap();
let path = Path::new(path).join("search_index");
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
let posts = posts::table
.filter(posts::published.eq(true))
.load::<Post>(conn)
.expect("Post::get_recents: loading error");
let len = posts.len();
for (i,post) in posts.iter().enumerate() {
println!("Importing {}/{} : {}", i+1, len, post.title);
searcher.update_document(conn, &post);
}
println!("Commiting result");
searcher.commit();
}
fn unlock<'a>(args: &ArgMatches<'a>) {
let path = args.value_of("path").unwrap();
let path = Path::new(path).join("search_index/.tantivy-indexer.lock");
remove_file(path).unwrap();
}

3
plume-models/Cargo.toml
View File

@ -10,14 +10,17 @@ bcrypt = "0.2"
canapi = "0.1"
guid-create = "0.1"
heck = "0.3.0"
itertools = "0.7.8"
lazy_static = "*"
openssl = "0.10.11"
reqwest = "0.9"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
tantivy = "0.7.1"
url = "1.7"
webfinger = "0.3.1"
whatlang = "0.5.0"
[dependencies.chrono]
features = ["serde"]

13
plume-models/src/blogs.rs
View File

@ -24,6 +24,7 @@ use plume_common::activity_pub::{
use posts::Post;
use safe_string::SafeString;
use schema::blogs;
use search::Searcher;
use users::User;
use {Connection, BASE_URL, USE_HTTPS};
@ -411,9 +412,9 @@ impl Blog {
json
}
pub fn delete(&self, conn: &Connection) {
pub fn delete(&self, conn: &Connection, searcher: &Searcher) {
for post in Post::get_for_blog(conn, &self) {
post.delete(conn);
post.delete(&(conn, searcher));
}
diesel::delete(self)
.execute(conn)
@ -509,6 +510,7 @@ pub(crate) mod tests {
use instance::tests as instance_tests;
use tests::db;
use users::tests as usersTests;
use search::tests::get_searcher;
use Connection as Conn;
pub(crate) fn fill_database(conn: &Conn) -> Vec<Blog> {
@ -756,7 +758,7 @@ pub(crate) mod tests {
conn.test_transaction::<_, (), _>(|| {
let blogs = fill_database(conn);
blogs[0].delete(conn);
blogs[0].delete(conn, &get_searcher());
assert!(Blog::get(conn, blogs[0].id).is_none());
Ok(())
@ -767,6 +769,7 @@ pub(crate) mod tests {
fn delete_via_user() {
let conn = &db();
conn.test_transaction::<_, (), _>(|| {
let searcher = get_searcher();
let user = usersTests::fill_database(conn);
fill_database(conn);
@ -818,10 +821,10 @@ pub(crate) mod tests {
},
);
user[0].delete(conn);
user[0].delete(conn, &searcher);
assert!(Blog::get(conn, blog[0].id).is_some());
assert!(Blog::get(conn, blog[1].id).is_none());
user[1].delete(conn);
user[1].delete(conn, &searcher);
assert!(Blog::get(conn, blog[0].id).is_none());
Ok(())

33
plume-models/src/lib.rs
View File

@ -10,6 +10,7 @@ extern crate chrono;
extern crate diesel;
extern crate guid_create;
extern crate heck;
extern crate itertools;
#[macro_use]
extern crate lazy_static;
extern crate openssl;
@ -22,8 +23,11 @@ extern crate serde;
extern crate serde_derive;
#[macro_use]
extern crate serde_json;
#[macro_use]
extern crate tantivy;
extern crate url;
extern crate webfinger;
extern crate whatlang;
#[cfg(test)]
#[macro_use]
@ -145,34 +149,6 @@ macro_rules! insert {
};
}
/// Adds a function to a model to save changes to a model.
/// The model should derive diesel::AsChangeset.
///
/// # Usage
///
/// ```rust
/// impl Model {
/// update!(model_table);
/// }
///
/// // Update and save changes
/// let m = Model::get(connection, 1);
/// m.foo = 42;
/// m.update(connection);
/// ```
macro_rules! update {
($table:ident) => {
pub fn update(&self, conn: &crate::Connection) -> Self {
diesel::update(self)
.set(self)
.execute(conn)
.expect(concat!("macro::update: Error updating ", stringify!($table)));
Self::get(conn, self.id)
.expect(concat!("macro::update: ", stringify!($table), " we just updated doesn't exist anymore???"))
}
};
}
/// Returns the last row of a table.
///
/// # Usage
@ -284,6 +260,7 @@ pub mod post_authors;
pub mod posts;
pub mod reshares;
pub mod safe_string;
pub mod search;
pub mod schema;
pub mod tags;
pub mod users;

59
plume-models/src/posts.rs
View File

@ -25,6 +25,7 @@ use plume_common::{
use post_authors::*;
use reshares::Reshare;
use safe_string::SafeString;
use search::Searcher;
use schema::posts;
use std::collections::HashSet;
use tags::Tag;
@ -64,11 +65,11 @@ pub struct NewPost {
pub cover_id: Option<i32>,
}
impl<'a> Provider<(&'a Connection, Option<i32>)> for Post {
impl<'a> Provider<(&'a Connection, &'a Searcher, Option<i32>)> for Post {
type Data = PostEndpoint;
fn get(
(conn, user_id): &(&'a Connection, Option<i32>),
(conn, _search, user_id): &(&'a Connection, &Searcher, Option<i32>),
id: i32,
) -> Result<PostEndpoint, Error> {
if let Some(post) = Post::get(conn, id) {
@ -90,7 +91,7 @@ impl<'a> Provider<(&'a Connection, Option<i32>)> for Post {
}
fn list(
(conn, user_id): &(&'a Connection, Option<i32>),
(conn, _searcher, user_id): &(&'a Connection, &Searcher, Option<i32>),
filter: PostEndpoint,
) -> Vec<PostEndpoint> {
let mut query = posts::table.into_boxed();
@ -123,37 +124,57 @@ impl<'a> Provider<(&'a Connection, Option<i32>)> for Post {
}
fn create(
(_conn, _user_id): &(&'a Connection, Option<i32>),
(_conn, _searcher, _user_id): &(&'a Connection, &Searcher, Option<i32>),
_query: PostEndpoint,
) -> Result<PostEndpoint, Error> {
unimplemented!()
}
fn update(
(_conn, _user_id): &(&'a Connection, Option<i32>),
(_conn, _searcher, _user_id): &(&'a Connection, &Searcher, Option<i32>),
_id: i32,
_new_data: PostEndpoint,
) -> Result<PostEndpoint, Error> {
unimplemented!()
}
fn delete((conn, user_id): &(&'a Connection, Option<i32>), id: i32) {
fn delete((conn, searcher, user_id): &(&'a Connection, &Searcher, Option<i32>), id: i32) {
let user_id = user_id.expect("Post as Provider::delete: not authenticated");
if let Some(post) = Post::get(conn, id) {
if post.is_author(conn, user_id) {
post.delete(conn);
post.delete(&(conn, searcher));
}
}
}
}
impl Post {
insert!(posts, NewPost);
get!(posts);
update!(posts);
find_by!(posts, find_by_slug, slug as &str, blog_id as i32);
find_by!(posts, find_by_ap_url, ap_url as &str);
last!(posts);
pub fn insert(conn: &Connection, new: NewPost, searcher: &Searcher) -> Self {
diesel::insert_into(posts::table)
.values(new)
.execute(conn)
.expect("Post::insert: Error saving in posts");
let post = Self::last(conn);
searcher.add_document(conn, &post);
post
}
pub fn update(&self, conn: &Connection, searcher: &Searcher) -> Self {
diesel::update(self)
.set(self)
.execute(conn)
.expect("Post::update: Error updating posts");
let post = Self::get(conn, self.id)
.expect("macro::update: posts we just updated doesn't exist anymore???");
searcher.update_document(conn, &post);
post
}
pub fn list_by_tag(conn: &Connection, tag: String, (min, max): (i32, i32)) -> Vec<Post> {
use schema::tags;
@ -560,7 +581,7 @@ impl Post {
act
}
pub fn handle_update(conn: &Connection, updated: &Article) {
pub fn handle_update(conn: &Connection, updated: &Article, searcher: &Searcher) {
let id = updated
.object_props
.id_string()
@ -620,7 +641,7 @@ impl Post {
post.update_hashtags(conn, hashtags);
}
post.update(conn);
post.update(conn, searcher);
}
pub fn update_mentions(&self, conn: &Connection, mentions: Vec<link::Mention>) {
@ -765,8 +786,8 @@ impl Post {
}
}
impl FromActivity<Article, Connection> for Post {
fn from_activity(conn: &Connection, article: Article, _actor: Id) -> Post {
impl<'a> FromActivity<Article, (&'a Connection, &'a Searcher)> for Post {
fn from_activity((conn, searcher): &(&'a Connection, &'a Searcher), article: Article, _actor: Id) -> Post {
if let Some(post) = Post::find_by_ap_url(
conn,
&article.object_props.id_string().unwrap_or_default(),
@ -838,6 +859,7 @@ impl FromActivity<Article, Connection> for Post {
.content,
cover_id: cover,
},
searcher,
);
for author in authors {
@ -877,8 +899,8 @@ impl FromActivity<Article, Connection> for Post {
}
}
impl Deletable<Connection, Delete> for Post {
fn delete(&self, conn: &Connection) -> Delete {
impl<'a> Deletable<(&'a Connection, &'a Searcher), Delete> for Post {
fn delete(&self, (conn, searcher): &(&Connection, &Searcher)) -> Delete {
let mut act = Delete::default();
act.delete_props
.set_actor_link(self.get_authors(conn)[0].clone().into_id())
@ -904,12 +926,13 @@ impl Deletable<Connection, Delete> for Post {
m.delete(conn);
}
diesel::delete(self)
.execute(conn)
.execute(*conn)
.expect("Post::delete: DB error");
searcher.delete_document(self);
act
}
fn delete_id(id: &str, actor_id: &str, conn: &Connection) {
fn delete_id(id: &str, actor_id: &str, (conn, searcher): &(&Connection, &Searcher)) {
let actor = User::find_by_ap_url(conn, actor_id);
let post = Post::find_by_ap_url(conn, id);
let can_delete = actor
@ -919,7 +942,7 @@ impl Deletable<Connection, Delete> for Post {
})
.unwrap_or(false);
if can_delete {
post.map(|p| p.delete(conn));
post.map(|p| p.delete(&(conn, searcher)));
}
}
}

167
plume-models/src/search/mod.rs Normal file
View File

@ -0,0 +1,167 @@
mod searcher;
mod query;
mod tokenizer;
pub use self::searcher::*;
pub use self::query::PlumeQuery as Query;
#[cfg(test)]
pub(crate) mod tests {
use super::{Query, Searcher};
use std::env::temp_dir;
use diesel::Connection;
use plume_common::activity_pub::inbox::Deletable;
use plume_common::utils::random_hex;
use blogs::tests::fill_database;
use posts::{NewPost, Post};
use post_authors::*;
use safe_string::SafeString;
use tests::db;
pub(crate) fn get_searcher() -> Searcher {
let dir = temp_dir().join("plume-test");
if dir.exists() {
Searcher::open(&dir)
} else {
Searcher::create(&dir)
}.unwrap()
}
#[test]
fn get_first_token() {
let vector = vec![
("+\"my token\" other", ("+\"my token\"", " other")),
("-\"my token\" other", ("-\"my token\"", " other")),
(" \"my token\" other", ("\"my token\"", " other")),
("\"my token\" other", ("\"my token\"", " other")),
("+my token other", ("+my", " token other")),
("-my token other", ("-my", " token other")),
(" my token other", ("my", " token other")),
("my token other", ("my", " token other")),
("+\"my token other", ("+\"my token other", "")),
("-\"my token other", ("-\"my token other", "")),
(" \"my token other", ("\"my token other", "")),
("\"my token other", ("\"my token other", "")),
];
for (source, res) in vector {
assert_eq!(Query::get_first_token(source), res);
}
}
#[test]
fn from_str() {
let vector = vec![
("", ""),
("a query", "a query"),
("\"a query\"", "\"a query\""),
("+a -\"query\"", "+a -query"),
("title:\"something\" a query", "a query title:something"),
("-title:\"something\" a query", "a query -title:something"),
("author:user@domain", "author:user@domain"),
("-author:@user@domain", "-author:user@domain"),
("before:2017-11-05 before:2018-01-01", "before:2017-11-05"),
("after:2017-11-05 after:2018-01-01", "after:2018-01-01"),
];
for (source, res) in vector {
assert_eq!(&Query::from_str(source).to_string(), res);
assert_eq!(Query::new().parse_query(source).to_string(), res);
}
}
#[test]
fn setters() {
let vector = vec![
("something", "title:something"),
("+something", "+title:something"),
("-something", "-title:something"),
("+\"something\"", "+title:something"),
("+some thing", "+title:\"some thing\""),
];
for (source, res) in vector {
assert_eq!(&Query::new().title(source, None).to_string(), res);
}
let vector = vec![
("something", "author:something"),
("+something", "+author:something"),
("-something", "-author:something"),
("+\"something\"", "+author:something"),
("+@someone@somewhere", "+author:someone@somewhere"),
];
for (source, res) in vector {
assert_eq!(&Query::new().author(source, None).to_string(), res);
}
}
#[test]
fn open() {
{get_searcher()};//make sure $tmp/plume-test-tantivy exist
let dir = temp_dir().join("plume-test");
Searcher::open(&dir).unwrap();
}
#[test]
fn create() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
assert!(Searcher::open(&dir).is_err());
{Searcher::create(&dir).unwrap();}
Searcher::open(&dir).unwrap();//verify it's well created
}
#[test]
fn search() {
let conn = &db();
conn.test_transaction::<_, (), _>(|| {
let searcher = get_searcher();
let blog = &fill_database(conn)[0];
let author = &blog.list_authors(conn)[0];
let title = random_hex()[..8].to_owned();
let mut post = Post::insert(conn, NewPost {
blog_id: blog.id,
slug: title.clone(),
title: title.clone(),
content: SafeString::new(""),
published: true,
license: "CC-BY-SA".to_owned(),
ap_url: "".to_owned(),
creation_date: None,
subtitle: "".to_owned(),
source: "".to_owned(),
cover_id: None,
}, &searcher);
PostAuthor::insert(conn, NewPostAuthor {
post_id: post.id,
author_id: author.id,
});
searcher.commit();
assert_eq!(searcher.search_document(conn, Query::from_str(&title), (0,1))[0].id, post.id);
let newtitle = random_hex()[..8].to_owned();
post.title = newtitle.clone();
post.update(conn, &searcher);
searcher.commit();
assert_eq!(searcher.search_document(conn, Query::from_str(&newtitle), (0,1))[0].id, post.id);
assert!(searcher.search_document(conn, Query::from_str(&title), (0,1)).is_empty());
post.delete(&(conn, &searcher));
searcher.commit();
assert!(searcher.search_document(conn, Query::from_str(&newtitle), (0,1)).is_empty());
Ok(())
});
}
#[test]
fn drop_writer() {
let searcher = get_searcher();
searcher.drop_writer();
get_searcher();
}
}

343
plume-models/src/search/query.rs Normal file
View File

@ -0,0 +1,343 @@
use chrono::{Datelike, naive::NaiveDate, offset::Utc};
use tantivy::{query::*, schema::*, Term};
use std::{cmp,ops::Bound};
use search::searcher::Searcher;
//Generate functions for advanced search
macro_rules! gen_func {
( $($field:ident),*; strip: $($strip:ident),* ) => {
$( //most fields go here, it's kinda the "default" way
pub fn $field(&mut self, mut val: &str, occur: Option<Occur>) -> &mut Self {
if !val.trim_matches(&[' ', '"', '+', '-'][..]).is_empty() {
let occur = if let Some(occur) = occur {
occur
} else {
if val.get(0..1).map(|v| v=="+").unwrap_or(false) {
val = &val[1..];
Occur::Must
} else if val.get(0..1).map(|v| v=="-").unwrap_or(false) {
val = &val[1..];
Occur::MustNot
} else {
Occur::Should
}
};
self.$field.push((occur, val.trim_matches(&[' ', '"'][..]).to_owned()));
}
self
}
)*
$( // blog and author go here, leading @ get dismissed
pub fn $strip(&mut self, mut val: &str, occur: Option<Occur>) -> &mut Self {
if !val.trim_matches(&[' ', '"', '+', '-'][..]).is_empty() {
let occur = if let Some(occur) = occur {
occur
} else {
if val.get(0..1).map(|v| v=="+").unwrap_or(false) {
val = &val[1..];
Occur::Must
} else if val.get(0..1).map(|v| v=="-").unwrap_or(false) {
val = &val[1..];
Occur::MustNot
} else {
Occur::Should
}
};
self.$strip.push((occur, val.trim_matches(&[' ', '"', '@'][..]).to_owned()));
}
self
}
)*
}
}
//generate the parser for advanced query from string
macro_rules! gen_parser {
( $self:ident, $query:ident, $occur:ident; normal: $($field:ident),*; date: $($date:ident),*) => {
$( // most fields go here
if $query.starts_with(concat!(stringify!($field), ':')) {
let new_query = &$query[concat!(stringify!($field), ':').len()..];
let (token, rest) = Self::get_first_token(new_query);
$query = rest;
$self.$field(token, Some($occur));
} else
)*
$( // dates (before/after) got here
if $query.starts_with(concat!(stringify!($date), ':')) {
let new_query = &$query[concat!(stringify!($date), ':').len()..];
let (token, rest) = Self::get_first_token(new_query);
$query = rest;
if let Ok(token) = NaiveDate::parse_from_str(token, "%Y-%m-%d") {
$self.$date(&token);
}
} else
)* // fields without 'fieldname:' prefix are considered bare words, and will be searched in title, subtitle and content
{
let (token, rest) = Self::get_first_token($query);
$query = rest;
$self.text(token, Some($occur));
}
}
}
// generate the to_string, giving back a textual query from a PlumeQuery
macro_rules! gen_to_string {
( $self:ident, $result:ident; normal: $($field:ident),*; date: $($date:ident),*) => {
$(
for (occur, val) in &$self.$field {
if val.contains(' ') {
$result.push_str(&format!("{}{}:\"{}\" ", Self::occur_to_str(&occur), stringify!($field), val));
} else {
$result.push_str(&format!("{}{}:{} ", Self::occur_to_str(&occur), stringify!($field), val));
}
}
)*
$(
for val in &$self.$date {
$result.push_str(&format!("{}:{} ", stringify!($date), NaiveDate::from_num_days_from_ce(*val as i32).format("%Y-%m-%d")));
}
)*
}
}
// convert PlumeQuery to Tantivy's Query
macro_rules! gen_to_query {
( $self:ident, $result:ident; normal: $($normal:ident),*; oneoff: $($oneoff:ident),*) => {
$( // classic fields
for (occur, token) in $self.$normal {
$result.push((occur, Self::token_to_query(&token, stringify!($normal))));
}
)*
$( // fields where having more than on Must make no sense in general, so it's considered a Must be one of these instead.
// Those fields are instance, author, blog, lang and license
let mut subresult = Vec::new();
for (occur, token) in $self.$oneoff {
match occur {
Occur::Must => subresult.push((Occur::Should, Self::token_to_query(&token, stringify!($oneoff)))),
occur => $result.push((occur, Self::token_to_query(&token, stringify!($oneoff)))),
}
}
if !subresult.is_empty() {
$result.push((Occur::Must, Box::new(BooleanQuery::from(subresult))));
}
)*
}
}
#[derive(Default)]
pub struct PlumeQuery {
text: Vec<(Occur, String)>,
title: Vec<(Occur, String)>,
subtitle: Vec<(Occur, String)>,
content: Vec<(Occur, String)>,
tag: Vec<(Occur, String)>,
instance: Vec<(Occur, String)>,
author: Vec<(Occur, String)>,
blog: Vec<(Occur, String)>,
lang: Vec<(Occur, String)>,
license: Vec<(Occur, String)>,
before: Option<i64>,
after: Option<i64>,
}
impl PlumeQuery {
/// Create a new empty Query
pub fn new() -> Self {
Default::default()
}
/// Create a new Query from &str
/// Same as doing
/// ```rust
/// # extern crate plume_models;
/// # use plume_models::search::Query;
/// let mut q = Query::new();
/// q.parse_query("some query");
/// ```
pub fn from_str(query: &str) -> Self {
let mut res: Self = Default::default();
res.from_str_req(&query.trim());
res
}
/// Parse a query string into this Query
pub fn parse_query(&mut self, query: &str) -> &mut Self {
self.from_str_req(&query.trim())
}
/// Convert this Query to a Tantivy Query
pub fn into_query(self) -> BooleanQuery {
let mut result: Vec<(Occur, Box<Query>)> = Vec::new();
gen_to_query!(self, result; normal: title, subtitle, content, tag;
oneoff: instance, author, blog, lang, license);
for (occur, token) in self.text { // text entries need to be added as multiple Terms
match occur {
Occur::Must => { // a Must mean this must be in one of title subtitle or content, not in all 3
let subresult = vec![
(Occur::Should, Self::token_to_query(&token, "title")),
(Occur::Should, Self::token_to_query(&token, "subtitle")),
(Occur::Should, Self::token_to_query(&token, "content")),
];
result.push((Occur::Must, Box::new(BooleanQuery::from(subresult))));
},
occur => {
result.push((occur, Self::token_to_query(&token, "title")));
result.push((occur, Self::token_to_query(&token, "subtitle")));
result.push((occur, Self::token_to_query(&token, "content")));
},
}
}
if self.before.is_some() || self.after.is_some() { // if at least one range bound is provided
let after = self.after.unwrap_or_else(|| i64::from(NaiveDate::from_ymd(2000, 1, 1).num_days_from_ce()));
let before = self.before.unwrap_or_else(|| i64::from(Utc::today().num_days_from_ce()));
let field = Searcher::schema().get_field("creation_date").unwrap();
let range = RangeQuery::new_i64_bounds(field, Bound::Included(after), Bound::Included(before));
result.push((Occur::Must, Box::new(range)));
}
result.into()
}
//generate most setters functions
gen_func!(text, title, subtitle, content, tag, instance, lang, license; strip: author, blog);
// documents newer than the provided date will be ignored
pub fn before<D: Datelike>(&mut self, date: &D) -> &mut Self {
let before = self.before.unwrap_or_else(|| i64::from(Utc::today().num_days_from_ce()));
self.before = Some(cmp::min(before, i64::from(date.num_days_from_ce())));
self
}
// documents older than the provided date will be ignored
pub fn after<D: Datelike>(&mut self, date: &D) -> &mut Self {
let after = self.after.unwrap_or_else(|| i64::from(NaiveDate::from_ymd(2000, 1, 1).num_days_from_ce()));
self.after = Some(cmp::max(after, i64::from(date.num_days_from_ce())));
self
}
// split a string into a token and a rest
pub fn get_first_token<'a>(mut query: &'a str) -> (&'a str, &'a str) {
query = query.trim();
if query.is_empty() {
("", "")
} else {
if query.get(0..1).map(|v| v=="\"").unwrap_or(false) {
if let Some(index) = query[1..].find('"') {
query.split_at(index+2)
} else {
(query, "")
}
} else if query.get(0..2).map(|v| v=="+\"" || v=="-\"").unwrap_or(false) {
if let Some(index) = query[2..].find('"') {
query.split_at(index+3)
} else {
(query, "")
}
} else {
if let Some(index) = query.find(' ') {
query.split_at(index)
} else {
(query, "")
}
}
}
}
// map each Occur state to a prefix
fn occur_to_str(occur: &Occur) -> &'static str {
match occur {
Occur::Should => "",
Occur::Must => "+",
Occur::MustNot => "-",
}
}
// recursive parser for query string
fn from_str_req(&mut self, mut query: &str) -> &mut Self {
query = query.trim_left();
if query.is_empty() {
self
} else {
let occur = if query.get(0..1).map(|v| v=="+").unwrap_or(false) {
query = &query[1..];
Occur::Must
} else if query.get(0..1).map(|v| v=="-").unwrap_or(false) {
query = &query[1..];
Occur::MustNot
} else {
Occur::Should
};
gen_parser!(self, query, occur; normal: title, subtitle, content, tag,
instance, author, blog, lang, license;
date: after, before);
self.from_str_req(query)
}
}
// map a token and it's field to a query
fn token_to_query(token: &str, field_name: &str) -> Box<Query> {
let token = token.to_lowercase();
let token = token.as_str();
let field = Searcher::schema().get_field(field_name).unwrap();
if token.contains('@') && (field_name=="author" || field_name=="blog") {
let pos = token.find('@').unwrap();
let user_term = Term::from_field_text(field, &token[..pos]);
let instance_term = Term::from_field_text(Searcher::schema().get_field("instance").unwrap(), &token[pos+1..]);
Box::new(BooleanQuery::from(vec![
(Occur::Must, Box::new(TermQuery::new(user_term, if field_name=="author" { IndexRecordOption::Basic }
else { IndexRecordOption::WithFreqsAndPositions }
)) as Box<dyn Query + 'static>),
(Occur::Must, Box::new(TermQuery::new(instance_term, IndexRecordOption::Basic))),
]))
} else if token.contains(' ') { // phrase query
match field_name {
"instance" | "author" | "tag" => // phrase query are not available on these fields, treat it as multiple Term queries
Box::new(BooleanQuery::from(token.split_whitespace()
.map(|token| {
let term = Term::from_field_text(field, token);
(Occur::Should, Box::new(TermQuery::new(term, IndexRecordOption::Basic))
as Box<dyn Query + 'static>)
})
.collect::<Vec<_>>())),
_ => Box::new(PhraseQuery::new(token.split_whitespace()
.map(|token| Term::from_field_text(field, token))
.collect()))
}
} else { // Term Query
let term = Term::from_field_text(field, token);
let index_option = match field_name {
"instance" | "author" | "tag" => IndexRecordOption::Basic,
_ => IndexRecordOption::WithFreqsAndPositions,
};
Box::new(TermQuery::new(term, index_option))
}
}
}
impl ToString for PlumeQuery {
fn to_string(&self) -> String {
let mut result = String::new();
for (occur, val) in &self.text {
if val.contains(' ') {
result.push_str(&format!("{}\"{}\" ", Self::occur_to_str(&occur), val));
} else {
result.push_str(&format!("{}{} ", Self::occur_to_str(&occur), val));
}
}
gen_to_string!(self, result; normal: title, subtitle, content, tag,
instance, author, blog, lang, license;
date: before, after);
result.pop();// remove trailing ' '
result
}
}

203
plume-models/src/search/searcher.rs Normal file
View File

@ -0,0 +1,203 @@
use instance::Instance;
use posts::Post;
use tags::Tag;
use Connection;
use chrono::Datelike;
use itertools::Itertools;
use tantivy::{
collector::TopCollector, directory::MmapDirectory,
schema::*, tokenizer::*, Index, IndexWriter, Term
};
use whatlang::{detect as detect_lang, Lang};
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use search::query::PlumeQuery;
use super::tokenizer;
#[derive(Debug)]
pub enum SearcherError{
IndexCreationError,
WriteLockAcquisitionError,
IndexOpeningError,
IndexEditionError,
}
pub struct Searcher {
index: Index,
writer: Mutex<Option<IndexWriter>>,
}
impl Searcher {
pub fn schema() -> Schema {
let tag_indexing = TextOptions::default()
.set_indexing_options(TextFieldIndexing::default()
.set_tokenizer("whitespace_tokenizer")
.set_index_option(IndexRecordOption::Basic));
let content_indexing = TextOptions::default()
.set_indexing_options(TextFieldIndexing::default()
.set_tokenizer("content_tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions));
let property_indexing = TextOptions::default()
.set_indexing_options(TextFieldIndexing::default()
.set_tokenizer("property_tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions));
let mut schema_builder = SchemaBuilder::default();
schema_builder.add_i64_field("post_id", INT_STORED | INT_INDEXED);
schema_builder.add_i64_field("creation_date", INT_INDEXED);
schema_builder.add_text_field("instance", tag_indexing.clone());
schema_builder.add_text_field("author", tag_indexing.clone());//todo move to a user_indexing with user_tokenizer function
schema_builder.add_text_field("tag", tag_indexing);
schema_builder.add_text_field("blog", content_indexing.clone());
schema_builder.add_text_field("content", content_indexing.clone());
schema_builder.add_text_field("subtitle", content_indexing.clone());
schema_builder.add_text_field("title", content_indexing);
schema_builder.add_text_field("lang", property_indexing.clone());
schema_builder.add_text_field("license", property_indexing);
schema_builder.build()
}
pub fn create(path: &AsRef<Path>) -> Result<Self,SearcherError> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer
.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false)
.filter(LowerCaser);
let schema = Self::schema();
create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
let index = Index::create(MmapDirectory::open(path).map_err(|_| SearcherError::IndexCreationError)?, schema).map_err(|_| SearcherError::IndexCreationError)?;
{
let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer);
}//to please the borrow checker
Ok(Self {
writer: Mutex::new(Some(index.writer(50_000_000).map_err(|_| SearcherError::WriteLockAcquisitionError)?)),
index
})
}
pub fn open(path: &AsRef<Path>) -> Result<Self, SearcherError> {
let whitespace_tokenizer = tokenizer::WhitespaceTokenizer
.filter(LowerCaser);
let content_tokenizer = SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser);
let property_tokenizer = NgramTokenizer::new(2, 8, false)
.filter(LowerCaser);
let index = Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?).map_err(|_| SearcherError::IndexOpeningError)?;
{
let tokenizer_manager = index.tokenizers();
tokenizer_manager.register("whitespace_tokenizer", whitespace_tokenizer);
tokenizer_manager.register("content_tokenizer", content_tokenizer);
tokenizer_manager.register("property_tokenizer", property_tokenizer);
}//to please the borrow checker
let mut writer = index.writer(50_000_000).map_err(|_| SearcherError::WriteLockAcquisitionError)?;
writer.garbage_collect_files().map_err(|_| SearcherError::IndexEditionError)?;
Ok(Self {
writer: Mutex::new(Some(writer)),
index,
})
}
pub fn add_document(&self, conn: &Connection, post: &Post) {
let schema = self.index.schema();
let post_id = schema.get_field("post_id").unwrap();
let creation_date = schema.get_field("creation_date").unwrap();
let instance = schema.get_field("instance").unwrap();
let author = schema.get_field("author").unwrap();
let tag = schema.get_field("tag").unwrap();
let blog_name = schema.get_field("blog").unwrap();
let content = schema.get_field("content").unwrap();
let subtitle = schema.get_field("subtitle").unwrap();
let title = schema.get_field("title").unwrap();
let lang = schema.get_field("lang").unwrap();
let license = schema.get_field("license").unwrap();
let mut writer = self.writer.lock().unwrap();
let writer = writer.as_mut().unwrap();
writer.add_document(doc!(
post_id => i64::from(post.id),
author => post.get_authors(conn).into_iter().map(|u| u.get_fqn(conn)).join(" "),
creation_date => i64::from(post.creation_date.num_days_from_ce()),
instance => Instance::get(conn, post.get_blog(conn).instance_id).unwrap().public_domain.clone(),
tag => Tag::for_post(conn, post.id).into_iter().map(|t| t.tag).join(" "),
blog_name => post.get_blog(conn).title,
content => post.content.get().clone(),
subtitle => post.subtitle.clone(),
title => post.title.clone(),
lang => detect_lang(post.content.get()).and_then(|i| if i.is_reliable() { Some(i.lang()) } else {None} ).unwrap_or(Lang::Eng).name(),
license => post.license.clone(),
));
}
pub fn delete_document(&self, post: &Post) {
let schema = self.index.schema();
let post_id = schema.get_field("post_id").unwrap();
let doc_id = Term::from_field_i64(post_id, i64::from(post.id));
let mut writer = self.writer.lock().unwrap();
let writer = writer.as_mut().unwrap();
writer.delete_term(doc_id);
}
pub fn update_document(&self, conn: &Connection, post: &Post) {
self.delete_document(post);
self.add_document(conn, post);
}
pub fn search_document(&self, conn: &Connection, query: PlumeQuery, (min, max): (i32, i32)) -> Vec<Post>{
let schema = self.index.schema();
let post_id = schema.get_field("post_id").unwrap();
let mut collector = TopCollector::with_limit(cmp::max(1,max) as usize);
let searcher = self.index.searcher();
searcher.search(&query.into_query(), &mut collector).unwrap();
collector.docs().get(min as usize..).unwrap_or(&[])
.into_iter()
.filter_map(|doc_add| {
let doc = searcher.doc(*doc_add).ok()?;
let id = doc.get_first(post_id)?;
Post::get(conn, id.i64_value() as i32)
//borrow checker don't want me to use filter_map or and_then here
})
.collect()
}
pub fn commit(&self) {
let mut writer = self.writer.lock().unwrap();
writer.as_mut().unwrap().commit().unwrap();
self.index.load_searchers().unwrap();
}
pub fn drop_writer(&self) {
self.writer.lock().unwrap().take();
}
}

67
plume-models/src/search/tokenizer.rs Normal file
View File

@ -0,0 +1,67 @@
use std::str::CharIndices;
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
/// Tokenize the text by splitting on whitespaces. Pretty much a copy of Tantivy's SimpleTokenizer,
/// but not splitting on punctuation
#[derive(Clone)]
pub struct WhitespaceTokenizer;
pub struct WhitespaceTokenStream<'a> {
text: &'a str,
chars: CharIndices<'a>,
token: Token,
}
impl<'a> Tokenizer<'a> for WhitespaceTokenizer {
type TokenStreamImpl = WhitespaceTokenStream<'a>;
fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
WhitespaceTokenStream {
text,
chars: text.char_indices(),
token: Token::default(),
}
}
}
impl<'a> WhitespaceTokenStream<'a> {
// search for the end of the current token.
fn search_token_end(&mut self) -> usize {
(&mut self.chars)
.filter(|&(_, ref c)| c.is_whitespace())
.map(|(offset, _)| offset)
.next()
.unwrap_or_else(|| self.text.len())
}
}
impl<'a> TokenStream for WhitespaceTokenStream<'a> {
fn advance(&mut self) -> bool {
self.token.text.clear();
self.token.position = self.token.position.wrapping_add(1);
loop {
match self.chars.next() {
Some((offset_from, c)) => {
if !c.is_whitespace() {
let offset_to = self.search_token_end();
self.token.offset_from = offset_from;
self.token.offset_to = offset_to;
self.token.text.push_str(&self.text[offset_from..offset_to]);
return true;
}
}
None => {
return false;
}
}
}
}
fn token(&self) -> &Token {
&self.token
}
fn token_mut(&mut self) -> &mut Token {
&mut self.token
}
}

10
plume-models/src/users.rs
View File

@ -41,6 +41,7 @@ use posts::Post;
use reshares::Reshare;
use safe_string::SafeString;
use schema::users;
use search::Searcher;
use {ap_url, Connection, BASE_URL, USE_HTTPS};
pub type CustomPerson = CustomObject<ApSignature, Person>;
@ -104,13 +105,13 @@ impl User {
.expect("User::one_by_instance: loading error")
}
pub fn delete(&self, conn: &Connection) {
pub fn delete(&self, conn: &Connection, searcher: &Searcher) {
use schema::post_authors;
Blog::find_for_author(conn, self)
.iter()
.filter(|b| b.list_authors(conn).len() <= 1)
.for_each(|b| b.delete(conn));
.for_each(|b| b.delete(conn, searcher));
// delete the posts if they is the only author
let all_their_posts_ids: Vec<i32> = post_authors::table
.filter(post_authors::author_id.eq(self.id))
@ -129,7 +130,7 @@ impl User {
if !has_other_authors {
Post::get(conn, post_id)
.expect("User::delete: post not found error")
.delete(conn);
.delete(&(conn, searcher));
}
}
@ -981,6 +982,7 @@ pub(crate) mod tests {
use super::*;
use diesel::Connection;
use instance::{tests as instance_tests, Instance};
use search::tests::get_searcher;
use tests::db;
use Connection as Conn;
@ -1077,7 +1079,7 @@ pub(crate) mod tests {
let inserted = fill_database(conn);
assert!(User::get(conn, inserted[0].id).is_some());
inserted[0].delete(conn);
inserted[0].delete(conn, &get_searcher());
assert!(User::get(conn, inserted[0].id).is_none());
Ok(())

10
src/api/posts.rs
View File

@ -9,18 +9,20 @@ use plume_models::{
Connection,
db_conn::DbConn,
posts::Post,
search::Searcher as UnmanagedSearcher,
};
use api::authorization::*;
use Searcher;
#[get("/posts/<id>")]
fn get(id: i32, conn: DbConn, auth: Option<Authorization<Read, Post>>) -> Json<serde_json::Value> {
let post = <Post as Provider<(&Connection, Option<i32>)>>::get(&(&*conn, auth.map(|a| a.0.user_id)), id).ok();
fn get(id: i32, conn: DbConn, auth: Option<Authorization<Read, Post>>, search: Searcher) -> Json<serde_json::Value> {
let post = <Post as Provider<(&Connection, &UnmanagedSearcher, Option<i32>)>>::get(&(&*conn, &search, auth.map(|a| a.0.user_id)), id).ok();
Json(json!(post))
}
#[get("/posts")]
fn list(conn: DbConn, uri: &Origin, auth: Option<Authorization<Read, Post>>) -> Json<serde_json::Value> {
fn list(conn: DbConn, uri: &Origin, auth: Option<Authorization<Read, Post>>, search: Searcher) -> Json<serde_json::Value> {
let query: PostEndpoint = serde_qs::from_str(uri.query().unwrap_or("")).expect("api::list: invalid query error");
let post = <Post as Provider<(&Connection, Option<i32>)>>::list(&(&*conn, auth.map(|a| a.0.user_id)), query);
let post = <Post as Provider<(&Connection, &UnmanagedSearcher, Option<i32>)>>::list(&(&*conn, &search, auth.map(|a| a.0.user_id)), query);
Json(json!(post))
}

10
src/inbox.rs
View File

@ -19,11 +19,11 @@ use plume_common::activity_pub::{
};
use plume_models::{
comments::Comment, follows::Follow, instance::Instance, likes, posts::Post, reshares::Reshare,
users::User, Connection,
users::User, search::Searcher, Connection,
};
pub trait Inbox {
fn received(&self, conn: &Connection, act: serde_json::Value) -> Result<(), Error> {
fn received(&self, conn: &Connection, searcher: &Searcher, act: serde_json::Value) -> Result<(), Error> {
let actor_id = Id::new(act["actor"].as_str().unwrap_or_else(|| {
act["actor"]["id"]
.as_str()
@ -37,7 +37,7 @@ pub trait Inbox {
}
"Create" => {
let act: Create = serde_json::from_value(act.clone())?;
if Post::try_from_activity(conn, act.clone())
if Post::try_from_activity(&(conn, searcher), act.clone())
|| Comment::try_from_activity(conn, act)
{
Ok(())
@ -53,7 +53,7 @@ pub trait Inbox {
.object_props
.id_string()?,
actor_id.as_ref(),
conn,
&(conn, searcher),
);
Ok(())
}
@ -113,7 +113,7 @@ pub trait Inbox {
}
"Update" => {
let act: Update = serde_json::from_value(act.clone())?;
Post::handle_update(conn, &act.update_props.object_object()?);
Post::handle_update(conn, &act.update_props.object_object()?, searcher);
Ok(())
}
_ => Err(InboxError::InvalidType)?,

39
src/main.rs
View File

@ -6,6 +6,7 @@ extern crate atom_syndication;
extern crate canapi;
extern crate chrono;
extern crate colored;
extern crate ctrlc;
extern crate diesel;
extern crate dotenv;
extern crate failure;
@ -13,6 +14,7 @@ extern crate gettextrs;
extern crate guid_create;
extern crate heck;
extern crate multipart;
extern crate num_cpus;
extern crate plume_api;
extern crate plume_common;
extern crate plume_models;
@ -22,6 +24,7 @@ extern crate rocket_contrib;
extern crate rocket_csrf;
extern crate rocket_i18n;
extern crate rpassword;
extern crate scheduled_thread_pool;
extern crate serde;
#[macro_use]
extern crate serde_derive;
@ -32,20 +35,24 @@ extern crate validator;
#[macro_use]
extern crate validator_derive;
extern crate webfinger;
extern crate workerpool;
use diesel::r2d2::ConnectionManager;
use rocket::State;
use rocket_contrib::Template;
use rocket_csrf::CsrfFairingBuilder;
use plume_models::{DATABASE_URL, Connection, db_conn::DbPool};
use workerpool::{Pool, thunk::ThunkWorker};
use plume_models::{DATABASE_URL, Connection,
db_conn::DbPool, search::Searcher as UnmanagedSearcher};
use scheduled_thread_pool::ScheduledThreadPool;
use std::process::exit;
use std::sync::Arc;
use std::time::Duration;
mod api;
mod inbox;
mod routes;
type Worker<'a> = State<'a, Pool<ThunkWorker<()>>>;
type Worker<'a> = State<'a, ScheduledThreadPool>;
type Searcher<'a> = State<'a, Arc<UnmanagedSearcher>>;
/// Initializes a database pool.
fn init_pool() -> Option<DbPool> {
@ -56,7 +63,21 @@ fn init_pool() -> Option<DbPool> {
}
fn main() {
let pool = init_pool().expect("main: database pool initialization error");
let dbpool = init_pool().expect("main: database pool initialization error");
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
let searcher = Arc::new(UnmanagedSearcher::open(&"search_index").unwrap());
let commiter = searcher.clone();
workpool.execute_with_fixed_delay(Duration::from_secs(5), Duration::from_secs(60*30), move || commiter.commit());
let search_unlocker = searcher.clone();
ctrlc::set_handler(move || {
search_unlocker.drop_writer();
exit(0);
}).expect("Error setting Ctrl-c handler");
rocket::ignite()
.mount("/", routes![
routes::blogs::paginated_details,
@ -119,6 +140,9 @@ fn main() {
routes::reshares::create,
routes::reshares::create_auth,
routes::search::index,
routes::search::query,
routes::session::new,
routes::session::new_message,
routes::session::create,
@ -167,8 +191,9 @@ fn main() {
routes::errors::not_found,
routes::errors::server_error
])
.manage(pool)
.manage(Pool::<ThunkWorker<()>>::new(4))
.manage(dbpool)
.manage(workpool)
.manage(searcher)
.attach(Template::custom(|engines| {
rocket_i18n::tera(&mut engines.tera);
}))

5
src/routes/blogs.rs
View File

@ -21,6 +21,7 @@ use plume_models::{
users::User
};
use routes::Page;
use Searcher;
#[get("/~/<name>?<page>", rank = 2)]
fn paginated_details(name: String, conn: DbConn, user: Option<User>, page: Page) -> Template {
@ -130,10 +131,10 @@ fn create(conn: DbConn, data: LenientForm<NewBlogForm>, user: User) -> Result<Re
}
#[post("/~/<name>/delete")]
fn delete(conn: DbConn, name: String, user: Option<User>) -> Result<Redirect, Option<Template>>{
fn delete(conn: DbConn, name: String, user: Option<User>, searcher: Searcher) -> Result<Redirect, Option<Template>>{
let blog = Blog::find_local(&*conn, &name).ok_or(None)?;
if user.map(|u| u.is_author_in(&*conn, &blog)).unwrap_or(false) {
blog.delete(&conn);
blog.delete(&conn, &searcher);
Ok(Redirect::to(uri!(super::instance::index)))
} else {
Err(Some(Template::render("errors/403", json!({// TODO actually return 403 error code

7
src/routes/comments.rs
View File

@ -1,13 +1,11 @@
use activitypub::object::Note;
use rocket::{
State,
request::LenientForm,
response::Redirect
};
use rocket_contrib::Template;
use serde_json;
use validator::Validate;
use workerpool::{Pool, thunk::*};
use plume_common::{utils, activity_pub::{broadcast, ApRequest, ActivityStream}};
use plume_models::{
@ -19,6 +17,7 @@ use plume_models::{
safe_string::SafeString,
users::User
};
use Worker;
#[derive(FromForm, Debug, Validate, Serialize)]
struct NewCommentForm {
@ -29,7 +28,7 @@ struct NewCommentForm {
}
#[post("/~/<blog_name>/<slug>/comment", data = "<data>")]
fn create(blog_name: String, slug: String, data: LenientForm<NewCommentForm>, user: User, conn: DbConn, worker: State<Pool<ThunkWorker<()>>>)
fn create(blog_name: String, slug: String, data: LenientForm<NewCommentForm>, user: User, conn: DbConn, worker: Worker)
-> Result<Redirect, Option<Template>> {
let blog = Blog::find_by_fqn(&*conn, &blog_name).ok_or(None)?;
let post = Post::find_by_slug(&*conn, &slug, blog.id).ok_or(None)?;
@ -56,7 +55,7 @@ fn create(blog_name: String, slug: String, data: LenientForm<NewCommentForm>, us
// federate
let dest = User::one_by_instance(&*conn);
let user_clone = user.clone();
worker.execute(Thunk::of(move || broadcast(&user_clone, new_comment, dest)));
worker.execute(move || broadcast(&user_clone, new_comment, dest));
Redirect::to(uri!(super::posts::details: blog = blog_name, slug = slug))
})

9
src/routes/instance.rs
View File

@ -18,6 +18,7 @@ use plume_models::{
};
use inbox::Inbox;
use routes::Page;
use Searcher;
#[get("/")]
fn index(conn: DbConn, user: Option<User>) -> Template {
@ -190,15 +191,15 @@ fn admin_users_paginated(admin: Admin, conn: DbConn, page: Page) -> Template {
}
#[post("/admin/users/<id>/ban")]
fn ban(_admin: Admin, conn: DbConn, id: i32) -> Redirect {
fn ban(_admin: Admin, conn: DbConn, id: i32, searcher: Searcher) -> Redirect {
if let Some(u) = User::get(&*conn, id) {
u.delete(&*conn);
u.delete(&*conn, &searcher);
}
Redirect::to(uri!(admin_users))
}
#[post("/inbox", data = "<data>")]
fn shared_inbox(conn: DbConn, data: String, headers: Headers) -> Result<String, status::BadRequest<&'static str>> {
fn shared_inbox(conn: DbConn, data: String, headers: Headers, searcher: Searcher) -> Result<String, status::BadRequest<&'static str>> {
let act: serde_json::Value = serde_json::from_str(&data[..]).expect("instance::shared_inbox: deserialization error");
let activity = act.clone();
@ -216,7 +217,7 @@ fn shared_inbox(conn: DbConn, data: String, headers: Headers) -> Result<String,
return Ok(String::new());
}
let instance = Instance::get_local(&*conn).expect("instance::shared_inbox: local instance not found error");
Ok(match instance.received(&*conn, act) {
Ok(match instance.received(&*conn, &searcher, act) {
Ok(_) => String::new(),
Err(e) => {
println!("Shared inbox error: {}\n{}", e.as_fail(), e.backtrace());

10
src/routes/likes.rs
View File

@ -1,5 +1,4 @@
use rocket::{State, response::{Redirect, Flash}};
use workerpool::{Pool, thunk::*};
use rocket::{response::{Redirect, Flash}};
use plume_common::activity_pub::{broadcast, inbox::{Notify, Deletable}};
use plume_common::utils;
@ -10,9 +9,10 @@ use plume_models::{
posts::Post,
users::User
};
use Worker;
#[post("/~/<blog>/<slug>/like")]
fn create(blog: String, slug: String, user: User, conn: DbConn, worker: State<Pool<ThunkWorker<()>>>) -> Option<Redirect> {
fn create(blog: String, slug: String, user: User, conn: DbConn, worker: Worker) -> Option<Redirect> {
let b = Blog::find_by_fqn(&*conn, &blog)?;
let post = Post::find_by_slug(&*conn, &slug, b.id)?;
@ -27,12 +27,12 @@ fn create(blog: String, slug: String, user: User, conn: DbConn, worker: State<Po
let dest = User::one_by_instance(&*conn);
let act = like.to_activity(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, act, dest)));
worker.execute(move || broadcast(&user, act, dest));
} else {
let like = likes::Like::find_by_user_on_post(&*conn, user.id, post.id).expect("likes::create: like exist but not found error");
let delete_act = like.delete(&*conn);
let dest = User::one_by_instance(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, delete_act, dest)));
worker.execute(move || broadcast(&user, delete_act, dest));
}
Some(Redirect::to(uri!(super::posts::details: blog = blog, slug = slug)))

16
src/routes/mod.rs
View File

@ -1,6 +1,8 @@
use atom_syndication::{ContentBuilder, Entry, EntryBuilder, LinkBuilder, Person, PersonBuilder};
use rocket::{
http::uri::{FromUriParam, UriDisplay},
http::{RawStr,
uri::{FromUriParam, UriDisplay}},
request::FromFormValue,
response::NamedFile
};
use std::{
@ -57,6 +59,17 @@ impl FromUriParam<i32> for Page {
}
}
impl<'v> FromFormValue<'v> for Page {
type Error = &'v RawStr;
fn from_form_value(form_value: &'v RawStr) -> Result<Page, &'v RawStr> {
match form_value.parse::<i32>() {
Ok(page) => Ok(Page{page}),
_ => Err(form_value),
}
}
}
impl Page {
pub fn first() -> Page {
Page {
@ -109,6 +122,7 @@ pub mod reshares;
pub mod session;
pub mod tags;
pub mod user;
pub mod search;
pub mod well_known;
#[get("/static/<file..>", rank = 2)]

27
src/routes/posts.rs
View File

@ -1,13 +1,12 @@
use activitypub::object::Article;
use chrono::Utc;
use heck::{CamelCase, KebabCase};
use rocket::{State, request::LenientForm};
use rocket::{request::LenientForm};
use rocket::response::{Redirect, Flash};
use rocket_contrib::Template;
use serde_json;
use std::{collections::{HashMap, HashSet}, borrow::Cow};
use validator::{Validate, ValidationError, ValidationErrors};
use workerpool::{Pool, thunk::*};
use plume_common::activity_pub::{broadcast, ActivityStream, ApRequest, inbox::Deletable};
use plume_common::utils;
@ -24,6 +23,8 @@ use plume_models::{
tags::*,
users::User
};
use Worker;
use Searcher;
#[derive(FromForm)]
struct CommentQuery {
@ -163,7 +164,7 @@ fn edit(blog: String, slug: String, user: User, conn: DbConn) -> Option<Template
}
#[post("/~/<blog>/<slug>/edit", data = "<data>")]
fn update(blog: String, slug: String, user: User, conn: DbConn, data: LenientForm<NewPostForm>, worker: State<Pool<ThunkWorker<()>>>)
fn update(blog: String, slug: String, user: User, conn: DbConn, data: LenientForm<NewPostForm>, worker: Worker, searcher: Searcher)
-> Result<Redirect, Option<Template>> {
let b = Blog::find_by_fqn(&*conn, &blog).ok_or(None)?;
let mut post = Post::find_by_slug(&*conn, &slug, b.id).ok_or(None)?;
@ -217,7 +218,7 @@ fn update(blog: String, slug: String, user: User, conn: DbConn, data: LenientFor
post.source = form.content.clone();
post.license = license;
post.cover_id = form.cover;
post.update(&*conn);
post.update(&*conn, &searcher);
let post = post.update_ap_url(&*conn);
if post.published {
@ -236,11 +237,11 @@ fn update(blog: String, slug: String, user: User, conn: DbConn, data: LenientFor
if newly_published {
let act = post.create_activity(&conn);
let dest = User::one_by_instance(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, act, dest)));
worker.execute(move || broadcast(&user, act, dest));
} else {
let act = post.update_activity(&*conn);
let dest = User::one_by_instance(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, act, dest)));
worker.execute(move || broadcast(&user, act, dest));
}
}
@ -284,7 +285,7 @@ fn valid_slug(title: &str) -> Result<(), ValidationError> {
}
#[post("/~/<blog_name>/new", data = "<data>")]
fn create(blog_name: String, data: LenientForm<NewPostForm>, user: User, conn: DbConn, worker: State<Pool<ThunkWorker<()>>>) -> Result<Redirect, Option<Template>> {
fn create(blog_name: String, data: LenientForm<NewPostForm>, user: User, conn: DbConn, worker: Worker, searcher: Searcher) -> Result<Redirect, Option<Template>> {
let blog = Blog::find_by_fqn(&*conn, &blog_name).ok_or(None)?;
let form = data.get();
let slug = form.title.to_string().to_kebab_case();
@ -324,7 +325,9 @@ fn create(blog_name: String, data: LenientForm<NewPostForm>, user: User, conn: D
subtitle: form.subtitle.clone(),
source: form.content.clone(),
cover_id: form.cover,
});
},
&searcher,
);
let post = post.update_ap_url(&*conn);
PostAuthor::insert(&*conn, NewPostAuthor {
post_id: post.id,
@ -357,7 +360,7 @@ fn create(blog_name: String, data: LenientForm<NewPostForm>, user: User, conn: D
let act = post.create_activity(&*conn);
let dest = User::one_by_instance(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, act, dest)));
worker.execute(move || broadcast(&user, act, dest));
}
Ok(Redirect::to(uri!(details: blog = blog_name, slug = slug)))
@ -377,7 +380,7 @@ fn create(blog_name: String, data: LenientForm<NewPostForm>, user: User, conn: D
}
#[post("/~/<blog_name>/<slug>/delete")]
fn delete(blog_name: String, slug: String, conn: DbConn, user: User, worker: State<Pool<ThunkWorker<()>>>) -> Redirect {
fn delete(blog_name: String, slug: String, conn: DbConn, user: User, worker: Worker, searcher: Searcher) -> Redirect {
let post = Blog::find_by_fqn(&*conn, &blog_name)
.and_then(|blog| Post::find_by_slug(&*conn, &slug, blog.id));
@ -386,8 +389,8 @@ fn delete(blog_name: String, slug: String, conn: DbConn, user: User, worker: Sta
Redirect::to(uri!(details: blog = blog_name.clone(), slug = slug.clone()))
} else {
let dest = User::one_by_instance(&*conn);
let delete_activity = post.delete(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, delete_activity, dest)));
let delete_activity = post.delete(&(&conn, &searcher));
worker.execute(move || broadcast(&user, delete_activity, dest));
Redirect::to(uri!(super::blogs::details: name = blog_name))
}

10
src/routes/reshares.rs
View File

@ -1,5 +1,4 @@
use rocket::{State, response::{Redirect, Flash}};
use workerpool::{Pool, thunk::*};
use rocket::{response::{Redirect, Flash}};
use plume_common::activity_pub::{broadcast, inbox::{Deletable, Notify}};
use plume_common::utils;
@ -10,9 +9,10 @@ use plume_models::{
reshares::*,
users::User
};
use Worker;
#[post("/~/<blog>/<slug>/reshare")]
fn create(blog: String, slug: String, user: User, conn: DbConn, worker: State<Pool<ThunkWorker<()>>>) -> Option<Redirect> {
fn create(blog: String, slug: String, user: User, conn: DbConn, worker: Worker) -> Option<Redirect> {
let b = Blog::find_by_fqn(&*conn, &blog)?;
let post = Post::find_by_slug(&*conn, &slug, b.id)?;
@ -27,13 +27,13 @@ fn create(blog: String, slug: String, user: User, conn: DbConn, worker: State<Po
let dest = User::one_by_instance(&*conn);
let act = reshare.to_activity(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, act, dest)));
worker.execute(move || broadcast(&user, act, dest));
} else {
let reshare = Reshare::find_by_user_on_post(&*conn, user.id, post.id)
.expect("reshares::create: reshare exist but not found error");
let delete_act = reshare.delete(&*conn);
let dest = User::one_by_instance(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, delete_act, dest)));
worker.execute(move || broadcast(&user, delete_act, dest));
}
Some(Redirect::to(uri!(super::posts::details: blog = blog, slug = slug)))

84
src/routes/search.rs Normal file
View File

@ -0,0 +1,84 @@
use chrono::offset::Utc;
use rocket_contrib::Template;
use serde_json;
use plume_models::{
db_conn::DbConn, users::User,
search::Query};
use routes::Page;
use Searcher;
#[get("/search")]
fn index(conn: DbConn, user: Option<User>) -> Template {
Template::render("search/index", json!({
"account": user.map(|u| u.to_json(&*conn)),
"now": format!("{}", Utc::today().format("%Y-%m-d")),
}))
}
#[derive(FromForm)]
struct SearchQuery {
q: Option<String>,
title: Option<String>,
subtitle: Option<String>,
content: Option<String>,
instance: Option<String>,
author: Option<String>,
tag: Option<String>,
blog: Option<String>,
lang: Option<String>,
license: Option<String>,
after: Option<String>,
before: Option<String>,
page: Option<Page>,
}
macro_rules! param_to_query {
( $query:ident, $parsed_query:ident; normal: $($field:ident),*; date: $($date:ident),*) => {
$(
if let Some(field) = $query.$field {
let mut rest = field.as_str();
while !rest.is_empty() {
let (token, r) = Query::get_first_token(rest);
rest = r;
$parsed_query.$field(token, None);
}
}
)*
$(
if let Some(field) = $query.$date {
let mut rest = field.as_str();
while !rest.is_empty() {
use chrono::naive::NaiveDate;
let (token, r) = Query::get_first_token(rest);
rest = r;
if let Ok(token) = NaiveDate::parse_from_str(token, "%Y-%m-%d") {
$parsed_query.$date(&token);
}
}
}
)*
}
}
#[get("/search?<query>")]
fn query(query: SearchQuery, conn: DbConn, searcher: Searcher, user: Option<User>) -> Template {
let page = query.page.unwrap_or(Page::first());
let mut parsed_query = Query::from_str(&query.q.unwrap_or_default());
param_to_query!(query, parsed_query; normal: title, subtitle, content, tag,
instance, author, blog, lang, license;
date: before, after);
let str_q = parsed_query.to_string();
let res = searcher.search_document(&conn, parsed_query, page.limits());
Template::render("search/result", json!({
"query":str_q,
"account": user.map(|u| u.to_json(&*conn)),
"next_page": if res.is_empty() { 0 } else { page.page+1 },
"posts": res.into_iter().map(|p| p.to_json(&*conn)).collect::<Vec<serde_json::Value>>(),
"page": page.page,
}))
}

41
src/routes/user.rs
View File

@ -8,7 +8,6 @@ use rocket::{
use rocket_contrib::Template;
use serde_json;
use validator::{Validate, ValidationError};
use workerpool::thunk::*;
use inbox::Inbox;
use plume_common::activity_pub::{
@ -24,6 +23,7 @@ use plume_models::{
};
use routes::Page;
use Worker;
use Searcher;
#[get("/me")]
fn me(user: Option<User>) -> Result<Redirect, Flash<Redirect>> {
@ -39,9 +39,10 @@ fn details(
conn: DbConn,
account: Option<User>,
worker: Worker,
fecth_articles_conn: DbConn,
fecth_followers_conn: DbConn,
fetch_articles_conn: DbConn,
fetch_followers_conn: DbConn,
update_conn: DbConn,
searcher: Searcher,
) -> Template {
may_fail!(
account.map(|a| a.to_json(&*conn)),
@ -56,12 +57,13 @@ fn details(
if !user.get_instance(&*conn).local {
// Fetch new articles
let user_clone = user.clone();
worker.execute(Thunk::of(move || {
let searcher = searcher.clone();
worker.execute(move || {
for create_act in user_clone.fetch_outbox::<Create>() {
match create_act.create_props.object_object::<Article>() {
Ok(article) => {
Post::from_activity(
&*fecth_articles_conn,
&(&fetch_articles_conn, &searcher),
article,
user_clone.clone().into_id(),
);
@ -72,20 +74,20 @@ fn details(
}
}
}
}));
});
// Fetch followers
let user_clone = user.clone();
worker.execute(Thunk::of(move || {
worker.execute(move || {
for user_id in user_clone.fetch_followers_ids() {
let follower =
User::find_by_ap_url(&*fecth_followers_conn, &user_id)
User::find_by_ap_url(&*fetch_followers_conn, &user_id)
.unwrap_or_else(|| {
User::fetch_from_url(&*fecth_followers_conn, &user_id)
User::fetch_from_url(&*fetch_followers_conn, &user_id)
.expect("user::details: Couldn't fetch follower")
});
follows::Follow::insert(
&*fecth_followers_conn,
&*fetch_followers_conn,
follows::NewFollow {
follower_id: follower.id,
following_id: user_clone.id,
@ -93,14 +95,14 @@ fn details(
},
);
}
}));
});
// Update profile information if needed
let user_clone = user.clone();
if user.needs_update() {
worker.execute(Thunk::of(move || {
worker.execute(move || {
user_clone.refetch(&*update_conn);
}))
});
}
}
@ -148,9 +150,9 @@ fn follow(name: String, conn: DbConn, user: User, worker: Worker) -> Option<Redi
let target = User::find_by_fqn(&*conn, &name)?;
if let Some(follow) = follows::Follow::find(&*conn, user.id, target.id) {
let delete_act = follow.delete(&*conn);
worker.execute(Thunk::of(move || {
worker.execute(move || {
broadcast(&user, delete_act, vec![target])
}));
});
} else {
let f = follows::Follow::insert(
&*conn,
@ -163,7 +165,7 @@ fn follow(name: String, conn: DbConn, user: User, worker: Worker) -> Option<Redi
f.notify(&*conn);
let act = f.to_activity(&*conn);
worker.execute(Thunk::of(move || broadcast(&user, act, vec![target])));
worker.execute(move || broadcast(&user, act, vec![target]));
}
Some(Redirect::to(uri!(details: name = name)))
}
@ -285,10 +287,10 @@ fn update(_name: String, conn: DbConn, user: User, data: LenientForm<UpdateUserF
}
#[post("/@/<name>/delete")]
fn delete(name: String, conn: DbConn, user: User, mut cookies: Cookies) -> Option<Redirect> {
fn delete(name: String, conn: DbConn, user: User, mut cookies: Cookies, searcher: Searcher) -> Option<Redirect> {
let account = User::find_by_fqn(&*conn, &name)?;
if user.id == account.id {
account.delete(&*conn);
account.delete(&*conn, &searcher);
if let Some(cookie) = cookies.get_private(AUTH_COOKIE) {
cookies.remove_private(cookie);
@ -393,6 +395,7 @@ fn inbox(
conn: DbConn,
data: String,
headers: Headers,
searcher: Searcher,
) -> Result<String, Option<status::BadRequest<&'static str>>> {
let user = User::find_local(&*conn, &name).ok_or(None)?;
let act: serde_json::Value =
@ -420,7 +423,7 @@ fn inbox(
if Instance::is_blocked(&*conn, actor_id) {
return Ok(String::new());
}
Ok(match user.received(&*conn, act) {
Ok(match user.received(&*conn, &searcher, act) {
Ok(_) => String::new(),
Err(e) => {
println!("User inbox error: {}\n{}", e.as_fail(), e.backtrace());

11
templates/macros.html.tera
View File

@ -27,7 +27,7 @@
</p>
</div>
{% endmacro post_card %}
{% macro input(name, label, errors, form, type="text", props="", optional=false, default='', details=' ') %}
{% macro input(name, label, errors="", form="", type="text", props="", optional=false, default='', details=' ') %}
<label for="{{ name }}">
{{ label | _ }}
{% if optional %}
@ -43,13 +43,16 @@
{% set default = default[name] | default(value="") %}
<input type="{{ type }}" id="{{ name }}" name="{{ name }}" value="{{ form[name] | default(value=default) }}" {{ props | safe }}/>
{% endmacro input %}
{% macro paginate(page, total, previous="Previous page", next="Next page") %}
{% macro paginate(page, total, previous="Previous page", next="Next page", query="") %}
{% if query %}
{% set query = query ~ "&" %}
{% endif %}
<div class="pagination">
{% if page != 1 %}
<a href="?page={{ page - 1 }}">{{ previous | _ }}</a>
<a href="?{{ query }}page={{ page - 1 }}">{{ previous | _ }}</a>
{% endif %}
{% if page < total %}
<a href="?page={{ page + 1 }}">{{ next | _ }}</a>
<a href="?{{ query }}page={{ page + 1 }}">{{ next | _ }}</a>
{% endif %}
</div>
{% endmacro %}

50
templates/search/index.html.tera Normal file
View File

@ -0,0 +1,50 @@
{% extends "base" %}
{% import "macros" as macros %}
{% block title %}
Search
{% endblock title %}
{% block head %}
<script>
window.onload = function(evt) {
var form = document.getElementById('form');
form.addEventListener('submit', function () {
for (var input of form.getElementsByTagName('input')) {
if (input.name === '') {
input.name = input.id
}
if (input.name && !input.value) {
input.name = '';
}
}
});
}
</script>
{% endblock head %}
{% block content %}
<h1>search</h1>
<form method="get" id="form">
<input id="q" name="q" placeholder="Your query" type="search">
<details>
<summary>Advanced search</summary>
{{ macros::input(name="title", label="Title matching these words", props='placeholder="Title"') }}
{{ macros::input(name="subtitle", label="Subtitle matching these words", props='placeholder="Subtitle"') }}
{{ macros::input(name="content", label="Content matching these words", props='placeholder="Content"') }}
{{ macros::input(name="after", label="From this date", type="date", props='max=' ~ now) }}
{{ macros::input(name="before", label="To this date", type="date", props='max=' ~ now) }}
{{ macros::input(name="instance", label="Sent from one of these instance", props='placeholder="Instance domain"') }}
{{ macros::input(name="author", label="Sent by one of these author", props='placeholder="Authors"') }}
{{ macros::input(name="tag", label="Containing these tags", props='placeholder="Tags"') }}
{{ macros::input(name="blog", label="In blog", props='placeholder="Blog title"') }}
{{ macros::input(name="lang", label="Language used", props='placeholder="Language"') }}
{{ macros::input(name="license", label="Using license", props='placeholder="License"') }}
</details>
<input type="submit" value="Search"/>
</form>
{% endblock content %}

26
templates/search/result.html.tera Normal file
View File

@ -0,0 +1,26 @@
{% extends "base" %}
{% import "macros" as macros %}
{% block title %}
Search result for "{{query}}"
{% endblock title %}
{% block content %}
<h1>Search result</h1>
<p>{{query}}</p>
<section>
{% if posts | length < 1 %}
{% if page == 1 %}
<p>No result for your query</p>
{% else %}
<p>No more result for your query</p>
{% endif %}
{% endif %}
<div class="cards">
{% for article in posts %}
{{ macros::post_card(article=article) }}
{% endfor %}
</div>
{{ macros::paginate(page=page, total=next_page, query= "q=" ~ query) }}
</section>
{% endblock content %}