From 13a866aeb0c24f20ed18ab40c0ea5616ef910676 Mon Sep 17 00:00:00 2001 From: asimons04 <69986579+asimons04@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:51:51 -0400 Subject: [PATCH 01/27] Update Dockerfile to run process as non-privileged user. (#3709) --- docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index e81d9d0c2..02c2e572c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,4 +37,9 @@ RUN apk add --no-cache libpq # Copy resources COPY --from=builder /app/lemmy_server /app/lemmy +# Create non-privileged user +RUN adduser -h /app -s sh -S -u 1000 lemmy +RUN chown -R lemmy /app +USER lemmy + CMD ["/app/lemmy"] From 3b09d8c882c2effcfad9fda4ae06350dd20c0e5f Mon Sep 17 00:00:00 2001 From: phiresky Date: Tue, 25 Jul 2023 18:46:00 +0200 Subject: [PATCH 02/27] prevent ordering by comment path without post filter (#3717) --- crates/db_views/src/comment_view.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 1b77168d8..6e06e1baa 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -340,8 +340,11 @@ impl<'a> CommentQuery<'a> { query = query.filter(nlevel(comment::path).le(depth_limit)); - // Always order by the parent path first - query = query.order_by(subpath(comment::path, 0, -1)); + // only order if filtering by a post id. DOS potential otherwise and max_depth + !post_id isn't used anyways (afaik) + if self.post_id.is_some() { + // Always order by the parent path first + query = query.order_by(subpath(comment::path, 0, -1)); + } // TODO limit question. Limiting does not work for comment threads ATM, only max_depth // For now, don't do any limiting for tree fetches From 95e758e7079727d86a4e4f542d6b261fbf0ccc37 Mon Sep 17 00:00:00 2001 From: Ben Wyatt Date: Tue, 25 Jul 2023 12:01:13 -0500 Subject: [PATCH 03/27] Bump version of dependency "webmention" (#3711) Webmention .4->.5 bumps a few internal dependencies that needed to be upgraded --- Cargo.lock | 322 ++++++------------------------------- crates/api_crud/Cargo.toml | 2 +- 2 files changed, 46 insertions(+), 278 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e8f5fc6b..0597cfeb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,7 +134,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.8.5", + "rand", "sha1", "smallvec", "tokio", @@ -345,7 +345,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check", ] @@ -357,7 +357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check", ] @@ -416,12 +416,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f8ebf5827e4ac4fd5946560e6a99776ea73b596d80898f357007317a7141e47" -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "assert-json-diff" version = "2.0.2" @@ -538,7 +532,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.8.5", + "rand", "rustls 0.20.7", "serde", "serde_json", @@ -677,7 +671,7 @@ checksum = "28d1c9c15093eb224f0baa400f38fcd713fc1391a6f1c389d886beef146d60a3" dependencies = [ "base64 0.21.2", "blowfish", - "getrandom 0.2.10", + "getrandom", "subtle", "zeroize", ] @@ -718,18 +712,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" -[[package]] -name = "bitvec" -version = "0.19.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f93d0ef3363c364d5976646a38f04cf67cfe1d4c8d160cdea02cab2c116b33" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -828,7 +810,7 @@ dependencies = [ "hound", "image", "lodepng", - "rand 0.8.5", + "rand", "serde_json", ] @@ -1018,7 +1000,7 @@ dependencies = [ "async-trait", "json5", "lazy_static", - "nom 7.1.1", + "nom", "pathdiff", "ron", "rust-ini", @@ -1730,7 +1712,7 @@ checksum = "2e1f6c3800b304a6be0012039e2a45a322a093539c45ab818d9e6895a39c90fe" dependencies = [ "proc-macro2", "quote", - "rand 0.8.5", + "rand", "syn 1.0.103", ] @@ -1881,12 +1863,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "funty" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" - [[package]] name = "futf" version = "0.1.5" @@ -2020,17 +1996,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.10" @@ -2105,7 +2070,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "flate2", - "nom 7.1.1", + "nom", "num-traits", ] @@ -2186,10 +2151,10 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be92446e11d68f5d71367d571c229d09ced1f24ab6d08ea0bff329d5f6c0b2a3" dependencies = [ - "html5ever 0.26.0", + "html5ever", "jni", "lazy_static", - "markup5ever_rcdom 0.2.0", + "markup5ever_rcdom", "percent-encoding", "regex", ] @@ -2200,25 +2165,11 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74cda84f06c1cc83476f79ae8e2e892b626bdadafcb227baec54c918cadc18a0" dependencies = [ - "html5ever 0.26.0", - "markup5ever 0.11.0", + "html5ever", + "markup5ever", "tendril", "unicode-width", - "xml5ever 0.17.0", -] - -[[package]] -name = "html5ever" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148" -dependencies = [ - "log", - "mac", - "markup5ever 0.10.1", - "proc-macro2", - "quote", - "syn 1.0.103", + "xml5ever", ] [[package]] @@ -2229,7 +2180,7 @@ checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" dependencies = [ "log", "mac", - "markup5ever 0.11.0", + "markup5ever", "proc-macro2", "quote", "syn 1.0.103", @@ -2627,7 +2578,7 @@ dependencies = [ "chrono", "encoding", "futures", - "getrandom 0.2.10", + "getrandom", "lemmy_db_schema", "lemmy_db_views", "lemmy_db_views_actor", @@ -2913,7 +2864,7 @@ dependencies = [ "idna 0.3.0", "mime", "native-tls", - "nom 7.1.1", + "nom", "once_cell", "quoted_printable", "socket2 0.4.9", @@ -2921,19 +2872,6 @@ dependencies = [ "tokio-native-tls", ] -[[package]] -name = "lexical-core" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" -dependencies = [ - "arrayvec", - "bitflags 1.3.2", - "cfg-if", - "ryu", - "static_assertions", -] - [[package]] name = "libc" version = "0.2.146" @@ -3073,20 +3011,6 @@ dependencies = [ "unicode-general-category", ] -[[package]] -name = "markup5ever" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" -dependencies = [ - "log", - "phf 0.8.0", - "phf_codegen 0.8.0", - "string_cache", - "string_cache_codegen", - "tendril", -] - [[package]] name = "markup5ever" version = "0.11.0" @@ -3095,34 +3019,22 @@ checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" dependencies = [ "log", "phf 0.10.1", - "phf_codegen 0.10.0", + "phf_codegen", "string_cache", "string_cache_codegen", "tendril", ] -[[package]] -name = "markup5ever_rcdom" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015da43bcd8d4f144559a3423f4591d69b8ce0652c905374da7205df336ae2b" -dependencies = [ - "html5ever 0.25.2", - "markup5ever 0.10.1", - "tendril", - "xml5ever 0.16.2", -] - [[package]] name = "markup5ever_rcdom" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9521dd6750f8e80ee6c53d65e2e4656d7de37064f3a7a5d2d11d05df93839c2" dependencies = [ - "html5ever 0.26.0", - "markup5ever 0.11.0", + "html5ever", + "markup5ever", "tendril", - "xml5ever 0.17.0", + "xml5ever", ] [[package]] @@ -3315,19 +3227,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" -[[package]] -name = "nom" -version = "6.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2" -dependencies = [ - "bitvec", - "funty", - "lexical-core", - "memchr", - "version_check", -] - [[package]] name = "nom" version = "7.1.1" @@ -3480,7 +3379,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project", - "rand 0.8.5", + "rand", "thiserror", ] @@ -3557,7 +3456,7 @@ dependencies = [ "once_cell", "opentelemetry_api", "percent-encoding", - "rand 0.8.5", + "rand", "thiserror", "tokio", "tokio-stream", @@ -3716,15 +3615,6 @@ dependencies = [ "sha1", ] -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_shared 0.8.0", -] - [[package]] name = "phf" version = "0.10.1" @@ -3743,36 +3633,16 @@ dependencies = [ "phf_shared 0.11.1", ] -[[package]] -name = "phf_codegen" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", -] - [[package]] name = "phf_codegen" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" dependencies = [ - "phf_generator 0.10.0", + "phf_generator", "phf_shared 0.10.0", ] -[[package]] -name = "phf_generator" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" -dependencies = [ - "phf_shared 0.8.0", - "rand 0.7.3", -] - [[package]] name = "phf_generator" version = "0.10.0" @@ -3780,16 +3650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ "phf_shared 0.10.0", - "rand 0.8.5", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher", + "rand", ] [[package]] @@ -3957,7 +3818,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.8.5", + "rand", "sha2", "stringprep", ] @@ -4177,26 +4038,6 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49" -[[package]] -name = "radium" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", - "rand_pcg", -] - [[package]] name = "rand" version = "0.8.5" @@ -4204,18 +4045,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -4225,16 +4056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -4243,25 +4065,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", + "getrandom", ] [[package]] @@ -4409,7 +4213,7 @@ checksum = "1b97ad83c2fc18113346b7158d79732242002427c30f620fa817c1f32901e0a8" dependencies = [ "anyhow", "async-trait", - "getrandom 0.2.10", + "getrandom", "matchit 0.7.0", "opentelemetry 0.16.0", "reqwest", @@ -4707,13 +4511,13 @@ dependencies = [ [[package]] name = "select" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee061f90afcc8678bef7a78d0d121683f0ba753f740ff7005f833ec445876b7" +checksum = "6f9da09dc3f4dfdb6374cbffff7a2cffcec316874d4429899eefdc97b3b94dcd" dependencies = [ "bit-set", - "html5ever 0.25.2", - "markup5ever_rcdom 0.1.0", + "html5ever", + "markup5ever_rcdom", ] [[package]] @@ -5004,12 +4808,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "storage-path-generator" version = "0.1.1" @@ -5036,7 +4834,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ - "phf_generator 0.10.0", + "phf_generator", "phf_shared 0.10.0", "proc-macro2", "quote", @@ -5140,12 +4938,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "task-local-extensions" version = "0.1.4" @@ -5517,7 +5309,7 @@ dependencies = [ "base32", "constant_time_eq", "hmac", - "rand 0.8.5", + "rand", "sha1", "sha2", "url", @@ -5535,7 +5327,7 @@ dependencies = [ "indexmap 1.9.1", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand", "slab", "tokio", "tokio-util", @@ -5887,7 +5679,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" dependencies = [ - "getrandom 0.2.10", + "getrandom", "serde", ] @@ -5936,12 +5728,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -6054,12 +5840,12 @@ dependencies = [ [[package]] name = "webmention" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c1f9ad3af9421b7e94faef6f884d32bd60b6ea00ff05d84df74392a89c2b9f" +checksum = "8d07b90492f7b6fe35f5298fcd01c663d3c453e8c302dc86c7292c6681b8117d" dependencies = [ "anyhow", - "nom 6.1.2", + "nom", "reqwest", "select", "serde", @@ -6073,8 +5859,8 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8598785beeb5af95abe95e7bb20c7e747d1188347080d6811d5a56d2b9a5f368" dependencies = [ - "html5ever 0.26.0", - "markup5ever_rcdom 0.2.0", + "html5ever", + "markup5ever_rcdom", "serde", "serde_json", ] @@ -6313,24 +6099,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "wyz" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" - -[[package]] -name = "xml5ever" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9234163818fd8e2418fcde330655e757900d4236acd8cc70fef345ef91f6d865" -dependencies = [ - "log", - "mac", - "markup5ever 0.10.1", - "time 0.1.44", -] - [[package]] name = "xml5ever" version = "0.17.0" @@ -6339,7 +6107,7 @@ checksum = "4034e1d05af98b51ad7214527730626f019682d797ba38b51689212118d8e650" dependencies = [ "log", "mac", - "markup5ever 0.11.0", + "markup5ever", ] [[package]] diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 1da8335ea..06e29044b 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -21,6 +21,6 @@ actix-web = { workspace = true } tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } -webmention = "0.4.0" +webmention = "0.5.0" chrono = { workspace = true } uuid = { workspace = true } From 9a1f9aad45f36d2d6541349d0f735affd07094c1 Mon Sep 17 00:00:00 2001 From: Freakazoid182 Date: Tue, 25 Jul 2023 19:26:54 +0200 Subject: [PATCH 04/27] detailed error message for blocked domains (#3698) (#3701) * detailed error message for blocked domains (#3698) * Pass the domain as an error param Not formatting the error message to support i18n --------- Co-authored-by: Freek van Zee --- crates/apub/src/lib.rs | 31 ++++++++++++++++++++----------- crates/utils/src/error.rs | 4 ++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index 9a45284f2..e920e05df 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -42,7 +42,21 @@ impl UrlVerifier for VerifyUrlData { let local_site_data = local_site_data_cached(&mut (&self.0).into()) .await .expect("read local site data"); - check_apub_id_valid(url, &local_site_data)?; + check_apub_id_valid(url, &local_site_data).map_err(|err| match err { + LemmyError { + error_type: LemmyErrorType::FederationDisabled, + .. + } => "Federation disabled", + LemmyError { + error_type: LemmyErrorType::DomainBlocked(_), + .. + } => "Domain is blocked", + LemmyError { + error_type: LemmyErrorType::DomainNotInAllowList(_), + .. + } => "Domain is not in allowlist", + _ => "Failed validating apub id", + })?; Ok(()) } } @@ -55,7 +69,7 @@ impl UrlVerifier for VerifyUrlData { /// - URL being in the allowlist (if it is active) /// - URL not being in the blocklist (if it is active) #[tracing::instrument(skip(local_site_data))] -fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result<(), &'static str> { +fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result<(), LemmyError> { let domain = apub_id.domain().expect("apud id has domain").to_string(); if !local_site_data @@ -64,7 +78,7 @@ fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result .map(|l| l.federation_enabled) .unwrap_or(true) { - return Err("Federation disabled"); + return Err(LemmyErrorType::FederationDisabled)?; } if local_site_data @@ -72,7 +86,7 @@ fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result .iter() .any(|i| domain.eq(&i.domain)) { - return Err("Domain is blocked"); + return Err(LemmyErrorType::DomainBlocked(domain))?; } // Only check this if there are instances in the allowlist @@ -82,7 +96,7 @@ fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result .iter() .any(|i| domain.eq(&i.domain)) { - return Err("Domain is not in allowlist"); + return Err(LemmyErrorType::DomainNotInAllowList(domain))?; } Ok(()) @@ -142,12 +156,7 @@ pub(crate) async fn check_apub_id_valid_with_strictness( } let local_site_data = local_site_data_cached(&mut context.pool()).await?; - check_apub_id_valid(apub_id, &local_site_data).map_err(|err| match err { - "Federation disabled" => LemmyErrorType::FederationDisabled, - "Domain is blocked" => LemmyErrorType::DomainBlocked, - "Domain is not in allowlist" => LemmyErrorType::DomainNotInAllowList, - _ => panic!("Could not handle apub error!"), - })?; + check_apub_id_valid(apub_id, &local_site_data)?; // Only check allowlist if this is a community, and there are instances in the allowlist if is_strict && !local_site_data.allowed_instances.is_empty() { diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs index ffc1723b4..c6e6ad018 100644 --- a/crates/utils/src/error.rs +++ b/crates/utils/src/error.rs @@ -195,8 +195,8 @@ pub enum LemmyErrorType { CouldntFindObject, RegistrationDenied(String), FederationDisabled, - DomainBlocked, - DomainNotInAllowList, + DomainBlocked(String), + DomainNotInAllowList(String), FederationDisabledByStrictAllowList, SiteNameRequired, SiteNameLengthOverflow, From cf2229d66520fdb3833c4f1b1d3aa569f41d8883 Mon Sep 17 00:00:00 2001 From: Into the V0id <57257404+into-the-v0id@users.noreply.github.com> Date: Tue, 25 Jul 2023 17:33:02 +0000 Subject: [PATCH 05/27] Dont authenticate user after successful password reset #3714 (#3715) Co-authored-by: Dessalines --- .../local_user/change_password_after_reset.rs | 38 ++++--------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/crates/api/src/local_user/change_password_after_reset.rs b/crates/api/src/local_user/change_password_after_reset.rs index 919c250f4..65587bcbf 100644 --- a/crates/api/src/local_user/change_password_after_reset.rs +++ b/crates/api/src/local_user/change_password_after_reset.rs @@ -5,15 +5,11 @@ use lemmy_api_common::{ person::{LoginResponse, PasswordChangeAfterReset}, utils::password_length_check, }; -use lemmy_db_schema::{ - source::{local_user::LocalUser, password_reset_request::PasswordResetRequest}, - RegistrationMode, -}; -use lemmy_db_views::structs::SiteView; -use lemmy_utils::{ - claims::Claims, - error::{LemmyError, LemmyErrorExt, LemmyErrorType}, +use lemmy_db_schema::source::{ + local_user::LocalUser, + password_reset_request::PasswordResetRequest, }; +use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; #[async_trait::async_trait(?Send)] impl Perform for PasswordChangeAfterReset { @@ -38,30 +34,12 @@ impl Perform for PasswordChangeAfterReset { // Update the user with the new password let password = data.password.clone(); - let updated_local_user = - LocalUser::update_password(&mut context.pool(), local_user_id, &password) - .await - .with_lemmy_type(LemmyErrorType::CouldntUpdateUser)?; - - // Return the jwt if login is allowed - let site_view = SiteView::read_local(&mut context.pool()).await?; - let jwt = if site_view.local_site.registration_mode == RegistrationMode::RequireApplication - && !updated_local_user.accepted_application - { - None - } else { - Some( - Claims::jwt( - updated_local_user.id.0, - &context.secret().jwt_secret, - &context.settings().hostname, - )? - .into(), - ) - }; + LocalUser::update_password(&mut context.pool(), local_user_id, &password) + .await + .with_lemmy_type(LemmyErrorType::CouldntUpdateUser)?; Ok(LoginResponse { - jwt, + jwt: None, verify_email_sent: false, registration_created: false, }) From 9b123f45eca1f364d038e52fab43ed3dea36fa98 Mon Sep 17 00:00:00 2001 From: dullbananas Date: Wed, 26 Jul 2023 03:22:14 -0700 Subject: [PATCH 06/27] Enable gzip for reqwest (#3696) This will reduce bandwidth used by federation Co-authored-by: Dessalines --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f5268be24..405ed9d4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ tracing-log = "0.1.3" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } url = { version = "2.4.0", features = ["serde"] } url_serde = "0.2.0" -reqwest = { version = "0.11.18", features = ["json", "blocking"] } +reqwest = { version = "0.11.18", features = ["json", "blocking", "gzip"] } reqwest-middleware = "0.2.2" reqwest-tracing = "0.4.5" clokwerk = "0.4.0" From f9351b651217cd437801e5f579891f5ad815e05d Mon Sep 17 00:00:00 2001 From: Domenic Horner Date: Wed, 26 Jul 2023 19:53:45 +0800 Subject: [PATCH 07/27] UI Settings - Blur NSFW & Auto Expand (#3377) * add new flag to api * add new ui settings for local user * remove extraneous def * add props to application reg. * fix clippy updated these * re-order db schema entries * remove dupe * update lemmy sdk * update lemmy js client --------- Co-authored-by: Nutomic --- api_tests/src/shared.ts | 4 ++++ crates/api/src/local_user/save_settings.rs | 2 ++ crates/api_common/src/person.rs | 2 ++ crates/db_schema/src/schema.rs | 2 ++ crates/db_schema/src/source/local_user.rs | 6 ++++++ crates/db_views/src/registration_application_view.rs | 2 ++ migrations/2023-06-27-065106_add_ui_settings/down.sql | 2 ++ migrations/2023-06-27-065106_add_ui_settings/up.sql | 6 ++++++ 8 files changed, 26 insertions(+) create mode 100644 migrations/2023-06-27-065106_add_ui_settings/down.sql create mode 100644 migrations/2023-06-27-065106_add_ui_settings/up.sql diff --git a/api_tests/src/shared.ts b/api_tests/src/shared.ts index bbd4eaaeb..e4306c94c 100644 --- a/api_tests/src/shared.ts +++ b/api_tests/src/shared.ts @@ -611,6 +611,8 @@ export async function registerUser( export async function saveUserSettingsBio(api: API): Promise { let form: SaveUserSettings = { show_nsfw: true, + blur_nsfw: false, + auto_expand: true, theme: "darkly", default_sort_type: "Active", default_listing_type: "All", @@ -631,6 +633,8 @@ export async function saveUserSettingsFederated( let bio = "a changed bio"; let form: SaveUserSettings = { show_nsfw: false, + blur_nsfw: true, + auto_expand: false, default_sort_type: "Hot", default_listing_type: "All", interface_language: "", diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index 4176a3f4c..c5038eb79 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -124,6 +124,8 @@ impl Perform for SaveUserSettings { .show_new_post_notifs(data.show_new_post_notifs) .send_notifications_to_email(data.send_notifications_to_email) .show_nsfw(data.show_nsfw) + .blur_nsfw(data.blur_nsfw) + .auto_expand(data.auto_expand) .show_bot_accounts(data.show_bot_accounts) .show_scores(data.show_scores) .default_sort_type(default_sort_type) diff --git a/crates/api_common/src/person.rs b/crates/api_common/src/person.rs index 031bc6c7e..79a0aa377 100644 --- a/crates/api_common/src/person.rs +++ b/crates/api_common/src/person.rs @@ -91,6 +91,8 @@ pub struct CaptchaResponse { pub struct SaveUserSettings { /// Show nsfw posts. pub show_nsfw: Option, + pub blur_nsfw: Option, + pub auto_expand: Option, /// Show post and comment scores. pub show_scores: Option, /// Your user's theme. diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 17a0f99f8..faebe9999 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -395,6 +395,8 @@ diesel::table! { totp_2fa_secret -> Nullable, totp_2fa_url -> Nullable, open_links_in_new_tab -> Bool, + blur_nsfw -> Bool, + auto_expand -> Bool, infinite_scroll_enabled -> Bool, } } diff --git a/crates/db_schema/src/source/local_user.rs b/crates/db_schema/src/source/local_user.rs index d9e1bde75..0d8db6693 100644 --- a/crates/db_schema/src/source/local_user.rs +++ b/crates/db_schema/src/source/local_user.rs @@ -53,6 +53,8 @@ pub struct LocalUser { pub totp_2fa_url: Option, /// Open links in a new tab. pub open_links_in_new_tab: bool, + pub blur_nsfw: bool, + pub auto_expand: bool, /// Whether infinite scroll is enabled. pub infinite_scroll_enabled: bool, } @@ -83,6 +85,8 @@ pub struct LocalUserInsertForm { pub totp_2fa_secret: Option>, pub totp_2fa_url: Option>, pub open_links_in_new_tab: Option, + pub blur_nsfw: Option, + pub auto_expand: Option, pub infinite_scroll_enabled: Option, } @@ -109,5 +113,7 @@ pub struct LocalUserUpdateForm { pub totp_2fa_secret: Option>, pub totp_2fa_url: Option>, pub open_links_in_new_tab: Option, + pub blur_nsfw: Option, + pub auto_expand: Option, pub infinite_scroll_enabled: Option, } diff --git a/crates/db_views/src/registration_application_view.rs b/crates/db_views/src/registration_application_view.rs index 106e41e43..8e2f58264 100644 --- a/crates/db_views/src/registration_application_view.rs +++ b/crates/db_views/src/registration_application_view.rs @@ -280,6 +280,8 @@ mod tests { person_id: inserted_sara_local_user.person_id, email: inserted_sara_local_user.email, show_nsfw: inserted_sara_local_user.show_nsfw, + auto_expand: inserted_sara_local_user.auto_expand, + blur_nsfw: inserted_sara_local_user.blur_nsfw, theme: inserted_sara_local_user.theme, default_sort_type: inserted_sara_local_user.default_sort_type, default_listing_type: inserted_sara_local_user.default_listing_type, diff --git a/migrations/2023-06-27-065106_add_ui_settings/down.sql b/migrations/2023-06-27-065106_add_ui_settings/down.sql new file mode 100644 index 000000000..51fa70467 --- /dev/null +++ b/migrations/2023-06-27-065106_add_ui_settings/down.sql @@ -0,0 +1,2 @@ +alter table local_user drop column blur_nsfw; +alter table local_user drop column auto_expand; diff --git a/migrations/2023-06-27-065106_add_ui_settings/up.sql b/migrations/2023-06-27-065106_add_ui_settings/up.sql new file mode 100644 index 000000000..3c66c9c79 --- /dev/null +++ b/migrations/2023-06-27-065106_add_ui_settings/up.sql @@ -0,0 +1,6 @@ + +-- Add the blur_nsfw to the local user table as a setting +alter table local_user add column blur_nsfw boolean not null default true; + +-- Add the auto_expand to the local user table as a setting +alter table local_user add column auto_expand boolean not null default false; From afac3ee7ffb0c577a7572b8e5d06080e59a57b71 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 26 Jul 2023 18:15:18 +0200 Subject: [PATCH 08/27] Skip fragile API tests (#3723) * Skip fragile API tests These tests fail very often for no reason, eg in #3712 and #3696. Better to disable them until they can be fixed. * killall -s1 --- api_tests/run-federation-test.sh | 2 +- api_tests/src/comment.spec.ts | 2 +- api_tests/src/post.spec.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api_tests/run-federation-test.sh b/api_tests/run-federation-test.sh index abced2ad6..0d241e2ab 100755 --- a/api_tests/run-federation-test.sh +++ b/api_tests/run-federation-test.sh @@ -13,7 +13,7 @@ popd yarn yarn api-test || true -killall lemmy_server +killall -s1 lemmy_server for INSTANCE in lemmy_alpha lemmy_beta lemmy_gamma lemmy_delta lemmy_epsilon; do psql "$LEMMY_DATABASE_URL" -c "DROP DATABASE $INSTANCE" diff --git a/api_tests/src/comment.spec.ts b/api_tests/src/comment.spec.ts index 932c7ffeb..19c2797b2 100644 --- a/api_tests/src/comment.spec.ts +++ b/api_tests/src/comment.spec.ts @@ -141,7 +141,7 @@ test("Delete a comment", async () => { assertCommentFederation(betaComment2, undeleteCommentRes.comment_view); }); -test("Remove a comment from admin and community on the same instance", async () => { +test.skip("Remove a comment from admin and community on the same instance", async () => { let commentRes = await createComment(alpha, postRes.post_view.post.id); // Get the id for beta diff --git a/api_tests/src/post.spec.ts b/api_tests/src/post.spec.ts index 8ea3ea912..532841b13 100644 --- a/api_tests/src/post.spec.ts +++ b/api_tests/src/post.spec.ts @@ -412,7 +412,7 @@ test("Enforce site ban for federated user", async () => { expect(alphaUserOnBeta2.person?.person.banned).toBe(false); }); -test("Enforce community ban for federated user", async () => { +test.skip("Enforce community ban for federated user", async () => { if (!betaCommunity) { throw "Missing beta community"; } From dc4572460e8458753907dc9659bdd015e2f7bb27 Mon Sep 17 00:00:00 2001 From: marsara9 <1316726+marsara9@users.noreply.github.com> Date: Wed, 26 Jul 2023 12:17:42 -0400 Subject: [PATCH 09/27] Make resolve_object not require auth #3685 (#3716) * Resolves issue #3685 If user isn't authenticated with resolve_object, only allow a local search instead of possibly making an http request. * Making sure to validate auth before doing a potential remote lookup. --- crates/api_common/src/site.rs | 2 +- crates/apub/src/api/resolve_object.rs | 37 ++++++++++++++++++--------- crates/apub/src/fetcher/search.rs | 12 +++++++++ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/crates/api_common/src/site.rs b/crates/api_common/src/site.rs index bc7687e3c..35b6d77ec 100644 --- a/crates/api_common/src/site.rs +++ b/crates/api_common/src/site.rs @@ -84,7 +84,7 @@ pub struct SearchResponse { pub struct ResolveObject { /// Can be the full url, or a shortened version like: !fediverse@lemmy.ml pub q: String, - pub auth: Sensitive, + pub auth: Option>, } #[skip_serializing_none] diff --git a/crates/apub/src/api/resolve_object.rs b/crates/apub/src/api/resolve_object.rs index d86c28d60..898cc8d51 100644 --- a/crates/apub/src/api/resolve_object.rs +++ b/crates/apub/src/api/resolve_object.rs @@ -1,11 +1,15 @@ -use crate::fetcher::search::{search_query_to_object_id, SearchableObjects}; +use crate::fetcher::search::{ + search_query_to_object_id, + search_query_to_object_id_local, + SearchableObjects, +}; use activitypub_federation::config::Data; use actix_web::web::{Json, Query}; use diesel::NotFound; use lemmy_api_common::{ context::LemmyContext, site::{ResolveObject, ResolveObjectResponse}, - utils::{check_private_instance, local_user_view_from_jwt}, + utils::{check_private_instance, local_user_view_from_jwt_opt}, }; use lemmy_db_schema::{newtypes::PersonId, source::local_site::LocalSite, utils::DbPool}; use lemmy_db_views::structs::{CommentView, PostView}; @@ -17,14 +21,23 @@ pub async fn resolve_object( data: Query, context: Data, ) -> Result, LemmyError> { - let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; + let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), &context).await; let local_site = LocalSite::read(&mut context.pool()).await?; - let person_id = local_user_view.person.id; - check_private_instance(&Some(local_user_view), &local_site)?; + check_private_instance(&local_user_view, &local_site)?; + let person_id = local_user_view.map(|v| v.person.id); + // If we get a valid personId back we can safely assume that the user is authenticated, + // if there's no personId then the JWT was missing or invalid. + let is_authenticated = person_id.is_some(); + + let res = if is_authenticated { + // user is fully authenticated; allow remote lookups as well. + search_query_to_object_id(&data.q, &context).await + } else { + // user isn't authenticated only allow a local search. + search_query_to_object_id_local(&data.q, &context).await + } + .with_lemmy_type(LemmyErrorType::CouldntFindObject)?; - let res = search_query_to_object_id(&data.q, &context) - .await - .with_lemmy_type(LemmyErrorType::CouldntFindObject)?; convert_response(res, person_id, &mut context.pool()) .await .with_lemmy_type(LemmyErrorType::CouldntFindObject) @@ -32,7 +45,7 @@ pub async fn resolve_object( async fn convert_response( object: SearchableObjects, - user_id: PersonId, + user_id: Option, pool: &mut DbPool<'_>, ) -> Result, LemmyError> { use SearchableObjects::*; @@ -45,15 +58,15 @@ async fn convert_response( } Community(c) => { removed_or_deleted = c.deleted || c.removed; - res.community = Some(CommunityView::read(pool, c.id, Some(user_id), None).await?) + res.community = Some(CommunityView::read(pool, c.id, user_id, None).await?) } Post(p) => { removed_or_deleted = p.deleted || p.removed; - res.post = Some(PostView::read(pool, p.id, Some(user_id), None).await?) + res.post = Some(PostView::read(pool, p.id, user_id, None).await?) } Comment(c) => { removed_or_deleted = c.deleted || c.removed; - res.comment = Some(CommentView::read(pool, c.id, Some(user_id)).await?) + res.comment = Some(CommentView::read(pool, c.id, user_id).await?) } }; // if the object was deleted from database, dont return it diff --git a/crates/apub/src/fetcher/search.rs b/crates/apub/src/fetcher/search.rs index 39ecbc1be..dd8ef2ca2 100644 --- a/crates/apub/src/fetcher/search.rs +++ b/crates/apub/src/fetcher/search.rs @@ -44,6 +44,18 @@ pub(crate) async fn search_query_to_object_id( }) } +/// Converts a search query to an object id. The query MUST bbe a URL which will bbe treated +/// as the ObjectId directly. If the query is a webfinger identifier (@user@example.com or +/// !community@example.com) this method will return an error. +#[tracing::instrument(skip_all)] +pub(crate) async fn search_query_to_object_id_local( + query: &str, + context: &Data, +) -> Result { + let url = Url::parse(query)?; + ObjectId::from(url).dereference_local(context).await +} + /// The types of ActivityPub objects that can be fetched directly by searching for their ID. #[derive(Debug)] pub(crate) enum SearchableObjects { From c890797b370417b9479186acd86bc065693f8691 Mon Sep 17 00:00:00 2001 From: Pavlos Smith <57727226+iByteABit256@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:07:05 +0300 Subject: [PATCH 10/27] Add controversial ranking (#3205) * Added controversy rank property to posts and comments, and ability to sort by it * Triggers instead of schedules tasks, integer -> double, TODO: comments don't seem to get updated with floats, divide SortTypes * Created PersonSortType * PersonSortType::MostComments case * Removed unused PartialOrd trait * Added new person sort type mappings * SortType -> PersonSortType * fixes * cargo fmt * fixes after merge with main * Fixed bug in controversy rank trigger, removed TopX sorts from PersonSortType and added CommentScore instead * Uncovered enum case * clippy * reset translation changes * translations * translations * Added additional hot ordering on controversial posts and comments * featured local and featured community added to controversy rank index, additional order_by removed (?), added post_score and post_count to PersonSortType * Woodpecker rerun * cargo fmt * woodpecker rerun * fixed controversy_rank order * fix * Readded migration as latest, removed second update statement for setting controversy rank --- crates/apub/src/api/search.rs | 6 +- crates/db_schema/src/aggregates/structs.rs | 6 +- crates/db_schema/src/lib.rs | 15 +++ crates/db_schema/src/schema.rs | 2 + crates/db_schema/src/utils.rs | 16 +++ crates/db_views/src/comment_report_view.rs | 1 + crates/db_views/src/comment_view.rs | 4 + crates/db_views/src/post_report_view.rs | 1 + crates/db_views/src/post_view.rs | 2 + crates/db_views/src/structs.rs | 8 +- .../db_views_actor/src/comment_reply_view.rs | 3 + crates/db_views_actor/src/community_view.rs | 3 +- .../db_views_actor/src/person_mention_view.rs | 3 + crates/db_views_actor/src/person_view.rs | 50 ++-------- crates/db_views_actor/src/structs.rs | 4 +- .../down.sql | 63 ++++++++++++ .../up.sql | 97 +++++++++++++++++++ 17 files changed, 232 insertions(+), 52 deletions(-) create mode 100644 migrations/2023-07-26-000217_create_controversial_indexes/down.sql create mode 100644 migrations/2023-07-26-000217_create_controversial_indexes/up.sql diff --git a/crates/apub/src/api/search.rs b/crates/apub/src/api/search.rs index ca84606ff..aaea69e07 100644 --- a/crates/apub/src/api/search.rs +++ b/crates/apub/src/api/search.rs @@ -8,7 +8,7 @@ use lemmy_api_common::{ }; use lemmy_db_schema::{ source::{community::Community, local_site::LocalSite}, - utils::post_to_comment_sort_type, + utils::{post_to_comment_sort_type, post_to_person_sort_type}, SearchType, }; use lemmy_db_views::{comment_view::CommentQuery, post_view::PostQuery}; @@ -98,7 +98,7 @@ pub async fn search( } SearchType::Users => { users = PersonQuery { - sort: (sort), + sort: (sort.map(post_to_person_sort_type)), search_term: (Some(q)), page: (page), limit: (limit), @@ -168,7 +168,7 @@ pub async fn search( vec![] } else { PersonQuery { - sort: (sort), + sort: (sort.map(post_to_person_sort_type)), search_term: (Some(q)), page: (page), limit: (limit), diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 1af94a800..3b3612bb7 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] use ts_rs::TS; -#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] +#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))] #[cfg_attr(feature = "full", diesel(table_name = comment_aggregates))] #[cfg_attr(feature = "full", diesel(belongs_to(crate::source::comment::Comment)))] @@ -28,6 +28,7 @@ pub struct CommentAggregates { /// The total number of children in this comment branch. pub child_count: i32, pub hot_rank: i32, + pub controversy_rank: f64, } #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] @@ -72,7 +73,7 @@ pub struct PersonAggregates { pub comment_score: i64, } -#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] +#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))] #[cfg_attr(feature = "full", diesel(table_name = post_aggregates))] #[cfg_attr(feature = "full", diesel(belongs_to(crate::source::post::Post)))] @@ -98,6 +99,7 @@ pub struct PostAggregates { pub hot_rank_active: i32, pub community_id: CommunityId, pub creator_id: PersonId, + pub controversy_rank: f64, } #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index acb069ca7..e5b86fe19 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -66,6 +66,7 @@ pub enum SortType { TopThreeMonths, TopSixMonths, TopNineMonths, + Controversial, } #[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy)] @@ -77,6 +78,20 @@ pub enum CommentSortType { Top, New, Old, + Controversial, +} + +#[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +/// The person sort types. See here for descriptions: https://join-lemmy.org/docs/en/users/03-votes-and-ranking.html +pub enum PersonSortType { + New, + Old, + MostComments, + CommentScore, + PostScore, + PostCount, } #[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index faebe9999..d6e2cf69c 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -93,6 +93,7 @@ diesel::table! { published -> Timestamp, child_count -> Int4, hot_rank -> Int4, + controversy_rank -> Float8, } } @@ -676,6 +677,7 @@ diesel::table! { hot_rank_active -> Int4, community_id -> Int4, creator_id -> Int4, + controversy_rank -> Float8, } } diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 94c867d6b..cd2005ad0 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -3,6 +3,7 @@ use crate::{ diesel_migrations::MigrationHarness, newtypes::DbUrl, CommentSortType, + PersonSortType, SortType, }; use activitypub_federation::{fetch::object_id::ObjectId, traits::Object}; @@ -347,6 +348,7 @@ pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType { SortType::Active | SortType::Hot => CommentSortType::Hot, SortType::New | SortType::NewComments | SortType::MostComments => CommentSortType::New, SortType::Old => CommentSortType::Old, + SortType::Controversial => CommentSortType::Controversial, SortType::TopHour | SortType::TopSixHour | SortType::TopTwelveHour @@ -361,6 +363,16 @@ pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType { } } +pub fn post_to_person_sort_type(sort: SortType) -> PersonSortType { + match sort { + SortType::Active | SortType::Hot | SortType::Controversial => PersonSortType::CommentScore, + SortType::New | SortType::NewComments => PersonSortType::New, + SortType::MostComments => PersonSortType::MostComments, + SortType::Old => PersonSortType::Old, + _ => PersonSortType::CommentScore, + } +} + static EMAIL_REGEX: Lazy = Lazy::new(|| { Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$") .expect("compile email regex") @@ -373,6 +385,10 @@ pub mod functions { fn hot_rank(score: BigInt, time: Timestamp) -> Integer; } + sql_function! { + fn controversy_rank(upvotes: BigInt, downvotes: BigInt, score: BigInt) -> Double; + } + sql_function!(fn lower(x: Text) -> Text); } diff --git a/crates/db_views/src/comment_report_view.rs b/crates/db_views/src/comment_report_view.rs index a09971dbe..0b1821c1b 100644 --- a/crates/db_views/src/comment_report_view.rs +++ b/crates/db_views/src/comment_report_view.rs @@ -478,6 +478,7 @@ mod tests { published: agg.published, child_count: 0, hot_rank: 1728, + controversy_rank: 0.0, }, my_vote: None, resolver: None, diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 6e06e1baa..2d233438a 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -366,6 +366,9 @@ impl<'a> CommentQuery<'a> { CommentSortType::Hot => query .then_order_by(comment_aggregates::hot_rank.desc()) .then_order_by(comment_aggregates::score.desc()), + CommentSortType::Controversial => { + query.then_order_by(comment_aggregates::controversy_rank.desc()) + } CommentSortType::New => query.then_order_by(comment::published.desc()), CommentSortType::Old => query.then_order_by(comment::published.asc()), CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), @@ -948,6 +951,7 @@ mod tests { published: agg.published, child_count: 5, hot_rank: 1728, + controversy_rank: 0.0, }, } } diff --git a/crates/db_views/src/post_report_view.rs b/crates/db_views/src/post_report_view.rs index a53762e21..8c47d8c52 100644 --- a/crates/db_views/src/post_report_view.rs +++ b/crates/db_views/src/post_report_view.rs @@ -470,6 +470,7 @@ mod tests { featured_local: false, hot_rank: 1728, hot_rank_active: 1728, + controversy_rank: 0.0, community_id: inserted_post.community_id, creator_id: inserted_post.creator_id, }, diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index d2f6ab759..e0f481edc 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -422,6 +422,7 @@ impl<'a> PostQuery<'a> { SortType::Hot => query .then_order_by(post_aggregates::hot_rank.desc()) .then_order_by(post_aggregates::published.desc()), + SortType::Controversial => query.then_order_by(post_aggregates::controversy_rank.desc()), SortType::New => query.then_order_by(post_aggregates::published.desc()), SortType::Old => query.then_order_by(post_aggregates::published.asc()), SortType::NewComments => query.then_order_by(post_aggregates::newest_comment_time.desc()), @@ -1141,6 +1142,7 @@ mod tests { featured_local: false, hot_rank: 1728, hot_rank_active: 1728, + controversy_rank: 0.0, community_id: inserted_post.community_id, creator_id: inserted_post.creator_id, }, diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index 403292909..536dd3597 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -25,7 +25,7 @@ use serde_with::skip_serializing_none; use ts_rs::TS; #[skip_serializing_none] -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A comment report view. @@ -43,7 +43,7 @@ pub struct CommentReportView { } #[skip_serializing_none] -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A comment view. @@ -71,7 +71,7 @@ pub struct LocalUserView { } #[skip_serializing_none] -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A post report view. @@ -88,7 +88,7 @@ pub struct PostReportView { } #[skip_serializing_none] -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A post view. diff --git a/crates/db_views_actor/src/comment_reply_view.rs b/crates/db_views_actor/src/comment_reply_view.rs index 4d7a8eac4..406bfcb97 100644 --- a/crates/db_views_actor/src/comment_reply_view.rs +++ b/crates/db_views_actor/src/comment_reply_view.rs @@ -266,6 +266,9 @@ impl CommentReplyQuery { query = match self.sort.unwrap_or(CommentSortType::New) { CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()), + CommentSortType::Controversial => { + query.then_order_by(comment_aggregates::controversy_rank.desc()) + } CommentSortType::New => query.then_order_by(comment_reply::published.desc()), CommentSortType::Old => query.then_order_by(comment_reply::published.asc()), CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), diff --git a/crates/db_views_actor/src/community_view.rs b/crates/db_views_actor/src/community_view.rs index 64dc09090..c31a2bd5d 100644 --- a/crates/db_views_actor/src/community_view.rs +++ b/crates/db_views_actor/src/community_view.rs @@ -170,7 +170,8 @@ impl<'a> CommunityQuery<'a> { } New => query = query.order_by(community::published.desc()), Old => query = query.order_by(community::published.asc()), - MostComments => query = query.order_by(community_aggregates::comments.desc()), + // Controversial is temporary until a CommentSortType is created + MostComments | Controversial => query = query.order_by(community_aggregates::comments.desc()), TopAll | TopYear | TopNineMonths => { query = query.order_by(community_aggregates::subscribers.desc()) } diff --git a/crates/db_views_actor/src/person_mention_view.rs b/crates/db_views_actor/src/person_mention_view.rs index 3e142254a..6bf107a3d 100644 --- a/crates/db_views_actor/src/person_mention_view.rs +++ b/crates/db_views_actor/src/person_mention_view.rs @@ -271,6 +271,9 @@ impl PersonMentionQuery { query = match self.sort.unwrap_or(CommentSortType::Hot) { CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()), + CommentSortType::Controversial => { + query.then_order_by(comment_aggregates::controversy_rank.desc()) + } CommentSortType::New => query.then_order_by(comment::published.desc()), CommentSortType::Old => query.then_order_by(comment::published.asc()), CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), diff --git a/crates/db_views_actor/src/person_view.rs b/crates/db_views_actor/src/person_view.rs index e6baa1fc6..908fbaab4 100644 --- a/crates/db_views_actor/src/person_view.rs +++ b/crates/db_views_actor/src/person_view.rs @@ -1,6 +1,6 @@ use crate::structs::PersonView; use diesel::{ - dsl::{now, IntervalDsl}, + dsl::now, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -16,7 +16,7 @@ use lemmy_db_schema::{ source::person::Person, traits::JoinView, utils::{fuzzy_search, get_conn, limit_and_offset, DbPool}, - SortType, + PersonSortType, }; use std::iter::Iterator; @@ -80,7 +80,7 @@ impl PersonView { #[derive(Default)] pub struct PersonQuery { - pub sort: Option, + pub sort: Option, pub search_term: Option, pub page: Option, pub limit: Option, @@ -101,43 +101,13 @@ impl PersonQuery { .or_filter(person::display_name.ilike(searcher)); } - query = match self.sort.unwrap_or(SortType::Hot) { - SortType::New | SortType::NewComments => query.order_by(person::published.desc()), - SortType::Old => query.order_by(person::published.asc()), - SortType::Hot | SortType::Active | SortType::TopAll => { - query.order_by(person_aggregates::comment_score.desc()) - } - SortType::MostComments => query.order_by(person_aggregates::comment_count.desc()), - SortType::TopYear => query - .filter(person::published.gt(now - 1.years())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopMonth => query - .filter(person::published.gt(now - 1.months())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopWeek => query - .filter(person::published.gt(now - 1.weeks())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopDay => query - .filter(person::published.gt(now - 1.days())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopHour => query - .filter(person::published.gt(now - 1.hours())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopSixHour => query - .filter(person::published.gt(now - 6.hours())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopTwelveHour => query - .filter(person::published.gt(now - 12.hours())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopThreeMonths => query - .filter(person::published.gt(now - 3.months())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopSixMonths => query - .filter(person::published.gt(now - 6.months())) - .order_by(person_aggregates::comment_score.desc()), - SortType::TopNineMonths => query - .filter(person::published.gt(now - 9.months())) - .order_by(person_aggregates::comment_score.desc()), + query = match self.sort.unwrap_or(PersonSortType::CommentScore) { + PersonSortType::New => query.order_by(person::published.desc()), + PersonSortType::Old => query.order_by(person::published.asc()), + PersonSortType::MostComments => query.order_by(person_aggregates::comment_count.desc()), + PersonSortType::CommentScore => query.order_by(person_aggregates::comment_score.desc()), + PersonSortType::PostScore => query.order_by(person_aggregates::post_score.desc()), + PersonSortType::PostCount => query.order_by(person_aggregates::post_count.desc()), }; let (limit, offset) = limit_and_offset(self.page, self.limit)?; diff --git a/crates/db_views_actor/src/structs.rs b/crates/db_views_actor/src/structs.rs index 74d3fbe89..35391776b 100644 --- a/crates/db_views_actor/src/structs.rs +++ b/crates/db_views_actor/src/structs.rs @@ -70,7 +70,7 @@ pub struct PersonBlockView { } #[skip_serializing_none] -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A person mention view. @@ -90,7 +90,7 @@ pub struct PersonMentionView { } #[skip_serializing_none] -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A comment reply view. diff --git a/migrations/2023-07-26-000217_create_controversial_indexes/down.sql b/migrations/2023-07-26-000217_create_controversial_indexes/down.sql new file mode 100644 index 000000000..a355546d1 --- /dev/null +++ b/migrations/2023-07-26-000217_create_controversial_indexes/down.sql @@ -0,0 +1,63 @@ +-- Update comment_aggregates_score trigger function to exclude controversy_rank update +create or replace function comment_aggregates_score() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'INSERT') THEN + update comment_aggregates ca + set score = score + NEW.score, + upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end, + downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end + where ca.comment_id = NEW.comment_id; + + ELSIF (TG_OP = 'DELETE') THEN + -- Join to comment because that comment may not exist anymore + update comment_aggregates ca + set score = score - OLD.score, + upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end, + downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end + from comment c + where ca.comment_id = c.id + and ca.comment_id = OLD.comment_id; + + END IF; + return null; +end $$; + +-- Update post_aggregates_score trigger function to exclude controversy_rank update +create or replace function post_aggregates_score() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'INSERT') THEN + update post_aggregates pa + set score = score + NEW.score, + upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end, + downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end + where pa.post_id = NEW.post_id; + + ELSIF (TG_OP = 'DELETE') THEN + -- Join to post because that post may not exist anymore + update post_aggregates pa + set score = score - OLD.score, + upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end, + downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end + from post p + where pa.post_id = p.id + and pa.post_id = OLD.post_id; + END IF; + return null; +end $$; + +-- Drop the indexes +drop index if exists idx_post_aggregates_featured_local_controversy; +drop index if exists idx_post_aggregates_featured_community_controversy; +drop index if exists idx_comment_aggregates_controversy; + +-- Remove the added columns from the tables +alter table post_aggregates drop column controversy_rank; +alter table comment_aggregates drop column controversy_rank; + +-- Remove function +drop function controversy_rank(numeric, numeric); + diff --git a/migrations/2023-07-26-000217_create_controversial_indexes/up.sql b/migrations/2023-07-26-000217_create_controversial_indexes/up.sql new file mode 100644 index 000000000..f50a706bc --- /dev/null +++ b/migrations/2023-07-26-000217_create_controversial_indexes/up.sql @@ -0,0 +1,97 @@ +-- Need to add immutable to the controversy_rank function in order to index by it + +-- Controversy Rank: +-- if downvotes <= 0 or upvotes <= 0: +-- 0 +-- else: +-- (upvotes + downvotes) * min(upvotes, downvotes) / max(upvotes, downvotes) +create or replace function controversy_rank(upvotes numeric, downvotes numeric) +returns float as $$ +begin + if downvotes <= 0 or upvotes <= 0 then + return 0; + else + return (upvotes + downvotes) * + case when upvotes > downvotes + then downvotes::float / upvotes::float + else upvotes::float / downvotes::float + end; + end if; +end; $$ +LANGUAGE plpgsql +IMMUTABLE; + +-- Aggregates +alter table post_aggregates add column controversy_rank float not null default 0; +alter table comment_aggregates add column controversy_rank float not null default 0; + +-- Populate them initially +-- Note: After initial population, these are updated with vote triggers +update post_aggregates set controversy_rank = controversy_rank(upvotes::numeric, downvotes::numeric); +update comment_aggregates set controversy_rank = controversy_rank(upvotes::numeric, downvotes::numeric); + +-- Create single column indexes +create index idx_post_aggregates_featured_local_controversy on post_aggregates (featured_local desc, controversy_rank desc); +create index idx_post_aggregates_featured_community_controversy on post_aggregates (featured_community desc, controversy_rank desc); +create index idx_comment_aggregates_controversy on comment_aggregates (controversy_rank desc); + +-- Update post_aggregates_score trigger function to include controversy_rank update +create or replace function post_aggregates_score() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'INSERT') THEN + update post_aggregates pa + set score = score + NEW.score, + upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end, + downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end, + controversy_rank = controversy_rank(pa.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, + pa.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric) + where pa.post_id = NEW.post_id; + + ELSIF (TG_OP = 'DELETE') THEN + -- Join to post because that post may not exist anymore + update post_aggregates pa + set score = score - OLD.score, + upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end, + downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end, + controversy_rank = controversy_rank(pa.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, + pa.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric) + from post p + where pa.post_id = p.id + and pa.post_id = OLD.post_id; + + END IF; + return null; +end $$; + +-- Update comment_aggregates_score trigger function to include controversy_rank update +create or replace function comment_aggregates_score() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'INSERT') THEN + update comment_aggregates ca + set score = score + NEW.score, + upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end, + downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end, + controversy_rank = controversy_rank(ca.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, + ca.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric) + where ca.comment_id = NEW.comment_id; + + ELSIF (TG_OP = 'DELETE') THEN + -- Join to comment because that comment may not exist anymore + update comment_aggregates ca + set score = score - OLD.score, + upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end, + downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end, + controversy_rank = controversy_rank(ca.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, + ca.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric) + from comment c + where ca.comment_id = c.id + and ca.comment_id = OLD.comment_id; + + END IF; + return null; +end $$; + From 2de994797e4fe8f569c903de35da55ccdf823fb8 Mon Sep 17 00:00:00 2001 From: biosfood Date: Wed, 26 Jul 2023 19:51:11 +0200 Subject: [PATCH 11/27] Add moderator view parameter to list posts (#3176) * add option to only show posts from moderated communities * rename moderated_only to moderator_view and show blocked users in moderator view * add test for moderator view * bump lemmy-js-client version for moderation view tests * fix yarn lockfile * retry build * Delete logfile * retry checks * remove unused select statement from list posts --------- Co-authored-by: Dessalines --- api_tests/src/community.spec.ts | 46 +++++++++++++++ api_tests/src/shared.ts | 13 ++++ api_tests/yarn.lock | 98 +++++++++++++++++-------------- crates/api_common/src/post.rs | 1 + crates/apub/src/api/list_posts.rs | 3 + crates/db_views/src/post_view.rs | 24 +++++++- 6 files changed, 141 insertions(+), 44 deletions(-) diff --git a/api_tests/src/community.spec.ts b/api_tests/src/community.spec.ts index a5a202ace..fd09f8f56 100644 --- a/api_tests/src/community.spec.ts +++ b/api_tests/src/community.spec.ts @@ -18,6 +18,9 @@ import { createPost, getPost, resolvePost, + registerUser, + API, + getPosts, } from "./shared"; beforeAll(async () => { @@ -233,3 +236,46 @@ test("Admin actions in remote community are not federated to origin", async () = let gammaPost2 = await getPost(gamma, gammaPost.post.id); expect(gammaPost2.post_view.creator_banned_from_community).toBe(false); }); + +test("moderator view", async () => { + // register a new user with their own community on alpha and post to it + let otherUser: API = { + auth: (await registerUser(alpha)).jwt ?? "", + client: alpha.client, + }; + expect(otherUser.auth).not.toBe(""); + let otherCommunity = (await createCommunity(otherUser)).community_view; + expect(otherCommunity.community.name).toBeDefined(); + let otherPost = (await createPost(otherUser, otherCommunity.community.id)) + .post_view; + expect(otherPost.post.id).toBeDefined(); + + // create a community and post on alpha + let alphaCommunity = (await createCommunity(alpha)).community_view; + expect(alphaCommunity.community.name).toBeDefined(); + let alphaPost = (await createPost(alpha, alphaCommunity.community.id)) + .post_view; + expect(alphaPost.post.id).toBeDefined(); + + // other user also posts on alpha's community + let otherAlphaPost = ( + await createPost(otherUser, alphaCommunity.community.id) + ).post_view; + expect(otherAlphaPost.post.id).toBeDefined(); + + // alpha lists posts on home page, should contain all posts that were made + let posts = (await getPosts(alpha)).posts; + expect(posts).toBeDefined(); + let postIds = posts.map(post => post.post.id); + expect(postIds).toContain(otherPost.post.id); + expect(postIds).toContain(alphaPost.post.id); + expect(postIds).toContain(otherAlphaPost.post.id); + + // in moderator view, alpha should not see otherPost, wich was posted on a community alpha doesn't moderate + posts = (await getPosts(alpha, true)).posts; + expect(posts).toBeDefined(); + postIds = posts.map(post => post.post.id); + expect(postIds).not.toContain(otherPost.post.id); + expect(postIds).toContain(alphaPost.post.id); + expect(postIds).toContain(otherAlphaPost.post.id); +}); diff --git a/api_tests/src/shared.ts b/api_tests/src/shared.ts index e4306c94c..f873e78c1 100644 --- a/api_tests/src/shared.ts +++ b/api_tests/src/shared.ts @@ -58,6 +58,8 @@ import { CommentReportResponse } from "lemmy-js-client/dist/types/CommentReportR import { CreateCommentReport } from "lemmy-js-client/dist/types/CreateCommentReport"; import { ListCommentReportsResponse } from "lemmy-js-client/dist/types/ListCommentReportsResponse"; import { ListCommentReports } from "lemmy-js-client/dist/types/ListCommentReports"; +import { GetPostsResponse } from "lemmy-js-client/dist/types/GetPostsResponse"; +import { GetPosts } from "lemmy-js-client/dist/types/GetPosts"; import { GetPersonDetailsResponse } from "lemmy-js-client/dist/types/GetPersonDetailsResponse"; import { GetPersonDetails } from "lemmy-js-client/dist/types/GetPersonDetails"; @@ -757,6 +759,17 @@ export async function listCommentReports( return api.client.listCommentReports(form); } +export function getPosts( + api: API, + moderator_view = false, +): Promise { + let form: GetPosts = { + moderator_view, + auth: api.auth, + }; + return api.client.getPosts(form); +} + export function delay(millis = 500) { return new Promise(resolve => setTimeout(resolve, millis)); } diff --git a/api_tests/yarn.lock b/api_tests/yarn.lock index 30f13014b..275dcd067 100644 --- a/api_tests/yarn.lock +++ b/api_tests/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@aashutoshrathi/word-wrap@^1.2.3": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" + integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== + "@ampproject/remapping@^2.2.0": version "2.2.1" resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" @@ -496,6 +501,13 @@ dependencies: "@sinclair/typebox" "^0.25.16" +"@jest/schemas@^29.6.0": + version "29.6.0" + resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.0.tgz#0f4cb2c8e3dca80c135507ba5635a4fd755b0040" + integrity sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ== + dependencies: + "@sinclair/typebox" "^0.27.8" + "@jest/source-map@^29.4.3": version "29.4.3" resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-29.4.3.tgz#ff8d05cbfff875d4a791ab679b4333df47951d20" @@ -621,6 +633,11 @@ resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.25.24.tgz#8c7688559979f7079aacaf31aa881c3aa410b718" integrity sha512-XJfwUVUKDHF5ugKwIcxEgc9k8b7HbznCp6eUfWgu710hMPNIO4aw4/zB5RogDQz8nd6gyCDpU9O/m6qYEWY6yQ== +"@sinclair/typebox@^0.27.8": + version "0.27.8" + resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" + integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== + "@sinonjs/commons@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-2.0.0.tgz#fd4ca5b063554307e8327b4564bd56d3b73924a3" @@ -1131,11 +1148,11 @@ convert-source-map@^2.0.0: integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== cross-fetch@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" - integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== + version "3.1.8" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82" + integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg== dependencies: - node-fetch "2.6.7" + node-fetch "^2.6.12" cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" @@ -2297,10 +2314,10 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== -node-fetch@2.6.7: - version "2.6.7" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" - integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== +node-fetch@^2.6.12: + version "2.6.12" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba" + integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g== dependencies: whatwg-url "^5.0.0" @@ -2310,9 +2327,9 @@ node-int64@^0.4.0: integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== node-releases@^2.0.8: - version "2.0.10" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f" - integrity sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w== + version "2.0.13" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.13.tgz#d5ed1627c23e3461e819b02e57b75e4899b1c81d" + integrity sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ== normalize-path@^3.0.0: version "3.0.0" @@ -2341,16 +2358,16 @@ onetime@^5.1.2: mimic-fn "^2.1.0" optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== + version "0.9.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" + integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== dependencies: + "@aashutoshrathi/word-wrap" "^1.2.3" deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" - word-wrap "^1.2.3" p-limit@^2.2.0: version "2.3.0" @@ -2438,9 +2455,9 @@ picomatch@^2.0.4, picomatch@^2.2.3, picomatch@^2.3.1: integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== pirates@^4.0.4: - version "4.0.5" - resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b" - integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ== + version "4.0.6" + resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" + integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== pkg-dir@^4.2.0: version "4.2.0" @@ -2467,11 +2484,11 @@ prettier@^3.0.0: integrity sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g== pretty-format@^29.0.0, pretty-format@^29.5.0: - version "29.5.0" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.5.0.tgz#283134e74f70e2e3e7229336de0e4fce94ccde5a" - integrity sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw== + version "29.6.1" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.6.1.tgz#ec838c288850b7c4f9090b867c2d4f4edbfb0f3e" + integrity sha512-7jRj+yXO0W7e4/tSJKoR7HRIHLPPjtNaUGG2xxKQnGvPNRkgWcQ0AZX6P4KBRJN4FcTBWb3sa7DVUJmocYuoog== dependencies: - "@jest/schemas" "^29.4.3" + "@jest/schemas" "^29.6.0" ansi-styles "^5.0.0" react-is "^18.0.0" @@ -2558,18 +2575,18 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -semver@7.x, semver@^7.3.5, semver@^7.3.7: - version "7.5.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.0.tgz#ed8c5dc8efb6c629c88b23d41dc9bf40c1d96cd0" - integrity sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA== +semver@^6.0.0, semver@^6.3.0: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.3.5, semver@^7.3.7, semver@^7.5.3: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" @@ -2724,9 +2741,9 @@ tr46@~0.0.3: integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== ts-jest@^29.1.0: - version "29.1.0" - resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.0.tgz#4a9db4104a49b76d2b368ea775b6c9535c603891" - integrity sha512-ZhNr7Z4PcYa+JjMl62ir+zPiNJfXJN6E8hSLnaUKhOgqcn8vb3e537cpkd0FuAfRK3sR1LSqM1MOhliXNgOFPA== + version "29.1.1" + resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.1.tgz#f58fe62c63caf7bfcc5cc6472082f79180f0815b" + integrity sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA== dependencies: bs-logger "0.x" fast-json-stable-stringify "2.x" @@ -2734,7 +2751,7 @@ ts-jest@^29.1.0: json5 "^2.2.3" lodash.memoize "4.x" make-error "1.x" - semver "7.x" + semver "^7.5.3" yargs-parser "^21.0.1" tslib@^1.8.1: @@ -2772,9 +2789,9 @@ type-fest@^0.21.3: integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== typescript@^5.0.4: - version "5.0.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.4.tgz#b217fd20119bd61a94d4011274e0ab369058da3b" - integrity sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw== + version "5.1.6" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.1.6.tgz#02f8ac202b6dad2c0dd5e0913745b47a37998274" + integrity sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA== update-browserslist-db@^1.0.10: version "1.0.11" @@ -2827,11 +2844,6 @@ which@^2.0.1: dependencies: isexe "^2.0.0" -word-wrap@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== - wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" diff --git a/crates/api_common/src/post.rs b/crates/api_common/src/post.rs index 962c4bb92..01a7db9ce 100644 --- a/crates/api_common/src/post.rs +++ b/crates/api_common/src/post.rs @@ -75,6 +75,7 @@ pub struct GetPosts { pub community_id: Option, pub community_name: Option, pub saved_only: Option, + pub moderator_view: Option, pub auth: Option>, } diff --git a/crates/apub/src/api/list_posts.rs b/crates/apub/src/api/list_posts.rs index 2ebd6b766..2635e149e 100644 --- a/crates/apub/src/api/list_posts.rs +++ b/crates/apub/src/api/list_posts.rs @@ -36,6 +36,8 @@ pub async fn list_posts( }; let saved_only = data.saved_only; + let moderator_view = data.moderator_view; + let listing_type = Some(listing_type_with_default( data.type_, &local_site, @@ -48,6 +50,7 @@ pub async fn list_posts( sort, community_id, saved_only, + moderator_view, page, limit, ..Default::default() diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index e0f481edc..e02310928 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -21,6 +21,7 @@ use lemmy_db_schema::{ community, community_block, community_follower, + community_moderator, community_person_ban, local_user_language, person, @@ -91,6 +92,13 @@ impl PostView { .and(community_follower::person_id.eq(person_id_join)), ), ) + .left_join( + community_moderator::table.on( + post::community_id + .eq(community_moderator::community_id) + .and(community_moderator::person_id.eq(person_id_join)), + ), + ) .left_join( post_saved::table.on( post_aggregates::post_id @@ -210,6 +218,7 @@ pub struct PostQuery<'a> { pub search_term: Option, pub url_search: Option, pub saved_only: Option, + pub moderator_view: Option, pub is_profile_view: Option, pub page: Option, pub limit: Option, @@ -244,6 +253,13 @@ impl<'a> PostQuery<'a> { .and(community_follower::person_id.eq(person_id_join)), ), ) + .left_join( + community_moderator::table.on( + post::community_id + .eq(community_moderator::community_id) + .and(community_moderator::person_id.eq(person_id_join)), + ), + ) .left_join( post_saved::table.on( post_aggregates::post_id @@ -396,6 +412,10 @@ impl<'a> PostQuery<'a> { if self.saved_only.unwrap_or(false) { query = query.filter(post_saved::post_id.is_not_null()); } + + if self.moderator_view.unwrap_or(false) { + query = query.filter(community_moderator::person_id.is_not_null()); + } // Only hide the read posts, if the saved_only is false. Otherwise ppl with the hide_read // setting wont be able to see saved posts. else if !self @@ -412,7 +432,9 @@ impl<'a> PostQuery<'a> { // Don't show blocked communities or persons query = query.filter(community_block::person_id.is_null()); - query = query.filter(person_block::person_id.is_null()); + if !self.moderator_view.unwrap_or(false) { + query = query.filter(person_block::person_id.is_null()); + } } query = match self.sort.unwrap_or(SortType::Hot) { From 3471f3533cb724b2cf6953d563aadfcc9f66c1d2 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 26 Jul 2023 20:01:15 +0200 Subject: [PATCH 12/27] Sanitize html (#3708) * HTML sanitization in apub code * Sanitize API inputs * fmt * Dont allow html a, img tags --------- Co-authored-by: Dessalines --- Cargo.lock | 20 ++++++ api_tests/src/post.spec.ts | 19 +++++ crates/api/src/comment_report/create.rs | 13 ++-- crates/api/src/community/ban.rs | 9 ++- crates/api/src/community/hide.rs | 4 +- crates/api/src/lib.rs | 2 +- crates/api/src/local_user/ban_person.rs | 4 +- crates/api/src/local_user/save_settings.rs | 16 +++-- crates/api/src/post_report/create.rs | 13 ++-- .../api/src/private_message_report/create.rs | 8 +-- crates/api/src/site/purge/comment.rs | 4 +- crates/api/src/site/purge/community.rs | 4 +- crates/api/src/site/purge/person.rs | 4 +- crates/api/src/site/purge/post.rs | 4 +- .../site/registration_applications/approve.rs | 2 +- crates/api_common/Cargo.toml | 2 + crates/api_common/src/utils.rs | 71 ++++++++++++------- crates/api_crud/src/comment/create.rs | 10 +-- crates/api_crud/src/comment/update.rs | 15 ++-- crates/api_crud/src/community/create.rs | 18 +++-- crates/api_crud/src/community/update.rs | 15 ++-- crates/api_crud/src/custom_emoji/create.rs | 12 ++-- crates/api_crud/src/custom_emoji/update.rs | 9 ++- crates/api_crud/src/post/create.rs | 11 ++- crates/api_crud/src/post/update.rs | 16 ++++- crates/api_crud/src/private_message/create.rs | 13 ++-- crates/api_crud/src/private_message/update.rs | 9 +-- crates/api_crud/src/site/create.rs | 24 +++++-- crates/api_crud/src/site/update.rs | 29 +++++--- crates/api_crud/src/user/create.rs | 4 +- .../apub/src/activities/block/block_user.rs | 6 +- .../src/activities/block/undo_block_user.rs | 6 +- .../apub/src/activities/community/report.rs | 6 +- crates/apub/src/activities/deletion/delete.rs | 4 +- crates/apub/src/objects/comment.rs | 10 ++- crates/apub/src/objects/instance.rs | 13 +++- crates/apub/src/objects/person.rs | 13 ++-- crates/apub/src/objects/post.rs | 12 +++- crates/apub/src/objects/private_message.rs | 10 ++- crates/apub/src/protocol/objects/group.rs | 16 +++-- crates/db_schema/src/utils.rs | 10 +-- 41 files changed, 339 insertions(+), 151 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0597cfeb5..6785fe50b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -386,6 +386,19 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "ammonia" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170" +dependencies = [ + "html5ever 0.26.0", + "maplit", + "once_cell", + "tendril", + "url", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -2574,6 +2587,7 @@ version = "0.18.1" dependencies = [ "activitypub_federation", "actix-web", + "ammonia", "anyhow", "chrono", "encoding", @@ -2988,6 +3002,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markdown-it" version = "0.5.1" diff --git a/api_tests/src/post.spec.ts b/api_tests/src/post.spec.ts index 532841b13..42173dba8 100644 --- a/api_tests/src/post.spec.ts +++ b/api_tests/src/post.spec.ts @@ -36,6 +36,7 @@ import { resolveCommunity, } from "./shared"; import { PostView } from "lemmy-js-client/dist/types/PostView"; +import { CreatePost } from "lemmy-js-client/dist/types/CreatePost"; let betaCommunity: CommunityView | undefined; @@ -504,3 +505,21 @@ test("Report a post", async () => { expect(betaReport.original_post_body).toBe(alphaReport.original_post_body); expect(betaReport.reason).toBe(alphaReport.reason); }); + +test("Sanitize HTML", async () => { + let betaCommunity = (await resolveBetaCommunity(beta)).community; + if (!betaCommunity) { + throw "Missing beta community"; + } + + let name = randomString(5); + let body = " hello"; + let form: CreatePost = { + name, + body, + auth: beta.auth, + community_id: betaCommunity.community.id, + }; + let post = await beta.client.createPost(form); + expect(post.post_view.post.body).toBe(" hello"); +}); diff --git a/crates/api/src/comment_report/create.rs b/crates/api/src/comment_report/create.rs index 3a89e1014..190e47a1e 100644 --- a/crates/api/src/comment_report/create.rs +++ b/crates/api/src/comment_report/create.rs @@ -3,7 +3,12 @@ use actix_web::web::Data; use lemmy_api_common::{ comment::{CommentReportResponse, CreateCommentReport}, context::LemmyContext, - utils::{check_community_ban, local_user_view_from_jwt, send_new_report_email_to_admins}, + utils::{ + check_community_ban, + local_user_view_from_jwt, + sanitize_html, + send_new_report_email_to_admins, + }, }; use lemmy_db_schema::{ source::{ @@ -29,8 +34,8 @@ impl Perform for CreateCommentReport { let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let local_site = LocalSite::read(&mut context.pool()).await?; - let reason = self.reason.trim(); - check_report_reason(reason, &local_site)?; + let reason = sanitize_html(self.reason.trim()); + check_report_reason(&reason, &local_site)?; let person_id = local_user_view.person.id; let comment_id = data.comment_id; @@ -42,7 +47,7 @@ impl Perform for CreateCommentReport { creator_id: person_id, comment_id, original_comment_text: comment_view.comment.content, - reason: reason.to_owned(), + reason, }; let report = CommentReport::report(&mut context.pool(), &report_form) diff --git a/crates/api/src/community/ban.rs b/crates/api/src/community/ban.rs index 33f6ef833..95c2bbc04 100644 --- a/crates/api/src/community/ban.rs +++ b/crates/api/src/community/ban.rs @@ -3,7 +3,12 @@ use actix_web::web::Data; use lemmy_api_common::{ community::{BanFromCommunity, BanFromCommunityResponse}, context::LemmyContext, - utils::{is_mod_or_admin, local_user_view_from_jwt, remove_user_data_in_community}, + utils::{ + is_mod_or_admin, + local_user_view_from_jwt, + remove_user_data_in_community, + sanitize_html_opt, + }, }; use lemmy_db_schema::{ source::{ @@ -81,7 +86,7 @@ impl Perform for BanFromCommunity { mod_person_id: local_user_view.person.id, other_person_id: data.person_id, community_id: data.community_id, - reason: data.reason.clone(), + reason: sanitize_html_opt(&data.reason), banned: Some(data.ban), expires, }; diff --git a/crates/api/src/community/hide.rs b/crates/api/src/community/hide.rs index 313e3d84a..4c05a71cf 100644 --- a/crates/api/src/community/hide.rs +++ b/crates/api/src/community/hide.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ build_response::build_community_response, community::{CommunityResponse, HideCommunity}, context::LemmyContext, - utils::{is_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -34,7 +34,7 @@ impl Perform for HideCommunity { let mod_hide_community_form = ModHideCommunityForm { community_id: data.community_id, mod_person_id: local_user_view.person.id, - reason: data.reason.clone(), + reason: sanitize_html_opt(&data.reason), hidden: Some(data.hidden), }; diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 9d3cf211c..b297f503f 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -60,7 +60,7 @@ pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> Result Result<(), LemmyError> { let slur_regex = &local_site_to_slur_regex(local_site); diff --git a/crates/api/src/local_user/ban_person.rs b/crates/api/src/local_user/ban_person.rs index 83768bc23..77e8e8056 100644 --- a/crates/api/src/local_user/ban_person.rs +++ b/crates/api/src/local_user/ban_person.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, person::{BanPerson, BanPersonResponse}, - utils::{is_admin, local_user_view_from_jwt, remove_user_data}, + utils::{is_admin, local_user_view_from_jwt, remove_user_data, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -63,7 +63,7 @@ impl Perform for BanPerson { let form = ModBanForm { mod_person_id: local_user_view.person.id, other_person_id: data.person_id, - reason: data.reason.clone(), + reason: sanitize_html_opt(&data.reason), banned: Some(data.ban), expires, }; diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index c5038eb79..152c11ad1 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, person::{LoginResponse, SaveUserSettings}, - utils::{local_user_view_from_jwt, send_verification_email}, + utils::{local_user_view_from_jwt, sanitize_html_opt, send_verification_email}, }; use lemmy_db_schema::{ source::{ @@ -37,13 +37,16 @@ impl Perform for SaveUserSettings { let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let site_view = SiteView::read_local(&mut context.pool()).await?; + let bio = sanitize_html_opt(&data.bio); + let display_name = sanitize_html_opt(&data.display_name); + let avatar = diesel_option_overwrite_to_url(&data.avatar)?; let banner = diesel_option_overwrite_to_url(&data.banner)?; - let bio = diesel_option_overwrite(&data.bio); - let display_name = diesel_option_overwrite(&data.display_name); - let matrix_user_id = diesel_option_overwrite(&data.matrix_user_id); + let bio = diesel_option_overwrite(bio); + let display_name = diesel_option_overwrite(display_name); + let matrix_user_id = diesel_option_overwrite(data.matrix_user_id.clone()); let email_deref = data.email.as_deref().map(str::to_lowercase); - let email = diesel_option_overwrite(&email_deref); + let email = diesel_option_overwrite(email_deref.clone()); if let Some(Some(email)) = &email { let previous_email = local_user_view.local_user.email.clone().unwrap_or_default(); @@ -85,6 +88,7 @@ impl Perform for SaveUserSettings { let person_id = local_user_view.person.id; let default_listing_type = data.default_listing_type; let default_sort_type = data.default_sort_type; + let theme = sanitize_html_opt(&data.theme); let person_form = PersonUpdateForm::builder() .display_name(display_name) @@ -130,7 +134,7 @@ impl Perform for SaveUserSettings { .show_scores(data.show_scores) .default_sort_type(default_sort_type) .default_listing_type(default_listing_type) - .theme(data.theme.clone()) + .theme(theme) .interface_language(data.interface_language.clone()) .totp_2fa_secret(totp_2fa_secret) .totp_2fa_url(totp_2fa_url) diff --git a/crates/api/src/post_report/create.rs b/crates/api/src/post_report/create.rs index 16c994d3b..a4081015c 100644 --- a/crates/api/src/post_report/create.rs +++ b/crates/api/src/post_report/create.rs @@ -3,7 +3,12 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, post::{CreatePostReport, PostReportResponse}, - utils::{check_community_ban, local_user_view_from_jwt, send_new_report_email_to_admins}, + utils::{ + check_community_ban, + local_user_view_from_jwt, + sanitize_html, + send_new_report_email_to_admins, + }, }; use lemmy_db_schema::{ source::{ @@ -26,8 +31,8 @@ impl Perform for CreatePostReport { let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let local_site = LocalSite::read(&mut context.pool()).await?; - let reason = self.reason.trim(); - check_report_reason(reason, &local_site)?; + let reason = sanitize_html(self.reason.trim()); + check_report_reason(&reason, &local_site)?; let person_id = local_user_view.person.id; let post_id = data.post_id; @@ -41,7 +46,7 @@ impl Perform for CreatePostReport { original_post_name: post_view.post.name, original_post_url: post_view.post.url, original_post_body: post_view.post.body, - reason: reason.to_owned(), + reason, }; let report = PostReport::report(&mut context.pool(), &report_form) diff --git a/crates/api/src/private_message_report/create.rs b/crates/api/src/private_message_report/create.rs index 88511bcf7..4ca1d7cd6 100644 --- a/crates/api/src/private_message_report/create.rs +++ b/crates/api/src/private_message_report/create.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, private_message::{CreatePrivateMessageReport, PrivateMessageReportResponse}, - utils::{local_user_view_from_jwt, send_new_report_email_to_admins}, + utils::{local_user_view_from_jwt, sanitize_html, send_new_report_email_to_admins}, }; use lemmy_db_schema::{ source::{ @@ -25,8 +25,8 @@ impl Perform for CreatePrivateMessageReport { let local_user_view = local_user_view_from_jwt(&self.auth, context).await?; let local_site = LocalSite::read(&mut context.pool()).await?; - let reason = self.reason.trim(); - check_report_reason(reason, &local_site)?; + let reason = sanitize_html(self.reason.trim()); + check_report_reason(&reason, &local_site)?; let person_id = local_user_view.person.id; let private_message_id = self.private_message_id; @@ -36,7 +36,7 @@ impl Perform for CreatePrivateMessageReport { creator_id: person_id, private_message_id, original_pm_text: private_message.content, - reason: reason.to_owned(), + reason: reason.clone(), }; let report = PrivateMessageReport::report(&mut context.pool(), &report_form) diff --git a/crates/api/src/site/purge/comment.rs b/crates/api/src/site/purge/comment.rs index 9334961e9..bfaf9cbb0 100644 --- a/crates/api/src/site/purge/comment.rs +++ b/crates/api/src/site/purge/comment.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, site::{PurgeComment, PurgeItemResponse}, - utils::{is_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -38,7 +38,7 @@ impl Perform for PurgeComment { Comment::delete(&mut context.pool(), comment_id).await?; // Mod tables - let reason = data.reason.clone(); + let reason = sanitize_html_opt(&data.reason); let form = AdminPurgeCommentForm { admin_person_id: local_user_view.person.id, reason, diff --git a/crates/api/src/site/purge/community.rs b/crates/api/src/site/purge/community.rs index 56e757176..bd8d9d386 100644 --- a/crates/api/src/site/purge/community.rs +++ b/crates/api/src/site/purge/community.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ context::LemmyContext, request::purge_image_from_pictrs, site::{PurgeCommunity, PurgeItemResponse}, - utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_community}, + utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_community, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -55,7 +55,7 @@ impl Perform for PurgeCommunity { Community::delete(&mut context.pool(), community_id).await?; // Mod tables - let reason = data.reason.clone(); + let reason = sanitize_html_opt(&data.reason); let form = AdminPurgeCommunityForm { admin_person_id: local_user_view.person.id, reason, diff --git a/crates/api/src/site/purge/person.rs b/crates/api/src/site/purge/person.rs index fa884147f..838b36070 100644 --- a/crates/api/src/site/purge/person.rs +++ b/crates/api/src/site/purge/person.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ context::LemmyContext, request::purge_image_from_pictrs, site::{PurgeItemResponse, PurgePerson}, - utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_person}, + utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_person, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -54,7 +54,7 @@ impl Perform for PurgePerson { Person::delete(&mut context.pool(), person_id).await?; // Mod tables - let reason = data.reason.clone(); + let reason = sanitize_html_opt(&data.reason); let form = AdminPurgePersonForm { admin_person_id: local_user_view.person.id, reason, diff --git a/crates/api/src/site/purge/post.rs b/crates/api/src/site/purge/post.rs index 6824e408b..ee0a3af09 100644 --- a/crates/api/src/site/purge/post.rs +++ b/crates/api/src/site/purge/post.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ context::LemmyContext, request::purge_image_from_pictrs, site::{PurgeItemResponse, PurgePost}, - utils::{is_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -50,7 +50,7 @@ impl Perform for PurgePost { Post::delete(&mut context.pool(), post_id).await?; // Mod tables - let reason = data.reason.clone(); + let reason = sanitize_html_opt(&data.reason); let form = AdminPurgePostForm { admin_person_id: local_user_view.person.id, reason, diff --git a/crates/api/src/site/registration_applications/approve.rs b/crates/api/src/site/registration_applications/approve.rs index 1a8521ca9..227f93243 100644 --- a/crates/api/src/site/registration_applications/approve.rs +++ b/crates/api/src/site/registration_applications/approve.rs @@ -30,7 +30,7 @@ impl Perform for ApproveRegistrationApplication { is_admin(&local_user_view)?; // Update the registration with reason, admin_id - let deny_reason = diesel_option_overwrite(&data.deny_reason); + let deny_reason = diesel_option_overwrite(data.deny_reason.clone()); let app_form = RegistrationApplicationUpdateForm { admin_id: Some(Some(local_user_view.person.id)), deny_reason, diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 8a23a4cb2..d74acd136 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -34,6 +34,7 @@ full = [ "actix-web", "futures", "once_cell", + "ammonia", ] [dependencies] @@ -66,3 +67,4 @@ once_cell = { workspace = true, optional = true } actix-web = { workspace = true, optional = true } # necessary for wasmt compilation getrandom = { version = "0.2.10", features = ["js"] } +ammonia = { version = "3.3.0", optional = true } diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index d259b9e4c..8ccb7d3fe 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -729,31 +729,6 @@ pub async fn delete_user_account( Ok(()) } -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used)] - #![allow(clippy::indexing_slicing)] - - use crate::utils::{honeypot_check, password_length_check}; - - #[test] - #[rustfmt::skip] - fn password_length() { - assert!(password_length_check("Õ¼¾°3yË,o¸ãtÌÈú|ÇÁÙAøüÒI©·¤(T]/ð>æºWæ[C¤bªWöaÃÎñ·{=û³&§½K/c").is_ok()); - assert!(password_length_check("1234567890").is_ok()); - assert!(password_length_check("short").is_err()); - assert!(password_length_check("looooooooooooooooooooooooooooooooooooooooooooooooooooooooooong").is_err()); - } - - #[test] - fn honeypot() { - assert!(honeypot_check(&None).is_ok()); - assert!(honeypot_check(&Some(String::new())).is_ok()); - assert!(honeypot_check(&Some("1".to_string())).is_err()); - assert!(honeypot_check(&Some("message".to_string())).is_err()); - } -} - pub enum EndpointType { Community, Person, @@ -819,3 +794,49 @@ pub fn generate_featured_url(actor_id: &DbUrl) -> Result { pub fn generate_moderators_url(community_id: &DbUrl) -> Result { Ok(Url::parse(&format!("{community_id}/moderators"))?.into()) } + +/// Sanitize HTML with default options. Additionally, dont allow bypassing markdown +/// links and images +pub fn sanitize_html(data: &str) -> String { + ammonia::Builder::default() + .rm_tags(&["a", "img"]) + .clean(data) + .to_string() +} + +pub fn sanitize_html_opt(data: &Option) -> Option { + data.as_ref().map(|d| sanitize_html(d)) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + #![allow(clippy::indexing_slicing)] + + use crate::utils::{honeypot_check, password_length_check, sanitize_html}; + + #[test] + #[rustfmt::skip] + fn password_length() { + assert!(password_length_check("Õ¼¾°3yË,o¸ãtÌÈú|ÇÁÙAøüÒI©·¤(T]/ð>æºWæ[C¤bªWöaÃÎñ·{=û³&§½K/c").is_ok()); + assert!(password_length_check("1234567890").is_ok()); + assert!(password_length_check("short").is_err()); + assert!(password_length_check("looooooooooooooooooooooooooooooooooooooooooooooooooooooooooong").is_err()); + } + + #[test] + fn honeypot() { + assert!(honeypot_check(&None).is_ok()); + assert!(honeypot_check(&Some(String::new())).is_ok()); + assert!(honeypot_check(&Some("1".to_string())).is_err()); + assert!(honeypot_check(&Some("message".to_string())).is_err()); + } + + #[test] + fn test_sanitize_html() { + let sanitized = sanitize_html(" hello"); + assert_eq!(sanitized, " hello"); + let sanitized = sanitize_html(" test"); + assert_eq!(sanitized, " test"); + } +} diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index 098d1a664..4a7513a4b 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -12,6 +12,7 @@ use lemmy_api_common::{ get_post, local_site_to_slur_regex, local_user_view_from_jwt, + sanitize_html, EndpointType, }, }; @@ -47,11 +48,12 @@ impl PerformCrud for CreateComment { let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let local_site = LocalSite::read(&mut context.pool()).await?; - let content_slurs_removed = remove_slurs( + let content = remove_slurs( &data.content.clone(), &local_site_to_slur_regex(&local_site), ); - is_valid_body_field(&Some(content_slurs_removed.clone()), false)?; + is_valid_body_field(&Some(content.clone()), false)?; + let content = sanitize_html(&content); // Check for a community ban let post_id = data.post_id; @@ -104,7 +106,7 @@ impl PerformCrud for CreateComment { }; let comment_form = CommentInsertForm::builder() - .content(content_slurs_removed.clone()) + .content(content.clone()) .post_id(data.post_id) .creator_id(local_user_view.person.id) .language_id(language_id) @@ -135,7 +137,7 @@ impl PerformCrud for CreateComment { .with_lemmy_type(LemmyErrorType::CouldntCreateComment)?; // Scan the comment for user mentions, add those rows - let mentions = scrape_text_for_mentions(&content_slurs_removed); + let mentions = scrape_text_for_mentions(&content); let recipient_ids = send_local_notifs( mentions, &updated_comment, diff --git a/crates/api_crud/src/comment/update.rs b/crates/api_crud/src/comment/update.rs index 0129e87c2..558965f62 100644 --- a/crates/api_crud/src/comment/update.rs +++ b/crates/api_crud/src/comment/update.rs @@ -4,7 +4,12 @@ use lemmy_api_common::{ build_response::{build_comment_response, send_local_notifs}, comment::{CommentResponse, EditComment}, context::LemmyContext, - utils::{check_community_ban, local_site_to_slur_regex, local_user_view_from_jwt}, + utils::{ + check_community_ban, + local_site_to_slur_regex, + local_user_view_from_jwt, + sanitize_html_opt, + }, }; use lemmy_db_schema::{ source::{ @@ -59,16 +64,16 @@ impl PerformCrud for EditComment { .await?; // Update the Content - let content_slurs_removed = data + let content = data .content .as_ref() .map(|c| remove_slurs(c, &local_site_to_slur_regex(&local_site))); - - is_valid_body_field(&content_slurs_removed, false)?; + is_valid_body_field(&content, false)?; + let content = sanitize_html_opt(&content); let comment_id = data.comment_id; let form = CommentUpdateForm::builder() - .content(content_slurs_removed) + .content(content) .language_id(data.language_id) .updated(Some(Some(naive_now()))) .build(); diff --git a/crates/api_crud/src/community/create.rs b/crates/api_crud/src/community/create.rs index 77ab833b9..7c84a2150 100644 --- a/crates/api_crud/src/community/create.rs +++ b/crates/api_crud/src/community/create.rs @@ -13,6 +13,8 @@ use lemmy_api_common::{ is_admin, local_site_to_slur_regex, local_user_view_from_jwt, + sanitize_html, + sanitize_html_opt, EndpointType, }, }; @@ -59,10 +61,14 @@ impl PerformCrud for CreateCommunity { let icon = diesel_option_overwrite_to_url_create(&data.icon)?; let banner = diesel_option_overwrite_to_url_create(&data.banner)?; + let name = sanitize_html(&data.name); + let title = sanitize_html(&data.title); + let description = sanitize_html_opt(&data.description); + let slur_regex = local_site_to_slur_regex(&local_site); - check_slurs(&data.name, &slur_regex)?; - check_slurs(&data.title, &slur_regex)?; - check_slurs_opt(&data.description, &slur_regex)?; + check_slurs(&name, &slur_regex)?; + check_slurs(&title, &slur_regex)?; + check_slurs_opt(&description, &slur_regex)?; is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?; is_valid_body_field(&data.description, false)?; @@ -83,9 +89,9 @@ impl PerformCrud for CreateCommunity { let keypair = generate_actor_keypair()?; let community_form = CommunityInsertForm::builder() - .name(data.name.clone()) - .title(data.title.clone()) - .description(data.description.clone()) + .name(name) + .title(title) + .description(description) .icon(icon) .banner(banner) .nsfw(data.nsfw) diff --git a/crates/api_crud/src/community/update.rs b/crates/api_crud/src/community/update.rs index 62c3776f4..128be036f 100644 --- a/crates/api_crud/src/community/update.rs +++ b/crates/api_crud/src/community/update.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ build_response::build_community_response, community::{CommunityResponse, EditCommunity}, context::LemmyContext, - utils::{local_site_to_slur_regex, local_user_view_from_jwt}, + utils::{local_site_to_slur_regex, local_user_view_from_jwt, sanitize_html_opt}, }; use lemmy_db_schema::{ newtypes::PersonId, @@ -32,15 +32,18 @@ impl PerformCrud for EditCommunity { let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let local_site = LocalSite::read(&mut context.pool()).await?; - let icon = diesel_option_overwrite_to_url(&data.icon)?; - let banner = diesel_option_overwrite_to_url(&data.banner)?; - let description = diesel_option_overwrite(&data.description); - let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.title, &slur_regex)?; check_slurs_opt(&data.description, &slur_regex)?; is_valid_body_field(&data.description, false)?; + let title = sanitize_html_opt(&data.title); + let description = sanitize_html_opt(&data.description); + + let icon = diesel_option_overwrite_to_url(&data.icon)?; + let banner = diesel_option_overwrite_to_url(&data.banner)?; + let description = diesel_option_overwrite(description); + // Verify its a mod (only mods can edit it) let community_id = data.community_id; let mods: Vec = @@ -64,7 +67,7 @@ impl PerformCrud for EditCommunity { } let community_form = CommunityUpdateForm::builder() - .title(data.title.clone()) + .title(title) .description(description) .icon(icon) .banner(banner) diff --git a/crates/api_crud/src/custom_emoji/create.rs b/crates/api_crud/src/custom_emoji/create.rs index dcf4fe7f9..93e7114ae 100644 --- a/crates/api_crud/src/custom_emoji/create.rs +++ b/crates/api_crud/src/custom_emoji/create.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, custom_emoji::{CreateCustomEmoji, CustomEmojiResponse}, - utils::{is_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt, sanitize_html}, }; use lemmy_db_schema::source::{ custom_emoji::{CustomEmoji, CustomEmojiInsertForm}, @@ -26,11 +26,15 @@ impl PerformCrud for CreateCustomEmoji { // Make sure user is an admin is_admin(&local_user_view)?; + let shortcode = sanitize_html(data.shortcode.to_lowercase().trim()); + let alt_text = sanitize_html(&data.alt_text); + let category = sanitize_html(&data.category); + let emoji_form = CustomEmojiInsertForm::builder() .local_site_id(local_site.id) - .shortcode(data.shortcode.to_lowercase().trim().to_string()) - .alt_text(data.alt_text.to_string()) - .category(data.category.to_string()) + .shortcode(shortcode) + .alt_text(alt_text) + .category(category) .image_url(data.clone().image_url.into()) .build(); let emoji = CustomEmoji::create(&mut context.pool(), &emoji_form).await?; diff --git a/crates/api_crud/src/custom_emoji/update.rs b/crates/api_crud/src/custom_emoji/update.rs index 7db3a5282..93708c379 100644 --- a/crates/api_crud/src/custom_emoji/update.rs +++ b/crates/api_crud/src/custom_emoji/update.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, custom_emoji::{CustomEmojiResponse, EditCustomEmoji}, - utils::{is_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt, sanitize_html}, }; use lemmy_db_schema::source::{ custom_emoji::{CustomEmoji, CustomEmojiUpdateForm}, @@ -26,10 +26,13 @@ impl PerformCrud for EditCustomEmoji { // Make sure user is an admin is_admin(&local_user_view)?; + let alt_text = sanitize_html(&data.alt_text); + let category = sanitize_html(&data.category); + let emoji_form = CustomEmojiUpdateForm::builder() .local_site_id(local_site.id) - .alt_text(data.alt_text.to_string()) - .category(data.category.to_string()) + .alt_text(alt_text) + .category(category) .image_url(data.clone().image_url.into()) .build(); let emoji = CustomEmoji::update(&mut context.pool(), data.id, &emoji_form).await?; diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 458fdb248..264cdbc82 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -14,6 +14,8 @@ use lemmy_api_common::{ local_site_to_slur_regex, local_user_view_from_jwt, mark_post_as_read, + sanitize_html, + sanitize_html_opt, EndpointType, }, }; @@ -91,6 +93,11 @@ pub async fn create_post( .map(|u| (u.title, u.description, u.embed_video_url)) .unwrap_or_default(); + let name = sanitize_html(data.name.trim()); + let body = sanitize_html_opt(&data.body); + let embed_title = sanitize_html_opt(&embed_title); + let embed_description = sanitize_html_opt(&embed_description); + // Only need to check if language is allowed in case user set it explicitly. When using default // language, it already only returns allowed languages. CommunityLanguage::is_allowed_community_language( @@ -114,9 +121,9 @@ pub async fn create_post( }; let post_form = PostInsertForm::builder() - .name(data.name.trim().to_owned()) + .name(name) .url(url) - .body(data.body.clone()) + .body(body) .community_id(data.community_id) .creator_id(local_user_view.person.id) .nsfw(data.nsfw) diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index fbbadbc61..f3be5f6af 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -5,7 +5,12 @@ use lemmy_api_common::{ context::LemmyContext, post::{EditPost, PostResponse}, request::fetch_site_data, - utils::{check_community_ban, local_site_to_slur_regex, local_user_view_from_jwt}, + utils::{ + check_community_ban, + local_site_to_slur_regex, + local_user_view_from_jwt, + sanitize_html_opt, + }, }; use lemmy_db_schema::{ source::{ @@ -39,7 +44,6 @@ impl PerformCrud for EditPost { // TODO No good way to handle a clear. // Issue link: https://github.com/LemmyNet/lemmy/issues/2287 let url = Some(data_url.map(clean_url_params).map(Into::into)); - let body = diesel_option_overwrite(&data.body); let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.name, &slur_regex)?; @@ -75,6 +79,12 @@ impl PerformCrud for EditPost { .map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url))) .unwrap_or_default(); + let name = sanitize_html_opt(&data.name); + let body = sanitize_html_opt(&data.body); + let body = diesel_option_overwrite(body); + let embed_title = embed_title.map(|e| sanitize_html_opt(&e)); + let embed_description = embed_description.map(|e| sanitize_html_opt(&e)); + let language_id = self.language_id; CommunityLanguage::is_allowed_community_language( &mut context.pool(), @@ -84,7 +94,7 @@ impl PerformCrud for EditPost { .await?; let post_form = PostUpdateForm::builder() - .name(data.name.clone()) + .name(name) .url(url) .body(body) .nsfw(data.nsfw) diff --git a/crates/api_crud/src/private_message/create.rs b/crates/api_crud/src/private_message/create.rs index 48f6bdd23..3b1a625f6 100644 --- a/crates/api_crud/src/private_message/create.rs +++ b/crates/api_crud/src/private_message/create.rs @@ -9,6 +9,7 @@ use lemmy_api_common::{ get_interface_language, local_site_to_slur_regex, local_user_view_from_jwt, + sanitize_html, send_email_to_user, EndpointType, }, @@ -39,11 +40,9 @@ impl PerformCrud for CreatePrivateMessage { let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let local_site = LocalSite::read(&mut context.pool()).await?; - let content_slurs_removed = remove_slurs( - &data.content.clone(), - &local_site_to_slur_regex(&local_site), - ); - is_valid_body_field(&Some(content_slurs_removed.clone()), false)?; + let content = sanitize_html(&data.content); + let content = remove_slurs(&content, &local_site_to_slur_regex(&local_site)); + is_valid_body_field(&Some(content.clone()), false)?; check_person_block( local_user_view.person.id, @@ -53,7 +52,7 @@ impl PerformCrud for CreatePrivateMessage { .await?; let private_message_form = PrivateMessageInsertForm::builder() - .content(content_slurs_removed.clone()) + .content(content.clone()) .creator_id(local_user_view.person.id) .recipient_id(data.recipient_id) .build(); @@ -92,7 +91,7 @@ impl PerformCrud for CreatePrivateMessage { send_email_to_user( &local_recipient, &lang.notification_private_message_subject(sender_name), - &lang.notification_private_message_body(inbox_link, &content_slurs_removed, sender_name), + &lang.notification_private_message_body(inbox_link, &content, sender_name), context.settings(), ) .await; diff --git a/crates/api_crud/src/private_message/update.rs b/crates/api_crud/src/private_message/update.rs index 4abf6f3cc..09b50540d 100644 --- a/crates/api_crud/src/private_message/update.rs +++ b/crates/api_crud/src/private_message/update.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, private_message::{EditPrivateMessage, PrivateMessageResponse}, - utils::{local_site_to_slur_regex, local_user_view_from_jwt}, + utils::{local_site_to_slur_regex, local_user_view_from_jwt, sanitize_html}, }; use lemmy_db_schema::{ source::{ @@ -41,15 +41,16 @@ impl PerformCrud for EditPrivateMessage { } // Doing the update - let content_slurs_removed = remove_slurs(&data.content, &local_site_to_slur_regex(&local_site)); - is_valid_body_field(&Some(content_slurs_removed.clone()), false)?; + let content = sanitize_html(&data.content); + let content = remove_slurs(&content, &local_site_to_slur_regex(&local_site)); + is_valid_body_field(&Some(content.clone()), false)?; let private_message_id = data.private_message_id; PrivateMessage::update( &mut context.pool(), private_message_id, &PrivateMessageUpdateForm::builder() - .content(Some(content_slurs_removed)) + .content(Some(content)) .updated(Some(Some(naive_now()))) .build(), ) diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 540b3c6c1..98d111a1d 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -12,6 +12,8 @@ use lemmy_api_common::{ is_admin, local_site_rate_limit_to_rate_limit_config, local_user_view_from_jwt, + sanitize_html, + sanitize_html_opt, }, }; use lemmy_db_schema::{ @@ -59,10 +61,14 @@ impl PerformCrud for CreateSite { let actor_id: DbUrl = Url::parse(&context.settings().get_protocol_and_hostname())?.into(); let inbox_url = Some(generate_site_inbox_url(&actor_id)?); let keypair = generate_actor_keypair()?; + let name = sanitize_html(&data.name); + let sidebar = sanitize_html_opt(&data.sidebar); + let description = sanitize_html_opt(&data.description); + let site_form = SiteUpdateForm::builder() - .name(Some(data.name.clone())) - .sidebar(diesel_option_overwrite(&data.sidebar)) - .description(diesel_option_overwrite(&data.description)) + .name(Some(name)) + .sidebar(diesel_option_overwrite(sidebar)) + .description(diesel_option_overwrite(description)) .icon(diesel_option_overwrite_to_url(&data.icon)?) .banner(diesel_option_overwrite_to_url(&data.banner)?) .actor_id(Some(actor_id)) @@ -76,6 +82,10 @@ impl PerformCrud for CreateSite { Site::update(&mut context.pool(), site_id, &site_form).await?; + let application_question = sanitize_html_opt(&data.application_question); + let default_theme = sanitize_html_opt(&data.default_theme); + let legal_information = sanitize_html_opt(&data.legal_information); + let local_site_form = LocalSiteUpdateForm::builder() // Set the site setup to true .site_setup(Some(true)) @@ -84,15 +94,15 @@ impl PerformCrud for CreateSite { .enable_nsfw(data.enable_nsfw) .community_creation_admin_only(data.community_creation_admin_only) .require_email_verification(data.require_email_verification) - .application_question(diesel_option_overwrite(&data.application_question)) + .application_question(diesel_option_overwrite(application_question)) .private_instance(data.private_instance) - .default_theme(data.default_theme.clone()) + .default_theme(default_theme) .default_post_listing_type(data.default_post_listing_type) - .legal_information(diesel_option_overwrite(&data.legal_information)) + .legal_information(diesel_option_overwrite(legal_information)) .application_email_admins(data.application_email_admins) .hide_modlog_mod_names(data.hide_modlog_mod_names) .updated(Some(Some(naive_now()))) - .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) + .slur_filter_regex(diesel_option_overwrite(data.slur_filter_regex.clone())) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) .captcha_enabled(data.captcha_enabled) diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index ea3c53aa7..2b8ce4c0f 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -6,7 +6,12 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, site::{EditSite, SiteResponse}, - utils::{is_admin, local_site_rate_limit_to_rate_limit_config, local_user_view_from_jwt}, + utils::{ + is_admin, + local_site_rate_limit_to_rate_limit_config, + local_user_view_from_jwt, + sanitize_html_opt, + }, }; use lemmy_db_schema::{ source::{ @@ -59,10 +64,14 @@ impl PerformCrud for EditSite { SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?; } + let name = sanitize_html_opt(&data.name); + let sidebar = sanitize_html_opt(&data.sidebar); + let description = sanitize_html_opt(&data.description); + let site_form = SiteUpdateForm::builder() - .name(data.name.clone()) - .sidebar(diesel_option_overwrite(&data.sidebar)) - .description(diesel_option_overwrite(&data.description)) + .name(name) + .sidebar(diesel_option_overwrite(sidebar)) + .description(diesel_option_overwrite(description)) .icon(diesel_option_overwrite_to_url(&data.icon)?) .banner(diesel_option_overwrite_to_url(&data.banner)?) .updated(Some(Some(naive_now()))) @@ -74,21 +83,25 @@ impl PerformCrud for EditSite { // Diesel will throw an error for empty update forms .ok(); + let application_question = sanitize_html_opt(&data.application_question); + let default_theme = sanitize_html_opt(&data.default_theme); + let legal_information = sanitize_html_opt(&data.legal_information); + let local_site_form = LocalSiteUpdateForm::builder() .enable_downvotes(data.enable_downvotes) .registration_mode(data.registration_mode) .enable_nsfw(data.enable_nsfw) .community_creation_admin_only(data.community_creation_admin_only) .require_email_verification(data.require_email_verification) - .application_question(diesel_option_overwrite(&data.application_question)) + .application_question(diesel_option_overwrite(application_question)) .private_instance(data.private_instance) - .default_theme(data.default_theme.clone()) + .default_theme(default_theme) .default_post_listing_type(data.default_post_listing_type) - .legal_information(diesel_option_overwrite(&data.legal_information)) + .legal_information(diesel_option_overwrite(legal_information)) .application_email_admins(data.application_email_admins) .hide_modlog_mod_names(data.hide_modlog_mod_names) .updated(Some(Some(naive_now()))) - .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) + .slur_filter_regex(diesel_option_overwrite(data.slur_filter_regex.clone())) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) .captcha_enabled(data.captcha_enabled) diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index caba9bd8a..f2af6940e 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -11,6 +11,7 @@ use lemmy_api_common::{ honeypot_check, local_site_to_slur_regex, password_length_check, + sanitize_html, send_new_applicant_email_to_admins, send_verification_email, EndpointType, @@ -92,6 +93,7 @@ impl PerformCrud for Register { let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.username, &slur_regex)?; check_slurs_opt(&data.answer, &slur_regex)?; + let username = sanitize_html(&data.username); let actor_keypair = generate_actor_keypair()?; is_valid_actor_name(&data.username, local_site.actor_name_max_length as usize)?; @@ -111,7 +113,7 @@ impl PerformCrud for Register { // Register the new person let person_form = PersonInsertForm::builder() - .name(data.username.clone()) + .name(username) .actor_id(Some(actor_id.clone())) .private_key(Some(actor_keypair.private_key)) .public_key(actor_keypair.public_key) diff --git a/crates/apub/src/activities/block/block_user.rs b/crates/apub/src/activities/block/block_user.rs index 55642f862..abfab8456 100644 --- a/crates/apub/src/activities/block/block_user.rs +++ b/crates/apub/src/activities/block/block_user.rs @@ -23,7 +23,7 @@ use anyhow::anyhow; use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, - utils::{remove_user_data, remove_user_data_in_community}, + utils::{remove_user_data, remove_user_data_in_community, sanitize_html_opt}, }; use lemmy_db_schema::{ source::{ @@ -177,7 +177,7 @@ impl ActivityHandler for BlockUser { let form = ModBanForm { mod_person_id: mod_person.id, other_person_id: blocked_person.id, - reason: self.summary, + reason: sanitize_html_opt(&self.summary), banned: Some(true), expires, }; @@ -211,7 +211,7 @@ impl ActivityHandler for BlockUser { mod_person_id: mod_person.id, other_person_id: blocked_person.id, community_id: community.id, - reason: self.summary, + reason: sanitize_html_opt(&self.summary), banned: Some(true), expires, }; diff --git a/crates/apub/src/activities/block/undo_block_user.rs b/crates/apub/src/activities/block/undo_block_user.rs index f68349794..2ebd053ba 100644 --- a/crates/apub/src/activities/block/undo_block_user.rs +++ b/crates/apub/src/activities/block/undo_block_user.rs @@ -17,7 +17,7 @@ use activitypub_federation::{ protocol::verification::verify_domains_match, traits::{ActivityHandler, Actor}, }; -use lemmy_api_common::context::LemmyContext; +use lemmy_api_common::{context::LemmyContext, utils::sanitize_html_opt}; use lemmy_db_schema::{ source::{ community::{CommunityPersonBan, CommunityPersonBanForm}, @@ -116,7 +116,7 @@ impl ActivityHandler for UndoBlockUser { let form = ModBanForm { mod_person_id: mod_person.id, other_person_id: blocked_person.id, - reason: self.object.summary, + reason: sanitize_html_opt(&self.object.summary), banned: Some(false), expires, }; @@ -135,7 +135,7 @@ impl ActivityHandler for UndoBlockUser { mod_person_id: mod_person.id, other_person_id: blocked_person.id, community_id: community.id, - reason: self.object.summary, + reason: sanitize_html_opt(&self.object.summary), banned: Some(false), expires, }; diff --git a/crates/apub/src/activities/community/report.rs b/crates/apub/src/activities/community/report.rs index 67b84644e..22a8c12be 100644 --- a/crates/apub/src/activities/community/report.rs +++ b/crates/apub/src/activities/community/report.rs @@ -16,7 +16,7 @@ use lemmy_api_common::{ comment::{CommentReportResponse, CreateCommentReport}, context::LemmyContext, post::{CreatePostReport, PostReportResponse}, - utils::local_user_view_from_jwt, + utils::{local_user_view_from_jwt, sanitize_html}, }; use lemmy_db_schema::{ source::{ @@ -131,7 +131,7 @@ impl ActivityHandler for Report { post_id: post.id, original_post_name: post.name.clone(), original_post_url: post.url.clone(), - reason: self.summary, + reason: sanitize_html(&self.summary), original_post_body: post.body.clone(), }; PostReport::report(&mut context.pool(), &report_form).await?; @@ -141,7 +141,7 @@ impl ActivityHandler for Report { creator_id: actor.id, comment_id: comment.id, original_comment_text: comment.content.clone(), - reason: self.summary, + reason: sanitize_html(&self.summary), }; CommentReport::report(&mut context.pool(), &report_form).await?; } diff --git a/crates/apub/src/activities/deletion/delete.rs b/crates/apub/src/activities/deletion/delete.rs index fcdede8d7..06f7463ae 100644 --- a/crates/apub/src/activities/deletion/delete.rs +++ b/crates/apub/src/activities/deletion/delete.rs @@ -8,7 +8,7 @@ use crate::{ protocol::{activities::deletion::delete::Delete, IdOrNestedObject}, }; use activitypub_federation::{config::Data, kinds::activity::DeleteType, traits::ActivityHandler}; -use lemmy_api_common::context::LemmyContext; +use lemmy_api_common::{context::LemmyContext, utils::sanitize_html_opt}; use lemmy_db_schema::{ source::{ comment::{Comment, CommentUpdateForm}, @@ -105,6 +105,8 @@ pub(in crate::activities) async fn receive_remove_action( reason: Option, context: &Data, ) -> Result<(), LemmyError> { + let reason = sanitize_html_opt(&reason); + match DeletableObjects::read_from_db(object, context).await? { DeletableObjects::Community(community) => { if community.local { diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index 2954de096..3b05ed394 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -16,7 +16,10 @@ use activitypub_federation::{ traits::Object, }; use chrono::NaiveDateTime; -use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex}; +use lemmy_api_common::{ + context::LemmyContext, + utils::{local_site_opt_to_slur_regex, sanitize_html}, +}; use lemmy_db_schema::{ source::{ comment::{Comment, CommentInsertForm, CommentUpdateForm}, @@ -154,14 +157,15 @@ impl Object for ApubComment { let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); - let content_slurs_removed = remove_slurs(&content, slur_regex); + let content = remove_slurs(&content, slur_regex); + let content = sanitize_html(&content); let language_id = LanguageTag::to_language_id_single(note.language, &mut context.pool()).await?; let form = CommentInsertForm { creator_id: creator.id, post_id: post.id, - content: content_slurs_removed, + content, removed: None, published: note.published.map(|u| u.naive_local()), updated: note.updated.map(|u| u.naive_local()), diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index 7933d4705..52fc210b0 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -16,7 +16,10 @@ use activitypub_federation::{ traits::{Actor, Object}, }; use chrono::NaiveDateTime; -use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex}; +use lemmy_api_common::{ + context::LemmyContext, + utils::{local_site_opt_to_slur_regex, sanitize_html_opt}, +}; use lemmy_db_schema::{ newtypes::InstanceId, source::{ @@ -129,13 +132,17 @@ impl Object for ApubSite { let domain = apub.id.inner().domain().expect("group id has domain"); let instance = DbInstance::read_or_create(&mut data.pool(), domain.to_string()).await?; + let sidebar = read_from_string_or_source_opt(&apub.content, &None, &apub.source); + let sidebar = sanitize_html_opt(&sidebar); + let description = sanitize_html_opt(&apub.summary); + let site_form = SiteInsertForm { name: apub.name.clone(), - sidebar: read_from_string_or_source_opt(&apub.content, &None, &apub.source), + sidebar, updated: apub.updated.map(|u| u.clone().naive_local()), icon: apub.icon.clone().map(|i| i.url.into()), banner: apub.image.clone().map(|i| i.url.into()), - description: apub.summary.clone(), + description, actor_id: Some(apub.id.clone().into()), last_refreshed_at: Some(naive_now()), inbox_url: Some(apub.inbox.clone().into()), diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index d28f8c7cf..2c238fb56 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -19,7 +19,7 @@ use activitypub_federation::{ use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, - utils::{generate_outbox_url, local_site_opt_to_slur_regex}, + utils::{generate_outbox_url, local_site_opt_to_slur_regex, sanitize_html, sanitize_html_opt}, }; use lemmy_db_schema::{ source::person::{Person as DbPerson, PersonInsertForm, PersonUpdateForm}, @@ -138,12 +138,17 @@ impl Object for ApubPerson { ) -> Result { let instance_id = fetch_instance_actor_for_object(&person.id, context).await?; + let name = sanitize_html(&person.preferred_username); + let display_name = sanitize_html_opt(&person.name); + let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source); + let bio = sanitize_html_opt(&bio); + // Some Mastodon users have `name: ""` (empty string), need to convert that to `None` // https://github.com/mastodon/mastodon/issues/25233 - let display_name = person.name.filter(|n| !n.is_empty()); + let display_name = display_name.filter(|n| !n.is_empty()); let person_form = PersonInsertForm { - name: person.preferred_username, + name, display_name, banned: None, ban_expires: None, @@ -153,7 +158,7 @@ impl Object for ApubPerson { published: person.published.map(|u| u.naive_local()), updated: person.updated.map(|u| u.naive_local()), actor_id: Some(person.id.into()), - bio: read_from_string_or_source_opt(&person.summary, &None, &person.source), + bio, local: Some(false), admin: Some(false), bot_account: Some(person.kind == UserTypes::Service), diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 48b573d30..f04e07ded 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -25,7 +25,13 @@ use html2md::parse_html; use lemmy_api_common::{ context::LemmyContext, request::fetch_site_data, - utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex}, + utils::{ + is_mod_or_admin, + local_site_opt_to_sensitive, + local_site_opt_to_slur_regex, + sanitize_html, + sanitize_html_opt, + }, }; use lemmy_db_schema::{ self, @@ -228,6 +234,10 @@ impl Object for ApubPost { let language_id = LanguageTag::to_language_id_single(page.language, &mut context.pool()).await?; + let name = sanitize_html(&name); + let embed_title = sanitize_html_opt(&embed_title); + let embed_description = sanitize_html_opt(&embed_description); + PostInsertForm { name, url: url.map(Into::into), diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 69a2638ad..a51cfe6b7 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -12,7 +12,10 @@ use activitypub_federation::{ traits::Object, }; use chrono::NaiveDateTime; -use lemmy_api_common::{context::LemmyContext, utils::check_person_block}; +use lemmy_api_common::{ + context::LemmyContext, + utils::{check_person_block, sanitize_html}, +}; use lemmy_db_schema::{ source::{ person::Person, @@ -118,10 +121,13 @@ impl Object for ApubPrivateMessage { let recipient = note.to[0].dereference(context).await?; check_person_block(creator.id, recipient.id, &mut context.pool()).await?; + let content = read_from_string_or_source(¬e.content, &None, ¬e.source); + let content = sanitize_html(&content); + let form = PrivateMessageInsertForm { creator_id: creator.id, recipient_id: recipient.id, - content: read_from_string_or_source(¬e.content, &None, ¬e.source), + content, published: note.published.map(|u| u.naive_local()), updated: note.updated.map(|u| u.naive_local()), deleted: Some(false), diff --git a/crates/apub/src/protocol/objects/group.rs b/crates/apub/src/protocol/objects/group.rs index 77cafc828..9c679fdf1 100644 --- a/crates/apub/src/protocol/objects/group.rs +++ b/crates/apub/src/protocol/objects/group.rs @@ -23,7 +23,10 @@ use activitypub_federation::{ }, }; use chrono::{DateTime, FixedOffset}; -use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex}; +use lemmy_api_common::{ + context::LemmyContext, + utils::{local_site_opt_to_slur_regex, sanitize_html, sanitize_html_opt}, +}; use lemmy_db_schema::{ newtypes::InstanceId, source::community::{CommunityInsertForm, CommunityUpdateForm}, @@ -94,10 +97,15 @@ impl Group { } pub(crate) fn into_insert_form(self, instance_id: InstanceId) -> CommunityInsertForm { + let name = sanitize_html(&self.preferred_username); + let title = sanitize_html(&self.name.unwrap_or(self.preferred_username)); + let description = read_from_string_or_source_opt(&self.summary, &None, &self.source); + let description = sanitize_html_opt(&description); + CommunityInsertForm { - name: self.preferred_username.clone(), - title: self.name.unwrap_or(self.preferred_username), - description: read_from_string_or_source_opt(&self.summary, &None, &self.source), + name, + title, + description, removed: None, published: self.published.map(|u| u.naive_local()), updated: self.updated.map(|u| u.naive_local()), diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index cd2005ad0..7e8204de9 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -197,12 +197,12 @@ pub fn is_email_regex(test: &str) -> bool { EMAIL_REGEX.is_match(test) } -pub fn diesel_option_overwrite(opt: &Option) -> Option> { +pub fn diesel_option_overwrite(opt: Option) -> Option> { match opt { // An empty string is an erase Some(unwrapped) => { if !unwrapped.eq("") { - Some(Some(unwrapped.clone())) + Some(Some(unwrapped)) } else { Some(None) } @@ -445,10 +445,10 @@ mod tests { #[test] fn test_diesel_option_overwrite() { - assert_eq!(diesel_option_overwrite(&None), None); - assert_eq!(diesel_option_overwrite(&Some(String::new())), Some(None)); + assert_eq!(diesel_option_overwrite(None), None); + assert_eq!(diesel_option_overwrite(Some(String::new())), Some(None)); assert_eq!( - diesel_option_overwrite(&Some("test".to_string())), + diesel_option_overwrite(Some("test".to_string())), Some(Some("test".to_string())) ); } From 9bfa86d16279aca5cab02b0f835540cb3ae3e824 Mon Sep 17 00:00:00 2001 From: phiresky Date: Thu, 27 Jul 2023 12:11:00 +0200 Subject: [PATCH 13/27] add trigram index to search (#3719) * add trigram index to search * add community index --- migrations/2023-07-24-232635_trigram-index/down.sql | 8 ++++++++ migrations/2023-07-24-232635_trigram-index/up.sql | 10 ++++++++++ 2 files changed, 18 insertions(+) create mode 100644 migrations/2023-07-24-232635_trigram-index/down.sql create mode 100644 migrations/2023-07-24-232635_trigram-index/up.sql diff --git a/migrations/2023-07-24-232635_trigram-index/down.sql b/migrations/2023-07-24-232635_trigram-index/down.sql new file mode 100644 index 000000000..6883178f8 --- /dev/null +++ b/migrations/2023-07-24-232635_trigram-index/down.sql @@ -0,0 +1,8 @@ +DROP INDEX idx_comment_content_trigram; + +DROP INDEX idx_post_trigram; + +DROP INDEX idx_person_trigram; + +DROP INDEX idx_community_trigram; + diff --git a/migrations/2023-07-24-232635_trigram-index/up.sql b/migrations/2023-07-24-232635_trigram-index/up.sql new file mode 100644 index 000000000..ed648b6e5 --- /dev/null +++ b/migrations/2023-07-24-232635_trigram-index/up.sql @@ -0,0 +1,10 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE INDEX IF NOT EXISTS idx_comment_content_trigram ON comment USING gin(content gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS idx_post_trigram ON post USING gin(name gin_trgm_ops, body gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS idx_person_trigram ON person USING gin(name gin_trgm_ops, display_name gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS idx_community_trigram ON community USING gin(name gin_trgm_ops, title gin_trgm_ops); + From 2d7b416652c7f5afe72cfcf9c0fdc082d53e922e Mon Sep 17 00:00:00 2001 From: Bao Date: Thu, 27 Jul 2023 05:12:18 -0500 Subject: [PATCH 14/27] Omit local instance from federated instances list (#3712) Co-authored-by: Nutomic --- crates/db_schema/src/impls/instance.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/db_schema/src/impls/instance.rs b/crates/db_schema/src/impls/instance.rs index d6a23a712..f92d261b5 100644 --- a/crates/db_schema/src/impls/instance.rs +++ b/crates/db_schema/src/impls/instance.rs @@ -1,7 +1,7 @@ use crate::{ diesel::dsl::IntervalDsl, newtypes::InstanceId, - schema::{federation_allowlist, federation_blocklist, instance}, + schema::{federation_allowlist, federation_blocklist, instance, local_site, site}, source::instance::{Instance, InstanceForm}, utils::{get_conn, naive_now, DbPool}, }; @@ -97,6 +97,10 @@ impl Instance { pub async fn linked(pool: &mut DbPool<'_>) -> Result, Error> { let conn = &mut get_conn(pool).await?; instance::table + // omit instance representing the local site + .left_join(site::table.inner_join(local_site::table)) + .filter(local_site::id.is_null()) + // omit instances in the blocklist .left_join(federation_blocklist::table) .filter(federation_blocklist::id.is_null()) .select(instance::all_columns) From 21a87ebaf2e5c038594eb70ef58bd51826259529 Mon Sep 17 00:00:00 2001 From: RocketDerp <113625597+RocketDerp@users.noreply.github.com> Date: Thu, 27 Jul 2023 03:17:40 -0700 Subject: [PATCH 15/27] Federation tests replication round1 - demonstrate absent replication of comment deletes (#3657) * more robust test of unlike a comment, confirm replication to instance downstream from community home * more robust 'delete a comment' test, confirm replication * Far more robust "Report a comment" test. Many comments about situation, this is currently failing because gamma does not get the report * typo and actually have Gamma comment check use gamma, not alpha * prepare-drone-federation-test.sh has some more echo output and note about the LEMMY_DATABASE_URL format (#3651) * Add http cache for webfingers (#3317) * Add http cache for webfingers * Remove the outgoing cache middleware & adjust the cache headers directive * Use 1h & 3day cache header * Update routes and adjust the cache headers location * revert apub caching --------- Co-authored-by: Dessalines Co-authored-by: Felix Ableitner * Rewrite activity lists to fix delete federation (fixes #3625) * Revert "typo and actually have Gamma comment check use gamma, not alpha" This reverts commit 7dfb6ee0f4885da3a2d10316422f5b510772806c. * Revert "Far more robust "Report a comment" test. Many comments about situation, this is currently failing because gamma does not get the report" This reverts commit 7bd3b20ae08a64324029491ddb3ce4295ba16787. * prettier TypeScript * revised comments, as ResolveObject isn't using routine replication * fmt * fix api tests * remove comment --------- Co-authored-by: cetra3 Co-authored-by: Dessalines Co-authored-by: Felix Ableitner --- Cargo.lock | 16 +++++- api_tests/src/comment.spec.ts | 49 ++++++++++++++++- .../apub/src/activities/community/announce.rs | 16 +++--- crates/apub/src/activity_lists.rs | 52 ++++++++----------- crates/apub/src/http/person.rs | 4 +- 5 files changed, 97 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6785fe50b..6ac601c0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -392,7 +392,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170" dependencies = [ - "html5ever 0.26.0", + "html5ever", "maplit", "once_cell", "tendril", @@ -439,6 +439,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-io" version = "1.13.0" @@ -4176,6 +4189,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ + "async-compression", "base64 0.21.2", "bytes", "encoding_rs", diff --git a/api_tests/src/comment.spec.ts b/api_tests/src/comment.spec.ts index 19c2797b2..d7d533119 100644 --- a/api_tests/src/comment.spec.ts +++ b/api_tests/src/comment.spec.ts @@ -112,8 +112,27 @@ test("Update a comment", async () => { }); test("Delete a comment", async () => { + // creating a comment on alpha (remote from home of community) let commentRes = await createComment(alpha, postRes.post_view.post.id); + // Find the comment on beta (home of community) + let betaComment = ( + await resolveComment(beta, commentRes.comment_view.comment) + ).comment; + + if (!betaComment) { + throw "Missing beta comment before delete"; + } + + // Find the comment on remote instance gamma + let gammaComment = ( + await resolveComment(gamma, commentRes.comment_view.comment) + ).comment; + + if (!gammaComment) { + throw "Missing gamma comment (remote-home-remote replication) before delete"; + } + let deleteCommentRes = await deleteComment( alpha, true, @@ -126,6 +145,12 @@ test("Delete a comment", async () => { resolveComment(beta, commentRes.comment_view.comment), ).rejects.toBe("couldnt_find_object"); + // Make sure that comment is undefined on gamma after delete + await expect( + resolveComment(gamma, commentRes.comment_view.comment), + ).rejects.toBe("couldnt_find_object"); + + // Test undeleting the comment let undeleteCommentRes = await deleteComment( alpha, false, @@ -225,10 +250,22 @@ test("Remove a comment from admin and community on different instance", async () test("Unlike a comment", async () => { let commentRes = await createComment(alpha, postRes.post_view.post.id); + + // Lemmy automatically creates 1 like (vote) by author of comment. + // Make sure that comment is liked (voted up) on gamma, downstream peer + // This is testing replication from remote-home-remote (alpha-beta-gamma) + let gammaComment1 = ( + await resolveComment(gamma, commentRes.comment_view.comment) + ).comment; + expect(gammaComment1).toBeDefined(); + expect(gammaComment1?.community.local).toBe(false); + expect(gammaComment1?.creator.local).toBe(false); + expect(gammaComment1?.counts.score).toBe(1); + let unlike = await likeComment(alpha, 0, commentRes.comment_view.comment); expect(unlike.comment_view.counts.score).toBe(0); - // Make sure that post is unliked on beta + // Make sure that comment is unliked on beta let betaComment = ( await resolveComment(beta, commentRes.comment_view.comment) ).comment; @@ -236,6 +273,16 @@ test("Unlike a comment", async () => { expect(betaComment?.community.local).toBe(true); expect(betaComment?.creator.local).toBe(false); expect(betaComment?.counts.score).toBe(0); + + // Make sure that comment is unliked on gamma, downstream peer + // This is testing replication from remote-home-remote (alpha-beta-gamma) + let gammaComment = ( + await resolveComment(gamma, commentRes.comment_view.comment) + ).comment; + expect(gammaComment).toBeDefined(); + expect(gammaComment?.community.local).toBe(false); + expect(gammaComment?.creator.local).toBe(false); + expect(gammaComment?.counts.score).toBe(0); }); test("Federated comment like", async () => { diff --git a/crates/apub/src/activities/community/announce.rs b/crates/apub/src/activities/community/announce.rs index ed489158e..6eb23f8da 100644 --- a/crates/apub/src/activities/community/announce.rs +++ b/crates/apub/src/activities/community/announce.rs @@ -50,17 +50,19 @@ impl ActivityHandler for RawAnnouncableActivities { if let AnnouncableActivities::Page(_) = activity { return Err(LemmyErrorType::CannotReceivePage)?; } - let community = activity.community(data).await?; - let actor_id = activity.actor().clone().into(); // verify and receive activity activity.verify(data).await?; - activity.receive(data).await?; + activity.clone().receive(data).await?; - // send to community followers - if community.local { - verify_person_in_community(&actor_id, &community, data).await?; - AnnounceActivity::send(self, &community, data).await?; + // if activity is in a community, send to followers + let community = activity.community(data).await; + if let Ok(community) = community { + if community.local { + let actor_id = activity.actor().clone().into(); + verify_person_in_community(&actor_id, &community, data).await?; + AnnounceActivity::send(self, &community, data).await?; + } } Ok(()) } diff --git a/crates/apub/src/activity_lists.rs b/crates/apub/src/activity_lists.rs index 4cce3372f..d4ca20c33 100644 --- a/crates/apub/src/activity_lists.rs +++ b/crates/apub/src/activity_lists.rs @@ -24,24 +24,32 @@ use crate::{ InCommunity, }, }; -use activitypub_federation::{ - config::Data, - protocol::context::WithContext, - traits::ActivityHandler, -}; +use activitypub_federation::{config::Data, traits::ActivityHandler}; use lemmy_api_common::context::LemmyContext; use lemmy_utils::error::LemmyError; use serde::{Deserialize, Serialize}; use url::Url; +/// List of activities which the shared inbox can handle. +/// +/// This could theoretically be defined as an enum with variants `GroupInboxActivities` and +/// `PersonInboxActivities`. In practice we need to write it out manually so that priorities +/// are handled correctly. #[derive(Debug, Deserialize, Serialize)] #[serde(untagged)] #[enum_delegate::implement(ActivityHandler)] pub enum SharedInboxActivities { - PersonInboxActivities(Box>), - GroupInboxActivities(Box>), + Follow(Follow), + AcceptFollow(AcceptFollow), + UndoFollow(UndoFollow), + CreateOrUpdatePrivateMessage(CreateOrUpdateChatMessage), + Report(Report), + AnnounceActivity(AnnounceActivity), + /// This is a catch-all and needs to be last + RawAnnouncableActivities(RawAnnouncableActivities), } +/// List of activities which the group inbox can handle. #[derive(Debug, Deserialize, Serialize)] #[serde(untagged)] #[enum_delegate::implement(ActivityHandler)] @@ -49,10 +57,11 @@ pub enum GroupInboxActivities { Follow(Follow), UndoFollow(UndoFollow), Report(Report), - // This is a catch-all and needs to be last + /// This is a catch-all and needs to be last AnnouncableActivities(RawAnnouncableActivities), } +/// List of activities which the person inbox can handle. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(untagged)] #[enum_delegate::implement(ActivityHandler)] @@ -64,17 +73,8 @@ pub enum PersonInboxActivities { Delete(Delete), UndoDelete(UndoDelete), AnnounceActivity(AnnounceActivity), -} - -/// This is necessary for user inbox, which can also receive some "announcable" activities, -/// eg a comment mention. This needs to be a separate enum so that announcables received in shared -/// inbox can fall through to be parsed as GroupInboxActivities::AnnouncableActivities. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(untagged)] -#[enum_delegate::implement(ActivityHandler)] -pub enum PersonInboxActivitiesWithAnnouncable { - PersonInboxActivities(Box), - AnnouncableActivities(Box), + /// User can also receive some "announcable" activities, eg a comment mention. + AnnouncableActivities(AnnouncableActivities), } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -138,12 +138,7 @@ mod tests { #![allow(clippy::indexing_slicing)] use crate::{ - activity_lists::{ - GroupInboxActivities, - PersonInboxActivities, - PersonInboxActivitiesWithAnnouncable, - SiteInboxActivities, - }, + activity_lists::{GroupInboxActivities, PersonInboxActivities, SiteInboxActivities}, protocol::tests::{test_json, test_parse_lemmy_item}, }; @@ -161,16 +156,15 @@ mod tests { fn test_person_inbox() { test_parse_lemmy_item::("assets/lemmy/activities/following/accept.json") .unwrap(); - test_parse_lemmy_item::( + test_parse_lemmy_item::( "assets/lemmy/activities/create_or_update/create_note.json", ) .unwrap(); - test_parse_lemmy_item::( + test_parse_lemmy_item::( "assets/lemmy/activities/create_or_update/create_private_message.json", ) .unwrap(); - test_json::("assets/mastodon/activities/follow.json") - .unwrap(); + test_json::("assets/mastodon/activities/follow.json").unwrap(); } #[test] diff --git a/crates/apub/src/http/person.rs b/crates/apub/src/http/person.rs index 16956ec47..254313634 100644 --- a/crates/apub/src/http/person.rs +++ b/crates/apub/src/http/person.rs @@ -1,5 +1,5 @@ use crate::{ - activity_lists::PersonInboxActivitiesWithAnnouncable, + activity_lists::PersonInboxActivities, fetcher::user_or_community::UserOrCommunity, http::{create_apub_response, create_apub_tombstone_response}, objects::person::ApubPerson, @@ -49,7 +49,7 @@ pub async fn person_inbox( body: Bytes, data: Data, ) -> Result { - receive_activity::, UserOrCommunity, LemmyContext>( + receive_activity::, UserOrCommunity, LemmyContext>( request, body, &data, ) .await From d909f3455ddb89d45f35c78c4ce96bd0896469b1 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 27 Jul 2023 15:42:52 +0200 Subject: [PATCH 16/27] Change readme installation link (#3736) Replaces https://github.com/LemmyNet/lemmy/pull/3551 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f27d8441e..bb77a1cd8 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,7 @@ Each Lemmy server can set its own moderation policy; appointing site-wide admins ## Installation -- [Docker](https://join-lemmy.org/docs/administration/install_docker.html) -- [Ansible](https://join-lemmy.org/docs/administration/install_ansible.html) +- [Lemmy Administration Docs](https://join-lemmy.org/docs/administration/administration.html) ## Lemmy Projects From 1253a2a0d531f55594109f1d548c56cd1b1fcb59 Mon Sep 17 00:00:00 2001 From: phiresky Date: Thu, 27 Jul 2023 23:35:53 +0200 Subject: [PATCH 17/27] remove performance-problematic and buggy duplicate site aggregates (#3732) --- .../down.sql | 11 +++++++++++ .../2023-07-26-222023_site-aggregates-one/up.sql | 15 +++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 migrations/2023-07-26-222023_site-aggregates-one/down.sql create mode 100644 migrations/2023-07-26-222023_site-aggregates-one/up.sql diff --git a/migrations/2023-07-26-222023_site-aggregates-one/down.sql b/migrations/2023-07-26-222023_site-aggregates-one/down.sql new file mode 100644 index 000000000..f384785f4 --- /dev/null +++ b/migrations/2023-07-26-222023_site-aggregates-one/down.sql @@ -0,0 +1,11 @@ +create or replace function site_aggregates_site() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'INSERT') THEN + insert into site_aggregates (site_id) values (NEW.id); + ELSIF (TG_OP = 'DELETE') THEN + delete from site_aggregates where site_id = OLD.id; + END IF; + return null; +end $$; \ No newline at end of file diff --git a/migrations/2023-07-26-222023_site-aggregates-one/up.sql b/migrations/2023-07-26-222023_site-aggregates-one/up.sql new file mode 100644 index 000000000..7ed40316d --- /dev/null +++ b/migrations/2023-07-26-222023_site-aggregates-one/up.sql @@ -0,0 +1,15 @@ +create or replace function site_aggregates_site() +returns trigger language plpgsql +as $$ +begin + -- we only ever want to have a single value in site_aggregate because the site_aggregate triggers update all rows in that table. + -- a cleaner check would be to insert it for the local_site but that would break assumptions at least in the tests + IF (TG_OP = 'INSERT') AND NOT EXISTS (select id from site_aggregates limit 1) THEN + insert into site_aggregates (site_id) values (NEW.id); + ELSIF (TG_OP = 'DELETE') THEN + delete from site_aggregates where site_id = OLD.id; + END IF; + return null; +end $$; + +delete from site_aggregates a where not exists (select id from local_site s where s.site_id = a.site_id); \ No newline at end of file From e315092ee38e7c16005cafa9045042c8307224b0 Mon Sep 17 00:00:00 2001 From: phiresky Date: Thu, 27 Jul 2023 23:36:51 +0200 Subject: [PATCH 18/27] remove n^2 part of person triggers, improve community aggregate trigger (#3739) * remove n^2 part of person triggers, improve community aggregate trigger * comment out comment_score tests since previously they only accidentally succeeded * empty --- .../src/aggregates/person_aggregates.rs | 12 ++- .../down.sql | 80 +++++++++++++++++++ .../up.sql | 47 +++++++++++ 3 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 migrations/2023-07-27-134652_remove-expensive-broken-trigger/down.sql create mode 100644 migrations/2023-07-27-134652_remove-expensive-broken-trigger/up.sql diff --git a/crates/db_schema/src/aggregates/person_aggregates.rs b/crates/db_schema/src/aggregates/person_aggregates.rs index 43feadd45..2e71844fa 100644 --- a/crates/db_schema/src/aggregates/person_aggregates.rs +++ b/crates/db_schema/src/aggregates/person_aggregates.rs @@ -161,7 +161,8 @@ mod tests { .await .unwrap(); assert_eq!(0, after_parent_comment_removed.comment_count); - assert_eq!(0, after_parent_comment_removed.comment_score); + // TODO: fix person aggregate comment score calculation + // assert_eq!(0, after_parent_comment_removed.comment_score); // Remove a parent comment (the scores should also be removed) Comment::delete(pool, inserted_comment.id).await.unwrap(); @@ -172,7 +173,8 @@ mod tests { .await .unwrap(); assert_eq!(0, after_parent_comment_delete.comment_count); - assert_eq!(0, after_parent_comment_delete.comment_score); + // TODO: fix person aggregate comment score calculation + // assert_eq!(0, after_parent_comment_delete.comment_score); // Add in the two comments again, then delete the post. let new_parent_comment = Comment::create(pool, &comment_form, None).await.unwrap(); @@ -186,13 +188,15 @@ mod tests { .await .unwrap(); assert_eq!(2, after_comment_add.comment_count); - assert_eq!(1, after_comment_add.comment_score); + // TODO: fix person aggregate comment score calculation + // assert_eq!(1, after_comment_add.comment_score); Post::delete(pool, inserted_post.id).await.unwrap(); let after_post_delete = PersonAggregates::read(pool, inserted_person.id) .await .unwrap(); - assert_eq!(0, after_post_delete.comment_score); + // TODO: fix person aggregate comment score calculation + // assert_eq!(0, after_post_delete.comment_score); assert_eq!(0, after_post_delete.comment_count); assert_eq!(0, after_post_delete.post_score); assert_eq!(0, after_post_delete.post_count); diff --git a/migrations/2023-07-27-134652_remove-expensive-broken-trigger/down.sql b/migrations/2023-07-27-134652_remove-expensive-broken-trigger/down.sql new file mode 100644 index 000000000..1a5c4ec4f --- /dev/null +++ b/migrations/2023-07-27-134652_remove-expensive-broken-trigger/down.sql @@ -0,0 +1,80 @@ +create or replace function person_aggregates_comment_count() + returns trigger language plpgsql +as $$ +begin + IF (was_restored_or_created(TG_OP, OLD, NEW)) THEN + update person_aggregates + set comment_count = comment_count + 1 where person_id = NEW.creator_id; + ELSIF (was_removed_or_deleted(TG_OP, OLD, NEW)) THEN + update person_aggregates + set comment_count = comment_count - 1 where person_id = OLD.creator_id; + + -- If the comment gets deleted, the score calculation trigger won't fire, + -- so you need to re-calculate + update person_aggregates ua + set comment_score = cd.score + from ( + select u.id, + coalesce(0, sum(cl.score)) as score + -- User join because comments could be empty + from person u + left join comment c on u.id = c.creator_id and c.deleted = 'f' and c.removed = 'f' + left join comment_like cl on c.id = cl.comment_id + group by u.id + ) cd + where ua.person_id = OLD.creator_id; + END IF; + return null; +end $$; + +create or replace function person_aggregates_post_count() + returns trigger language plpgsql +as $$ +begin + IF (was_restored_or_created(TG_OP, OLD, NEW)) THEN + update person_aggregates + set post_count = post_count + 1 where person_id = NEW.creator_id; + + ELSIF (was_removed_or_deleted(TG_OP, OLD, NEW)) THEN + update person_aggregates + set post_count = post_count - 1 where person_id = OLD.creator_id; + + -- If the post gets deleted, the score calculation trigger won't fire, + -- so you need to re-calculate + update person_aggregates ua + set post_score = pd.score + from ( + select u.id, + coalesce(0, sum(pl.score)) as score + -- User join because posts could be empty + from person u + left join post p on u.id = p.creator_id and p.deleted = 'f' and p.removed = 'f' + left join post_like pl on p.id = pl.post_id + group by u.id + ) pd + where ua.person_id = OLD.creator_id; + + END IF; + return null; +end $$; + +create or replace function community_aggregates_comment_count() + returns trigger language plpgsql +as $$ +begin + IF (was_restored_or_created(TG_OP, OLD, NEW)) THEN +update community_aggregates ca +set comments = comments + 1 from comment c, post p +where p.id = c.post_id + and p.id = NEW.post_id + and ca.community_id = p.community_id; +ELSIF (was_removed_or_deleted(TG_OP, OLD, NEW)) THEN +update community_aggregates ca +set comments = comments - 1 from comment c, post p +where p.id = c.post_id + and p.id = OLD.post_id + and ca.community_id = p.community_id; + +END IF; +return null; +end $$; \ No newline at end of file diff --git a/migrations/2023-07-27-134652_remove-expensive-broken-trigger/up.sql b/migrations/2023-07-27-134652_remove-expensive-broken-trigger/up.sql new file mode 100644 index 000000000..66a78371b --- /dev/null +++ b/migrations/2023-07-27-134652_remove-expensive-broken-trigger/up.sql @@ -0,0 +1,47 @@ +create or replace function person_aggregates_comment_count() + returns trigger language plpgsql +as $$ +begin + IF (was_restored_or_created(TG_OP, OLD, NEW)) THEN + update person_aggregates + set comment_count = comment_count + 1 where person_id = NEW.creator_id; + ELSIF (was_removed_or_deleted(TG_OP, OLD, NEW)) THEN + update person_aggregates + set comment_count = comment_count - 1 where person_id = OLD.creator_id; + END IF; + return null; +end $$; + +create or replace function person_aggregates_post_count() + returns trigger language plpgsql +as $$ +begin + IF (was_restored_or_created(TG_OP, OLD, NEW)) THEN + update person_aggregates + set post_count = post_count + 1 where person_id = NEW.creator_id; + + ELSIF (was_removed_or_deleted(TG_OP, OLD, NEW)) THEN + update person_aggregates + set post_count = post_count - 1 where person_id = OLD.creator_id; + END IF; + return null; +end $$; + +create or replace function community_aggregates_comment_count() + returns trigger language plpgsql +as $$ +begin + IF (was_restored_or_created(TG_OP, OLD, NEW)) THEN +update community_aggregates ca +set comments = comments + 1 from post p +where p.id = NEW.post_id + and ca.community_id = p.community_id; +ELSIF (was_removed_or_deleted(TG_OP, OLD, NEW)) THEN +update community_aggregates ca +set comments = comments - 1 from post p +where p.id = OLD.post_id + and ca.community_id = p.community_id; + +END IF; +return null; +end $$; \ No newline at end of file From 9a5a13c734a1792511e1bfef7b9ac4121e0e7371 Mon Sep 17 00:00:00 2001 From: dullbananas Date: Fri, 28 Jul 2023 01:36:50 -0700 Subject: [PATCH 19/27] Use same table join code for both read and list functions (#3663) * Try stuff * Revert "Try stuff" This reverts commit 3da5f83a8b4928368bf58e0061091c270259a226. * Revert "Revert "Try stuff"" This reverts commit 178bd43cac8c7674d30d2c285ed47ca0493ad659. * Revert "Revert "Revert "Try stuff""" This reverts commit b9f9a2316e7cd37082319608c606a1c7db057206. * Revert "Revert "Revert "Revert "Try stuff"""" This reverts commit ccd498dd7228050ae05d2022e9106034fd4132f8. * Try more stuff * Add queries function * Simplify queries function * Move aliases to db_schema * Revert "Move aliases to db_schema" This reverts commit 69afed05c1807c3fef8d5b5872546fa22e60b4d0. * Add ReadFuture and ListFuture * Refactor queries function and add Queries struct * Box futures in Queries::new * Use from_tuple * Add comment_view::queries and improve comment_report_view::queries * Add local_user_view::queries * Add post_report_view::queries * Ad post_view::queries * Add private_message_report_view::queries * private_message_view, registration_application_view * Use 'a in BoxedQuery * comment_reply_view, community_view * Change aliases to inline module * person_mention_view * person_view * Use separate community_person_ban joins instead of including boolean literal in join-on clause * Fix comment_view * rerun ci --- crates/db_schema/src/lib.rs | 5 + crates/db_schema/src/utils.rs | 91 +++- crates/db_views/src/comment_report_view.rs | 248 +++++----- crates/db_views/src/comment_view.rs | 464 ++++++++---------- crates/db_views/src/local_user_view.rs | 178 +++---- crates/db_views/src/post_report_view.rs | 186 +++---- crates/db_views/src/post_view.rs | 302 +++++------- .../src/private_message_report_view.rs | 140 +++--- crates/db_views/src/private_message_view.rs | 143 +++--- .../src/registration_application_view.rs | 135 +++-- .../db_views_actor/src/comment_reply_view.rs | 207 +++----- crates/db_views_actor/src/community_view.rs | 247 +++++----- .../db_views_actor/src/person_mention_view.rs | 269 ++++------ crates/db_views_actor/src/person_view.rs | 139 +++--- 14 files changed, 1233 insertions(+), 1521 deletions(-) diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index e5b86fe19..e3232c402 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -28,6 +28,11 @@ pub mod newtypes; #[rustfmt::skip] #[allow(clippy::wildcard_imports)] pub mod schema; +#[cfg(feature = "full")] +pub mod aliases { + use crate::schema::person; + diesel::alias!(person as person1: Person1, person as person2: Person2); +} pub mod source; #[cfg(feature = "full")] pub mod traits; diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 7e8204de9..dc26bedfc 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -2,6 +2,7 @@ use crate::{ diesel::Connection, diesel_migrations::MigrationHarness, newtypes::DbUrl, + traits::JoinView, CommentSortType, PersonSortType, SortType, @@ -26,7 +27,7 @@ use diesel_async::{ }, }; use diesel_migrations::EmbeddedMigrations; -use futures_util::{future::BoxFuture, FutureExt}; +use futures_util::{future::BoxFuture, Future, FutureExt}; use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType}, settings::structs::Settings, @@ -420,6 +421,94 @@ where } } +pub type ResultFuture<'a, T> = BoxFuture<'a, Result>; + +pub trait ReadFn<'a, T: JoinView, Args>: + Fn(DbConn<'a>, Args) -> ResultFuture<'a, ::JoinTuple> +{ +} + +impl< + 'a, + T: JoinView, + Args, + F: Fn(DbConn<'a>, Args) -> ResultFuture<'a, ::JoinTuple>, + > ReadFn<'a, T, Args> for F +{ +} + +pub trait ListFn<'a, T: JoinView, Args>: + Fn(DbConn<'a>, Args) -> ResultFuture<'a, Vec<::JoinTuple>> +{ +} + +impl< + 'a, + T: JoinView, + Args, + F: Fn(DbConn<'a>, Args) -> ResultFuture<'a, Vec<::JoinTuple>>, + > ListFn<'a, T, Args> for F +{ +} + +/// Allows read and list functions to capture a shared closure that has an inferred return type, which is useful for join logic +pub struct Queries { + pub read_fn: RF, + pub list_fn: LF, +} + +// `()` is used to prevent type inference error +impl Queries<(), ()> { + pub fn new<'a, RFut, LFut, RT, LT, RA, LA, RF2, LF2>( + read_fn: RF2, + list_fn: LF2, + ) -> Queries, impl ListFn<'a, LT, LA>> + where + RFut: Future::JoinTuple, DieselError>> + Sized + Send + 'a, + LFut: + Future::JoinTuple>, DieselError>> + Sized + Send + 'a, + RT: JoinView, + LT: JoinView, + RF2: Fn(DbConn<'a>, RA) -> RFut, + LF2: Fn(DbConn<'a>, LA) -> LFut, + { + Queries { + read_fn: move |conn, args| read_fn(conn, args).boxed(), + list_fn: move |conn, args| list_fn(conn, args).boxed(), + } + } +} + +impl Queries { + pub async fn read<'a, T, Args>( + self, + pool: &'a mut DbPool<'_>, + args: Args, + ) -> Result + where + T: JoinView, + RF: ReadFn<'a, T, Args>, + { + let conn = get_conn(pool).await?; + let res = (self.read_fn)(conn, args).await?; + Ok(T::from_tuple(res)) + } + + pub async fn list<'a, T, Args>( + self, + pool: &'a mut DbPool<'_>, + args: Args, + ) -> Result, DieselError> + where + T: JoinView, + LF: ListFn<'a, T, Args>, + { + let conn = get_conn(pool).await?; + let res = (self.list_fn)(conn, args).await?; + Ok(res.into_iter().map(T::from_tuple).collect()) + } +} + #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] diff --git a/crates/db_views/src/comment_report_view.rs b/crates/db_views/src/comment_report_view.rs index 0b1821c1b..a92b38063 100644 --- a/crates/db_views/src/comment_report_view.rs +++ b/crates/db_views/src/comment_report_view.rs @@ -1,6 +1,7 @@ use crate::structs::CommentReportView; use diesel::{ dsl::now, + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -11,6 +12,7 @@ use diesel::{ use diesel_async::RunQueryDsl; use lemmy_db_schema::{ aggregates::structs::CommentAggregates, + aliases, newtypes::{CommentReportId, CommunityId, PersonId}, schema::{ comment, @@ -31,9 +33,119 @@ use lemmy_db_schema::{ post::Post, }, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, }; +fn queries<'a>() -> Queries< + impl ReadFn<'a, CommentReportView, (CommentReportId, PersonId)>, + impl ListFn<'a, CommentReportView, (CommentReportQuery, &'a Person)>, +> { + let all_joins = |query: comment_report::BoxedQuery<'a, Pg>, my_person_id: PersonId| { + query + .inner_join(comment::table) + .inner_join(post::table.on(comment::post_id.eq(post::id))) + .inner_join(community::table.on(post::community_id.eq(community::id))) + .inner_join(person::table.on(comment_report::creator_id.eq(person::id))) + .inner_join(aliases::person1.on(comment::creator_id.eq(aliases::person1.field(person::id)))) + .inner_join( + comment_aggregates::table.on(comment_report::comment_id.eq(comment_aggregates::comment_id)), + ) + .left_join( + comment_like::table.on( + comment::id + .eq(comment_like::comment_id) + .and(comment_like::person_id.eq(my_person_id)), + ), + ) + .left_join( + aliases::person2 + .on(comment_report::resolver_id.eq(aliases::person2.field(person::id).nullable())), + ) + }; + + let selection = ( + comment_report::all_columns, + comment::all_columns, + post::all_columns, + community::all_columns, + person::all_columns, + aliases::person1.fields(person::all_columns), + comment_aggregates::all_columns, + community_person_ban::all_columns.nullable(), + comment_like::score.nullable(), + aliases::person2.fields(person::all_columns).nullable(), + ); + + let read = move |mut conn: DbConn<'a>, (report_id, my_person_id): (CommentReportId, PersonId)| async move { + all_joins( + comment_report::table.find(report_id).into_boxed(), + my_person_id, + ) + .left_join( + community_person_ban::table.on( + community::id + .eq(community_person_ban::community_id) + .and(community_person_ban::person_id.eq(comment::creator_id)), + ), + ) + .select(selection) + .first::<::JoinTuple>(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, (options, my_person): (CommentReportQuery, &'a Person)| async move { + let mut query = all_joins(comment_report::table.into_boxed(), my_person.id) + .left_join( + community_person_ban::table.on( + community::id + .eq(community_person_ban::community_id) + .and(community_person_ban::person_id.eq(comment::creator_id)) + .and( + community_person_ban::expires + .is_null() + .or(community_person_ban::expires.gt(now)), + ), + ), + ) + .select(selection); + + if let Some(community_id) = options.community_id { + query = query.filter(post::community_id.eq(community_id)); + } + + if options.unresolved_only.unwrap_or(false) { + query = query.filter(comment_report::resolved.eq(false)); + } + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query = query + .order_by(comment_report::published.desc()) + .limit(limit) + .offset(offset); + + // If its not an admin, get only the ones you mod + if !my_person.admin { + query + .inner_join( + community_moderator::table.on( + community_moderator::community_id + .eq(post::community_id) + .and(community_moderator::person_id.eq(my_person.id)), + ), + ) + .load::<::JoinTuple>(&mut conn) + .await + } else { + query + .load::<::JoinTuple>(&mut conn) + .await + } + }; + + Queries::new(read, list) +} + impl CommentReportView { /// returns the CommentReportView for the provided report_id /// @@ -43,54 +155,7 @@ impl CommentReportView { report_id: CommentReportId, my_person_id: PersonId, ) -> Result { - let conn = &mut get_conn(pool).await?; - - let (person_alias_1, person_alias_2) = diesel::alias!(person as person1, person as person2); - - let res = comment_report::table - .find(report_id) - .inner_join(comment::table) - .inner_join(post::table.on(comment::post_id.eq(post::id))) - .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person::table.on(comment_report::creator_id.eq(person::id))) - .inner_join(person_alias_1.on(comment::creator_id.eq(person_alias_1.field(person::id)))) - .inner_join( - comment_aggregates::table.on(comment_report::comment_id.eq(comment_aggregates::comment_id)), - ) - .left_join( - community_person_ban::table.on( - community::id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)), - ), - ) - .left_join( - comment_like::table.on( - comment::id - .eq(comment_like::comment_id) - .and(comment_like::person_id.eq(my_person_id)), - ), - ) - .left_join( - person_alias_2 - .on(comment_report::resolver_id.eq(person_alias_2.field(person::id).nullable())), - ) - .select(( - comment_report::all_columns, - comment::all_columns, - post::all_columns, - community::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - comment_like::score.nullable(), - person_alias_2.fields(person::all_columns).nullable(), - )) - .first::<::JoinTuple>(conn) - .await?; - - Ok(Self::from_tuple(res)) + queries().read(pool, (report_id, my_person_id)).await } /// Returns the current unresolved post report count for the communities you mod @@ -150,90 +215,7 @@ impl CommentReportQuery { pool: &mut DbPool<'_>, my_person: &Person, ) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - - let (person_alias_1, person_alias_2) = diesel::alias!(person as person1, person as person2); - - let mut query = comment_report::table - .inner_join(comment::table) - .inner_join(post::table.on(comment::post_id.eq(post::id))) - .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person::table.on(comment_report::creator_id.eq(person::id))) - .inner_join(person_alias_1.on(comment::creator_id.eq(person_alias_1.field(person::id)))) - .inner_join( - comment_aggregates::table.on(comment_report::comment_id.eq(comment_aggregates::comment_id)), - ) - .left_join( - community_person_ban::table.on( - community::id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), - ), - ) - .left_join( - comment_like::table.on( - comment::id - .eq(comment_like::comment_id) - .and(comment_like::person_id.eq(my_person.id)), - ), - ) - .left_join( - person_alias_2 - .on(comment_report::resolver_id.eq(person_alias_2.field(person::id).nullable())), - ) - .select(( - comment_report::all_columns, - comment::all_columns, - post::all_columns, - community::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - comment_like::score.nullable(), - person_alias_2.fields(person::all_columns).nullable(), - )) - .into_boxed(); - - if let Some(community_id) = self.community_id { - query = query.filter(post::community_id.eq(community_id)); - } - - if self.unresolved_only.unwrap_or(false) { - query = query.filter(comment_report::resolved.eq(false)); - } - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - query = query - .order_by(comment_report::published.desc()) - .limit(limit) - .offset(offset); - - // If its not an admin, get only the ones you mod - let res = if !my_person.admin { - query - .inner_join( - community_moderator::table.on( - community_moderator::community_id - .eq(post::community_id) - .and(community_moderator::person_id.eq(my_person.id)), - ), - ) - .load::<::JoinTuple>(conn) - .await? - } else { - query - .load::<::JoinTuple>(conn) - .await? - }; - - Ok(res.into_iter().map(CommentReportView::from_tuple).collect()) + queries().list(pool, (self, my_person)).await } } diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 2d233438a..26787cceb 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -1,5 +1,6 @@ use crate::structs::{CommentView, LocalUserView}; use diesel::{ + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -35,7 +36,7 @@ use lemmy_db_schema::{ post::Post, }, traits::JoinView, - utils::{fuzzy_search, get_conn, limit_and_offset, DbPool}, + utils::{fuzzy_search, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, CommentSortType, ListingType, }; @@ -53,30 +54,14 @@ type CommentViewTuple = ( Option, ); -impl CommentView { - pub async fn read( - pool: &mut DbPool<'_>, - comment_id: CommentId, - my_person_id: Option, - ) -> Result { - let conn = &mut get_conn(pool).await?; - +fn queries<'a>() -> Queries< + impl ReadFn<'a, CommentView, (CommentId, Option)>, + impl ListFn<'a, CommentView, CommentQuery<'a>>, +> { + let all_joins = |query: comment::BoxedQuery<'a, Pg>, my_person_id: Option| { // The left join below will return None in this case let person_id_join = my_person_id.unwrap_or(PersonId(-1)); - - let ( - comment, - creator, - post, - community, - counts, - creator_banned_from_community, - follower, - saved, - creator_blocked, - comment_like, - ) = comment::table - .find(comment_id) + query .inner_join(person::table) .inner_join(post::table) .inner_join(community::table.on(post::community_id.eq(community::id))) @@ -116,41 +101,201 @@ impl CommentView { .and(comment_like::person_id.eq(person_id_join)), ), ) - .select(( - comment::all_columns, - person::all_columns, - post::all_columns, - community::all_columns, - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - community_follower::all_columns.nullable(), - comment_saved::all_columns.nullable(), - person_block::all_columns.nullable(), - comment_like::score.nullable(), - )) - .first::(conn) - .await?; + }; - // If a person is given, then my_vote, if None, should be 0, not null - // Necessary to differentiate between other person's votes - let my_vote = if my_person_id.is_some() && comment_like.is_none() { - Some(0) - } else { - comment_like + let selection = ( + comment::all_columns, + person::all_columns, + post::all_columns, + community::all_columns, + comment_aggregates::all_columns, + community_person_ban::all_columns.nullable(), + community_follower::all_columns.nullable(), + comment_saved::all_columns.nullable(), + person_block::all_columns.nullable(), + comment_like::score.nullable(), + ); + + let read = move |mut conn: DbConn<'a>, + (comment_id, my_person_id): (CommentId, Option)| async move { + all_joins(comment::table.find(comment_id).into_boxed(), my_person_id) + .select(selection) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, options: CommentQuery<'a>| async move { + let person_id = options.local_user.map(|l| l.person.id); + let local_user_id = options.local_user.map(|l| l.local_user.id); + + // The left join below will return None in this case + let person_id_join = person_id.unwrap_or(PersonId(-1)); + let local_user_id_join = local_user_id.unwrap_or(LocalUserId(-1)); + + let mut query = all_joins(comment::table.into_boxed(), person_id) + .left_join( + community_block::table.on( + community::id + .eq(community_block::community_id) + .and(community_block::person_id.eq(person_id_join)), + ), + ) + .left_join( + local_user_language::table.on( + comment::language_id + .eq(local_user_language::language_id) + .and(local_user_language::local_user_id.eq(local_user_id_join)), + ), + ) + .select(selection); + + if let Some(creator_id) = options.creator_id { + query = query.filter(comment::creator_id.eq(creator_id)); }; - Ok(CommentView { - comment, - post, - creator, - community, - counts, - creator_banned_from_community: creator_banned_from_community.is_some(), - subscribed: CommunityFollower::to_subscribed_type(&follower), - saved: saved.is_some(), - creator_blocked: creator_blocked.is_some(), - my_vote, - }) + if let Some(post_id) = options.post_id { + query = query.filter(comment::post_id.eq(post_id)); + }; + + if let Some(parent_path) = options.parent_path.as_ref() { + query = query.filter(comment::path.contained_by(parent_path)); + }; + + if let Some(search_term) = options.search_term { + query = query.filter(comment::content.ilike(fuzzy_search(&search_term))); + }; + + if let Some(community_id) = options.community_id { + query = query.filter(post::community_id.eq(community_id)); + } + + if let Some(listing_type) = options.listing_type { + match listing_type { + ListingType::Subscribed => { + query = query.filter(community_follower::person_id.is_not_null()) + } // TODO could be this: and(community_follower::person_id.eq(person_id_join)), + ListingType::Local => { + query = query.filter(community::local.eq(true)).filter( + community::hidden + .eq(false) + .or(community_follower::person_id.eq(person_id_join)), + ) + } + ListingType::All => { + query = query.filter( + community::hidden + .eq(false) + .or(community_follower::person_id.eq(person_id_join)), + ) + } + } + } + + if options.saved_only.unwrap_or(false) { + query = query.filter(comment_saved::comment_id.is_not_null()); + } + + let is_profile_view = options.is_profile_view.unwrap_or(false); + let is_creator = options.creator_id == options.local_user.map(|l| l.person.id); + // only show deleted comments to creator + if !is_creator { + query = query.filter(comment::deleted.eq(false)); + } + + let is_admin = options.local_user.map(|l| l.person.admin).unwrap_or(false); + // only show removed comments to admin when viewing user profile + if !(is_profile_view && is_admin) { + query = query.filter(comment::removed.eq(false)); + } + + if !options + .local_user + .map(|l| l.local_user.show_bot_accounts) + .unwrap_or(true) + { + query = query.filter(person::bot_account.eq(false)); + }; + + if options.local_user.is_some() { + // Filter out the rows with missing languages + query = query.filter(local_user_language::language_id.is_not_null()); + + // Don't show blocked communities or persons + if options.post_id.is_none() { + query = query.filter(community_block::person_id.is_null()); + } + query = query.filter(person_block::person_id.is_null()); + } + + // A Max depth given means its a tree fetch + let (limit, offset) = if let Some(max_depth) = options.max_depth { + let depth_limit = if let Some(parent_path) = options.parent_path.as_ref() { + parent_path.0.split('.').count() as i32 + max_depth + // Add one because of root "0" + } else { + max_depth + 1 + }; + + query = query.filter(nlevel(comment::path).le(depth_limit)); + + // only order if filtering by a post id. DOS potential otherwise and max_depth + !post_id isn't used anyways (afaik) + if options.post_id.is_some() { + // Always order by the parent path first + query = query.order_by(subpath(comment::path, 0, -1)); + } + + // TODO limit question. Limiting does not work for comment threads ATM, only max_depth + // For now, don't do any limiting for tree fetches + // https://stackoverflow.com/questions/72983614/postgres-ltree-how-to-limit-the-max-number-of-children-at-any-given-level + + // Don't use the regular error-checking one, many more comments must ofter be fetched. + // This does not work for comment trees, and the limit should be manually set to a high number + // + // If a max depth is given, then you know its a tree fetch, and limits should be ignored + // TODO a kludge to prevent attacks. Limit comments to 300 for now. + // (i64::MAX, 0) + (300, 0) + } else { + // limit_and_offset_unlimited(options.page, options.limit) + limit_and_offset(options.page, options.limit)? + }; + + query = match options.sort.unwrap_or(CommentSortType::Hot) { + CommentSortType::Hot => query + .then_order_by(comment_aggregates::hot_rank.desc()) + .then_order_by(comment_aggregates::score.desc()), + CommentSortType::Controversial => { + query.then_order_by(comment_aggregates::controversy_rank.desc()) + } + CommentSortType::New => query.then_order_by(comment::published.desc()), + CommentSortType::Old => query.then_order_by(comment::published.asc()), + CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), + }; + + // Note: deleted and removed comments are done on the front side + query + .limit(limit) + .offset(offset) + .load::(&mut conn) + .await + }; + + Queries::new(read, list) +} + +impl CommentView { + pub async fn read( + pool: &mut DbPool<'_>, + comment_id: CommentId, + my_person_id: Option, + ) -> Result { + // If a person is given, then my_vote (res.9), if None, should be 0, not null + // Necessary to differentiate between other person's votes + let mut res = queries().read(pool, (comment_id, my_person_id)).await?; + if my_person_id.is_some() && res.my_vote.is_none() { + res.my_vote = Some(0); + } + Ok(res) } } @@ -174,214 +319,7 @@ pub struct CommentQuery<'a> { impl<'a> CommentQuery<'a> { pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - - // The left join below will return None in this case - let person_id_join = self.local_user.map(|l| l.person.id).unwrap_or(PersonId(-1)); - let local_user_id_join = self - .local_user - .map(|l| l.local_user.id) - .unwrap_or(LocalUserId(-1)); - - let mut query = comment::table - .inner_join(person::table) - .inner_join(post::table) - .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(comment_aggregates::table) - .left_join( - community_person_ban::table.on( - community::id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)), - ), - ) - .left_join( - community_follower::table.on( - post::community_id - .eq(community_follower::community_id) - .and(community_follower::person_id.eq(person_id_join)), - ), - ) - .left_join( - comment_saved::table.on( - comment::id - .eq(comment_saved::comment_id) - .and(comment_saved::person_id.eq(person_id_join)), - ), - ) - .left_join( - person_block::table.on( - comment::creator_id - .eq(person_block::target_id) - .and(person_block::person_id.eq(person_id_join)), - ), - ) - .left_join( - community_block::table.on( - community::id - .eq(community_block::community_id) - .and(community_block::person_id.eq(person_id_join)), - ), - ) - .left_join( - comment_like::table.on( - comment::id - .eq(comment_like::comment_id) - .and(comment_like::person_id.eq(person_id_join)), - ), - ) - .left_join( - local_user_language::table.on( - comment::language_id - .eq(local_user_language::language_id) - .and(local_user_language::local_user_id.eq(local_user_id_join)), - ), - ) - .select(( - comment::all_columns, - person::all_columns, - post::all_columns, - community::all_columns, - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - community_follower::all_columns.nullable(), - comment_saved::all_columns.nullable(), - person_block::all_columns.nullable(), - comment_like::score.nullable(), - )) - .into_boxed(); - - if let Some(creator_id) = self.creator_id { - query = query.filter(comment::creator_id.eq(creator_id)); - }; - - if let Some(post_id) = self.post_id { - query = query.filter(comment::post_id.eq(post_id)); - }; - - if let Some(parent_path) = self.parent_path.as_ref() { - query = query.filter(comment::path.contained_by(parent_path)); - }; - - if let Some(search_term) = self.search_term { - query = query.filter(comment::content.ilike(fuzzy_search(&search_term))); - }; - - if let Some(community_id) = self.community_id { - query = query.filter(post::community_id.eq(community_id)); - } - - if let Some(listing_type) = self.listing_type { - match listing_type { - ListingType::Subscribed => { - query = query.filter(community_follower::person_id.is_not_null()) - } // TODO could be this: and(community_follower::person_id.eq(person_id_join)), - ListingType::Local => { - query = query.filter(community::local.eq(true)).filter( - community::hidden - .eq(false) - .or(community_follower::person_id.eq(person_id_join)), - ) - } - ListingType::All => { - query = query.filter( - community::hidden - .eq(false) - .or(community_follower::person_id.eq(person_id_join)), - ) - } - } - } - - if self.saved_only.unwrap_or(false) { - query = query.filter(comment_saved::comment_id.is_not_null()); - } - - let is_profile_view = self.is_profile_view.unwrap_or(false); - let is_creator = self.creator_id == self.local_user.map(|l| l.person.id); - // only show deleted comments to creator - if !is_creator { - query = query.filter(comment::deleted.eq(false)); - } - - let is_admin = self.local_user.map(|l| l.person.admin).unwrap_or(false); - // only show removed comments to admin when viewing user profile - if !(is_profile_view && is_admin) { - query = query.filter(comment::removed.eq(false)); - } - - if !self - .local_user - .map(|l| l.local_user.show_bot_accounts) - .unwrap_or(true) - { - query = query.filter(person::bot_account.eq(false)); - }; - - if self.local_user.is_some() { - // Filter out the rows with missing languages - query = query.filter(local_user_language::language_id.is_not_null()); - - // Don't show blocked communities or persons - if self.post_id.is_none() { - query = query.filter(community_block::person_id.is_null()); - } - query = query.filter(person_block::person_id.is_null()); - } - - // A Max depth given means its a tree fetch - let (limit, offset) = if let Some(max_depth) = self.max_depth { - let depth_limit = if let Some(parent_path) = self.parent_path.as_ref() { - parent_path.0.split('.').count() as i32 + max_depth - // Add one because of root "0" - } else { - max_depth + 1 - }; - - query = query.filter(nlevel(comment::path).le(depth_limit)); - - // only order if filtering by a post id. DOS potential otherwise and max_depth + !post_id isn't used anyways (afaik) - if self.post_id.is_some() { - // Always order by the parent path first - query = query.order_by(subpath(comment::path, 0, -1)); - } - - // TODO limit question. Limiting does not work for comment threads ATM, only max_depth - // For now, don't do any limiting for tree fetches - // https://stackoverflow.com/questions/72983614/postgres-ltree-how-to-limit-the-max-number-of-children-at-any-given-level - - // Don't use the regular error-checking one, many more comments must ofter be fetched. - // This does not work for comment trees, and the limit should be manually set to a high number - // - // If a max depth is given, then you know its a tree fetch, and limits should be ignored - // TODO a kludge to prevent attacks. Limit comments to 300 for now. - // (i64::MAX, 0) - (300, 0) - } else { - // limit_and_offset_unlimited(self.page, self.limit) - limit_and_offset(self.page, self.limit)? - }; - - query = match self.sort.unwrap_or(CommentSortType::Hot) { - CommentSortType::Hot => query - .then_order_by(comment_aggregates::hot_rank.desc()) - .then_order_by(comment_aggregates::score.desc()), - CommentSortType::Controversial => { - query.then_order_by(comment_aggregates::controversy_rank.desc()) - } - CommentSortType::New => query.then_order_by(comment::published.desc()), - CommentSortType::Old => query.then_order_by(comment::published.asc()), - CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), - }; - - // Note: deleted and removed comments are done on the front side - let res = query - .limit(limit) - .offset(offset) - .load::(conn) - .await?; - - Ok(res.into_iter().map(CommentView::from_tuple).collect()) + queries().list(pool, self).await } } diff --git a/crates/db_views/src/local_user_view.rs b/crates/db_views/src/local_user_view.rs index 567ca3feb..23a8b8f05 100644 --- a/crates/db_views/src/local_user_view.rs +++ b/crates/db_views/src/local_user_view.rs @@ -7,136 +7,102 @@ use lemmy_db_schema::{ schema::{local_user, person, person_aggregates}, source::{local_user::LocalUser, person::Person}, traits::JoinView, - utils::{functions::lower, get_conn, DbPool}, + utils::{functions::lower, DbConn, DbPool, ListFn, Queries, ReadFn}, }; type LocalUserViewTuple = (LocalUser, Person, PersonAggregates); +enum ReadBy<'a> { + Id(LocalUserId), + Person(PersonId), + Name(&'a str), + NameOrEmail(&'a str), + Email(&'a str), +} + +enum ListMode { + AdminsWithEmails, +} + +fn queries<'a>( +) -> Queries>, impl ListFn<'a, LocalUserView, ListMode>> { + let selection = ( + local_user::all_columns, + person::all_columns, + person_aggregates::all_columns, + ); + + let read = move |mut conn: DbConn<'a>, search: ReadBy<'a>| async move { + let mut query = local_user::table.into_boxed(); + query = match search { + ReadBy::Id(local_user_id) => query.filter(local_user::id.eq(local_user_id)), + ReadBy::Email(from_email) => query.filter(local_user::email.eq(from_email)), + _ => query, + }; + let mut query = query.inner_join(person::table); + query = match search { + ReadBy::Person(person_id) => query.filter(person::id.eq(person_id)), + ReadBy::Name(name) => query.filter(lower(person::name).eq(name.to_lowercase())), + ReadBy::NameOrEmail(name_or_email) => query.filter( + lower(person::name) + .eq(lower(name_or_email)) + .or(local_user::email.eq(name_or_email)), + ), + _ => query, + }; + query + .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) + .select(selection) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, mode: ListMode| async move { + match mode { + ListMode::AdminsWithEmails => { + local_user::table + .filter(local_user::email.is_not_null()) + .filter(person::admin.eq(true)) + .inner_join(person::table) + .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) + .select(selection) + .load::(&mut conn) + .await + } + } + }; + + Queries::new(read, list) +} + impl LocalUserView { pub async fn read(pool: &mut DbPool<'_>, local_user_id: LocalUserId) -> Result { - let conn = &mut get_conn(pool).await?; - - let (local_user, person, counts) = local_user::table - .find(local_user_id) - .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) - .select(( - local_user::all_columns, - person::all_columns, - person_aggregates::all_columns, - )) - .first::(conn) - .await?; - Ok(Self { - local_user, - person, - counts, - }) + queries().read(pool, ReadBy::Id(local_user_id)).await } pub async fn read_person(pool: &mut DbPool<'_>, person_id: PersonId) -> Result { - let conn = &mut get_conn(pool).await?; - let (local_user, person, counts) = local_user::table - .filter(person::id.eq(person_id)) - .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) - .select(( - local_user::all_columns, - person::all_columns, - person_aggregates::all_columns, - )) - .first::(conn) - .await?; - Ok(Self { - local_user, - person, - counts, - }) + queries().read(pool, ReadBy::Person(person_id)).await } pub async fn read_from_name(pool: &mut DbPool<'_>, name: &str) -> Result { - let conn = &mut get_conn(pool).await?; - let (local_user, person, counts) = local_user::table - .filter(lower(person::name).eq(name.to_lowercase())) - .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) - .select(( - local_user::all_columns, - person::all_columns, - person_aggregates::all_columns, - )) - .first::(conn) - .await?; - Ok(Self { - local_user, - person, - counts, - }) + queries().read(pool, ReadBy::Name(name)).await } pub async fn find_by_email_or_name( pool: &mut DbPool<'_>, name_or_email: &str, ) -> Result { - let conn = &mut get_conn(pool).await?; - let (local_user, person, counts) = local_user::table - .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) - .filter( - lower(person::name) - .eq(lower(name_or_email)) - .or(local_user::email.eq(name_or_email)), - ) - .select(( - local_user::all_columns, - person::all_columns, - person_aggregates::all_columns, - )) - .first::(conn) - .await?; - Ok(Self { - local_user, - person, - counts, - }) + queries() + .read(pool, ReadBy::NameOrEmail(name_or_email)) + .await } pub async fn find_by_email(pool: &mut DbPool<'_>, from_email: &str) -> Result { - let conn = &mut get_conn(pool).await?; - let (local_user, person, counts) = local_user::table - .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) - .filter(local_user::email.eq(from_email)) - .select(( - local_user::all_columns, - person::all_columns, - person_aggregates::all_columns, - )) - .first::(conn) - .await?; - Ok(Self { - local_user, - person, - counts, - }) + queries().read(pool, ReadBy::Email(from_email)).await } pub async fn list_admins_with_emails(pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let res = local_user::table - .filter(person::admin.eq(true)) - .filter(local_user::email.is_not_null()) - .inner_join(person::table) - .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) - .select(( - local_user::all_columns, - person::all_columns, - person_aggregates::all_columns, - )) - .load::(conn) - .await?; - - Ok(res.into_iter().map(LocalUserView::from_tuple).collect()) + queries().list(pool, ListMode::AdminsWithEmails).await } } diff --git a/crates/db_views/src/post_report_view.rs b/crates/db_views/src/post_report_view.rs index 8c47d8c52..4ef6067fd 100644 --- a/crates/db_views/src/post_report_view.rs +++ b/crates/db_views/src/post_report_view.rs @@ -1,5 +1,6 @@ use crate::structs::PostReportView; use diesel::{ + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -10,6 +11,7 @@ use diesel::{ use diesel_async::RunQueryDsl; use lemmy_db_schema::{ aggregates::structs::PostAggregates, + aliases, newtypes::{CommunityId, PersonId, PostReportId}, schema::{ community, @@ -28,7 +30,7 @@ use lemmy_db_schema::{ post_report::PostReport, }, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, }; type PostReportViewTuple = ( @@ -43,34 +45,16 @@ type PostReportViewTuple = ( Option, ); -impl PostReportView { - /// returns the PostReportView for the provided report_id - /// - /// * `report_id` - the report id to obtain - pub async fn read( - pool: &mut DbPool<'_>, - report_id: PostReportId, - my_person_id: PersonId, - ) -> Result { - let conn = &mut get_conn(pool).await?; - let (person_alias_1, person_alias_2) = diesel::alias!(person as person1, person as person2); - - let ( - post_report, - post, - community, - creator, - post_creator, - creator_banned_from_community, - post_like, - counts, - resolver, - ) = post_report::table - .find(report_id) +fn queries<'a>() -> Queries< + impl ReadFn<'a, PostReportView, (PostReportId, PersonId)>, + impl ListFn<'a, PostReportView, (PostReportQuery, &'a Person)>, +> { + let all_joins = |query: post_report::BoxedQuery<'a, Pg>, my_person_id: PersonId| { + query .inner_join(post::table) .inner_join(community::table.on(post::community_id.eq(community::id))) .inner_join(person::table.on(post_report::creator_id.eq(person::id))) - .inner_join(person_alias_1.on(post::creator_id.eq(person_alias_1.field(person::id)))) + .inner_join(aliases::person1.on(post::creator_id.eq(aliases::person1.field(person::id)))) .left_join( community_person_ban::table.on( post::community_id @@ -87,35 +71,79 @@ impl PostReportView { ) .inner_join(post_aggregates::table.on(post_report::post_id.eq(post_aggregates::post_id))) .left_join( - person_alias_2.on(post_report::resolver_id.eq(person_alias_2.field(person::id).nullable())), + aliases::person2 + .on(post_report::resolver_id.eq(aliases::person2.field(person::id).nullable())), ) .select(( post_report::all_columns, post::all_columns, community::all_columns, person::all_columns, - person_alias_1.fields(person::all_columns), + aliases::person1.fields(person::all_columns), community_person_ban::all_columns.nullable(), post_like::score.nullable(), post_aggregates::all_columns, - person_alias_2.fields(person::all_columns.nullable()), + aliases::person2.fields(person::all_columns.nullable()), )) - .first::(conn) - .await?; + }; - let my_vote = post_like; + let read = move |mut conn: DbConn<'a>, (report_id, my_person_id): (PostReportId, PersonId)| async move { + all_joins( + post_report::table.find(report_id).into_boxed(), + my_person_id, + ) + .first::(&mut conn) + .await + }; - Ok(Self { - post_report, - post, - community, - creator, - post_creator, - creator_banned_from_community: creator_banned_from_community.is_some(), - my_vote, - counts, - resolver, - }) + let list = move |mut conn: DbConn<'a>, (options, my_person): (PostReportQuery, &'a Person)| async move { + let mut query = all_joins(post_report::table.into_boxed(), my_person.id); + + if let Some(community_id) = options.community_id { + query = query.filter(post::community_id.eq(community_id)); + } + + if options.unresolved_only.unwrap_or(false) { + query = query.filter(post_report::resolved.eq(false)); + } + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query = query + .order_by(post_report::published.desc()) + .limit(limit) + .offset(offset); + + // If its not an admin, get only the ones you mod + if !my_person.admin { + query + .inner_join( + community_moderator::table.on( + community_moderator::community_id + .eq(post::community_id) + .and(community_moderator::person_id.eq(my_person.id)), + ), + ) + .load::(&mut conn) + .await + } else { + query.load::(&mut conn).await + } + }; + + Queries::new(read, list) +} + +impl PostReportView { + /// returns the PostReportView for the provided report_id + /// + /// * `report_id` - the report id to obtain + pub async fn read( + pool: &mut DbPool<'_>, + report_id: PostReportId, + my_person_id: PersonId, + ) -> Result { + queries().read(pool, (report_id, my_person_id)).await } /// returns the current unresolved post report count for the communities you mod @@ -172,77 +200,7 @@ impl PostReportQuery { pool: &mut DbPool<'_>, my_person: &Person, ) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let (person_alias_1, person_alias_2) = diesel::alias!(person as person1, person as person2); - - let mut query = post_report::table - .inner_join(post::table) - .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person::table.on(post_report::creator_id.eq(person::id))) - .inner_join(person_alias_1.on(post::creator_id.eq(person_alias_1.field(person::id)))) - .left_join( - community_person_ban::table.on( - post::community_id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(post::creator_id)), - ), - ) - .left_join( - post_like::table.on( - post::id - .eq(post_like::post_id) - .and(post_like::person_id.eq(my_person.id)), - ), - ) - .inner_join(post_aggregates::table.on(post_report::post_id.eq(post_aggregates::post_id))) - .left_join( - person_alias_2.on(post_report::resolver_id.eq(person_alias_2.field(person::id).nullable())), - ) - .select(( - post_report::all_columns, - post::all_columns, - community::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - community_person_ban::all_columns.nullable(), - post_like::score.nullable(), - post_aggregates::all_columns, - person_alias_2.fields(person::all_columns.nullable()), - )) - .into_boxed(); - - if let Some(community_id) = self.community_id { - query = query.filter(post::community_id.eq(community_id)); - } - - if self.unresolved_only.unwrap_or(false) { - query = query.filter(post_report::resolved.eq(false)); - } - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - query = query - .order_by(post_report::published.desc()) - .limit(limit) - .offset(offset); - - // If its not an admin, get only the ones you mod - let res = if !my_person.admin { - query - .inner_join( - community_moderator::table.on( - community_moderator::community_id - .eq(post::community_id) - .and(community_moderator::person_id.eq(my_person.id)), - ), - ) - .load::(conn) - .await? - } else { - query.load::(conn).await? - }; - - Ok(res.into_iter().map(PostReportView::from_tuple).collect()) + queries().list(pool, (self, my_person)).await } } diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index e02310928..9f6d0735f 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -40,7 +40,7 @@ use lemmy_db_schema::{ post::{Post, PostRead, PostSaved}, }, traits::JoinView, - utils::{fuzzy_search, get_conn, limit_and_offset, DbPool}, + utils::{fuzzy_search, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, ListingType, SortType, }; @@ -62,19 +62,15 @@ type PostViewTuple = ( sql_function!(fn coalesce(x: sql_types::Nullable, y: sql_types::BigInt) -> sql_types::BigInt); -impl PostView { - pub async fn read( - pool: &mut DbPool<'_>, - post_id: PostId, - my_person_id: Option, - is_mod_or_admin: Option, - ) -> Result { - let conn = &mut get_conn(pool).await?; - +fn queries<'a>() -> Queries< + impl ReadFn<'a, PostView, (PostId, Option, Option)>, + impl ListFn<'a, PostView, PostQuery<'a>>, +> { + let all_joins = |query: post_aggregates::BoxedQuery<'a, Pg>, my_person_id: Option| { // The left join below will return None in this case let person_id_join = my_person_id.unwrap_or(PersonId(-1)); - let mut query = post_aggregates::table - .filter(post_aggregates::post_id.eq(post_id)) + + query .inner_join(person::table) .inner_join(community::table) .left_join( @@ -134,23 +130,41 @@ impl PostView { .and(person_post_aggregates::person_id.eq(person_id_join)), ), ) - .select(( - post::all_columns, - person::all_columns, - community::all_columns, - community_person_ban::all_columns.nullable(), - post_aggregates::all_columns, - community_follower::all_columns.nullable(), - post_saved::all_columns.nullable(), - post_read::all_columns.nullable(), - person_block::all_columns.nullable(), - post_like::score.nullable(), - coalesce( - post_aggregates::comments.nullable() - person_post_aggregates::read_comments.nullable(), - post_aggregates::comments, - ), - )) - .into_boxed(); + }; + + let selection = ( + post::all_columns, + person::all_columns, + community::all_columns, + community_person_ban::all_columns.nullable(), + post_aggregates::all_columns, + community_follower::all_columns.nullable(), + post_saved::all_columns.nullable(), + post_read::all_columns.nullable(), + person_block::all_columns.nullable(), + post_like::score.nullable(), + coalesce( + post_aggregates::comments.nullable() - person_post_aggregates::read_comments.nullable(), + post_aggregates::comments, + ), + ); + + let read = move |mut conn: DbConn<'a>, + (post_id, my_person_id, is_mod_or_admin): ( + PostId, + Option, + Option, + )| async move { + // The left join below will return None in this case + let person_id_join = my_person_id.unwrap_or(PersonId(-1)); + + let mut query = all_joins( + post_aggregates::table + .filter(post_aggregates::post_id.eq(post_id)) + .into_boxed(), + my_person_id, + ) + .select(selection); // Hide deleted and removed for non-admins or mods if !is_mod_or_admin.unwrap_or(false) { @@ -170,117 +184,18 @@ impl PostView { ); } - let ( - post, - creator, - community, - creator_banned_from_community, - counts, - follower, - saved, - read, - creator_blocked, - post_like, - unread_comments, - ) = query.first::(conn).await?; + query.first::(&mut conn).await + }; - // If a person is given, then my_vote, if None, should be 0, not null - // Necessary to differentiate between other person's votes - let my_vote = if my_person_id.is_some() && post_like.is_none() { - Some(0) - } else { - post_like - }; - - Ok(PostView { - post, - creator, - community, - creator_banned_from_community: creator_banned_from_community.is_some(), - counts, - subscribed: CommunityFollower::to_subscribed_type(&follower), - saved: saved.is_some(), - read: read.is_some(), - creator_blocked: creator_blocked.is_some(), - my_vote, - unread_comments, - }) - } -} - -#[derive(Default)] -pub struct PostQuery<'a> { - pub listing_type: Option, - pub sort: Option, - pub creator_id: Option, - pub community_id: Option, - pub local_user: Option<&'a LocalUserView>, - pub search_term: Option, - pub url_search: Option, - pub saved_only: Option, - pub moderator_view: Option, - pub is_profile_view: Option, - pub page: Option, - pub limit: Option, -} - -impl<'a> PostQuery<'a> { - pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; + let list = move |mut conn: DbConn<'a>, options: PostQuery<'a>| async move { + let person_id = options.local_user.map(|l| l.person.id); + let local_user_id = options.local_user.map(|l| l.local_user.id); // The left join below will return None in this case - let person_id_join = self.local_user.map(|l| l.person.id).unwrap_or(PersonId(-1)); - let local_user_id_join = self - .local_user - .map(|l| l.local_user.id) - .unwrap_or(LocalUserId(-1)); + let person_id_join = person_id.unwrap_or(PersonId(-1)); + let local_user_id_join = local_user_id.unwrap_or(LocalUserId(-1)); - let mut query = post_aggregates::table - .inner_join(person::table) - .inner_join(post::table) - .inner_join(community::table) - .left_join( - community_person_ban::table.on( - post_aggregates::community_id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(post_aggregates::creator_id)), - ), - ) - .left_join( - community_follower::table.on( - post_aggregates::community_id - .eq(community_follower::community_id) - .and(community_follower::person_id.eq(person_id_join)), - ), - ) - .left_join( - community_moderator::table.on( - post::community_id - .eq(community_moderator::community_id) - .and(community_moderator::person_id.eq(person_id_join)), - ), - ) - .left_join( - post_saved::table.on( - post_aggregates::post_id - .eq(post_saved::post_id) - .and(post_saved::person_id.eq(person_id_join)), - ), - ) - .left_join( - post_read::table.on( - post_aggregates::post_id - .eq(post_read::post_id) - .and(post_read::person_id.eq(person_id_join)), - ), - ) - .left_join( - person_block::table.on( - post_aggregates::creator_id - .eq(person_block::target_id) - .and(person_block::person_id.eq(person_id_join)), - ), - ) + let mut query = all_joins(post_aggregates::table.into_boxed(), person_id) .left_join( community_block::table.on( post_aggregates::community_id @@ -288,20 +203,6 @@ impl<'a> PostQuery<'a> { .and(community_block::person_id.eq(person_id_join)), ), ) - .left_join( - post_like::table.on( - post_aggregates::post_id - .eq(post_like::post_id) - .and(post_like::person_id.eq(person_id_join)), - ), - ) - .left_join( - person_post_aggregates::table.on( - post_aggregates::post_id - .eq(person_post_aggregates::post_id) - .and(person_post_aggregates::person_id.eq(person_id_join)), - ), - ) .left_join( local_user_language::table.on( post::language_id @@ -309,26 +210,10 @@ impl<'a> PostQuery<'a> { .and(local_user_language::local_user_id.eq(local_user_id_join)), ), ) - .select(( - post::all_columns, - person::all_columns, - community::all_columns, - community_person_ban::all_columns.nullable(), - post_aggregates::all_columns, - community_follower::all_columns.nullable(), - post_saved::all_columns.nullable(), - post_read::all_columns.nullable(), - person_block::all_columns.nullable(), - post_like::score.nullable(), - coalesce( - post_aggregates::comments.nullable() - person_post_aggregates::read_comments.nullable(), - post_aggregates::comments, - ), - )) - .into_boxed(); + .select(selection); - let is_profile_view = self.is_profile_view.unwrap_or(false); - let is_creator = self.creator_id == self.local_user.map(|l| l.person.id); + let is_profile_view = options.is_profile_view.unwrap_or(false); + let is_creator = options.creator_id == options.local_user.map(|l| l.person.id); // only show deleted posts to creator if is_creator { query = query @@ -336,7 +221,7 @@ impl<'a> PostQuery<'a> { .filter(post::deleted.eq(false)); } - let is_admin = self.local_user.map(|l| l.person.admin).unwrap_or(false); + let is_admin = options.local_user.map(|l| l.person.admin).unwrap_or(false); // only show removed posts to admin when viewing user profile if !(is_profile_view && is_admin) { query = query @@ -344,19 +229,19 @@ impl<'a> PostQuery<'a> { .filter(post::removed.eq(false)); } - if self.community_id.is_none() { + if options.community_id.is_none() { query = query.then_order_by(post_aggregates::featured_local.desc()); - } else if let Some(community_id) = self.community_id { + } else if let Some(community_id) = options.community_id { query = query .filter(post_aggregates::community_id.eq(community_id)) .then_order_by(post_aggregates::featured_community.desc()); } - if let Some(creator_id) = self.creator_id { + if let Some(creator_id) = options.creator_id { query = query.filter(post_aggregates::creator_id.eq(creator_id)); } - if let Some(listing_type) = self.listing_type { + if let Some(listing_type) = options.listing_type { match listing_type { ListingType::Subscribed => { query = query.filter(community_follower::person_id.is_not_null()) @@ -378,11 +263,11 @@ impl<'a> PostQuery<'a> { } } - if let Some(url_search) = self.url_search { + if let Some(url_search) = options.url_search { query = query.filter(post::url.eq(url_search)); } - if let Some(search_term) = self.search_term { + if let Some(search_term) = options.search_term { let searcher = fuzzy_search(&search_term); query = query.filter( post::name @@ -391,7 +276,7 @@ impl<'a> PostQuery<'a> { ); } - if !self + if !options .local_user .map(|l| l.local_user.show_nsfw) .unwrap_or(false) @@ -401,7 +286,7 @@ impl<'a> PostQuery<'a> { .filter(community::nsfw.eq(false)); }; - if !self + if !options .local_user .map(|l| l.local_user.show_bot_accounts) .unwrap_or(true) @@ -409,16 +294,16 @@ impl<'a> PostQuery<'a> { query = query.filter(person::bot_account.eq(false)); }; - if self.saved_only.unwrap_or(false) { + if options.saved_only.unwrap_or(false) { query = query.filter(post_saved::post_id.is_not_null()); } - if self.moderator_view.unwrap_or(false) { + if options.moderator_view.unwrap_or(false) { query = query.filter(community_moderator::person_id.is_not_null()); } // Only hide the read posts, if the saved_only is false. Otherwise ppl with the hide_read // setting wont be able to see saved posts. - else if !self + else if !options .local_user .map(|l| l.local_user.show_read_posts) .unwrap_or(true) @@ -426,18 +311,18 @@ impl<'a> PostQuery<'a> { query = query.filter(post_read::post_id.is_null()); } - if self.local_user.is_some() { + if options.local_user.is_some() { // Filter out the rows with missing languages query = query.filter(local_user_language::language_id.is_not_null()); // Don't show blocked communities or persons query = query.filter(community_block::person_id.is_null()); - if !self.moderator_view.unwrap_or(false) { + if !options.moderator_view.unwrap_or(false) { query = query.filter(person_block::person_id.is_null()); } } - query = match self.sort.unwrap_or(SortType::Hot) { + query = match options.sort.unwrap_or(SortType::Hot) { SortType::Active => query .then_order_by(post_aggregates::hot_rank_active.desc()) .then_order_by(post_aggregates::published.desc()), @@ -496,15 +381,58 @@ impl<'a> PostQuery<'a> { .then_order_by(post_aggregates::published.desc()), }; - let (limit, offset) = limit_and_offset(self.page, self.limit)?; + let (limit, offset) = limit_and_offset(options.page, options.limit)?; query = query.limit(limit).offset(offset); debug!("Post View Query: {:?}", debug_query::(&query)); - let res = query.load::(conn).await?; + query.load::(&mut conn).await + }; - Ok(res.into_iter().map(PostView::from_tuple).collect()) + Queries::new(read, list) +} + +impl PostView { + pub async fn read( + pool: &mut DbPool<'_>, + post_id: PostId, + my_person_id: Option, + is_mod_or_admin: Option, + ) -> Result { + let mut res = queries() + .read(pool, (post_id, my_person_id, is_mod_or_admin)) + .await?; + + // If a person is given, then my_vote, if None, should be 0, not null + // Necessary to differentiate between other person's votes + if my_person_id.is_some() && res.my_vote.is_none() { + res.my_vote = Some(0) + }; + + Ok(res) + } +} + +#[derive(Default)] +pub struct PostQuery<'a> { + pub listing_type: Option, + pub sort: Option, + pub creator_id: Option, + pub community_id: Option, + pub local_user: Option<&'a LocalUserView>, + pub search_term: Option, + pub url_search: Option, + pub saved_only: Option, + pub moderator_view: Option, + pub is_profile_view: Option, + pub page: Option, + pub limit: Option, +} + +impl<'a> PostQuery<'a> { + pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { + queries().list(pool, self).await } } diff --git a/crates/db_views/src/private_message_report_view.rs b/crates/db_views/src/private_message_report_view.rs index 7ceca271a..ce87abaec 100644 --- a/crates/db_views/src/private_message_report_view.rs +++ b/crates/db_views/src/private_message_report_view.rs @@ -1,7 +1,15 @@ use crate::structs::PrivateMessageReportView; -use diesel::{result::Error, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl}; +use diesel::{ + pg::Pg, + result::Error, + ExpressionMethods, + JoinOnDsl, + NullableExpressionMethods, + QueryDsl, +}; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ + aliases, newtypes::PrivateMessageReportId, schema::{person, private_message, private_message_report}, source::{ @@ -10,7 +18,7 @@ use lemmy_db_schema::{ private_message_report::PrivateMessageReport, }, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, }; type PrivateMessageReportViewTuple = ( @@ -21,6 +29,57 @@ type PrivateMessageReportViewTuple = ( Option, ); +fn queries<'a>() -> Queries< + impl ReadFn<'a, PrivateMessageReportView, PrivateMessageReportId>, + impl ListFn<'a, PrivateMessageReportView, PrivateMessageReportQuery>, +> { + let all_joins = + |query: private_message_report::BoxedQuery<'a, Pg>| { + query + .inner_join(private_message::table) + .inner_join(person::table.on(private_message::creator_id.eq(person::id))) + .inner_join( + aliases::person1 + .on(private_message_report::creator_id.eq(aliases::person1.field(person::id))), + ) + .left_join(aliases::person2.on( + private_message_report::resolver_id.eq(aliases::person2.field(person::id).nullable()), + )) + .select(( + private_message_report::all_columns, + private_message::all_columns, + person::all_columns, + aliases::person1.fields(person::all_columns), + aliases::person2.fields(person::all_columns).nullable(), + )) + }; + + let read = move |mut conn: DbConn<'a>, report_id: PrivateMessageReportId| async move { + all_joins(private_message_report::table.find(report_id).into_boxed()) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, options: PrivateMessageReportQuery| async move { + let mut query = all_joins(private_message_report::table.into_boxed()); + + if options.unresolved_only.unwrap_or(false) { + query = query.filter(private_message_report::resolved.eq(false)); + } + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query + .order_by(private_message::published.desc()) + .limit(limit) + .offset(offset) + .load::(&mut conn) + .await + }; + + Queries::new(read, list) +} + impl PrivateMessageReportView { /// returns the PrivateMessageReportView for the provided report_id /// @@ -29,40 +88,7 @@ impl PrivateMessageReportView { pool: &mut DbPool<'_>, report_id: PrivateMessageReportId, ) -> Result { - let conn = &mut get_conn(pool).await?; - let (person_alias_1, person_alias_2) = diesel::alias!(person as person1, person as person2); - - let (private_message_report, private_message, private_message_creator, creator, resolver) = - private_message_report::table - .find(report_id) - .inner_join(private_message::table) - .inner_join(person::table.on(private_message::creator_id.eq(person::id))) - .inner_join( - person_alias_1 - .on(private_message_report::creator_id.eq(person_alias_1.field(person::id))), - ) - .left_join( - person_alias_2.on( - private_message_report::resolver_id.eq(person_alias_2.field(person::id).nullable()), - ), - ) - .select(( - private_message_report::all_columns, - private_message::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - person_alias_2.fields(person::all_columns).nullable(), - )) - .first::(conn) - .await?; - - Ok(Self { - private_message_report, - private_message, - private_message_creator, - creator, - resolver, - }) + queries().read(pool, report_id).await } /// Returns the current unresolved post report count for the communities you mod @@ -89,47 +115,7 @@ pub struct PrivateMessageReportQuery { impl PrivateMessageReportQuery { pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let (person_alias_1, person_alias_2) = diesel::alias!(person as person1, person as person2); - - let mut query = private_message_report::table - .inner_join(private_message::table) - .inner_join(person::table.on(private_message::creator_id.eq(person::id))) - .inner_join( - person_alias_1.on(private_message_report::creator_id.eq(person_alias_1.field(person::id))), - ) - .left_join( - person_alias_2 - .on(private_message_report::resolver_id.eq(person_alias_2.field(person::id).nullable())), - ) - .select(( - private_message_report::all_columns, - private_message::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - person_alias_2.fields(person::all_columns).nullable(), - )) - .into_boxed(); - - if self.unresolved_only.unwrap_or(false) { - query = query.filter(private_message_report::resolved.eq(false)); - } - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - query = query - .order_by(private_message::published.desc()) - .limit(limit) - .offset(offset); - - let res = query.load::(conn).await?; - - Ok( - res - .into_iter() - .map(PrivateMessageReportView::from_tuple) - .collect(), - ) + queries().list(pool, self).await } } diff --git a/crates/db_views/src/private_message_view.rs b/crates/db_views/src/private_message_view.rs index 863db8125..55d6583ba 100644 --- a/crates/db_views/src/private_message_view.rs +++ b/crates/db_views/src/private_message_view.rs @@ -10,44 +10,87 @@ use diesel::{ }; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ + aliases, newtypes::{PersonId, PrivateMessageId}, schema::{person, private_message}, source::{person::Person, private_message::PrivateMessage}, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, }; use tracing::debug; type PrivateMessageViewTuple = (PrivateMessage, Person, Person); +fn queries<'a>() -> Queries< + impl ReadFn<'a, PrivateMessageView, PrivateMessageId>, + impl ListFn<'a, PrivateMessageView, (PrivateMessageQuery, PersonId)>, +> { + let all_joins = |query: private_message::BoxedQuery<'a, Pg>| { + query + .inner_join(person::table.on(private_message::creator_id.eq(person::id))) + .inner_join( + aliases::person1.on(private_message::recipient_id.eq(aliases::person1.field(person::id))), + ) + }; + + let selection = ( + private_message::all_columns, + person::all_columns, + aliases::person1.fields(person::all_columns), + ); + + let read = move |mut conn: DbConn<'a>, private_message_id: PrivateMessageId| async move { + all_joins(private_message::table.find(private_message_id).into_boxed()) + .order_by(private_message::published.desc()) + .select(selection) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, + (options, recipient_id): (PrivateMessageQuery, PersonId)| async move { + let mut query = all_joins(private_message::table.into_boxed()).select(selection); + + // If its unread, I only want the ones to me + if options.unread_only.unwrap_or(false) { + query = query + .filter(private_message::read.eq(false)) + .filter(private_message::recipient_id.eq(recipient_id)); + } + // Otherwise, I want the ALL view to show both sent and received + else { + query = query.filter( + private_message::recipient_id + .eq(recipient_id) + .or(private_message::creator_id.eq(recipient_id)), + ) + } + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query = query + .filter(private_message::deleted.eq(false)) + .limit(limit) + .offset(offset) + .order_by(private_message::published.desc()); + + debug!( + "Private Message View Query: {:?}", + debug_query::(&query) + ); + + query.load::(&mut conn).await + }; + + Queries::new(read, list) +} + impl PrivateMessageView { pub async fn read( pool: &mut DbPool<'_>, private_message_id: PrivateMessageId, ) -> Result { - let conn = &mut get_conn(pool).await?; - let person_alias_1 = diesel::alias!(person as person1); - - let (private_message, creator, recipient) = private_message::table - .find(private_message_id) - .inner_join(person::table.on(private_message::creator_id.eq(person::id))) - .inner_join( - person_alias_1.on(private_message::recipient_id.eq(person_alias_1.field(person::id))), - ) - .order_by(private_message::published.desc()) - .select(( - private_message::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - )) - .first::(conn) - .await?; - - Ok(PrivateMessageView { - private_message, - creator, - recipient, - }) + queries().read(pool, private_message_id).await } /// Gets the number of unread messages @@ -80,57 +123,7 @@ impl PrivateMessageQuery { pool: &mut DbPool<'_>, recipient_id: PersonId, ) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let person_alias_1 = diesel::alias!(person as person1); - - let mut query = private_message::table - .inner_join(person::table.on(private_message::creator_id.eq(person::id))) - .inner_join( - person_alias_1.on(private_message::recipient_id.eq(person_alias_1.field(person::id))), - ) - .select(( - private_message::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns), - )) - .into_boxed(); - - // If its unread, I only want the ones to me - if self.unread_only.unwrap_or(false) { - query = query - .filter(private_message::read.eq(false)) - .filter(private_message::recipient_id.eq(recipient_id)); - } - // Otherwise, I want the ALL view to show both sent and received - else { - query = query.filter( - private_message::recipient_id - .eq(recipient_id) - .or(private_message::creator_id.eq(recipient_id)), - ) - } - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - query = query - .filter(private_message::deleted.eq(false)) - .limit(limit) - .offset(offset) - .order_by(private_message::published.desc()); - - debug!( - "Private Message View Query: {:?}", - debug_query::(&query) - ); - - let res = query.load::(conn).await?; - - Ok( - res - .into_iter() - .map(PrivateMessageView::from_tuple) - .collect(), - ) + queries().list(pool, (self, recipient_id)).await } } diff --git a/crates/db_views/src/registration_application_view.rs b/crates/db_views/src/registration_application_view.rs index 8e2f58264..6064bf055 100644 --- a/crates/db_views/src/registration_application_view.rs +++ b/crates/db_views/src/registration_application_view.rs @@ -1,6 +1,7 @@ use crate::structs::RegistrationApplicationView; use diesel::{ dsl::count, + pg::Pg, result::Error, ExpressionMethods, JoinOnDsl, @@ -9,6 +10,7 @@ use diesel::{ }; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ + aliases, schema::{local_user, person, registration_application}, source::{ local_user::LocalUser, @@ -16,47 +18,75 @@ use lemmy_db_schema::{ registration_application::RegistrationApplication, }, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, }; type RegistrationApplicationViewTuple = (RegistrationApplication, LocalUser, Person, Option); +fn queries<'a>() -> Queries< + impl ReadFn<'a, RegistrationApplicationView, i32>, + impl ListFn<'a, RegistrationApplicationView, RegistrationApplicationQuery>, +> { + let all_joins = |query: registration_application::BoxedQuery<'a, Pg>| { + query + .inner_join(local_user::table.on(registration_application::local_user_id.eq(local_user::id))) + .inner_join(person::table.on(local_user::person_id.eq(person::id))) + .left_join( + aliases::person1 + .on(registration_application::admin_id.eq(aliases::person1.field(person::id).nullable())), + ) + .order_by(registration_application::published.desc()) + .select(( + registration_application::all_columns, + local_user::all_columns, + person::all_columns, + aliases::person1.fields(person::all_columns).nullable(), + )) + }; + + let read = move |mut conn: DbConn<'a>, registration_application_id: i32| async move { + all_joins( + registration_application::table + .find(registration_application_id) + .into_boxed(), + ) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, options: RegistrationApplicationQuery| async move { + let mut query = all_joins(registration_application::table.into_boxed()); + + if options.unread_only.unwrap_or(false) { + query = query.filter(registration_application::admin_id.is_null()) + } + + if options.verified_email_only.unwrap_or(false) { + query = query.filter(local_user::email_verified.eq(true)) + } + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query = query + .limit(limit) + .offset(offset) + .order_by(registration_application::published.desc()); + + query + .load::(&mut conn) + .await + }; + + Queries::new(read, list) +} + impl RegistrationApplicationView { pub async fn read( pool: &mut DbPool<'_>, registration_application_id: i32, ) -> Result { - let conn = &mut get_conn(pool).await?; - let person_alias_1 = diesel::alias!(person as person1); - - let (registration_application, creator_local_user, creator, admin) = - registration_application::table - .find(registration_application_id) - .inner_join( - local_user::table.on(registration_application::local_user_id.eq(local_user::id)), - ) - .inner_join(person::table.on(local_user::person_id.eq(person::id))) - .left_join( - person_alias_1 - .on(registration_application::admin_id.eq(person_alias_1.field(person::id).nullable())), - ) - .order_by(registration_application::published.desc()) - .select(( - registration_application::all_columns, - local_user::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns).nullable(), - )) - .first::(conn) - .await?; - - Ok(RegistrationApplicationView { - registration_application, - creator_local_user, - creator, - admin, - }) + queries().read(pool, registration_application_id).await } /// Returns the current unread registration_application count @@ -101,48 +131,7 @@ impl RegistrationApplicationQuery { self, pool: &mut DbPool<'_>, ) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let person_alias_1 = diesel::alias!(person as person1); - - let mut query = registration_application::table - .inner_join(local_user::table.on(registration_application::local_user_id.eq(local_user::id))) - .inner_join(person::table.on(local_user::person_id.eq(person::id))) - .left_join( - person_alias_1 - .on(registration_application::admin_id.eq(person_alias_1.field(person::id).nullable())), - ) - .order_by(registration_application::published.desc()) - .select(( - registration_application::all_columns, - local_user::all_columns, - person::all_columns, - person_alias_1.fields(person::all_columns).nullable(), - )) - .into_boxed(); - - if self.unread_only.unwrap_or(false) { - query = query.filter(registration_application::admin_id.is_null()) - } - - if self.verified_email_only.unwrap_or(false) { - query = query.filter(local_user::email_verified.eq(true)) - } - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - query = query - .limit(limit) - .offset(offset) - .order_by(registration_application::published.desc()); - - let res = query.load::(conn).await?; - - Ok( - res - .into_iter() - .map(RegistrationApplicationView::from_tuple) - .collect(), - ) + queries().list(pool, self).await } } diff --git a/crates/db_views_actor/src/comment_reply_view.rs b/crates/db_views_actor/src/comment_reply_view.rs index 406bfcb97..869345200 100644 --- a/crates/db_views_actor/src/comment_reply_view.rs +++ b/crates/db_views_actor/src/comment_reply_view.rs @@ -1,5 +1,6 @@ use crate::structs::CommentReplyView; use diesel::{ + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -10,6 +11,7 @@ use diesel::{ use diesel_async::RunQueryDsl; use lemmy_db_schema::{ aggregates::structs::CommentAggregates, + aliases, newtypes::{CommentReplyId, PersonId}, schema::{ comment, @@ -33,7 +35,7 @@ use lemmy_db_schema::{ post::Post, }, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, CommentSortType, }; @@ -52,38 +54,20 @@ type CommentReplyViewTuple = ( Option, ); -impl CommentReplyView { - pub async fn read( - pool: &mut DbPool<'_>, - comment_reply_id: CommentReplyId, - my_person_id: Option, - ) -> Result { - let conn = &mut get_conn(pool).await?; - let person_alias_1 = diesel::alias!(person as person1); - +fn queries<'a>() -> Queries< + impl ReadFn<'a, CommentReplyView, (CommentReplyId, Option)>, + impl ListFn<'a, CommentReplyView, CommentReplyQuery>, +> { + let all_joins = |query: comment_reply::BoxedQuery<'a, Pg>, my_person_id: Option| { // The left join below will return None in this case let person_id_join = my_person_id.unwrap_or(PersonId(-1)); - let ( - comment_reply, - comment, - creator, - post, - community, - recipient, - counts, - creator_banned_from_community, - follower, - saved, - creator_blocked, - my_vote, - ) = comment_reply::table - .find(comment_reply_id) + query .inner_join(comment::table) .inner_join(person::table.on(comment::creator_id.eq(person::id))) .inner_join(post::table.on(comment::post_id.eq(post::id))) .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person_alias_1) + .inner_join(aliases::person1) .inner_join(comment_aggregates::table.on(comment::id.eq(comment_aggregates::comment_id))) .left_join( community_person_ban::table.on( @@ -126,7 +110,7 @@ impl CommentReplyView { person::all_columns, post::all_columns, community::all_columns, - person_alias_1.fields(person::all_columns), + aliases::person1.fields(person::all_columns), comment_aggregates::all_columns, community_person_ban::all_columns.nullable(), community_follower::all_columns.nullable(), @@ -134,23 +118,63 @@ impl CommentReplyView { person_block::all_columns.nullable(), comment_like::score.nullable(), )) - .first::(conn) - .await?; + }; - Ok(CommentReplyView { - comment_reply, - comment, - creator, - post, - community, - recipient, - counts, - creator_banned_from_community: creator_banned_from_community.is_some(), - subscribed: CommunityFollower::to_subscribed_type(&follower), - saved: saved.is_some(), - creator_blocked: creator_blocked.is_some(), - my_vote, - }) + let read = + move |mut conn: DbConn<'a>, + (comment_reply_id, my_person_id): (CommentReplyId, Option)| async move { + all_joins( + comment_reply::table.find(comment_reply_id).into_boxed(), + my_person_id, + ) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, options: CommentReplyQuery| async move { + let mut query = all_joins(comment_reply::table.into_boxed(), options.my_person_id); + + if let Some(recipient_id) = options.recipient_id { + query = query.filter(comment_reply::recipient_id.eq(recipient_id)); + } + + if options.unread_only.unwrap_or(false) { + query = query.filter(comment_reply::read.eq(false)); + } + + if !options.show_bot_accounts.unwrap_or(true) { + query = query.filter(person::bot_account.eq(false)); + }; + + query = match options.sort.unwrap_or(CommentSortType::New) { + CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()), + CommentSortType::Controversial => { + query.then_order_by(comment_aggregates::controversy_rank.desc()) + } + CommentSortType::New => query.then_order_by(comment_reply::published.desc()), + CommentSortType::Old => query.then_order_by(comment_reply::published.asc()), + CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), + }; + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query + .limit(limit) + .offset(offset) + .load::(&mut conn) + .await + }; + + Queries::new(read, list) +} + +impl CommentReplyView { + pub async fn read( + pool: &mut DbPool<'_>, + comment_reply_id: CommentReplyId, + my_person_id: Option, + ) -> Result { + queries().read(pool, (comment_reply_id, my_person_id)).await } /// Gets the number of unread replies @@ -187,102 +211,7 @@ pub struct CommentReplyQuery { impl CommentReplyQuery { pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - - let person_alias_1 = diesel::alias!(person as person1); - - // The left join below will return None in this case - let person_id_join = self.my_person_id.unwrap_or(PersonId(-1)); - - let mut query = comment_reply::table - .inner_join(comment::table) - .inner_join(person::table.on(comment::creator_id.eq(person::id))) - .inner_join(post::table.on(comment::post_id.eq(post::id))) - .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person_alias_1) - .inner_join(comment_aggregates::table.on(comment::id.eq(comment_aggregates::comment_id))) - .left_join( - community_person_ban::table.on( - community::id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)), - ), - ) - .left_join( - community_follower::table.on( - post::community_id - .eq(community_follower::community_id) - .and(community_follower::person_id.eq(person_id_join)), - ), - ) - .left_join( - comment_saved::table.on( - comment::id - .eq(comment_saved::comment_id) - .and(comment_saved::person_id.eq(person_id_join)), - ), - ) - .left_join( - person_block::table.on( - comment::creator_id - .eq(person_block::target_id) - .and(person_block::person_id.eq(person_id_join)), - ), - ) - .left_join( - comment_like::table.on( - comment::id - .eq(comment_like::comment_id) - .and(comment_like::person_id.eq(person_id_join)), - ), - ) - .select(( - comment_reply::all_columns, - comment::all_columns, - person::all_columns, - post::all_columns, - community::all_columns, - person_alias_1.fields(person::all_columns), - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - community_follower::all_columns.nullable(), - comment_saved::all_columns.nullable(), - person_block::all_columns.nullable(), - comment_like::score.nullable(), - )) - .into_boxed(); - - if let Some(recipient_id) = self.recipient_id { - query = query.filter(comment_reply::recipient_id.eq(recipient_id)); - } - - if self.unread_only.unwrap_or(false) { - query = query.filter(comment_reply::read.eq(false)); - } - - if !self.show_bot_accounts.unwrap_or(true) { - query = query.filter(person::bot_account.eq(false)); - }; - - query = match self.sort.unwrap_or(CommentSortType::New) { - CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()), - CommentSortType::Controversial => { - query.then_order_by(comment_aggregates::controversy_rank.desc()) - } - CommentSortType::New => query.then_order_by(comment_reply::published.desc()), - CommentSortType::Old => query.then_order_by(comment_reply::published.asc()), - CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), - }; - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - let res = query - .limit(limit) - .offset(offset) - .load::(conn) - .await?; - - Ok(res.into_iter().map(CommentReplyView::from_tuple).collect()) + queries().list(pool, self).await } } diff --git a/crates/db_views_actor/src/community_view.rs b/crates/db_views_actor/src/community_view.rs index c31a2bd5d..9ca5c218c 100644 --- a/crates/db_views_actor/src/community_view.rs +++ b/crates/db_views_actor/src/community_view.rs @@ -1,5 +1,6 @@ use crate::structs::{CommunityModeratorView, CommunityView, PersonView}; use diesel::{ + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -19,7 +20,7 @@ use lemmy_db_schema::{ local_user::LocalUser, }, traits::JoinView, - utils::{fuzzy_search, get_conn, limit_and_offset, DbPool}, + utils::{fuzzy_search, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, ListingType, SortType, }; @@ -31,19 +32,15 @@ type CommunityViewTuple = ( Option, ); -impl CommunityView { - pub async fn read( - pool: &mut DbPool<'_>, - community_id: CommunityId, - my_person_id: Option, - is_mod_or_admin: Option, - ) -> Result { - let conn = &mut get_conn(pool).await?; +fn queries<'a>() -> Queries< + impl ReadFn<'a, CommunityView, (CommunityId, Option, Option)>, + impl ListFn<'a, CommunityView, CommunityQuery<'a>>, +> { + let all_joins = |query: community::BoxedQuery<'a, Pg>, my_person_id: Option| { // The left join below will return None in this case let person_id_join = my_person_id.unwrap_or(PersonId(-1)); - let mut query = community::table - .find(community_id) + query .inner_join(community_aggregates::table) .left_join( community_follower::table.on( @@ -59,29 +56,126 @@ impl CommunityView { .and(community_block::person_id.eq(person_id_join)), ), ) - .select(( - community::all_columns, - community_aggregates::all_columns, - community_follower::all_columns.nullable(), - community_block::all_columns.nullable(), - )) - .into_boxed(); + }; + + let selection = ( + community::all_columns, + community_aggregates::all_columns, + community_follower::all_columns.nullable(), + community_block::all_columns.nullable(), + ); + + let not_removed_or_deleted = community::removed + .eq(false) + .and(community::deleted.eq(false)); + + let read = move |mut conn: DbConn<'a>, + (community_id, my_person_id, is_mod_or_admin): ( + CommunityId, + Option, + Option, + )| async move { + let mut query = all_joins( + community::table.find(community_id).into_boxed(), + my_person_id, + ) + .select(selection); // Hide deleted and removed for non-admins or mods if !is_mod_or_admin.unwrap_or(false) { - query = query - .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)); + query = query.filter(not_removed_or_deleted); } - let (community, counts, follower, blocked) = query.first::(conn).await?; + query.first::(&mut conn).await + }; - Ok(CommunityView { - community, - subscribed: CommunityFollower::to_subscribed_type(&follower), - blocked: blocked.is_some(), - counts, - }) + let list = move |mut conn: DbConn<'a>, options: CommunityQuery<'a>| async move { + use SortType::*; + + let my_person_id = options.local_user.map(|l| l.person_id); + + // The left join below will return None in this case + let person_id_join = my_person_id.unwrap_or(PersonId(-1)); + + let mut query = all_joins(community::table.into_boxed(), my_person_id) + .left_join(local_user::table.on(local_user::person_id.eq(person_id_join))) + .select(selection); + + if let Some(search_term) = options.search_term { + let searcher = fuzzy_search(&search_term); + query = query + .filter(community::name.ilike(searcher.clone())) + .or_filter(community::title.ilike(searcher)) + } + + // Hide deleted and removed for non-admins or mods + if !options.is_mod_or_admin.unwrap_or(false) { + query = query.filter(not_removed_or_deleted).filter( + community::hidden + .eq(false) + .or(community_follower::person_id.eq(person_id_join)), + ); + } + + match options.sort.unwrap_or(Hot) { + Hot | Active => query = query.order_by(community_aggregates::hot_rank.desc()), + NewComments | TopDay | TopTwelveHour | TopSixHour | TopHour => { + query = query.order_by(community_aggregates::users_active_day.desc()) + } + New => query = query.order_by(community::published.desc()), + Old => query = query.order_by(community::published.asc()), + // Controversial is temporary until a CommentSortType is created + MostComments | Controversial => query = query.order_by(community_aggregates::comments.desc()), + TopAll | TopYear | TopNineMonths => { + query = query.order_by(community_aggregates::subscribers.desc()) + } + TopSixMonths | TopThreeMonths => { + query = query.order_by(community_aggregates::users_active_half_year.desc()) + } + TopMonth => query = query.order_by(community_aggregates::users_active_month.desc()), + TopWeek => query = query.order_by(community_aggregates::users_active_week.desc()), + }; + + if let Some(listing_type) = options.listing_type { + query = match listing_type { + ListingType::Subscribed => query.filter(community_follower::person_id.is_not_null()), // TODO could be this: and(community_follower::person_id.eq(person_id_join)), + ListingType::Local => query.filter(community::local.eq(true)), + _ => query, + }; + } + + // Don't show blocked communities or nsfw communities if not enabled in profile + if options.local_user.is_some() { + query = query.filter(community_block::person_id.is_null()); + query = query.filter(community::nsfw.eq(false).or(local_user::show_nsfw.eq(true))); + } else { + // No person in request, only show nsfw communities if show_nsfw is passed into request + if !options.show_nsfw.unwrap_or(false) { + query = query.filter(community::nsfw.eq(false)); + } + } + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + query + .limit(limit) + .offset(offset) + .load::(&mut conn) + .await + }; + + Queries::new(read, list) +} + +impl CommunityView { + pub async fn read( + pool: &mut DbPool<'_>, + community_id: CommunityId, + my_person_id: Option, + is_mod_or_admin: Option, + ) -> Result { + queries() + .read(pool, (community_id, my_person_id, is_mod_or_admin)) + .await } pub async fn is_mod_or_admin( @@ -113,102 +207,7 @@ pub struct CommunityQuery<'a> { impl<'a> CommunityQuery<'a> { pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - use SortType::*; - - let conn = &mut get_conn(pool).await?; - - // The left join below will return None in this case - let person_id_join = self.local_user.map(|l| l.person_id).unwrap_or(PersonId(-1)); - - let mut query = community::table - .inner_join(community_aggregates::table) - .left_join(local_user::table.on(local_user::person_id.eq(person_id_join))) - .left_join( - community_follower::table.on( - community::id - .eq(community_follower::community_id) - .and(community_follower::person_id.eq(person_id_join)), - ), - ) - .left_join( - community_block::table.on( - community::id - .eq(community_block::community_id) - .and(community_block::person_id.eq(person_id_join)), - ), - ) - .select(( - community::all_columns, - community_aggregates::all_columns, - community_follower::all_columns.nullable(), - community_block::all_columns.nullable(), - )) - .into_boxed(); - - if let Some(search_term) = self.search_term { - let searcher = fuzzy_search(&search_term); - query = query - .filter(community::name.ilike(searcher.clone())) - .or_filter(community::title.ilike(searcher)); - }; - - // Hide deleted and removed for non-admins or mods - if !self.is_mod_or_admin.unwrap_or(false) { - query = query - .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)) - .filter( - community::hidden - .eq(false) - .or(community_follower::person_id.eq(person_id_join)), - ); - } - match self.sort.unwrap_or(Hot) { - Hot | Active => query = query.order_by(community_aggregates::hot_rank.desc()), - NewComments | TopDay | TopTwelveHour | TopSixHour | TopHour => { - query = query.order_by(community_aggregates::users_active_day.desc()) - } - New => query = query.order_by(community::published.desc()), - Old => query = query.order_by(community::published.asc()), - // Controversial is temporary until a CommentSortType is created - MostComments | Controversial => query = query.order_by(community_aggregates::comments.desc()), - TopAll | TopYear | TopNineMonths => { - query = query.order_by(community_aggregates::subscribers.desc()) - } - TopSixMonths | TopThreeMonths => { - query = query.order_by(community_aggregates::users_active_half_year.desc()) - } - TopMonth => query = query.order_by(community_aggregates::users_active_month.desc()), - TopWeek => query = query.order_by(community_aggregates::users_active_week.desc()), - }; - - if let Some(listing_type) = self.listing_type { - query = match listing_type { - ListingType::Subscribed => query.filter(community_follower::person_id.is_not_null()), // TODO could be this: and(community_follower::person_id.eq(person_id_join)), - ListingType::Local => query.filter(community::local.eq(true)), - _ => query, - }; - } - - // Don't show blocked communities or nsfw communities if not enabled in profile - if self.local_user.is_some() { - query = query.filter(community_block::person_id.is_null()); - query = query.filter(community::nsfw.eq(false).or(local_user::show_nsfw.eq(true))); - } else { - // No person in request, only show nsfw communities if show_nsfw is passed into request - if !self.show_nsfw.unwrap_or(false) { - query = query.filter(community::nsfw.eq(false)); - } - } - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - let res = query - .limit(limit) - .offset(offset) - .load::(conn) - .await?; - - Ok(res.into_iter().map(CommunityView::from_tuple).collect()) + queries().list(pool, self).await } } diff --git a/crates/db_views_actor/src/person_mention_view.rs b/crates/db_views_actor/src/person_mention_view.rs index 6bf107a3d..6528ab5da 100644 --- a/crates/db_views_actor/src/person_mention_view.rs +++ b/crates/db_views_actor/src/person_mention_view.rs @@ -1,6 +1,7 @@ use crate::structs::PersonMentionView; use diesel::{ dsl::now, + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -11,6 +12,7 @@ use diesel::{ use diesel_async::RunQueryDsl; use lemmy_db_schema::{ aggregates::structs::CommentAggregates, + aliases, newtypes::{PersonId, PersonMentionId}, schema::{ comment, @@ -34,7 +36,7 @@ use lemmy_db_schema::{ post::Post, }, traits::JoinView, - utils::{get_conn, limit_and_offset, DbPool}, + utils::{get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, CommentSortType, }; @@ -53,46 +55,21 @@ type PersonMentionViewTuple = ( Option, ); -impl PersonMentionView { - pub async fn read( - pool: &mut DbPool<'_>, - person_mention_id: PersonMentionId, - my_person_id: Option, - ) -> Result { - let conn = &mut get_conn(pool).await?; - let person_alias_1 = diesel::alias!(person as person1); - +fn queries<'a>() -> Queries< + impl ReadFn<'a, PersonMentionView, (PersonMentionId, Option)>, + impl ListFn<'a, PersonMentionView, PersonMentionQuery>, +> { + let all_joins = |query: person_mention::BoxedQuery<'a, Pg>, my_person_id: Option| { // The left join below will return None in this case let person_id_join = my_person_id.unwrap_or(PersonId(-1)); - let ( - person_mention, - comment, - creator, - post, - community, - recipient, - counts, - creator_banned_from_community, - follower, - saved, - creator_blocked, - my_vote, - ) = person_mention::table - .find(person_mention_id) + query .inner_join(comment::table) .inner_join(person::table.on(comment::creator_id.eq(person::id))) .inner_join(post::table.on(comment::post_id.eq(post::id))) .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person_alias_1) + .inner_join(aliases::person1) .inner_join(comment_aggregates::table.on(comment::id.eq(comment_aggregates::comment_id))) - .left_join( - community_person_ban::table.on( - community::id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)), - ), - ) .left_join( community_follower::table.on( post::community_id @@ -121,37 +98,101 @@ impl PersonMentionView { .and(comment_like::person_id.eq(person_id_join)), ), ) - .select(( - person_mention::all_columns, - comment::all_columns, - person::all_columns, - post::all_columns, - community::all_columns, - person_alias_1.fields(person::all_columns), - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - community_follower::all_columns.nullable(), - comment_saved::all_columns.nullable(), - person_block::all_columns.nullable(), - comment_like::score.nullable(), - )) - .first::(conn) - .await?; + }; - Ok(PersonMentionView { - person_mention, - comment, - creator, - post, - community, - recipient, - counts, - creator_banned_from_community: creator_banned_from_community.is_some(), - subscribed: CommunityFollower::to_subscribed_type(&follower), - saved: saved.is_some(), - creator_blocked: creator_blocked.is_some(), - my_vote, - }) + let selection = ( + person_mention::all_columns, + comment::all_columns, + person::all_columns, + post::all_columns, + community::all_columns, + aliases::person1.fields(person::all_columns), + comment_aggregates::all_columns, + community_person_ban::all_columns.nullable(), + community_follower::all_columns.nullable(), + comment_saved::all_columns.nullable(), + person_block::all_columns.nullable(), + comment_like::score.nullable(), + ); + + let read = + move |mut conn: DbConn<'a>, + (person_mention_id, my_person_id): (PersonMentionId, Option)| async move { + all_joins( + person_mention::table.find(person_mention_id).into_boxed(), + my_person_id, + ) + .left_join( + community_person_ban::table.on( + community::id + .eq(community_person_ban::community_id) + .and(community_person_ban::person_id.eq(comment::creator_id)), + ), + ) + .select(selection) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, options: PersonMentionQuery| async move { + let mut query = all_joins(person_mention::table.into_boxed(), options.my_person_id) + .left_join( + community_person_ban::table.on( + community::id + .eq(community_person_ban::community_id) + .and(community_person_ban::person_id.eq(comment::creator_id)) + .and( + community_person_ban::expires + .is_null() + .or(community_person_ban::expires.gt(now)), + ), + ), + ) + .select(selection); + + if let Some(recipient_id) = options.recipient_id { + query = query.filter(person_mention::recipient_id.eq(recipient_id)); + } + + if options.unread_only.unwrap_or(false) { + query = query.filter(person_mention::read.eq(false)); + } + + if !options.show_bot_accounts.unwrap_or(true) { + query = query.filter(person::bot_account.eq(false)); + }; + + query = match options.sort.unwrap_or(CommentSortType::Hot) { + CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()), + CommentSortType::Controversial => { + query.then_order_by(comment_aggregates::controversy_rank.desc()) + } + CommentSortType::New => query.then_order_by(comment::published.desc()), + CommentSortType::Old => query.then_order_by(comment::published.asc()), + CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), + }; + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + + query + .limit(limit) + .offset(offset) + .load::(&mut conn) + .await + }; + + Queries::new(read, list) +} + +impl PersonMentionView { + pub async fn read( + pool: &mut DbPool<'_>, + person_mention_id: PersonMentionId, + my_person_id: Option, + ) -> Result { + queries() + .read(pool, (person_mention_id, my_person_id)) + .await } /// Gets the number of unread mentions @@ -187,107 +228,7 @@ pub struct PersonMentionQuery { impl PersonMentionQuery { pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - - let person_alias_1 = diesel::alias!(person as person1); - - // The left join below will return None in this case - let person_id_join = self.my_person_id.unwrap_or(PersonId(-1)); - - let mut query = person_mention::table - .inner_join(comment::table) - .inner_join(person::table.on(comment::creator_id.eq(person::id))) - .inner_join(post::table.on(comment::post_id.eq(post::id))) - .inner_join(community::table.on(post::community_id.eq(community::id))) - .inner_join(person_alias_1) - .inner_join(comment_aggregates::table.on(comment::id.eq(comment_aggregates::comment_id))) - .left_join( - community_person_ban::table.on( - community::id - .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), - ), - ) - .left_join( - community_follower::table.on( - post::community_id - .eq(community_follower::community_id) - .and(community_follower::person_id.eq(person_id_join)), - ), - ) - .left_join( - comment_saved::table.on( - comment::id - .eq(comment_saved::comment_id) - .and(comment_saved::person_id.eq(person_id_join)), - ), - ) - .left_join( - person_block::table.on( - comment::creator_id - .eq(person_block::target_id) - .and(person_block::person_id.eq(person_id_join)), - ), - ) - .left_join( - comment_like::table.on( - comment::id - .eq(comment_like::comment_id) - .and(comment_like::person_id.eq(person_id_join)), - ), - ) - .select(( - person_mention::all_columns, - comment::all_columns, - person::all_columns, - post::all_columns, - community::all_columns, - person_alias_1.fields(person::all_columns), - comment_aggregates::all_columns, - community_person_ban::all_columns.nullable(), - community_follower::all_columns.nullable(), - comment_saved::all_columns.nullable(), - person_block::all_columns.nullable(), - comment_like::score.nullable(), - )) - .into_boxed(); - - if let Some(recipient_id) = self.recipient_id { - query = query.filter(person_mention::recipient_id.eq(recipient_id)); - } - - if self.unread_only.unwrap_or(false) { - query = query.filter(person_mention::read.eq(false)); - } - - if !self.show_bot_accounts.unwrap_or(true) { - query = query.filter(person::bot_account.eq(false)); - }; - - query = match self.sort.unwrap_or(CommentSortType::Hot) { - CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()), - CommentSortType::Controversial => { - query.then_order_by(comment_aggregates::controversy_rank.desc()) - } - CommentSortType::New => query.then_order_by(comment::published.desc()), - CommentSortType::Old => query.then_order_by(comment::published.asc()), - CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), - }; - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - - let res = query - .limit(limit) - .offset(offset) - .load::(conn) - .await?; - - Ok(res.into_iter().map(PersonMentionView::from_tuple).collect()) + queries().list(pool, self).await } } diff --git a/crates/db_views_actor/src/person_view.rs b/crates/db_views_actor/src/person_view.rs index 908fbaab4..dd4edcaf0 100644 --- a/crates/db_views_actor/src/person_view.rs +++ b/crates/db_views_actor/src/person_view.rs @@ -1,6 +1,7 @@ use crate::structs::PersonView; use diesel::{ dsl::now, + pg::Pg, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -15,23 +16,82 @@ use lemmy_db_schema::{ schema::{person, person_aggregates}, source::person::Person, traits::JoinView, - utils::{fuzzy_search, get_conn, limit_and_offset, DbPool}, + utils::{fuzzy_search, get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, PersonSortType, }; -use std::iter::Iterator; type PersonViewTuple = (Person, PersonAggregates); -impl PersonView { - pub async fn read(pool: &mut DbPool<'_>, person_id: PersonId) -> Result { - let conn = &mut get_conn(pool).await?; - let res = person::table - .find(person_id) +enum ListMode { + Admins, + Banned, + Query(PersonQuery), +} + +fn queries<'a>( +) -> Queries, impl ListFn<'a, PersonView, ListMode>> { + let all_joins = |query: person::BoxedQuery<'a, Pg>| { + query .inner_join(person_aggregates::table) .select((person::all_columns, person_aggregates::all_columns)) - .first::(conn) - .await?; - Ok(Self::from_tuple(res)) + }; + + let read = move |mut conn: DbConn<'a>, person_id: PersonId| async move { + all_joins(person::table.find(person_id).into_boxed()) + .first::(&mut conn) + .await + }; + + let list = move |mut conn: DbConn<'a>, mode: ListMode| async move { + let mut query = all_joins(person::table.into_boxed()); + match mode { + ListMode::Admins => { + query = query + .filter(person::admin.eq(true)) + .filter(person::deleted.eq(false)) + .order_by(person::published); + } + ListMode::Banned => { + query = query + .filter( + person::banned.eq(true).and( + person::ban_expires + .is_null() + .or(person::ban_expires.gt(now)), + ), + ) + .filter(person::deleted.eq(false)); + } + ListMode::Query(options) => { + if let Some(search_term) = options.search_term { + let searcher = fuzzy_search(&search_term); + query = query + .filter(person::name.ilike(searcher.clone())) + .or_filter(person::display_name.ilike(searcher)); + } + + query = match options.sort.unwrap_or(PersonSortType::CommentScore) { + PersonSortType::New => query.order_by(person::published.desc()), + PersonSortType::Old => query.order_by(person::published.asc()), + PersonSortType::MostComments => query.order_by(person_aggregates::comment_count.desc()), + PersonSortType::CommentScore => query.order_by(person_aggregates::comment_score.desc()), + PersonSortType::PostScore => query.order_by(person_aggregates::post_score.desc()), + PersonSortType::PostCount => query.order_by(person_aggregates::post_count.desc()), + }; + + let (limit, offset) = limit_and_offset(options.page, options.limit)?; + query = query.limit(limit).offset(offset); + } + } + query.load::(&mut conn).await + }; + + Queries::new(read, list) +} + +impl PersonView { + pub async fn read(pool: &mut DbPool<'_>, person_id: PersonId) -> Result { + queries().read(pool, person_id).await } pub async fn is_admin(pool: &mut DbPool<'_>, person_id: PersonId) -> Result { @@ -44,37 +104,13 @@ impl PersonView { .await?; Ok(is_admin) } - pub async fn admins(pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let admins = person::table - .inner_join(person_aggregates::table) - .select((person::all_columns, person_aggregates::all_columns)) - .filter(person::admin.eq(true)) - .filter(person::deleted.eq(false)) - .order_by(person::published) - .load::(conn) - .await?; - Ok(admins.into_iter().map(Self::from_tuple).collect()) + pub async fn admins(pool: &mut DbPool<'_>) -> Result, Error> { + queries().list(pool, ListMode::Admins).await } pub async fn banned(pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let banned = person::table - .inner_join(person_aggregates::table) - .select((person::all_columns, person_aggregates::all_columns)) - .filter( - person::banned.eq(true).and( - person::ban_expires - .is_null() - .or(person::ban_expires.gt(now)), - ), - ) - .filter(person::deleted.eq(false)) - .load::(conn) - .await?; - - Ok(banned.into_iter().map(Self::from_tuple).collect()) + queries().list(pool, ListMode::Banned).await } } @@ -88,34 +124,7 @@ pub struct PersonQuery { impl PersonQuery { pub async fn list(self, pool: &mut DbPool<'_>) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - let mut query = person::table - .inner_join(person_aggregates::table) - .select((person::all_columns, person_aggregates::all_columns)) - .into_boxed(); - - if let Some(search_term) = self.search_term { - let searcher = fuzzy_search(&search_term); - query = query - .filter(person::name.ilike(searcher.clone())) - .or_filter(person::display_name.ilike(searcher)); - } - - query = match self.sort.unwrap_or(PersonSortType::CommentScore) { - PersonSortType::New => query.order_by(person::published.desc()), - PersonSortType::Old => query.order_by(person::published.asc()), - PersonSortType::MostComments => query.order_by(person_aggregates::comment_count.desc()), - PersonSortType::CommentScore => query.order_by(person_aggregates::comment_score.desc()), - PersonSortType::PostScore => query.order_by(person_aggregates::post_score.desc()), - PersonSortType::PostCount => query.order_by(person_aggregates::post_count.desc()), - }; - - let (limit, offset) = limit_and_offset(self.page, self.limit)?; - query = query.limit(limit).offset(offset); - - let res = query.load::(conn).await?; - - Ok(res.into_iter().map(PersonView::from_tuple).collect()) + queries().list(pool, ListMode::Query(self)).await } } From 0e6fb082841b9b6601cce25c5c0a744ac0219b14 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 28 Jul 2023 14:23:46 +0200 Subject: [PATCH 20/27] Restore markdown quotes after sanitize (#3708) (#3749) --- crates/api_common/src/utils.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 8ccb7d3fe..5a678191b 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -798,10 +798,12 @@ pub fn generate_moderators_url(community_id: &DbUrl) -> Result String { - ammonia::Builder::default() + let sanitized = ammonia::Builder::default() .rm_tags(&["a", "img"]) .clean(data) - .to_string() + .to_string(); + // restore markdown quotes + sanitized.replace(">", ">") } pub fn sanitize_html_opt(data: &Option) -> Option { From a34e0d477eb8bcc9acc5b3df44f843bf636b173a Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 28 Jul 2023 15:33:23 +0200 Subject: [PATCH 21/27] Only run slow CI checks for actual code changes (#3750) * Only run slow CI checks for actual code changes Its a waste of time to compile Rust and run tests when only something like the readme was changed. * also make cache restore/rebuild conditional * remove unused condition --- .woodpecker.yml | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 58ab2f0f9..6353da5cf 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -48,6 +48,8 @@ pipeline: - "api_tests/node_modules" secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] + when: + path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] toml_fmt: image: tamasfe/taplo:0.8.1 @@ -65,8 +67,6 @@ pipeline: - rustup toolchain install nightly-2023-07-10 - rustup component add rustfmt --toolchain nightly-2023-07-10 - cargo +nightly-2023-07-10 fmt -- --check - # when: - # platform: linux/amd64 # make sure api builds with default features (used by other crates relying on lemmy api) check_api_common_default_features: @@ -75,8 +75,8 @@ pipeline: CARGO_HOME: .cargo commands: - cargo check --package lemmy_api_common - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations"] lemmy_api_common_doesnt_depend_on_diesel: image: *muslrust_image @@ -84,8 +84,8 @@ pipeline: CARGO_HOME: .cargo commands: - "! cargo tree -p lemmy_api_common --no-default-features -i diesel" - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations"] lemmy_api_common_works_with_wasm: image: *muslrust_image @@ -94,6 +94,8 @@ pipeline: commands: - "rustup target add wasm32-unknown-unknown" - "cargo check --target wasm32-unknown-unknown -p lemmy_api_common" + when: + path: ["crates", "src", "Cargo.toml", "migrations"] check_defaults_hjson_updated: image: *muslrust_image @@ -103,8 +105,8 @@ pipeline: - export LEMMY_CONFIG_LOCATION=./config/config.hjson - ./scripts/update_config_defaults.sh config/defaults_current.hjson - diff config/defaults.hjson config/defaults_current.hjson - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations"] check_diesel_schema: image: willsquire/diesel-cli @@ -115,6 +117,8 @@ pipeline: - diesel migration run - diesel print-schema --config-file=diesel.toml > tmp.schema - diff tmp.schema crates/db_schema/src/schema.rs + when: + path: ["crates", "src", "Cargo.toml", "migrations"] check_diesel_migration_revertable: image: willsquire/diesel-cli @@ -124,6 +128,8 @@ pipeline: commands: - diesel migration run - diesel migration redo + when: + path: ["crates", "src", "Cargo.toml", "migrations"] cargo_clippy: image: *muslrust_image @@ -147,8 +153,8 @@ pipeline: -D clippy::needless_collect -D clippy::unwrap_used -D clippy::indexing_slicing - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations"] cargo_test: image: *muslrust_image @@ -159,8 +165,8 @@ pipeline: commands: - export LEMMY_CONFIG_LOCATION=../../config/config.hjson - cargo test --workspace --no-fail-fast - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations"] cargo_build: image: *muslrust_image @@ -169,8 +175,8 @@ pipeline: commands: - cargo build - mv target/x86_64-unknown-linux-musl/debug/lemmy_server target/lemmy_server - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] run_federation_tests: image: node:alpine @@ -183,8 +189,8 @@ pipeline: - cd api_tests/ - yarn - yarn api-test - # when: - # platform: linux/amd64 + when: + path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] rebuild-cache: image: meltwater/drone-cache:v1 @@ -208,6 +214,8 @@ pipeline: - "api_tests/node_modules" secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] + when: + path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] publish_release_docker: image: woodpeckerci/plugin-docker-buildx @@ -257,5 +265,3 @@ services: environment: POSTGRES_USER: lemmy POSTGRES_PASSWORD: password - # when: - # platform: linux/amd64 From db76c5b7ffeaad8094998f9a76541ef4b345159a Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 28 Jul 2023 10:33:45 -0400 Subject: [PATCH 22/27] Adding 0.18.3 Release notes. (#3753) --- RELEASES.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index bca2f02f1..a81512800 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,99 @@ +# Lemmy v0.18.3 Release (2023-07-28) + +## What is Lemmy? + +Lemmy is a self-hosted social link aggregation and discussion platform. It is completely free and open, and not controlled by any company. This means that there is no advertising, tracking, or secret algorithms. Content is organized into communities, so it is easy to subscribe to topics that you are interested in, and ignore others. Voting is used to bring the most interesting items to the top. + +## Major Changes + +This version brings major optimizations to the database queries, which significantly reduces CPU usage. There is also a change to the way federation activities are stored, which reduces database size by around 80%. Special thanks to @phiresky for their work on DB optimizations. + +The federation code now includes a check for dead instances which is used when sending activities. This helps to reduce the amount of outgoing POST requests, and also reduce server load. + +In terms of security, Lemmy now performs HTML sanitization on all messages which are submitted through the API or received via federation. Together with the tightened content-security-policy from 0.18.2, cross-site scripting attacks are now much more difficult. + +Other than that, there are numerous bug fixes and minor enhancements. + +## Support development + +@dessalines and @nutomic are working full-time on Lemmy to integrate community contributions, fix bugs, optimize performance and much more. This work is funded exclusively through donations. + +If you like using Lemmy, and want to make sure that we will always be available to work full time building it, consider [donating to support its development](https://join-lemmy.org/donate). No one likes recurring donations, but they’ve proven to be the only way that open-source software like Lemmy can stay independent and alive. + +- [Liberapay](https://liberapay.com/Lemmy) (preferred option) +- [Open Collective](https://opencollective.com/lemmy) +- [Patreon](https://www.patreon.com/dessalines) +- [Cryptocurrency](https://join-lemmy.org/donate) (scroll to bottom of page) + +## Upgrade instructions + +Follow the upgrade instructions for [ansible](https://github.com/LemmyNet/lemmy-ansible#upgrading) or [docker](https://join-lemmy.org/docs/en/administration/install_docker.html#updating). There are no config or API changes with this release. + +This upgrade takes ~5 minutes for the database migrations to complete. + +If you need help with the upgrade, you can ask in our [support forum](https://lemmy.ml/c/lemmy_support) or on the [Matrix Chat](https://matrix.to/#/#lemmy-admin-support-topics:discuss.online). + +## Changes + +### Lemmy + +- Restore markdown quotes after sanitize ([#3708](https://github.com/LemmyNet/lemmy/issues/3708)) ([#3749](https://github.com/LemmyNet/lemmy/issues/3749)) +- remove performance-problematic and buggy duplicate site aggregates ([#3732](https://github.com/LemmyNet/lemmy/issues/3732)) +- remove n^2 part of person triggers, improve community aggregate trigger ([#3739](https://github.com/LemmyNet/lemmy/issues/3739)) +- Revert "Add controversial ranking ([#3205](https://github.com/LemmyNet/lemmy/issues/3205))" +- Omit local instance from federated instances list ([#3712](https://github.com/LemmyNet/lemmy/issues/3712)) +- add trigram index to search ([#3719](https://github.com/LemmyNet/lemmy/issues/3719)) +- Federation tests replication round1 - demonstrate absent replication of comment deletes ([#3657](https://github.com/LemmyNet/lemmy/issues/3657)) +- Make resolve_object not require auth [#3685](https://github.com/LemmyNet/lemmy/issues/3685) ([#3716](https://github.com/LemmyNet/lemmy/issues/3716)) +- Sanitize html ([#3708](https://github.com/LemmyNet/lemmy/issues/3708)) +- Add controversial ranking ([#3205](https://github.com/LemmyNet/lemmy/issues/3205)) +- Skip fragile API tests ([#3723](https://github.com/LemmyNet/lemmy/issues/3723)) +- Enable gzip for reqwest ([#3696](https://github.com/LemmyNet/lemmy/issues/3696)) +- Dont authenticate user after successful password reset [#3714](https://github.com/LemmyNet/lemmy/issues/3714) ([#3715](https://github.com/LemmyNet/lemmy/issues/3715)) +- Bump version of dependency "webmention" ([#3711](https://github.com/LemmyNet/lemmy/issues/3711)) +- prevent ordering by comment path without post filter ([#3717](https://github.com/LemmyNet/lemmy/issues/3717)) +- Update Dockerfile to run process as non-privileged user. ([#3709](https://github.com/LemmyNet/lemmy/issues/3709)) +- Dont show removed comments to unauthenticated users (release branch) ([#3689](https://github.com/LemmyNet/lemmy/issues/3689)) +- Add dev profile to strip symbols and disable debug info (ref [#3610](https://github.com/LemmyNet/lemmy/issues/3610)) ([#3611](https://github.com/LemmyNet/lemmy/issues/3611)) +- Dont publish releases to crates.io (fixes [#3272](https://github.com/LemmyNet/lemmy/issues/3272)) ([#3664](https://github.com/LemmyNet/lemmy/issues/3664)) +- Change logic for determining comment default language (fixes [#3451](https://github.com/LemmyNet/lemmy/issues/3451)) ([#3672](https://github.com/LemmyNet/lemmy/issues/3672)) +- Post remove delete federation outbound fix0 ([#3613](https://github.com/LemmyNet/lemmy/issues/3613)) +- disable rustfmt feature on rosetta-build ([#3679](https://github.com/LemmyNet/lemmy/issues/3679)) +- Make sure comments are sorted by hot_rank, then score. ([#3667](https://github.com/LemmyNet/lemmy/issues/3667)) +- Ignore errors when fetching community mods (fixes [#3460](https://github.com/LemmyNet/lemmy/issues/3460)) ([#3674](https://github.com/LemmyNet/lemmy/issues/3674)) +- Upgrade activitypub library to 0.4.6 (fixes [#3222](https://github.com/LemmyNet/lemmy/issues/3222)) ([#3675](https://github.com/LemmyNet/lemmy/issues/3675)) +- Denormalize community_id into post_aggregates for a 1000x speed-up when loading posts ([#3653](https://github.com/LemmyNet/lemmy/issues/3653)) +- Fixing hot_ranks and scores to append a published sort. ([#3618](https://github.com/LemmyNet/lemmy/issues/3618)) +- Use local_site.default_post_listing_type as the initial default listing type for new users ([#3666](https://github.com/LemmyNet/lemmy/issues/3666)) +- Don't panic when scheduled tasks can't connect to database ([#3634](https://github.com/LemmyNet/lemmy/issues/3634)) +- Add http cache for webfingers ([#3317](https://github.com/LemmyNet/lemmy/issues/3317)) +- Optimize hot rank updates ([#3617](https://github.com/LemmyNet/lemmy/issues/3617)) +- Split activity table into sent and received parts (fixes [#3103](https://github.com/LemmyNet/lemmy/issues/3103)) ([#3583](https://github.com/LemmyNet/lemmy/issues/3583)) +- work around race condition on community fetch ([#3414](https://github.com/LemmyNet/lemmy/issues/3414)) +- Make `lemmy_api_common` wasm-compatible ([#3587](https://github.com/LemmyNet/lemmy/issues/3587)) +- Check for dead federated instances (fixes [#2221](https://github.com/LemmyNet/lemmy/issues/2221)) ([#3427](https://github.com/LemmyNet/lemmy/issues/3427)) +- Fix wrong SMTP port when TLS is being used (fixes [#3574](https://github.com/LemmyNet/lemmy/issues/3574)) ([#3607](https://github.com/LemmyNet/lemmy/issues/3607)) +- Add infinite scroll user option ([#3572](https://github.com/LemmyNet/lemmy/issues/3572)) +- Shrink capacity in `RateLimitStorage::remove_older_than` ([#3536](https://github.com/LemmyNet/lemmy/issues/3536)) +- Fix [#3501](https://github.com/LemmyNet/lemmy/issues/3501) - Fix aggregation counts for elements removed and deleted ([#3543](https://github.com/LemmyNet/lemmy/issues/3543)) + +### Lemmy-UI + +- Fixing comment report showing dot. ([#1989](https://github.com/LemmyNet/lemmy-ui/issues/1989)) +- Make sure comment score color matches your vote. ([#1988](https://github.com/LemmyNet/lemmy-ui/issues/1988)) +- Allow limited set of markdown in title rendering ([#1977](https://github.com/LemmyNet/lemmy-ui/issues/1977)) +- Allow selecting from all languages in person settings (fixes [#1971](https://github.com/LemmyNet/lemmy-ui/issues/1971)) ([#1985](https://github.com/LemmyNet/lemmy-ui/issues/1985)) +- Separate final comment row + add classes ([#1982](https://github.com/LemmyNet/lemmy-ui/issues/1982)) +- Fix CSP in dev mode ([#1918](https://github.com/LemmyNet/lemmy-ui/issues/1918)) +- Fix base.output (see [#1911](https://github.com/LemmyNet/lemmy-ui/issues/1911)) ([#1943](https://github.com/LemmyNet/lemmy-ui/issues/1943)) +- Add show/hide button to password fields ([#1861](https://github.com/LemmyNet/lemmy-ui/issues/1861)) +- Fix start_url and scope ([#1931](https://github.com/LemmyNet/lemmy-ui/issues/1931)) +- Remove lodash.merge dependency ([#1911](https://github.com/LemmyNet/lemmy-ui/issues/1911)) +- Set person_id to myId in handleLeaveModTeam ([#1929](https://github.com/LemmyNet/lemmy-ui/issues/1929)) +- Remove invalid default option from language list ([#1919](https://github.com/LemmyNet/lemmy-ui/issues/1919)) +- Comment border tweak ([#1820](https://github.com/LemmyNet/lemmy-ui/issues/1820)) +- Add Toast Messages for Bad Logins ([#1874](https://github.com/LemmyNet/lemmy-ui/issues/1874)) + # Lemmy v0.18.1 Release (2023-07-07) ## What is Lemmy? From 37998b3398ed925a7640a9b67d1dc6ef871893a9 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 28 Jul 2023 16:39:38 +0200 Subject: [PATCH 23/27] Rewrite some API handlers to remove Perform trait (#3735) * Rewrite some API handlers to remove Perform trait * Convert CreateComment * ci --- Cargo.lock | 1 + crates/api/Cargo.toml | 1 + crates/api/src/comment/distinguish.rs | 80 +++-- crates/api/src/comment/mod.rs | 6 +- crates/api/src/comment/save.rs | 62 ++-- crates/api/src/comment_report/list.rs | 47 ++- crates/api/src/comment_report/mod.rs | 6 +- crates/api/src/comment_report/resolve.rs | 63 ++-- crates/api/src/lib.rs | 18 +- crates/api/src/local_user/mod.rs | 26 +- .../notifications/mark_reply_read.rs | 65 ++-- .../api/src/local_user/notifications/mod.rs | 12 +- crates/api_common/src/build_response.rs | 2 +- crates/api_common/src/send_activity.rs | 3 +- crates/api_crud/src/comment/create.rs | 282 +++++++++--------- crates/api_crud/src/comment/delete.rs | 3 +- crates/api_crud/src/comment/mod.rs | 10 +- crates/api_crud/src/comment/read.rs | 27 +- crates/api_crud/src/comment/remove.rs | 3 +- crates/api_crud/src/comment/update.rs | 3 +- crates/api_crud/src/community/list.rs | 67 ++--- crates/api_crud/src/community/mod.rs | 10 +- crates/api_crud/src/lib.rs | 12 +- crates/api_crud/src/post/mod.rs | 8 +- crates/api_crud/src/post/read.rs | 191 ++++++------ crates/api_crud/src/private_message/mod.rs | 8 +- crates/api_crud/src/private_message/read.rs | 65 ++-- crates/api_crud/src/site/create.rs | 178 ++++++----- crates/api_crud/src/site/mod.rs | 6 +- crates/api_crud/src/site/read.rs | 119 ++++---- crates/api_crud/src/site/update.rs | 276 +++++++++-------- .../activities/create_or_update/comment.rs | 37 +-- crates/apub/src/activities/mod.rs | 25 +- crates/apub/src/api/read_community.rs | 2 +- crates/utils/translations | 2 +- src/api_routes_http.rs | 95 ++---- 36 files changed, 858 insertions(+), 963 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ac601c0e..bac96d5ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2573,6 +2573,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" name = "lemmy_api" version = "0.18.1" dependencies = [ + "activitypub_federation", "actix-web", "anyhow", "async-trait", diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index be3065e4d..17f40d57d 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -20,6 +20,7 @@ lemmy_db_views = { workspace = true, features = ["full"] } lemmy_db_views_moderator = { workspace = true, features = ["full"] } lemmy_db_views_actor = { workspace = true, features = ["full"] } lemmy_api_common = { workspace = true, features = ["full"] } +activitypub_federation = { workspace = true } bcrypt = { workspace = true } serde = { workspace = true } actix-web = { workspace = true } diff --git a/crates/api/src/comment/distinguish.rs b/crates/api/src/comment/distinguish.rs index 47c23d3d2..540c19a3d 100644 --- a/crates/api/src/comment/distinguish.rs +++ b/crates/api/src/comment/distinguish.rs @@ -1,5 +1,4 @@ -use crate::Perform; -use actix_web::web::Data; +use actix_web::web::{Data, Json}; use lemmy_api_common::{ comment::{CommentResponse, DistinguishComment}, context::LemmyContext, @@ -12,50 +11,47 @@ use lemmy_db_schema::{ use lemmy_db_views::structs::CommentView; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; -#[async_trait::async_trait(?Send)] -impl Perform for DistinguishComment { - type Response = CommentResponse; +#[tracing::instrument(skip(context))] +pub async fn distinguish_comment( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &DistinguishComment = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; + let comment_id = data.comment_id; + let orig_comment = CommentView::read(&mut context.pool(), comment_id, None).await?; - let comment_id = data.comment_id; - let orig_comment = CommentView::read(&mut context.pool(), comment_id, None).await?; + check_community_ban( + local_user_view.person.id, + orig_comment.community.id, + &mut context.pool(), + ) + .await?; - check_community_ban( - local_user_view.person.id, - orig_comment.community.id, - &mut context.pool(), - ) - .await?; + // Verify that only a mod or admin can distinguish a comment + is_mod_or_admin( + &mut context.pool(), + local_user_view.person.id, + orig_comment.community.id, + ) + .await?; - // Verify that only a mod or admin can distinguish a comment - is_mod_or_admin( - &mut context.pool(), - local_user_view.person.id, - orig_comment.community.id, - ) - .await?; + // Update the Comment + let comment_id = data.comment_id; + let form = CommentUpdateForm::builder() + .distinguished(Some(data.distinguished)) + .build(); + Comment::update(&mut context.pool(), comment_id, &form) + .await + .with_lemmy_type(LemmyErrorType::CouldntUpdateComment)?; - // Update the Comment - let comment_id = data.comment_id; - let form = CommentUpdateForm::builder() - .distinguished(Some(data.distinguished)) - .build(); - Comment::update(&mut context.pool(), comment_id, &form) - .await - .with_lemmy_type(LemmyErrorType::CouldntUpdateComment)?; + let comment_id = data.comment_id; + let person_id = local_user_view.person.id; + let comment_view = CommentView::read(&mut context.pool(), comment_id, Some(person_id)).await?; - let comment_id = data.comment_id; - let person_id = local_user_view.person.id; - let comment_view = CommentView::read(&mut context.pool(), comment_id, Some(person_id)).await?; - - Ok(CommentResponse { - comment_view, - recipient_ids: Vec::new(), - form_id: None, - }) - } + Ok(Json(CommentResponse { + comment_view, + recipient_ids: Vec::new(), + form_id: None, + })) } diff --git a/crates/api/src/comment/mod.rs b/crates/api/src/comment/mod.rs index 27584c360..8caeaf8b0 100644 --- a/crates/api/src/comment/mod.rs +++ b/crates/api/src/comment/mod.rs @@ -1,3 +1,3 @@ -mod distinguish; -mod like; -mod save; +pub mod distinguish; +pub mod like; +pub mod save; diff --git a/crates/api/src/comment/save.rs b/crates/api/src/comment/save.rs index 7161c8e9c..8c9d90555 100644 --- a/crates/api/src/comment/save.rs +++ b/crates/api/src/comment/save.rs @@ -1,5 +1,4 @@ -use crate::Perform; -use actix_web::web::Data; +use actix_web::web::{Data, Json}; use lemmy_api_common::{ comment::{CommentResponse, SaveComment}, context::LemmyContext, @@ -12,38 +11,35 @@ use lemmy_db_schema::{ use lemmy_db_views::structs::CommentView; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; -#[async_trait::async_trait(?Send)] -impl Perform for SaveComment { - type Response = CommentResponse; +#[tracing::instrument(skip(context))] +pub async fn save_comment( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &SaveComment = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; + let comment_saved_form = CommentSavedForm { + comment_id: data.comment_id, + person_id: local_user_view.person.id, + }; - let comment_saved_form = CommentSavedForm { - comment_id: data.comment_id, - person_id: local_user_view.person.id, - }; - - if data.save { - CommentSaved::save(&mut context.pool(), &comment_saved_form) - .await - .with_lemmy_type(LemmyErrorType::CouldntSaveComment)?; - } else { - CommentSaved::unsave(&mut context.pool(), &comment_saved_form) - .await - .with_lemmy_type(LemmyErrorType::CouldntSaveComment)?; - } - - let comment_id = data.comment_id; - let person_id = local_user_view.person.id; - let comment_view = CommentView::read(&mut context.pool(), comment_id, Some(person_id)).await?; - - Ok(CommentResponse { - comment_view, - recipient_ids: Vec::new(), - form_id: None, - }) + if data.save { + CommentSaved::save(&mut context.pool(), &comment_saved_form) + .await + .with_lemmy_type(LemmyErrorType::CouldntSaveComment)?; + } else { + CommentSaved::unsave(&mut context.pool(), &comment_saved_form) + .await + .with_lemmy_type(LemmyErrorType::CouldntSaveComment)?; } + + let comment_id = data.comment_id; + let person_id = local_user_view.person.id; + let comment_view = CommentView::read(&mut context.pool(), comment_id, Some(person_id)).await?; + + Ok(Json(CommentResponse { + comment_view, + recipient_ids: Vec::new(), + form_id: None, + })) } diff --git a/crates/api/src/comment_report/list.rs b/crates/api/src/comment_report/list.rs index b67ec333c..0ca093c74 100644 --- a/crates/api/src/comment_report/list.rs +++ b/crates/api/src/comment_report/list.rs @@ -1,5 +1,4 @@ -use crate::Perform; -use actix_web::web::Data; +use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ comment::{ListCommentReports, ListCommentReportsResponse}, context::LemmyContext, @@ -10,32 +9,26 @@ use lemmy_utils::error::LemmyError; /// Lists comment reports for a community if an id is supplied /// or returns all comment reports for communities a user moderates -#[async_trait::async_trait(?Send)] -impl Perform for ListCommentReports { - type Response = ListCommentReportsResponse; +#[tracing::instrument(skip(context))] +pub async fn list_comment_reports( + data: Query, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; - #[tracing::instrument(skip(context))] - async fn perform( - &self, - context: &Data, - ) -> Result { - let data: &ListCommentReports = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; + let community_id = data.community_id; + let unresolved_only = data.unresolved_only; - let community_id = data.community_id; - let unresolved_only = data.unresolved_only; - - let page = data.page; - let limit = data.limit; - let comment_reports = CommentReportQuery { - community_id, - unresolved_only, - page, - limit, - } - .list(&mut context.pool(), &local_user_view.person) - .await?; - - Ok(ListCommentReportsResponse { comment_reports }) + let page = data.page; + let limit = data.limit; + let comment_reports = CommentReportQuery { + community_id, + unresolved_only, + page, + limit, } + .list(&mut context.pool(), &local_user_view.person) + .await?; + + Ok(Json(ListCommentReportsResponse { comment_reports })) } diff --git a/crates/api/src/comment_report/mod.rs b/crates/api/src/comment_report/mod.rs index 375fde4c3..3bb1a9b46 100644 --- a/crates/api/src/comment_report/mod.rs +++ b/crates/api/src/comment_report/mod.rs @@ -1,3 +1,3 @@ -mod create; -mod list; -mod resolve; +pub mod create; +pub mod list; +pub mod resolve; diff --git a/crates/api/src/comment_report/resolve.rs b/crates/api/src/comment_report/resolve.rs index 111495276..8e03484e8 100644 --- a/crates/api/src/comment_report/resolve.rs +++ b/crates/api/src/comment_report/resolve.rs @@ -1,5 +1,4 @@ -use crate::Perform; -use actix_web::web::Data; +use actix_web::web::{Data, Json}; use lemmy_api_common::{ comment::{CommentReportResponse, ResolveCommentReport}, context::LemmyContext, @@ -10,41 +9,35 @@ use lemmy_db_views::structs::CommentReportView; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; /// Resolves or unresolves a comment report and notifies the moderators of the community -#[async_trait::async_trait(?Send)] -impl Perform for ResolveCommentReport { - type Response = CommentReportResponse; +#[tracing::instrument(skip(context))] +pub async fn resolve_comment_report( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; - #[tracing::instrument(skip(context))] - async fn perform( - &self, - context: &Data, - ) -> Result { - let data: &ResolveCommentReport = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; + let report_id = data.report_id; + let person_id = local_user_view.person.id; + let report = CommentReportView::read(&mut context.pool(), report_id, person_id).await?; - let report_id = data.report_id; - let person_id = local_user_view.person.id; - let report = CommentReportView::read(&mut context.pool(), report_id, person_id).await?; + let person_id = local_user_view.person.id; + is_mod_or_admin(&mut context.pool(), person_id, report.community.id).await?; - let person_id = local_user_view.person.id; - is_mod_or_admin(&mut context.pool(), person_id, report.community.id).await?; - - if data.resolved { - CommentReport::resolve(&mut context.pool(), report_id, person_id) - .await - .with_lemmy_type(LemmyErrorType::CouldntResolveReport)?; - } else { - CommentReport::unresolve(&mut context.pool(), report_id, person_id) - .await - .with_lemmy_type(LemmyErrorType::CouldntResolveReport)?; - } - - let report_id = data.report_id; - let comment_report_view = - CommentReportView::read(&mut context.pool(), report_id, person_id).await?; - - Ok(CommentReportResponse { - comment_report_view, - }) + if data.resolved { + CommentReport::resolve(&mut context.pool(), report_id, person_id) + .await + .with_lemmy_type(LemmyErrorType::CouldntResolveReport)?; + } else { + CommentReport::unresolve(&mut context.pool(), report_id, person_id) + .await + .with_lemmy_type(LemmyErrorType::CouldntResolveReport)?; } + + let report_id = data.report_id; + let comment_report_view = + CommentReportView::read(&mut context.pool(), report_id, person_id).await?; + + Ok(Json(CommentReportResponse { + comment_report_view, + })) } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index b297f503f..cc5fb8e8e 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -9,15 +9,15 @@ use lemmy_utils::{ }; use std::io::Cursor; -mod comment; -mod comment_report; -mod community; -mod local_user; -mod post; -mod post_report; -mod private_message; -mod private_message_report; -mod site; +pub mod comment; +pub mod comment_report; +pub mod community; +pub mod local_user; +pub mod post; +pub mod post_report; +pub mod private_message; +pub mod private_message_report; +pub mod site; #[async_trait::async_trait(?Send)] pub trait Perform { diff --git a/crates/api/src/local_user/mod.rs b/crates/api/src/local_user/mod.rs index 3a92beda5..806fa66a2 100644 --- a/crates/api/src/local_user/mod.rs +++ b/crates/api/src/local_user/mod.rs @@ -1,13 +1,13 @@ -mod add_admin; -mod ban_person; -mod block; -mod change_password; -mod change_password_after_reset; -mod get_captcha; -mod list_banned; -mod login; -mod notifications; -mod report_count; -mod reset_password; -mod save_settings; -mod verify_email; +pub mod add_admin; +pub mod ban_person; +pub mod block; +pub mod change_password; +pub mod change_password_after_reset; +pub mod get_captcha; +pub mod list_banned; +pub mod login; +pub mod notifications; +pub mod report_count; +pub mod reset_password; +pub mod save_settings; +pub mod verify_email; diff --git a/crates/api/src/local_user/notifications/mark_reply_read.rs b/crates/api/src/local_user/notifications/mark_reply_read.rs index 4071a466d..9ae9f5251 100644 --- a/crates/api/src/local_user/notifications/mark_reply_read.rs +++ b/crates/api/src/local_user/notifications/mark_reply_read.rs @@ -1,5 +1,4 @@ -use crate::Perform; -use actix_web::web::Data; +use actix_web::web::{Data, Json}; use lemmy_api_common::{ context::LemmyContext, person::{CommentReplyResponse, MarkCommentReplyAsRead}, @@ -12,41 +11,35 @@ use lemmy_db_schema::{ use lemmy_db_views_actor::structs::CommentReplyView; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; -#[async_trait::async_trait(?Send)] -impl Perform for MarkCommentReplyAsRead { - type Response = CommentReplyResponse; +#[tracing::instrument(skip(context))] +pub async fn mark_reply_as_read( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; - #[tracing::instrument(skip(context))] - async fn perform( - &self, - context: &Data, - ) -> Result { - let data = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; + let comment_reply_id = data.comment_reply_id; + let read_comment_reply = CommentReply::read(&mut context.pool(), comment_reply_id).await?; - let comment_reply_id = data.comment_reply_id; - let read_comment_reply = CommentReply::read(&mut context.pool(), comment_reply_id).await?; - - if local_user_view.person.id != read_comment_reply.recipient_id { - return Err(LemmyErrorType::CouldntUpdateComment)?; - } - - let comment_reply_id = read_comment_reply.id; - let read = Some(data.read); - - CommentReply::update( - &mut context.pool(), - comment_reply_id, - &CommentReplyUpdateForm { read }, - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntUpdateComment)?; - - let comment_reply_id = read_comment_reply.id; - let person_id = local_user_view.person.id; - let comment_reply_view = - CommentReplyView::read(&mut context.pool(), comment_reply_id, Some(person_id)).await?; - - Ok(CommentReplyResponse { comment_reply_view }) + if local_user_view.person.id != read_comment_reply.recipient_id { + return Err(LemmyErrorType::CouldntUpdateComment)?; } + + let comment_reply_id = read_comment_reply.id; + let read = Some(data.read); + + CommentReply::update( + &mut context.pool(), + comment_reply_id, + &CommentReplyUpdateForm { read }, + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntUpdateComment)?; + + let comment_reply_id = read_comment_reply.id; + let person_id = local_user_view.person.id; + let comment_reply_view = + CommentReplyView::read(&mut context.pool(), comment_reply_id, Some(person_id)).await?; + + Ok(Json(CommentReplyResponse { comment_reply_view })) } diff --git a/crates/api/src/local_user/notifications/mod.rs b/crates/api/src/local_user/notifications/mod.rs index ab98053fe..35567afde 100644 --- a/crates/api/src/local_user/notifications/mod.rs +++ b/crates/api/src/local_user/notifications/mod.rs @@ -1,6 +1,6 @@ -mod list_mentions; -mod list_replies; -mod mark_all_read; -mod mark_mention_read; -mod mark_reply_read; -mod unread_count; +pub mod list_mentions; +pub mod list_replies; +pub mod mark_all_read; +pub mod mark_mention_read; +pub mod mark_reply_read; +pub mod unread_count; diff --git a/crates/api_common/src/build_response.rs b/crates/api_common/src/build_response.rs index 8a63f7ad4..b8c02457d 100644 --- a/crates/api_common/src/build_response.rs +++ b/crates/api_common/src/build_response.rs @@ -23,7 +23,7 @@ use lemmy_db_views_actor::structs::CommunityView; use lemmy_utils::{error::LemmyError, utils::mention::MentionData}; pub async fn build_comment_response( - context: &Data, + context: &LemmyContext, comment_id: CommentId, local_user_view: Option, form_id: Option, diff --git a/crates/api_common/src/send_activity.rs b/crates/api_common/src/send_activity.rs index 6c91258ec..994aea2a7 100644 --- a/crates/api_common/src/send_activity.rs +++ b/crates/api_common/src/send_activity.rs @@ -1,7 +1,7 @@ use crate::context::LemmyContext; use activitypub_federation::config::Data; use futures::future::BoxFuture; -use lemmy_db_schema::source::post::Post; +use lemmy_db_schema::source::{comment::Comment, post::Post}; use lemmy_utils::{error::LemmyResult, SYNCHRONOUS_FEDERATION}; use once_cell::sync::{Lazy, OnceCell}; use tokio::{ @@ -22,6 +22,7 @@ pub static MATCH_OUTGOING_ACTIVITIES: OnceCell = O #[derive(Debug)] pub enum SendActivityData { CreatePost(Post), + CreateComment(Comment), } // TODO: instead of static, move this into LemmyContext. make sure that stopping the process with diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index 4a7513a4b..f334efe5e 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -1,9 +1,10 @@ -use crate::PerformCrud; -use actix_web::web::Data; +use activitypub_federation::config::Data; +use actix_web::web::Json; use lemmy_api_common::{ build_response::{build_comment_response, send_local_notifs}, comment::{CommentResponse, CreateComment}, context::LemmyContext, + send_activity::{ActivityChannel, SendActivityData}, utils::{ check_community_ban, check_community_deleted_or_removed, @@ -35,169 +36,174 @@ use lemmy_utils::{ validation::is_valid_body_field, }, }; +use std::ops::Deref; const MAX_COMMENT_DEPTH_LIMIT: usize = 100; -#[async_trait::async_trait(?Send)] -impl PerformCrud for CreateComment { - type Response = CommentResponse; +#[tracing::instrument(skip(context))] +pub async fn create_comment( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; + let local_site = LocalSite::read(&mut context.pool()).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &CreateComment = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - let local_site = LocalSite::read(&mut context.pool()).await?; + let content = remove_slurs( + &data.content.clone(), + &local_site_to_slur_regex(&local_site), + ); + is_valid_body_field(&Some(content.clone()), false)?; + let content = sanitize_html(&content); - let content = remove_slurs( - &data.content.clone(), - &local_site_to_slur_regex(&local_site), - ); - is_valid_body_field(&Some(content.clone()), false)?; - let content = sanitize_html(&content); + // Check for a community ban + let post_id = data.post_id; + let post = get_post(post_id, &mut context.pool()).await?; + let community_id = post.community_id; - // Check for a community ban - let post_id = data.post_id; - let post = get_post(post_id, &mut context.pool()).await?; - let community_id = post.community_id; + check_community_ban(local_user_view.person.id, community_id, &mut context.pool()).await?; + check_community_deleted_or_removed(community_id, &mut context.pool()).await?; + check_post_deleted_or_removed(&post)?; - check_community_ban(local_user_view.person.id, community_id, &mut context.pool()).await?; - check_community_deleted_or_removed(community_id, &mut context.pool()).await?; - check_post_deleted_or_removed(&post)?; + // Check if post is locked, no new comments + if post.locked { + return Err(LemmyErrorType::Locked)?; + } - // Check if post is locked, no new comments - if post.locked { - return Err(LemmyErrorType::Locked)?; + // Fetch the parent, if it exists + let parent_opt = if let Some(parent_id) = data.parent_id { + Comment::read(&mut context.pool(), parent_id).await.ok() + } else { + None + }; + + // If there's a parent_id, check to make sure that comment is in that post + // Strange issue where sometimes the post ID of the parent comment is incorrect + if let Some(parent) = parent_opt.as_ref() { + if parent.post_id != post_id { + return Err(LemmyErrorType::CouldntCreateComment)?; } + check_comment_depth(parent)?; + } - // Fetch the parent, if it exists - let parent_opt = if let Some(parent_id) = data.parent_id { - Comment::read(&mut context.pool(), parent_id).await.ok() - } else { - None - }; + CommunityLanguage::is_allowed_community_language( + &mut context.pool(), + data.language_id, + community_id, + ) + .await?; - // If there's a parent_id, check to make sure that comment is in that post - // Strange issue where sometimes the post ID of the parent comment is incorrect - if let Some(parent) = parent_opt.as_ref() { - if parent.post_id != post_id { - return Err(LemmyErrorType::CouldntCreateComment)?; - } - check_comment_depth(parent)?; + // attempt to set default language if none was provided + let language_id = match data.language_id { + Some(lid) => Some(lid), + None => { + default_post_language( + &mut context.pool(), + community_id, + local_user_view.local_user.id, + ) + .await? } + }; - CommunityLanguage::is_allowed_community_language( - &mut context.pool(), - data.language_id, - community_id, - ) - .await?; + let comment_form = CommentInsertForm::builder() + .content(content.clone()) + .post_id(data.post_id) + .creator_id(local_user_view.person.id) + .language_id(language_id) + .build(); - // attempt to set default language if none was provided - let language_id = match data.language_id { - Some(lid) => Some(lid), - None => { - default_post_language( - &mut context.pool(), - community_id, - local_user_view.local_user.id, - ) - .await? - } - }; - - let comment_form = CommentInsertForm::builder() - .content(content.clone()) - .post_id(data.post_id) - .creator_id(local_user_view.person.id) - .language_id(language_id) - .build(); - - // Create the comment - let parent_path = parent_opt.clone().map(|t| t.path); - let inserted_comment = - Comment::create(&mut context.pool(), &comment_form, parent_path.as_ref()) - .await - .with_lemmy_type(LemmyErrorType::CouldntCreateComment)?; - - // Necessary to update the ap_id - let inserted_comment_id = inserted_comment.id; - let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - - let apub_id = generate_local_apub_endpoint( - EndpointType::Comment, - &inserted_comment_id.to_string(), - &protocol_and_hostname, - )?; - let updated_comment = Comment::update( - &mut context.pool(), - inserted_comment_id, - &CommentUpdateForm::builder().ap_id(Some(apub_id)).build(), - ) + // Create the comment + let parent_path = parent_opt.clone().map(|t| t.path); + let inserted_comment = Comment::create(&mut context.pool(), &comment_form, parent_path.as_ref()) .await .with_lemmy_type(LemmyErrorType::CouldntCreateComment)?; - // Scan the comment for user mentions, add those rows - let mentions = scrape_text_for_mentions(&content); - let recipient_ids = send_local_notifs( - mentions, - &updated_comment, - &local_user_view.person, - &post, - true, - context, - ) - .await?; + // Necessary to update the ap_id + let inserted_comment_id = inserted_comment.id; + let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - // You like your own comment by default - let like_form = CommentLikeForm { - comment_id: inserted_comment.id, - post_id: post.id, - person_id: local_user_view.person.id, - score: 1, - }; + let apub_id = generate_local_apub_endpoint( + EndpointType::Comment, + &inserted_comment_id.to_string(), + &protocol_and_hostname, + )?; + let updated_comment = Comment::update( + &mut context.pool(), + inserted_comment_id, + &CommentUpdateForm::builder().ap_id(Some(apub_id)).build(), + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntCreateComment)?; - CommentLike::like(&mut context.pool(), &like_form) + // Scan the comment for user mentions, add those rows + let mentions = scrape_text_for_mentions(&content); + let recipient_ids = send_local_notifs( + mentions, + &updated_comment, + &local_user_view.person, + &post, + true, + &context, + ) + .await?; + + // You like your own comment by default + let like_form = CommentLikeForm { + comment_id: inserted_comment.id, + post_id: post.id, + person_id: local_user_view.person.id, + score: 1, + }; + + CommentLike::like(&mut context.pool(), &like_form) + .await + .with_lemmy_type(LemmyErrorType::CouldntLikeComment)?; + + ActivityChannel::submit_activity( + SendActivityData::CreateComment(updated_comment.clone()), + &context, + ) + .await?; + + // If its a reply, mark the parent as read + if let Some(parent) = parent_opt { + let parent_id = parent.id; + let comment_reply = CommentReply::read_by_comment(&mut context.pool(), parent_id).await; + if let Ok(reply) = comment_reply { + CommentReply::update( + &mut context.pool(), + reply.id, + &CommentReplyUpdateForm { read: Some(true) }, + ) .await - .with_lemmy_type(LemmyErrorType::CouldntLikeComment)?; - - // If its a reply, mark the parent as read - if let Some(parent) = parent_opt { - let parent_id = parent.id; - let comment_reply = CommentReply::read_by_comment(&mut context.pool(), parent_id).await; - if let Ok(reply) = comment_reply { - CommentReply::update( - &mut context.pool(), - reply.id, - &CommentReplyUpdateForm { read: Some(true) }, - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntUpdateReplies)?; - } - - // If the parent has PersonMentions mark them as read too - let person_id = local_user_view.person.id; - let person_mention = - PersonMention::read_by_comment_and_person(&mut context.pool(), parent_id, person_id).await; - if let Ok(mention) = person_mention { - PersonMention::update( - &mut context.pool(), - mention.id, - &PersonMentionUpdateForm { read: Some(true) }, - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntUpdatePersonMentions)?; - } + .with_lemmy_type(LemmyErrorType::CouldntUpdateReplies)?; } + // If the parent has PersonMentions mark them as read too + let person_id = local_user_view.person.id; + let person_mention = + PersonMention::read_by_comment_and_person(&mut context.pool(), parent_id, person_id).await; + if let Ok(mention) = person_mention { + PersonMention::update( + &mut context.pool(), + mention.id, + &PersonMentionUpdateForm { read: Some(true) }, + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntUpdatePersonMentions)?; + } + } + + Ok(Json( build_comment_response( - context, + context.deref(), inserted_comment.id, Some(local_user_view), - self.form_id.clone(), + data.form_id.clone(), recipient_ids, ) - .await - } + .await?, + )) } pub fn check_comment_depth(comment: &Comment) -> Result<(), LemmyError> { diff --git a/crates/api_crud/src/comment/delete.rs b/crates/api_crud/src/comment/delete.rs index c42924de7..eba7d1dec 100644 --- a/crates/api_crud/src/comment/delete.rs +++ b/crates/api_crud/src/comment/delete.rs @@ -15,6 +15,7 @@ use lemmy_db_schema::{ }; use lemmy_db_views::structs::CommentView; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; +use std::ops::Deref; #[async_trait::async_trait(?Send)] impl PerformCrud for DeleteComment { @@ -68,7 +69,7 @@ impl PerformCrud for DeleteComment { .await?; build_comment_response( - context, + context.deref(), updated_comment.id, Some(local_user_view), None, diff --git a/crates/api_crud/src/comment/mod.rs b/crates/api_crud/src/comment/mod.rs index d3d789a02..8bb842b70 100644 --- a/crates/api_crud/src/comment/mod.rs +++ b/crates/api_crud/src/comment/mod.rs @@ -1,5 +1,5 @@ -mod create; -mod delete; -mod read; -mod remove; -mod update; +pub mod create; +pub mod delete; +pub mod read; +pub mod remove; +pub mod update; diff --git a/crates/api_crud/src/comment/read.rs b/crates/api_crud/src/comment/read.rs index e6899fdc7..1a794dc5d 100644 --- a/crates/api_crud/src/comment/read.rs +++ b/crates/api_crud/src/comment/read.rs @@ -1,5 +1,4 @@ -use crate::PerformCrud; -use actix_web::web::Data; +use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ build_response::build_comment_response, comment::{CommentResponse, GetComment}, @@ -8,19 +7,19 @@ use lemmy_api_common::{ }; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::error::LemmyError; +use std::ops::Deref; -#[async_trait::async_trait(?Send)] -impl PerformCrud for GetComment { - type Response = CommentResponse; +#[tracing::instrument(skip(context))] +pub async fn get_comment( + data: Query, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), &context).await; + let local_site = LocalSite::read(&mut context.pool()).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data = self; - let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), context).await; - let local_site = LocalSite::read(&mut context.pool()).await?; + check_private_instance(&local_user_view, &local_site)?; - check_private_instance(&local_user_view, &local_site)?; - - build_comment_response(context, data.id, local_user_view, None, vec![]).await - } + Ok(Json( + build_comment_response(context.deref(), data.id, local_user_view, None, vec![]).await?, + )) } diff --git a/crates/api_crud/src/comment/remove.rs b/crates/api_crud/src/comment/remove.rs index e87eb425b..cfc3ccff7 100644 --- a/crates/api_crud/src/comment/remove.rs +++ b/crates/api_crud/src/comment/remove.rs @@ -16,6 +16,7 @@ use lemmy_db_schema::{ }; use lemmy_db_views::structs::CommentView; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; +use std::ops::Deref; #[async_trait::async_trait(?Send)] impl PerformCrud for RemoveComment { @@ -76,7 +77,7 @@ impl PerformCrud for RemoveComment { .await?; build_comment_response( - context, + context.deref(), updated_comment.id, Some(local_user_view), None, diff --git a/crates/api_crud/src/comment/update.rs b/crates/api_crud/src/comment/update.rs index 558965f62..5d4d75a37 100644 --- a/crates/api_crud/src/comment/update.rs +++ b/crates/api_crud/src/comment/update.rs @@ -29,6 +29,7 @@ use lemmy_utils::{ validation::is_valid_body_field, }, }; +use std::ops::Deref; #[async_trait::async_trait(?Send)] impl PerformCrud for EditComment { @@ -95,7 +96,7 @@ impl PerformCrud for EditComment { .await?; build_comment_response( - context, + context.deref(), updated_comment.id, Some(local_user_view), self.form_id.clone(), diff --git a/crates/api_crud/src/community/list.rs b/crates/api_crud/src/community/list.rs index bd8189951..c8ce9e58c 100644 --- a/crates/api_crud/src/community/list.rs +++ b/crates/api_crud/src/community/list.rs @@ -1,5 +1,4 @@ -use crate::PerformCrud; -use actix_web::web::Data; +use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ community::{ListCommunities, ListCommunitiesResponse}, context::LemmyContext, @@ -9,42 +8,36 @@ use lemmy_db_schema::source::local_site::LocalSite; use lemmy_db_views_actor::community_view::CommunityQuery; use lemmy_utils::error::LemmyError; -#[async_trait::async_trait(?Send)] -impl PerformCrud for ListCommunities { - type Response = ListCommunitiesResponse; +#[tracing::instrument(skip(context))] +pub async fn list_communities( + data: Query, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), &context).await; + let local_site = LocalSite::read(&mut context.pool()).await?; + let is_admin = local_user_view.as_ref().map(|luv| is_admin(luv).is_ok()); - #[tracing::instrument(skip(context))] - async fn perform( - &self, - context: &Data, - ) -> Result { - let data: &ListCommunities = self; - let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), context).await; - let local_site = LocalSite::read(&mut context.pool()).await?; - let is_admin = local_user_view.as_ref().map(|luv| is_admin(luv).is_ok()); + check_private_instance(&local_user_view, &local_site)?; - check_private_instance(&local_user_view, &local_site)?; - - let sort = data.sort; - let listing_type = data.type_; - let show_nsfw = data.show_nsfw; - let page = data.page; - let limit = data.limit; - let local_user = local_user_view.map(|l| l.local_user); - let communities = CommunityQuery { - listing_type, - show_nsfw, - sort, - local_user: local_user.as_ref(), - page, - limit, - is_mod_or_admin: is_admin, - ..Default::default() - } - .list(&mut context.pool()) - .await?; - - // Return the jwt - Ok(ListCommunitiesResponse { communities }) + let sort = data.sort; + let listing_type = data.type_; + let show_nsfw = data.show_nsfw; + let page = data.page; + let limit = data.limit; + let local_user = local_user_view.map(|l| l.local_user); + let communities = CommunityQuery { + listing_type, + show_nsfw, + sort, + local_user: local_user.as_ref(), + page, + limit, + is_mod_or_admin: is_admin, + ..Default::default() } + .list(&mut context.pool()) + .await?; + + // Return the jwt + Ok(Json(ListCommunitiesResponse { communities })) } diff --git a/crates/api_crud/src/community/mod.rs b/crates/api_crud/src/community/mod.rs index 3fc741652..4bd028482 100644 --- a/crates/api_crud/src/community/mod.rs +++ b/crates/api_crud/src/community/mod.rs @@ -1,5 +1,5 @@ -mod create; -mod delete; -mod list; -mod remove; -mod update; +pub mod create; +pub mod delete; +pub mod list; +pub mod remove; +pub mod update; diff --git a/crates/api_crud/src/lib.rs b/crates/api_crud/src/lib.rs index e79342865..edd5c46f2 100644 --- a/crates/api_crud/src/lib.rs +++ b/crates/api_crud/src/lib.rs @@ -2,13 +2,13 @@ use actix_web::web::Data; use lemmy_api_common::context::LemmyContext; use lemmy_utils::error::LemmyError; -mod comment; -mod community; -mod custom_emoji; +pub mod comment; +pub mod community; +pub mod custom_emoji; pub mod post; -mod private_message; -mod site; -mod user; +pub mod private_message; +pub mod site; +pub mod user; #[async_trait::async_trait(?Send)] pub trait PerformCrud { diff --git a/crates/api_crud/src/post/mod.rs b/crates/api_crud/src/post/mod.rs index 437955561..8bb842b70 100644 --- a/crates/api_crud/src/post/mod.rs +++ b/crates/api_crud/src/post/mod.rs @@ -1,5 +1,5 @@ pub mod create; -mod delete; -mod read; -mod remove; -mod update; +pub mod delete; +pub mod read; +pub mod remove; +pub mod update; diff --git a/crates/api_crud/src/post/read.rs b/crates/api_crud/src/post/read.rs index e668517d3..efa0c87b8 100644 --- a/crates/api_crud/src/post/read.rs +++ b/crates/api_crud/src/post/read.rs @@ -1,5 +1,4 @@ -use crate::PerformCrud; -use actix_web::web::Data; +use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ context::LemmyContext, post::{GetPost, GetPostResponse}, @@ -19,107 +18,103 @@ use lemmy_db_views::{post_view::PostQuery, structs::PostView}; use lemmy_db_views_actor::structs::{CommunityModeratorView, CommunityView}; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; -#[async_trait::async_trait(?Send)] -impl PerformCrud for GetPost { - type Response = GetPostResponse; +#[tracing::instrument(skip(context))] +pub async fn get_post( + data: Query, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), &context).await; + let local_site = LocalSite::read(&mut context.pool()).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &GetPost = self; - let local_user_view = local_user_view_from_jwt_opt(data.auth.as_ref(), context).await; - let local_site = LocalSite::read(&mut context.pool()).await?; + check_private_instance(&local_user_view, &local_site)?; - check_private_instance(&local_user_view, &local_site)?; + let person_id = local_user_view.as_ref().map(|u| u.person.id); - let person_id = local_user_view.as_ref().map(|u| u.person.id); + // I'd prefer fetching the post_view by a comment join, but it adds a lot of boilerplate + let post_id = if let Some(id) = data.id { + id + } else if let Some(comment_id) = data.comment_id { + Comment::read(&mut context.pool(), comment_id) + .await + .with_lemmy_type(LemmyErrorType::CouldntFindPost)? + .post_id + } else { + Err(LemmyErrorType::CouldntFindPost)? + }; - // I'd prefer fetching the post_view by a comment join, but it adds a lot of boilerplate - let post_id = if let Some(id) = data.id { - id - } else if let Some(comment_id) = data.comment_id { - Comment::read(&mut context.pool(), comment_id) - .await - .with_lemmy_type(LemmyErrorType::CouldntFindPost)? - .post_id - } else { - Err(LemmyErrorType::CouldntFindPost)? - }; + // Check to see if the person is a mod or admin, to show deleted / removed + let community_id = Post::read(&mut context.pool(), post_id).await?.community_id; + let is_mod_or_admin = is_mod_or_admin_opt( + &mut context.pool(), + local_user_view.as_ref(), + Some(community_id), + ) + .await + .is_ok(); - // Check to see if the person is a mod or admin, to show deleted / removed - let community_id = Post::read(&mut context.pool(), post_id).await?.community_id; - let is_mod_or_admin = is_mod_or_admin_opt( - &mut context.pool(), - local_user_view.as_ref(), - Some(community_id), - ) - .await - .is_ok(); + let post_view = PostView::read( + &mut context.pool(), + post_id, + person_id, + Some(is_mod_or_admin), + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntFindPost)?; - let post_view = PostView::read( - &mut context.pool(), - post_id, - person_id, - Some(is_mod_or_admin), - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntFindPost)?; - - // Mark the post as read - let post_id = post_view.post.id; - if let Some(person_id) = person_id { - mark_post_as_read(person_id, post_id, &mut context.pool()).await?; - } - - // Necessary for the sidebar subscribed - let community_view = CommunityView::read( - &mut context.pool(), - community_id, - person_id, - Some(is_mod_or_admin), - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntFindCommunity)?; - - // Insert into PersonPostAggregates - // to update the read_comments count - if let Some(person_id) = person_id { - let read_comments = post_view.counts.comments; - let person_post_agg_form = PersonPostAggregatesForm { - person_id, - post_id, - read_comments, - ..PersonPostAggregatesForm::default() - }; - PersonPostAggregates::upsert(&mut context.pool(), &person_post_agg_form) - .await - .with_lemmy_type(LemmyErrorType::CouldntFindPost)?; - } - - let moderators = - CommunityModeratorView::for_community(&mut context.pool(), community_id).await?; - - // Fetch the cross_posts - let cross_posts = if let Some(url) = &post_view.post.url { - let mut x_posts = PostQuery { - url_search: Some(url.inner().as_str().into()), - ..Default::default() - } - .list(&mut context.pool()) - .await?; - - // Don't return this post as one of the cross_posts - x_posts.retain(|x| x.post.id != post_id); - x_posts - } else { - Vec::new() - }; - - // Return the jwt - Ok(GetPostResponse { - post_view, - community_view, - moderators, - cross_posts, - }) + // Mark the post as read + let post_id = post_view.post.id; + if let Some(person_id) = person_id { + mark_post_as_read(person_id, post_id, &mut context.pool()).await?; } + + // Necessary for the sidebar subscribed + let community_view = CommunityView::read( + &mut context.pool(), + community_id, + person_id, + Some(is_mod_or_admin), + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntFindCommunity)?; + + // Insert into PersonPostAggregates + // to update the read_comments count + if let Some(person_id) = person_id { + let read_comments = post_view.counts.comments; + let person_post_agg_form = PersonPostAggregatesForm { + person_id, + post_id, + read_comments, + ..PersonPostAggregatesForm::default() + }; + PersonPostAggregates::upsert(&mut context.pool(), &person_post_agg_form) + .await + .with_lemmy_type(LemmyErrorType::CouldntFindPost)?; + } + + let moderators = CommunityModeratorView::for_community(&mut context.pool(), community_id).await?; + + // Fetch the cross_posts + let cross_posts = if let Some(url) = &post_view.post.url { + let mut x_posts = PostQuery { + url_search: Some(url.inner().as_str().into()), + ..Default::default() + } + .list(&mut context.pool()) + .await?; + + // Don't return this post as one of the cross_posts + x_posts.retain(|x| x.post.id != post_id); + x_posts + } else { + Vec::new() + }; + + // Return the jwt + Ok(Json(GetPostResponse { + post_view, + community_view, + moderators, + cross_posts, + })) } diff --git a/crates/api_crud/src/private_message/mod.rs b/crates/api_crud/src/private_message/mod.rs index 716832376..ab7fa4390 100644 --- a/crates/api_crud/src/private_message/mod.rs +++ b/crates/api_crud/src/private_message/mod.rs @@ -1,4 +1,4 @@ -mod create; -mod delete; -mod read; -mod update; +pub mod create; +pub mod delete; +pub mod read; +pub mod update; diff --git a/crates/api_crud/src/private_message/read.rs b/crates/api_crud/src/private_message/read.rs index 87d8ee66e..ec4f5c102 100644 --- a/crates/api_crud/src/private_message/read.rs +++ b/crates/api_crud/src/private_message/read.rs @@ -1,5 +1,4 @@ -use crate::PerformCrud; -use actix_web::web::Data; +use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ context::LemmyContext, private_message::{GetPrivateMessages, PrivateMessagesResponse}, @@ -8,40 +7,34 @@ use lemmy_api_common::{ use lemmy_db_views::private_message_view::PrivateMessageQuery; use lemmy_utils::error::LemmyError; -#[async_trait::async_trait(?Send)] -impl PerformCrud for GetPrivateMessages { - type Response = PrivateMessagesResponse; +#[tracing::instrument(skip(context))] +pub async fn get_private_message( + data: Query, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(data.auth.as_ref(), &context).await?; + let person_id = local_user_view.person.id; - #[tracing::instrument(skip(self, context))] - async fn perform( - &self, - context: &Data, - ) -> Result { - let data: &GetPrivateMessages = self; - let local_user_view = local_user_view_from_jwt(data.auth.as_ref(), context).await?; - let person_id = local_user_view.person.id; - - let page = data.page; - let limit = data.limit; - let unread_only = data.unread_only; - let mut messages = PrivateMessageQuery { - page, - limit, - unread_only, - } - .list(&mut context.pool(), person_id) - .await?; - - // Messages sent by ourselves should be marked as read. The `read` column in database is only - // for the recipient, and shouldnt be exposed to sender. - messages.iter_mut().for_each(|pmv| { - if pmv.creator.id == person_id { - pmv.private_message.read = true - } - }); - - Ok(PrivateMessagesResponse { - private_messages: messages, - }) + let page = data.page; + let limit = data.limit; + let unread_only = data.unread_only; + let mut messages = PrivateMessageQuery { + page, + limit, + unread_only, } + .list(&mut context.pool(), person_id) + .await?; + + // Messages sent by ourselves should be marked as read. The `read` column in database is only + // for the recipient, and shouldnt be exposed to sender. + messages.iter_mut().for_each(|pmv| { + if pmv.creator.id == person_id { + pmv.private_message.read = true + } + }); + + Ok(Json(PrivateMessagesResponse { + private_messages: messages, + })) } diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 98d111a1d..59a57ff8f 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -1,9 +1,6 @@ -use crate::{ - site::{application_question_check, site_default_post_listing_type_check}, - PerformCrud, -}; +use crate::site::{application_question_check, site_default_post_listing_type_check}; use activitypub_federation::http_signatures::generate_actor_keypair; -use actix_web::web::Data; +use actix_web::web::{Data, Json}; use lemmy_api_common::{ context::LemmyContext, site::{CreateSite, SiteResponse}, @@ -43,108 +40,105 @@ use lemmy_utils::{ }; use url::Url; -#[async_trait::async_trait(?Send)] -impl PerformCrud for CreateSite { - type Response = SiteResponse; +#[tracing::instrument(skip(context))] +pub async fn create_site( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; + let local_site = LocalSite::read(&mut context.pool()).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &CreateSite = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - let local_site = LocalSite::read(&mut context.pool()).await?; + // Make sure user is an admin; other types of users should not create site data... + is_admin(&local_user_view)?; - // Make sure user is an admin; other types of users should not create site data... - is_admin(&local_user_view)?; + validate_create_payload(&local_site, &data)?; - validate_create_payload(&local_site, data)?; + let actor_id: DbUrl = Url::parse(&context.settings().get_protocol_and_hostname())?.into(); + let inbox_url = Some(generate_site_inbox_url(&actor_id)?); + let keypair = generate_actor_keypair()?; + let name = sanitize_html(&data.name); + let sidebar = sanitize_html_opt(&data.sidebar); + let description = sanitize_html_opt(&data.description); - let actor_id: DbUrl = Url::parse(&context.settings().get_protocol_and_hostname())?.into(); - let inbox_url = Some(generate_site_inbox_url(&actor_id)?); - let keypair = generate_actor_keypair()?; - let name = sanitize_html(&data.name); - let sidebar = sanitize_html_opt(&data.sidebar); - let description = sanitize_html_opt(&data.description); + let site_form = SiteUpdateForm::builder() + .name(Some(name)) + .sidebar(diesel_option_overwrite(sidebar)) + .description(diesel_option_overwrite(description)) + .icon(diesel_option_overwrite_to_url(&data.icon)?) + .banner(diesel_option_overwrite_to_url(&data.banner)?) + .actor_id(Some(actor_id)) + .last_refreshed_at(Some(naive_now())) + .inbox_url(inbox_url) + .private_key(Some(Some(keypair.private_key))) + .public_key(Some(keypair.public_key)) + .build(); - let site_form = SiteUpdateForm::builder() - .name(Some(name)) - .sidebar(diesel_option_overwrite(sidebar)) - .description(diesel_option_overwrite(description)) - .icon(diesel_option_overwrite_to_url(&data.icon)?) - .banner(diesel_option_overwrite_to_url(&data.banner)?) - .actor_id(Some(actor_id)) - .last_refreshed_at(Some(naive_now())) - .inbox_url(inbox_url) - .private_key(Some(Some(keypair.private_key))) - .public_key(Some(keypair.public_key)) - .build(); + let site_id = local_site.site_id; - let site_id = local_site.site_id; + Site::update(&mut context.pool(), site_id, &site_form).await?; - Site::update(&mut context.pool(), site_id, &site_form).await?; + let application_question = sanitize_html_opt(&data.application_question); + let default_theme = sanitize_html_opt(&data.default_theme); + let legal_information = sanitize_html_opt(&data.legal_information); - let application_question = sanitize_html_opt(&data.application_question); - let default_theme = sanitize_html_opt(&data.default_theme); - let legal_information = sanitize_html_opt(&data.legal_information); + let local_site_form = LocalSiteUpdateForm::builder() + // Set the site setup to true + .site_setup(Some(true)) + .enable_downvotes(data.enable_downvotes) + .registration_mode(data.registration_mode) + .enable_nsfw(data.enable_nsfw) + .community_creation_admin_only(data.community_creation_admin_only) + .require_email_verification(data.require_email_verification) + .application_question(diesel_option_overwrite(application_question)) + .private_instance(data.private_instance) + .default_theme(default_theme) + .default_post_listing_type(data.default_post_listing_type) + .legal_information(diesel_option_overwrite(legal_information)) + .application_email_admins(data.application_email_admins) + .hide_modlog_mod_names(data.hide_modlog_mod_names) + .updated(Some(Some(naive_now()))) + .slur_filter_regex(diesel_option_overwrite(data.slur_filter_regex.clone())) + .actor_name_max_length(data.actor_name_max_length) + .federation_enabled(data.federation_enabled) + .captcha_enabled(data.captcha_enabled) + .captcha_difficulty(data.captcha_difficulty.clone()) + .build(); - let local_site_form = LocalSiteUpdateForm::builder() - // Set the site setup to true - .site_setup(Some(true)) - .enable_downvotes(data.enable_downvotes) - .registration_mode(data.registration_mode) - .enable_nsfw(data.enable_nsfw) - .community_creation_admin_only(data.community_creation_admin_only) - .require_email_verification(data.require_email_verification) - .application_question(diesel_option_overwrite(application_question)) - .private_instance(data.private_instance) - .default_theme(default_theme) - .default_post_listing_type(data.default_post_listing_type) - .legal_information(diesel_option_overwrite(legal_information)) - .application_email_admins(data.application_email_admins) - .hide_modlog_mod_names(data.hide_modlog_mod_names) - .updated(Some(Some(naive_now()))) - .slur_filter_regex(diesel_option_overwrite(data.slur_filter_regex.clone())) - .actor_name_max_length(data.actor_name_max_length) - .federation_enabled(data.federation_enabled) - .captcha_enabled(data.captcha_enabled) - .captcha_difficulty(data.captcha_difficulty.clone()) - .build(); + LocalSite::update(&mut context.pool(), &local_site_form).await?; - LocalSite::update(&mut context.pool(), &local_site_form).await?; + let local_site_rate_limit_form = LocalSiteRateLimitUpdateForm::builder() + .message(data.rate_limit_message) + .message_per_second(data.rate_limit_message_per_second) + .post(data.rate_limit_post) + .post_per_second(data.rate_limit_post_per_second) + .register(data.rate_limit_register) + .register_per_second(data.rate_limit_register_per_second) + .image(data.rate_limit_image) + .image_per_second(data.rate_limit_image_per_second) + .comment(data.rate_limit_comment) + .comment_per_second(data.rate_limit_comment_per_second) + .search(data.rate_limit_search) + .search_per_second(data.rate_limit_search_per_second) + .build(); - let local_site_rate_limit_form = LocalSiteRateLimitUpdateForm::builder() - .message(data.rate_limit_message) - .message_per_second(data.rate_limit_message_per_second) - .post(data.rate_limit_post) - .post_per_second(data.rate_limit_post_per_second) - .register(data.rate_limit_register) - .register_per_second(data.rate_limit_register_per_second) - .image(data.rate_limit_image) - .image_per_second(data.rate_limit_image_per_second) - .comment(data.rate_limit_comment) - .comment_per_second(data.rate_limit_comment_per_second) - .search(data.rate_limit_search) - .search_per_second(data.rate_limit_search_per_second) - .build(); + LocalSiteRateLimit::update(&mut context.pool(), &local_site_rate_limit_form).await?; - LocalSiteRateLimit::update(&mut context.pool(), &local_site_rate_limit_form).await?; + let site_view = SiteView::read_local(&mut context.pool()).await?; - let site_view = SiteView::read_local(&mut context.pool()).await?; + let new_taglines = data.taglines.clone(); + let taglines = Tagline::replace(&mut context.pool(), local_site.id, new_taglines).await?; - let new_taglines = data.taglines.clone(); - let taglines = Tagline::replace(&mut context.pool(), local_site.id, new_taglines).await?; + let rate_limit_config = + local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit); + context + .settings_updated_channel() + .send(rate_limit_config) + .await?; - let rate_limit_config = - local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit); - context - .settings_updated_channel() - .send(rate_limit_config) - .await?; - - Ok(SiteResponse { - site_view, - taglines, - }) - } + Ok(Json(SiteResponse { + site_view, + taglines, + })) } fn validate_create_payload(local_site: &LocalSite, create_site: &CreateSite) -> LemmyResult<()> { diff --git a/crates/api_crud/src/site/mod.rs b/crates/api_crud/src/site/mod.rs index 652b9e656..e4911ba48 100644 --- a/crates/api_crud/src/site/mod.rs +++ b/crates/api_crud/src/site/mod.rs @@ -1,9 +1,9 @@ use lemmy_db_schema::{ListingType, RegistrationMode}; use lemmy_utils::error::{LemmyErrorType, LemmyResult}; -mod create; -mod read; -mod update; +pub mod create; +pub mod read; +pub mod update; /// Checks whether the default post listing type is valid for a site. pub fn site_default_post_listing_type_check( diff --git a/crates/api_crud/src/site/read.rs b/crates/api_crud/src/site/read.rs index e74eeefbd..62d96492a 100644 --- a/crates/api_crud/src/site/read.rs +++ b/crates/api_crud/src/site/read.rs @@ -1,5 +1,4 @@ -use crate::PerformCrud; -use actix_web::web::Data; +use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ context::LemmyContext, sensitive::Sensitive, @@ -28,76 +27,72 @@ use lemmy_utils::{ version, }; -#[async_trait::async_trait(?Send)] -impl PerformCrud for GetSite { - type Response = GetSiteResponse; +#[tracing::instrument(skip(context))] +pub async fn get_site( + data: Query, + context: Data, +) -> Result, LemmyError> { + let site_view = SiteView::read_local(&mut context.pool()).await?; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &GetSite = self; + let admins = PersonView::admins(&mut context.pool()).await?; - let site_view = SiteView::read_local(&mut context.pool()).await?; + // Build the local user + let my_user = if let Some(local_user_view) = + local_user_settings_view_from_jwt_opt(data.auth.as_ref(), &context).await + { + let person_id = local_user_view.person.id; + let local_user_id = local_user_view.local_user.id; - let admins = PersonView::admins(&mut context.pool()).await?; + let follows = CommunityFollowerView::for_person(&mut context.pool(), person_id) + .await + .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - // Build the local user - let my_user = if let Some(local_user_view) = - local_user_settings_view_from_jwt_opt(data.auth.as_ref(), context).await - { - let person_id = local_user_view.person.id; - let local_user_id = local_user_view.local_user.id; + let person_id = local_user_view.person.id; + let community_blocks = CommunityBlockView::for_person(&mut context.pool(), person_id) + .await + .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - let follows = CommunityFollowerView::for_person(&mut context.pool(), person_id) - .await - .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; + let person_id = local_user_view.person.id; + let person_blocks = PersonBlockView::for_person(&mut context.pool(), person_id) + .await + .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - let person_id = local_user_view.person.id; - let community_blocks = CommunityBlockView::for_person(&mut context.pool(), person_id) - .await - .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; + let moderates = CommunityModeratorView::for_person(&mut context.pool(), person_id) + .await + .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - let person_id = local_user_view.person.id; - let person_blocks = PersonBlockView::for_person(&mut context.pool(), person_id) - .await - .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; + let discussion_languages = LocalUserLanguage::read(&mut context.pool(), local_user_id) + .await + .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - let moderates = CommunityModeratorView::for_person(&mut context.pool(), person_id) - .await - .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - - let discussion_languages = LocalUserLanguage::read(&mut context.pool(), local_user_id) - .await - .with_lemmy_type(LemmyErrorType::SystemErrLogin)?; - - Some(MyUserInfo { - local_user_view, - follows, - moderates, - community_blocks, - person_blocks, - discussion_languages, - }) - } else { - None - }; - - let all_languages = Language::read_all(&mut context.pool()).await?; - let discussion_languages = SiteLanguage::read_local_raw(&mut context.pool()).await?; - let taglines = Tagline::get_all(&mut context.pool(), site_view.local_site.id).await?; - let custom_emojis = - CustomEmojiView::get_all(&mut context.pool(), site_view.local_site.id).await?; - - Ok(GetSiteResponse { - site_view, - admins, - version: version::VERSION.to_string(), - my_user, - all_languages, + Some(MyUserInfo { + local_user_view, + follows, + moderates, + community_blocks, + person_blocks, discussion_languages, - taglines, - custom_emojis, }) - } + } else { + None + }; + + let all_languages = Language::read_all(&mut context.pool()).await?; + let discussion_languages = SiteLanguage::read_local_raw(&mut context.pool()).await?; + let taglines = Tagline::get_all(&mut context.pool(), site_view.local_site.id).await?; + let custom_emojis = + CustomEmojiView::get_all(&mut context.pool(), site_view.local_site.id).await?; + + Ok(Json(GetSiteResponse { + site_view, + admins, + version: version::VERSION.to_string(), + my_user, + all_languages, + discussion_languages, + taglines, + custom_emojis, + })) } #[tracing::instrument(skip_all)] diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index 2b8ce4c0f..f560c7295 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -1,8 +1,5 @@ -use crate::{ - site::{application_question_check, site_default_post_listing_type_check}, - PerformCrud, -}; -use actix_web::web::Data; +use crate::site::{application_question_check, site_default_post_listing_type_check}; +use actix_web::web::{Data, Json}; use lemmy_api_common::{ context::LemmyContext, site::{EditSite, SiteResponse}, @@ -43,147 +40,142 @@ use lemmy_utils::{ }, }; -#[async_trait::async_trait(?Send)] -impl PerformCrud for EditSite { - type Response = SiteResponse; +#[tracing::instrument(skip(context))] +pub async fn update_site( + data: Json, + context: Data, +) -> Result, LemmyError> { + let local_user_view = local_user_view_from_jwt(&data.auth, &context).await?; + let site_view = SiteView::read_local(&mut context.pool()).await?; + let local_site = site_view.local_site; + let site = site_view.site; - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let data: &EditSite = self; - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - let site_view = SiteView::read_local(&mut context.pool()).await?; - let local_site = site_view.local_site; - let site = site_view.site; + // Make sure user is an admin; other types of users should not update site data... + is_admin(&local_user_view)?; - // Make sure user is an admin; other types of users should not update site data... - is_admin(&local_user_view)?; + validate_update_payload(&local_site, &data)?; - validate_update_payload(&local_site, data)?; - - if let Some(discussion_languages) = data.discussion_languages.clone() { - SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?; - } - - let name = sanitize_html_opt(&data.name); - let sidebar = sanitize_html_opt(&data.sidebar); - let description = sanitize_html_opt(&data.description); - - let site_form = SiteUpdateForm::builder() - .name(name) - .sidebar(diesel_option_overwrite(sidebar)) - .description(diesel_option_overwrite(description)) - .icon(diesel_option_overwrite_to_url(&data.icon)?) - .banner(diesel_option_overwrite_to_url(&data.banner)?) - .updated(Some(Some(naive_now()))) - .build(); - - Site::update(&mut context.pool(), site.id, &site_form) - .await - // Ignore errors for all these, so as to not throw errors if no update occurs - // Diesel will throw an error for empty update forms - .ok(); - - let application_question = sanitize_html_opt(&data.application_question); - let default_theme = sanitize_html_opt(&data.default_theme); - let legal_information = sanitize_html_opt(&data.legal_information); - - let local_site_form = LocalSiteUpdateForm::builder() - .enable_downvotes(data.enable_downvotes) - .registration_mode(data.registration_mode) - .enable_nsfw(data.enable_nsfw) - .community_creation_admin_only(data.community_creation_admin_only) - .require_email_verification(data.require_email_verification) - .application_question(diesel_option_overwrite(application_question)) - .private_instance(data.private_instance) - .default_theme(default_theme) - .default_post_listing_type(data.default_post_listing_type) - .legal_information(diesel_option_overwrite(legal_information)) - .application_email_admins(data.application_email_admins) - .hide_modlog_mod_names(data.hide_modlog_mod_names) - .updated(Some(Some(naive_now()))) - .slur_filter_regex(diesel_option_overwrite(data.slur_filter_regex.clone())) - .actor_name_max_length(data.actor_name_max_length) - .federation_enabled(data.federation_enabled) - .captcha_enabled(data.captcha_enabled) - .captcha_difficulty(data.captcha_difficulty.clone()) - .reports_email_admins(data.reports_email_admins) - .build(); - - let update_local_site = LocalSite::update(&mut context.pool(), &local_site_form) - .await - .ok(); - - let local_site_rate_limit_form = LocalSiteRateLimitUpdateForm::builder() - .message(data.rate_limit_message) - .message_per_second(data.rate_limit_message_per_second) - .post(data.rate_limit_post) - .post_per_second(data.rate_limit_post_per_second) - .register(data.rate_limit_register) - .register_per_second(data.rate_limit_register_per_second) - .image(data.rate_limit_image) - .image_per_second(data.rate_limit_image_per_second) - .comment(data.rate_limit_comment) - .comment_per_second(data.rate_limit_comment_per_second) - .search(data.rate_limit_search) - .search_per_second(data.rate_limit_search_per_second) - .build(); - - LocalSiteRateLimit::update(&mut context.pool(), &local_site_rate_limit_form) - .await - .ok(); - - // Replace the blocked and allowed instances - let allowed = data.allowed_instances.clone(); - FederationAllowList::replace(&mut context.pool(), allowed).await?; - let blocked = data.blocked_instances.clone(); - FederationBlockList::replace(&mut context.pool(), blocked).await?; - - // TODO can't think of a better way to do this. - // If the server suddenly requires email verification, or required applications, no old users - // will be able to log in. It really only wants this to be a requirement for NEW signups. - // So if it was set from false, to true, you need to update all current users columns to be verified. - - let old_require_application = - local_site.registration_mode == RegistrationMode::RequireApplication; - let new_require_application = update_local_site - .as_ref() - .map(|ols| ols.registration_mode == RegistrationMode::RequireApplication) - .unwrap_or(false); - if !old_require_application && new_require_application { - LocalUser::set_all_users_registration_applications_accepted(&mut context.pool()) - .await - .with_lemmy_type(LemmyErrorType::CouldntSetAllRegistrationsAccepted)?; - } - - let new_require_email_verification = update_local_site - .as_ref() - .map(|ols| ols.require_email_verification) - .unwrap_or(false); - if !local_site.require_email_verification && new_require_email_verification { - LocalUser::set_all_users_email_verified(&mut context.pool()) - .await - .with_lemmy_type(LemmyErrorType::CouldntSetAllEmailVerified)?; - } - - let new_taglines = data.taglines.clone(); - let taglines = Tagline::replace(&mut context.pool(), local_site.id, new_taglines).await?; - - let site_view = SiteView::read_local(&mut context.pool()).await?; - - let rate_limit_config = - local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit); - context - .settings_updated_channel() - .send(rate_limit_config) - .await?; - - let res = SiteResponse { - site_view, - taglines, - }; - - Ok(res) + if let Some(discussion_languages) = data.discussion_languages.clone() { + SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?; } + + let name = sanitize_html_opt(&data.name); + let sidebar = sanitize_html_opt(&data.sidebar); + let description = sanitize_html_opt(&data.description); + + let site_form = SiteUpdateForm::builder() + .name(name) + .sidebar(diesel_option_overwrite(sidebar)) + .description(diesel_option_overwrite(description)) + .icon(diesel_option_overwrite_to_url(&data.icon)?) + .banner(diesel_option_overwrite_to_url(&data.banner)?) + .updated(Some(Some(naive_now()))) + .build(); + + Site::update(&mut context.pool(), site.id, &site_form) + .await + // Ignore errors for all these, so as to not throw errors if no update occurs + // Diesel will throw an error for empty update forms + .ok(); + + let application_question = sanitize_html_opt(&data.application_question); + let default_theme = sanitize_html_opt(&data.default_theme); + let legal_information = sanitize_html_opt(&data.legal_information); + + let local_site_form = LocalSiteUpdateForm::builder() + .enable_downvotes(data.enable_downvotes) + .registration_mode(data.registration_mode) + .enable_nsfw(data.enable_nsfw) + .community_creation_admin_only(data.community_creation_admin_only) + .require_email_verification(data.require_email_verification) + .application_question(diesel_option_overwrite(application_question)) + .private_instance(data.private_instance) + .default_theme(default_theme) + .default_post_listing_type(data.default_post_listing_type) + .legal_information(diesel_option_overwrite(legal_information)) + .application_email_admins(data.application_email_admins) + .hide_modlog_mod_names(data.hide_modlog_mod_names) + .updated(Some(Some(naive_now()))) + .slur_filter_regex(diesel_option_overwrite(data.slur_filter_regex.clone())) + .actor_name_max_length(data.actor_name_max_length) + .federation_enabled(data.federation_enabled) + .captcha_enabled(data.captcha_enabled) + .captcha_difficulty(data.captcha_difficulty.clone()) + .reports_email_admins(data.reports_email_admins) + .build(); + + let update_local_site = LocalSite::update(&mut context.pool(), &local_site_form) + .await + .ok(); + + let local_site_rate_limit_form = LocalSiteRateLimitUpdateForm::builder() + .message(data.rate_limit_message) + .message_per_second(data.rate_limit_message_per_second) + .post(data.rate_limit_post) + .post_per_second(data.rate_limit_post_per_second) + .register(data.rate_limit_register) + .register_per_second(data.rate_limit_register_per_second) + .image(data.rate_limit_image) + .image_per_second(data.rate_limit_image_per_second) + .comment(data.rate_limit_comment) + .comment_per_second(data.rate_limit_comment_per_second) + .search(data.rate_limit_search) + .search_per_second(data.rate_limit_search_per_second) + .build(); + + LocalSiteRateLimit::update(&mut context.pool(), &local_site_rate_limit_form) + .await + .ok(); + + // Replace the blocked and allowed instances + let allowed = data.allowed_instances.clone(); + FederationAllowList::replace(&mut context.pool(), allowed).await?; + let blocked = data.blocked_instances.clone(); + FederationBlockList::replace(&mut context.pool(), blocked).await?; + + // TODO can't think of a better way to do this. + // If the server suddenly requires email verification, or required applications, no old users + // will be able to log in. It really only wants this to be a requirement for NEW signups. + // So if it was set from false, to true, you need to update all current users columns to be verified. + + let old_require_application = + local_site.registration_mode == RegistrationMode::RequireApplication; + let new_require_application = update_local_site + .as_ref() + .map(|ols| ols.registration_mode == RegistrationMode::RequireApplication) + .unwrap_or(false); + if !old_require_application && new_require_application { + LocalUser::set_all_users_registration_applications_accepted(&mut context.pool()) + .await + .with_lemmy_type(LemmyErrorType::CouldntSetAllRegistrationsAccepted)?; + } + + let new_require_email_verification = update_local_site + .as_ref() + .map(|ols| ols.require_email_verification) + .unwrap_or(false); + if !local_site.require_email_verification && new_require_email_verification { + LocalUser::set_all_users_email_verified(&mut context.pool()) + .await + .with_lemmy_type(LemmyErrorType::CouldntSetAllEmailVerified)?; + } + + let new_taglines = data.taglines.clone(); + let taglines = Tagline::replace(&mut context.pool(), local_site.id, new_taglines).await?; + + let site_view = SiteView::read_local(&mut context.pool()).await?; + + let rate_limit_config = + local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit); + context + .settings_updated_channel() + .send(rate_limit_config) + .await?; + + Ok(Json(SiteResponse { + site_view, + taglines, + })) } fn validate_update_payload(local_site: &LocalSite, edit_site: &EditSite) -> LemmyResult<()> { diff --git a/crates/apub/src/activities/create_or_update/comment.rs b/crates/apub/src/activities/create_or_update/comment.rs index 51b87ed27..fa235a7f2 100644 --- a/crates/apub/src/activities/create_or_update/comment.rs +++ b/crates/apub/src/activities/create_or_update/comment.rs @@ -25,7 +25,7 @@ use activitypub_federation::{ }; use lemmy_api_common::{ build_response::send_local_notifs, - comment::{CommentResponse, CreateComment, EditComment}, + comment::{CommentResponse, EditComment}, context::LemmyContext, utils::{check_post_deleted_or_removed, is_mod_or_admin}, }; @@ -43,25 +43,6 @@ use lemmy_db_schema::{ use lemmy_utils::{error::LemmyError, utils::mention::scrape_text_for_mentions}; use url::Url; -#[async_trait::async_trait] -impl SendActivity for CreateComment { - type Response = CommentResponse; - - async fn send_activity( - _request: &Self, - response: &Self::Response, - context: &Data, - ) -> Result<(), LemmyError> { - CreateOrUpdateNote::send( - &response.comment_view.comment, - response.comment_view.creator.id, - CreateOrUpdateType::Create, - context, - ) - .await - } -} - #[async_trait::async_trait] impl SendActivity for EditComment { type Response = CommentResponse; @@ -72,10 +53,10 @@ impl SendActivity for EditComment { context: &Data, ) -> Result<(), LemmyError> { CreateOrUpdateNote::send( - &response.comment_view.comment, + response.comment_view.comment.clone(), response.comment_view.creator.id, CreateOrUpdateType::Update, - context, + context.reset_request_count(), ) .await } @@ -83,11 +64,11 @@ impl SendActivity for EditComment { impl CreateOrUpdateNote { #[tracing::instrument(skip(comment, person_id, kind, context))] - async fn send( - comment: &Comment, + pub(crate) async fn send( + comment: Comment, person_id: PersonId, kind: CreateOrUpdateType, - context: &Data, + context: Data, ) -> Result<(), LemmyError> { // TODO: might be helpful to add a comment method to retrieve community directly let post_id = comment.post_id; @@ -102,7 +83,7 @@ impl CreateOrUpdateNote { kind.clone(), &context.settings().get_protocol_and_hostname(), )?; - let note = ApubComment(comment.clone()).into_json(context).await?; + let note = ApubComment(comment).into_json(&context).await?; let create_or_update = CreateOrUpdateNote { actor: person.id().into(), @@ -130,12 +111,12 @@ impl CreateOrUpdateNote { .collect(); let mut inboxes = vec![]; for t in tagged_users { - let person = t.dereference(context).await?; + let person = t.dereference(&context).await?; inboxes.push(person.shared_inbox_or_inbox()); } let activity = AnnouncableActivities::CreateOrUpdateComment(create_or_update); - send_activity_in_community(activity, &person, &community, inboxes, false, context).await + send_activity_in_community(activity, &person, &community, inboxes, false, &context).await } } diff --git a/crates/apub/src/activities/mod.rs b/crates/apub/src/activities/mod.rs index 02ad0b6b1..c7d19e372 100644 --- a/crates/apub/src/activities/mod.rs +++ b/crates/apub/src/activities/mod.rs @@ -1,6 +1,9 @@ use crate::{ objects::{community::ApubCommunity, person::ApubPerson}, - protocol::activities::{create_or_update::page::CreateOrUpdatePage, CreateOrUpdateType}, + protocol::activities::{ + create_or_update::{note::CreateOrUpdateNote, page::CreateOrUpdatePage}, + CreateOrUpdateType, + }, CONTEXT, }; use activitypub_federation::{ @@ -217,15 +220,17 @@ pub async fn match_outgoing_activities( data: SendActivityData, context: &Data, ) -> LemmyResult<()> { - let fed_task = match data { - SendActivityData::CreatePost(post) => { - let creator_id = post.creator_id; - CreateOrUpdatePage::send( - post, - creator_id, - CreateOrUpdateType::Create, - context.reset_request_count(), - ) + let context = context.reset_request_count(); + let fed_task = async { + match data { + SendActivityData::CreatePost(post) => { + let creator_id = post.creator_id; + CreateOrUpdatePage::send(post, creator_id, CreateOrUpdateType::Create, context).await + } + SendActivityData::CreateComment(comment) => { + let creator_id = comment.creator_id; + CreateOrUpdateNote::send(comment, creator_id, CreateOrUpdateType::Create, context).await + } } }; if *SYNCHRONOUS_FEDERATION { diff --git a/crates/apub/src/api/read_community.rs b/crates/apub/src/api/read_community.rs index 12e17dac6..1bdfb88a0 100644 --- a/crates/apub/src/api/read_community.rs +++ b/crates/apub/src/api/read_community.rs @@ -16,7 +16,7 @@ use lemmy_db_views_actor::structs::{CommunityModeratorView, CommunityView}; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorExt2, LemmyErrorType}; #[tracing::instrument(skip(context))] -pub async fn read_community( +pub async fn get_community( data: Query, context: Data, ) -> Result, LemmyError> { diff --git a/crates/utils/translations b/crates/utils/translations index 713ceed9c..1c42c5794 160000 --- a/crates/utils/translations +++ b/crates/utils/translations @@ -1 +1 @@ -Subproject commit 713ceed9c7ef84deaa222e68361e670e0763cd83 +Subproject commit 1c42c579460871de7b4ea18e58dc25543b80d289 diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index bc4340e3c..e372e340e 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -1,19 +1,12 @@ use actix_web::{guard, web, Error, HttpResponse, Result}; -use lemmy_api::Perform; +use lemmy_api::{ + comment::{distinguish::distinguish_comment, save::save_comment}, + comment_report::{list::list_comment_reports, resolve::resolve_comment_report}, + local_user::notifications::mark_reply_read::mark_reply_as_read, + Perform, +}; use lemmy_api_common::{ - comment::{ - CreateComment, - CreateCommentLike, - CreateCommentReport, - DeleteComment, - DistinguishComment, - EditComment, - GetComment, - ListCommentReports, - RemoveComment, - ResolveCommentReport, - SaveComment, - }, + comment::{CreateCommentLike, CreateCommentReport, DeleteComment, EditComment, RemoveComment}, community::{ AddModToCommunity, BanFromCommunity, @@ -23,7 +16,6 @@ use lemmy_api_common::{ EditCommunity, FollowCommunity, HideCommunity, - ListCommunities, RemoveCommunity, TransferCommunity, }, @@ -43,7 +35,6 @@ use lemmy_api_common::{ GetUnreadCount, Login, MarkAllAsRead, - MarkCommentReplyAsRead, MarkPersonMentionAsRead, PasswordChangeAfterReset, PasswordReset, @@ -57,7 +48,6 @@ use lemmy_api_common::{ DeletePost, EditPost, FeaturePost, - GetPost, GetSiteMetadata, ListPostReports, LockPost, @@ -71,18 +61,14 @@ use lemmy_api_common::{ CreatePrivateMessageReport, DeletePrivateMessage, EditPrivateMessage, - GetPrivateMessages, ListPrivateMessageReports, MarkPrivateMessageAsRead, ResolvePrivateMessageReport, }, site::{ ApproveRegistrationApplication, - CreateSite, - EditSite, GetFederatedInstances, GetModlog, - GetSite, GetUnreadRegistrationApplicationCount, LeaveAdmin, ListRegistrationApplications, @@ -92,12 +78,19 @@ use lemmy_api_common::{ PurgePost, }, }; -use lemmy_api_crud::{post::create::create_post, PerformCrud}; +use lemmy_api_crud::{ + comment::{create::create_comment, read::get_comment}, + community::list::list_communities, + post::{create::create_post, read::get_post}, + private_message::read::get_private_message, + site::{create::create_site, read::get_site, update::update_site}, + PerformCrud, +}; use lemmy_apub::{ api::{ list_comments::list_comments, list_posts::list_posts, - read_community::read_community, + read_community::get_community, read_person::read_person, resolve_object::resolve_object, search::search, @@ -114,10 +107,10 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .service( web::scope("/site") .wrap(rate_limit.message()) - .route("", web::get().to(route_get_crud::)) + .route("", web::get().to(get_site)) // Admin Actions - .route("", web::post().to(route_post_crud::)) - .route("", web::put().to(route_post_crud::)), + .route("", web::post().to(create_site)) + .route("", web::put().to(update_site)), ) .service( web::resource("/modlog") @@ -144,10 +137,10 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .service( web::scope("/community") .wrap(rate_limit.message()) - .route("", web::get().to(read_community)) + .route("", web::get().to(get_community)) .route("", web::put().to(route_post_crud::)) .route("/hide", web::put().to(route_post::)) - .route("/list", web::get().to(route_get_crud::)) + .route("/list", web::get().to(list_communities)) .route("/follow", web::post().to(route_post::)) .route("/block", web::post().to(route_post::)) .route( @@ -179,7 +172,7 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .service( web::scope("/post") .wrap(rate_limit.message()) - .route("", web::get().to(route_get_crud::)) + .route("", web::get().to(get_post)) .route("", web::put().to(route_post_crud::)) .route("/delete", web::post().to(route_post_crud::)) .route("/remove", web::post().to(route_post_crud::)) @@ -209,41 +202,29 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { web::resource("/comment") .guard(guard::Post()) .wrap(rate_limit.comment()) - .route(web::post().to(route_post_crud::)), + .route(web::post().to(create_comment)), ) .service( web::scope("/comment") .wrap(rate_limit.message()) - .route("", web::get().to(route_get_crud::)) + .route("", web::get().to(get_comment)) .route("", web::put().to(route_post_crud::)) .route("/delete", web::post().to(route_post_crud::)) .route("/remove", web::post().to(route_post_crud::)) - .route( - "/mark_as_read", - web::post().to(route_post::), - ) - .route( - "/distinguish", - web::post().to(route_post::), - ) + .route("/mark_as_read", web::post().to(mark_reply_as_read)) + .route("/distinguish", web::post().to(distinguish_comment)) .route("/like", web::post().to(route_post::)) - .route("/save", web::put().to(route_post::)) + .route("/save", web::put().to(save_comment)) .route("/list", web::get().to(list_comments)) .route("/report", web::post().to(route_post::)) - .route( - "/report/resolve", - web::put().to(route_post::), - ) - .route( - "/report/list", - web::get().to(route_get::), - ), + .route("/report/resolve", web::put().to(resolve_comment_report)) + .route("/report/list", web::get().to(list_comment_reports)), ) // Private Message .service( web::scope("/private_message") .wrap(rate_limit.message()) - .route("/list", web::get().to(route_get_crud::)) + .route("/list", web::get().to(get_private_message)) .route("", web::post().to(route_post_crud::)) .route("", web::put().to(route_post_crud::)) .route( @@ -447,22 +428,6 @@ where Ok(HttpResponse::Ok().json(&res)) } -async fn route_get_crud<'a, Data>( - data: web::Query, - context: web::Data, - apub_data: activitypub_federation::config::Data, -) -> Result -where - Data: PerformCrud - + SendActivity::Response> - + Clone - + Deserialize<'a> - + Send - + 'static, -{ - perform_crud::(data.0, context, apub_data).await -} - async fn route_post_crud<'a, Data>( data: web::Json, context: web::Data, From e365d48769ba9ab5d394491ff64784e9f0818f0a Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 28 Jul 2023 17:11:18 +0200 Subject: [PATCH 24/27] Move entry from .rgignore to .gitignore (#3752) rg command respects gitignore, and sql dumps should be ignored by git as well. So theres no reason to have a separate file. --- .gitignore | 3 +++ .rgignore | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 .rgignore diff --git a/.gitignore b/.gitignore index e48248836..186713e1f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ bindings # Database cluster and sockets for testing dev_pgdata/ *.PGSQL.* + +# database dumps +*.sqldump diff --git a/.rgignore b/.rgignore deleted file mode 100644 index eab207b73..000000000 --- a/.rgignore +++ /dev/null @@ -1 +0,0 @@ -*.sqldump From 39752fa096b86f0858bc009945847d65d0ab7570 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 28 Jul 2023 17:14:26 +0200 Subject: [PATCH 25/27] Delete migrations_testing folder (#3751) Its completely unused as far as I can see --- .../down.sql | 211 ------------ .../up.sql | 324 ------------------ 2 files changed, 535 deletions(-) delete mode 100644 migrations_testing/2020-01-13-025151_create_materialized_views/down.sql delete mode 100644 migrations_testing/2020-01-13-025151_create_materialized_views/up.sql diff --git a/migrations_testing/2020-01-13-025151_create_materialized_views/down.sql b/migrations_testing/2020-01-13-025151_create_materialized_views/down.sql deleted file mode 100644 index ba801ba54..000000000 --- a/migrations_testing/2020-01-13-025151_create_materialized_views/down.sql +++ /dev/null @@ -1,211 +0,0 @@ --- functions and triggers -drop trigger refresh_user on user_; -drop function refresh_user(); -drop trigger refresh_post on post; -drop function refresh_post(); -drop trigger refresh_post_like on post_like; -drop function refresh_post_like(); -drop trigger refresh_community on community; -drop function refresh_community(); -drop trigger refresh_community_follower on community_follower; -drop function refresh_community_follower(); -drop trigger refresh_comment on comment; -drop function refresh_comment(); -drop trigger refresh_comment_like on comment_like; -drop function refresh_comment_like(); - --- post --- Recreate the view -drop materialized view post_view; -create view post_view as -with all_post as -( - select - p.*, - (select u.banned from user_ u where p.creator_id = u.id) as banned, - (select cb.id::bool from community_user_ban cb where p.creator_id = cb.user_id and p.community_id = cb.community_id) as banned_from_community, - (select name from user_ where p.creator_id = user_.id) as creator_name, - (select avatar from user_ where p.creator_id = user_.id) as creator_avatar, - (select name from community where p.community_id = community.id) as community_name, - (select removed from community c where p.community_id = c.id) as community_removed, - (select deleted from community c where p.community_id = c.id) as community_deleted, - (select nsfw from community c where p.community_id = c.id) as community_nsfw, - (select count(*) from comment where comment.post_id = p.id) as number_of_comments, - coalesce(sum(pl.score), 0) as score, - count (case when pl.score = 1 then 1 else null end) as upvotes, - count (case when pl.score = -1 then 1 else null end) as downvotes, - hot_rank(coalesce(sum(pl.score) , 0), p.published) as hot_rank - from post p - left join post_like pl on p.id = pl.post_id - group by p.id -) - -select -ap.*, -u.id as user_id, -coalesce(pl.score, 0) as my_vote, -(select cf.id::bool from community_follower cf where u.id = cf.user_id and cf.community_id = ap.community_id) as subscribed, -(select pr.id::bool from post_read pr where u.id = pr.user_id and pr.post_id = ap.id) as read, -(select ps.id::bool from post_saved ps where u.id = ps.user_id and ps.post_id = ap.id) as saved -from user_ u -cross join all_post ap -left join post_like pl on u.id = pl.user_id and ap.id = pl.post_id - -union all - -select -ap.*, -null as user_id, -null as my_vote, -null as subscribed, -null as read, -null as saved -from all_post ap -; - - -drop materialized view user_view; -create view user_view as -select id, -name, -avatar, -email, -fedi_name, -admin, -banned, -show_avatars, -send_notifications_to_email, -published, -(select count(*) from post p where p.creator_id = u.id) as number_of_posts, -(select coalesce(sum(score), 0) from post p, post_like pl where u.id = p.creator_id and p.id = pl.post_id) as post_score, -(select count(*) from comment c where c.creator_id = u.id) as number_of_comments, -(select coalesce(sum(score), 0) from comment c, comment_like cl where u.id = c.creator_id and c.id = cl.comment_id) as comment_score -from user_ u; - - --- community -drop materialized view community_view; -create view community_view as -with all_community as -( - select *, - (select name from user_ u where c.creator_id = u.id) as creator_name, - (select avatar from user_ u where c.creator_id = u.id) as creator_avatar, - (select name from category ct where c.category_id = ct.id) as category_name, - (select count(*) from community_follower cf where cf.community_id = c.id) as number_of_subscribers, - (select count(*) from post p where p.community_id = c.id) as number_of_posts, - (select count(*) from comment co, post p where c.id = p.community_id and p.id = co.post_id) as number_of_comments, - hot_rank((select count(*) from community_follower cf where cf.community_id = c.id), c.published) as hot_rank - from community c -) - -select -ac.*, -u.id as user_id, -(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.id = cf.community_id) as subscribed -from user_ u -cross join all_community ac - -union all - -select -ac.*, -null as user_id, -null as subscribed -from all_community ac -; - --- reply and comment view -drop view reply_view; -drop view user_mention_view; -drop materialized view comment_view; -create view comment_view as -with all_comment as -( - select - c.*, - (select community_id from post p where p.id = c.post_id), - (select u.banned from user_ u where c.creator_id = u.id) as banned, - (select cb.id::bool from community_user_ban cb, post p where c.creator_id = cb.user_id and p.id = c.post_id and p.community_id = cb.community_id) as banned_from_community, - (select name from user_ where c.creator_id = user_.id) as creator_name, - (select avatar from user_ where c.creator_id = user_.id) as creator_avatar, - coalesce(sum(cl.score), 0) as score, - count (case when cl.score = 1 then 1 else null end) as upvotes, - count (case when cl.score = -1 then 1 else null end) as downvotes - from comment c - left join comment_like cl on c.id = cl.comment_id - group by c.id -) - -select -ac.*, -u.id as user_id, -coalesce(cl.score, 0) as my_vote, -(select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved -from user_ u -cross join all_comment ac -left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id - -union all - -select - ac.*, - null as user_id, - null as my_vote, - null as saved -from all_comment ac -; - -create view reply_view as -with closereply as ( - select - c2.id, - c2.creator_id as sender_id, - c.creator_id as recipient_id - from comment c - inner join comment c2 on c.id = c2.parent_id - where c2.creator_id != c.creator_id - -- Do union where post is null - union - select - c.id, - c.creator_id as sender_id, - p.creator_id as recipient_id - from comment c, post p - where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id -) -select cv.*, -closereply.recipient_id -from comment_view cv, closereply -where closereply.id = cv.id -; - --- user mention -create view user_mention_view as -select - c.id, - um.id as user_mention_id, - c.creator_id, - c.post_id, - c.parent_id, - c.content, - c.removed, - um.read, - c.published, - c.updated, - c.deleted, - c.community_id, - c.banned, - c.banned_from_community, - c.creator_name, - c.creator_avatar, - c.score, - c.upvotes, - c.downvotes, - c.user_id, - c.my_vote, - c.saved, - um.recipient_id -from user_mention um, comment_view c -where um.comment_id = c.id; - diff --git a/migrations_testing/2020-01-13-025151_create_materialized_views/up.sql b/migrations_testing/2020-01-13-025151_create_materialized_views/up.sql deleted file mode 100644 index 33b0442f1..000000000 --- a/migrations_testing/2020-01-13-025151_create_materialized_views/up.sql +++ /dev/null @@ -1,324 +0,0 @@ --- post -drop view post_view; -create materialized view post_view as -with all_post as -( - select - p.*, - (select u.banned from user_ u where p.creator_id = u.id) as banned, - (select cb.id::bool from community_user_ban cb where p.creator_id = cb.user_id and p.community_id = cb.community_id) as banned_from_community, - (select name from user_ where p.creator_id = user_.id) as creator_name, - (select avatar from user_ where p.creator_id = user_.id) as creator_avatar, - (select name from community where p.community_id = community.id) as community_name, - (select removed from community c where p.community_id = c.id) as community_removed, - (select deleted from community c where p.community_id = c.id) as community_deleted, - (select nsfw from community c where p.community_id = c.id) as community_nsfw, - (select count(*) from comment where comment.post_id = p.id) as number_of_comments, - coalesce(sum(pl.score), 0) as score, - count (case when pl.score = 1 then 1 else null end) as upvotes, - count (case when pl.score = -1 then 1 else null end) as downvotes, - hot_rank(coalesce(sum(pl.score) , 0), p.published) as hot_rank - from post p - left join post_like pl on p.id = pl.post_id - group by p.id -) - -select -ap.*, -u.id as user_id, -coalesce(pl.score, 0) as my_vote, -(select cf.id::bool from community_follower cf where u.id = cf.user_id and cf.community_id = ap.community_id) as subscribed, -(select pr.id::bool from post_read pr where u.id = pr.user_id and pr.post_id = ap.id) as read, -(select ps.id::bool from post_saved ps where u.id = ps.user_id and ps.post_id = ap.id) as saved -from user_ u -cross join all_post ap -left join post_like pl on u.id = pl.user_id and ap.id = pl.post_id - -union all - -select -ap.*, -null as user_id, -null as my_vote, -null as subscribed, -null as read, -null as saved -from all_post ap -with data -; - -create unique index idx_post_view_unique on post_view (id, user_id); -create index idx_post_view_user_id on post_view (user_id); -create index idx_post_view_hot_rank_published on post_view (hot_rank desc, published desc); -create index idx_post_view_published on post_view (published desc); -create index idx_post_view_score on post_view (score desc); - --- user_view -drop view user_view; -create materialized view user_view as -select id, -name, -avatar, -email, -fedi_name, -admin, -banned, -show_avatars, -send_notifications_to_email, -published, -(select count(*) from post p where p.creator_id = u.id) as number_of_posts, -(select coalesce(sum(score), 0) from post p, post_like pl where u.id = p.creator_id and p.id = pl.post_id) as post_score, -(select count(*) from comment c where c.creator_id = u.id) as number_of_comments, -(select coalesce(sum(score), 0) from comment c, comment_like cl where u.id = c.creator_id and c.id = cl.comment_id) as comment_score -from user_ u; - -create unique index idx_user_view_unique on user_view (id); -create index idx_user_view_comment_published on user_view (comment_score desc, published desc); -create index idx_user_view_admin on user_view (admin); -create index idx_user_view_banned on user_view (banned); - --- community -drop view community_view; -create materialized view community_view as -with all_community as -( - select *, - (select name from user_ u where c.creator_id = u.id) as creator_name, - (select avatar from user_ u where c.creator_id = u.id) as creator_avatar, - (select name from category ct where c.category_id = ct.id) as category_name, - (select count(*) from community_follower cf where cf.community_id = c.id) as number_of_subscribers, - (select count(*) from post p where p.community_id = c.id) as number_of_posts, - (select count(*) from comment co, post p where c.id = p.community_id and p.id = co.post_id) as number_of_comments, - hot_rank((select count(*) from community_follower cf where cf.community_id = c.id), c.published) as hot_rank - from community c -) - -select -ac.*, -u.id as user_id, -(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.id = cf.community_id) as subscribed -from user_ u -cross join all_community ac - -union all - -select -ac.*, -null as user_id, -null as subscribed -from all_community ac -; - -create unique index idx_community_view_unique on community_view (id, user_id); -create index idx_community_view_user_id on community_view (user_id); -create index idx_community_view_hot_rank_subscribed on community_view (hot_rank desc, number_of_subscribers desc); - - --- reply and comment view -drop view reply_view; -drop view user_mention_view; -drop view comment_view; -create materialized view comment_view as -with all_comment as -( - select - c.*, - (select community_id from post p where p.id = c.post_id), - (select u.banned from user_ u where c.creator_id = u.id) as banned, - (select cb.id::bool from community_user_ban cb, post p where c.creator_id = cb.user_id and p.id = c.post_id and p.community_id = cb.community_id) as banned_from_community, - (select name from user_ where c.creator_id = user_.id) as creator_name, - (select avatar from user_ where c.creator_id = user_.id) as creator_avatar, - coalesce(sum(cl.score), 0) as score, - count (case when cl.score = 1 then 1 else null end) as upvotes, - count (case when cl.score = -1 then 1 else null end) as downvotes - from comment c - left join comment_like cl on c.id = cl.comment_id - group by c.id -) - -select -ac.*, -u.id as user_id, -coalesce(cl.score, 0) as my_vote, -(select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved -from user_ u -cross join all_comment ac -left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id - -union all - -select - ac.*, - null as user_id, - null as my_vote, - null as saved -from all_comment ac -; - -create unique index idx_comment_view_unique on comment_view (id, user_id); -create index idx_comment_view_user_id on comment_view (user_id); -create index idx_comment_view_creator_id on comment_view (creator_id); -create index idx_comment_view_post_id on comment_view (post_id); -create index idx_comment_view_score on comment_view (score desc); - -create view reply_view as -with closereply as ( - select - c2.id, - c2.creator_id as sender_id, - c.creator_id as recipient_id - from comment c - inner join comment c2 on c.id = c2.parent_id - where c2.creator_id != c.creator_id - -- Do union where post is null - union - select - c.id, - c.creator_id as sender_id, - p.creator_id as recipient_id - from comment c, post p - where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id -) -select cv.*, -closereply.recipient_id -from comment_view cv, closereply -where closereply.id = cv.id -; - --- user mention -create view user_mention_view as -select - c.id, - um.id as user_mention_id, - c.creator_id, - c.post_id, - c.parent_id, - c.content, - c.removed, - um.read, - c.published, - c.updated, - c.deleted, - c.community_id, - c.banned, - c.banned_from_community, - c.creator_name, - c.creator_avatar, - c.score, - c.upvotes, - c.downvotes, - c.user_id, - c.my_vote, - c.saved, - um.recipient_id -from user_mention um, comment_view c -where um.comment_id = c.id; - --- user -create or replace function refresh_user() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently comment_view; -- cause of bans - refresh materialized view concurrently post_view; - return null; -end $$; - -create trigger refresh_user -after insert or update or delete or truncate -on user_ -for each statement -execute procedure refresh_user(); - --- post -create or replace function refresh_post() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently post_view; - return null; -end $$; - -create trigger refresh_post -after insert or update or delete or truncate -on post -for each statement -execute procedure refresh_post(); - --- post_like -create or replace function refresh_post_like() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently post_view; - return null; -end $$; - -create trigger refresh_post_like -after insert or update or delete or truncate -on post_like -for each statement -execute procedure refresh_post_like(); - --- community -create or replace function refresh_community() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently post_view; - refresh materialized view concurrently community_view; - return null; -end $$; - -create trigger refresh_community -after insert or update or delete or truncate -on community -for each statement -execute procedure refresh_community(); - --- community_follower -create or replace function refresh_community_follower() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently community_view; - refresh materialized view concurrently post_view; - return null; -end $$; - -create trigger refresh_community_follower -after insert or update or delete or truncate -on community_follower -for each statement -execute procedure refresh_community_follower(); - --- comment -create or replace function refresh_comment() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently post_view; - refresh materialized view concurrently comment_view; - return null; -end $$; - -create trigger refresh_comment -after insert or update or delete or truncate -on comment -for each statement -execute procedure refresh_comment(); - --- comment_like -create or replace function refresh_comment_like() -returns trigger language plpgsql -as $$ -begin - refresh materialized view concurrently comment_view; - return null; -end $$; - -create trigger refresh_comment_like -after insert or update or delete or truncate -on comment_like -for each statement -execute procedure refresh_comment_like(); From 543bd99900816eb82e88daaea891a418b8e97109 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 28 Jul 2023 17:16:18 +0200 Subject: [PATCH 26/27] Update CODEOWNERS (#3748) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 90ae73365..3d1bd7c72 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,3 @@ * @Nutomic @dessalines @phiresky +crates/apub/ @Nutomic +migrations/ @dessalines @phiresky From 963d04b3526f8a5e9ff762960bfb5215e353bb27 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 28 Jul 2023 22:44:45 +0200 Subject: [PATCH 27/27] Revert "Only run slow CI checks for actual code changes (#3750)" This reverts commit a34e0d477eb8bcc9acc5b3df44f843bf636b173a. --- .woodpecker.yml | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 6353da5cf..58ab2f0f9 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -48,8 +48,6 @@ pipeline: - "api_tests/node_modules" secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - when: - path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] toml_fmt: image: tamasfe/taplo:0.8.1 @@ -67,6 +65,8 @@ pipeline: - rustup toolchain install nightly-2023-07-10 - rustup component add rustfmt --toolchain nightly-2023-07-10 - cargo +nightly-2023-07-10 fmt -- --check + # when: + # platform: linux/amd64 # make sure api builds with default features (used by other crates relying on lemmy api) check_api_common_default_features: @@ -75,8 +75,8 @@ pipeline: CARGO_HOME: .cargo commands: - cargo check --package lemmy_api_common - when: - path: ["crates", "src", "Cargo.toml", "migrations"] + # when: + # platform: linux/amd64 lemmy_api_common_doesnt_depend_on_diesel: image: *muslrust_image @@ -84,8 +84,8 @@ pipeline: CARGO_HOME: .cargo commands: - "! cargo tree -p lemmy_api_common --no-default-features -i diesel" - when: - path: ["crates", "src", "Cargo.toml", "migrations"] + # when: + # platform: linux/amd64 lemmy_api_common_works_with_wasm: image: *muslrust_image @@ -94,8 +94,6 @@ pipeline: commands: - "rustup target add wasm32-unknown-unknown" - "cargo check --target wasm32-unknown-unknown -p lemmy_api_common" - when: - path: ["crates", "src", "Cargo.toml", "migrations"] check_defaults_hjson_updated: image: *muslrust_image @@ -105,8 +103,8 @@ pipeline: - export LEMMY_CONFIG_LOCATION=./config/config.hjson - ./scripts/update_config_defaults.sh config/defaults_current.hjson - diff config/defaults.hjson config/defaults_current.hjson - when: - path: ["crates", "src", "Cargo.toml", "migrations"] + # when: + # platform: linux/amd64 check_diesel_schema: image: willsquire/diesel-cli @@ -117,8 +115,6 @@ pipeline: - diesel migration run - diesel print-schema --config-file=diesel.toml > tmp.schema - diff tmp.schema crates/db_schema/src/schema.rs - when: - path: ["crates", "src", "Cargo.toml", "migrations"] check_diesel_migration_revertable: image: willsquire/diesel-cli @@ -128,8 +124,6 @@ pipeline: commands: - diesel migration run - diesel migration redo - when: - path: ["crates", "src", "Cargo.toml", "migrations"] cargo_clippy: image: *muslrust_image @@ -153,8 +147,8 @@ pipeline: -D clippy::needless_collect -D clippy::unwrap_used -D clippy::indexing_slicing - when: - path: ["crates", "src", "Cargo.toml", "migrations"] + # when: + # platform: linux/amd64 cargo_test: image: *muslrust_image @@ -165,8 +159,8 @@ pipeline: commands: - export LEMMY_CONFIG_LOCATION=../../config/config.hjson - cargo test --workspace --no-fail-fast - when: - path: ["crates", "src", "Cargo.toml", "migrations"] + # when: + # platform: linux/amd64 cargo_build: image: *muslrust_image @@ -175,8 +169,8 @@ pipeline: commands: - cargo build - mv target/x86_64-unknown-linux-musl/debug/lemmy_server target/lemmy_server - when: - path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] + # when: + # platform: linux/amd64 run_federation_tests: image: node:alpine @@ -189,8 +183,8 @@ pipeline: - cd api_tests/ - yarn - yarn api-test - when: - path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] + # when: + # platform: linux/amd64 rebuild-cache: image: meltwater/drone-cache:v1 @@ -214,8 +208,6 @@ pipeline: - "api_tests/node_modules" secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - when: - path: ["crates", "src", "Cargo.toml", "migrations", "api_tests"] publish_release_docker: image: woodpeckerci/plugin-docker-buildx @@ -265,3 +257,5 @@ services: environment: POSTGRES_USER: lemmy POSTGRES_PASSWORD: password + # when: + # platform: linux/amd64