From 7d3894d5dd86b1b08cbe74904a852e4551110934 Mon Sep 17 00:00:00 2001 From: dullbananas Date: Fri, 30 Jun 2023 00:50:30 -0700 Subject: [PATCH 1/4] Automatically manage database when running scripts/test.sh (#3389) * Update .gitignore * Create start-dev-db.sh * Rename start-dev-db.sh to start_dev_db.sh * Update .gitignore * Update start_dev_db.sh * Update start_dev_db.sh * Update start_dev_db.sh * Update start_dev_db.sh * h * Update test.sh * Update start_dev_db.sh * made it work * Make test.sh work when run from scripts dir --- .gitignore | 4 ++++ scripts/start_dev_db.sh | 24 ++++++++++++++++++++++++ scripts/test.sh | 11 ++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 scripts/start_dev_db.sh diff --git a/.gitignore b/.gitignore index 0461784f0..e48248836 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,7 @@ pictrs/ # The generated typescript bindings bindings + +# Database cluster and sockets for testing +dev_pgdata/ +*.PGSQL.* diff --git a/scripts/start_dev_db.sh b/scripts/start_dev_db.sh new file mode 100644 index 000000000..f192defa6 --- /dev/null +++ b/scripts/start_dev_db.sh @@ -0,0 +1,24 @@ +# This script is meant to be run with `source` so it can set environment variables. + +export PGDATA="$PWD/dev_pgdata" +export PGHOST=$PWD +export LEMMY_DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD" + +# If cluster exists, stop the server and delete the cluster +if [ -d $PGDATA ] +then + # Prevent `stop` from failing if server already stopped + pg_ctl restart > /dev/null + pg_ctl stop + rm -rf $PGDATA +fi + +# Create cluster +initdb --username=postgres --auth=trust --no-instructions + +# Start server that only listens to socket in current directory +pg_ctl start --options="-c listen_addresses= -c unix_socket_directories=$PWD" > /dev/null + +# Setup database +psql -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres +psql -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres diff --git a/scripts/test.sh b/scripts/test.sh index f117238bd..2a5efb30d 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,13 +1,15 @@ #!/usr/bin/env bash set -e +CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +cd $CWD/../ + PACKAGE="$1" echo "$PACKAGE" -psql -U lemmy -d postgres -c "DROP DATABASE lemmy;" -psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;" +source scripts/start_dev_db.sh -export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy # tests are executed in working directory crates/api (or similar), # so to load the config we need to traverse to the repo root export LEMMY_CONFIG_LOCATION=../../config/config.hjson @@ -21,3 +23,6 @@ else fi # Add this to do printlns: -- --nocapture + +pg_ctl stop +rm -rf $PGDATA From fcc010b5dc7c0835e758dbd34fbe33b1c0ff6103 Mon Sep 17 00:00:00 2001 From: Dominic Mazzoni Date: Fri, 30 Jun 2023 03:36:38 -0700 Subject: [PATCH 2/4] Fix concatenation of audio captcha wav files (#3350) * Fix concatenation of audio captcha wav files * Log errors rather than crashing * Return Result from captcha_as_wav_base64 * Change to return LemmyError * Check for wav write error, format * Remove unused import * Rewrite to avoid clippy warnings --- crates/api/Cargo.toml | 1 + crates/api/src/lib.rs | 39 +++++++++++++++++++----- crates/api/src/local_user/get_captcha.rs | 2 +- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index ca792809b..be3065e4d 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -30,6 +30,7 @@ captcha = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } chrono = { workspace = true } +wav = "1.0.0" [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 615a8a314..7ac3cec72 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -3,6 +3,7 @@ use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; +use std::io::Cursor; mod comment; mod comment_report; @@ -22,18 +23,42 @@ pub trait Perform { } /// Converts the captcha to a base64 encoded wav audio file -pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { +pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> Result { let letters = captcha.as_wav(); - let mut concat_letters: Vec = Vec::new(); - + // Decode each wav file, concatenate the samples + let mut concat_samples: Vec = Vec::new(); + let mut any_header: Option = None; for letter in letters { - let bytes = letter.unwrap_or_default(); - concat_letters.extend(bytes); + let mut cursor = Cursor::new(letter.unwrap_or_default()); + let (header, samples) = wav::read(&mut cursor)?; + any_header = Some(header); + if let Some(samples16) = samples.as_sixteen() { + concat_samples.extend(samples16); + } else { + return Err(LemmyError::from_message("couldnt_create_audio_captcha")); + } } - // Convert to base64 - base64::encode(concat_letters) + // Encode the concatenated result as a wav file + let mut output_buffer = Cursor::new(vec![]); + let header = match any_header { + Some(header) => header, + None => return Err(LemmyError::from_message("couldnt_create_audio_captcha")), + }; + let wav_write_result = wav::write( + header, + &wav::BitDepth::Sixteen(concat_samples), + &mut output_buffer, + ); + if let Err(e) = wav_write_result { + return Err(LemmyError::from_error_message( + e, + "couldnt_create_audio_captcha", + )); + } + + Ok(base64::encode(output_buffer.into_inner())) } /// Check size of report and remove whitespace diff --git a/crates/api/src/local_user/get_captcha.rs b/crates/api/src/local_user/get_captcha.rs index 133044248..92653cfc6 100644 --- a/crates/api/src/local_user/get_captcha.rs +++ b/crates/api/src/local_user/get_captcha.rs @@ -33,7 +33,7 @@ impl Perform for GetCaptcha { let png = captcha.as_base64().expect("failed to generate captcha"); - let wav = captcha_as_wav_base64(&captcha); + let wav = captcha_as_wav_base64(&captcha)?; let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer }; // Stores the captcha item in the db From 810762762fedcf4f20c571c263603aa7753234df Mon Sep 17 00:00:00 2001 From: "Lemmus.org" <137361180+lemmus-org@users.noreply.github.com> Date: Fri, 30 Jun 2023 03:42:42 -0700 Subject: [PATCH 3/4] Update federated posts to not cache sensitive images if not allow by local site (#3253) * Update federated posts to not cache sensitive images if not allow by local site * Refactor thumbnail match to simplify logic --- crates/api_common/src/request.rs | 4 ++++ crates/api_common/src/utils.rs | 7 +++++++ crates/api_crud/src/post/create.rs | 2 +- crates/api_crud/src/post/update.rs | 2 +- crates/apub/src/objects/post.rs | 25 ++++++++++++++++++++----- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index 9f7f9db59..64563bc0d 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -193,6 +193,7 @@ pub async fn fetch_site_data( client: &ClientWithMiddleware, settings: &Settings, url: Option<&Url>, + include_image: bool, ) -> (Option, Option) { match &url { Some(url) => { @@ -200,6 +201,9 @@ pub async fn fetch_site_data( // Ignore errors, since it may be an image, or not have the data. // Warning, this may ignore SSL errors let metadata_option = fetch_site_metadata(client, url).await.ok(); + if !include_image { + return (metadata_option, None); + } let missing_pictrs_file = |r: PictrsResponse| r.files.first().expect("missing pictrs file").file.clone(); diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index e3e761c90..f400cc9a8 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -428,6 +428,13 @@ pub fn local_site_opt_to_slur_regex(local_site: &Option) -> Option) -> bool { + local_site + .as_ref() + .map(|site| site.enable_nsfw) + .unwrap_or(false) +} + pub fn send_application_approved_email( user: &LocalUserView, settings: &Settings, diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 8ff1b678a..0bbbabcb0 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -79,7 +79,7 @@ impl PerformCrud for CreatePost { // Fetch post links and pictrs cached image let (metadata_res, thumbnail_url) = - fetch_site_data(context.client(), context.settings(), data_url).await; + fetch_site_data(context.client(), context.settings(), data_url, true).await; let (embed_title, embed_description, embed_video_url) = metadata_res .map(|u| (u.title, u.description, u.embed_video_url)) .unwrap_or_default(); diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index a540f454f..253340834 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -69,7 +69,7 @@ impl PerformCrud for EditPost { // Fetch post links and Pictrs cached image let data_url = data.url.as_ref(); let (metadata_res, thumbnail_url) = - fetch_site_data(context.client(), context.settings(), data_url).await; + fetch_site_data(context.client(), context.settings(), data_url, true).await; let (embed_title, embed_description, embed_video_url) = metadata_res .map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url))) .unwrap_or_default(); diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 4ef9351ab..7878fcf12 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -25,7 +25,7 @@ use html2md::parse_html; use lemmy_api_common::{ context::LemmyContext, request::fetch_site_data, - utils::{is_mod_or_admin, local_site_opt_to_slur_regex}, + utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex}, }; use lemmy_db_schema::{ self, @@ -197,18 +197,33 @@ impl Object for ApubPost { } else { None }; + + let local_site = LocalSite::read(context.pool()).await.ok(); + let allow_sensitive = local_site_opt_to_sensitive(&local_site); + let page_is_sensitive = page.sensitive.unwrap_or(false); + let include_image = allow_sensitive || !page_is_sensitive; + // Only fetch metadata if the post has a url and was not seen previously. We dont want to // waste resources by fetching metadata for the same post multiple times. - let (metadata_res, thumbnail_url) = match &url { + // Additionally, only fetch image if content is not sensitive or is allowed on local site. + let (metadata_res, thumbnail) = match &url { Some(url) if old_post.is_err() => { - fetch_site_data(context.client(), context.settings(), Some(url)).await + fetch_site_data( + context.client(), + context.settings(), + Some(url), + include_image, + ) + .await } - _ => (None, page.image.map(|i| i.url.into())), + _ => (None, None), }; + // If no image was included with metadata, use post image instead when available. + let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into())); + let (embed_title, embed_description, embed_video_url) = metadata_res .map(|u| (u.title, u.description, u.embed_video_url)) .unwrap_or_default(); - let local_site = LocalSite::read(context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); let body_slurs_removed = From cb28af508d030683561bdc9dde2299cf818f2c6b Mon Sep 17 00:00:00 2001 From: phiresky Date: Fri, 30 Jun 2023 15:32:43 +0200 Subject: [PATCH 4/4] don't strip, log trace if requested (#3425) --- Cargo.lock | 19 +++++++++++++++++++ Cargo.toml | 7 +------ crates/utils/src/error.rs | 6 +++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fb6746e0..ba527f0ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,6 +399,9 @@ name = "anyhow" version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +dependencies = [ + "backtrace", +] [[package]] name = "argparse" @@ -2590,6 +2593,7 @@ dependencies = [ "tokio", "tracing", "uuid", + "wav", ] [[package]] @@ -4445,6 +4449,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "riff" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1" + [[package]] name = "ring" version = "0.16.20" @@ -6256,6 +6266,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wav" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609" +dependencies = [ + "riff", +] + [[package]] name = "web-sys" version = "0.3.60" diff --git a/Cargo.toml b/Cargo.toml index 6fbd91d1b..e7ab77090 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,14 +21,9 @@ repository.workspace = true doctest = false [profile.release] -strip = "symbols" debug = 0 lto = "thin" -[profile.dev] -strip = "symbols" -debug = 0 - [features] embed-pictrs = ["pict-rs"] console = ["console-subscriber", "opentelemetry", "opentelemetry-otlp", "tracing-opentelemetry", "reqwest-tracing/opentelemetry_0_16"] @@ -85,7 +80,7 @@ base64 = "0.13.1" uuid = { version = "1.3.4", features = ["serde", "v4"] } async-trait = "0.1.68" captcha = "0.0.9" -anyhow = "1.0.71" +anyhow = { version = "1.0.71", features = ["backtrace"] } # backtrace is on by default on nightly, but not stable rust diesel_ltree = "0.3.0" typed-builder = "0.10.0" serial_test = "0.9.0" diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs index 1aa3e1c62..c89e84dbb 100644 --- a/crates/utils/src/error.rs +++ b/crates/utils/src/error.rs @@ -90,7 +90,11 @@ impl Display for LemmyError { if let Some(message) = &self.message { write!(f, "{message}: ")?; } - writeln!(f, "{}", self.inner)?; + // print anyhow including trace + // https://docs.rs/anyhow/latest/anyhow/struct.Error.html#display-representations + // this will print the anyhow trace (only if it exists) + // and if RUST_BACKTRACE=1, also a full backtrace + writeln!(f, "{:?}", self.inner)?; fmt::Display::fmt(&self.context, f) } }