Merge remote-tracking branch 'upstream/main' into db-traits-refactor

This commit is contained in:
dull b 2023-06-30 19:17:13 +00:00
commit 6846210074
14 changed files with 128 additions and 25 deletions

4
.gitignore vendored
View File

@ -26,3 +26,7 @@ pictrs/
# The generated typescript bindings
bindings
# Database cluster and sockets for testing
dev_pgdata/
*.PGSQL.*

19
Cargo.lock generated
View File

@ -399,6 +399,9 @@ name = "anyhow"
version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
dependencies = [
"backtrace",
]
[[package]]
name = "argparse"
@ -2590,6 +2593,7 @@ dependencies = [
"tokio",
"tracing",
"uuid",
"wav",
]
[[package]]
@ -4445,6 +4449,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "riff"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"
[[package]]
name = "ring"
version = "0.16.20"
@ -6256,6 +6266,15 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wav"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609"
dependencies = [
"riff",
]
[[package]]
name = "web-sys"
version = "0.3.60"

View File

@ -21,14 +21,9 @@ repository.workspace = true
doctest = false
[profile.release]
strip = "symbols"
debug = 0
lto = "thin"
[profile.dev]
strip = "symbols"
debug = 0
[features]
embed-pictrs = ["pict-rs"]
console = ["console-subscriber", "opentelemetry", "opentelemetry-otlp", "tracing-opentelemetry", "reqwest-tracing/opentelemetry_0_16"]
@ -85,7 +80,7 @@ base64 = "0.13.1"
uuid = { version = "1.3.4", features = ["serde", "v4"] }
async-trait = "0.1.68"
captcha = "0.0.9"
anyhow = "1.0.71"
anyhow = { version = "1.0.71", features = ["backtrace"] } # backtrace is on by default on nightly, but not stable rust
diesel_ltree = "0.3.0"
typed-builder = "0.10.0"
serial_test = "0.9.0"

View File

@ -30,6 +30,7 @@ captcha = { workspace = true }
anyhow = { workspace = true }
tracing = { workspace = true }
chrono = { workspace = true }
wav = "1.0.0"
[dev-dependencies]
serial_test = { workspace = true }

View File

@ -3,6 +3,7 @@ use captcha::Captcha;
use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex};
use lemmy_db_schema::source::local_site::LocalSite;
use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs};
use std::io::Cursor;
mod comment;
mod comment_report;
@ -22,18 +23,42 @@ pub trait Perform {
}
/// Converts the captcha to a base64 encoded wav audio file
pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String {
pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> Result<String, LemmyError> {
let letters = captcha.as_wav();
let mut concat_letters: Vec<u8> = Vec::new();
// Decode each wav file, concatenate the samples
let mut concat_samples: Vec<i16> = Vec::new();
let mut any_header: Option<wav::Header> = None;
for letter in letters {
let bytes = letter.unwrap_or_default();
concat_letters.extend(bytes);
let mut cursor = Cursor::new(letter.unwrap_or_default());
let (header, samples) = wav::read(&mut cursor)?;
any_header = Some(header);
if let Some(samples16) = samples.as_sixteen() {
concat_samples.extend(samples16);
} else {
return Err(LemmyError::from_message("couldnt_create_audio_captcha"));
}
}
// Convert to base64
base64::encode(concat_letters)
// Encode the concatenated result as a wav file
let mut output_buffer = Cursor::new(vec![]);
let header = match any_header {
Some(header) => header,
None => return Err(LemmyError::from_message("couldnt_create_audio_captcha")),
};
let wav_write_result = wav::write(
header,
&wav::BitDepth::Sixteen(concat_samples),
&mut output_buffer,
);
if let Err(e) = wav_write_result {
return Err(LemmyError::from_error_message(
e,
"couldnt_create_audio_captcha",
));
}
Ok(base64::encode(output_buffer.into_inner()))
}
/// Check size of report and remove whitespace

View File

@ -33,7 +33,7 @@ impl Perform for GetCaptcha {
let png = captcha.as_base64().expect("failed to generate captcha");
let wav = captcha_as_wav_base64(&captcha);
let wav = captcha_as_wav_base64(&captcha)?;
let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer };
// Stores the captcha item in the db

View File

@ -193,6 +193,7 @@ pub async fn fetch_site_data(
client: &ClientWithMiddleware,
settings: &Settings,
url: Option<&Url>,
include_image: bool,
) -> (Option<SiteMetadata>, Option<DbUrl>) {
match &url {
Some(url) => {
@ -200,6 +201,9 @@ pub async fn fetch_site_data(
// Ignore errors, since it may be an image, or not have the data.
// Warning, this may ignore SSL errors
let metadata_option = fetch_site_metadata(client, url).await.ok();
if !include_image {
return (metadata_option, None);
}
let missing_pictrs_file =
|r: PictrsResponse| r.files.first().expect("missing pictrs file").file.clone();

View File

@ -432,6 +432,13 @@ pub fn local_site_opt_to_slur_regex(local_site: &Option<LocalSite>) -> Option<Re
.unwrap_or(None)
}
pub fn local_site_opt_to_sensitive(local_site: &Option<LocalSite>) -> bool {
local_site
.as_ref()
.map(|site| site.enable_nsfw)
.unwrap_or(false)
}
pub fn send_application_approved_email(
user: &LocalUserView,
settings: &Settings,

View File

@ -84,7 +84,7 @@ impl PerformCrud for CreatePost {
// Fetch post links and pictrs cached image
let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url).await;
fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default();

View File

@ -69,7 +69,7 @@ impl PerformCrud for EditPost {
// Fetch post links and Pictrs cached image
let data_url = data.url.as_ref();
let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url).await;
fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
.unwrap_or_default();

View File

@ -25,7 +25,7 @@ use html2md::parse_html;
use lemmy_api_common::{
context::LemmyContext,
request::fetch_site_data,
utils::{is_mod_or_admin, local_site_opt_to_slur_regex},
utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex},
};
use lemmy_db_schema::{
self,
@ -197,18 +197,33 @@ impl Object for ApubPost {
} else {
None
};
let local_site = LocalSite::read(&mut *context.conn().await?).await.ok();
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
let page_is_sensitive = page.sensitive.unwrap_or(false);
let include_image = allow_sensitive || !page_is_sensitive;
// Only fetch metadata if the post has a url and was not seen previously. We dont want to
// waste resources by fetching metadata for the same post multiple times.
let (metadata_res, thumbnail_url) = match &url {
// Additionally, only fetch image if content is not sensitive or is allowed on local site.
let (metadata_res, thumbnail) = match &url {
Some(url) if old_post.is_err() => {
fetch_site_data(context.client(), context.settings(), Some(url)).await
fetch_site_data(
context.client(),
context.settings(),
Some(url),
include_image,
)
.await
}
_ => (None, page.image.map(|i| i.url.into())),
_ => (None, None),
};
// If no image was included with metadata, use post image instead when available.
let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into()));
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default();
let local_site = LocalSite::read(&mut *context.conn().await?).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let body_slurs_removed =

View File

@ -90,7 +90,11 @@ impl Display for LemmyError {
if let Some(message) = &self.message {
write!(f, "{message}: ")?;
}
writeln!(f, "{}", self.inner)?;
// print anyhow including trace
// https://docs.rs/anyhow/latest/anyhow/struct.Error.html#display-representations
// this will print the anyhow trace (only if it exists)
// and if RUST_BACKTRACE=1, also a full backtrace
writeln!(f, "{:?}", self.inner)?;
fmt::Display::fmt(&self.context, f)
}
}

24
scripts/start_dev_db.sh Normal file
View File

@ -0,0 +1,24 @@
# This script is meant to be run with `source` so it can set environment variables.
export PGDATA="$PWD/dev_pgdata"
export PGHOST=$PWD
export LEMMY_DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD"
# If cluster exists, stop the server and delete the cluster
if [ -d $PGDATA ]
then
# Prevent `stop` from failing if server already stopped
pg_ctl restart > /dev/null
pg_ctl stop
rm -rf $PGDATA
fi
# Create cluster
initdb --username=postgres --auth=trust --no-instructions
# Start server that only listens to socket in current directory
pg_ctl start --options="-c listen_addresses= -c unix_socket_directories=$PWD" > /dev/null
# Setup database
psql -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres
psql -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres

View File

@ -1,13 +1,15 @@
#!/usr/bin/env bash
set -e
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
PACKAGE="$1"
echo "$PACKAGE"
psql -U lemmy -d postgres -c "DROP DATABASE lemmy;"
psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;"
source scripts/start_dev_db.sh
export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
# tests are executed in working directory crates/api (or similar),
# so to load the config we need to traverse to the repo root
export LEMMY_CONFIG_LOCATION=../../config/config.hjson
@ -21,3 +23,6 @@ else
fi
# Add this to do printlns: -- --nocapture
pg_ctl stop
rm -rf $PGDATA