From 4db65c191c8b9198913c12f192fb641bbb709c08 Mon Sep 17 00:00:00 2001 From: Vijay Ramesh Date: Mon, 19 Jun 2023 23:17:54 -0700 Subject: [PATCH 01/10] after 30 days post deletion, replace comment.content and post.body with 'Permanently Deleted' --- crates/db_schema/src/impls/comment.rs | 5 ++- crates/db_schema/src/impls/post.rs | 18 +++++---- crates/db_schema/src/utils.rs | 3 ++ src/scheduled_tasks.rs | 55 ++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 46045cd10..8aa8c1793 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -11,7 +11,7 @@ use crate::{ CommentUpdateForm, }, traits::{Crud, Likeable, Saveable}, - utils::{get_conn, naive_now, DbPool}, + utils::{get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT}, }; use diesel::{ dsl::{insert_into, sql_query}, @@ -29,9 +29,10 @@ impl Comment { for_creator_id: PersonId, ) -> Result, Error> { let conn = &mut get_conn(pool).await?; + diesel::update(comment.filter(creator_id.eq(for_creator_id))) .set(( - content.eq("*Permananently Deleted*"), + content.eq(DELETED_REPLACEMENT_TEXT), deleted.eq(true), updated.eq(naive_now()), )) diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index a3309428c..7f59d29ec 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -27,7 +27,14 @@ use crate::{ PostUpdateForm, }, traits::{Crud, Likeable, Readable, Saveable}, - utils::{get_conn, naive_now, DbPool, FETCH_LIMIT_MAX}, + utils::{ + get_conn, + naive_now, + DbPool, + DELETED_REPLACEMENT_TEXT, + DELETED_REPLACEMENT_URL, + FETCH_LIMIT_MAX, + }, }; use ::url::Url; use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl, TextExpressionMethods}; @@ -111,14 +118,11 @@ impl Post { ) -> Result, Error> { let conn = &mut get_conn(pool).await?; - let perma_deleted = "*Permananently Deleted*"; - let perma_deleted_url = "https://deleted.com"; - diesel::update(post.filter(creator_id.eq(for_creator_id))) .set(( - name.eq(perma_deleted), - url.eq(perma_deleted_url), - body.eq(perma_deleted), + name.eq(DELETED_REPLACEMENT_TEXT), + url.eq(DELETED_REPLACEMENT_URL), + body.eq(DELETED_REPLACEMENT_TEXT), deleted.eq(true), updated.eq(naive_now()), )) diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 98d3952ab..9954f623b 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -222,6 +222,9 @@ pub mod functions { sql_function!(fn lower(x: Text) -> Text); } +pub const DELETED_REPLACEMENT_TEXT: &str = "*Permanently Deleted*"; +pub const DELETED_REPLACEMENT_URL: &str = "https://join-lemmy.org/"; + impl ToSql for DbUrl { fn to_sql(&self, out: &mut Output) -> diesel::serialize::Result { >::to_sql(&self.0.to_string(), &mut out.reborrow()) diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 9fb1ba702..891dca365 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -3,6 +3,7 @@ use diesel::{ dsl::{now, IntervalDsl}, Connection, ExpressionMethods, + NullableExpressionMethods, QueryDsl, }; // Import week days and WeekDay @@ -11,15 +12,17 @@ use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::{ schema::{ activity, + comment, comment_aggregates, community_aggregates, community_person_ban, instance, person, + post, post_aggregates, }, source::instance::{Instance, InstanceForm}, - utils::{functions::hot_rank, naive_now}, + utils::{functions::hot_rank, naive_now, DELETED_REPLACEMENT_TEXT}, }; use lemmy_routes::nodeinfo::NodeInfo; use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT}; @@ -66,6 +69,13 @@ pub fn setup( context_1.settings_updated_channel().remove_older_than(hour); }); + // Overwrite deleted & removed posts and comments every day + let url = db_url.clone(); + scheduler.every(CTimeUnits::days(1)).run(move || { + let mut conn = PgConnection::establish(&url).expect("could not establish connection"); + overwrite_deleted_posts_and_comments(&mut conn); + }); + // Update the Instance Software scheduler.every(CTimeUnits::days(1)).run(move || { let mut conn = PgConnection::establish(&db_url).expect("could not establish connection"); @@ -86,6 +96,7 @@ fn startup_jobs(db_url: &str) { update_hot_ranks(&mut conn, false); update_banned_when_expired(&mut conn); clear_old_activities(&mut conn); + overwrite_deleted_posts_and_comments(&mut conn); } /// Update the hot_rank columns for the aggregates tables @@ -166,6 +177,48 @@ fn clear_old_activities(conn: &mut PgConnection) { } } +/// overwrite posts and comments 30d after deletion +fn overwrite_deleted_posts_and_comments(conn: &mut PgConnection) { + info!("Overwriting deleted posts..."); + match diesel::update( + post::table + .filter(post::deleted.eq(true)) + .filter(post::updated.lt(now.nullable() - 1.months())) + .filter(post::body.ne(DELETED_REPLACEMENT_TEXT)), + ) + .set(( + post::body.eq(DELETED_REPLACEMENT_TEXT), + post::name.eq(DELETED_REPLACEMENT_TEXT), + )) + .execute(conn) + { + Ok(_) => { + info!("Done."); + } + Err(e) => { + error!("Failed to overwrite deleted posts: {}", e) + } + } + + info!("Overwriting deleted comments..."); + match diesel::update( + comment::table + .filter(comment::deleted.eq(true)) + .filter(comment::updated.lt(now.nullable() - 1.months())) + .filter(comment::content.ne(DELETED_REPLACEMENT_TEXT)), + ) + .set(comment::content.eq(DELETED_REPLACEMENT_TEXT)) + .execute(conn) + { + Ok(_) => { + info!("Done."); + } + Err(e) => { + error!("Failed to overwrite deleted comments: {}", e) + } + } +} + /// Re-calculate the site and community active counts every 12 hours fn active_counts(conn: &mut PgConnection) { info!("Updating active site and community aggregates ..."); From 95378785195c552a6618acb44d7ca31b084ee480 Mon Sep 17 00:00:00 2001 From: TKilFree Date: Fri, 23 Jun 2023 10:47:12 +0100 Subject: [PATCH 02/10] feat: re-added captcha checks (#3249) --- Cargo.lock | 2 + crates/api/Cargo.toml | 1 + crates/api/src/lib.rs | 16 ++ crates/api/src/local_user/get_captcha.rs | 53 ++++++ crates/api/src/local_user/mod.rs | 1 + crates/api_crud/Cargo.toml | 1 + crates/api_crud/src/user/create.rs | 18 ++ crates/db_schema/src/diesel_ltree.patch | 26 +-- crates/db_schema/src/impls/captcha_answer.rs | 164 ++++++++++++++++++ crates/db_schema/src/impls/mod.rs | 1 + crates/db_schema/src/schema.rs | 9 + crates/db_schema/src/source/captcha_answer.rs | 14 ++ crates/db_schema/src/source/mod.rs | 1 + .../2023-06-21-153242_add_captcha/down.sql | 1 + .../2023-06-21-153242_add_captcha/up.sql | 5 + src/api_routes_http.rs | 7 + 16 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 crates/api/src/local_user/get_captcha.rs create mode 100644 crates/db_schema/src/impls/captcha_answer.rs create mode 100644 crates/db_schema/src/source/captcha_answer.rs create mode 100644 migrations/2023-06-21-153242_add_captcha/down.sql create mode 100644 migrations/2023-06-21-153242_add_captcha/up.sql diff --git a/Cargo.lock b/Cargo.lock index cee02f798..08437ecf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,6 +2526,7 @@ dependencies = [ "base64 0.13.1", "bcrypt", "captcha", + "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", @@ -2576,6 +2577,7 @@ dependencies = [ "actix-web", "async-trait", "bcrypt", + "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 2488f2c2c..ca792809b 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -29,6 +29,7 @@ async-trait = { workspace = true } captcha = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } +chrono = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 9ff1677d0..615a8a314 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,4 +1,5 @@ use actix_web::web::Data; +use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; @@ -20,6 +21,21 @@ pub trait Perform { async fn perform(&self, context: &Data) -> Result; } +/// Converts the captcha to a base64 encoded wav audio file +pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { + let letters = captcha.as_wav(); + + let mut concat_letters: Vec = Vec::new(); + + for letter in letters { + let bytes = letter.unwrap_or_default(); + concat_letters.extend(bytes); + } + + // Convert to base64 + base64::encode(concat_letters) +} + /// Check size of report and remove whitespace pub(crate) fn check_report_reason(reason: &str, local_site: &LocalSite) -> Result<(), LemmyError> { let slur_regex = &local_site_to_slur_regex(local_site); diff --git a/crates/api/src/local_user/get_captcha.rs b/crates/api/src/local_user/get_captcha.rs new file mode 100644 index 000000000..6dbc34823 --- /dev/null +++ b/crates/api/src/local_user/get_captcha.rs @@ -0,0 +1,53 @@ +use crate::{captcha_as_wav_base64, Perform}; +use actix_web::web::Data; +use captcha::{gen, Difficulty}; +use chrono::Duration; +use lemmy_api_common::{ + context::LemmyContext, + person::{CaptchaResponse, GetCaptcha, GetCaptchaResponse}, +}; +use lemmy_db_schema::{ + source::{captcha_answer::CaptchaAnswer, local_site::LocalSite}, + utils::naive_now, +}; +use lemmy_utils::error::LemmyError; + +#[async_trait::async_trait(?Send)] +impl Perform for GetCaptcha { + type Response = GetCaptchaResponse; + + #[tracing::instrument(skip(context))] + async fn perform(&self, context: &Data) -> Result { + let local_site = LocalSite::read(context.pool()).await?; + + if !local_site.captcha_enabled { + return Ok(GetCaptchaResponse { ok: None }); + } + + let captcha = gen(match local_site.captcha_difficulty.as_str() { + "easy" => Difficulty::Easy, + "hard" => Difficulty::Hard, + _ => Difficulty::Medium, + }); + + let answer = captcha.chars_as_string(); + + let png = captcha.as_base64().expect("failed to generate captcha"); + + let uuid = uuid::Uuid::new_v4().to_string(); + + let wav = captcha_as_wav_base64(&captcha); + + let captcha: CaptchaAnswer = CaptchaAnswer { + answer, + uuid: uuid.clone(), + expires: naive_now() + Duration::minutes(10), // expires in 10 minutes + }; + // Stores the captcha item in the db + CaptchaAnswer::insert(context.pool(), &captcha).await?; + + Ok(GetCaptchaResponse { + ok: Some(CaptchaResponse { png, wav, uuid }), + }) + } +} diff --git a/crates/api/src/local_user/mod.rs b/crates/api/src/local_user/mod.rs index 9244f825d..3a92beda5 100644 --- a/crates/api/src/local_user/mod.rs +++ b/crates/api/src/local_user/mod.rs @@ -3,6 +3,7 @@ mod ban_person; mod block; mod change_password; mod change_password_after_reset; +mod get_captcha; mod list_banned; mod login; mod notifications; diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 1fb1e5a66..21320a3c3 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -22,3 +22,4 @@ tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } webmention = "0.4.0" +chrono = { worspace = true } diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index f5a26f756..871a05d6f 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -1,6 +1,7 @@ use crate::PerformCrud; use activitypub_federation::http_signatures::generate_actor_keypair; use actix_web::web::Data; +use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, person::{LoginResponse, Register}, @@ -19,6 +20,7 @@ use lemmy_api_common::{ use lemmy_db_schema::{ aggregates::structs::PersonAggregates, source::{ + captcha_answer::CaptchaAnswer, local_user::{LocalUser, LocalUserInsertForm}, person::{Person, PersonInsertForm}, registration_application::{RegistrationApplication, RegistrationApplicationInsertForm}, @@ -71,6 +73,22 @@ impl PerformCrud for Register { return Err(LemmyError::from_message("passwords_dont_match")); } + if local_site.site_setup && local_site.captcha_enabled { + let check = CaptchaAnswer::check_captcha( + context.pool(), + CaptchaAnswer { + uuid: data.captcha_uuid.clone().unwrap_or_default(), + answer: data.captcha_answer.clone().unwrap_or_default(), + // not used when checking + expires: NaiveDateTime::MIN, + }, + ) + .await?; + if !check { + return Err(LemmyError::from_message("captcha_incorrect")); + } + } + let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.username, &slur_regex)?; check_slurs_opt(&data.answer, &slur_regex)?; diff --git a/crates/db_schema/src/diesel_ltree.patch b/crates/db_schema/src/diesel_ltree.patch index d7d49f03e..2607eb68b 100644 --- a/crates/db_schema/src/diesel_ltree.patch +++ b/crates/db_schema/src/diesel_ltree.patch @@ -1,28 +1,17 @@ -diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs -index 255c6422..f2ccf5e2 100644 ---- a/crates/db_schema/src/schema.rs -+++ b/crates/db_schema/src/schema.rs -@@ -2,16 +2,12 @@ - - pub mod sql_types { - #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "listing_type_enum"))] +--- schema.rs 2023-06-21 22:25:50.252384233 +0100 ++++ "schema copy.rs" 2023-06-21 22:26:50.452378651 +0100 +@@ -6,10 +6,6 @@ pub struct ListingTypeEnum; -- #[derive(diesel::sql_types::SqlType)] + #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "ltree"))] - pub struct Ltree; - - #[derive(diesel::sql_types::SqlType)] +- #[derive(diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "registration_mode_enum"))] pub struct RegistrationModeEnum; - #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "sort_type_enum"))] -@@ -67,13 +63,13 @@ diesel::table! { - when_ -> Timestamp, - } - } +@@ -78,7 +74,7 @@ diesel::table! { use diesel::sql_types::*; @@ -31,6 +20,3 @@ index 255c6422..f2ccf5e2 100644 comment (id) { id -> Int4, - creator_id -> Int4, - post_id -> Int4, - content -> Text, diff --git a/crates/db_schema/src/impls/captcha_answer.rs b/crates/db_schema/src/impls/captcha_answer.rs new file mode 100644 index 000000000..afd18181c --- /dev/null +++ b/crates/db_schema/src/impls/captcha_answer.rs @@ -0,0 +1,164 @@ +use crate::{ + schema::captcha_answer, + source::captcha_answer::CaptchaAnswer, + utils::{functions::lower, get_conn, naive_now, DbPool}, +}; +use diesel::{ + delete, + dsl::exists, + insert_into, + result::Error, + select, + ExpressionMethods, + QueryDsl, +}; +use diesel_async::RunQueryDsl; + +impl CaptchaAnswer { + pub async fn insert(pool: &DbPool, captcha: &CaptchaAnswer) -> Result { + let conn = &mut get_conn(pool).await?; + + insert_into(captcha_answer::table) + .values(captcha) + .get_result::(conn) + .await + } + + pub async fn check_captcha(pool: &DbPool, to_check: CaptchaAnswer) -> Result { + let conn = &mut get_conn(pool).await?; + + // delete any expired captchas + delete(captcha_answer::table.filter(captcha_answer::expires.lt(&naive_now()))) + .execute(conn) + .await?; + + // fetch requested captcha + let captcha_exists = select(exists( + captcha_answer::dsl::captcha_answer + .filter((captcha_answer::dsl::uuid).eq(to_check.uuid.clone())) + .filter(lower(captcha_answer::dsl::answer).eq(to_check.answer.to_lowercase().clone())), + )) + .get_result::(conn) + .await?; + + // delete checked captcha + delete(captcha_answer::table.filter(captcha_answer::uuid.eq(to_check.uuid.clone()))) + .execute(conn) + .await?; + + Ok(captcha_exists) + } +} + +#[cfg(test)] +mod tests { + use crate::{ + source::captcha_answer::CaptchaAnswer, + utils::{build_db_pool_for_tests, naive_now}, + }; + use chrono::Duration; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_captcha_happy_path() { + let pool = &build_db_pool_for_tests().await; + + let captcha_a_id = "a".to_string(); + + let _ = CaptchaAnswer::insert( + pool, + &CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "XYZ".to_string(), + expires: naive_now() + Duration::minutes(10), + }, + ) + .await; + + let result = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + assert!(result.is_ok()); + assert!(result.unwrap()); + } + + #[tokio::test] + #[serial] + async fn test_captcha_repeat_answer_fails() { + let pool = &build_db_pool_for_tests().await; + + let captcha_a_id = "a".to_string(); + + let _ = CaptchaAnswer::insert( + pool, + &CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "XYZ".to_string(), + expires: naive_now() + Duration::minutes(10), + }, + ) + .await; + + let result = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + let result_repeat = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + assert!(result_repeat.is_ok()); + assert!(!result_repeat.unwrap()); + } + + #[tokio::test] + #[serial] + async fn test_captcha_expired_fails() { + let pool = &build_db_pool_for_tests().await; + + let expired_id = "already_expired".to_string(); + + let _ = CaptchaAnswer::insert( + pool, + &CaptchaAnswer { + uuid: expired_id.clone(), + answer: "xyz".to_string(), + expires: naive_now() - Duration::seconds(1), + }, + ) + .await; + + let expired_result = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: expired_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + assert!(expired_result.is_ok()); + assert!(!expired_result.unwrap()); + } +} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index 915d1c8e2..f13004d01 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -1,5 +1,6 @@ pub mod activity; pub mod actor_language; +pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index ac4ddc47a..f244ae664 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -64,6 +64,14 @@ diesel::table! { } } +diesel::table! { + captcha_answer (uuid) { + uuid -> Text, + answer -> Text, + expires -> Timestamp, + } +} + diesel::table! { use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; use diesel_ltree::sql_types::Ltree; @@ -916,6 +924,7 @@ diesel::allow_tables_to_appear_in_same_query!( admin_purge_community, admin_purge_person, admin_purge_post, + captcha_answer, comment, comment_aggregates, comment_like, diff --git a/crates/db_schema/src/source/captcha_answer.rs b/crates/db_schema/src/source/captcha_answer.rs new file mode 100644 index 000000000..113b7c96a --- /dev/null +++ b/crates/db_schema/src/source/captcha_answer.rs @@ -0,0 +1,14 @@ +#[cfg(feature = "full")] +use crate::schema::captcha_answer; +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; + +#[skip_serializing_none] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(Queryable, Insertable, AsChangeset))] +#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] +pub struct CaptchaAnswer { + pub uuid: String, + pub answer: String, + pub expires: chrono::NaiveDateTime, +} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index 9aab4b90b..926e23e73 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -1,6 +1,7 @@ #[cfg(feature = "full")] pub mod activity; pub mod actor_language; +pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/migrations/2023-06-21-153242_add_captcha/down.sql b/migrations/2023-06-21-153242_add_captcha/down.sql new file mode 100644 index 000000000..4e5b83042 --- /dev/null +++ b/migrations/2023-06-21-153242_add_captcha/down.sql @@ -0,0 +1 @@ +drop table captcha_answer; \ No newline at end of file diff --git a/migrations/2023-06-21-153242_add_captcha/up.sql b/migrations/2023-06-21-153242_add_captcha/up.sql new file mode 100644 index 000000000..71467be61 --- /dev/null +++ b/migrations/2023-06-21-153242_add_captcha/up.sql @@ -0,0 +1,5 @@ +create table captcha_answer ( + uuid text not null primary key, + answer text not null, + expires timestamp not null +); diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index a2abfa690..375630a92 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -38,6 +38,7 @@ use lemmy_api_common::{ ChangePassword, DeleteAccount, GetBannedPersons, + GetCaptcha, GetPersonDetails, GetPersonMentions, GetReplies, @@ -272,6 +273,12 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .wrap(rate_limit.register()) .route(web::post().to(route_post_crud::)), ) + .service( + // Handle captcha separately + web::resource("/user/get_captcha") + .wrap(rate_limit.post()) + .route(web::get().to(route_get::)), + ) // User actions .service( web::scope("/user") From aea5f6a38b9ae3e913e261dc4b978c2857292703 Mon Sep 17 00:00:00 2001 From: Kresten Laust Date: Fri, 23 Jun 2023 11:50:35 +0200 Subject: [PATCH 03/10] Reversed requirement question logic (#3283) * Reversed requirement question logic * Changed required state to 'true' --- .github/ISSUE_TEMPLATE/BUG_REPORT.yml | 4 ++-- .github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml index 206b4da85..a4028afd0 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml @@ -20,8 +20,8 @@ body: required: true - label: Is this only a single bug? Do not put multiple bugs in one issue. required: true - - label: Is this a UI / front end issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo. - required: false + - label: Is this a backend issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo for UI / frontend issues. + required: true - type: textarea id: summary attributes: diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml index 5f35bb030..40ef2caf3 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml @@ -18,8 +18,8 @@ body: required: true - label: Is this only a feature request? Do not put multiple feature requests in one issue. required: true - - label: Is this a UI / front end issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo. - required: false + - label: Is this a backend issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo for UI / frontend issues. + required: true - type: textarea id: problem attributes: From b6cd1bde8eae74d22c0c3c80ee7990b413d0ab27 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 06:53:46 -0400 Subject: [PATCH 04/10] Fixing removed posts showing. Fixes #2875 (#3279) * Fixing removed posts showing. Fixes #2875 * Fixing clippy. --- crates/db_views/src/post_view.rs | 58 +++++++++++++++++++-- crates/db_views_actor/src/community_view.rs | 6 +-- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index 79cc875cc..14cf7fe19 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -147,10 +147,12 @@ impl PostView { .into_boxed(); // Hide deleted and removed for non-admins or mods - if !is_mod_or_admin.unwrap_or(true) { + if !is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)); + .filter(community::deleted.eq(false)) + .filter(post::removed.eq(false)) + .filter(post::deleted.eq(false)); } let ( @@ -305,10 +307,12 @@ impl<'a> PostQuery<'a> { // Hide deleted and removed for non-admins or mods // TODO This eventually needs to show posts where you are the creator - if !self.is_mod_or_admin.unwrap_or(true) { + if !self.is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)); + .filter(community::deleted.eq(false)) + .filter(post::removed.eq(false)) + .filter(post::deleted.eq(false)); } if self.community_id.is_none() { @@ -475,7 +479,7 @@ mod tests { local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, person::{Person, PersonInsertForm}, person_block::{PersonBlock, PersonBlockForm}, - post::{Post, PostInsertForm, PostLike, PostLikeForm}, + post::{Post, PostInsertForm, PostLike, PostLikeForm, PostUpdateForm}, }, traits::{Blockable, Crud, Likeable}, utils::{build_db_pool_for_tests, DbPool}, @@ -870,6 +874,50 @@ mod tests { cleanup(data, pool).await; } + #[tokio::test] + #[serial] + async fn post_listings_deleted() { + let pool = &build_db_pool_for_tests().await; + let data = init_data(pool).await; + + // Delete the post + Post::update( + pool, + data.inserted_post.id, + &PostUpdateForm::builder().deleted(Some(true)).build(), + ) + .await + .unwrap(); + + // Make sure you don't see the deleted post in the results + let post_listings_no_admin = PostQuery::builder() + .pool(pool) + .sort(Some(SortType::New)) + .local_user(Some(&data.inserted_local_user)) + .is_mod_or_admin(Some(false)) + .build() + .list() + .await + .unwrap(); + + assert_eq!(1, post_listings_no_admin.len()); + + // Make sure they see both + let post_listings_is_admin = PostQuery::builder() + .pool(pool) + .sort(Some(SortType::New)) + .local_user(Some(&data.inserted_local_user)) + .is_mod_or_admin(Some(true)) + .build() + .list() + .await + .unwrap(); + + assert_eq!(2, post_listings_is_admin.len()); + + cleanup(data, pool).await; + } + async fn cleanup(data: Data, pool: &DbPool) { let num_deleted = Post::delete(pool, data.inserted_post.id).await.unwrap(); Community::delete(pool, data.inserted_community.id) diff --git a/crates/db_views_actor/src/community_view.rs b/crates/db_views_actor/src/community_view.rs index ee6066c22..c4f5920b7 100644 --- a/crates/db_views_actor/src/community_view.rs +++ b/crates/db_views_actor/src/community_view.rs @@ -69,7 +69,7 @@ impl CommunityView { .into_boxed(); // Hide deleted and removed for non-admins or mods - if !is_mod_or_admin.unwrap_or(true) { + if !is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) .filter(community::deleted.eq(false)); @@ -170,7 +170,7 @@ impl<'a> CommunityQuery<'a> { }; // Hide deleted and removed for non-admins or mods - if !self.is_mod_or_admin.unwrap_or(true) { + if !self.is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) .filter(community::deleted.eq(false)) @@ -213,8 +213,6 @@ impl<'a> CommunityQuery<'a> { let res = query .limit(limit) .offset(offset) - .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)) .load::(conn) .await?; From 37a47de3a871830ad002ec1f5c4da672ca430518 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 07:02:05 -0400 Subject: [PATCH 05/10] Revert "feat: re-added captcha checks (#3249)" (#3288) This reverts commit 8a086c82405bc8e2c8cb2fbbcceb10418f231d1b. --- Cargo.lock | 2 - crates/api/Cargo.toml | 1 - crates/api/src/lib.rs | 16 -- crates/api/src/local_user/get_captcha.rs | 53 ------ crates/api/src/local_user/mod.rs | 1 - crates/api_crud/Cargo.toml | 1 - crates/api_crud/src/user/create.rs | 18 -- crates/db_schema/src/diesel_ltree.patch | 26 ++- crates/db_schema/src/impls/captcha_answer.rs | 164 ------------------ crates/db_schema/src/impls/mod.rs | 1 - crates/db_schema/src/schema.rs | 9 - crates/db_schema/src/source/captcha_answer.rs | 14 -- crates/db_schema/src/source/mod.rs | 1 - .../2023-06-21-153242_add_captcha/down.sql | 1 - .../2023-06-21-153242_add_captcha/up.sql | 5 - src/api_routes_http.rs | 7 - 16 files changed, 20 insertions(+), 300 deletions(-) delete mode 100644 crates/api/src/local_user/get_captcha.rs delete mode 100644 crates/db_schema/src/impls/captcha_answer.rs delete mode 100644 crates/db_schema/src/source/captcha_answer.rs delete mode 100644 migrations/2023-06-21-153242_add_captcha/down.sql delete mode 100644 migrations/2023-06-21-153242_add_captcha/up.sql diff --git a/Cargo.lock b/Cargo.lock index 08437ecf7..cee02f798 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,7 +2526,6 @@ dependencies = [ "base64 0.13.1", "bcrypt", "captcha", - "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", @@ -2577,7 +2576,6 @@ dependencies = [ "actix-web", "async-trait", "bcrypt", - "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index ca792809b..2488f2c2c 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -29,7 +29,6 @@ async-trait = { workspace = true } captcha = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } -chrono = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 615a8a314..9ff1677d0 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,5 +1,4 @@ use actix_web::web::Data; -use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; @@ -21,21 +20,6 @@ pub trait Perform { async fn perform(&self, context: &Data) -> Result; } -/// Converts the captcha to a base64 encoded wav audio file -pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { - let letters = captcha.as_wav(); - - let mut concat_letters: Vec = Vec::new(); - - for letter in letters { - let bytes = letter.unwrap_or_default(); - concat_letters.extend(bytes); - } - - // Convert to base64 - base64::encode(concat_letters) -} - /// Check size of report and remove whitespace pub(crate) fn check_report_reason(reason: &str, local_site: &LocalSite) -> Result<(), LemmyError> { let slur_regex = &local_site_to_slur_regex(local_site); diff --git a/crates/api/src/local_user/get_captcha.rs b/crates/api/src/local_user/get_captcha.rs deleted file mode 100644 index 6dbc34823..000000000 --- a/crates/api/src/local_user/get_captcha.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::{captcha_as_wav_base64, Perform}; -use actix_web::web::Data; -use captcha::{gen, Difficulty}; -use chrono::Duration; -use lemmy_api_common::{ - context::LemmyContext, - person::{CaptchaResponse, GetCaptcha, GetCaptchaResponse}, -}; -use lemmy_db_schema::{ - source::{captcha_answer::CaptchaAnswer, local_site::LocalSite}, - utils::naive_now, -}; -use lemmy_utils::error::LemmyError; - -#[async_trait::async_trait(?Send)] -impl Perform for GetCaptcha { - type Response = GetCaptchaResponse; - - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let local_site = LocalSite::read(context.pool()).await?; - - if !local_site.captcha_enabled { - return Ok(GetCaptchaResponse { ok: None }); - } - - let captcha = gen(match local_site.captcha_difficulty.as_str() { - "easy" => Difficulty::Easy, - "hard" => Difficulty::Hard, - _ => Difficulty::Medium, - }); - - let answer = captcha.chars_as_string(); - - let png = captcha.as_base64().expect("failed to generate captcha"); - - let uuid = uuid::Uuid::new_v4().to_string(); - - let wav = captcha_as_wav_base64(&captcha); - - let captcha: CaptchaAnswer = CaptchaAnswer { - answer, - uuid: uuid.clone(), - expires: naive_now() + Duration::minutes(10), // expires in 10 minutes - }; - // Stores the captcha item in the db - CaptchaAnswer::insert(context.pool(), &captcha).await?; - - Ok(GetCaptchaResponse { - ok: Some(CaptchaResponse { png, wav, uuid }), - }) - } -} diff --git a/crates/api/src/local_user/mod.rs b/crates/api/src/local_user/mod.rs index 3a92beda5..9244f825d 100644 --- a/crates/api/src/local_user/mod.rs +++ b/crates/api/src/local_user/mod.rs @@ -3,7 +3,6 @@ mod ban_person; mod block; mod change_password; mod change_password_after_reset; -mod get_captcha; mod list_banned; mod login; mod notifications; diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 21320a3c3..1fb1e5a66 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -22,4 +22,3 @@ tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } webmention = "0.4.0" -chrono = { worspace = true } diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index 871a05d6f..f5a26f756 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -1,7 +1,6 @@ use crate::PerformCrud; use activitypub_federation::http_signatures::generate_actor_keypair; use actix_web::web::Data; -use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, person::{LoginResponse, Register}, @@ -20,7 +19,6 @@ use lemmy_api_common::{ use lemmy_db_schema::{ aggregates::structs::PersonAggregates, source::{ - captcha_answer::CaptchaAnswer, local_user::{LocalUser, LocalUserInsertForm}, person::{Person, PersonInsertForm}, registration_application::{RegistrationApplication, RegistrationApplicationInsertForm}, @@ -73,22 +71,6 @@ impl PerformCrud for Register { return Err(LemmyError::from_message("passwords_dont_match")); } - if local_site.site_setup && local_site.captcha_enabled { - let check = CaptchaAnswer::check_captcha( - context.pool(), - CaptchaAnswer { - uuid: data.captcha_uuid.clone().unwrap_or_default(), - answer: data.captcha_answer.clone().unwrap_or_default(), - // not used when checking - expires: NaiveDateTime::MIN, - }, - ) - .await?; - if !check { - return Err(LemmyError::from_message("captcha_incorrect")); - } - } - let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.username, &slur_regex)?; check_slurs_opt(&data.answer, &slur_regex)?; diff --git a/crates/db_schema/src/diesel_ltree.patch b/crates/db_schema/src/diesel_ltree.patch index 2607eb68b..d7d49f03e 100644 --- a/crates/db_schema/src/diesel_ltree.patch +++ b/crates/db_schema/src/diesel_ltree.patch @@ -1,17 +1,28 @@ ---- schema.rs 2023-06-21 22:25:50.252384233 +0100 -+++ "schema copy.rs" 2023-06-21 22:26:50.452378651 +0100 -@@ -6,10 +6,6 @@ +diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs +index 255c6422..f2ccf5e2 100644 +--- a/crates/db_schema/src/schema.rs ++++ b/crates/db_schema/src/schema.rs +@@ -2,16 +2,12 @@ + + pub mod sql_types { + #[derive(diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "listing_type_enum"))] pub struct ListingTypeEnum; - #[derive(diesel::sql_types::SqlType)] +- #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "ltree"))] - pub struct Ltree; - -- #[derive(diesel::sql_types::SqlType)] + #[derive(diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "registration_mode_enum"))] pub struct RegistrationModeEnum; -@@ -78,7 +74,7 @@ + #[derive(diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "sort_type_enum"))] +@@ -67,13 +63,13 @@ diesel::table! { + when_ -> Timestamp, + } + } diesel::table! { use diesel::sql_types::*; @@ -20,3 +31,6 @@ comment (id) { id -> Int4, + creator_id -> Int4, + post_id -> Int4, + content -> Text, diff --git a/crates/db_schema/src/impls/captcha_answer.rs b/crates/db_schema/src/impls/captcha_answer.rs deleted file mode 100644 index afd18181c..000000000 --- a/crates/db_schema/src/impls/captcha_answer.rs +++ /dev/null @@ -1,164 +0,0 @@ -use crate::{ - schema::captcha_answer, - source::captcha_answer::CaptchaAnswer, - utils::{functions::lower, get_conn, naive_now, DbPool}, -}; -use diesel::{ - delete, - dsl::exists, - insert_into, - result::Error, - select, - ExpressionMethods, - QueryDsl, -}; -use diesel_async::RunQueryDsl; - -impl CaptchaAnswer { - pub async fn insert(pool: &DbPool, captcha: &CaptchaAnswer) -> Result { - let conn = &mut get_conn(pool).await?; - - insert_into(captcha_answer::table) - .values(captcha) - .get_result::(conn) - .await - } - - pub async fn check_captcha(pool: &DbPool, to_check: CaptchaAnswer) -> Result { - let conn = &mut get_conn(pool).await?; - - // delete any expired captchas - delete(captcha_answer::table.filter(captcha_answer::expires.lt(&naive_now()))) - .execute(conn) - .await?; - - // fetch requested captcha - let captcha_exists = select(exists( - captcha_answer::dsl::captcha_answer - .filter((captcha_answer::dsl::uuid).eq(to_check.uuid.clone())) - .filter(lower(captcha_answer::dsl::answer).eq(to_check.answer.to_lowercase().clone())), - )) - .get_result::(conn) - .await?; - - // delete checked captcha - delete(captcha_answer::table.filter(captcha_answer::uuid.eq(to_check.uuid.clone()))) - .execute(conn) - .await?; - - Ok(captcha_exists) - } -} - -#[cfg(test)] -mod tests { - use crate::{ - source::captcha_answer::CaptchaAnswer, - utils::{build_db_pool_for_tests, naive_now}, - }; - use chrono::Duration; - use serial_test::serial; - - #[tokio::test] - #[serial] - async fn test_captcha_happy_path() { - let pool = &build_db_pool_for_tests().await; - - let captcha_a_id = "a".to_string(); - - let _ = CaptchaAnswer::insert( - pool, - &CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "XYZ".to_string(), - expires: naive_now() + Duration::minutes(10), - }, - ) - .await; - - let result = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - assert!(result.is_ok()); - assert!(result.unwrap()); - } - - #[tokio::test] - #[serial] - async fn test_captcha_repeat_answer_fails() { - let pool = &build_db_pool_for_tests().await; - - let captcha_a_id = "a".to_string(); - - let _ = CaptchaAnswer::insert( - pool, - &CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "XYZ".to_string(), - expires: naive_now() + Duration::minutes(10), - }, - ) - .await; - - let result = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - let result_repeat = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - assert!(result_repeat.is_ok()); - assert!(!result_repeat.unwrap()); - } - - #[tokio::test] - #[serial] - async fn test_captcha_expired_fails() { - let pool = &build_db_pool_for_tests().await; - - let expired_id = "already_expired".to_string(); - - let _ = CaptchaAnswer::insert( - pool, - &CaptchaAnswer { - uuid: expired_id.clone(), - answer: "xyz".to_string(), - expires: naive_now() - Duration::seconds(1), - }, - ) - .await; - - let expired_result = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: expired_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - assert!(expired_result.is_ok()); - assert!(!expired_result.unwrap()); - } -} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index f13004d01..915d1c8e2 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -1,6 +1,5 @@ pub mod activity; pub mod actor_language; -pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index f244ae664..ac4ddc47a 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -64,14 +64,6 @@ diesel::table! { } } -diesel::table! { - captcha_answer (uuid) { - uuid -> Text, - answer -> Text, - expires -> Timestamp, - } -} - diesel::table! { use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; use diesel_ltree::sql_types::Ltree; @@ -924,7 +916,6 @@ diesel::allow_tables_to_appear_in_same_query!( admin_purge_community, admin_purge_person, admin_purge_post, - captcha_answer, comment, comment_aggregates, comment_like, diff --git a/crates/db_schema/src/source/captcha_answer.rs b/crates/db_schema/src/source/captcha_answer.rs deleted file mode 100644 index 113b7c96a..000000000 --- a/crates/db_schema/src/source/captcha_answer.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[cfg(feature = "full")] -use crate::schema::captcha_answer; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; - -#[skip_serializing_none] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[cfg_attr(feature = "full", derive(Queryable, Insertable, AsChangeset))] -#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] -pub struct CaptchaAnswer { - pub uuid: String, - pub answer: String, - pub expires: chrono::NaiveDateTime, -} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index 926e23e73..9aab4b90b 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -1,7 +1,6 @@ #[cfg(feature = "full")] pub mod activity; pub mod actor_language; -pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/migrations/2023-06-21-153242_add_captcha/down.sql b/migrations/2023-06-21-153242_add_captcha/down.sql deleted file mode 100644 index 4e5b83042..000000000 --- a/migrations/2023-06-21-153242_add_captcha/down.sql +++ /dev/null @@ -1 +0,0 @@ -drop table captcha_answer; \ No newline at end of file diff --git a/migrations/2023-06-21-153242_add_captcha/up.sql b/migrations/2023-06-21-153242_add_captcha/up.sql deleted file mode 100644 index 71467be61..000000000 --- a/migrations/2023-06-21-153242_add_captcha/up.sql +++ /dev/null @@ -1,5 +0,0 @@ -create table captcha_answer ( - uuid text not null primary key, - answer text not null, - expires timestamp not null -); diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index 375630a92..a2abfa690 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -38,7 +38,6 @@ use lemmy_api_common::{ ChangePassword, DeleteAccount, GetBannedPersons, - GetCaptcha, GetPersonDetails, GetPersonMentions, GetReplies, @@ -273,12 +272,6 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .wrap(rate_limit.register()) .route(web::post().to(route_post_crud::)), ) - .service( - // Handle captcha separately - web::resource("/user/get_captcha") - .wrap(rate_limit.post()) - .route(web::get().to(route_get::)), - ) // User actions .service( web::scope("/user") From 3f6e9a7f236a464e72b43a5316f3c877525f8a03 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 07:07:51 -0400 Subject: [PATCH 06/10] Version 0.18.0-rc.8 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cee02f798..e29f89982 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2518,7 +2518,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "actix-web", "anyhow", @@ -2541,7 +2541,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "actix-rt", "actix-web", @@ -2570,7 +2570,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-web", @@ -2589,7 +2589,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-rt", @@ -2627,7 +2627,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "async-trait", @@ -2659,7 +2659,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "diesel", "diesel-async", @@ -2676,7 +2676,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "diesel", "diesel-async", @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "diesel", "diesel-async", @@ -2701,7 +2701,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-web", @@ -2726,7 +2726,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-cors", @@ -2763,7 +2763,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index c05c1a57e..a86252576 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.0-rc.6", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.0-rc.6", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.0-rc.6", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.0-rc.6", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.0-rc.6", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.0-rc.6", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.0-rc.6", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.0-rc.6", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.0-rc.6", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.0-rc.6", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.0-rc.8", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.0-rc.8", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.0-rc.8", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.0-rc.8", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.0-rc.8", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.0-rc.8", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.0-rc.8", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.0-rc.8", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.0-rc.8", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.0-rc.8", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0" From e4d78b09747ecd3700f423bce9e6b8f5db347b24 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 08:41:20 -0400 Subject: [PATCH 07/10] Version 0.18.0 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- docker/docker-compose.yml | 4 ++-- docker/federation/docker-compose.yml | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e29f89982..9590e8f13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2518,7 +2518,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "actix-web", "anyhow", @@ -2541,7 +2541,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "actix-rt", "actix-web", @@ -2570,7 +2570,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-web", @@ -2589,7 +2589,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-rt", @@ -2627,7 +2627,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "async-trait", @@ -2659,7 +2659,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "diesel", "diesel-async", @@ -2676,7 +2676,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "diesel", "diesel-async", @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "diesel", "diesel-async", @@ -2701,7 +2701,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-web", @@ -2726,7 +2726,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-cors", @@ -2763,7 +2763,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index a86252576..430deb082 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.0-rc.8" +version = "0.18.0" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.0-rc.8", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.0-rc.8", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.0-rc.8", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.0-rc.8", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.0-rc.8", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.0-rc.8", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.0-rc.8", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.0-rc.8", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.0-rc.8", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.0-rc.8", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.0", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.0", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.0", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.0", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.0", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.0", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.0", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.0", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.0", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.0", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2846f547a..2033ee8c7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -35,7 +35,7 @@ services: logging: *default-logging lemmy: - # image: dessalines/lemmy:dev + # image: dessalines/lemmy:0.18.0 # use this to build your local lemmy server image for development # run docker compose up --build build: @@ -60,7 +60,7 @@ services: logging: *default-logging lemmy-ui: - image: dessalines/lemmy-ui:0.17.1 + image: dessalines/lemmy-ui:0.18.0 # use this to build your local lemmy ui image for development # run docker compose up --build # assuming lemmy-ui is cloned besides lemmy directory diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 569f264c4..b3464bacd 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" x-ui-default: &ui-default init: true - image: dessalines/lemmy-ui:0.17.3 + image: dessalines/lemmy-ui:0.18.0 # assuming lemmy-ui is cloned besides lemmy directory # build: # context: ../../../lemmy-ui From d6b580a530563d4a2be76d077e015f9aecc75479 Mon Sep 17 00:00:00 2001 From: cetra3 Date: Mon, 26 Jun 2023 17:54:11 +0930 Subject: [PATCH 08/10] Remove `actix_rt` & use standard tokio spawn (#3158) * Remove `actix_rt` & use standard tokio spawn * Adjust rust log back down * Format correctly * Update cargo lock * Add DB settings * Change name and update to latest rev * Clean up formatting changes * Move `worker_count` and `worker_retry_count` to settings * Update defaults * Use `0.4.4` instead of git branch --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- config/defaults.hjson | 4 ++++ crates/api_common/Cargo.toml | 2 +- crates/api_common/src/request.rs | 2 +- crates/api_common/src/site.rs | 3 --- crates/api_crud/src/site/create.rs | 1 - crates/api_crud/src/site/update.rs | 1 - crates/apub/Cargo.toml | 2 +- crates/apub/src/collections/community_moderators.rs | 2 +- crates/apub/src/objects/comment.rs | 6 +++--- crates/apub/src/objects/community.rs | 2 +- crates/apub/src/objects/instance.rs | 2 +- crates/apub/src/objects/person.rs | 4 ++-- crates/apub/src/objects/post.rs | 2 +- crates/apub/src/objects/private_message.rs | 4 ++-- crates/db_schema/src/schema.rs | 1 - crates/db_schema/src/source/local_site.rs | 4 ---- crates/utils/src/settings/structs.rs | 6 ++++++ .../down.sql | 1 + .../up.sql | 1 + src/lib.rs | 13 ++++++------- src/main.rs | 2 +- 23 files changed, 36 insertions(+), 35 deletions(-) create mode 100644 migrations/2023-06-19-055530_add_retry_worker_setting/down.sql create mode 100644 migrations/2023-06-19-055530_add_retry_worker_setting/up.sql diff --git a/Cargo.lock b/Cargo.lock index 9590e8f13..9d575f578 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2543,7 +2543,6 @@ dependencies = [ name = "lemmy_api_common" version = "0.18.0" dependencies = [ - "actix-rt", "actix-web", "anyhow", "chrono", @@ -2561,6 +2560,7 @@ dependencies = [ "rosetta-i18n", "serde", "serde_with", + "tokio", "tracing", "ts-rs", "url", @@ -2592,7 +2592,6 @@ name = "lemmy_apub" version = "0.18.0" dependencies = [ "activitypub_federation", - "actix-rt", "actix-web", "anyhow", "assert-json-diff", @@ -2620,6 +2619,7 @@ dependencies = [ "sha2", "strum_macros", "task-local-extensions", + "tokio", "tracing", "url", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 430deb082..07e41ab3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ anyhow = "1.0.71" diesel_ltree = "0.3.0" typed-builder = "0.10.0" serial_test = "0.9.0" -tokio = "1.28.2" +tokio = { version = "1.28.2", features = ["full"] } sha2 = "0.10.6" regex = "1.8.4" once_cell = "1.18.0" diff --git a/config/defaults.hjson b/config/defaults.hjson index 4c38ddd45..6032f8fc9 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -76,4 +76,8 @@ port: 8536 # Whether the site is available over TLS. Needs to be true for federation to work. tls_enabled: true + # The number of activitypub federation workers that can be in-flight concurrently + worker_count: 0 + # The number of activitypub federation retry workers that can be in-flight concurrently + retry_count: 0 } diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 46045d805..339d233a1 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -38,7 +38,7 @@ encoding = { version = "0.2.33", optional = true } anyhow = { workspace = true } futures = { workspace = true } uuid = { workspace = true } -actix-rt = { workspace = true } +tokio = { workspace = true } reqwest = { workspace = true } ts-rs = { workspace = true, optional = true } actix-web = { workspace = true } diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index c6f71b868..3139193a6 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -271,7 +271,7 @@ mod tests { use url::Url; // These helped with testing - #[actix_rt::test] + #[tokio::test] async fn test_site_metadata() { let settings = &SETTINGS.clone(); let client = reqwest::Client::builder() diff --git a/crates/api_common/src/site.rs b/crates/api_common/src/site.rs index 4d488ec1b..865acc0dc 100644 --- a/crates/api_common/src/site.rs +++ b/crates/api_common/src/site.rs @@ -177,7 +177,6 @@ pub struct CreateSite { pub rate_limit_search_per_second: Option, pub federation_enabled: Option, pub federation_debug: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub allowed_instances: Option>, @@ -250,8 +249,6 @@ pub struct EditSite { pub federation_enabled: Option, /// Enables federation debugging. pub federation_debug: Option, - /// The number of federation workers. - pub federation_worker_count: Option, /// Whether to enable captchas for signups. pub captcha_enabled: Option, /// The captcha difficulty. Can be easy, medium, or hard diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 2a51309a4..a1669baef 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -122,7 +122,6 @@ impl PerformCrud for CreateSite { .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) - .federation_worker_count(data.federation_worker_count) .captcha_enabled(data.captcha_enabled) .captcha_difficulty(data.captcha_difficulty.clone()) .build(); diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index fadde0a0b..6664d549a 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -123,7 +123,6 @@ impl PerformCrud for EditSite { .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) - .federation_worker_count(data.federation_worker_count) .captcha_enabled(data.captcha_enabled) .captcha_difficulty(data.captcha_difficulty.clone()) .reports_email_admins(data.reports_email_admins) diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index 2007b541a..8570541f7 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -25,7 +25,7 @@ chrono = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } actix-web = { workspace = true } -actix-rt = { workspace = true } +tokio = {workspace = true} tracing = { workspace = true } strum_macros = { workspace = true } url = { workspace = true } diff --git a/crates/apub/src/collections/community_moderators.rs b/crates/apub/src/collections/community_moderators.rs index c439da710..d53f86280 100644 --- a/crates/apub/src/collections/community_moderators.rs +++ b/crates/apub/src/collections/community_moderators.rs @@ -120,7 +120,7 @@ mod tests { }; use serial_test::serial; - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_community_moderators() { let context = init_context().await; diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index e2a03b8b3..16cb1542b 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -223,7 +223,7 @@ pub(crate) mod tests { LocalSite::delete(context.pool()).await.unwrap(); } - #[actix_rt::test] + #[tokio::test] #[serial] pub(crate) async fn test_parse_lemmy_comment() { let context = init_context().await; @@ -249,7 +249,7 @@ pub(crate) mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_comment() { let context = init_context().await; @@ -279,7 +279,7 @@ pub(crate) mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_html_to_markdown_sanitize() { let parsed = parse_html("hello"); diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 6526d2d26..888a7f458 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -242,7 +242,7 @@ pub(crate) mod tests { community } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_community() { let context = init_context().await; diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index 72d133441..6cd27fbbd 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -221,7 +221,7 @@ pub(crate) mod tests { site } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_instance() { let context = init_context().await; diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index c71d46ccf..3eeb733fd 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -223,7 +223,7 @@ pub(crate) mod tests { (person, site) } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_person() { let context = init_context().await; @@ -236,7 +236,7 @@ pub(crate) mod tests { cleanup((person, site), &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_person() { let context = init_context().await; diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index b255ffb9b..4ef9351ab 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -281,7 +281,7 @@ mod tests { use lemmy_db_schema::source::site::Site; use serial_test::serial; - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_post() { let context = init_context().await; diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 01f576ff8..ae2637c58 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -187,7 +187,7 @@ mod tests { Site::delete(context.pool(), data.2.id).await.unwrap(); } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_pm() { let context = init_context().await; @@ -213,7 +213,7 @@ mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_pm() { let context = init_context().await; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index ac4ddc47a..6714913f4 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -339,7 +339,6 @@ diesel::table! { slur_filter_regex -> Nullable, actor_name_max_length -> Int4, federation_enabled -> Bool, - federation_worker_count -> Int4, captcha_enabled -> Bool, #[max_length = 255] captcha_difficulty -> Varchar, diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index e65a61535..be93717a9 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -50,8 +50,6 @@ pub struct LocalSite { pub actor_name_max_length: i32, /// Whether federation is enabled. pub federation_enabled: bool, - /// The number of concurrent federation http workers. - pub federation_worker_count: i32, /// Whether captcha is enabled. pub captcha_enabled: bool, /// The captcha difficulty. @@ -85,7 +83,6 @@ pub struct LocalSiteInsertForm { pub slur_filter_regex: Option, pub actor_name_max_length: Option, pub federation_enabled: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub registration_mode: Option, @@ -112,7 +109,6 @@ pub struct LocalSiteUpdateForm { pub slur_filter_regex: Option>, pub actor_name_max_length: Option, pub federation_enabled: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub registration_mode: Option, diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 6e200b224..5d0e642f6 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -39,6 +39,12 @@ pub struct Settings { #[default(None)] #[doku(skip)] pub opentelemetry_url: Option, + /// The number of activitypub federation workers that can be in-flight concurrently + #[default(0)] + pub worker_count: usize, + /// The number of activitypub federation retry workers that can be in-flight concurrently + #[default(0)] + pub retry_count: usize, } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql new file mode 100644 index 000000000..e3c200a15 --- /dev/null +++ b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql @@ -0,0 +1 @@ +alter table local_site add column federation_worker_count int default 64 not null; \ No newline at end of file diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql new file mode 100644 index 000000000..2aac86f85 --- /dev/null +++ b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql @@ -0,0 +1 @@ +alter table local_site drop column federation_worker_count; \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 86cf400b6..d919acc05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,24 +139,23 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { }); } + let settings_bind = settings.clone(); + let federation_config = FederationConfig::builder() .domain(settings.hostname.clone()) .app_data(context.clone()) .client(client.clone()) .http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT) - .worker_count(local_site.federation_worker_count as usize) + .worker_count(settings.worker_count) + .retry_count(settings.retry_count) .debug(cfg!(debug_assertions)) .http_signature_compat(true) .url_verifier(Box::new(VerifyUrlData(context.pool().clone()))) .build() - .await - .expect("configure federation"); + .await?; // Create Http server with websocket support - let settings_bind = settings.clone(); HttpServer::new(move || { - let context = context.clone(); - let cors_config = if cfg!(debug_assertions) { Cors::permissive() } else { @@ -173,7 +172,7 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { )) .wrap(cors_config) .wrap(TracingLogger::::new()) - .app_data(Data::new(context)) + .app_data(Data::new(context.clone())) .app_data(Data::new(rate_limit_cell.clone())) .wrap(FederationMiddleware::new(federation_config.clone())) // The routes diff --git a/src/main.rs b/src/main.rs index 315fe84be..5fc03ed02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use lemmy_server::{init_logging, start_lemmy_server}; use lemmy_utils::{error::LemmyError, settings::SETTINGS}; -#[actix_web::main] +#[tokio::main] pub async fn main() -> Result<(), LemmyError> { init_logging(&SETTINGS.opentelemetry_url)?; #[cfg(not(feature = "embed-pictrs"))] From 6b28f8c616593be2c6905a39e98e1df2de956fe2 Mon Sep 17 00:00:00 2001 From: Sander Saarend Date: Mon, 26 Jun 2023 11:25:38 +0300 Subject: [PATCH 09/10] Add support for sslmode=require for diesel-async DB connections (#3189) --- Cargo.lock | 92 +++++++++++++++++++++++++++++------ Cargo.toml | 9 ++++ crates/db_schema/Cargo.toml | 6 ++- crates/db_schema/src/utils.rs | 64 ++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d575f578..99d94948f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,7 +215,7 @@ dependencies = [ "futures-util", "mio", "num_cpus", - "socket2", + "socket2 0.4.9", "tokio", "tracing", ] @@ -245,7 +245,7 @@ dependencies = [ "http", "log", "pin-project-lite", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util 0.7.4", "webpki-roots", ] @@ -297,7 +297,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.4.9", "time 0.3.15", "url", ] @@ -496,7 +496,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.8.5", - "rustls", + "rustls 0.20.7", "serde", "serde_json", "serde_urlencoded", @@ -2262,7 +2262,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -2640,9 +2640,11 @@ dependencies = [ "diesel-derive-newtype", "diesel_ltree", "diesel_migrations", + "futures-util", "lemmy_utils", "once_cell", "regex", + "rustls 0.21.2", "serde", "serde_json", "serde_with", @@ -2651,6 +2653,8 @@ dependencies = [ "strum", "strum_macros", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "ts-rs", "typed-builder", @@ -2736,6 +2740,7 @@ dependencies = [ "diesel", "diesel-async", "doku", + "futures-util", "lemmy_api", "lemmy_api_common", "lemmy_api_crud", @@ -2749,9 +2754,12 @@ dependencies = [ "reqwest", "reqwest-middleware", "reqwest-tracing", + "rustls 0.21.2", "serde", "serde_json", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "tracing-actix-web 0.6.2", "tracing-error", @@ -2820,7 +2828,7 @@ dependencies = [ "nom 7.1.1", "once_cell", "quoted_printable", - "socket2", + "socket2 0.4.9", ] [[package]] @@ -3932,11 +3940,11 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" dependencies = [ - "base64 0.13.1", + "base64 0.21.2", "byteorder", "bytes", "fallible-iterator", @@ -4495,6 +4503,28 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.9" @@ -4859,6 +4889,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -5304,7 +5344,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -5343,9 +5383,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a12c1b3e0704ae7dfc25562629798b29c72e6b1d0a681b6f29ab4ae5e7f7bf" +checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" dependencies = [ "async-trait", "byteorder", @@ -5360,22 +5400,46 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2", + "socket2 0.5.3", "tokio", "tokio-util 0.7.4", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" +dependencies = [ + "futures", + "ring", + "rustls 0.21.2", + "tokio", + "tokio-postgres", + "tokio-rustls 0.24.1", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.7", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.2", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index 07e41ab3b..2ee5a5308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,10 @@ rand = "0.8.5" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } tracing-opentelemetry = { version = "0.17.4" } ts-rs = { version = "6.2", features = ["serde-compat", "format", "chrono-impl"] } +rustls = { version ="0.21.2", features = ["dangerous_configuration"]} +futures-util = "0.3.28" +tokio-postgres = "0.7.8" +tokio-postgres-rustls = "0.10.0" [dependencies] lemmy_api = { workspace = true } @@ -140,3 +144,8 @@ opentelemetry-otlp = { version = "0.10.0", optional = true } pict-rs = { version = "0.4.0-rc.3", optional = true } tokio.workspace = true actix-cors = "0.6.4" +rustls = { workspace = true } +futures-util = { workspace = true } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } + diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index aa26382c0..6c89ab9fa 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -43,7 +43,11 @@ async-trait = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } deadpool = { version = "0.9.5", features = ["rt_tokio_1"], optional = true } -ts-rs = { workspace = true, optional = true } +ts-rs = { workspace = true, optional = true } +rustls = { workspace = true } +futures-util = { workspace = true } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 9954f623b..7ba5d5e7b 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -12,7 +12,7 @@ use diesel::{ backend::Backend, deserialize::FromSql, pg::Pg, - result::{Error as DieselError, Error::QueryBuilderError}, + result::{ConnectionError, ConnectionResult, Error as DieselError, Error::QueryBuilderError}, serialize::{Output, ToSql}, sql_types::Text, PgConnection, @@ -25,11 +25,21 @@ use diesel_async::{ }, }; use diesel_migrations::EmbeddedMigrations; +use futures_util::{future::BoxFuture, FutureExt}; use lemmy_utils::{error::LemmyError, settings::structs::Settings}; use once_cell::sync::Lazy; use regex::Regex; -use std::{env, env::VarError, time::Duration}; -use tracing::info; +use rustls::{ + client::{ServerCertVerified, ServerCertVerifier}, + ServerName, +}; +use std::{ + env, + env::VarError, + sync::Arc, + time::{Duration, SystemTime}, +}; +use tracing::{error, info}; use url::Url; const FETCH_LIMIT_DEFAULT: i64 = 10; @@ -136,7 +146,15 @@ pub fn diesel_option_overwrite_to_url_create( async fn build_db_pool_settings_opt(settings: Option<&Settings>) -> Result { let db_url = get_database_url(settings); let pool_size = settings.map(|s| s.database.pool_size).unwrap_or(5); - let manager = AsyncDieselConnectionManager::::new(&db_url); + // We only support TLS with sslmode=require currently + let tls_enabled = db_url.contains("sslmode=require"); + let manager = if tls_enabled { + // diesel-async does not support any TLS connections out of the box, so we need to manually + // provide a setup function which handles creating the connection + AsyncDieselConnectionManager::::new_with_setup(&db_url, establish_connection) + } else { + AsyncDieselConnectionManager::::new(&db_url) + }; let pool = Pool::builder(manager) .max_size(pool_size) .wait_timeout(POOL_TIMEOUT) @@ -153,6 +171,44 @@ async fn build_db_pool_settings_opt(settings: Option<&Settings>) -> Result BoxFuture> { + let fut = async { + let rustls_config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(Arc::new(NoCertVerifier {})) + .with_no_client_auth(); + + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config); + let (client, conn) = tokio_postgres::connect(config, tls) + .await + .map_err(|e| ConnectionError::BadConnection(e.to_string()))?; + tokio::spawn(async move { + if let Err(e) = conn.await { + error!("Database connection failed: {e}"); + } + }); + AsyncPgConnection::try_from(client).await + }; + fut.boxed() +} + +struct NoCertVerifier {} + +impl ServerCertVerifier for NoCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: SystemTime, + ) -> Result { + // Will verify all (even invalid) certs without any checks (sslmode=require) + Ok(ServerCertVerified::assertion()) + } +} + pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); pub fn run_migrations(db_url: &str) { From ce0cf0e41bf00d5b741f0cb4789c508df72edd79 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 26 Jun 2023 10:45:37 +0200 Subject: [PATCH 10/10] Remove DELETED_REPLACEMENT_URL --- crates/db_schema/src/impls/post.rs | 11 ++--------- crates/db_schema/src/utils.rs | 1 - 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 7f59d29ec..6a4d53d3a 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -27,14 +27,7 @@ use crate::{ PostUpdateForm, }, traits::{Crud, Likeable, Readable, Saveable}, - utils::{ - get_conn, - naive_now, - DbPool, - DELETED_REPLACEMENT_TEXT, - DELETED_REPLACEMENT_URL, - FETCH_LIMIT_MAX, - }, + utils::{get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT, FETCH_LIMIT_MAX}, }; use ::url::Url; use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl, TextExpressionMethods}; @@ -121,7 +114,7 @@ impl Post { diesel::update(post.filter(creator_id.eq(for_creator_id))) .set(( name.eq(DELETED_REPLACEMENT_TEXT), - url.eq(DELETED_REPLACEMENT_URL), + url.eq(Option::<&str>::None), body.eq(DELETED_REPLACEMENT_TEXT), deleted.eq(true), updated.eq(naive_now()), diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 7ba5d5e7b..fdd445637 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -279,7 +279,6 @@ pub mod functions { } pub const DELETED_REPLACEMENT_TEXT: &str = "*Permanently Deleted*"; -pub const DELETED_REPLACEMENT_URL: &str = "https://join-lemmy.org/"; impl ToSql for DbUrl { fn to_sql(&self, out: &mut Output) -> diesel::serialize::Result {