diff --git a/crates/apub/src/activities/receive/community.rs b/crates/apub/src/activities/receive/community.rs
index 854e75f26..cf85ad106 100644
--- a/crates/apub/src/activities/receive/community.rs
+++ b/crates/apub/src/activities/receive/community.rs
@@ -1,4 +1,7 @@
-use crate::{activities::receive::verify_activity_domains_valid, inbox::is_addressed_to_public};
+use crate::{
+  activities::receive::verify_activity_domains_valid,
+  inbox::verify_is_addressed_to_public,
+};
 use activitystreams::{
   activity::{ActorAndObjectRefExt, Delete, Remove, Undo},
   base::{AnyBase, ExtendsExt},
@@ -47,7 +50,7 @@ pub(crate) async fn receive_remove_community(
 ) -> Result<(), LemmyError> {
   let remove = Remove::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&remove, expected_domain, true)?;
-  is_addressed_to_public(&remove)?;
+  verify_is_addressed_to_public(&remove)?;
 
   let community_uri = remove
     .object()
@@ -89,11 +92,11 @@ pub(crate) async fn receive_undo_delete_community(
   community: Community,
   expected_domain: &Url,
 ) -> Result<(), LemmyError> {
-  is_addressed_to_public(&undo)?;
+  verify_is_addressed_to_public(&undo)?;
   let inner = undo.object().to_owned().one().context(location_info!())?;
   let delete = Delete::from_any_base(inner)?.context(location_info!())?;
   verify_activity_domains_valid(&delete, expected_domain, true)?;
-  is_addressed_to_public(&delete)?;
+  verify_is_addressed_to_public(&delete)?;
 
   let deleted_community = blocking(context.pool(), move |conn| {
     Community::update_deleted(conn, community.id, false)
@@ -124,12 +127,12 @@ pub(crate) async fn receive_undo_remove_community(
   undo: Undo,
   expected_domain: &Url,
 ) -> Result<(), LemmyError> {
-  is_addressed_to_public(&undo)?;
+  verify_is_addressed_to_public(&undo)?;
 
   let inner = undo.object().to_owned().one().context(location_info!())?;
   let remove = Remove::from_any_base(inner)?.context(location_info!())?;
   verify_activity_domains_valid(&remove, &expected_domain, true)?;
-  is_addressed_to_public(&remove)?;
+  verify_is_addressed_to_public(&remove)?;
 
   let community_uri = remove
     .object()
diff --git a/crates/apub/src/activities/send/community.rs b/crates/apub/src/activities/send/community.rs
index ffbb456c1..ff445ac60 100644
--- a/crates/apub/src/activities/send/community.rs
+++ b/crates/apub/src/activities/send/community.rs
@@ -219,7 +219,8 @@ pub async fn send_add_mod(
   add
     .set_many_contexts(lemmy_context()?)
     .set_id(generate_activity_id(AddType::Add)?)
-    .set_many_tos(vec![community.actor_id.to_owned().into_inner(), public()])
+    .set_to(public())
+    .set_many_ccs(vec![community.actor_id()])
     .set_target(generate_moderators_url(&community.actor_id)?.into_inner());
 
   if community.local {
@@ -245,7 +246,8 @@ pub async fn send_remove_mod(
   remove
     .set_many_contexts(lemmy_context()?)
     .set_id(generate_activity_id(RemoveType::Remove)?)
-    .set_many_tos(vec![community.actor_id.to_owned().into_inner(), public()])
+    .set_to(public())
+    .set_many_ccs(vec![community.actor_id()])
     .set_target(generate_moderators_url(&community.actor_id)?.into_inner());
 
   if community.local {
diff --git a/crates/apub/src/inbox/community_inbox.rs b/crates/apub/src/inbox/community_inbox.rs
index 43072c512..d357ee07c 100644
--- a/crates/apub/src/inbox/community_inbox.rs
+++ b/crates/apub/src/inbox/community_inbox.rs
@@ -6,7 +6,6 @@ use crate::{
     get_activity_to_and_cc,
     inbox_verify_http_signature,
     is_activity_already_known,
-    is_addressed_to_public,
     receive_for_community::{
       receive_add_for_community,
       receive_create_for_community,
@@ -17,6 +16,7 @@ use crate::{
       receive_undo_for_community,
       receive_update_for_community,
     },
+    verify_is_addressed_to_public,
   },
   insert_activity,
   ActorType,
@@ -164,18 +164,18 @@ pub(crate) async fn community_receive_message(
       true
     }
     CommunityValidTypes::Add => {
-      receive_add_for_community(context, any_base.clone(), &actor_url, request_counter).await?;
+      receive_add_for_community(context, any_base.clone(), None, request_counter).await?;
       true
     }
     CommunityValidTypes::Remove => {
-      receive_remove_for_community(context, any_base.clone(), &actor_url, request_counter).await?;
+      receive_remove_for_community(context, any_base.clone(), None, request_counter).await?;
       true
     }
   };
 
   if do_announce {
     // Check again that the activity is public, just to be sure
-    is_addressed_to_public(&activity)?;
+    verify_is_addressed_to_public(&activity)?;
     to_community
       .send_announce(activity.into_any_base()?, context)
       .await?;
diff --git a/crates/apub/src/inbox/mod.rs b/crates/apub/src/inbox/mod.rs
index 21585aa6a..314f57ca1 100644
--- a/crates/apub/src/inbox/mod.rs
+++ b/crates/apub/src/inbox/mod.rs
@@ -84,7 +84,7 @@ where
   to_and_cc
 }
 
-pub(crate) fn is_addressed_to_public<T, Kind>(activity: &T) -> Result<(), LemmyError>
+pub(crate) fn verify_is_addressed_to_public<T, Kind>(activity: &T) -> Result<(), LemmyError>
 where
   T: AsBase<Kind> + AsObject<Kind> + ActorAndObjectRefExt,
 {
diff --git a/crates/apub/src/inbox/receive_for_community.rs b/crates/apub/src/inbox/receive_for_community.rs
index c857b0b40..4a548bc08 100644
--- a/crates/apub/src/inbox/receive_for_community.rs
+++ b/crates/apub/src/inbox/receive_for_community.rs
@@ -36,7 +36,8 @@ use crate::{
     user::get_or_fetch_and_upsert_user,
   },
   find_post_or_comment_by_id,
-  inbox::is_addressed_to_public,
+  generate_moderators_url,
+  inbox::verify_is_addressed_to_public,
   ActorType,
   PostOrComment,
 };
@@ -44,6 +45,7 @@ use activitystreams::{
   activity::{
     ActorAndObjectRef,
     Add,
+    Announce,
     Create,
     Delete,
     Dislike,
@@ -54,6 +56,7 @@ use activitystreams::{
     Update,
   },
   base::AnyBase,
+  object::AsObject,
   prelude::*,
 };
 use anyhow::{anyhow, Context};
@@ -92,7 +95,7 @@ pub(in crate::inbox) async fn receive_create_for_community(
 ) -> Result<(), LemmyError> {
   let create = Create::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&create, &expected_domain, true)?;
-  is_addressed_to_public(&create)?;
+  verify_is_addressed_to_public(&create)?;
 
   let kind = create
     .object()
@@ -114,7 +117,7 @@ pub(in crate::inbox) async fn receive_update_for_community(
 ) -> Result<(), LemmyError> {
   let update = Update::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&update, &expected_domain, true)?;
-  is_addressed_to_public(&update)?;
+  verify_is_addressed_to_public(&update)?;
 
   let kind = update
     .object()
@@ -136,7 +139,7 @@ pub(in crate::inbox) async fn receive_like_for_community(
 ) -> Result<(), LemmyError> {
   let like = Like::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&like, &expected_domain, false)?;
-  is_addressed_to_public(&like)?;
+  verify_is_addressed_to_public(&like)?;
 
   let object_id = like
     .object()
@@ -167,7 +170,7 @@ pub(in crate::inbox) async fn receive_dislike_for_community(
 
   let dislike = Dislike::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&dislike, &expected_domain, false)?;
-  is_addressed_to_public(&dislike)?;
+  verify_is_addressed_to_public(&dislike)?;
 
   let object_id = dislike
     .object()
@@ -191,7 +194,7 @@ pub(in crate::inbox) async fn receive_delete_for_community(
 ) -> Result<(), LemmyError> {
   let delete = Delete::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&delete, &expected_domain, true)?;
-  is_addressed_to_public(&delete)?;
+  verify_is_addressed_to_public(&delete)?;
 
   let object = delete
     .object()
@@ -210,18 +213,18 @@ pub(in crate::inbox) async fn receive_delete_for_community(
 /// A post or comment being removed by a mod/admin
 pub(in crate::inbox) async fn receive_remove_for_community(
   context: &LemmyContext,
-  activity: AnyBase,
-  expected_domain: &Url,
+  remove_any_base: AnyBase,
+  announce: Option<Announce>,
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
-  let remove = Remove::from_any_base(activity.to_owned())?.context(location_info!())?;
-  verify_activity_domains_valid(&remove, &expected_domain, false)?;
-  is_addressed_to_public(&remove)?;
+  let remove = Remove::from_any_base(remove_any_base.to_owned())?.context(location_info!())?;
+  let community = extract_community_from_cc(&remove, context).await?;
+
+  verify_mod_activity(&remove, announce, &community, context).await?;
+  verify_is_addressed_to_public(&remove)?;
+  verify_actor_is_community_mod(&remove, &community, context).await?;
 
-  // Remove a moderator from community
   if remove.target().is_some() {
-    let community = verify_actor_is_community_mod(&remove, context).await?;
-
     let remove_mod = remove
       .object()
       .as_single_xsd_any_uri()
@@ -235,32 +238,18 @@ pub(in crate::inbox) async fn receive_remove_for_community(
       CommunityModerator::leave(conn, &form)
     })
     .await??;
-    community.send_announce(activity, context).await?;
+    community.send_announce(remove_any_base, context).await?;
     // TODO: send websocket notification about removed mod
     Ok(())
   }
   // Remove a post or comment
   else {
-    let cc = remove
-      .cc()
-      .map(|c| c.as_many())
-      .flatten()
-      .context(location_info!())?;
-    let community_id = cc
-      .first()
-      .map(|c| c.as_xsd_any_uri())
-      .flatten()
-      .context(location_info!())?;
-
     let object = remove
       .object()
       .to_owned()
       .single_xsd_any_uri()
       .context(location_info!())?;
 
-    // Ensure that remove activity comes from the same domain as the community
-    remove.id(community_id.domain().context(location_info!())?)?;
-
     match find_post_or_comment_by_id(context, object).await {
       Ok(PostOrComment::Post(p)) => receive_remove_post(context, remove, *p).await,
       Ok(PostOrComment::Comment(c)) => receive_remove_comment(context, remove, *c).await,
@@ -287,7 +276,7 @@ pub(in crate::inbox) async fn receive_undo_for_community(
 ) -> Result<(), LemmyError> {
   let undo = Undo::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&undo, &expected_domain.to_owned(), true)?;
-  is_addressed_to_public(&undo)?;
+  verify_is_addressed_to_public(&undo)?;
 
   use UndoableActivities::*;
   match undo
@@ -316,7 +305,7 @@ pub(in crate::inbox) async fn receive_undo_delete_for_community(
   let delete = Delete::from_any_base(undo.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
   verify_activity_domains_valid(&delete, &expected_domain, true)?;
-  is_addressed_to_public(&delete)?;
+  verify_is_addressed_to_public(&delete)?;
 
   let object = delete
     .object()
@@ -340,7 +329,7 @@ pub(in crate::inbox) async fn receive_undo_remove_for_community(
   let remove = Remove::from_any_base(undo.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
   verify_activity_domains_valid(&remove, &expected_domain, false)?;
-  is_addressed_to_public(&remove)?;
+  verify_is_addressed_to_public(&remove)?;
 
   let object = remove
     .object()
@@ -365,7 +354,7 @@ pub(in crate::inbox) async fn receive_undo_like_for_community(
   let like = Like::from_any_base(undo.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
   verify_activity_domains_valid(&like, &expected_domain, false)?;
-  is_addressed_to_public(&like)?;
+  verify_is_addressed_to_public(&like)?;
 
   let object_id = like
     .object()
@@ -384,14 +373,17 @@ pub(in crate::inbox) async fn receive_undo_like_for_community(
 /// Add a new mod to the community (can only be done by an existing mod).
 pub(in crate::inbox) async fn receive_add_for_community(
   context: &LemmyContext,
-  activity: AnyBase,
-  expected_domain: &Url,
+  add_any_base: AnyBase,
+  announce: Option<Announce>,
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
-  let add = Add::from_any_base(activity.to_owned())?.context(location_info!())?;
-  verify_activity_domains_valid(&add, &expected_domain, false)?;
-  is_addressed_to_public(&add)?;
-  let community = verify_actor_is_community_mod(&add, context).await?;
+  let add = Add::from_any_base(add_any_base.to_owned())?.context(location_info!())?;
+  let community = extract_community_from_cc(&add, context).await?;
+
+  verify_mod_activity(&add, announce, &community, context).await?;
+  verify_is_addressed_to_public(&add)?;
+  verify_actor_is_community_mod(&add, &community, context).await?;
+  verify_add_remove_moderator_target(&add, &community)?;
 
   let new_mod = add
     .object()
@@ -417,7 +409,7 @@ pub(in crate::inbox) async fn receive_add_for_community(
     .await??;
   }
   if community.local {
-    community.send_announce(activity, context).await?;
+    community.send_announce(add_any_base, context).await?;
   }
   // TODO: send websocket notification about added mod
   Ok(())
@@ -433,7 +425,7 @@ pub(in crate::inbox) async fn receive_undo_dislike_for_community(
   let dislike = Dislike::from_any_base(undo.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
   verify_activity_domains_valid(&dislike, &expected_domain, false)?;
-  is_addressed_to_public(&dislike)?;
+  verify_is_addressed_to_public(&dislike)?;
 
   let object_id = dislike
     .object()
@@ -465,26 +457,39 @@ async fn fetch_post_or_comment_by_id(
   Err(NotFound.into())
 }
 
-async fn verify_actor_is_community_mod<T, Kind>(
+async fn extract_community_from_cc<T, Kind>(
   activity: &T,
   context: &LemmyContext,
 ) -> Result<Community, LemmyError>
 where
-  T: ActorAndObjectRef + BaseExt<Kind> + OptTargetRef,
+  T: AsObject<Kind>,
 {
-  // should be the moderators collection of a local community
-  let target = activity
-    .target()
-    .map(|t| t.as_single_xsd_any_uri())
+  let cc = activity
+    .cc()
+    .map(|c| c.as_many())
     .flatten()
     .context(location_info!())?;
-  // TODO: very hacky, we should probably store the moderators url in db
-  let community_id: DbUrl = Url::parse(&target.to_string().replace("/moderators", ""))?.into();
+  let community_id = cc
+    .first()
+    .map(|c| c.as_xsd_any_uri())
+    .flatten()
+    .context(location_info!())?;
+  let community_id: DbUrl = community_id.to_owned().into();
   let community = blocking(&context.pool(), move |conn| {
     Community::read_from_apub_id(&conn, &community_id)
   })
   .await??;
+  Ok(community)
+}
 
+async fn verify_actor_is_community_mod<T, Kind>(
+  activity: &T,
+  community: &Community,
+  context: &LemmyContext,
+) -> Result<(), LemmyError>
+where
+  T: ActorAndObjectRef + BaseExt<Kind>,
+{
   let actor = activity
     .actor()?
     .as_single_xsd_any_uri()
@@ -507,6 +512,43 @@ where
     return Err(anyhow!("Not a mod").into());
   }
 
-  // TODO: the function name doesnt make sense if we return the community
-  Ok(community)
+  Ok(())
+}
+
+async fn verify_mod_activity<T, Kind>(
+  mod_action: &T,
+  announce: Option<Announce>,
+  community: &Community,
+  context: &LemmyContext,
+) -> Result<(), LemmyError>
+where
+  T: ActorAndObjectRef + OptTargetRef + BaseExt<Kind>,
+{
+  // Remove was sent by community to user, we just check that it came from the right domain
+  if let Some(announce) = announce {
+    verify_activity_domains_valid(&announce, &community.actor_id.to_owned().into(), false)?;
+  }
+  // Remove was sent by a remote mod to community, check that they are actually mod
+  else {
+    verify_actor_is_community_mod(mod_action, community, context).await?;
+  }
+
+  Ok(())
+}
+fn verify_add_remove_moderator_target<T, Kind>(
+  activity: &T,
+  community: &Community,
+) -> Result<(), LemmyError>
+where
+  T: ActorAndObjectRef + BaseExt<Kind> + OptTargetRef,
+{
+  let target = activity
+    .target()
+    .map(|t| t.as_single_xsd_any_uri())
+    .flatten()
+    .context(location_info!())?;
+  if target != &generate_moderators_url(&community.actor_id)?.into_inner() {
+    return Err(anyhow!("Unkown target url").into());
+  }
+  Ok(())
 }
diff --git a/crates/apub/src/inbox/user_inbox.rs b/crates/apub/src/inbox/user_inbox.rs
index d99092fc4..28e1365f7 100644
--- a/crates/apub/src/inbox/user_inbox.rs
+++ b/crates/apub/src/inbox/user_inbox.rs
@@ -26,7 +26,6 @@ use crate::{
     is_activity_already_known,
     is_addressed_to_community_followers,
     is_addressed_to_local_user,
-    is_addressed_to_public,
     receive_for_community::{
       receive_add_for_community,
       receive_create_for_community,
@@ -37,6 +36,7 @@ use crate::{
       receive_undo_for_community,
       receive_update_for_community,
     },
+    verify_is_addressed_to_public,
   },
   insert_activity,
   ActorType,
@@ -265,7 +265,7 @@ pub async fn receive_announce(
 ) -> Result<(), LemmyError> {
   let announce = Announce::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&announce, &actor.actor_id(), false)?;
-  is_addressed_to_public(&announce)?;
+  verify_is_addressed_to_public(&announce)?;
 
   let kind = announce
     .object()
@@ -299,13 +299,13 @@ pub async fn receive_announce(
     }
     Some(Delete) => receive_delete_for_community(context, inner_activity, &inner_id).await,
     Some(Remove) => {
-      receive_remove_for_community(context, inner_activity, &inner_id, request_counter).await
+      receive_remove_for_community(context, inner_activity, Some(announce), request_counter).await
     }
     Some(Undo) => {
       receive_undo_for_community(context, inner_activity, &inner_id, request_counter).await
     }
     Some(Add) => {
-      receive_add_for_community(context, inner_activity, &inner_id, request_counter).await
+      receive_add_for_community(context, inner_activity, Some(announce), request_counter).await
     }
     _ => receive_unhandled_activity(inner_activity),
   }
@@ -319,7 +319,7 @@ async fn receive_create(
 ) -> Result<(), LemmyError> {
   let create = Create::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&create, &expected_domain, true)?;
-  if is_addressed_to_public(&create).is_ok() {
+  if verify_is_addressed_to_public(&create).is_ok() {
     receive_create_comment(create, context, request_counter).await
   } else {
     receive_create_private_message(&context, create, expected_domain, request_counter).await
@@ -334,7 +334,7 @@ async fn receive_update(
 ) -> Result<(), LemmyError> {
   let update = Update::from_any_base(activity)?.context(location_info!())?;
   verify_activity_domains_valid(&update, &expected_domain, true)?;
-  if is_addressed_to_public(&update).is_ok() {
+  if verify_is_addressed_to_public(&update).is_ok() {
     receive_update_comment(update, context, request_counter).await
   } else {
     receive_update_private_message(&context, update, expected_domain, request_counter).await