Fix fetching of community posts (fixes #4283) (#4293)

* Fix fetching of community posts (fixes #4283)

Also use spawn_try_task to fetch community outbox, mods etc to avoid
delay/timeout when fetching new community.

* prettier

* fix test

* fix api test

* prettier

* add delay

* Update run-federation-test.sh

* fix test
increase_max_api_items
Nutomic 2024-01-04 17:42:18 +01:00 committed by GitHub
parent 952c162dac
commit 023c9f4fcd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 30 deletions

View File

@ -31,6 +31,7 @@ import {
searchPostLocal,
resolveBetaCommunity,
longDelay,
delay,
} from "./shared";
import { EditSite } from "lemmy-js-client";
@ -377,7 +378,9 @@ test("User blocks instance, communities are hidden", async () => {
test("Community follower count is federated", async () => {
// Follow the beta community from alpha
let resolved = await resolveBetaCommunity(alpha);
let community = await createCommunity(beta);
let community_id = community.community_view.community.actor_id;
let resolved = await resolveCommunity(alpha, community_id);
if (!resolved.community) {
throw "Missing beta community";
}
@ -385,7 +388,7 @@ test("Community follower count is federated", async () => {
await followCommunity(alpha, true, resolved.community.community.id);
let followed = (
await waitUntil(
() => resolveBetaCommunity(alpha),
() => resolveCommunity(alpha, community_id),
c => c.community?.subscribed === "Subscribed",
)
).community;
@ -394,7 +397,7 @@ test("Community follower count is federated", async () => {
expect(followed?.counts.subscribers).toBe(1);
// Follow the community from gamma
resolved = await resolveBetaCommunity(gamma);
resolved = await resolveCommunity(gamma, community_id);
if (!resolved.community) {
throw "Missing beta community";
}
@ -402,7 +405,7 @@ test("Community follower count is federated", async () => {
await followCommunity(gamma, true, resolved.community.community.id);
followed = (
await waitUntil(
() => resolveBetaCommunity(gamma),
() => resolveCommunity(gamma, community_id),
c => c.community?.subscribed === "Subscribed",
)
).community;
@ -411,7 +414,7 @@ test("Community follower count is federated", async () => {
expect(followed?.counts?.subscribers).toBe(2);
// Follow the community from delta
resolved = await resolveBetaCommunity(delta);
resolved = await resolveCommunity(delta, community_id);
if (!resolved.community) {
throw "Missing beta community";
}
@ -419,7 +422,7 @@ test("Community follower count is federated", async () => {
await followCommunity(delta, true, resolved.community.community.id);
followed = (
await waitUntil(
() => resolveBetaCommunity(delta),
() => resolveCommunity(delta, community_id),
c => c.community?.subscribed === "Subscribed",
)
).community;
@ -480,3 +483,31 @@ test("Dont receive community activities after unsubscribe", async () => {
let postResBeta = searchPostLocal(beta, postRes.post_view.post);
expect((await postResBeta).posts.length).toBe(0);
});
test("Fetch community, includes posts", async () => {
let communityRes = await createCommunity(alpha);
expect(communityRes.community_view.community.name).toBeDefined();
expect(communityRes.community_view.counts.subscribers).toBe(1);
let postRes = await createPost(
alpha,
communityRes.community_view.community.id,
);
expect(postRes.post_view.post).toBeDefined();
let resolvedCommunity = await waitUntil(
() =>
resolveCommunity(beta, communityRes.community_view.community.actor_id),
c => c.community?.community.id != undefined,
);
let betaCommunity = resolvedCommunity.community;
expect(betaCommunity?.community.actor_id).toBe(
communityRes.community_view.community.actor_id,
);
await longDelay();
let post_listing = await getPosts(beta, "All", betaCommunity?.community.id);
expect(post_listing.posts.length).toBe(1);
expect(post_listing.posts[0].post.ap_id).toBe(postRes.post_view.post.ap_id);
});

View File

@ -805,10 +805,12 @@ export async function listCommentReports(
export function getPosts(
api: LemmyHttp,
listingType?: ListingType,
community_id?: number,
): Promise<GetPostsResponse> {
let form: GetPosts = {
type_: listingType,
limit: 50,
community_id,
};
return api.getPosts(form);
}

View File

@ -96,11 +96,15 @@ impl Collection for ApubCommunityOutbox {
// process items in parallel, to avoid long delay from fetch_site_metadata() and other processing
join_all(outbox_activities.into_iter().map(|activity| {
async {
// use separate request counter for each item, otherwise there will be problems with
// parallel processing
let verify = activity.verify(data).await;
if verify.is_ok() {
activity.receive(data).await.ok();
// Receiving announce requires at least one local community follower for anti spam purposes.
// This won't be the case for newly fetched communities, so we extract the inner activity
// and handle it directly to bypass this check.
let inner = activity.object.object(data).await.map(TryInto::try_into);
if let Ok(Ok(AnnouncableActivities::CreateOrUpdatePost(inner))) = inner {
let verify = inner.verify(data).await;
if verify.is_ok() {
inner.receive(data).await.ok();
}
}
}
}))

View File

@ -28,9 +28,8 @@ use lemmy_db_schema::{
traits::{ApubActor, Crud},
};
use lemmy_db_views_actor::structs::CommunityFollowerView;
use lemmy_utils::{error::LemmyError, utils::markdown::markdown_to_html};
use lemmy_utils::{error::LemmyError, spawn_try_task, utils::markdown::markdown_to_html};
use std::ops::Deref;
use tracing::debug;
use url::Url;
#[derive(Clone, Debug)]
@ -142,21 +141,16 @@ impl Object for ApubCommunity {
// Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
// we need to ignore these errors so that tests can work entirely offline.
let fetch_outbox = group.outbox.dereference(&community, context);
let fetch_followers = group.followers.dereference(&community, context);
if let Some(moderators) = group.attributed_to {
let fetch_moderators = moderators.dereference(&community, context);
// Fetch mods, outbox and followers in parallel
let res = tokio::join!(fetch_outbox, fetch_moderators, fetch_followers);
res.0.map_err(|e| debug!("{}", e)).ok();
res.1.map_err(|e| debug!("{}", e)).ok();
res.2.map_err(|e| debug!("{}", e)).ok();
} else {
let res = tokio::join!(fetch_outbox, fetch_followers);
res.0.map_err(|e| debug!("{}", e)).ok();
res.1.map_err(|e| debug!("{}", e)).ok();
}
let community_ = community.clone();
let context_ = context.reset_request_count();
spawn_try_task(async move {
group.outbox.dereference(&community_, &context_).await?;
group.followers.dereference(&community_, &context_).await?;
if let Some(moderators) = group.attributed_to {
moderators.dereference(&community_, &context_).await?;
}
Ok(())
});
Ok(community)
}
@ -241,8 +235,6 @@ pub(crate) mod tests {
let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward")?;
ApubCommunity::verify(&json, &url, &context2).await?;
let community = ApubCommunity::from_json(json, &context2).await?;
// this makes requests to the (intentionally broken) outbox and followers collections
assert_eq!(context2.request_count(), 2);
Ok(community)
}