diff --git a/lib/Activity/Listener.php b/lib/Activity/Listener.php
index 232225b6826..57410417ff6 100644
--- a/lib/Activity/Listener.php
+++ b/lib/Activity/Listener.php
@@ -232,6 +232,11 @@ public function generateInvitationActivity(Room $room, array $participants): voi
 			return;
 		}
 
+		// We know the new participant is in the room,
+		// so skip loading them just to make sure they can read it.
+		// Must be overwritten later on for one-to-one chats.
+		$roomName = $room->getDisplayName($actorId);
+
 		foreach ($participants as $participant) {
 			if ($participant['actorType'] !== Attendee::ACTOR_USERS) {
 				// No user => no activity
@@ -244,7 +249,10 @@ public function generateInvitationActivity(Room $room, array $participants): voi
 			}
 
 			try {
-				$roomName = $room->getDisplayName($participant['actorId']);
+				if ($room->getType() === Room::TYPE_ONE_TO_ONE) {
+					// Overwrite the room name with the other participant
+					$roomName = $room->getDisplayName($participant['actorId']);
+				}
 				$event
 					->setObject('room', $room->getId(), $roomName)
 					->setSubject('invitation', [