diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 2873b604..dd303a8d 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -233,7 +233,6 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 
 	// make a map from domain to device keys
 	domainToDeviceKeys := make(map[string]map[string][]string)
-	domainToCrossSigningKeys := make(map[string]struct{})
 	for userID, deviceIDs := range req.UserToDevices {
 		_, serverName, err := gomatrixserverlib.SplitID('@', userID)
 		if err != nil {
@@ -285,20 +284,25 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 			domainToDeviceKeys[domain][userID] = append(domainToDeviceKeys[domain][userID], deviceIDs...)
 		}
 		// work out if our cross-signing request for this user was
-		// satisfied
-		if _, ok := domainToCrossSigningKeys[userID]; !ok {
-			if _, ok := res.MasterKeys[userID]; !ok {
-				domainToCrossSigningKeys[userID] = struct{}{}
+		// satisfied, if not add them to the list of things to fetch
+		if _, ok := domainToDeviceKeys[domain]; !ok {
+			domainToDeviceKeys[domain] = make(map[string][]string)
+		}
+		if _, ok := res.MasterKeys[userID]; !ok {
+			if _, ok := domainToDeviceKeys[domain][userID]; !ok {
+				domainToDeviceKeys[domain][userID] = []string{}
 			}
-			if _, ok := res.SelfSigningKeys[userID]; !ok {
-				domainToCrossSigningKeys[userID] = struct{}{}
+		}
+		if _, ok := res.SelfSigningKeys[userID]; !ok {
+			if _, ok := domainToDeviceKeys[domain][userID]; !ok {
+				domainToDeviceKeys[domain][userID] = []string{}
 			}
 		}
 	}
 
 	// attempt to satisfy key queries from the local database first as we should get device updates pushed to us
 	domainToDeviceKeys = a.remoteKeysFromDatabase(ctx, res, domainToDeviceKeys)
-	if len(domainToDeviceKeys) == 0 && len(domainToCrossSigningKeys) == 0 {
+	if len(domainToDeviceKeys) == 0 {
 		return // nothing to query
 	}