diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index dd303a8d..5fbaefb0 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -285,16 +285,21 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 		}
 		// work out if our cross-signing request for this user was
 		// satisfied, if not add them to the list of things to fetch
-		if _, ok := domainToDeviceKeys[domain]; !ok {
-			domainToDeviceKeys[domain] = make(map[string][]string)
-		}
 		if _, ok := res.MasterKeys[userID]; !ok {
+			if _, ok := domainToDeviceKeys[domain]; !ok {
+				domainToDeviceKeys[domain] = make(map[string][]string)
+			}
 			if _, ok := domainToDeviceKeys[domain][userID]; !ok {
+				util.GetLogger(ctx).Infof("Request cross-signing keys from %s for %s", domain, userID)
 				domainToDeviceKeys[domain][userID] = []string{}
 			}
 		}
 		if _, ok := res.SelfSigningKeys[userID]; !ok {
+			if _, ok := domainToDeviceKeys[domain]; !ok {
+				domainToDeviceKeys[domain] = make(map[string][]string)
+			}
 			if _, ok := domainToDeviceKeys[domain][userID]; !ok {
+				util.GetLogger(ctx).Infof("Request cross-signing keys from %s for %s", domain, userID)
 				domainToDeviceKeys[domain][userID] = []string{}
 			}
 		}