diff --git a/common/flatpak-oci-registry.c b/common/flatpak-oci-registry.c
index 6576ae81..48c8dd04 100644
--- a/common/flatpak-oci-registry.c
+++ b/common/flatpak-oci-registry.c
@@ -1036,6 +1036,18 @@ get_token_for_www_auth (FlatpakOciRegistry *self,
   if (body == NULL)
     return NULL;
 
+  if (!SOUP_STATUS_IS_SUCCESSFUL (auth_msg->status_code))
+    {
+      if (auth_msg->status_code == SOUP_STATUS_UNAUTHORIZED)
+        {
+          flatpak_fail_error (error, FLATPAK_ERROR_NOT_AUTHORIZED, _("Authorization failed: %s"), (char *)g_bytes_get_data (body, NULL));
+          return NULL;
+        }
+
+      flatpak_fail (error, _("Unexpected response status %d when requesting token: %s"), auth_msg->status_code, (char *)g_bytes_get_data (body, NULL));
+      return NULL;
+    }
+
   json = json_from_string ((char *)g_bytes_get_data (body, NULL), error);
   if (json == NULL)
     return NULL;
diff --git a/oci-authenticator/flatpak-oci-authenticator.c b/oci-authenticator/flatpak-oci-authenticator.c
index 85a36496..a8413eb5 100644
--- a/oci-authenticator/flatpak-oci-authenticator.c
+++ b/oci-authenticator/flatpak-oci-authenticator.c
@@ -539,6 +539,8 @@ handle_request_ref_tokens (FlatpakAuthenticator *f_authenticator,
           if (test_auth == NULL)
             return cancel_request (request, sender);
 
+          g_clear_error (&error);
+
           first_token = get_token_for_ref (registry, ref_data, test_auth, &error);
           if (first_token != NULL)
             {
@@ -547,14 +549,19 @@ handle_request_ref_tokens (FlatpakAuthenticator *f_authenticator,
             }
           else
             {
-              g_debug ("Failed to get token: %s", error->message);
-              g_clear_error (&error);
+              if (!g_error_matches (error, FLATPAK_ERROR, FLATPAK_ERROR_NOT_AUTHORIZED))
+                return error_request (request, sender, error);
+              else
+                {
+                  g_debug ("Auth failed getting token: %s", error->message);
+                  /* Keep error for reporting below, or clear on next iteration start */
+                }
             }
         }
     }
 
   if (!have_auth)
-    return error_request_raw (request, sender, FLATPAK_ERROR_AUTHENTICATION_FAILED, _("Authentication failed"));
+    return error_request (request, sender, error);
 
   g_variant_builder_init (&tokens, G_VARIANT_TYPE ("a{sas}"));