diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 82927570333a..d7a957376a90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -371,11 +371,9 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 	p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
 		initial_bytes_moved;
 
-	if (unlikely(r)) {
-		if (r != -ERESTARTSYS && domain != bo->allowed_domains) {
-			domain = bo->allowed_domains;
-			goto retry;
-		}
+	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
+		domain = bo->allowed_domains;
+		goto retry;
 	}
 
 	return r;