diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index cc4b2ee5b4..aa320773ad 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -1024,7 +1024,7 @@ class Trainer:
         else:
             loss.backward()
 
-        return loss
+        return loss.detach()
 
     def is_local_master(self) -> bool:
         """