diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index a45ab05b729..8ae9681688c 100755
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -2005,7 +2005,7 @@ def _prepare_ipex_or_xpu(self, *args):
                 optimizer = obj
         if optimizer is not None and model is not None:
             dtype = torch.bfloat16 if self.state.mixed_precision == "bf16" else None
-            if self.device.type == "xpu":
+            if self.device.type == "xpu" and model.device.type == "cpu":
                 model = model.to(self.device)
             # ipex.optimize() is available only for IPEX, both IPEX-CPU and IPEX-XPU
             if is_ipex_available():
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 239f37887e6..47d718704a6 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -703,7 +703,7 @@ def default_device(self) -> torch.device:
         elif torch.cuda.is_available():
             return torch.device("cuda")
         elif is_xpu_available():
-            return torch.device("xpu:0")
+            return torch.device("xpu")
         else:
             return torch.device("cpu")