diff --git a/libai/models/gpt_model.py b/libai/models/gpt_model.py
index 27f6bc8e9..ac11cc416 100644
--- a/libai/models/gpt_model.py
+++ b/libai/models/gpt_model.py
@@ -244,7 +244,9 @@ def forward(self, input_ids, past_length=0):
         bsz, seq_length = input_ids.size()
 
         position_ids = self.position_ids[:, past_length : past_length + seq_length]
-        position_ids = position_ids.expand_as(input_ids).to_global(sbp=input_ids.sbp)
+        position_ids = position_ids.expand_as(input_ids).to_global(
+            sbp=input_ids.sbp, placement=input_ids.placement
+        )
 
         token_embeds = self.token_embeddings(input_ids)
         position_embeds = self.position_embeddings(position_ids)
diff --git a/tools/train_net.py b/tools/train_net.py
index 458849c75..eb63bc64b 100644
--- a/tools/train_net.py
+++ b/tools/train_net.py
@@ -36,7 +36,8 @@ def main(args):
 
     seed_for_rank = cfg.train.seed + flow.env.get_rank()
     flow.manual_seed(seed_for_rank)
-    flow.cuda.manual_seed(seed_for_rank)
+    if flow.cuda.is_available():
+        flow.cuda.manual_seed(seed_for_rank)
     np.random.seed(seed_for_rank)
     random.seed(seed_for_rank)