feat: add sequence parallelism instructions and clean up imports

- Add bash script header and comments to `sp_fsdp_dense.sh` explaining how to enable sequence parallelism with ulysses_size
- Remove duplicate `import os` statement in transformers.py for cleaner code
- Fix minor formatting by removing extra blank line in transformers_utils.py

Author: meichangsu1

cookbook/transformers/sp_fsdp_dense.sh

@@ -1 +1,10 @@
+#!/bin/bash
+# To enabele sequence parallelism, please set ulysses_size > 1
+# device_mesh = DeviceMesh(
+#     device_type="cuda",
+#     mesh=np.arange(4).reshape(2, 2),
+#     mesh_dim_names=("dp", "fsdp"),
+#     ulysses_size=2,
+# )
+#
 CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --nproc_per_node=4 sp_fsdp_dense.py

src/twinkle/model/transformers/transformers.py

@@ -2,7 +2,6 @@
 import contextlib
 import os
 import json
-import os
 import re
 from dataclasses import dataclass, field
 from typing import Dict, Any, List, Literal, Callable

src/twinkle/utils/transformers_utils.py

@@ -135,7 +135,6 @@ def get_modules_to_not_convert(model):
             res.append(n)
     return res if res else None
 
-
 def get_llm_model(model, *, model_meta=None, inner_backbone: bool = True):
     """Best-effort extraction of the LLM module from a (possibly wrapped) model.