diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 000000000..6460a78c6
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,51 @@
+name: Megatron-LM-X-CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  pull_request:
+
+jobs:
+  code-lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - run: echo "🎉 Begin Megatron-LM-X Python Lint."
+      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
+      - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
+      - uses: actions/checkout@v4
+      - run: git config --global --add safe.directory /github/workspace
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+            python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Install Autoflake
+        run: pip install autoflake==v2.3.1
+      - name: Run Autoflake
+        if: always()
+        run: |
+          output=$(autoflake . --remove-all-unused-imports --remove-unused-variables --expand-star-imports --ignore-init-module-imports --recursive)
+          if [[ -n $output ]]
+          then
+            echo "Autoflake check failed: $output"
+            exit 1
+          else
+            echo "Autoflake check success."
+            exit 0
+          fi
+      - uses: isort/isort-action@v1
+        if: always()
+        with:
+            configuration: "--profile black --check-only --diff"
+      - uses: psf/black@stable
+        if: always()
+        with:
+          options: "--check --diff --color --verbose --line-length=110"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..29cd260e3
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,43 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+# Install
+# 1. pip install pre-commit
+# 2. pre-commit install(the first time you download the repo, it will be cached for future use)
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+      - id: autoflake
+        args:
+          [
+            "--remove-all-unused-imports",
+            "--remove-unused-variables",
+            "--expand-star-imports",
+            "--ignore-init-module-imports",
+            "--recursive",
+            "--in-place",
+          ]
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.8.0
+    hooks:
+      - id: black
+        args: ["--verbose", "--line-length=110"]
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.10.0.1
+    hooks:
+      - id: shellcheck
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/third_party/README.md b/third_party/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/README.md b/xcore/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/__init__.py b/xcore/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/data/README.md b/xcore/data/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/data/__init__.py b/xcore/data/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/model/README.md b/xcore/model/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/model/__init__.py b/xcore/model/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/tokenizer/README.md b/xcore/tokenizer/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/xcore/tokenizer/__init__.py b/xcore/tokenizer/__init__.py
new file mode 100644
index 000000000..e69de29bb