From f3f8c8861db75cc69acf1f4edbe608831e941488 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Tue, 27 Feb 2024 15:38:29 +0530
Subject: [PATCH 01/26] Update finetune_lora.sh

Changed the data_path and image_folder paths
---
 scripts/finetune_lora.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index fc02e09d7..7a2889de9 100644
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -19,8 +19,8 @@ deepspeed llava/train/train_mem.py \
     --lora_enable True \
     --model_name_or_path ./checkpoints/$MODEL_VERSION \
     --version $PROMPT_VERSION \
-    --data_path ./playground/data/llava_instruct_80k.json \
-    --image_folder /path/to/coco/train2017 \
+    --data_path /home/akash/data/LLAVA_data.json \
+    --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
     --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
     --mm_vision_select_layer -2 \
@@ -45,5 +45,4 @@ deepspeed llava/train/train_mem.py \
     --model_max_length 2048 \
     --gradient_checkpointing True \
     --lazy_preprocess True \
-    --dataloader_num_workers 4 \
-    --report_to wandb
+    --dataloader_num_workers 4 

From b55ac50a4558769ddbec4688aeb6c6898a02b347 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Tue, 27 Feb 2024 15:54:37 +0530
Subject: [PATCH 02/26] Update finetune_lora.sh

Changed the path for train_mem.py to absolute path
---
 scripts/finetune_lora.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index 7a2889de9..bf13ded21 100644
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -14,7 +14,7 @@
 # MODEL_VERSION="llama-2-7b-chat"
 ################## LLaMA-2 ##################
 
-deepspeed llava/train/train_mem.py \
+deepspeed /home/akash/LLaVA/llava/train/train_mem.py \
     --deepspeed ./scripts/zero2.json \
     --lora_enable True \
     --model_name_or_path ./checkpoints/$MODEL_VERSION \

From e68ddbde6ac5ed31db2e29c0fa8f73fc08b63e52 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Tue, 27 Feb 2024 17:29:25 +0530
Subject: [PATCH 03/26] Update finetune_lora.sh

uncommented the PROMPT_VERSION AND MODEL_VERSION
---
 scripts/finetune_lora.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index bf13ded21..39b5b8f85 100644
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -10,8 +10,8 @@
 ################## VICUNA ##################
 
 ################## LLaMA-2 ##################
-# PROMPT_VERSION="llava_llama_2"
-# MODEL_VERSION="llama-2-7b-chat"
+PROMPT_VERSION="llava_llama_2"
+MODEL_VERSION="llama-2-7b-chat"
 ################## LLaMA-2 ##################
 
 deepspeed /home/akash/LLaVA/llava/train/train_mem.py \

From 2324760e492d87bdc6a75745cf9947dd4c097de9 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Fri, 8 Mar 2024 07:11:10 +0000
Subject: [PATCH 04/26] updating path

---
 scripts/finetune_lora.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index 39b5b8f85..6e39b0bce 100644
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -14,7 +14,7 @@ PROMPT_VERSION="llava_llama_2"
 MODEL_VERSION="llama-2-7b-chat"
 ################## LLaMA-2 ##################
 
-deepspeed /home/akash/LLaVA/llava/train/train_mem.py \
+deepspeed llava/train/train_mem.py \
     --deepspeed ./scripts/zero2.json \
     --lora_enable True \
     --model_name_or_path ./checkpoints/$MODEL_VERSION \

From a689fd8c73e7d3cdaf5cb8f5a0686f1dbe2c4afc Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Sun, 10 Mar 2024 09:47:59 +0000
Subject: [PATCH 05/26] modified the checkpoint to hf loaction

---
 scripts/convert_gqa_for_eval.py             | 0
 scripts/convert_mmbench_for_submission.py   | 0
 scripts/convert_mmvet_for_eval.py           | 0
 scripts/convert_seed_for_submission.py      | 0
 scripts/convert_sqa_to_llava.py             | 0
 scripts/convert_sqa_to_llava_base_prompt.py | 0
 scripts/convert_vizwiz_for_submission.py    | 0
 scripts/convert_vqav2_for_submission.py     | 0
 scripts/extract_mm_projector.py             | 0
 scripts/finetune.sh                         | 0
 scripts/finetune_full_schedule.sh           | 0
 scripts/finetune_lora.sh                    | 2 +-
 scripts/finetune_qlora.sh                   | 0
 scripts/finetune_sqa.sh                     | 0
 scripts/merge_lora_weights.py               | 0
 scripts/pretrain.sh                         | 0
 scripts/pretrain_xformers.sh                | 0
 scripts/sqa_eval_batch.sh                   | 0
 scripts/sqa_eval_gather.sh                  | 0
 scripts/zero2.json                          | 0
 scripts/zero3.json                          | 0
 scripts/zero3_offload.json                  | 0
 22 files changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 scripts/convert_gqa_for_eval.py
 mode change 100644 => 100755 scripts/convert_mmbench_for_submission.py
 mode change 100644 => 100755 scripts/convert_mmvet_for_eval.py
 mode change 100644 => 100755 scripts/convert_seed_for_submission.py
 mode change 100644 => 100755 scripts/convert_sqa_to_llava.py
 mode change 100644 => 100755 scripts/convert_sqa_to_llava_base_prompt.py
 mode change 100644 => 100755 scripts/convert_vizwiz_for_submission.py
 mode change 100644 => 100755 scripts/convert_vqav2_for_submission.py
 mode change 100644 => 100755 scripts/extract_mm_projector.py
 mode change 100644 => 100755 scripts/finetune.sh
 mode change 100644 => 100755 scripts/finetune_full_schedule.sh
 mode change 100644 => 100755 scripts/finetune_lora.sh
 mode change 100644 => 100755 scripts/finetune_qlora.sh
 mode change 100644 => 100755 scripts/finetune_sqa.sh
 mode change 100644 => 100755 scripts/merge_lora_weights.py
 mode change 100644 => 100755 scripts/pretrain.sh
 mode change 100644 => 100755 scripts/pretrain_xformers.sh
 mode change 100644 => 100755 scripts/sqa_eval_batch.sh
 mode change 100644 => 100755 scripts/sqa_eval_gather.sh
 mode change 100644 => 100755 scripts/zero2.json
 mode change 100644 => 100755 scripts/zero3.json
 mode change 100644 => 100755 scripts/zero3_offload.json

diff --git a/scripts/convert_gqa_for_eval.py b/scripts/convert_gqa_for_eval.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_mmbench_for_submission.py b/scripts/convert_mmbench_for_submission.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_mmvet_for_eval.py b/scripts/convert_mmvet_for_eval.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_seed_for_submission.py b/scripts/convert_seed_for_submission.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_sqa_to_llava.py b/scripts/convert_sqa_to_llava.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_sqa_to_llava_base_prompt.py b/scripts/convert_sqa_to_llava_base_prompt.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_vizwiz_for_submission.py b/scripts/convert_vizwiz_for_submission.py
old mode 100644
new mode 100755
diff --git a/scripts/convert_vqav2_for_submission.py b/scripts/convert_vqav2_for_submission.py
old mode 100644
new mode 100755
diff --git a/scripts/extract_mm_projector.py b/scripts/extract_mm_projector.py
old mode 100644
new mode 100755
diff --git a/scripts/finetune.sh b/scripts/finetune.sh
old mode 100644
new mode 100755
diff --git a/scripts/finetune_full_schedule.sh b/scripts/finetune_full_schedule.sh
old mode 100644
new mode 100755
diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
old mode 100644
new mode 100755
index 6e39b0bce..55c51b83b
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -27,7 +27,7 @@ deepspeed llava/train/train_mem.py \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \
     --bf16 True \
-    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
+    --output_dir liuhaotian/llava-llama-2-7b-chat-lightning-lora-preview
     --num_train_epochs 1 \
     --per_device_train_batch_size 16 \
     --per_device_eval_batch_size 4 \
diff --git a/scripts/finetune_qlora.sh b/scripts/finetune_qlora.sh
old mode 100644
new mode 100755
diff --git a/scripts/finetune_sqa.sh b/scripts/finetune_sqa.sh
old mode 100644
new mode 100755
diff --git a/scripts/merge_lora_weights.py b/scripts/merge_lora_weights.py
old mode 100644
new mode 100755
diff --git a/scripts/pretrain.sh b/scripts/pretrain.sh
old mode 100644
new mode 100755
diff --git a/scripts/pretrain_xformers.sh b/scripts/pretrain_xformers.sh
old mode 100644
new mode 100755
diff --git a/scripts/sqa_eval_batch.sh b/scripts/sqa_eval_batch.sh
old mode 100644
new mode 100755
diff --git a/scripts/sqa_eval_gather.sh b/scripts/sqa_eval_gather.sh
old mode 100644
new mode 100755
diff --git a/scripts/zero2.json b/scripts/zero2.json
old mode 100644
new mode 100755
diff --git a/scripts/zero3.json b/scripts/zero3.json
old mode 100644
new mode 100755
diff --git a/scripts/zero3_offload.json b/scripts/zero3_offload.json
old mode 100644
new mode 100755

From 696bbcc5049eaddaf8e574388a9f40e8a0ab019d Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Sun, 10 Mar 2024 10:50:08 +0000
Subject: [PATCH 06/26] changed the model_name_or_path to hf model path

---
 scripts/finetune_lora.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index 55c51b83b..3ad783751 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -17,7 +17,7 @@ MODEL_VERSION="llama-2-7b-chat"
 deepspeed llava/train/train_mem.py \
     --deepspeed ./scripts/zero2.json \
     --lora_enable True \
-    --model_name_or_path ./checkpoints/$MODEL_VERSION \
+    --model_name_or_path liuhaotian/llava-llama-2-7b-chat-lightning-lora-preview \
     --version $PROMPT_VERSION \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
@@ -27,7 +27,7 @@ deepspeed llava/train/train_mem.py \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \
     --bf16 True \
-    --output_dir liuhaotian/llava-llama-2-7b-chat-lightning-lora-preview
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \
     --per_device_train_batch_size 16 \
     --per_device_eval_batch_size 4 \

From 403ef5cd581b011b067f8dff94712886ec8b51f4 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 11:22:52 +0530
Subject: [PATCH 07/26] Update finetune_lora.sh

changed path names for model_name_or_path and output_dir
---
 scripts/finetune_lora.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index 3ad783751..e272a77f0 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -17,7 +17,7 @@ MODEL_VERSION="llama-2-7b-chat"
 deepspeed llava/train/train_mem.py \
     --deepspeed ./scripts/zero2.json \
     --lora_enable True \
-    --model_name_or_path liuhaotian/llava-llama-2-7b-chat-lightning-lora-preview \
+    --model_name_or_path /home/akash/llava-v1.5-7b \
     --version $PROMPT_VERSION \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
@@ -27,7 +27,7 @@ deepspeed llava/train/train_mem.py \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \
     --bf16 True \
-    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
+    --output_dir /home/akash/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \
     --per_device_train_batch_size 16 \
     --per_device_eval_batch_size 4 \

From c9f11eca099d5f35987625999b67284699df8169 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 11:37:39 +0530
Subject: [PATCH 08/26] Update finetune_lora.sh

Added --mm_projector_type mlp2x_gelu \ as per wandb blog and commented the pretrain_mm_mlp_adapter line
---
 scripts/finetune_lora.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index e272a77f0..d3ded9cae 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -22,7 +22,8 @@ deepspeed llava/train/train_mem.py \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
-    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    # --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
+    --mm_projector_type mlp2x_gelu \
     --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \

From c9628fe5431295845433d4a66844e7fb96191738 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 11:48:59 +0530
Subject: [PATCH 09/26] Update finetune_lora.sh

changed the pretrain_mm_mlp_adapter path to suit the llava-v1.5-7b path in home dir and also removed the --mm_projector_type command
---
 scripts/finetune_lora.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index d3ded9cae..9984c867a 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -22,8 +22,8 @@ deepspeed llava/train/train_mem.py \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
-    # --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
-    --mm_projector_type mlp2x_gelu \
+    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin \
+    # --mm_projector_type mlp2x_gelu \  #added this from wandb blog, but was giving errors
     --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \

From 4ff13a98e8f27a48d8991dad2c07bae7fc424aa3 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 11:58:01 +0530
Subject: [PATCH 10/26] Update finetune_lora.sh

mm_vision_select_layer commented
---
 scripts/finetune_lora.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index 9984c867a..a4a70b34e 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -24,7 +24,7 @@ deepspeed llava/train/train_mem.py \
     --vision_tower openai/clip-vit-large-patch14 \
     --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin \
     # --mm_projector_type mlp2x_gelu \  #added this from wandb blog, but was giving errors
-    --mm_vision_select_layer -2 \
+    # --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \
     --bf16 True \

From ef51af7181fcfe5894532c6c287c6a4d82b5050e Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:02:36 +0530
Subject: [PATCH 11/26] Update finetune_lora.sh

Removed comments, was second guessing that might be issue of recurrent command not found prompts
---
 scripts/finetune_lora.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index a4a70b34e..c7b20b2de 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -22,9 +22,8 @@ deepspeed llava/train/train_mem.py \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
-    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin \
-    # --mm_projector_type mlp2x_gelu \  #added this from wandb blog, but was giving errors
-    # --mm_vision_select_layer -2 \
+    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin 
+    --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \
     --bf16 True \

From 580696af1b1bf36373d528877ae9872f5773c3f9 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:05:23 +0530
Subject: [PATCH 12/26] Update finetune_lora.sh

Commented the commands not found
---
 scripts/finetune_lora.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index c7b20b2de..ac7109b32 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -23,9 +23,9 @@ deepspeed llava/train/train_mem.py \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
     --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin 
-    --mm_vision_select_layer -2 \
-    --mm_use_im_start_end False \
-    --mm_use_im_patch_token False \
+    # --mm_vision_select_layer -2 \
+    # --mm_use_im_start_end False \
+    # --mm_use_im_patch_token False \
     --bf16 True \
     --output_dir /home/akash/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \

From 423ad7f07f2fac1e6f2dd4e3b23e353d27dc8c10 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:11:17 +0530
Subject: [PATCH 13/26] Update finetune_lora.sh

---
 scripts/finetune_lora.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index ac7109b32..a29984007 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -26,7 +26,7 @@ deepspeed llava/train/train_mem.py \
     # --mm_vision_select_layer -2 \
     # --mm_use_im_start_end False \
     # --mm_use_im_patch_token False \
-    --bf16 True \
+    # --bf16 True \
     --output_dir /home/akash/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \
     --per_device_train_batch_size 16 \

From ca7e22bbac9e9427de921696d990033386e1d2c4 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:13:08 +0530
Subject: [PATCH 14/26] Update finetune_lora.sh

Stupid mistake of missing a "\" after pretarin_mm_mlp_adapter
---
 scripts/finetune_lora.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index a29984007..aaa50cfc3 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -22,11 +22,11 @@ deepspeed llava/train/train_mem.py \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
-    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin 
-    # --mm_vision_select_layer -2 \
-    # --mm_use_im_start_end False \
-    # --mm_use_im_patch_token False \
-    # --bf16 True \
+    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
     --output_dir /home/akash/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \
     --per_device_train_batch_size 16 \

From a417a4ce930cde5d68dda0f103064e0b13b62aeb Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:15:32 +0530
Subject: [PATCH 15/26] Update finetune_lora.sh

code spacing typos
---
 scripts/finetune_lora.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index aaa50cfc3..af21434e2 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -22,7 +22,7 @@ deepspeed llava/train/train_mem.py \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14 \
-    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b \mm_projector.bin \
+    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b\mm_projector.bin \
     --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \

From 4fa482f8903386188e06b5d46abcdb5e5d62cde6 Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Thu, 14 Mar 2024 14:27:13 +0530
Subject: [PATCH 16/26] Update finetune_lora.sh

Changed the file paths for the v1.5 script
---
 scripts/v1_5/finetune_lora.sh | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scripts/v1_5/finetune_lora.sh b/scripts/v1_5/finetune_lora.sh
index 90f00707c..fc0d9360e 100644
--- a/scripts/v1_5/finetune_lora.sh
+++ b/scripts/v1_5/finetune_lora.sh
@@ -3,10 +3,10 @@
 deepspeed llava/train/train_mem.py \
     --lora_enable True --lora_r 128 --lora_alpha 256 --mm_projector_lr 2e-5 \
     --deepspeed ./scripts/zero3.json \
-    --model_name_or_path lmsys/vicuna-13b-v1.5 \
+    --model_name_or_path vicuna-13b-v1.5 \
     --version v1 \
-    --data_path ./playground/data/llava_v1_5_mix665k.json \
-    --image_folder ./playground/data \
+    --data_path /home/akash/data/LLAVA_data.json \
+    --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14-336 \
     --pretrain_mm_mlp_adapter ./checkpoints/llava-v1.5-13b-pretrain/mm_projector.bin \
     --mm_projector_type mlp2x_gelu \
@@ -34,5 +34,4 @@ deepspeed llava/train/train_mem.py \
     --model_max_length 2048 \
     --gradient_checkpointing True \
     --dataloader_num_workers 4 \
-    --lazy_preprocess True \
-    --report_to wandb
+    --lazy_preprocess True

From c8f3b6a73f10c099a0f724e1a5bbdd51954c9c01 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Sat, 23 Mar 2024 10:44:22 +0000
Subject: [PATCH 17/26] night call changes

---
 LICENSE                            | 0
 README.md                          | 0
 cog.yaml                           | 0
 predict.py                         | 0
 pyproject.toml                     | 0
 scripts/finetune_lora.sh           | 2 +-
 scripts/v1_5/finetune.sh           | 0
 scripts/v1_5/finetune_lora.sh      | 8 ++++----
 scripts/v1_5/finetune_task.sh      | 0
 scripts/v1_5/finetune_task_lora.sh | 0
 scripts/v1_5/pretrain.sh           | 0
 scripts/zero3.json                 | 7 +++++--
 12 files changed, 10 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 LICENSE
 mode change 100644 => 100755 README.md
 mode change 100644 => 100755 cog.yaml
 mode change 100644 => 100755 predict.py
 mode change 100644 => 100755 pyproject.toml
 mode change 100644 => 100755 scripts/v1_5/finetune.sh
 mode change 100644 => 100755 scripts/v1_5/finetune_lora.sh
 mode change 100644 => 100755 scripts/v1_5/finetune_task.sh
 mode change 100644 => 100755 scripts/v1_5/finetune_task_lora.sh
 mode change 100644 => 100755 scripts/v1_5/pretrain.sh

diff --git a/LICENSE b/LICENSE
old mode 100644
new mode 100755
diff --git a/README.md b/README.md
old mode 100644
new mode 100755
diff --git a/cog.yaml b/cog.yaml
old mode 100644
new mode 100755
diff --git a/predict.py b/predict.py
old mode 100644
new mode 100755
diff --git a/pyproject.toml b/pyproject.toml
old mode 100644
new mode 100755
diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index af21434e2..ea1e11b75 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -29,7 +29,7 @@ deepspeed llava/train/train_mem.py \
     --bf16 True \
     --output_dir /home/akash/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \
-    --per_device_train_batch_size 16 \
+    --per_device_train_batch_size 4 \
     --per_device_eval_batch_size 4 \
     --gradient_accumulation_steps 1 \
     --evaluation_strategy "no" \
diff --git a/scripts/v1_5/finetune.sh b/scripts/v1_5/finetune.sh
old mode 100644
new mode 100755
diff --git a/scripts/v1_5/finetune_lora.sh b/scripts/v1_5/finetune_lora.sh
old mode 100644
new mode 100755
index fc0d9360e..3f3955b60
--- a/scripts/v1_5/finetune_lora.sh
+++ b/scripts/v1_5/finetune_lora.sh
@@ -1,14 +1,14 @@
 #!/bin/bash
 
+
 deepspeed llava/train/train_mem.py \
     --lora_enable True --lora_r 128 --lora_alpha 256 --mm_projector_lr 2e-5 \
     --deepspeed ./scripts/zero3.json \
-    --model_name_or_path vicuna-13b-v1.5 \
+    --model_name_or_path liuhaotian/llava-v1.5-7b \
     --version v1 \
     --data_path /home/akash/data/LLAVA_data.json \
     --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14-336 \
-    --pretrain_mm_mlp_adapter ./checkpoints/llava-v1.5-13b-pretrain/mm_projector.bin \
     --mm_projector_type mlp2x_gelu \
     --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
@@ -16,9 +16,9 @@ deepspeed llava/train/train_mem.py \
     --image_aspect_ratio pad \
     --group_by_modality_length True \
     --bf16 True \
-    --output_dir ./checkpoints/llava-v1.5-13b-lora \
+    --output_dir ./checkpoints/llava-v1.6-mistral-7b \
     --num_train_epochs 1 \
-    --per_device_train_batch_size 16 \
+    --per_device_train_batch_size 4 \
     --per_device_eval_batch_size 4 \
     --gradient_accumulation_steps 1 \
     --evaluation_strategy "no" \
diff --git a/scripts/v1_5/finetune_task.sh b/scripts/v1_5/finetune_task.sh
old mode 100644
new mode 100755
diff --git a/scripts/v1_5/finetune_task_lora.sh b/scripts/v1_5/finetune_task_lora.sh
old mode 100644
new mode 100755
diff --git a/scripts/v1_5/pretrain.sh b/scripts/v1_5/pretrain.sh
old mode 100644
new mode 100755
diff --git a/scripts/zero3.json b/scripts/zero3.json
index 6917317af..f3e1c4408 100755
--- a/scripts/zero3.json
+++ b/scripts/zero3.json
@@ -24,5 +24,8 @@
         "stage3_max_live_parameters": 1e9,
         "stage3_max_reuse_distance": 1e9,
         "stage3_gather_16bit_weights_on_model_save": true
-    }
-}
\ No newline at end of file
+    },
+    "wandb": {
+    "enabled": false
+  }
+}

From 7d806f1d6772abf4014222fbb1ca277703a67af9 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Fri, 29 Mar 2024 15:24:21 +0000
Subject: [PATCH 18/26] changes made to the scripts/v1_5/finetune_task_lora.sh
 file

---
 scripts/v1_5/finetune_task_lora.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/v1_5/finetune_task_lora.sh b/scripts/v1_5/finetune_task_lora.sh
index f11303f29..050282d66 100755
--- a/scripts/v1_5/finetune_task_lora.sh
+++ b/scripts/v1_5/finetune_task_lora.sh
@@ -3,10 +3,10 @@
 deepspeed llava/train/train_mem.py \
     --lora_enable True --lora_r 128 --lora_alpha 256 --mm_projector_lr 2e-5 \
     --deepspeed ./scripts/zero3.json \
-    --model_name_or_path liuhaotian/llava-v1.5-13b \
+    --model_name_or_path liuhaotian/llava-v1.5-7b \
     --version v1 \
-    --data_path ./playground/data/llava_v1_5_mix665k.json \
-    --image_folder ./playground/data \
+    --data_path /home/akash/data/LLAVA_data.json \
+    --image_folder /home/akash/data/ \
     --vision_tower openai/clip-vit-large-patch14-336 \
     --mm_projector_type mlp2x_gelu \
     --mm_vision_select_layer -2 \
@@ -15,9 +15,9 @@ deepspeed llava/train/train_mem.py \
     --image_aspect_ratio pad \
     --group_by_modality_length True \
     --bf16 True \
-    --output_dir ./checkpoints/llava-v1.5-13b-task-lora \
+    --output_dir ./checkpoints/llava-v1.5-7b-task-lora \
     --num_train_epochs 1 \
-    --per_device_train_batch_size 16 \
+    --per_device_train_batch_size 4 \
     --per_device_eval_batch_size 4 \
     --gradient_accumulation_steps 1 \
     --evaluation_strategy "no" \
@@ -34,4 +34,4 @@ deepspeed llava/train/train_mem.py \
     --gradient_checkpointing True \
     --dataloader_num_workers 4 \
     --lazy_preprocess True \
-    --report_to wandb
+

From c53a30a8aa1de13b33267c50bf7dfe7673f72010 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Sun, 14 Apr 2024 05:12:50 +0000
Subject: [PATCH 19/26] Dummy change in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5e2509592..a9c1c4dc9 100755
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # 🌋 LLaVA: Large Language and Vision Assistant
 
-*Visual instruction tuning towards large language and vision models with GPT-4 level capabilities.*
+*Visual instruction tuning towards large language and vision models with GPT-4 level capabilities*
 
 [📢 [LLaVA-NeXT Blog](https://llava-vl.github.io/blog/2024-01-30-llava-next/)] [[Project Page](https://llava-vl.github.io/)] [[Demo](https://llava.hliu.cc/)]  [[Data](https://github.com/haotian-liu/LLaVA/blob/main/docs/Data.md)] [[Model Zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)]
 

From 98d97dc671d43f98a875fd92c8258d8e4c3f317f Mon Sep 17 00:00:00 2001
From: Akds_2000 <44442562+akash-das2000@users.noreply.github.com>
Date: Mon, 15 Apr 2024 19:13:11 +0530
Subject: [PATCH 20/26] Update finetune_lora.sh

Reverted the finetune_lora.sh file to og
---
 scripts/finetune_lora.sh | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/scripts/finetune_lora.sh b/scripts/finetune_lora.sh
index ea1e11b75..fc02e09d7 100755
--- a/scripts/finetune_lora.sh
+++ b/scripts/finetune_lora.sh
@@ -10,26 +10,26 @@
 ################## VICUNA ##################
 
 ################## LLaMA-2 ##################
-PROMPT_VERSION="llava_llama_2"
-MODEL_VERSION="llama-2-7b-chat"
+# PROMPT_VERSION="llava_llama_2"
+# MODEL_VERSION="llama-2-7b-chat"
 ################## LLaMA-2 ##################
 
 deepspeed llava/train/train_mem.py \
     --deepspeed ./scripts/zero2.json \
     --lora_enable True \
-    --model_name_or_path /home/akash/llava-v1.5-7b \
+    --model_name_or_path ./checkpoints/$MODEL_VERSION \
     --version $PROMPT_VERSION \
-    --data_path /home/akash/data/LLAVA_data.json \
-    --image_folder /home/akash/data/ \
+    --data_path ./playground/data/llava_instruct_80k.json \
+    --image_folder /path/to/coco/train2017 \
     --vision_tower openai/clip-vit-large-patch14 \
-    --pretrain_mm_mlp_adapter /home/akash/llava-v1.5-7b\mm_projector.bin \
+    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
     --mm_vision_select_layer -2 \
     --mm_use_im_start_end False \
     --mm_use_im_patch_token False \
     --bf16 True \
-    --output_dir /home/akash/llava-$MODEL_VERSION-finetune_lora \
+    --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
     --num_train_epochs 1 \
-    --per_device_train_batch_size 4 \
+    --per_device_train_batch_size 16 \
     --per_device_eval_batch_size 4 \
     --gradient_accumulation_steps 1 \
     --evaluation_strategy "no" \
@@ -45,4 +45,5 @@ deepspeed llava/train/train_mem.py \
     --model_max_length 2048 \
     --gradient_checkpointing True \
     --lazy_preprocess True \
-    --dataloader_num_workers 4 
+    --dataloader_num_workers 4 \
+    --report_to wandb

From 8f8fa40bfa2e29964273a1ebaf12bd6d698bc160 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Thu, 18 Apr 2024 16:11:44 +0000
Subject: [PATCH 21/26] Changes for data prep

---
 .gitattributes                 |  6 +--
 .gitignore                     |  1 +
 data_prep/LLaVA_InitialJson.py | 40 ++++++++++++++++
 data_prep/LLaVA_dataSplit.py   | 87 ++++++++++++++++++++++++++++++++++
 4 files changed, 130 insertions(+), 4 deletions(-)
 create mode 100644 data_prep/LLaVA_InitialJson.py
 create mode 100644 data_prep/LLaVA_dataSplit.py

diff --git a/.gitattributes b/.gitattributes
index 5462cde72..78248ee02 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,9 +1,7 @@
 # https://git-scm.com/docs/gitattributes
-
 # Set the default behavior, in case people don't have core.autocrlf set.
 # https://git-scm.com/docs/gitattributes#_end_of_line_conversion
 * text=auto
-
 # common python attributes, taken from https://github.com/alexkaratarakis/gitattributes/blob/710900479a2bedeec7003d381719521ffbb18bf8/Python.gitattributes
 # Source files
 # ============
@@ -14,7 +12,6 @@
 *.pyx    text diff=python
 *.pyz    text diff=python
 *.pyi    text diff=python
-
 # Binary files
 # ============
 *.db     binary
@@ -24,6 +21,7 @@
 *.pyc    binary export-ignore
 *.pyo    binary export-ignore
 *.pyd    binary
-
 # Jupyter notebook
 *.ipynb  text eol=lf
+llava_data_r1.zip filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
index 6ff6a3dc8..d3bb920a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,4 @@ ckpts*
 
 # Demo
 serve_images/
+llava/
diff --git a/data_prep/LLaVA_InitialJson.py b/data_prep/LLaVA_InitialJson.py
new file mode 100644
index 000000000..cd5398bc7
--- /dev/null
+++ b/data_prep/LLaVA_InitialJson.py
@@ -0,0 +1,40 @@
+import os
+import json
+
+# Function to extract conversation data from .gui file
+def extract_conversations_from_gui(gui_file):
+    with open(gui_file, 'r') as file:
+        gui_text = file.read().strip()
+        return [{'from': 'human', 'value': '<image>\nWrite a code for the given UI'}, {'from': 'gpt', 'value': gui_text}]
+
+
+# Function to convert data to JSON format
+def convert_data_to_json(input_folder, output_folder):
+    data = []
+    for filename in os.listdir(input_folder):
+        if filename.endswith('.gui'):
+            sample_id = filename.split('.')[0]
+            image_path = "./Sketch2Code_og/" + os.path.relpath(os.path.join(input_folder, f"{sample_id}.png"), output_folder)
+            gui_path = os.path.join(input_folder, filename)
+            conversations = extract_conversations_from_gui(gui_path)
+            sample = {
+                'id': sample_id,
+                'image': image_path,
+                'conversations': conversations
+            }
+            data.append(sample)
+    
+    output_path = os.path.join(output_folder, 'samples.json')
+    with open(output_path, 'w') as json_file:
+        json.dump(data, json_file, indent=2)
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Convert data to JSON format")
+    parser.add_argument("input_folder", help="Input folder containing .gui files and corresponding images")
+    parser.add_argument("output_folder", help="Output folder to store the JSON file")
+    
+    args = parser.parse_args()
+    
+    convert_data_to_json(args.input_folder, args.output_folder)
diff --git a/data_prep/LLaVA_dataSplit.py b/data_prep/LLaVA_dataSplit.py
new file mode 100644
index 000000000..7813d0560
--- /dev/null
+++ b/data_prep/LLaVA_dataSplit.py
@@ -0,0 +1,87 @@
+import os
+import json
+import random
+import shutil
+import argparse
+
+def split_dataset(input_folder, output_folder, test_split=10, validation_split=10, seed=42):
+    # Create output folders for train, test, and validation sets
+    train_folder = os.path.join(output_folder, 'train')
+    test_folder = os.path.join(output_folder, 'test')
+    validation_folder = os.path.join(output_folder, 'validation')
+    split_json_folder = os.path.join(output_folder, 'split_json_files')
+
+    for folder in [train_folder, test_folder, validation_folder, split_json_folder]:
+        os.makedirs(folder, exist_ok=True)
+
+    # Load the JSON file containing the data
+    json_file_path = os.path.join(input_folder, 'samples.json')
+    with open(json_file_path, 'r') as json_file:
+        data = json.load(json_file)
+
+    # Calculate the number of samples for test and validation sets
+    num_samples = len(data)
+    num_test_samples = num_samples * test_split // 100
+    num_validation_samples = num_samples * validation_split // 100
+
+    # Set random seed for reproducibility
+    random.seed(seed)
+
+    # Randomly select indices for test set
+    test_indices = random.sample(range(num_samples), num_test_samples)
+
+    # Remove test indices from the list of all indices
+    remaining_indices = [i for i in range(num_samples) if i not in test_indices]
+
+    # Randomly select indices for validation set from remaining indices
+    validation_indices = random.sample(remaining_indices, num_validation_samples)
+
+    # The remaining indices are for the training set
+    train_indices = [i for i in remaining_indices if i not in validation_indices]
+
+    # Copy files to corresponding folders and update JSON files
+    for idx, sample in enumerate(data):
+        source_image = sample['image']
+        source_gui = os.path.join(input_folder, 'data', f"{sample['id']}.gui")
+        destination_folder = None
+        if idx in test_indices:
+            destination_folder = test_folder
+        elif idx in validation_indices:
+            destination_folder = validation_folder
+        else:
+            destination_folder = train_folder
+
+        # Copy files to destination folder
+        image_filename = os.path.basename(source_image)
+        gui_filename = f"{sample['id']}.gui"
+        destination_image = os.path.join(destination_folder, image_filename)
+        destination_gui = os.path.join(destination_folder, gui_filename)
+        shutil.copy(source_image, destination_image)
+        shutil.copy(source_gui, destination_gui)
+
+        # Update JSON data with relative paths
+        relative_image_path = os.path.relpath(destination_image, output_folder)
+        relative_gui_path = os.path.relpath(destination_gui, output_folder)
+        sample['image'] = "./" + relative_image_path
+        sample['gui'] = "./" + relative_gui_path
+
+    # Create updated JSON files for each split
+    splits = {'train': train_indices, 'test': test_indices, 'validation': validation_indices}
+    for split, indices in splits.items():
+        split_data = [data[i] for i in indices]
+        split_json_path = os.path.join(split_json_folder, f"{split}_json.json")
+        with open(split_json_path, 'w') as json_file:
+            json.dump(split_data, json_file, indent=2)
+
+    print("Dataset splitting and JSON file creation completed.")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Split dataset and generate JSON files")
+    parser.add_argument("input_folder", help="Path to the input folder containing the dataset")
+    parser.add_argument("output_folder", help="Path to the output folder to save the split dataset")
+    parser.add_argument("--test_split", type=int, default=10, help="Percentage of data to use for test (default: 10)")
+    parser.add_argument("--validation_split", type=int, default=10, help="Percentage of data to use for validation (default: 10)")
+    parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility (default: 42)")
+    args = parser.parse_args()
+
+    split_dataset(args.input_folder, args.output_folder, args.test_split, args.validation_split, args.seed)

From c824499d5e9925fcd782be163a14b783bcf25853 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Mon, 22 Apr 2024 08:34:30 +0000
Subject: [PATCH 22/26] Adding the readme file for the scripts to split the
 data

---
 data_prep/Readme.md | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 data_prep/Readme.md

diff --git a/data_prep/Readme.md b/data_prep/Readme.md
new file mode 100644
index 000000000..54bbb8319
--- /dev/null
+++ b/data_prep/Readme.md
@@ -0,0 +1,8 @@
+Use the following command line prompt to runs the .py files to generate the data splits
+
+
+python LLaVA_InitialJson.py ~/LLaVA/data_prep/Sketch2Code_og/data ~/LLaVA/data_prep/Sketch2Code_og
+
+python LLaVA_dataSplit.py ~/LLaVA/data_prep/Sketch2Code_og/ ~/splitted_data_verify/
+
+

From bb28667b6b6bc6bef52c04d4feaf82c5bb77b866 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Mon, 22 Apr 2024 08:36:57 +0000
Subject: [PATCH 23/26] Adding the readme file for the scripts to split the
 data

---
 data_prep/Readme.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/data_prep/Readme.md b/data_prep/Readme.md
index 54bbb8319..656f25f26 100644
--- a/data_prep/Readme.md
+++ b/data_prep/Readme.md
@@ -1,3 +1,10 @@
+The python file LLaVA_InitialJson.py will make the json data in the customised required format for LLaVA model to work on.
+The python file LLaVA_dataSplit.py will perform the data splitting.
+
+
+
+
+
 Use the following command line prompt to runs the .py files to generate the data splits
 
 

From 36527202ba76ebccb1d0a061f9cace4f5491b62d Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Wed, 29 May 2024 01:55:20 +0000
Subject: [PATCH 24/26] Adding wiki and updated finetune script with relative
 paths

---
 scripts/v1_5/finetune_task_lora.sh |  8 ++---
 wiki/navigation.md                 | 53 ++++++++++++++++++++++++++++++
 wiki/navigation.md.save            | 53 ++++++++++++++++++++++++++++++
 3 files changed, 110 insertions(+), 4 deletions(-)
 create mode 100644 wiki/navigation.md
 create mode 100644 wiki/navigation.md.save

diff --git a/scripts/v1_5/finetune_task_lora.sh b/scripts/v1_5/finetune_task_lora.sh
index 050282d66..ec9dfb593 100755
--- a/scripts/v1_5/finetune_task_lora.sh
+++ b/scripts/v1_5/finetune_task_lora.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
 
-deepspeed llava/train/train_mem.py \
+deepspeed "${PWD}/../../llava/train/train_mem.py" \
     --lora_enable True --lora_r 128 --lora_alpha 256 --mm_projector_lr 2e-5 \
-    --deepspeed ./scripts/zero3.json \
+    --deepspeed "${PWD}/../zero3.json" \
     --model_name_or_path liuhaotian/llava-v1.5-7b \
     --version v1 \
-    --data_path /home/akash/data/LLAVA_data.json \
-    --image_folder /home/akash/data/ \
+    --data_path "${PWD}/../../data_prep/splitted_data/split_json_files/train_json.json" \
+    --image_folder "${PWD}/../../data_prep/splitted_data/" \
     --vision_tower openai/clip-vit-large-patch14-336 \
     --mm_projector_type mlp2x_gelu \
     --mm_vision_select_layer -2 \
diff --git a/wiki/navigation.md b/wiki/navigation.md
new file mode 100644
index 000000000..e90e634dc
--- /dev/null
+++ b/wiki/navigation.md
@@ -0,0 +1,53 @@
+# Welcome to the LLaVA DSL Gen Project Wiki
+
+## Table of Contents
+- [Introduction](#introduction)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Contributing](#contributing)
+- [FAQ](#faq)
+
+## Introduction
+Welcome to the LLaVA DSL Gen Project! This project is designed to demonstrate how to install and navigate through this repository.
+
+## Installation
+
+### Step-by-Step LLaVA Installation Guide from the Github Repository
+
+1. **Clone this repository and navigate to the LLaVA folder:**
+    ```shell
+    git clone https://github.com/haotian-liu/LLaVA.git
+    cd LLaVA
+    ```
+
+2. **Install Package:**
+    ```shell
+    conda create -n llava python=3.10 -y
+    conda activate llava
+    pip install --upgrade pip  # enable PEP 660 support
+    pip install -e .
+    ```
+
+3. **Install additional packages for training cases:**
+    ```shell
+    pip install -e ".[train]"
+    pip install flash-attn --no-build-isolation
+    ```
+
+_For the purpose of running the current codes use the llava_new venev._
+
+### Finetuning Guide
+To start finetuning, run the _LLaVA/scripts/v1_5/finetune_task_lora.sh_ with the desired hyperparameter settings. 
+
+### Data Preperation Guide
+In addition to existing steps, the current repository also offers additional feature to split your data into train_eval_test split according to your desired split ratio. Follow the below steps for the same.
+
+1. _LLaVA_InitialJson.py_ will prepare your initial custom data inot the desired LLaVA dataset format and return a .json file useful for the next steps.
+   ```shell
+   Syntax: python file_name.py --input_folder_with_gui_and_png_files --output_folder_to_store_json_file
+   ```
+
+2. _LLaVA_dataSplit.py_ will split your data intp the required train_eval_test split. Default value will be 80_10_10.
+   ```shell
+   Syntax: python LLaVA_dataSpit.py --path_to_input_folder_containing_data_and_json_file --output_folder_to_save_the_splitted_dataset
+   ```
diff --git a/wiki/navigation.md.save b/wiki/navigation.md.save
new file mode 100644
index 000000000..169526eba
--- /dev/null
+++ b/wiki/navigation.md.save
@@ -0,0 +1,53 @@
+# Welcome to the LLaVA DSL Gen Project Wiki
+
+## Table of Contents
+- [Introduction](#introduction)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Contributing](#contributing)
+- [FAQ](#faq)
+
+## Introduction
+Welcome to the LLaVA DSL Gen Project! This project is designed to demonstrate how to install and navigate through this repository.
+
+## Installation
+
+### Step-by-Step LLaVA Installation Guide from the Github Repository
+
+1. **Clone this repository and navigate to the LLaVA folder:**
+    ```shell
+    git clone https://github.com/haotian-liu/LLaVA.git
+    cd LLaVA
+    ```
+
+2. **Install Package:**
+    ```shell
+    conda create -n llava python=3.10 -y
+    conda activate llava
+    pip install --upgrade pip  # enable PEP 660 support
+    pip install -e .
+    ```
+
+3. **Install additional packages for training cases:**
+    ```shell
+    pip install -e ".[train]"
+    pip install flash-attn --no-build-isolation
+    ```
+
+_For the purpose of running the current codes use the llava_new venev._
+
+### Finetuning Guide
+To start finetuning, run the _LLaVA/scripts/v1_5/finetune_task_lora.sh_ with the desired hyperparameter settings. 
+
+### Data Preperation Guide
+In addition to existing steps, the current repository also offers additional feature to split your data into train_eval_test split according to your desired split ratio. Follow the below steps for the same.
+1. _  
+
+
+
+## Usage
+After installation, you can start the project with:
+```bash
+npm start
+
+_

From a6b42a165b50072a0754ffd26fa172d1d99f067f Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Wed, 29 May 2024 02:00:02 +0000
Subject: [PATCH 25/26] Editied wiki/naviagtion.md

---
 wiki/navigation.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/wiki/navigation.md b/wiki/navigation.md
index e90e634dc..04dda3189 100644
--- a/wiki/navigation.md
+++ b/wiki/navigation.md
@@ -4,8 +4,6 @@
 - [Introduction](#introduction)
 - [Installation](#installation)
 - [Usage](#usage)
-- [Contributing](#contributing)
-- [FAQ](#faq)
 
 ## Introduction
 Welcome to the LLaVA DSL Gen Project! This project is designed to demonstrate how to install and navigate through this repository.
@@ -36,6 +34,7 @@ Welcome to the LLaVA DSL Gen Project! This project is designed to demonstrate ho
 
 _For the purpose of running the current codes use the llava_new venev._
 
+## Usage
 ### Finetuning Guide
 To start finetuning, run the _LLaVA/scripts/v1_5/finetune_task_lora.sh_ with the desired hyperparameter settings. 
 
@@ -44,7 +43,7 @@ In addition to existing steps, the current repository also offers additional fea
 
 1. _LLaVA_InitialJson.py_ will prepare your initial custom data inot the desired LLaVA dataset format and return a .json file useful for the next steps.
    ```shell
-   Syntax: python file_name.py --input_folder_with_gui_and_png_files --output_folder_to_store_json_file
+   Syntax: python LLaVA_IntialJson.py --input_folder_with_gui_and_png_files --output_folder_to_store_json_file
    ```
 
 2. _LLaVA_dataSplit.py_ will split your data intp the required train_eval_test split. Default value will be 80_10_10.

From 0c771a97894f4d7f3c5a4fd457b77f804503ba65 Mon Sep 17 00:00:00 2001
From: Akash Das <akash@2d0fe940-24d6-401a-b886-b7683809e9ae.tensordock.cloud>
Date: Mon, 10 Jun 2024 03:22:52 +0000
Subject: [PATCH 26/26] Adding the changes made to builder.py to tackle the
 offload issue

---
 llava/model/builder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llava/model/builder.py b/llava/model/builder.py
index e3d50829f..009a21103 100644
--- a/llava/model/builder.py
+++ b/llava/model/builder.py
@@ -24,7 +24,7 @@
 
 
 def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", use_flash_attn=False, **kwargs):
-    kwargs = {"device_map": device_map, **kwargs}
+    kwargs = {"device_map": device_map, "offload_folder": "offload", **kwargs}
 
     if device != "cuda":
         kwargs['device_map'] = {"": device}