admin/openVLA-Qwen2-0.5B
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
hanluzhi0429@gmail.com
2025-04-13 09:30:46 +08:00
commit 4f8972e88a
248 changed files with 34689 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/archived/convert_gqa_for_eval.py
+18
View File
@@ -0,0 +1,18 @@
import os
import json
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--src", type=str)
parser.add_argument("--dst", type=str)
args = parser.parse_args()
all_answers = []
for line_idx, line in enumerate(open(args.src)):
res = json.loads(line)
question_id = res["question_id"]
text = res["text"].rstrip(".").lower()
all_answers.append({"questionId": question_id, "prediction": text})
with open(args.dst, "w") as f:
json.dump(all_answers, f)
scripts/archived/convert_mmvet_for_eval.py
+18
View File
@@ -0,0 +1,18 @@
import os
import json
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--src", type=str)
parser.add_argument("--dst", type=str)
args = parser.parse_args()
cur_result = {}
for line in open(args.src):
data = json.loads(line)
qid = data["question_id"]
cur_result[f"v1_{qid}"] = data["text"]
with open(args.dst, "w") as f:
json.dump(cur_result, f, indent=2)
scripts/archived/convert_sqa_to_llava.py
+88
View File
@@ -0,0 +1,88 @@
import json
import os
import fire
import re
from convert_sqa_to_llava_base_prompt import build_prompt_chatbot
def convert_to_llava(base_dir, split, prompt_format="QCM-LEA"):
split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[split]
problems = json.load(open(os.path.join(base_dir, "problems.json")))
split_problems = build_prompt_chatbot(problems, split_indices, prompt_format, use_caption=False, is_test=False)
target_format = []
for prob_id, (input, output) in split_problems.items():
if input.startswith("Question: "):
input = input.replace("Question: ", "")
if output.startswith("Answer: "):
output = output.replace("Answer: ", "")
raw_prob_data = problems[prob_id]
if raw_prob_data["image"] is None:
target_format.append(
{
"id": prob_id,
"conversations": [
{"from": "human", "value": f"{input}"},
{"from": "gpt", "value": f"{output}"},
],
}
)
else:
target_format.append(
{
"id": prob_id,
"image": os.path.join(prob_id, raw_prob_data["image"]),
"conversations": [
{"from": "human", "value": f"{input}\n<image>"},
{"from": "gpt", "value": f"{output}"},
],
}
)
print(f"Number of samples: {len(target_format)}")
with open(os.path.join(base_dir, f"llava_{split}_{prompt_format}.json"), "w") as f:
json.dump(target_format, f, indent=2)
def convert_to_jsonl(base_dir, split, prompt_format="QCM-LEPA"):
split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[split]
problems = json.load(open(os.path.join(base_dir, "problems.json")))
split_problems = build_prompt_chatbot(problems, split_indices, prompt_format, use_caption=False, is_test=False)
writer = open(os.path.join(base_dir, f"scienceqa_{split}_{prompt_format}.jsonl"), "w")
for prob_id, (input, output) in split_problems.items():
if input.startswith("Question: "):
input = input.replace("Question: ", "")
if output.startswith("Answer: "):
output = output.replace("Answer: ", "")
raw_prob_data = problems[prob_id]
if raw_prob_data["image"] is None:
data = {
"id": prob_id,
"instruction": f"{input}",
"output": f"{output}",
}
else:
data = {
"id": prob_id,
"image": os.path.join(prob_id, raw_prob_data["image"]),
"instruction": f"{input}\n<image>",
"output": f"{output}",
}
writer.write(json.dumps(data) + "\n")
writer.close()
def main(task, **kwargs):
globals()[task](**kwargs)
if __name__ == "__main__":
fire.Fire(main)
scripts/archived/convert_sqa_to_llava_base_prompt.py
+298
View File
@@ -0,0 +1,298 @@
def get_question_text(problem):
question = problem["question"]
return question
def get_context_text(problem, use_caption):
txt_context = problem["hint"]
img_context = problem["caption"] if use_caption else ""
context = " ".join([txt_context, img_context]).strip()
if context == "":
context = "N/A"
return context
def get_choice_text(probelm, options):
choices = probelm["choices"]
choice_list = []
for i, c in enumerate(choices):
choice_list.append("({}) {}".format(options[i], c))
choice_txt = " ".join(choice_list)
# print(choice_txt)
return choice_txt
def get_answer(problem, options):
return options[problem["answer"]]
def get_lecture_text(problem):
# \\n: GPT-3 can generate the lecture with more tokens.
lecture = problem["lecture"].replace("\n", "\\n")
return lecture
def get_solution_text(problem):
# \\n: GPT-3 can generate the solution with more tokens
solution = problem["solution"].replace("\n", "\\n")
return solution
def create_one_example_chatbot(format, question, context, choice, answer, lecture, solution, test_example=True):
input_format, output_format = format.split("-")
## Inputs
if input_format == "CQM":
input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
elif input_format == "QCM":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
# upper bound experiment
elif input_format == "QCML":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
elif input_format == "QCME":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
elif input_format == "QCMLE":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
elif input_format == "QCLM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
elif input_format == "QCEM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
elif input_format == "QCLEM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
# Outputs
if test_example:
output = "Answer:"
elif output_format == "A":
output = f"Answer: The answer is {answer}."
elif output_format == "AL":
output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
elif output_format == "AE":
output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
elif output_format == "ALE":
output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
elif output_format == "AEL":
output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
elif output_format == "LA":
output = f"Answer: {lecture} The answer is {answer}."
elif output_format == "EA":
output = f"Answer: {solution} The answer is {answer}."
elif output_format == "LEA":
output = f"Answer: {lecture} {solution} The answer is {answer}."
elif output_format == "ELA":
output = f"Answer: {solution} {lecture} The answer is {answer}."
elif output_format == "LEPA":
output = ""
if len(lecture.strip()) > 0:
output += f"LECTURE: {lecture}\n"
if len(solution.strip()) > 0:
output += f"SOLUTION: {solution}\n"
output += "###\n"
output += f"ANSWER: {answer}."
input = input.replace(" ", " ").strip()
output = output.replace(" ", " ").strip()
if input.endswith("BECAUSE:"):
input = input.replace("BECAUSE:", "").strip()
if output.endswith("BECAUSE:"):
output = output.replace("BECAUSE:", "").strip()
return input, output
def create_one_example(format, question, context, choice, answer, lecture, solution, test_example=True):
input_format, output_format = format.split("-")
## Inputs
if input_format == "CQM":
input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
elif input_format == "QCM":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
# upper bound experiment
elif input_format == "QCML":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
elif input_format == "QCME":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
elif input_format == "QCMLE":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
elif input_format == "QCLM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
elif input_format == "QCEM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
elif input_format == "QCLEM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
# Outputs
if test_example:
output = "Answer:"
elif output_format == "A":
output = f"Answer: The answer is {answer}."
elif output_format == "AL":
output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
elif output_format == "AE":
output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
elif output_format == "ALE":
output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
elif output_format == "AEL":
output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
elif output_format == "LA":
output = f"Answer: {lecture} The answer is {answer}."
elif output_format == "EA":
output = f"Answer: {solution} The answer is {answer}."
elif output_format == "LEA":
output = f"Answer: {lecture} {solution} The answer is {answer}."
elif output_format == "ELA":
output = f"Answer: {solution} {lecture} The answer is {answer}."
text = input + output
text = text.replace(" ", " ").strip()
if text.endswith("BECAUSE:"):
text = text.replace("BECAUSE:", "").strip()
return text
def create_one_example_gpt4(format, question, context, choice, answer, lecture, solution, test_example=True):
input_format, output_format = format.split("-")
## Inputs
if input_format == "CQM":
input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
elif input_format == "QCM":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
# upper bound experiment
elif input_format == "QCML":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
elif input_format == "QCME":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
elif input_format == "QCMLE":
input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
elif input_format == "QCLM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
elif input_format == "QCEM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
elif input_format == "QCLEM":
input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
# Outputs
if test_example:
output = "Answer:"
elif output_format == "A":
output = f"Answer: The answer is {answer}."
elif output_format == "AL":
output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
elif output_format == "AE":
output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
elif output_format == "ALE":
output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
elif output_format == "AEL":
output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
elif output_format == "LA":
output = f"Answer: {lecture} The answer is {answer}."
elif output_format == "EA":
output = f"Answer: {solution} The answer is {answer}."
elif output_format == "LEA":
output = f"Answer: {lecture} {solution} The answer is {answer}."
elif output_format == "ELA":
output = f"Answer: {solution} {lecture} The answer is {answer}."
input = input.replace(" ", " ").strip()
output = output.replace(" ", " ").strip()
if output.endswith("BECAUSE:"):
output = output.replace("BECAUSE:", "").strip()
user_prompt = {"role": "user", "content": f"Can you explain {input}?"}
assistant_prompt = {"role": "assistant", "content": f"{output}"}
return user_prompt, assistant_prompt
def build_prompt_chatbot(problems, shot_qids, prompt_format, use_caption=False, options=["A", "B", "C", "D", "E"], is_test=False):
examples = {}
for qid in shot_qids:
question = get_question_text(problems[qid])
context = get_context_text(problems[qid], use_caption)
choice = get_choice_text(problems[qid], options)
answer = get_answer(problems[qid], options)
lecture = get_lecture_text(problems[qid]).replace("\\n", "\n")
solution = get_solution_text(problems[qid]).replace("\\n", "\n")
train_example = create_one_example_chatbot(prompt_format, question, context, choice, answer, lecture, solution, test_example=is_test)
examples[qid] = train_example
return examples
def build_prompt(problems, shot_qids, test_qid, args):
examples = []
# n-shot training examples
for qid in shot_qids:
question = get_question_text(problems[qid])
context = get_context_text(problems[qid], args.use_caption)
choice = get_choice_text(problems[qid], args.options)
answer = get_answer(problems[qid], args.options)
lecture = get_lecture_text(problems[qid])
solution = get_solution_text(problems[qid])
train_example = create_one_example(args.prompt_format, question, context, choice, answer, lecture, solution, test_example=False)
examples.append(train_example)
# test example
question = get_question_text(problems[test_qid])
context = get_context_text(problems[test_qid], args.use_caption)
choice = get_choice_text(problems[test_qid], args.options)
answer = get_answer(problems[test_qid], args.options)
lecture = get_lecture_text(problems[test_qid])
solution = get_solution_text(problems[test_qid])
test_example = create_one_example(args.prompt_format, question, context, choice, answer, lecture, solution, test_example=True)
examples.append(test_example)
# create the prompt input
prompt_input = "\n\n".join(examples)
return prompt_input
def build_prompt_gpt4(problems, shot_qids, test_qid, args):
prompt_array = [{"role": "system", "content": "You are a helpful assistant."}]
# n-shot training examples
for qid in shot_qids:
question = get_question_text(problems[qid])
context = get_context_text(problems[qid], args.use_caption)
choice = get_choice_text(problems[qid], args.options)
answer = get_answer(problems[qid], args.options)
lecture = get_lecture_text(problems[qid])
solution = get_solution_text(problems[qid])
user_prompt, assistant_prompt = create_one_example_gpt4(args.prompt_format, question, context, choice, answer, lecture, solution, test_example=False)
prompt_array.append(user_prompt)
prompt_array.append(assistant_prompt)
# test example
question = get_question_text(problems[test_qid])
context = get_context_text(problems[test_qid], args.use_caption)
choice = get_choice_text(problems[test_qid], args.options)
answer = get_answer(problems[test_qid], args.options)
lecture = get_lecture_text(problems[test_qid])
solution = get_solution_text(problems[test_qid])
user_prompt, assistant_prompt = create_one_example_gpt4(args.prompt_format, question, context, choice, answer, lecture, solution, test_example=True)
prompt_array.append(user_prompt)
prompt_array.append(assistant_prompt)
return prompt_array
scripts/archived/convert_vizwiz_for_submission.py
+45
View File
@@ -0,0 +1,45 @@
import os
import argparse
import json
from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--annotation-file", type=str, required=True)
parser.add_argument("--result-file", type=str, required=True)
parser.add_argument("--result-upload-file", type=str, required=True)
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
os.makedirs(os.path.dirname(args.result_upload_file), exist_ok=True)
results = []
error_line = 0
for line_idx, line in enumerate(open(args.result_file)):
try:
results.append(json.loads(line))
except:
error_line += 1
results = {x["question_id"]: x["text"] for x in results}
test_split = [json.loads(line) for line in open(args.annotation_file)]
split_ids = set([x["question_id"] for x in test_split])
print(f"total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}")
all_answers = []
answer_processor = EvalAIAnswerProcessor()
for x in test_split:
# import pdb; pdb.set_trace()
assert x["question_id"] in results, print(x)
all_answers.append({"image": x["image"], "answer": answer_processor(results[x["question_id"]])})
with open(args.result_upload_file, "w") as f:
json.dump(all_answers, f)
scripts/archived/convert_vqav2_for_submission.py
+50
View File
@@ -0,0 +1,50 @@
import os
import argparse
import json
from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--dir", type=str, default="./playground/data/eval/vqav2")
parser.add_argument("--ckpt", type=str, required=True)
parser.add_argument("--split", type=str, required=True)
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
src = os.path.join(args.dir, "answers", args.split, args.ckpt, "merge.jsonl")
test_split = os.path.join(args.dir, "llava_vqav2_mscoco_test2015.jsonl")
dst = os.path.join(args.dir, "answers_upload", args.split, f"{args.ckpt}.json")
os.makedirs(os.path.dirname(dst), exist_ok=True)
results = []
error_line = 0
for line_idx, line in enumerate(open(src)):
try:
results.append(json.loads(line))
except:
error_line += 1
results = {x["question_id"]: x["text"] for x in results}
test_split = [json.loads(line) for line in open(test_split)]
split_ids = set([x["question_id"] for x in test_split])
print(f"total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}")
all_answers = []
answer_processor = EvalAIAnswerProcessor()
for x in test_split:
if x["question_id"] not in results:
all_answers.append({"question_id": x["question_id"], "answer": ""})
else:
all_answers.append({"question_id": x["question_id"], "answer": answer_processor(results[x["question_id"]])})
with open(dst, "w") as f:
json.dump(all_answers, open(dst, "w"))
scripts/archived/data_info.py
+124
View File
@@ -0,0 +1,124 @@
import json
import os
from PIL import Image
from tqdm import tqdm
import matplotlib.pyplot as plt
import numpy as np
def load_data(json_path):
with open(json_path, "r") as f:
return json.load(f)
def filter_data(data):
# filtered_data = [item for item in data if "image" in item and "text" in item["image"]]
filtered_data = [item for item in data if "image" in item]
return filtered_data
from multiprocessing import Pool
import functools
def calculate_image_dimension(item, images_folder):
image_path = os.path.join(images_folder, item["image"])
try:
with Image.open(image_path) as img:
width, height = img.size
return width, height
except Exception as e:
print(f"Error opening {image_path}: {e}")
return None, None
def calculate_image_dimensions_multiprocess(filtered_data, images_folder, num_processes=256):
with Pool(num_processes) as p:
dimensions = list(tqdm(p.imap(functools.partial(calculate_image_dimension, images_folder=images_folder), filtered_data), total=len(filtered_data), desc="Calculating image dimensions"))
widths, heights = zip(*[dim for dim in dimensions if dim[0] is not None])
return list(widths), list(heights)
def tokenize(text):
return text.split()
def calculate_tokenized_lengths(data):
lengths = []
for item in tqdm(data, desc="Tokenizing conversations"):
for conversation in item["conversations"]:
tokenized_value = tokenize(conversation["value"])
lengths.append(len(tokenized_value))
return lengths
import argparse
def main():
parser = argparse.ArgumentParser(description="Process data for LLaVA_Next project.")
parser.add_argument("--json_path", type=str, help="Path to the JSON file containing data.")
parser.add_argument("--images_folder", type=str, default="/mnt/bn/vl-research/data/llava_data", help="Path to the folder containing images.")
args = parser.parse_args()
llava_instruct_name = args.json_path.split("/")[-1].replace(".json", "")
json_path = args.json_path
llava_instruct_name = os.path.basename(json_path).replace(".json", "")
images_folder = args.images_folder
data = load_data(json_path)
filtered_data = filter_data(data)
if len(filtered_data) != 0:
print(f"Total data items: {len(data)}, Filtered data items: {len(filtered_data)}")
widths, heights = calculate_image_dimensions_multiprocess(filtered_data, images_folder)
max_width = max(widths)
max_height = max(heights)
print(f"Max width: {max_width}, Max height: {max_height}")
tokenized_lengths = calculate_tokenized_lengths(data)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 12))
if len(filtered_data) != 0:
# Plot 2D histogram
if min(widths) == max(widths):
widths_bins = [min(widths), max(widths) + 1]
else:
widths_bins = np.arange(min(widths), max(widths) + 100, 100)
if min(heights) == max(heights):
heights_bins = [min(heights), max(heights) + 1]
else:
heights_bins = np.arange(min(heights), max(heights) + 100, 100)
h, xedges, yedges, image = ax1.hist2d(widths, heights, bins=[widths_bins, heights_bins], cmap=plt.cm.jet, density=True)
fig.colorbar(image, ax=ax1)
ax1.set_xlabel("Width")
ax1.set_ylabel("Height")
ax1.set_title(f"dist_{llava_instruct_name}_2d_w_h\nMax width: {max(widths)}, Max height: {max(heights)}", fontsize=10)
# Plot histogram
hist, bin_edges = np.histogram(tokenized_lengths, bins=np.arange(0, max(tokenized_lengths) + 10, 100))
bins = np.arange(0, max(tokenized_lengths) + 10, 100)
ax2.bar(bin_edges[:-1], hist, width=7, edgecolor="black", log=True)
# Display every nth label on the x-axis
n = 8 # Adjust this value to control the number of labels displayed
ticks = bins[::n]
tick_labels = [int(tick) for tick in ticks]
ax2.set_xticks(ticks)
ax2.set_xticklabels(tick_labels, rotation=90, fontsize=8)
ax2.set_xlim(min(bin_edges), max(bin_edges))
ax2.set_xlabel("Tokenized Length")
ax2.set_ylabel("Count (log scale)")
ax2.set_title(f"dist_{llava_instruct_name}_tokenized_length", fontsize=8)
plt.tight_layout()
plt.savefig(f"/mnt/bn/vl-research/workspace/boli01/projects/LLaVA_Next/notebooks/sft_data/dist_{llava_instruct_name}_combined.png")
print(f"Plots saved to /mnt/bn/vl-research/workspace/boli01/projects/LLaVA_Next/notebooks/sft_data/dist_{llava_instruct_name}_combined.png")
if __name__ == "__main__":
main()
scripts/archived/dpo_data_info.py
+67
View File
@@ -0,0 +1,67 @@
import json
import re
json_path = "/mnt/bn/vl-research/workspace/boli01/projects/sft_data_workspace/vlfeedback_80k.jsonl"
with open(json_path, "r") as f:
data = f.readlines()
data = [json.loads(d) for d in data]
def convert_format(original_data, dimension="Visual Faithfulness"):
converted_data = []
for item in original_data:
# Assuming the best response is the one with the highest helpfulness rating
best_completion = max(item["completions"], key=lambda x: int(x["annotations"]["Helpfulness"]["Rating"]))
best_response = best_completion["response"]
best_model = best_completion["model"]
if "†source" in best_response:
print(best_response)
# Regex pattern to match the pattern 【digit†source】
pattern = r"【\d+†source】"
# Replace the matched patterns with an empty string
cleaned_text = re.sub(pattern, "", best_response)
best_response = cleaned_text
print(f"*****************************************")
print(best_response)
# Assuming the worst response is the one with the lowest helpfulness rating
worst_completion = min(item["completions"], key=lambda x: int(x["annotations"]["Helpfulness"]["Rating"]))
worst_response = worst_completion["response"]
if "†source" in worst_response:
print(worst_response)
# Regex pattern to match the pattern ��digit†source】
pattern = r"【\d+†source】"
# Replace the matched patterns with an empty string
cleaned_text = re.sub(pattern, "", worst_response)
worst_response = cleaned_text
print(f"*****************************************")
print(worst_response)
# Extract scores
best_score = int(best_completion["annotations"][dimension]["Rating"])
worst_score = int(worst_completion["annotations"][dimension]["Rating"])
# Construct the new format
new_item = {
"id": item["id"],
"prompt": item["prompt"],
"answer": "",
"image": f"silkie_dpo/{item['id']}.jpg", # Assuming the video ID is the last part of the original ID
"chosen": best_response,
"rejected": worst_response,
"chosen_score": best_score,
"rejected_score": worst_score,
}
converted_data.append(new_item)
return converted_data
for dimension in ["Visual Faithfulness", "Helpfulness", "Ethical Considerations"]:
converted_data = convert_format(data, dimension=dimension)
with open(f"/mnt/bn/vl-research/data/llava_instruct/dpo_data/silkie_dpo_data_{dimension.replace(' ', '_').lower()}_{len(converted_data)}.json", "w") as f:
json.dump(converted_data, f, indent=4)
scripts/archived/entry_cmd.sh
+30
View File
@@ -0,0 +1,30 @@
python3 -m pip install --upgrade pip;
export http_proxy=http://sys-proxy-rd-relay.byted.org:8118;
export https_proxy=http://sys-proxy-rd-relay.byted.org:8118;
export HF_HOME=/mnt/bn/vl-research-boli01-cn/.cache/huggingface;
export HF_TOKEN="";
export HF_HUB_ENABLE_HF_TRANSFER="1";
cd /mnt/bn/vl-research-boli01-cn/projects/zzz/lmms-eval;
pip install -e .;
cd /mnt/bn/vl-research-boli01-cn/projects/zzz/LLaVA_Next;
pip install -e .;
python3 -m pip install ninja;
python3 -m pip install flash-attn --no-build-isolation;
bash /mnt/bn/vl-research-boli01-cn/projects/zzz/LLaVA_Next/cn_scripts/vicuna/internal0.6m_finetune_llava1.6mix_7b_v0.2_unfreeze.sh
accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
--model llava \
--model_args pretrained="/mnt/bn/vl-research-boli01-cn/projects/zzz/LLaVA_Next/internal_project_checkpoints/llavanext-lmsys_vicuna-7b-v1.5-clip-vit-large-patch14-336-mlp2x_gelu-pretrain_internal0.6m_vicuna_v1_finetune_llava1.6_datamix_unfreezeVIS_1e" \
--tasks ok_vqa,textcaps_val,mme_test,mmmu,cmmmu,coco2017_cap_val,vizwiz_vqa_val,ai2d,chartqa,pope \
--batch_size 1 \
--log_samples \
--log_samples_suffix debug \
--output_path ./logs/ \
--wandb_args 'project=llava-next-lmms-eval,job_type=eval';
scripts/archived/finetune.sh
+95
View File
@@ -0,0 +1,95 @@
#!/bin/bash
cd /mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install -U transformers
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Get the latest version of deepspeed from PyPI
latest_version=$(yolk -V deepspeed | cut -d ' ' -f 2)
# Check if the installed version is not the latest
# pip install deepspeed==0.12.2
if [ "$installed_version" != "$latest_version" ]; then
pip install deepspeed==0.12.2
fi
# Install flash-attn if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## VICUNA ##################
PROMPT_VERSION=v1
MODEL_VERSION="vicuna-7b-v1-5"
################## VICUNA ##################
################## project ##################
PROJECT_NAME="ds_llava-vicuna-7b-v1-5-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME="mixtral_instruct_158K_V1"
# wandb configure
export WANDB_API_KEY="03fc62d68025c9498cf6493432551badd7d4f953"
wandb login $WANDB_API_KEY
export WANDB_NAME=$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME
export WANDB_PROJECT=LLaVA_Mixtral
export WANDB_MODE=online
# wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data/coco/train2017 \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/$PROJECT_NAME/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_projector_type mlp2x_gelu \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava--$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME--finetune \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/finetune_1.5.sh
+99
View File
@@ -0,0 +1,99 @@
#!/bin/bash
dataset_name=$1
# Uncomment and set the following variables correspondingly to run this script:
cd /mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install -U transformers
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Get the latest version of deepspeed from PyPI
latest_version=$(yolk -V deepspeed | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install deepspeed==0.12.2
fi
# Install yolk3k if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## VICUNA ##################
PROMPT_VERSION=v1
MODEL_VERSION="vicuna-7b-v1-5"
################## VICUNA ##################
################## project ##################
PROJECT_NAME="ds_llava-vicuna-7b-v1-5-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME=$dataset_name
# wandb configure
export WANDB_API_KEY="03fc62d68025c9498cf6493432551badd7d4f953"
wandb login $WANDB_API_KEY
export WANDB_NAME=$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME
export WANDB_PROJECT=LLaVA_Mixtral
export WANDB_MODE=online
wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/$PROJECT_NAME/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_projector_type mlp2x_gelu \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava--$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME--finetune \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/finetune_full_schedule.sh
+46
View File
@@ -0,0 +1,46 @@
#!/bin/bash
# Uncomment and set the following variables correspondingly to run this script:
################## VICUNA ##################
# PROMPT_VERSION=v1
# MODEL_VERSION="vicuna-v1-3-7b"
################## VICUNA ##################
################## LLaMA-2 ##################
# PROMPT_VERSION="llava_llama_2"
# MODEL_VERSION="llama-2-7b-chat"
################## LLaMA-2 ##################
deepspeed llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/llava_instruct_158k.json \
--image_folder /path/to/coco/train2017 \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava-$MODEL_VERSION-finetune \
--num_train_epochs 3 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/finetune_lora.sh
+47
View File
@@ -0,0 +1,47 @@
#!/bin/bash
# Uncomment and set the following variables correspondingly to run this script:
################## VICUNA ##################
# PROMPT_VERSION=v1
# MODEL_VERSION="vicuna-v1-3-7b"
################## VICUNA ##################
################## LLaMA-2 ##################
# PROMPT_VERSION="llava_llama_2"
# MODEL_VERSION="llama-2-7b-chat"
################## LLaMA-2 ##################
deepspeed llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--lora_enable True \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/llava_instruct_80k.json \
--image_folder /path/to/coco/train2017 \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--lazy_preprocess True \
--dataloader_num_workers 16 \
--report_to wandb
scripts/archived/finetune_mixtral.sh
+96
View File
@@ -0,0 +1,96 @@
#!/bin/bash
cd /mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install -U transformers
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Get the latest version of deepspeed from PyPI
latest_version=$(yolk -V deepspeed | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install deepspeed==0.12.2
fi
# Install yolk3k if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## MISTRAL ##################
PROMPT_VERSION=mistral_instruct
MODEL_VERSION="Mistral-7B-Instruct-v0.2"
################## VICUNA ##################
################## project ##################
PROJECT_NAME="ds_llava-Mistral-7B-Instruct-v0.2-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME="mixtral_instruct_158K_V1"
# wandb configure
export WANDB_API_KEY="03fc62d68025c9498cf6493432551badd7d4f953"
wandb login $WANDB_API_KEY
export WANDB_NAME=$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME
export WANDB_PROJECT=LLaVA_Mixtral
export WANDB_MODE=online
wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data/coco/train2017 \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/$PROJECT_NAME/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_projector_type mlp2x_gelu \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava--$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME--finetune \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/finetune_mixtral_1.5.sh
+96
View File
@@ -0,0 +1,96 @@
#!/bin/bash
dataset_name=$1
cd /mnt/bn/vl-research/workspace/yhzhang/LLaVA
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install -U transformers
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Get the latest version of deepspeed from PyPI
# latest_version=$(yolk -V deepspeed | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "0.12.2" ]; then
pip install deepspeed==0.12.2
fi
# Install yolk3k if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## MISTRAL ##################
PROMPT_VERSION=mistral_instruct
MODEL_VERSION="Mistral-7B-Instruct-v0.2"
################## MISTRAL ##################
################## project ##################
PROJECT_NAME="ds_llava-Mistral-7B-Instruct-v0.2-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME=$dataset_name
# wandb configure
export WANDB_API_KEY="03fc62d68025c9498cf6493432551badd7d4f953"
wandb login $WANDB_API_KEY
export WANDB_NAME=$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME
export WANDB_PROJECT=LLaVA_Mixtral
export WANDB_MODE=online
wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/$PROJECT_NAME/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_projector_type mlp2x_gelu \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava--$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME--finetune \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True
# --report_to wandb
scripts/archived/finetune_mixtral_1.6_336px_anyres.sh
+100
View File
@@ -0,0 +1,100 @@
#!/bin/bash
dataset_name=$1
cd /mnt/bn/vl-research/workspace/boli01/projects/LLaVA_Next
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
pip install pydantic
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "4.36.2" ]; then
pip install transformers==4.36.2
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "0.12.2" ]; then
pip install deepspeed==0.12.2
fi
# Install flash-atten if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## MISTRAL ##################
PROMPT_VERSION=mistral_instruct
MODEL_VERSION="Mistral-7B-Instruct-v0.2"
################## MISTRAL ##################
################## project ##################
PROJECT_NAME="ds_llava-Mistral-7B-Instruct-v0.2-clip_large_336px-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME=$dataset_name
# wandb configure
export WANDB_API_KEY=e464cc107357c7b38e87f239bc3eb2ce5fb73c7c
export WANDB_PROJECT=llava
export WANDB_NAME=$PROJECT_NAME--$DATA_NAME--336px--anyres--sft
export WANDB_MODE=online
wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero3.json \
--model_name_or_path /mnt/bn/vl-research/workspace/project/2023/LLaVA/checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data \
--vision_tower openai/clip-vit-large-patch14-336 \
--pretrain_mm_mlp_adapter /mnt/bn/vl-research/workspace/project/2023/LLaVA/checkpoints/ds_llava-Mistral-7B-Instruct-v0.2-clip_large_336px-mlp2x_gelu-pretrain_blip558k_plain/mm_projector.bin \
--mm_projector_type mlp2x_gelu \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--group_by_modality_length True \
--unfreeze_mm_vision_tower True \
--mm_vision_tower_lr 2e-6 \
--image_aspect_ratio anyres \
--image_grid_pinpoints "[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]" \
--mm_patch_merge_type spatial_unpad \
--bf16 True \
--output_dir ./checkpoints/$PROJECT_NAME--$DATA_NAME--336px--anyres--sft \
--num_train_epochs 9 \
--per_device_train_batch_size 8 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "epoch" \
--save_steps 1500 \
--learning_rate 5e-6 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 4096 \
--gradient_checkpointing True \
--dataloader_num_workers 8 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/finetune_mixtral_1.6_336px_anyres_freeze_vision.sh
+97
View File
@@ -0,0 +1,97 @@
#!/bin/bash
dataset_name=$1
cd /mnt/bn/vl-research/workspace/yhzhang/LLaVA
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
pip install pydantic
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "4.36.2" ]; then
pip install transformers==4.36.2
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "0.12.2" ]; then
pip install deepspeed==0.12.2
fi
# Install flash-atten if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## MISTRAL ##################
PROMPT_VERSION=mistral_instruct
MODEL_VERSION="Mistral-7B-Instruct-v0.2"
################## MISTRAL ##################
################## project ##################
PROJECT_NAME="ds_llava-Mistral-7B-Instruct-v0.2-clip_large_336px-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME=$dataset_name
# wandb configure
export WANDB_API_KEY=e464cc107357c7b38e87f239bc3eb2ce5fb73c7c
export WANDB_PROJECT=llava
export WANDB_NAME=$PROJECT_NAME--$DATA_NAME--336px--unfreeze--anyres--sft
export WANDB_MODE=online
wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero3.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data \
--vision_tower openai/clip-vit-large-patch14-336 \
--pretrain_mm_mlp_adapter /mnt/bn/vl-research/workspace/project/2023/LLaVA/checkpoints/ds_llava-Mistral-7B-Instruct-v0.2-clip_large_336px-mlp2x_gelu-pretrain_blip558k_plain/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_projector_type mlp2x_gelu \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--group_by_modality_length True \
--image_aspect_ratio anyres \
--image_grid_pinpoints "[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]" \
--mm_patch_merge_type spatial_unpad \
--bf16 True \
--output_dir ./checkpoints/$PROJECT_NAME--$DATA_NAME--336px--anyres--unfreeze--sft \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
scripts/archived/finetune_mixtral_1.6_336px_anyres_lmms_eval.sh
+78
View File
@@ -0,0 +1,78 @@
#!/bin/bash
# set up wandb
export WANDB_API_KEY=a651c244635bc6f913ab654af3f0eebaecdc9381
export WANDB_ENTITY=llava-vl
export WANDB_PROJECT=llava-next
export PYTHONWARNINGS="ignore"
cd /mnt/bn/vl-research/workspace/boli01/projects/lmms-eval
pip install -e .
# set up llava dev env
cd /mnt/bn/vl-research/workspace/boli01/projects/LLaVA_Next
################## MISTRAL ##################
PROMPT_VERSION=mistral_instruct
MODEL_VERSION="Mistral-7B-Instruct-v0.2"
################## MISTRAL ##################
################## project ##################
PROJECT_NAME="ds_llava-Mistral-7B-Instruct-v0.2-clip_large_336px-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME='llava_caps20k_chartqa19k'
export WANDB_NAME=$PROJECT_NAME--$DATA_NAME--336px--anyres--sft
export WANDB_MODE=online
wandb online
CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" deepspeed --master_port 26000 --include localhost:0,1,2,3,4,5,6,7 llava/train/train_mem.py \
--deepspeed ./scripts/zero3_offload.json \
--model_name_or_path mistralai/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/llava_instruct/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/data/llava \
--vision_tower openai/clip-vit-large-patch14-336 \
--mm_projector_type mlp2x_gelu \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--group_by_modality_length True \
--unfreeze_mm_vision_tower True \
--mm_vision_tower_lr 2e-6 \
--image_aspect_ratio anyres \
--image_grid_pinpoints "[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]" \
--mm_patch_merge_type spatial_unpad \
--bf16 True \
--output_dir ./checkpoints/$PROJECT_NAME--llava1.6--336px--anyres--sft \
--num_train_epochs 1 \
--per_device_train_batch_size 8 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 1500 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 4096 \
--gradient_checkpointing True \
--dataloader_num_workers 32 \
--lazy_preprocess True \
--report_to wandb \
--run_name $WANDB_NAME
# starting here is the args for evaluation
--eval_num_processes 4 \
--task_names mme,docvqa_val \
--model_args pretrained=./checkpoints/$PROJECT_NAME--$DATA_NAME--336px--anyres--sft \
--limit 8 \
--batch_size 1 \
--log_samples \
--log_samples_suffix debug \
--output_path ./logs/
scripts/archived/finetune_mixtral_copy.sh
+96
View File
@@ -0,0 +1,96 @@
#!/bin/bash
cd /mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA
# Install yolk3k if not installed
if ! pip show yolk3k > /dev/null 2>&1; then
pip install yolk3k
fi
# Get the installed version of transformers
installed_version=$(pip show transformers | grep Version | cut -d ' ' -f 2)
# Get the latest version of transformers from PyPI
latest_version=$(yolk -V transformers | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install -U transformers
fi
# Get the installed version of deepspeed
installed_version=$(pip show deepspeed | grep Version | cut -d ' ' -f 2)
# Get the latest version of deepspeed from PyPI
latest_version=$(yolk -V deepspeed | cut -d ' ' -f 2)
# Check if the installed version is not the latest
if [ "$installed_version" != "$latest_version" ]; then
pip install deepspeed==0.12.2
fi
# Install yolk3k if not installed
if ! pip show flash-attn > /dev/null 2>&1; then
pip install flash-attn --no-build-isolation
fi
################## MISTRAL ##################
PROMPT_VERSION=mistral_instruct
MODEL_VERSION="Mistral-7B-Instruct-v0.2"
################## VICUNA ##################
################## project ##################
PROJECT_NAME="ds_llava-Mistral-7B-Instruct-v0.2-mlp2x_gelu-pretrain_blip558k_plain"
################## data ##################
DATA_NAME="llava_instruct_150k"
# wandb configure
export WANDB_API_KEY="03fc62d68025c9498cf6493432551badd7d4f953"
wandb login $WANDB_API_KEY
export WANDB_NAME=$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME
export WANDB_PROJECT=LLaVA_Mixtral
export WANDB_MODE=online
wandb online
deepspeed --master_port 26000 \
llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/$DATA_NAME.json \
--image_folder /mnt/bn/vl-research/workspace/boli01/data/playground/data/coco/train2017 \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/$PROJECT_NAME/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_projector_type mlp2x_gelu \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava--$PROJECT_NAME--$MODEL_VERSION--$DATA_NAME--finetune \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/finetune_qlora.sh
+48
View File
@@ -0,0 +1,48 @@
#!/bin/bash
# Uncomment and set the following variables correspondingly to run this script:
################## VICUNA ##################
# PROMPT_VERSION=v1
# MODEL_VERSION="vicuna-v1-3-7b"
################## VICUNA ##################
################## LLaMA-2 ##################
# PROMPT_VERSION="llava_llama_2"
# MODEL_VERSION="llama-2-7b-chat"
################## LLaMA-2 ##################
deepspeed llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--lora_enable True \
--bits 4 \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path ./playground/data/llava_instruct_80k.json \
--image_folder /path/to/coco/train2017 \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--lazy_preprocess True \
--dataloader_num_workers 16 \
--report_to wandb
scripts/archived/finetune_sqa.sh
+34
View File
@@ -0,0 +1,34 @@
#!/bin/bash
deepspeed llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path lmsys/vicuna-13b-v1.3 \
--version $PROMPT_VERSION \
--data_path /Data/ScienceQA/data/scienceqa/llava_train_QCM-LEA.json \
--image_folder /Data/ScienceQA/data/scienceqa/images/train \
--vision_tower openai/clip-vit-large-patch14 \
--pretrain_mm_mlp_adapter ./checkpoints/huggingface/liuhaotian/llava-pretrain-vicuna-13b-v1.3/mm_projector.bin \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava-vicuna-13b-v1.3-pretrain_lcs558k_plain-ScienceQA_QCM_LEA-12e \
--num_train_epochs 12 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/merge_lora_weights.py
+22
View File
@@ -0,0 +1,22 @@
import argparse
from llava.model.builder import load_pretrained_model
from llava.mm_utils import get_model_name_from_path
def merge_lora(args):
model_name = get_model_name_from_path(args.model_path)
tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, device_map="cpu")
model.save_pretrained(args.save_model_path)
tokenizer.save_pretrained(args.save_model_path)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", type=str, required=True)
parser.add_argument("--model-base", type=str, required=True)
parser.add_argument("--save-model-path", type=str, required=True)
args = parser.parse_args()
merge_lora(args)
scripts/archived/pretrain.sh
+43
View File
@@ -0,0 +1,43 @@
#!/bin/bash
# Uncomment and set the following variables correspondingly to run this script:
# MODEL_VERSION=vicuna-v1-3-7b
# MODEL_VERSION=llama-2-7b-chat
########### DO NOT CHANGE ###########
########### USE THIS FOR BOTH ###########
PROMPT_VERSION=plain
########### DO NOT CHANGE ###########
deepspeed llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/$MODEL_VERSION \
--version $PROMPT_VERSION \
--data_path /path/to/pretrain_data.json \
--image_folder /path/to/images \
--vision_tower openai/clip-vit-large-patch14 \
--tune_mm_mlp_adapter True \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--bf16 True \
--output_dir ./checkpoints/llava-$MODEL_VERSION-pretrain \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 24000 \
--learning_rate 2e-3 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 16 \
--lazy_preprocess True \
--report_to wandb
scripts/archived/quick_check.py
+57
View File
@@ -0,0 +1,57 @@
import json
import os
import argparse
from tqdm import tqdm
import yaml
def check_missing_images(json_path, images_folder):
data = json.load(open(json_path, "r"))
missing_data = []
for i, d in enumerate(tqdm(data)):
image = d["image"] if "image" in d else ""
if image != "":
path = os.path.join(images_folder, image)
if not os.path.exists(path):
print(f"Missing image: {path}")
missing_data.append(d)
return missing_data
def read_yaml_to_llava_data(yaml_path, images_folder):
print(f"Reading YAML file: {yaml_path}")
with open(yaml_path, "r") as f:
data = yaml.safe_load(f)
llava_json_paths = data["datasets"]
for item in llava_json_paths:
json_path = item["json_path"]
missing_data = check_missing_images(json_path, images_folder)
if len(missing_data) > 0:
print(f"Missing images in {json_path}:")
for d in missing_data:
print(d)
def direct_check_llava_data(json_path, images_folder):
missing_data = check_missing_images(json_path, images_folder)
if len(missing_data) > 0:
print(f"Missing images in {json_path}:")
for d in missing_data:
print(d)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Check for missing images in dataset.")
parser.add_argument("--yaml_path", type=str, default="", help="Path to the YAML file containing the dataset.")
parser.add_argument("--json_path", type=str, default="", help="Path to the JSON file containing the dataset.")
parser.add_argument("--images_folder", type=str, default="/mnt/bn/vl-research/data/llava_data", help="Path to the folder containing the images.")
args = parser.parse_args()
if args.json_path != "":
direct_check_llava_data(args.json_path, args.images_folder)
elif args.yaml_path != "":
read_yaml_to_llava_data(args.yaml_path, args.images_folder)
scripts/archived/sqa_eval_batch.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
CHUNKS=8
for IDX in {0..7}; do
CUDA_VISIBLE_DEVICES=$IDX python -m llava.eval.model_vqa_science \
--model-path liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3 \
--question-file ~/haotian/datasets/ScienceQA/data/scienceqa/llava_test_QCM-LEA.json \
--image-folder ~/haotian/datasets/ScienceQA/data/scienceqa/images/test \
--answers-file ./test_llava-13b-chunk$CHUNKS_$IDX.jsonl \
--num-chunks $CHUNKS \
--chunk-idx $IDX \
--conv-mode llava_v1 &
done
scripts/archived/sqa_eval_gather.sh
+18
View File
@@ -0,0 +1,18 @@
#!/bin/bash
CHUNKS=8
output_file="test_llava-13b.jsonl"
# Clear out the output file if it exists.
> "$output_file"
# Loop through the indices and concatenate each file.
for idx in $(seq 0 $((CHUNKS-1))); do
cat "./test_llava-13b-chunk${idx}.jsonl" >> "$output_file"
done
python llava/eval/eval_science_qa.py \
--base-dir ~/haotian/datasets/ScienceQA/data/scienceqa \
--result-file ./test_llava-13b.jsonl \
--output-file ./test_llava-13b_output.json \
--output-result ./test_llava-13b_result.json