diff --git a/playground/2d_hist.py b/playground/2d_hist.py deleted file mode 100644 index 3592348..0000000 --- a/playground/2d_hist.py +++ /dev/null @@ -1,132 +0,0 @@ -import json -import os -from PIL import Image -from tqdm import tqdm -import matplotlib.pyplot as plt -import numpy as np -from multiprocessing import Pool -import functools -import argparse - - -def load_data(json_path): - with open(json_path, "r") as f: - return json.load(f) - - -def filter_data(data): - filtered_data = [item for item in data if "image" in item] - return filtered_data - - -def calculate_image_dimension(image_path, images_folder): - full_path = os.path.join(images_folder, image_path) - try: - with Image.open(full_path) as img: - width, height = img.size - return width, height - except Exception as e: - print(f"Error opening {full_path}: {e}") - return None, None - - -def calculate_image_dimensions_multiprocess(filtered_data, images_folder, num_processes=256): - image_paths = [] - for item in filtered_data: - if isinstance(item["image"], list): - image_paths.extend(item["image"]) - else: - image_paths.append(item["image"]) - - with Pool(num_processes) as p: - dimensions = list( - tqdm( - p.imap(functools.partial(calculate_image_dimension, images_folder=images_folder), image_paths), - total=len(image_paths), - desc="Calculating image dimensions", - ) - ) - widths, heights = zip(*[dim for dim in dimensions if dim[0] is not None]) - return list(widths), list(heights) - - -def tokenize(text): - return text.split() - - -def calculate_tokenized_lengths(data): - lengths = [] - for item in tqdm(data, desc="Tokenizing conversations"): - for conversation in item["conversations"]: - tokenized_value = tokenize(conversation["value"]) - lengths.append(len(tokenized_value)) - return lengths - - -def main(): - parser = argparse.ArgumentParser(description="Process data for LLaVA_Next project.") - parser.add_argument( - "--json_path", - type=str, - help="Path to the JSON file containing data.", - default="/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/llava_ofa_DEMON-FULL.json", - ) - parser.add_argument( - "--images_folder", - type=str, - default="/mnt/bn/vl-research/data/llava_data", - help="Path to the folder containing images.", - ) - args = parser.parse_args() - - llava_instruct_name = os.path.basename(args.json_path).replace(".json", "") - images_folder = args.images_folder - - data = load_data(args.json_path) - filtered_data = filter_data(data) - - print(f"Total data items: {len(data)}, Filtered data items: {len(filtered_data)}") - widths, heights = calculate_image_dimensions_multiprocess(filtered_data, images_folder) - max_width, max_height = max(widths), max(heights) - print(f"Max width: {max_width}, Max height: {max_height}") - - tokenized_lengths = calculate_tokenized_lengths(filtered_data) - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 12)) - - # Plot 2D histogram - widths_bins = [min(widths), max(widths) + 1] if min(widths) == max(widths) else np.arange(min(widths), max(widths) + 100, 100) - heights_bins = [min(heights), max(heights) + 1] if min(heights) == max(heights) else np.arange(min(heights), max(heights) + 100, 100) - - h, xedges, yedges, image = ax1.hist2d(widths, heights, bins=[widths_bins, heights_bins], cmap=plt.cm.jet, density=True) - fig.colorbar(image, ax=ax1) - ax1.set_xlabel("Width") - ax1.set_ylabel("Height") - ax1.set_title( - f"dist_{llava_instruct_name}_2d_w_h\nMax width: {max(widths)}, Max height: {max(heights)}", - fontsize=10, - ) - - # Plot histogram - hist, bin_edges = np.histogram(tokenized_lengths, bins=np.arange(0, max(tokenized_lengths) + 10, 10)) - bins = np.arange(0, max(tokenized_lengths) + 10, 10) - ax2.bar(bin_edges[:-1], hist, width=7, edgecolor="black", log=True) - - # Display every nth label on the x-axis - n = 8 # Adjust this value to control the number of labels displayed - ticks = bins[::n] - tick_labels = [int(tick) for tick in ticks] - ax2.set_xticks(ticks) - ax2.set_xticklabels(tick_labels, rotation=90, fontsize=8) - - ax2.set_xlim(min(bin_edges), max(bin_edges)) - ax2.set_xlabel("Tokenized Length") - ax2.set_ylabel("Count (log scale)") - ax2.set_title(f"dist_{llava_instruct_name}_tokenized_length", fontsize=8) - - plt.tight_layout() - plt.savefig(f"./dist_{llava_instruct_name}_combined.png") - - -if __name__ == "__main__": - main() diff --git a/playground/data_checker.py b/playground/data_checker.py deleted file mode 100644 index 9d145ca..0000000 --- a/playground/data_checker.py +++ /dev/null @@ -1,364 +0,0 @@ -import json -import os -from tqdm import tqdm -from multiprocessing import Pool, cpu_count -import yaml - - -class DataProcessor: - def __init__(self, file_path, image_root, video_root): - self.file_path = file_path - self.image_root = image_root - self.data = None - self.video_root = video_root - self.load_data() - - def load_data(self): - if self.file_path.endswith(".json"): - with open(self.file_path, "r") as f: - self.data = json.load(f) - elif self.file_path.endswith(".yaml"): - with open(self.file_path, "r") as f: - self.data = yaml.safe_load(f) - elif self.file_path.endswith(".jsonl"): - with open(self.file_path, "r") as f: - self.data = [json.loads(line) for line in f.readlines()] - else: - raise ValueError("Unsupported file format") - - def load_json_data(self, json_path): - if json_path.endswith(".jsonl"): - cur_data_dict = [] - with open(json_path, "r") as json_file: - for line in json_file: - cur_data_dict.append(json.loads(line.strip())) - return cur_data_dict - elif json_path.endswith(".json"): - with open(json_path, "r") as f: - return json.load(f) - else: - raise ValueError("Unsupported file format") - - def check_image_existence(self, data): - if "image" in data: - if type(data["image"]) == list: - images = data["image"] - else: - images = [data["image"]] - - for image in images: - full_image_path = os.path.join(self.image_root, image) - if not os.path.exists(full_image_path): - print(f"WARNING!!! {full_image_path} not exists !!!") - - if "video" in data: - full_video_path = os.path.join(self.video_root, data["video"]) - if not os.path.exists(full_video_path): - print(f"WARNING!!! {full_video_path} not exists !!!") - - # if data["conversations"][0]["value"].count("") > 1: - # print(f"WARNING!!! {data['conversations'][0]['value']} has more than one !!!") - - def check_item_structure(self, item): - if not all(key in item for key in ["conversations"]): - print(f"WARNING!!! Item {item.get('id', 'unknown')} is missing required fields!") - return False - - conversations = item["conversations"] - if not isinstance(conversations, list) or len(conversations) < 2 or len(conversations) % 2 != 0: - print(f"WARNING!!! Item {item['id']} has invalid conversations structure!") - return False - - for i, conv in enumerate(conversations): - if not all(key in conv for key in ["from", "value"]): - print(f"WARNING!!! Item {item['id']} has invalid conversation format!") - return False - - expected_from = "human" if i % 2 == 0 else "gpt" - if conv["from"] != expected_from: - print(f"WARNING!!! Item {item['id']} has incorrect conversation order!") - return False - - return True - - def check_image_and_structure(self, item): - if not self.check_item_structure(item): - return - - # self.check_image_existence(item) - - def process_images(self): - if isinstance(self.data, list): - args = [d for d in self.data] - with Pool(processes=cpu_count()) as pool: - list(tqdm(pool.imap(self.check_image_and_structure, args), total=len(self.data))) - elif isinstance(self.data, dict): - for d in self.data["datasets"]: - dd_json_path = d["json_path"] - data = self.load_json_data(dd_json_path) - args = [d for d in data] - with Pool(processes=cpu_count()) as pool: - list(tqdm(pool.imap(self.check_image_and_structure, args), total=len(data), desc=f"Processing {dd_json_path}")) - - def count_items(self): - if isinstance(self.data, list): # Assuming JSON data loaded directly - return len(self.data) - elif isinstance(self.data, dict): # Assuming YAML data loaded - total_items_count = 0 - for d in self.data["datasets"]: - dd_json_path = d["json_path"] - data = self.load_json_data(dd_json_path) - current_items_count = len(data) - - sampling_strategy = d["sampling_strategy"] - try: - if sampling_strategy != "all": - percentage = float(sampling_strategy.split(":")[-1].replace("%", "")) / 100.0 - else: - percentage = 1.0 - except Exception as e: - print(f"Error: {e}") - percentage = 1.0 - - sampling_count = int(current_items_count * percentage) - total_items_count += sampling_count - print(f"{dd_json_path}: {sampling_count}") - return total_items_count - - def stat_data(self): - if isinstance(self.data, dict): - cur_lens_list = [] - single_image_count = 0 - multiple_image_count = 0 - video_count = 0 - total_count = 0 - text_count = 0 - max_tokens_item = None - max_tokens = 0 - - for d in self.data["datasets"]: - dd_json_path = d["json_path"] - data = self.load_json_data(dd_json_path) - sampling_strategy = d["sampling_strategy"] - - try: - if sampling_strategy != "all": - percentage = float(sampling_strategy.split(":")[-1].replace("%", "")) / 100.0 - else: - percentage = 1.0 - except Exception as e: - print(f"Error parsing sampling strategy: {e}") - percentage = 1.0 - - sampled_count = int(len(data) * percentage) - print(f"{dd_json_path}: {sampled_count} (sampled from {len(data)})") - - for item in data[:sampled_count]: - conversations = item["conversations"] - cur_len = sum([len(conv["value"].split()) for conv in conversations]) - cur_lens_list.append(cur_len) - - if cur_len > max_tokens: - max_tokens = cur_len - max_tokens_item = item - - total_count += 1 - if "image" in item: - if isinstance(item["image"], list): - if len(item["image"]) > 1: - multiple_image_count += 1 - else: - single_image_count += 1 - else: - single_image_count += 1 - elif "video" in item: - video_count += 1 - else: - text_count += 1 - - print(f"Max length: {max(cur_lens_list)}, Min length: {min(cur_lens_list)}, Average length: {sum(cur_lens_list) / len(cur_lens_list)}") - print(f"Total items: {total_count}") - print(f"Text items: {text_count} ({text_count/total_count*100:.2f}%)") - print(f"Single image items: {single_image_count} ({single_image_count/total_count*100:.2f}%)") - print(f"Multiple image items: {multiple_image_count} ({multiple_image_count/total_count*100:.2f}%)") - print(f"Video items: {video_count} ({video_count/total_count*100:.2f}%)") - - print("\nItem with the largest number of tokens:") - print(f"Token count: {max_tokens}") - print("Item content:") - print(json.dumps(max_tokens_item, indent=2)) - - def filter_data(self): - if isinstance(self.data, dict): - for d in self.data["datasets"]: - dd_json_path = d["json_path"] - print(f"Processing {dd_json_path}") - data = self.load_json_data(dd_json_path) - - filtered_data = [] - mismatch_data = [] - mismatch_flag = False - for item in data: - try: - if "image" in item: - num_image = len(item["image"]) if isinstance(item["image"], list) else 1 - else: - num_image = 0 - - if "video" in item: - num_video = len(item["video"]) if isinstance(item["video"], list) else 1 - else: - num_video = 0 - - num_visuals = num_image + num_video - conv_text = "" - for conv in item["conversations"]: - conv_text += conv["value"] - - num_img_token_appearance = conv_text.count("") - if len(conv_text) == 0: - print(f"Conversation text is empty for {item}") - - if num_img_token_appearance == num_visuals or num_img_token_appearance < num_visuals and len(conv_text) > 0: - filtered_data.append(item) - elif num_img_token_appearance > num_visuals: - item["num_img_token_appearance"] = num_img_token_appearance - item["num_visuals"] = num_visuals - mismatch_data.append(item) - - if not mismatch_flag: - print(f"Data mismatch for {item}") - - mismatch_flag = True - except Exception as e: - print(f"Error: {e}") - print() - - if mismatch_flag: - print(f"Data mismatch for {dd_json_path}") - - if len(filtered_data) < len(data): - saving_dd_json_path = dd_json_path.replace(".jsonl", f"fltd_{len(filtered_data)}.json").replace(".json", f"fltd_{len(filtered_data)}.json") - with open(saving_dd_json_path, "w") as f: - json.dump(filtered_data, f, indent=2) - print(f"Filtered data count: {len(filtered_data)}") - else: - pass - - def stat_and_filter_data(self, threshold): - if isinstance(self.data, dict): - cur_lens_list = [] - single_image_count = 0 - multiple_image_count = 0 - video_count = 0 - total_count = 0 - text_count = 0 - - for d in self.data["datasets"]: - dd_json_path = d["json_path"] - data = self.load_json_data(dd_json_path) - sampling_strategy = d["sampling_strategy"] - filtered_data = [] - - try: - if sampling_strategy != "all": - percentage = float(sampling_strategy.split(":")[-1].replace("%", "")) / 100.0 - else: - percentage = 1.0 - except Exception as e: - print(f"Error parsing sampling strategy: {e}") - percentage = 1.0 - - sampled_count = int(len(data) * percentage) - print(f"{dd_json_path}: {sampled_count} (sampled from {len(data)})") - - save_flag = False - for item in data: - total_count += 1 - conversations = item["conversations"] - filtered_conversations = [] - current_token_count = 0 - - for i in range(0, len(conversations), 2): - if i + 1 < len(conversations): - human_conv = conversations[i] - gpt_conv = conversations[i + 1] - pair_tokens = len(human_conv["value"].split()) + len(gpt_conv["value"].split()) - - if current_token_count + pair_tokens <= threshold: - filtered_conversations.extend([human_conv, gpt_conv]) - current_token_count += pair_tokens - else: - save_flag = True - break - - if filtered_conversations: - item["conversations"] = filtered_conversations - cur_len = sum([len(conv["value"].split()) for conv in filtered_conversations]) - cur_lens_list.append(cur_len) - filtered_data.append(item) - - if "image" in item: - if isinstance(item["image"], list): - if len(item["image"]) > 1: - multiple_image_count += 1 - else: - single_image_count += 1 - else: - single_image_count += 1 - elif "video" in item: - video_count += 1 - else: - text_count += 1 - - # Save filtered data for each dataset - if filtered_data and save_flag: - if dd_json_path.endswith(".jsonl"): - output_file = dd_json_path.replace(".jsonl", f"_filtered_{threshold}tokens_{len(filtered_data)}.jsonl") - with open(output_file, "w") as f: - for item in filtered_data: - f.write(json.dumps(item) + "\n") - else: - output_file = dd_json_path.replace(".json", f"_filtered_{threshold}tokens_{len(filtered_data)}.json") - with open(output_file, "w") as f: - json.dump(filtered_data, f, indent=2) - print(f"Filtered data for {dd_json_path} saved to: {output_file}") - - print(f"Max length: {max(cur_lens_list)}, Min length: {min(cur_lens_list)}, Average length: {sum(cur_lens_list) / len(cur_lens_list)}") - print(f"Total items: {total_count}") - print(f"Text items: {text_count} ({text_count/total_count*100:.2f}%)") - print(f"Single image items: {single_image_count} ({single_image_count/total_count*100:.2f}%)") - print(f"Multiple image items: {multiple_image_count} ({multiple_image_count/total_count*100:.2f}%)") - print(f"Video items: {video_count} ({video_count/total_count*100:.2f}%)") - - -def main(file_path, image_root, operation, video_root, threshold=None): - processor = DataProcessor(file_path, image_root, video_root) - if operation == "check": - processor.process_images() - elif operation == "count": - total_items = processor.count_items() - print(f"Total items: {total_items}") - elif operation == "filter": - processor.filter_data() - elif operation == "stat": - processor.stat_data() - elif operation == "stat_and_filter": - if threshold is None: - raise ValueError("Threshold must be provided for stat_and_filter operation") - processor.stat_and_filter_data(threshold) - else: - raise ValueError("Unsupported operation") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--file_path", type=str, default="/mnt/bn/vl-research/workspace/boli01/projects/LLaVA_Next/scripts/i18n/scale_llms/next_continual.yaml") - parser.add_argument("--image_root", type=str, default="/mnt/bn/vl-research/data/llava_data") - parser.add_argument("--video_root", type=str, default="/mnt/bn/vl-research/data/llava_video") - parser.add_argument("--operation", type=str, default="filter") - parser.add_argument("--threshold", type=int, default=None, help="Threshold for stat_and_filter operation") - args = parser.parse_args() - main(args.file_path, args.image_root, args.operation, args.video_root, args.threshold) diff --git a/playground/demo/video_demo.py b/playground/demo/video_demo.py deleted file mode 100644 index d93f658..0000000 --- a/playground/demo/video_demo.py +++ /dev/null @@ -1,335 +0,0 @@ -import argparse -import torch - -from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN -from llava.conversation import conv_templates, SeparatorStyle -from llava.model.builder import load_pretrained_model -from llava.utils import disable_torch_init -from llava.mm_utils import process_anyres_image,tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria - -import json -import os -import math -from tqdm import tqdm -from decord import VideoReader, cpu - -from transformers import AutoConfig - -import cv2 -import base64 -import openai - -from PIL import Image - - - -import numpy as np - -def split_list(lst, n): - """Split a list into n (roughly) equal-sized chunks""" - chunk_size = math.ceil(len(lst) / n) # integer division - return [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] - - -def get_chunk(lst, n, k): - chunks = split_list(lst, n) - return chunks[k] - - -def parse_args(): - """ - Parse command-line arguments. - """ - parser = argparse.ArgumentParser() - - # Define the command-line arguments - parser.add_argument("--video_path", help="Path to the video files.", required=True) - parser.add_argument("--output_dir", help="Directory to save the model results JSON.", required=True) - parser.add_argument("--output_name", help="Name of the file for storing results JSON.", required=True) - parser.add_argument("--model-path", type=str, default="facebook/opt-350m") - parser.add_argument("--model-base", type=str, default=None) - parser.add_argument("--conv-mode", type=str, default=None) - parser.add_argument("--chunk-idx", type=int, default=0) - parser.add_argument("--mm_resampler_type", type=str, default="spatial_pool") - parser.add_argument("--mm_spatial_pool_stride", type=int, default=4) - parser.add_argument("--mm_spatial_pool_out_channels", type=int, default=1024) - parser.add_argument("--mm_spatial_pool_mode", type=str, default="average") - parser.add_argument("--image_aspect_ratio", type=str, default="anyres") - parser.add_argument("--image_grid_pinpoints", type=str, default="[(224, 448), (224, 672), (224, 896), (448, 448), (448, 224), (672, 224), (896, 224)]") - parser.add_argument("--mm_patch_merge_type", type=str, default="spatial_unpad") - parser.add_argument("--overwrite", type=lambda x: (str(x).lower() == 'true'), default=True) - parser.add_argument("--for_get_frames_num", type=int, default=4) - parser.add_argument("--load_8bit", type=lambda x: (str(x).lower() == 'true'), default=False) - parser.add_argument("--prompt", type=str, default=None) - parser.add_argument("--api_key", type=str, help="OpenAI API key") - parser.add_argument("--mm_newline_position", type=str, default="no_token") - parser.add_argument("--force_sample", type=lambda x: (str(x).lower() == 'true'), default=False) - parser.add_argument("--add_time_instruction", type=str, default=False) - return parser.parse_args() - -def load_video(video_path,args): - if args.for_get_frames_num == 0: - return np.zeros((1, 336, 336, 3)) - vr = VideoReader(video_path, ctx=cpu(0),num_threads=1) - total_frame_num = len(vr) - video_time = total_frame_num / vr.get_avg_fps() - fps = round(vr.get_avg_fps()) - frame_idx = [i for i in range(0, len(vr), fps)] - frame_time = [i/fps for i in frame_idx] - if len(frame_idx) > args.for_get_frames_num or args.force_sample: - sample_fps = args.for_get_frames_num - uniform_sampled_frames = np.linspace(0, total_frame_num - 1, sample_fps, dtype=int) - frame_idx = uniform_sampled_frames.tolist() - frame_time = [i/vr.get_avg_fps() for i in frame_idx] - frame_time = ",".join([f"{i:.2f}s" for i in frame_time]) - spare_frames = vr.get_batch(frame_idx).asnumpy() - # import pdb;pdb.set_trace() - - return spare_frames,frame_time,video_time - - - - -def load_video_base64(path): - video = cv2.VideoCapture(path) - - base64Frames = [] - while video.isOpened(): - success, frame = video.read() - if not success: - break - _, buffer = cv2.imencode(".jpg", frame) - base64Frames.append(base64.b64encode(buffer).decode("utf-8")) - - video.release() - # print(len(base64Frames), "frames read.") - return base64Frames - - -def run_inference(args): - """ - Run inference on ActivityNet QA DataSet using the Video-ChatGPT model. - - Args: - args: Command-line arguments. - """ - # Initialize the model - if "gpt4v" != args.model_path: - model_name = get_model_name_from_path(args.model_path) - # Set model configuration parameters if they exist - if args.overwrite == True: - overwrite_config = {} - overwrite_config["mm_spatial_pool_mode"] = args.mm_spatial_pool_mode - overwrite_config["mm_spatial_pool_stride"] = args.mm_spatial_pool_stride - overwrite_config["mm_newline_position"] = args.mm_newline_position - - cfg_pretrained = AutoConfig.from_pretrained(args.model_path) - - # import pdb;pdb.set_trace() - if "qwen" not in args.model_path.lower(): - if "224" in cfg_pretrained.mm_vision_tower: - # suppose the length of text tokens is around 1000, from bo's report - least_token_number = args.for_get_frames_num*(16//args.mm_spatial_pool_stride)**2 + 1000 - else: - least_token_number = args.for_get_frames_num*(24//args.mm_spatial_pool_stride)**2 + 1000 - - scaling_factor = math.ceil(least_token_number/4096) - if scaling_factor >= 2: - if "vicuna" in cfg_pretrained._name_or_path.lower(): - print(float(scaling_factor)) - overwrite_config["rope_scaling"] = {"factor": float(scaling_factor), "type": "linear"} - overwrite_config["max_sequence_length"] = 4096 * scaling_factor - overwrite_config["tokenizer_model_max_length"] = 4096 * scaling_factor - - tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, load_8bit=args.load_8bit, overwrite_config=overwrite_config) - else: - tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name) - else: - pass - - # import pdb;pdb.set_trace() - if getattr(model.config, "force_sample", None) is not None: - args.force_sample = model.config.force_sample - else: - args.force_sample = False - - # import pdb;pdb.set_trace() - - if getattr(model.config, "add_time_instruction", None) is not None: - args.add_time_instruction = model.config.add_time_instruction - else: - args.add_time_instruction = False - - # Create the output directory if it doesn't exist - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - output_name = args.output_name - answers_file = os.path.join(args.output_dir, f"{output_name}.json") - ans_file = open(answers_file, "w") - - video_path = args.video_path - - all_video_pathes = [] - - # Check if the video_path is a directory or a file - if os.path.isdir(video_path): - # If it's a directory, loop over all files in the directory - for filename in os.listdir(video_path): - # Load the video file - cur_video_path = os.path.join(video_path, f"{filename}") - all_video_pathes.append(os.path.join(video_path, cur_video_path)) - else: - # If it's a file, just process the video - all_video_pathes.append(video_path) - - # import pdb;pdb.set_trace() - for video_path in all_video_pathes: - - sample_set = {} - question = args.prompt - sample_set["Q"] = question - sample_set["video_name"] = video_path - - - # Check if the video exists - if os.path.exists(video_path): - if "gpt4v" != args.model_path: - video,frame_time,video_time = load_video(video_path, args) - video = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].half().cuda() - video = [video] - else: - spare_frames,frame_time,video_time = load_video_base64(video_path) - interval = int(len(video) / args.for_get_frames_num) - - # try: - # Run inference on the video and add the output to the list - if "gpt4v" != args.model_path: - qs = question - if args.add_time_instruction: - time_instruciton = f"The video lasts for {video_time:.2f} seconds, and {len(video[0])} frames are uniformly sampled from it. These frames are located at {frame_time}.Please answer the following questions related to this video." - qs = f'{time_instruciton}\n{qs}' - if model.config.mm_use_im_start_end: - qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs - else: - qs = DEFAULT_IMAGE_TOKEN + "\n" + qs - - conv = conv_templates[args.conv_mode].copy() - conv.append_message(conv.roles[0], qs) - conv.append_message(conv.roles[1], None) - prompt = conv.get_prompt() - - input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).cuda() - if tokenizer.pad_token_id is None: - if "qwen" in tokenizer.name_or_path.lower(): - print("Setting pad token to bos token for qwen model.") - tokenizer.pad_token_id = 151643 - - attention_masks = input_ids.ne(tokenizer.pad_token_id).long().cuda() - - stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 - keywords = [stop_str] - stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) - - cur_prompt = question - else: - prompt = question - - system_error = "" - - if "gpt4v" != args.model_path: - - - with torch.inference_mode(): - # model.update_prompt([[cur_prompt]]) - # import pdb;pdb.set_trace() - # output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=True, temperature=0.2, max_new_tokens=1024, use_cache=True, stopping_criteria=[stopping_criteria]) - if "mistral" not in cfg_pretrained._name_or_path.lower(): - output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=False, temperature=0.0, max_new_tokens=1024, top_p=0.1,num_beams=1,use_cache=True, stopping_criteria=[stopping_criteria]) - # output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=True, temperature=0.2, max_new_tokens=1024, use_cache=True, stopping_criteria=[stopping_criteria]) - else: - output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=False, temperature=0.0, max_new_tokens=1024, top_p=0.1, num_beams=1, use_cache=True) - # output_ids = model.generate(inputs=input_ids, images=video, attention_mask=attention_masks, modalities="video", do_sample=True, temperature=0.2, max_new_tokens=1024, use_cache=True) - else: - openai.api_key = args.api_key # Your API key here - - max_num_retries = 0 - retry = 5 - PROMPT_MESSAGES = [ - { - "role": "user", - "content": [ - f"These are frames from a video that I want to upload. Answer me one question of this video: {prompt}", - *map(lambda x: {"image": x, "resize": 336}, video[0::interval]), - ], - }, - ] - params = { - "model": "gpt-4-vision-preview", #gpt-4-1106-vision-preview - "messages": PROMPT_MESSAGES, - "max_tokens": 1024, - } - sucess_flag=False - while max_num_retries < retry: - try: - result = openai.ChatCompletion.create(**params) - outputs = result.choices[0].message.content - sucess_flag = True - break - except Exception as inst : - if 'error' in dir(inst): - # import pdb;pdb.set_trace() - if inst.error.code == 'rate_limit_exceeded': - if "TPM" in inst.error.message: - time.sleep(30) - continue - else: - import pdb;pdb.set_trace() - elif inst.error.code == 'insufficient_quota': - print(f'insufficient_quota key') - exit() - elif inst.error.code == 'content_policy_violation': - print(f'content_policy_violation') - system_error = "content_policy_violation" - - break - print('Find error message in response: ',str(inst.error.message), 'error code: ', str(inst.error.code)) - - continue - if not sucess_flag: - print(f'Calling OpenAI failed after retrying for {max_num_retries} times. Check the logs for details.') - exit() - - if "gpt4v" != args.model_path: - outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() - else: - print(len(video[0::interval])) - - print(f"Question: {prompt}\n") - print(f"Response: {outputs}\n") - - if "gpt4v" == args.model_path: - if system_error == 'content_policy_violation': - continue - elif system_error == "": - continue - else: - import pdb;pdb.set_trace() - - # import pdb;pdb.set_trace() - if "mistral" not in cfg_pretrained._name_or_path.lower(): - if outputs.endswith(stop_str): - outputs = outputs[: -len(stop_str)] - - outputs = outputs.strip() - - sample_set["pred"] = outputs - ans_file.write(json.dumps(sample_set, ensure_ascii=False) + "\n") - ans_file.flush() - - ans_file.close() - - -if __name__ == "__main__": - args = parse_args() - run_inference(args) \ No newline at end of file diff --git a/playground/demo/xU25MMA2N4aVtYay.mp4 b/playground/demo/xU25MMA2N4aVtYay.mp4 deleted file mode 100644 index 5bc7f52..0000000 Binary files a/playground/demo/xU25MMA2N4aVtYay.mp4 and /dev/null differ diff --git a/playground/equal_splitter.py b/playground/equal_splitter.py deleted file mode 100644 index 8f89f2d..0000000 --- a/playground/equal_splitter.py +++ /dev/null @@ -1,38 +0,0 @@ -import json -from math import ceil - - -def split_json_file(input_file, n_splits): - # Read the JSON file - with open(input_file, "r") as file: - data = json.load(file) - - # Calculate the size of each split - total_items = len(data) - items_per_split = ceil(total_items / n_splits) - - # Split the data and save into separate files - for i in range(n_splits): - start_index = i * items_per_split - end_index = min((i + 1) * items_per_split, total_items) - split_data = data[start_index:end_index] - - # Write the split data to a new JSON file - with open(f"{input_file.split('.')[0]}_split_{i}.json", "w") as split_file: - json.dump(split_data, split_file, indent=4) - - -def main(): - import argparse - - parser = argparse.ArgumentParser(description="Split a JSON file into multiple parts.") - parser.add_argument("--input_file", type=str, help="The JSON file to split") - parser.add_argument("--n_splits", type=int, help="The number of splits") - - args = parser.parse_args() - - split_json_file(args.input_file, args.n_splits) - - -if __name__ == "__main__": - main() diff --git a/playground/remove_mid_ckpt.py b/playground/remove_mid_ckpt.py deleted file mode 100644 index 612b717..0000000 --- a/playground/remove_mid_ckpt.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import shutil -import glob - - -def remove_checkpoints(directory, pattern): - # Walk through the directory - for root, dirs, files in os.walk(directory): - # Use glob to find paths matching the pattern - for file_path in glob.glob(os.path.join(root, pattern)): - # Check if it is a directory - if "llava-1.6-mistral-7b" in file_path: - continue - if os.path.isdir(file_path): - # Remove the directory - print(f"Removing {file_path}") - input("Press Enter to continue...") - shutil.rmtree(file_path) - print(f"Removed directory: {file_path}") - else: - print(f"Removing {file_path}") - input("Press Enter to continue...") - # Remove the file - os.remove(file_path) - print(f"Removed file: {file_path}") - - -# Directory containing the checkpoints -directory = "/mnt/bn/vl-research/checkpoints/feng/" - -# Pattern to match in the file names -pattern = "global_step*" - -# Call the function -remove_checkpoints(directory, pattern) diff --git a/playground/sgl_llava_inference_multinode.py b/playground/sgl_llava_inference_multinode.py deleted file mode 100644 index 869099d..0000000 --- a/playground/sgl_llava_inference_multinode.py +++ /dev/null @@ -1,125 +0,0 @@ -import argparse -import json -import time -import os -import tqdm -import sglang as sgl -from sglang.test.test_utils import select_sglang_backend -from sglang.utils import dump_state_text - - -@sgl.function -def image_description(s, image_file): - prompt = "Please generate detailed descriptions of the given image." - s += sgl.user(sgl.image(image_file) + prompt) - s += sgl.assistant(sgl.gen("answer", max_tokens=1024, temperature=0.0)) - - -def load_progress(progress_file): - print(f"Load progress from {progress_file}") - if os.path.exists(progress_file): - with open(progress_file, "r") as f: - return json.load(f) - return {"last_index": -1, "last_chunk": -1, "results": [], "annotations": []} - - -def save_progress(progress_file, progress_data): - with open(progress_file, "w") as f: - json.dump(progress_data, f, indent=2) - - -def find_images_in_subfolders(folder_path): - image_extensions = (".png", ".jpg", ".jpeg", ".gif", ".bmp") - image_files = [] - for root, dirs, files in os.walk(folder_path): - for file in files: - if file.endswith(image_extensions): - image_files.append(os.path.join(root, file)) - return image_files - - -def main(args): - dist_rank = args.dist - dist_size = args.total_dist - - base_dir = os.path.dirname(args.result_file) - os.makedirs(base_dir, exist_ok=True) # Ensure the base directory exists - progress_file = f"{base_dir}/progress_{dist_rank}_or_{dist_size}.json" - progress_data = load_progress(progress_file) - - with open(args.json_path, "r") as fp: - data = json.load(fp) - - image_files = [os.path.join(args.images_root, item["image"]) for item in data] - image_files = image_files[: args.limit] if args.limit > 0 else image_files - - # Shard the data - shard_size = len(image_files) // dist_size - start_index = shard_size * dist_rank - end_index = start_index + shard_size if dist_rank < dist_size - 1 else len(image_files) - shard_files = image_files[start_index:end_index] - - print(f"Querying {len(shard_files)} images from index {start_index} to {end_index - 1}") - - # Select backend - backend = select_sglang_backend(args) - sgl.set_default_backend(backend) - - tic = time.time() - batch_size = args.parallel - for batch_start in tqdm.tqdm(range(0, len(shard_files), batch_size)): - batch_end = min(batch_start + batch_size, len(shard_files)) - if batch_start <= progress_data.get("last_index", -1): - print(f"Skipping already processed batch starting at {batch_start}") - continue - batch_arguments = [{"image_file": image_file} for image_file in shard_files[batch_start:batch_end]] - try: - batch_states = image_description.run_batch(batch_arguments, temperature=0, num_threads=args.parallel, progress_bar=False) - for i, ret in enumerate(batch_states): - image_file = batch_arguments[i]["image_file"] - caption = ret.text().split("ASSISTANT:")[-1].strip() - progress_data["annotations"].append({"image_file": image_file, "caption": caption}) - progress_data["last_index"] = batch_start + i # Update last_index relative to this rank's shard - - save_progress(progress_file, progress_data) - except Exception as e: - print(f"Error during batch processing: {e}") - save_progress(progress_file, progress_data) - break - - latency = time.time() - tic - print(f"Latency: {latency:.3f}") - - value = { - "task": "image_captioning", - "backend": args.backend, - "num_gpus": 1, - "latency": round(latency, 3), - "num_requests": len(shard_files), - "parallel": args.parallel, - "results": progress_data["annotations"], - } - - result_file = args.result_file.replace(".json", f"_shard_{dist_rank}_or_{dist_size}.json") - print(f"Write output to {result_file}") - with open(result_file, "w") as fout: - json.dump(value, fout, indent=2) - - save_progress(progress_file, progress_data) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--images_root", type=str, default="/mnt/bn/vl-research/data/llava_data/cc3m") - parser.add_argument("--json_path", type=str, default="/mnt/bn/vl-research/data/llava_instruct/cc3m_recap_requery_363707.json") - parser.add_argument("--max_tokens", type=int, default=1024) - parser.add_argument("--parallel", type=int, default=32) - parser.add_argument("--backend", type=str, default="srt") - parser.add_argument("--host", type=str, default="http://127.0.0.1") - parser.add_argument("--port", type=int, default=30000) - parser.add_argument("--result_file", type=str, default="/mnt/bn/vl-research/workspace/boli01/projects/LLaVA_Next/playground/sgl_llava_inference.json") - parser.add_argument("--limit", type=int, default=-1) - parser.add_argument("--dist", type=int, default=0, help="The rank of the distributed machine") - parser.add_argument("--total_dist", type=int, default=6, help="Total number of distributed machines") - args = parser.parse_args() - main(args) diff --git a/playground/upload_data.py b/playground/upload_data.py deleted file mode 100644 index d1acf2b..0000000 --- a/playground/upload_data.py +++ /dev/null @@ -1,217 +0,0 @@ -from datasets import Dataset, Features, Value, ClassLabel, Sequence, Image -import json -import PIL.Image as pil_image -from io import BytesIO -from tqdm import tqdm - -json_paths = [ - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/mavis_math_metagen_87358.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/mavis_math_rule_geo_100000.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/k12_printing_train_256646.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/iiit5k_annotations_2000.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/hme100k_train_clean_74502.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/ai2d_azuregpt_detailed_understanding_4874.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/infographic_vqa_4404.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/infographic_azuregpt4v_1992.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/lrv_chart_1787.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/lrv_normal_gpt4v_filtered_10500.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/scienceqa_nona_context_19218.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/allava_instruct_vflan4v_20000.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/allava_instruct_laion4v_50000.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/textocr_gpt4v_train_converted_25114.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/ai2d_train_internvl_single_12413.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/textcaps_train_21952.json", - # "/mnt/bn/vl-research/data/llava_instruct/ureader_new/ureader_qa_sft.json", - # "/mnt/bn/vl-research/data/llava_instruct/ureader_new/ureader_cap_sft.json", - # "/mnt/bn/vl-research/data/llava_instruct/ureader_new/ureader_ie_sft.json", - # "/mnt/bn/vl-research/data/llava_instruct/ureader_new/ureader_kg_sft.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/vision_flan_filtered_186070.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/mathqa_29837.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/geo3k_2101.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/geo170k_qa_converted_67833.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/geo170k_align_converted_60252.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/sharegpt4v-coco-50k.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/sharegpt4v-knowledge-2k.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/sharegpt4v-llava-30k.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/sharegpt4v-sam-20k.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_CLEVR-Math_5290.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_FigureQA_17597.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_Geometry3K_9734.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_GeoQA+_17172.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_GEOS_508.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_IconQA_22599.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_MapQA_5235.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_PMC-VQA_35958.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_Super-CLEVR_8652.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_TabMWP_22462.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_UniGeo_11959.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/MathV360K_VizWiz_6614.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/magpie_pro_qwen2_72b_st_300000_sp_token_fltd_299992.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/magpie_pro_l3_80b_st_300000.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/magpie_pro_l3_80b_mt_300000_sp_token_fltd_299998.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/image_textualization_dataset_filtered.json", - # "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/cambrian_filtered_gpt4vo_sp_token_fltd_max10k.json", - "/mnt/bn/vl-research/data/llava_instruct/real_vision_flan/sharegpt4o_dataset.jsonl", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/ai2d_llava_format_2434.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/aokvqa_16539_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/chart2text_26961.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/chartqa_18265_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/clevr_70000_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/diagram_image_to_text_300.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/dvqa_200000_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/figureqa_100000_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/geomverse_9303.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/hateful_memes_8500_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/hitab_2500_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/iam_5663.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/raven_42000.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/iconqa_llava_format_27307.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/infographic_vqa_2118_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/intergps_1280_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/mapqa_37417_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/multihiertt_7619.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/rendered_text_10000.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/robut_sqa_8514.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/robut_wikisql_74989.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/robut_wtq_38246_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/screen2words_15730.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/scienceqa_llava_format_4976.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/tabmwp_22722.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/tallyqa_98680_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/st_vqa_17247_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/tqa_llava_format_27307.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/visual7w_llava_format_14366.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/visualmrc_3027.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/vqarad_313_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/vsr_2157_llava_format.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/vistext_9969.json", - "/mnt/bn/vl-research/data/llava_instruct/cauldron/websight_10000.json" -] - -short_names = [ - # "mavis_math_metagen", - # "mavis_math_rule_geo", - # "k12_printing", - # "iiit5k", - # "hme100k", - # "ai2d(gpt4v)", - # "infographic_vqa", - # "infographic(gpt4v)", - # "lrv_chart", - # "lrv_normal(filtered)", - # "scienceqa(nona_context)", - # "allava_instruct_vflan4v", - # "allava_instruct_laion4v", - # "textocr(gpt4v)", - # "ai2d(internvl)", - # "textcaps", - # "ureader_qa", # need to re-upload - # "ureader_cap", # need to re-upload - # "ureader_ie", # need to re-upload - # "ureader_kg", # need to re-upload - # "vision_flan(filtered)", - # "mathqa", - # "geo3k", - # "geo170k(qa)", - # "geo170k(align)", - # "sharegpt4v(coco)", - # "sharegpt4v(knowledge)", - # "sharegpt4v(llava)", - # "sharegpt4v(sam)", - # "CLEVR-Math(MathV360K)", - # "FigureQA(MathV360K)", - # "Geometry3K(MathV360K)", - # "GeoQA+(MathV360K)", - # "GEOS(MathV360K)", - # "IconQA(MathV360K)", - # "MapQA(MathV360K)", - # "PMC-VQA(MathV360K)", - # "Super-CLEVR(MathV360K)", - # "TabMWP(MathV360K)", - # "UniGeo(MathV360K)", - # "VizWiz(MathV360K)", - # "magpie_pro(qwen2_72b_st)", - # "magpie_pro(l3_80b_st)", - # "magpie_pro(l3_80b_mt)", - # "image_textualization(filtered)", - # "cambrian(filtered_gpt4vo)", # need to re-upload - "sharegpt4o", - "ai2d(cauldron,llava_format)", - "aokvqa(cauldron,llava_format)", - "chart2text(cauldron)", - "chartqa(cauldron,llava_format)", - "clevr(cauldron,llava_format)", - "diagram_image_to_text(cauldron)", - "dvqa(cauldron,llava_format)", - "figureqa(cauldron,llava_format)", - "geomverse(cauldron)", - "hateful_memes(cauldron,llava_format)", - "hitab(cauldron,llava_format)", - "iam(cauldron)", - "raven(cauldron)", - "iconqa(cauldron,llava_format)", - "infographic_vqa_llava_format", - "intergps(cauldron,llava_format)", - "mapqa(cauldron,llava_format)", - "multihiertt(cauldron)", - "rendered_text(cauldron)", - "robut_sqa(cauldron)", - "robut_wikisql(cauldron)", - "robut_wtq(cauldron,llava_format)", - "screen2words(cauldron)", - "scienceqa(cauldron,llava_format)", - "tabmwp(cauldron)", - "tallyqa(cauldron,llava_format)", - "st_vqa(cauldron,llava_format)", - "tqa(cauldron,llava_format)", - "visual7w(cauldron,llava_format)", - "visualmrc(cauldron)", - "vqarad(cauldron,llava_format)", - "vsr(cauldron,llava_format)", - "vistext(cauldron)", - "websight(cauldron)" -] - -def upload_data(json_path, short_name): - def gen(): - if json_path.endswith(".jsonl"): - with open(json_path, "r") as f: - data = [json.loads(line) for line in f] - else: - with open(json_path, "r") as f: - data = json.load(f) - - preview_index = 5 - idx = 0 - for item in tqdm(data): - if preview_index > 0: - preview_index -= 1 - print(item) - continue - - try: - if "image" in item: - image_path = f"/mnt/bn/vl-research/data/llava_data/{item['image']}" - try: - with open(image_path, "rb") as img_file: - image = pil_image.open(BytesIO(img_file.read())) - except: - print(f"Failed to load image {item['image']}") - continue - else: - image = None - - item_id = item["id"] if "id" in item else f"{idx:06d}" - yield {"id": item_id, "image": image, "conversations": item["conversations"], "data_source": short_name} - idx += 1 - - except Exception as e: - print(e) - continue - - - hf_dataset = Dataset.from_generator(generator=gen, num_proc=32) - hf_dataset.push_to_hub("lmms-lab/LLaVA-OneVision-Data", config_name=short_name, split="train") - -for json_path, short_name in zip(json_paths, short_names): - upload_data(json_path, short_name) \ No newline at end of file