RagVL
Official PyTorch Implementation of MLLM Is a Strong Reranker: Advancing Multimodal Retrieval-augmented Generation via Knowledge-enhanced Reranking and Noise-injected Training.
File Explorer
Download Latest Version (.zip)- framework.png
- finetune_task_lora.sh
- zero2.json
- zero3.json
- zero3_offload.json
- finetune_lora.sh
- zero2.json
- zero3.json
- zero3_offload.json
- ds_config_zero2.json
- ds_config_zero3.json
- finetune.py
- finetune_ds.sh
- finetune_lora_ds.sh
- finetune_lora_single_gpu.sh
- finetune_qlora_ds.sh
- finetune_qlora_single_gpu.sh
- run_internvl.py
- configuration_internlm2.py
- modeling_internlm2.py
- tokenization_internlm2.py
- tokenization_internlm2_fast.py
- __init__.py
- configuration_intern_vit.py
- configuration_internvl_chat.py
- flash_attention.py
- modeling_intern_vit.py
- modeling_internvl_chat.py
- configuration_phi3.py
- modeling_phi3.py
- __init__.py
- __init__.py
- llama2_flash_attn_monkey_patch.py
- llama_flash_attn_monkey_patch.py
- llama_rmsnorm_monkey_patch.py
- pad_data_collator.py
- train_sampler_patch.py
- __init__.py
- constants.py
- dataset.py
- internvl_chat_finetune.py
- internvl_chat_pretrain.py
- trainer_monkey_patch.py
- conversation.py
- dist_utils.py
- internvl_2_finetune_flickr30k_rerank.json
- internvl_2_finetune_mmqa_qa.json
- internvl_2_finetune_mmqa_rerank.json
- internvl_2_finetune_mscoco_rerank.json
- internvl_2_finetune_webqa_qa.json
- internvl_2_finetune_webqa_rerank.json
- internvl_chat_v1_2_hermes2_yi34b_448_res_2nd_finetune_full.sh
- internvl_chat_v1_2_hermes2_yi34b_448_res_2nd_finetune_lora.sh
- internvl_chat_v1_2_hermes2_yi34b_448_res_finetune.sh
- internvl_chat_v1_5_internlm2_1_8b_dynamic_res_2nd_finetune_full.sh
- internvl_chat_v1_5_internlm2_1_8b_dynamic_res_2nd_finetune_lora.sh
- internvl_chat_v1_5_internlm2_20b_dynamic_res_2nd_finetune_full.sh
- internvl_chat_v1_5_internlm2_20b_dynamic_res_2nd_finetune_lora.sh
- internvl_chat_v1_5_phi3_3_8b_dynamic_res_2nd_finetune_full.sh
- internvl_chat_v1_5_phi3_3_8b_dynamic_res_2nd_finetune_lora.sh
- internvl_chat_v1_5_hermes2_yi34b_dynamic_res_finetune.sh
- internvl_chat_v1_5_hermes2_yi34b_dynamic_res_pretrain.sh
- internvl_chat_v1_5_internlm2_1_8b_dynamic_res_finetune.sh
- internvl_chat_v1_5_internlm2_1_8b_dynamic_res_pretrain.sh
- internvl_chat_v1_5_internlm2_20b_dynamic_res_finetune.sh
- internvl_chat_v1_5_internlm2_20b_dynamic_res_pretrain.sh
- internvl_chat_v1_5_phi3_3_8b_dynamic_res_finetune.sh
- internvl_chat_v1_5_phi3_3_8b_dynamic_res_pretrain.sh
- internvl2_1b_qwen2_0_5b_dynamic_res_2nd_finetune_full.sh
- internvl2_1b_qwen2_0_5b_dynamic_res_2nd_finetune_lora.sh
- internvl2_26b_internlm2_20b_dynamic_res_2nd_finetune_full.sh
- internvl2_26b_internlm2_20b_dynamic_res_2nd_finetune_lora.sh
- internvl2_2b_internlm2_1_8b_dynamic_res_2nd_finetune_full.sh
- internvl2_2b_internlm2_1_8b_dynamic_res_2nd_finetune_lora.sh
- internvl2_2b_internlm2_1_8b_dynamic_res_2nd_finetune_lora_coco.sh
- internvl2_40b_hermes2_yi_34b_dynamic_res_2nd_finetune_full.sh
- internvl2_40b_hermes2_yi_34b_dynamic_res_2nd_finetune_lora.sh
- internvl2_4b_phi3_3_8b_dynamic_res_2nd_finetune_full.sh
- internvl2_4b_phi3_3_8b_dynamic_res_2nd_finetune_lora.sh
- internvl2_76b_hermes2_llama3_70b_dynamic_res_2nd_finetune_full.sh
- internvl2_76b_hermes2_llama3_70b_dynamic_res_2nd_finetune_lora.sh
- internvl2_8b_internlm2_7b_dynamic_res_2nd_finetune_full.sh
- internvl2_8b_internlm2_7b_dynamic_res_2nd_finetune_lora.sh
- pyproject.toml
- README.md
- zero_stage1_config.json
- zero_stage2_config.json
- zero_stage3_config.json
- zero_stage3_config_100b.json
- zero_stage3_config_34b.json
- zero_stage3_config_70b.json
- answer_alpaca-13b.jsonl
- answer_bard.jsonl
- answer_gpt35.jsonl
- answer_llama-13b.jsonl
- answer_vicuna-13b.jsonl
- test_sqa_llava_13b_v0.json
- test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json
- review_alpaca-13b_vicuna-13b.jsonl
- review_bard_vicuna-13b.jsonl
- review_gpt35_vicuna-13b.jsonl
- review_llama-13b_vicuna-13b.jsonl
- caps_boxes_coco2014_val_80.jsonl
- model.jsonl
- prompt.jsonl
- question.jsonl
- reviewer.jsonl
- rule.json
- alpaca.png
- bard.jpg
- chatgpt.svg
- llama.jpg
- swords_FILL0_wght300_GRAD0_opsz48.svg
- vicuna.jpeg
- index.html
- script.js
- styles.css
- eval_gpt_review.py
- eval_gpt_review_bench.py
- eval_gpt_review_visual.py
- eval_pope.py
- eval_science_qa.py
- eval_science_qa_gpt4.py
- eval_science_qa_gpt4_requery.py
- eval_textvqa.py
- generate_webpage_data_from_table.py
- m4c_evaluator.py
- model_qa.py
- model_vqa.py
- model_vqa_loader.py
- model_vqa_mmbench.py
- model_vqa_science.py
- qa_baseline_gpt35.py
- run_llava.py
- summarize_gpt_review.py
- llava_llama.py
- llava_mistral.py
- llava_mpt.py
- builder.py
- clip_encoder.py
- builder.py
- __init__.py
- apply_delta.py
- builder.py
- consolidate.py
- llava_arch.py
- make_delta.py
- utils.py
- extreme_ironing.jpg
- waterview.jpg
- __init__.py
- cli.py
- controller.py
- gradio_web_server.py
- model_worker.py
- register_worker.py
- sglang_worker.py
- test_message.py
- llama_flash_attn_monkey_patch.py
- llama_xformers_attn_monkey_patch.py
- llava_trainer.py
- train.py
- train_mem.py
- train_xformers.py
- __init__.py
- constants.py
- conversation.py
- mm_utils.py
- utils.py
- __init__.py
- evaluate_caption.py
- evaluate_mmbench.py
- evaluate_mme.py
- evaluate_mmmu.py
- evaluate_vqa.py
- EVALUATION.md
- mmbench_converter.py
- run_mplug_owl2.py
- vqa.py
- vqa_eval.py
- extreme_ironing.jpg
- Rebecca_(1939_poster)_Small.jpeg
- __init__.py
- local_web_server.py
- model_worker.py
- __init__.py
- builder.py
- configuration_mplug_owl2.py
- configuration_qwen.py
- convert_mplug_owl2_weight_to_hf.py
- modeling_attn_mask_utils.py
- modeling_llama2.py
- modeling_mplug_owl2.py
- modeling_qwen.py
- multiway.py
- utils.py
- visual_encoder.py
- extreme_ironing.jpg
- Rebecca_(1939_poster)_Small.jpeg
- __init__.py
- cli.py
- controller.py
- gradio_web_server.py
- model_worker.py
- register_workers.py
- llama_flash_attn_monkey_patch.py
- mplug_owl2_trainer.py
- train.py
- train_mem.py
- __init__.py
- constants.py
- conversation.py
- mm_utils.py
- utils.py
- run_qwenvl.py
- EVA01-CLIP-B-16.json
- EVA01-CLIP-g-14-plus.json
- EVA01-CLIP-g-14.json
- EVA02-CLIP-B-16.json
- EVA02-CLIP-bigE-14-plus.json
- EVA02-CLIP-bigE-14.json
- EVA02-CLIP-L-14-336.json
- EVA02-CLIP-L-14.json
- __init__.py
- bpe_simple_vocab_16e6.txt.gz
- constants.py
- eva_vit_model.py
- factory.py
- hf_configs.py
- hf_model.py
- loss.py
- model.py
- modified_resnet.py
- openai.py
- pretrained.py
- rope.py
- timm_model.py
- tokenizer.py
- transform.py
- transformer.py
- utils.py
- __init__.py
- modeling.py
- README.md
- __init__.py
- __init__.py
- indexing_faiss.py
- metrics.py
- model_series.py
- utils.py
- vcd_add_noise.py
- vcd_sample.py
- .gitignore
- flickr30k_pipeline.py
- LICENSE
- merge_lora.py
- mmqa_oracle.py
- mmqa_pipeline.py
- mscoco_pipeline.py
- README.md
- requirements.txt
- webqa_oracle.py
- webqa_pipeline.py
// repository documentation
Was this content helpful?
(0 ratings)