diff --git a/configs/trainers/PromptSRC/vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets.yaml b/configs/trainers/PromptSRC/vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets.yaml index c062461..895912a 100644 --- a/configs/trainers/PromptSRC/vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets.yaml +++ b/configs/trainers/PromptSRC/vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets.yaml @@ -23,6 +23,7 @@ OPTIM: WARMUP_CONS_LR: 1e-5 TRAIN: + CHECKPOINT_FREQ: 5 PRINT_FREQ: 20 MODEL: diff --git a/configs/trainers/PromptSRC/vit_b16_c2_ep50_batch4_4+4ctx_few_shot.yaml b/configs/trainers/PromptSRC/vit_b16_c2_ep5_batch4_4+4ctx_cross_datasets.yaml similarity index 68% rename from configs/trainers/PromptSRC/vit_b16_c2_ep50_batch4_4+4ctx_few_shot.yaml rename to configs/trainers/PromptSRC/vit_b16_c2_ep5_batch4_4+4ctx_cross_datasets.yaml index b3215a9..2dbbacd 100644 --- a/configs/trainers/PromptSRC/vit_b16_c2_ep50_batch4_4+4ctx_few_shot.yaml +++ b/configs/trainers/PromptSRC/vit_b16_c2_ep5_batch4_4+4ctx_cross_datasets.yaml @@ -16,7 +16,7 @@ INPUT: OPTIM: NAME: "sgd" LR: 0.0025 - MAX_EPOCH: 50 + MAX_EPOCH: 5 LR_SCHEDULER: "cosine" WARMUP_EPOCH: 1 WARMUP_TYPE: "constant" @@ -35,13 +35,9 @@ TRAINER: N_CTX_TEXT: 4 CTX_INIT: "a photo of a" PREC: "fp16" - PROMPT_DEPTH_VISION: 9 - PROMPT_DEPTH_TEXT: 9 + PROMPT_DEPTH_VISION: 3 + PROMPT_DEPTH_TEXT: 3 TEXT_LOSS_WEIGHT: 25 IMAGE_LOSS_WEIGHT: 10 -# Use the below configuration for: ImageNet, Caltech101, OxfordPets, Food101, UCF101 and SUN397 - GPA_MEAN: 30 - GPA_STD: 30 -# Use the below configuration for: StanfordCars, Flowers102, FGVCAircraft, DTD and EuroSAT -# GPA_MEAN: 45 -# GPA_STD: 5 \ No newline at end of file + GPA_MEAN: 6 + GPA_STD: 10 diff --git a/scripts/promptsrc/few_shot.sh b/scripts/promptsrc/few_shot.sh deleted file mode 100644 index 8a63cf9..0000000 --- a/scripts/promptsrc/few_shot.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# custom config -DATA="/path/to/dataset/folder" -TRAINER=PromptSRC - -DATASET=$1 -CFG=vit_b16_c2_ep50_batch4_4+4ctx_few_shot -SHOTS=$2 - -for SEED in 1 2 3 -do - DIR=output/${DATASET}/${TRAINER}/${CFG}_${SHOTS}shots/seed${SEED} - if [ -d "$DIR" ]; then - echo " The results exist at ${DIR}" - else - echo "Run this job and save the output to ${DIR}" - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR} \ - DATASET.NUM_SHOTS ${SHOTS} - fi -done diff --git a/scripts/promptsrc/reproduce_base2novel_setting.sh b/scripts/promptsrc/reproduce_base2novel_setting.sh deleted file mode 100644 index 3a966ca..0000000 --- a/scripts/promptsrc/reproduce_base2novel_setting.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# custom config -DATA="/path/to/dataset/folder" -TRAINER=PromptSRC - -DATASET=$1 -SEED=$2 -WEIGHTSPATH=$3 - -CFG=vit_b16_c2_ep20_batch4_4+4ctx -SHOTS=16 -LOADEP=20 -SUB_base=base -SUB_novel=new - -COMMON_DIR=${DATASET}/shots_${SHOTS}/${TRAINER}/${CFG}/seed${SEED} -MODEL_DIR=${WEIGHTSPATH}/base/seed${SEED} -DIR_base=output/base2new/test_${SUB_base}/${COMMON_DIR} -DIR_novel=output/base2new/test_${SUB_novel}/${COMMON_DIR} -if [ -d "$DIR" ]; then - echo "Results are already available in ${DIR}. Skipping..." -else - echo "Evaluating model" - echo "Runing the first phase job and save the output to ${DIR}" - # Evaluate on base classes - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR_base} \ - --model-dir ${MODEL_DIR} \ - --load-epoch ${LOADEP} \ - --eval-only \ - DATASET.NUM_SHOTS ${SHOTS} \ - DATASET.SUBSAMPLE_CLASSES ${SUB_base} - - # Evaluate on novel classes - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR_novel} \ - --model-dir ${MODEL_DIR} \ - --load-epoch ${LOADEP} \ - --eval-only \ - DATASET.NUM_SHOTS ${SHOTS} \ - DATASET.SUBSAMPLE_CLASSES ${SUB_novel} - -fi \ No newline at end of file diff --git a/scripts/promptsrc/reproduce_few_shot.sh b/scripts/promptsrc/reproduce_few_shot.sh deleted file mode 100644 index 91206bd..0000000 --- a/scripts/promptsrc/reproduce_few_shot.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# custom config -DATA="/path/to/dataset/folder" -TRAINER=PromptSRC - -DATASET=$1 -SHOTS=$2 -WEIGHTSPATH=$3 - -CFG=vit_b16_c2_ep50_batch4_4+4ctx_few_shot -LOADEP=50 - -for SEED in 1 2 3 -do - MODEL_DIR=${WEIGHTSPATH}/${SHOTS}shot/seed${SEED} - DIR=output/few_shot/${DATASET}/${TRAINER}/${CFG}_${SHOTS}shots/seed${SEED} - if [ -d "$DIR" ]; then - echo " The results exist at ${DIR}" - else - echo "Run this job and save the output to ${DIR}" - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR} \ - --model-dir ${MODEL_DIR} \ - --load-epoch ${LOADEP} \ - --eval-only \ - DATASET.NUM_SHOTS ${SHOTS} - fi -done \ No newline at end of file diff --git a/scripts/promptsrc/reproduce_xd.sh b/scripts/promptsrc/reproduce_xd.sh deleted file mode 100644 index e12083f..0000000 --- a/scripts/promptsrc/reproduce_xd.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# custom config -DATA="/path/to/dataset/folder" -TRAINER=PromptSRC - -DATASET=$1 -SEED=$2 -WEIGHTSPATH=$3 - -CFG=vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets -SHOTS=16 -LOADEP=20 - -MODEL_DIR=${WEIGHTSPATH}/seed${SEED} - -DIR=output/evaluation/${TRAINER}/${CFG}_${SHOTS}shots/${DATASET}/seed${SEED} -if [ -d "$DIR" ]; then - echo "Results are already available in ${DIR}. Skipping..." -else - echo "Evaluating model" - echo "Runing the first phase job and save the output to ${DIR}" - # Evaluate on evaluation datasets - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR} \ - --model-dir ${MODEL_DIR} \ - --load-epoch ${LOADEP} \ - --eval-only \ - DATASET.NUM_SHOTS ${SHOTS} \ - -fi \ No newline at end of file diff --git a/scripts/promptsrc/xd_test.sh b/scripts/promptsrc/xd_test.sh deleted file mode 100644 index 4be1f70..0000000 --- a/scripts/promptsrc/xd_test.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - - -# custom config -DATA="/path/to/dataset/folder" -TRAINER=PromptSRC - -DATASET=$1 -SEED=$2 - -CFG=vit_b16_c2_ep5_batch4_4+4ctx_cross_datasets -SHOTS=16 - - -DIR=output/evaluation/${TRAINER}/${CFG}_${SHOTS}shots/${DATASET}/seed${SEED} -if [ -d "$DIR" ]; then - echo "Results are available in ${DIR}. Skip this job" -else - echo "Run this job and save the output to ${DIR}" - - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR} \ - --model-dir output/imagenet/${TRAINER}/${CFG}_${SHOTS}shots/seed${SEED} \ - --load-epoch 20 \ - --eval-only -fi \ No newline at end of file diff --git a/scripts/promptsrc/xd_train.sh b/scripts/promptsrc/xd_train.sh index 0da25e3..131f8c4 100644 --- a/scripts/promptsrc/xd_train.sh +++ b/scripts/promptsrc/xd_train.sh @@ -1,29 +1,30 @@ #!/bin/bash -# custom config -DATA="/path/to/dataset/folder" +DATA=" ~/Datasets/CoOp" TRAINER=PromptSRC - -DATASET=$1 -SEED=$2 - -CFG=vit_b16_c2_ep5_batch4_4+4ctx_cross_datasets +SRC_DATASETS=imagenet SHOTS=16 +CFG=vit_b16_c2_ep5_batch4_4+4ctx_cross_datasets -DIR=output/${DATASET}/${TRAINER}/${CFG}_${SHOTS}shots/seed${SEED} -if [ -d "$DIR" ]; then - echo "Results are available in ${DIR}." -else - echo "Run this job and save the output to ${DIR}" +for SEED in 1 2 3 +do + DIR=output_xd/base2new/train_base/${SRC_DATASETS}/shots_${SHOTS}/${TRAINER}/${CFG}/seed${SEED} + + if [ -d "$DIR" ]; then + echo "Results are available in ${DIR}. Skip this job" + else + echo "Run this job and save the output to ${DIR}" + + CUDA_VISIBLE_DEVICES=0 python train.py \ + --root ${DATA} \ + --seed ${SEED} \ + --trainer ${TRAINER} \ + --dataset-config-file configs/datasets/${SRC_DATASETS}.yaml \ + --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ + --output-dir ${DIR} \ + DATASET.NUM_SHOTS ${SHOTS} + fi +done - python train.py \ - --root ${DATA} \ - --seed ${SEED} \ - --trainer ${TRAINER} \ - --dataset-config-file configs/datasets/${DATASET}.yaml \ - --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ - --output-dir ${DIR} \ - DATASET.NUM_SHOTS ${SHOTS} -fi \ No newline at end of file diff --git a/scripts/promptsrc/xda_test.sh b/scripts/promptsrc/xda_test.sh new file mode 100644 index 0000000..7a54645 --- /dev/null +++ b/scripts/promptsrc/xda_test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + + +# custom config +DATA=" ~/Datasets/CoOp" +TRAINER=PromptSRC + + +SRC_DATASETS=imagenet +SHOTS=16 +CFG=vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets +LOADEP=20 + + +DATASETS=(dtd eurosat fgvc_aircraft food101 oxford_flowers oxford_pets stanford_cars ucf101 caltech101 sun397) +SEEDS=(1 2 3) + + +for DATASET in "${DATASETS[@]}" +do + for SEED in "${SEEDS[@]}" + do + MODEL_DIR=output_xd/base2new/train_base/${SRC_DATASETS}/shots_${SHOTS}/${TRAINER}/${CFG}/seed${SEED} + + DIR=output_xd/base2new/test_new/${DATASET}/shots_${SHOTS}/${TRAINER}/${CFG}/seed${SEED} + + if [ -d "$DIR" ]; then + echo "Results are available in ${DIR}. Skip this job" + else + echo "Run this job and save the output to ${DIR}" + echo "Loading model from ${MODEL_DIR}" + + CUDA_VISIBLE_DEVICES=0 python train.py \ + --root ${DATA} \ + --seed ${SEED} \ + --trainer ${TRAINER} \ + --dataset-config-file configs/datasets/${DATASET}.yaml \ + --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ + --output-dir ${DIR} \ + --model-dir ${MODEL_DIR} \ + --load-epoch ${LOADEP} \ + --eval-only + fi + done +done + diff --git a/scripts/promptsrc/xdo_test.sh b/scripts/promptsrc/xdo_test.sh new file mode 100644 index 0000000..00221fb --- /dev/null +++ b/scripts/promptsrc/xdo_test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + + +# custom config +DATA=" ~/Datasets/CoOp" +TRAINER=PromptSRC + + +SRC_DATASETS=imagenet +SHOTS=16 +CFG=vit_b16_c2_ep20_batch4_4+4ctx_cross_datasets +LOADEP=20 + + +DATASETS=(imagenetv2 imagenet_sketch imagenet_a imagenet_r) +SEEDS=(1 2 3) + + +for DATASET in "${DATASETS[@]}" +do + for SEED in "${SEEDS[@]}" + do + MODEL_DIR=output_xd/base2new/train_base/${SRC_DATASETS}/shots_${SHOTS}/${TRAINER}/${CFG}/seed${SEED} + + DIR=output_xd/base2new/test_new/${DATASET}/shots_${SHOTS}/${TRAINER}/${CFG}/seed${SEED} + + if [ -d "$DIR" ]; then + echo "Results are available in ${DIR}. Skip this job" + else + echo "Run this job and save the output to ${DIR}" + echo "Loading model from ${MODEL_DIR}" + + CUDA_VISIBLE_DEVICES=0 python train.py \ + --root ${DATA} \ + --seed ${SEED} \ + --trainer ${TRAINER} \ + --dataset-config-file configs/datasets/${DATASET}.yaml \ + --config-file configs/trainers/${TRAINER}/${CFG}.yaml \ + --output-dir ${DIR} \ + --model-dir ${MODEL_DIR} \ + --load-epoch ${LOADEP} \ + --eval-only + fi + done +done +