Release of PromptSRC with pretrained models.
This commit is contained in:
uzair khattak
17
lpclip/README.md
Normal file
17
lpclip/README.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# Linear Probe CLIP
|
||||
|
||||
To run linear probe baselines, make sure that your current working directory is `lpclip/`.
|
||||
|
||||
Step 1: Extract Features using the CLIP Image Encoder
|
||||
```bash
|
||||
sh feat_extractor.sh
|
||||
```
|
||||
|
||||
Step 2: Train few-shot linear probe
|
||||
```bash
|
||||
sh linear_probe.sh
|
||||
```
|
||||
|
||||
We follow the instructions stated in the Appendix A3 (pp.38) of [the original CLIP paper](https://arxiv.org/pdf/2103.00020.pdf), with a careful hyperparameter sweep.
|
||||
|
||||
Note: please pull the latest Dassl (version >= `606a2c6`).
|
||||
189
lpclip/feat_extractor.py
Normal file
189
lpclip/feat_extractor.py
Normal file
@@ -0,0 +1,189 @@
|
||||
import os, argparse
|
||||
import numpy as np
|
||||
import torch
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.abspath(".."))
|
||||
|
||||
from datasets.oxford_pets import OxfordPets
|
||||
from datasets.oxford_flowers import OxfordFlowers
|
||||
from datasets.fgvc_aircraft import FGVCAircraft
|
||||
from datasets.dtd import DescribableTextures
|
||||
from datasets.eurosat import EuroSAT
|
||||
from datasets.stanford_cars import StanfordCars
|
||||
from datasets.food101 import Food101
|
||||
from datasets.sun397 import SUN397
|
||||
from datasets.caltech101 import Caltech101
|
||||
from datasets.ucf101 import UCF101
|
||||
from datasets.imagenet import ImageNet
|
||||
from datasets.imagenetv2 import ImageNetV2
|
||||
from datasets.imagenet_sketch import ImageNetSketch
|
||||
from datasets.imagenet_a import ImageNetA
|
||||
from datasets.imagenet_r import ImageNetR
|
||||
|
||||
from dassl.utils import setup_logger, set_random_seed, collect_env_info
|
||||
from dassl.config import get_cfg_default
|
||||
from dassl.data.transforms import build_transform
|
||||
from dassl.data import DatasetWrapper
|
||||
|
||||
import clip
|
||||
|
||||
# import pdb; pdb.set_trace()
|
||||
|
||||
|
||||
def print_args(args, cfg):
|
||||
print("***************")
|
||||
print("** Arguments **")
|
||||
print("***************")
|
||||
optkeys = list(args.__dict__.keys())
|
||||
optkeys.sort()
|
||||
for key in optkeys:
|
||||
print("{}: {}".format(key, args.__dict__[key]))
|
||||
print("************")
|
||||
print("** Config **")
|
||||
print("************")
|
||||
print(cfg)
|
||||
|
||||
|
||||
def reset_cfg(cfg, args):
|
||||
if args.root:
|
||||
cfg.DATASET.ROOT = args.root
|
||||
|
||||
if args.output_dir:
|
||||
cfg.OUTPUT_DIR = args.output_dir
|
||||
|
||||
if args.trainer:
|
||||
cfg.TRAINER.NAME = args.trainer
|
||||
|
||||
if args.backbone:
|
||||
cfg.MODEL.BACKBONE.NAME = args.backbone
|
||||
|
||||
if args.head:
|
||||
cfg.MODEL.HEAD.NAME = args.head
|
||||
|
||||
|
||||
def extend_cfg(cfg):
|
||||
"""
|
||||
Add new config variables.
|
||||
|
||||
E.g.
|
||||
from yacs.config import CfgNode as CN
|
||||
cfg.TRAINER.MY_MODEL = CN()
|
||||
cfg.TRAINER.MY_MODEL.PARAM_A = 1.
|
||||
cfg.TRAINER.MY_MODEL.PARAM_B = 0.5
|
||||
cfg.TRAINER.MY_MODEL.PARAM_C = False
|
||||
"""
|
||||
from yacs.config import CfgNode as CN
|
||||
|
||||
cfg.TRAINER.OURS = CN()
|
||||
cfg.TRAINER.OURS.N_CTX = 10 # number of context vectors
|
||||
cfg.TRAINER.OURS.CSC = False # class-specific context
|
||||
cfg.TRAINER.OURS.CTX_INIT = "" # initialize context vectors with given words
|
||||
cfg.TRAINER.OURS.WEIGHT_U = 0.1 # weight for the unsupervised loss
|
||||
|
||||
|
||||
def setup_cfg(args):
|
||||
cfg = get_cfg_default()
|
||||
extend_cfg(cfg)
|
||||
|
||||
# 1. From the dataset config file
|
||||
if args.dataset_config_file:
|
||||
cfg.merge_from_file(args.dataset_config_file)
|
||||
|
||||
# 2. From the method config file
|
||||
if args.config_file:
|
||||
cfg.merge_from_file(args.config_file)
|
||||
|
||||
# 3. From input arguments
|
||||
reset_cfg(cfg, args)
|
||||
|
||||
cfg.freeze()
|
||||
|
||||
return cfg
|
||||
|
||||
|
||||
def main(args):
|
||||
cfg = setup_cfg(args)
|
||||
if cfg.SEED >= 0:
|
||||
print("Setting fixed seed: {}".format(cfg.SEED))
|
||||
set_random_seed(cfg.SEED)
|
||||
setup_logger(cfg.OUTPUT_DIR)
|
||||
|
||||
if torch.cuda.is_available() and cfg.USE_CUDA:
|
||||
torch.backends.cudnn.benchmark = True
|
||||
|
||||
print_args(args, cfg)
|
||||
print("Collecting env info ...")
|
||||
print("** System info **\n{}\n".format(collect_env_info()))
|
||||
|
||||
######################################
|
||||
# Setup DataLoader
|
||||
######################################
|
||||
dataset = eval(cfg.DATASET.NAME)(cfg)
|
||||
|
||||
if args.split == "train":
|
||||
dataset_input = dataset.train_x
|
||||
elif args.split == "val":
|
||||
dataset_input = dataset.val
|
||||
else:
|
||||
dataset_input = dataset.test
|
||||
|
||||
tfm_train = build_transform(cfg, is_train=False)
|
||||
data_loader = torch.utils.data.DataLoader(
|
||||
DatasetWrapper(cfg, dataset_input, transform=tfm_train, is_train=False),
|
||||
batch_size=cfg.DATALOADER.TRAIN_X.BATCH_SIZE,
|
||||
sampler=None,
|
||||
shuffle=False,
|
||||
num_workers=cfg.DATALOADER.NUM_WORKERS,
|
||||
drop_last=False,
|
||||
pin_memory=(torch.cuda.is_available() and cfg.USE_CUDA),
|
||||
)
|
||||
|
||||
########################################
|
||||
# Setup Network
|
||||
########################################
|
||||
clip_model, _ = clip.load("RN50", "cuda", jit=False)
|
||||
clip_model.eval()
|
||||
###################################################################################################################
|
||||
# Start Feature Extractor
|
||||
feature_list = []
|
||||
label_list = []
|
||||
train_dataiter = iter(data_loader)
|
||||
for train_step in range(1, len(train_dataiter) + 1):
|
||||
batch = next(train_dataiter)
|
||||
data = batch["img"].cuda()
|
||||
feature = clip_model.visual(data)
|
||||
feature = feature.cpu()
|
||||
for idx in range(len(data)):
|
||||
feature_list.append(feature[idx].tolist())
|
||||
label_list.extend(batch["label"].tolist())
|
||||
save_dir = os.path.join(cfg.OUTPUT_DIR, cfg.DATASET.NAME)
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
save_filename = f"{args.split}"
|
||||
np.savez(
|
||||
os.path.join(save_dir, save_filename),
|
||||
feature_list=feature_list,
|
||||
label_list=label_list,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=str, default="", help="path to dataset")
|
||||
parser.add_argument("--output-dir", type=str, default="", help="output directory")
|
||||
parser.add_argument("--config-file", type=str, default="", help="path to config file")
|
||||
parser.add_argument(
|
||||
"--dataset-config-file",
|
||||
type=str,
|
||||
default="",
|
||||
help="path to config file for dataset setup",
|
||||
)
|
||||
parser.add_argument("--num-shot", type=int, default=1, help="number of shots")
|
||||
parser.add_argument("--split", type=str, choices=["train", "val", "test"], help="which split")
|
||||
parser.add_argument("--trainer", type=str, default="", help="name of trainer")
|
||||
parser.add_argument("--backbone", type=str, default="", help="name of CNN backbone")
|
||||
parser.add_argument("--head", type=str, default="", help="name of head")
|
||||
parser.add_argument("--seed", type=int, default=-1, help="only positive value enables a fixed seed")
|
||||
parser.add_argument("--eval-only", action="store_true", help="evaluation only")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
20
lpclip/feat_extractor.sh
Normal file
20
lpclip/feat_extractor.sh
Normal file
@@ -0,0 +1,20 @@
|
||||
# sh feat_extractor.sh
|
||||
DATA=/path/to/datasets
|
||||
OUTPUT='./clip_feat/'
|
||||
SEED=1
|
||||
|
||||
# oxford_pets oxford_flowers fgvc_aircraft dtd eurosat stanford_cars food101 sun397 caltech101 ucf101 imagenet
|
||||
for DATASET in oxford_pets
|
||||
do
|
||||
for SPLIT in train val test
|
||||
do
|
||||
python feat_extractor.py \
|
||||
--split ${SPLIT} \
|
||||
--root ${DATA} \
|
||||
--seed ${SEED} \
|
||||
--dataset-config-file ../configs/datasets/${DATASET}.yaml \
|
||||
--config-file ../configs/trainers/CoOp/rn50_val.yaml \
|
||||
--output-dir ${OUTPUT} \
|
||||
--eval-only
|
||||
done
|
||||
done
|
||||
129
lpclip/linear_probe.py
Normal file
129
lpclip/linear_probe.py
Normal file
@@ -0,0 +1,129 @@
|
||||
import numpy as np
|
||||
import os
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dataset", type=str, default="", help="path to dataset")
|
||||
parser.add_argument("--num_step", type=int, default=8, help="number of steps")
|
||||
parser.add_argument("--num_run", type=int, default=10, help="number of runs")
|
||||
parser.add_argument("--feature_dir", type=str, default="clip_feat", help="feature dir path")
|
||||
args = parser.parse_args()
|
||||
|
||||
dataset = args.dataset
|
||||
dataset_path = os.path.join(f"{args.feature_dir}", dataset)
|
||||
|
||||
train_file = np.load(os.path.join(dataset_path, "train.npz"))
|
||||
train_feature, train_label = train_file["feature_list"], train_file["label_list"]
|
||||
val_file = np.load(os.path.join(dataset_path, "val.npz"))
|
||||
val_feature, val_label = val_file["feature_list"], val_file["label_list"]
|
||||
test_file = np.load(os.path.join(dataset_path, "test.npz"))
|
||||
test_feature, test_label = test_file["feature_list"], test_file["label_list"]
|
||||
|
||||
os.makedirs("report", exist_ok=True)
|
||||
|
||||
val_shot_list = {1: 1, 2: 2, 4: 4, 8: 4, 16: 4}
|
||||
|
||||
for num_shot in [1, 2, 4, 8, 16]:
|
||||
test_acc_step_list = np.zeros([args.num_run, args.num_step])
|
||||
for seed in range(1, args.num_run + 1):
|
||||
np.random.seed(seed)
|
||||
print(f"-- Seed: {seed} --------------------------------------------------------------")
|
||||
# Sampling
|
||||
all_label_list = np.unique(train_label)
|
||||
selected_idx_list = []
|
||||
for label in all_label_list:
|
||||
label_collection = np.where(train_label == label)[0]
|
||||
selected_idx = np.random.choice(label_collection, size=num_shot, replace=False)
|
||||
selected_idx_list.extend(selected_idx)
|
||||
|
||||
fewshot_train_feature = train_feature[selected_idx_list]
|
||||
fewshot_train_label = train_label[selected_idx_list]
|
||||
|
||||
val_num_shot = val_shot_list[num_shot]
|
||||
val_selected_idx_list = []
|
||||
for label in all_label_list:
|
||||
label_collection = np.where(val_label == label)[0]
|
||||
selected_idx = np.random.choice(label_collection, size=val_num_shot, replace=False)
|
||||
val_selected_idx_list.extend(selected_idx)
|
||||
|
||||
fewshot_val_feature = val_feature[val_selected_idx_list]
|
||||
fewshot_val_label = val_label[val_selected_idx_list]
|
||||
|
||||
# search initialization
|
||||
search_list = [1e6, 1e4, 1e2, 1, 1e-2, 1e-4, 1e-6]
|
||||
acc_list = []
|
||||
for c_weight in search_list:
|
||||
clf = LogisticRegression(solver="lbfgs", max_iter=1000, penalty="l2", C=c_weight).fit(fewshot_train_feature, fewshot_train_label)
|
||||
pred = clf.predict(fewshot_val_feature)
|
||||
acc_val = sum(pred == fewshot_val_label) / len(fewshot_val_label)
|
||||
acc_list.append(acc_val)
|
||||
|
||||
print(acc_list, flush=True)
|
||||
|
||||
# binary search
|
||||
peak_idx = np.argmax(acc_list)
|
||||
c_peak = search_list[peak_idx]
|
||||
c_left, c_right = 1e-1 * c_peak, 1e1 * c_peak
|
||||
|
||||
def binary_search(c_left, c_right, seed, step, test_acc_step_list):
|
||||
clf_left = LogisticRegression(solver="lbfgs", max_iter=1000, penalty="l2", C=c_left).fit(fewshot_train_feature, fewshot_train_label)
|
||||
pred_left = clf_left.predict(fewshot_val_feature)
|
||||
acc_left = sum(pred_left == fewshot_val_label) / len(fewshot_val_label)
|
||||
print("Val accuracy (Left): {:.2f}".format(100 * acc_left), flush=True)
|
||||
|
||||
clf_right = LogisticRegression(solver="lbfgs", max_iter=1000, penalty="l2", C=c_right).fit(fewshot_train_feature, fewshot_train_label)
|
||||
pred_right = clf_right.predict(fewshot_val_feature)
|
||||
acc_right = sum(pred_right == fewshot_val_label) / len(fewshot_val_label)
|
||||
print("Val accuracy (Right): {:.2f}".format(100 * acc_right), flush=True)
|
||||
|
||||
# find maximum and update ranges
|
||||
if acc_left < acc_right:
|
||||
c_final = c_right
|
||||
clf_final = clf_right
|
||||
# range for the next step
|
||||
c_left = 0.5 * (np.log10(c_right) + np.log10(c_left))
|
||||
c_right = np.log10(c_right)
|
||||
else:
|
||||
c_final = c_left
|
||||
clf_final = clf_left
|
||||
# range for the next step
|
||||
c_right = 0.5 * (np.log10(c_right) + np.log10(c_left))
|
||||
c_left = np.log10(c_left)
|
||||
|
||||
pred = clf_final.predict(test_feature)
|
||||
test_acc = 100 * sum(pred == test_label) / len(pred)
|
||||
print("Test Accuracy: {:.2f}".format(test_acc), flush=True)
|
||||
test_acc_step_list[seed - 1, step] = test_acc
|
||||
|
||||
saveline = "{}, seed {}, {} shot, weight {}, test_acc {:.2f}\n".format(dataset, seed, num_shot, c_final, test_acc)
|
||||
with open(
|
||||
"./report/{}_s{}r{}_details.txt".format(args.feature_dir, args.num_step, args.num_run),
|
||||
"a+",
|
||||
) as writer:
|
||||
writer.write(saveline)
|
||||
return (
|
||||
np.power(10, c_left),
|
||||
np.power(10, c_right),
|
||||
seed,
|
||||
step,
|
||||
test_acc_step_list,
|
||||
)
|
||||
|
||||
for step in range(args.num_step):
|
||||
print(
|
||||
f"{dataset}, {num_shot} Shot, Round {step}: {c_left}/{c_right}",
|
||||
flush=True,
|
||||
)
|
||||
c_left, c_right, seed, step, test_acc_step_list = binary_search(c_left, c_right, seed, step, test_acc_step_list)
|
||||
# save results of last step
|
||||
test_acc_list = test_acc_step_list[:, -1]
|
||||
acc_mean = np.mean(test_acc_list)
|
||||
acc_std = np.std(test_acc_list)
|
||||
save_line = "{}, {} Shot, Test acc stat: {:.2f} ({:.2f})\n".format(dataset, num_shot, acc_mean, acc_std)
|
||||
print(save_line, flush=True)
|
||||
with open(
|
||||
"./report/{}_s{}r{}.txt".format(args.feature_dir, args.num_step, args.num_run),
|
||||
"a+",
|
||||
) as writer:
|
||||
writer.write(save_line)
|
||||
10
lpclip/linear_probe.sh
Normal file
10
lpclip/linear_probe.sh
Normal file
@@ -0,0 +1,10 @@
|
||||
feature_dir=clip_feat
|
||||
|
||||
for DATASET in OxfordPets
|
||||
do
|
||||
python linear_probe.py \
|
||||
--dataset ${DATASET} \
|
||||
--feature_dir ${feature_dir} \
|
||||
--num_step 8 \
|
||||
--num_run 3
|
||||
done
|
||||
Reference in New Issue
Block a user