130 lines
5.7 KiB
Python
130 lines
5.7 KiB
Python
import numpy as np
|
|
import os
|
|
from sklearn.linear_model import LogisticRegression
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--dataset", type=str, default="", help="path to dataset")
|
|
parser.add_argument("--num_step", type=int, default=8, help="number of steps")
|
|
parser.add_argument("--num_run", type=int, default=10, help="number of runs")
|
|
parser.add_argument("--feature_dir", type=str, default="clip_feat", help="feature dir path")
|
|
args = parser.parse_args()
|
|
|
|
dataset = args.dataset
|
|
dataset_path = os.path.join(f"{args.feature_dir}", dataset)
|
|
|
|
train_file = np.load(os.path.join(dataset_path, "train.npz"))
|
|
train_feature, train_label = train_file["feature_list"], train_file["label_list"]
|
|
val_file = np.load(os.path.join(dataset_path, "val.npz"))
|
|
val_feature, val_label = val_file["feature_list"], val_file["label_list"]
|
|
test_file = np.load(os.path.join(dataset_path, "test.npz"))
|
|
test_feature, test_label = test_file["feature_list"], test_file["label_list"]
|
|
|
|
os.makedirs("report", exist_ok=True)
|
|
|
|
val_shot_list = {1: 1, 2: 2, 4: 4, 8: 4, 16: 4}
|
|
|
|
for num_shot in [1, 2, 4, 8, 16]:
|
|
test_acc_step_list = np.zeros([args.num_run, args.num_step])
|
|
for seed in range(1, args.num_run + 1):
|
|
np.random.seed(seed)
|
|
print(f"-- Seed: {seed} --------------------------------------------------------------")
|
|
# Sampling
|
|
all_label_list = np.unique(train_label)
|
|
selected_idx_list = []
|
|
for label in all_label_list:
|
|
label_collection = np.where(train_label == label)[0]
|
|
selected_idx = np.random.choice(label_collection, size=num_shot, replace=False)
|
|
selected_idx_list.extend(selected_idx)
|
|
|
|
fewshot_train_feature = train_feature[selected_idx_list]
|
|
fewshot_train_label = train_label[selected_idx_list]
|
|
|
|
val_num_shot = val_shot_list[num_shot]
|
|
val_selected_idx_list = []
|
|
for label in all_label_list:
|
|
label_collection = np.where(val_label == label)[0]
|
|
selected_idx = np.random.choice(label_collection, size=val_num_shot, replace=False)
|
|
val_selected_idx_list.extend(selected_idx)
|
|
|
|
fewshot_val_feature = val_feature[val_selected_idx_list]
|
|
fewshot_val_label = val_label[val_selected_idx_list]
|
|
|
|
# search initialization
|
|
search_list = [1e6, 1e4, 1e2, 1, 1e-2, 1e-4, 1e-6]
|
|
acc_list = []
|
|
for c_weight in search_list:
|
|
clf = LogisticRegression(solver="lbfgs", max_iter=1000, penalty="l2", C=c_weight).fit(fewshot_train_feature, fewshot_train_label)
|
|
pred = clf.predict(fewshot_val_feature)
|
|
acc_val = sum(pred == fewshot_val_label) / len(fewshot_val_label)
|
|
acc_list.append(acc_val)
|
|
|
|
print(acc_list, flush=True)
|
|
|
|
# binary search
|
|
peak_idx = np.argmax(acc_list)
|
|
c_peak = search_list[peak_idx]
|
|
c_left, c_right = 1e-1 * c_peak, 1e1 * c_peak
|
|
|
|
def binary_search(c_left, c_right, seed, step, test_acc_step_list):
|
|
clf_left = LogisticRegression(solver="lbfgs", max_iter=1000, penalty="l2", C=c_left).fit(fewshot_train_feature, fewshot_train_label)
|
|
pred_left = clf_left.predict(fewshot_val_feature)
|
|
acc_left = sum(pred_left == fewshot_val_label) / len(fewshot_val_label)
|
|
print("Val accuracy (Left): {:.2f}".format(100 * acc_left), flush=True)
|
|
|
|
clf_right = LogisticRegression(solver="lbfgs", max_iter=1000, penalty="l2", C=c_right).fit(fewshot_train_feature, fewshot_train_label)
|
|
pred_right = clf_right.predict(fewshot_val_feature)
|
|
acc_right = sum(pred_right == fewshot_val_label) / len(fewshot_val_label)
|
|
print("Val accuracy (Right): {:.2f}".format(100 * acc_right), flush=True)
|
|
|
|
# find maximum and update ranges
|
|
if acc_left < acc_right:
|
|
c_final = c_right
|
|
clf_final = clf_right
|
|
# range for the next step
|
|
c_left = 0.5 * (np.log10(c_right) + np.log10(c_left))
|
|
c_right = np.log10(c_right)
|
|
else:
|
|
c_final = c_left
|
|
clf_final = clf_left
|
|
# range for the next step
|
|
c_right = 0.5 * (np.log10(c_right) + np.log10(c_left))
|
|
c_left = np.log10(c_left)
|
|
|
|
pred = clf_final.predict(test_feature)
|
|
test_acc = 100 * sum(pred == test_label) / len(pred)
|
|
print("Test Accuracy: {:.2f}".format(test_acc), flush=True)
|
|
test_acc_step_list[seed - 1, step] = test_acc
|
|
|
|
saveline = "{}, seed {}, {} shot, weight {}, test_acc {:.2f}\n".format(dataset, seed, num_shot, c_final, test_acc)
|
|
with open(
|
|
"./report/{}_s{}r{}_details.txt".format(args.feature_dir, args.num_step, args.num_run),
|
|
"a+",
|
|
) as writer:
|
|
writer.write(saveline)
|
|
return (
|
|
np.power(10, c_left),
|
|
np.power(10, c_right),
|
|
seed,
|
|
step,
|
|
test_acc_step_list,
|
|
)
|
|
|
|
for step in range(args.num_step):
|
|
print(
|
|
f"{dataset}, {num_shot} Shot, Round {step}: {c_left}/{c_right}",
|
|
flush=True,
|
|
)
|
|
c_left, c_right, seed, step, test_acc_step_list = binary_search(c_left, c_right, seed, step, test_acc_step_list)
|
|
# save results of last step
|
|
test_acc_list = test_acc_step_list[:, -1]
|
|
acc_mean = np.mean(test_acc_list)
|
|
acc_std = np.std(test_acc_list)
|
|
save_line = "{}, {} Shot, Test acc stat: {:.2f} ({:.2f})\n".format(dataset, num_shot, acc_mean, acc_std)
|
|
print(save_line, flush=True)
|
|
with open(
|
|
"./report/{}_s{}r{}.txt".format(args.feature_dir, args.num_step, args.num_run),
|
|
"a+",
|
|
) as writer:
|
|
writer.write(save_line)
|