init

main_1_18.py

@@ -0,0 +1,383 @@
+import json
+import os
+import random
+import shutil
+import time
+from clip import clip
+import numpy as np
+import torch.backends.cudnn as cudnn
+import torch.nn as nn
+import torch.optim
+from data.prepare_data_shot2 import generate_dataloader  # Prepare the data and dataloader
+from opts import opts  # The options for the project
+from trainer_1_18 import train  # For the training process
+from trainer_1_18 import warm_train  
+from trainer_1_18 import validate  # For the validate (test) process
+from models.DomainClassifierTarget import DClassifierForTarget
+from models.DomainClassifierSource import DClassifierForSource
+from engine import partial_model
+from clip.model import ModifiedResNet, VisionTransformer
+from datasets import build_dataset
+from datasets.utils import build_data_loader
+import torchvision.transforms as transforms
+import math
+import shutil
+
+
+best_prec1 = 0
+
+class Weight_Adapter(nn.Module):
+    def __init__(self, n_input, n_output,adapter_weights):
+        super().__init__()
+        self.linear1 = nn.Linear(n_input, n_output, bias=False)
+        self.linear1.weight.data = adapter_weights  # Initialize linear layer weights
+
+    def forward(self, x):
+        x = self.linear1(x.float())
+        return x
+class Adapter(nn.Module):
+    def __init__(self, n_input,n_output):
+        super().__init__()
+        self.residual_ratio = 0.2
+        self.linear1 = nn.Linear(n_input, n_output, bias=False)
+        # self.linear1.weight.data = adapter_weights  # Initialize linear layer weights
+        self.relu=nn.ReLU()
+
+    def forward(self, x):
+        a=x
+        x = self.linear1(x.float())
+        x=self.relu(x)
+        # x = self.residual_ratio * x + (1 - self.residual_ratio) * a
+
+        return x
+def zeroshot_classifier(classname, templates, CLIP_Text):
+    with torch.no_grad():
+        classname = classname.replace('_', ' ')
+        str_prompts = [template.format(classname) for template in templates]
+        prompts = torch.cat([clip.tokenize(p) for p in str_prompts]).cuda()
+        features, eot_indices = CLIP_Text(prompts)
+    return features, eot_indices
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the precision@k for the specified values of k"""
+    maxk = max(topk)
+    batch_size = target.size(0)
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    _2, pred2 = output.topk(1, 1, True, True)
+    a = target.view(1, -1)
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+    # print(correct)
+    res = []
+    for k in topk:
+        correct_k = correct[:k].contiguous().view(-1).float().sum(0, keepdim=True)
+        res.append(correct_k.mul_(100.0 / batch_size))
+    return res
+
+
+def all_classifier(classnames, templates, model):
+    with torch.no_grad():
+        zeroshot_weights = []
+        for classname in classnames:
+            classname = classname.replace('_', ' ')
+            texts = [template.format(classname) for template in templates]  # format with class
+            texts = clip.tokenize(texts).cuda()  # tokenizeclip.tokenize向量化文字
+            class_embeddings = model.encode_text(texts)  # embed with text encoder
+            class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)
+            class_embedding = class_embeddings.mean(dim=0)
+            class_embedding /= class_embedding.norm()
+            zeroshot_weights.append(class_embedding)
+
+        zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()
+    return zeroshot_weights
+
+def validate_train(classnames, templates,val_loader, model, args, zero_shots, criterion,
+                   optimizer, scheduler, alpha, beta, gama, CLIP_Text, CLIP_Image,Image_Encoder,Text_Encoder,adapter):
+    global best_target_acc
+    Compu1_acc = AverageMeter()
+    losses = AverageMeter()
+    CLIP_Text.eval()
+    CLIP_Image.eval()
+    Image_Encoder.eval()
+    Text_Encoder.eval()
+    adapter.eval()
+    logit_scale = 4.60517
+    logit_scale = math.exp(logit_scale)
+    # switch to evaluate mode
+
+    for i, (image, label) in enumerate(val_loader):
+        image = image.cuda()
+        label = label.cuda()
+        zeroshot_weights = []
+        for j in range(len(label)):
+            features, eot_indices = zeroshot_classifier(classnames[label[j]], templates, CLIP_Text)
+            with torch.no_grad():
+                class_embeddings = Text_Encoder(features, eot_indices)
+                class_embeddings = class_embeddings / class_embeddings.norm(dim=-1, keepdim=True)
+                class_embedding = class_embeddings.mean(dim=0)
+                class_embedding = class_embedding / class_embedding.norm()
+                class_embedding = class_embedding / class_embedding.norm(dim=-1, keepdim=True)
+                zeroshot_weights.append(class_embedding)
+        zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()
+        input_source = zeroshot_weights
+        input_source = input_source.T
+        input_target = image.cuda()
+        target_target = label.cuda()
+        target_source = label.cuda()
+
+        input_target_clip = model.encode_image(input_target)
+        # clip图片编码器
+        with torch.no_grad():
+            input_target_temp = CLIP_Image(input_target)
+            input_target_add = Image_Encoder(input_target_temp)
+            output_source = adapter(input_source) * logit_scale
+            output_target = adapter(input_target_add) * logit_scale
+
+        # 3
+        loss_source = criterion(output_source[:, :len(classnames)], target_source)
+        loss_target = criterion(output_target[:, len(classnames):], target_target)
+
+        # measure accuracy and record loss
+        prec1_source, _ = accuracy(output_source.data[:, :len(classnames)], target_source, topk=(1, 5))
+        prec1_target, _ = accuracy(output_target.data[:, len(classnames):], target_target, topk=(1, 5))
+
+
+        # 2
+        logits2 =100.* input_target_clip.float() @ zero_shots.float()
+
+        # 3
+        logits3 = output_target[:, len(classnames):]
+
+        # compu1:1-2+3:
+        compu1 = beta*logits2 + gama * logits3
+
+        compu1_acc = accuracy(compu1, target_target, topk=(1, 5))
+        loss = criterion(compu1, target_target)
+        Compu1_acc.update(compu1_acc[0].item(), image.size(0))
+        losses.update(loss.item(), image.size(0))
+        print('loss:', loss.item())
+        print(i, '/', len(val_loader))
+        print('Compu1_acc:', Compu1_acc.val, 'alpha:', alpha.item(), 'beta:', beta.item(), 'gama:', gama.item())
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        scheduler.step()
+
+    print('Compu1_acc.avg', Compu1_acc.avg, 'alpha:', alpha.item(), 'beta:', beta.item(), 'gama:', gama.item(),
+          'losses.avg', losses.avg)
+    return Compu1_acc.avg, alpha.item(), beta.item(), gama.item()
+
+
+def main():
+
+    seed = 2023
+    random.seed(seed)
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+    global args, best_prec1
+    current_epoch = 0
+    args = opts()
+    clip.available_models()
+    model, preprocess = clip.load(args.name)
+    # model = model.cuda()
+    model.float()
+
+    if os.path.exists(args.filename_dir):
+        print('exist')
+    else:
+        os.makedirs(args.filename_dir)
+
+    filename=args.filename_dir+args.dataset_name+'.txt'
+    if os.path.exists(filename):
+        print(filename + " exist!")
+    else:
+        print("create " + filename)
+        f = open(filename, "w")
+        f.close()
+
+    epx_dir=args.savedir+args.dataset_name+'_epx/'+str(args.shot)+'shot'+'/'
+    if os.path.exists(epx_dir):
+        print('epx_dir exist')
+    else:
+        os.makedirs(epx_dir)
+
+
+    dataset = build_dataset(args.dataset_name, args.dataset_dir, args.shot)
+    classnames=dataset.classnames
+    templates=dataset.template
+
+
+    # loader = build_data_loader(data_source=dataset.val, batch_size=64, is_train=False, tfm=preprocess,
+    #                                shuffle=False)
+    loader = build_data_loader(data_source=dataset.test, batch_size=64, is_train=False, tfm=preprocess,
+                                    shuffle=False)
+
+    train_tranform = transforms.Compose([
+        transforms.RandomResizedCrop(size=224, scale=(0.5, 1), interpolation=transforms.InterpolationMode.BICUBIC),
+        transforms.RandomHorizontalFlip(p=0.5),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711))
+    ])
+    #
+    # train_loader_cache = build_data_loader(data_source=dataset.train_x, batch_size=256, tfm=train_tranform,
+    #                                        is_train=True, shuffle=False)
+    train_loader_shuffle = build_data_loader(data_source=dataset.train_x, batch_size=256, tfm=train_tranform, is_train=True,
+                                       shuffle=True)
+
+
+    criterion = nn.CrossEntropyLoss().cuda()
+    if not os.path.isdir(args.log):
+        os.makedirs(args.log)
+    log = open(os.path.join(args.log, 'log.txt'), 'a')
+    state = {k: v for k, v in args._get_kwargs()}
+    log.write(json.dumps(state) + '\n')
+    log.close()
+
+    cudnn.benchmark = True  # Benchmark模式会提升计算速度，但是由于计算中有随机性，每次网络前馈结果略有差异
+
+    log = open(os.path.join(args.log, 'log.txt'), 'a')
+    log.write('\n-------------------------------------------\n')
+    log.write(time.asctime(time.localtime(time.time())))
+    log.write('\n-------------------------------------------')
+    log.close()
+
+    # process the data and prepare the dataloaders.
+    # train_loader_shuffle, loader = generate_dataloader(args, preprocess)
+
+
+    #拆分CLIP图像编码器
+    if args.name   =="ViT-B/16":
+        CLIP_Text,Text_Encoder=partial_model.get_text(model,text_layer_idx=0)
+        assert type(model.visual) == VisionTransformer
+        CLIP_Image,Image_Encoder=partial_model.get_image_vit(model.visual, image_layer_idx=0)
+    elif args.name =="ViT-B/32":
+        CLIP_Text, Text_Encoder = partial_model.get_text(model, text_layer_idx=0)
+        assert type(model.visual) == VisionTransformer
+        CLIP_Image, Image_Encoder = partial_model.get_image_vit(model.visual, image_layer_idx=0)
+    elif args.name == "RN50":
+        CLIP_Text,Text_Encoder =partial_model.get_text(model,text_layer_idx=0)
+        assert type(model.visual) == ModifiedResNet
+        CLIP_Image,Image_Encoder=partial_model.get_image_resnet(model.visual, image_layer_idx=1)
+    elif args.name == "RN101":
+        CLIP_Text, Text_Encoder = partial_model.get_text(model, text_layer_idx=0)
+        assert type(model.visual) == ModifiedResNet
+        CLIP_Image, Image_Encoder = partial_model.get_image_resnet(model.visual, image_layer_idx=0)
+    elif args.name == "RN50x16":
+        CLIP_Text, Text_Encoder = partial_model.get_text(model, text_layer_idx=0)
+        assert type(model.visual) == ModifiedResNet
+        CLIP_Image, Image_Encoder = partial_model.get_image_resnet(model.visual, image_layer_idx=0)
+
+    # 1000类标签经过clip
+    model=model.cuda()
+    zero_weights = all_classifier(classnames, templates, model)
+    CLIP_Text,Text_Encoder,CLIP_Image,Image_Encoder=CLIP_Text.cuda(),Text_Encoder.cuda(),CLIP_Image.cuda(),Image_Encoder.cuda()
+    weights_path = None
+    best_epoch=0
+    best_init_acc=0
+    criterion_classifier_target = DClassifierForTarget(nClass=len(classnames)).cuda()
+    criterion_classifier_source = DClassifierForSource(nClass=len(classnames)).cuda()
+    text_weights=zero_weights
+
+    adapter_weights=torch.cat([text_weights,text_weights],dim=1).T
+    adapter = Weight_Adapter(1024, 2 * len(classnames),adapter_weights).cuda()
+    if args.shot>=18:
+        optimizer = torch.optim.AdamW([{'params': adapter.parameters(), 'lr': 0.001},
+                                       {'params': Image_Encoder.parameters(), 'lr':0.00001},
+                                       {'params': Text_Encoder.parameters(), 'lr': 0.00001}],
+                                      eps=1e-5)
+    else:
+        optimizer = torch.optim.AdamW([{'params': adapter.parameters(), 'lr': 0.0001},
+                                       {'params': Image_Encoder.parameters(), 'lr':0.00001},
+                                       {'params': Text_Encoder.parameters(), 'lr': 0.00001}],
+                                      eps=1e-5)
+    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs * len(train_loader_shuffle))
+    source_train_loader_batch = enumerate(train_loader_shuffle)
+    dir=args.savedir+args.dataset_name+'_epx/'+str(args.shot)+'shot'+'/'
+    torch.save(CLIP_Text, dir + '/CLIP_Text.pth')
+    torch.save(CLIP_Image, dir + '/CLIP_Image.pth')
+    while (current_epoch < args.epochs):
+        if(current_epoch<40):
+            source_train_loader_batch, current_epoch, new_epoch_flag = warm_train(classnames, templates,
+            train_loader_shuffle,
+            source_train_loader_batch,
+            model,
+            adapter,
+            criterion_classifier_source,
+            criterion_classifier_target,
+            optimizer,
+            current_epoch,
+            args, scheduler, criterion, CLIP_Text,Text_Encoder,CLIP_Image,Image_Encoder)
+            
+        else:
+            source_train_loader_batch, current_epoch, new_epoch_flag = train(classnames, templates,
+            train_loader_shuffle,
+            source_train_loader_batch,
+            model,
+            adapter,
+            criterion_classifier_source,
+            criterion_classifier_target,
+            optimizer,
+            current_epoch,
+            args, scheduler, criterion, CLIP_Text,Text_Encoder,CLIP_Image,Image_Encoder)
+        # evaluate on the val data
+        if new_epoch_flag:
+            if (current_epoch + 1) % args.test_freq == 0 or current_epoch == 0:
+                if current_epoch >=args.valepoch:
+                    prec1 = validate(classnames, templates,loader, model, adapter, current_epoch, args, zero_weights, criterion,
+                                                 CLIP_Text,Text_Encoder,CLIP_Image,Image_Encoder)
+                    # record the best prec1 and save checkpoint
+                    is_best = prec1 > best_prec1
+                    if prec1 > args.valacc:
+                        save_dir = dir+'/epoch_' + str(current_epoch) + '_' + str(
+                            prec1)
+                        if not os.path.isdir(save_dir):
+                            os.mkdir(save_dir)
+                        torch.save(adapter, save_dir + '/_adapter_extractor.pth')
+                        torch.save(Text_Encoder, save_dir + '/Text_Encoder.pth')
+                        torch.save(Image_Encoder, save_dir + '/Image_Encoder.pth')
+                    best_prec1 = max(prec1, best_prec1)
+                    if is_best:
+                        weights_path=save_dir
+                        best_init_acc=best_prec1
+                        best_epoch=current_epoch
+                        log = open(os.path.join(args.log, 'log.txt'), 'a')
+                        log.write('Best acc: %3f' % (best_prec1))
+                        log.close()
+    filename=args.filename_dir+args.dataset_name+'.txt'
+    strr=str(args.shot)+'shot'+' '+'best_epoch'+' '+str(best_epoch)+' '+'best_init_acc'+' '+str(best_init_acc)
+    with open(filename, 'a') as f:
+        f.write(strr+ '\n')
+        f.close()
+    log = open(os.path.join(args.log, 'log.txt'), 'a')
+    log.write('\n-------------------------------------------\n')
+    log.write(time.asctime(time.localtime(time.time())))
+    log.write('\n-------------------------------------------\n')
+    log.close()
+
+
+if __name__ == '__main__':
+    main()