Skip to content

fix: typo Standford to Stanford #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

fix: typo Standford to Stanford #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
99d6147
fix: typo `Standford` to `Stanford`
aquastripe Oct 19, 2022
c7bde0c
chore: remove tensorflow experiments
aquastripe Oct 20, 2022
40e4d80
chore: reformat
aquastripe Oct 20, 2022
e8563d2
fix: typo `Standford` to `Stanford`
aquastripe Oct 20, 2022
5e6905b
fix: StopIteration error.
aquastripe Oct 20, 2022
73fa4a0
fix: StopIteration error.
aquastripe Oct 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 0 additions & 89 deletions benchmark/configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,50 +49,6 @@
"train_objective" : SUPERVISED,
"pretrained_weights" : None
},
{
"name" : "imagenet_supervised_tf",
"display_name" : "ImageNet Supervised (tensorflow)",
"color" : "grey",
"format" : TENSORFLOW,
"backbone" : RESNET50,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/supervised/resnet50_no_top.h5',
"training_dataset" : IMAGENET,
"train_objective" : SUPERVISED,
"pretrained_weights" : None
},
{
"name" : "imagenet_simclr",
"display_name" : "ImageNet SimCLR",
"color" : "C0",
"format" : TENSORFLOW,
"backbone" : RESNET50,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/simclr_v1/resnet50_simclr_v1_imagenet_no_top.h5',
"training_dataset" : IMAGENET,
"train_objective" : SIMCLR,
"pretrained_weights" : None
},
{
"name" : "imagenet_simclr_x4",
"display_name" : "ImageNet SimCLR x4",
"color" : "C1",
"format" : TENSORFLOW,
"backbone" : RESNET50_X4,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/simclr_v1_4x/225206_resnet50_simclr_v1_4x_imagenet_no_top.h5',
"training_dataset" : IMAGENET,
"train_objective" : SIMCLR,
"pretrained_weights" : None
},
{
"name" : "imagenet_simclr_v2",
"display_name" : "ImageNet SimCLR v2",
"color" : "C2",
"format" : TENSORFLOW,
"backbone" : RESNET50,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/simclr_v2/250228_resnet50_simclr_v2_1x_imagenet_no_top.h5',
"training_dataset" : IMAGENET,
"train_objective" : SIMCLR_V2,
"pretrained_weights" : None
},
{
"name" : "imagenet_swav",
"display_name" : "ImageNet SwAV",
Expand Down Expand Up @@ -137,17 +93,6 @@
"train_objective" : SUPERVISED,
"pretrained_weights" : None
},
{
"name" : "inat2021_simclr",
"display_name" : "iNat2021 SimCLR",
"color" : "C5",
"format" : TENSORFLOW,
"backbone" : RESNET50,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021/simclr_v1/2623871_resnet50_simclr_v1_inat20_no_top.h5',
"training_dataset" : INAT2021,
"train_objective" : SIMCLR,
"pretrained_weights" : None
},
{
"name" : "inat2021_mini_supervised",
"display_name" : "iNat2021 Mini Supervised",
Expand All @@ -170,39 +115,6 @@
"train_objective" : SUPERVISED,
"pretrained_weights" : None
},
{
"name" : "inat2021_mini_simclr",
"display_name" : "iNat2021 Mini SimCLR",
"color" : "tab:blue",
"format" : TENSORFLOW,
"backbone" : RESNET50,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021_mini/simclr_v1/resnet50_simclr_v1_inat20_mini_no_top.h5',
"training_dataset" : INAT2021_MINI,
"train_objective" : SIMCLR,
"pretrained_weights" : None
},
{
"name" : "inat2021_mini_simclr_x4",
"display_name" : "iNat2021 Mini SimCLR x4",
"color" : "tab:orange",
"format" : TENSORFLOW,
"backbone" : RESNET50_X4,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021_mini/simclr_v1_4x/488282_resnet50_simclr_4x_inat_mini_no_top.h5',
"training_dataset" : INAT2021_MINI,
"train_objective" : SIMCLR,
"pretrained_weights" : None
},
{
"name" : "inat2021_mini_simclr_v2",
"display_name" : "iNat2021 Mini SimCLR v2",
"color" : "tab:green",
"format" : TENSORFLOW,
"backbone" : RESNET50,
"weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021_mini/simclr_v2/488282_resnet50_simclr_v2_inat20_mini_no_top.h5',
"training_dataset" : INAT2021_MINI,
"train_objective" : SIMCLR_V2,
"pretrained_weights" : None
},
{
"name" : "inat2021_mini_swav",
"display_name" : "iNat2021 Mini SwAV",
Expand Down Expand Up @@ -247,5 +159,4 @@
"train_objective" : SUPERVISED,
"pretrained_weights" : IMAGENET
},

]
6 changes: 3 additions & 3 deletions benchmark/dataset_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,9 @@ def load_dataset(dataset_name, dataset_path):
return load_cub(dataset_path)
elif dataset_name == 'OxfordFlowers':
return load_oxford_flowers(dataset_path)
elif dataset_name == 'StandfordDogs':
elif dataset_name == 'StanfordDogs':
return load_stanford_dogs(dataset_path)
elif dataset_name == 'StandfordCars':
elif dataset_name == 'StanfordCars':
return load_stanford_cars(dataset_path)
else:
raise ValueError("Unknown dataset name: %s" % dataset_name)
raise ValueError("Unknown dataset name: %s" % dataset_name)
76 changes: 36 additions & 40 deletions benchmark/make_fg_plots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,16 @@
SMALL_SIZE = 12
MEDIUM_SIZE = 14
BIGGER_SIZE = 18
plt.rc('font', size=SMALL_SIZE) # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE) # legend fontsize
plt.rc('font', size=SMALL_SIZE) # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE) # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title


def create_fg_plots(fg_results_dir, output_dir):

fg_results_df = pd.DataFrame(None, columns=['model_name', 'name', 'acc'])

for model_spec in configs.model_specs:
Expand All @@ -42,18 +41,21 @@ def create_fg_plots(fg_results_dir, output_dir):
# models are the rows
fg_model_results_df = fg_results_df.pivot(index='model_name', columns='name', values='acc')
fg_model_results_df = fg_model_results_df.loc[[model_spec['name'] for model_spec in configs.model_specs]]
fg_model_results_df = fg_model_results_df[['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StandfordDogs', 'StandfordCars']]
fg_model_results_df = fg_model_results_df[
['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs']]

# datasets are the rows
fg_dataset_results_df = fg_results_df.pivot(index='name', columns='model_name', values='acc')
fg_dataset_results_df = fg_dataset_results_df.loc[['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StandfordDogs', 'StandfordCars']]
fg_dataset_results_df = fg_dataset_results_df.loc[
['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs']]
fg_dataset_results_df = fg_dataset_results_df[[model_spec['name'] for model_spec in configs.model_specs]]

#####################
# Stem plot for FG tasks
datasets = ['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StandfordDogs', 'StandfordCars']
# datasets = ['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StanfordDogs']
datasets = ['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs']

result_names=[
result_names = [
'imagenet_simclr',
'imagenet_simclr_x4',
'imagenet_simclr_v2',
Expand All @@ -72,20 +74,21 @@ def create_fg_plots(fg_results_dir, output_dir):

baseline_scores = fg_model_results_df.loc['imagenet_supervised'][datasets].values
exp_results = []
for model_name in result_names:

model_spec = next(model_spec for model_spec in configs.model_specs if model_spec['name'] == model_name)

model_specs = [
model_spec
for model_name in result_names
for model_spec in configs.model_specs if model_spec['name'] == model_name
]
for model_spec in model_specs:
if model_spec['name'] == 'imagenet_supervised':
continue

task_scores = fg_model_results_df.loc[ model_spec['name']][datasets].values - baseline_scores

task_scores = fg_model_results_df.loc[model_spec['name']][datasets].values - baseline_scores
r = {
'name' : model_spec['name'],
'scores' : task_scores,
'color' : model_spec['color'],
'display_name' : model_spec['display_name'],
'name': model_spec['name'],
'scores': task_scores,
'color': model_spec['color'],
'display_name': model_spec['display_name'],
}

if model_spec['name'] == 'random':
Expand All @@ -112,10 +115,9 @@ def create_fg_plots(fg_results_dir, output_dir):

exp_results.append(r)


result_df = pd.DataFrame(exp_results)

task_labels = ['Flowers102', 'CUB', 'CUBExpert', 'NABirds', 'StanfordDogs', 'StanfordCars']
task_labels = ['Flowers102', 'CUB', 'NABirds', 'StanfordDogs']

plot_utils.task_stem_plot(
result_df,
Expand All @@ -138,7 +140,7 @@ def create_fg_plots(fg_results_dir, output_dir):
##############
# Latex Table of results

result_names=[
result_names = [
'imagenet_supervised',
'imagenet_simclr',
'imagenet_simclr_x4',
Expand All @@ -157,7 +159,11 @@ def create_fg_plots(fg_results_dir, output_dir):
'inat2021_mini_swav',
'inat2021_mini_moco_v2'
]

model_specs = [
model_spec
for model_name in result_names
for model_spec in configs.model_specs if model_spec['name'] == model_name
]

num_cols = len(datasets)
num_rows = len(result_names)
Expand All @@ -176,23 +182,18 @@ def create_fg_plots(fg_results_dir, output_dir):
print(" & \t".join(header) + "\\\\")
print("\hline\hline")

for model_name in result_names:

model_spec = next(model_spec for model_spec in configs.model_specs if model_spec['name'] == model_name)

for model_spec in model_specs:
model_scores = fg_model_results_df.loc[model_spec['name']]

ys = []
ry = []
for i, label in enumerate(datasets):

v = model_scores[label]
ys.append(
"%0.3f" % v
)
ry.append(v)


td = model_spec['training_dataset'] if model_spec['training_dataset'] is not None else ""
to = model_spec['train_objective']
if model_spec['backbone'] == configs.RESNET50_X4:
Expand All @@ -211,26 +212,21 @@ def create_fg_plots(fg_results_dir, output_dir):
print("\\end{table*}")



def parse_args():

parser = argparse.ArgumentParser(description='Create the stem plot figure and latex table of results for the FG datasets.')

parser = argparse.ArgumentParser(
description='Create the stem plot figure and latex table of results for the FG datasets.')
parser.add_argument('--result_dir', dest='result_dir',
help='Path to the directory containing the FG results.', type=str,
required=True)

parser.add_argument('--output_dir', dest='output_dir',
help='Path to the directory to save figures and tables.', type=str,
required=True)

parsed_args = parser.parse_args()

return parsed_args

if __name__ == '__main__':


if __name__ == '__main__':
args = parse_args()

if not os.path.exists(args.result_dir):
Expand All @@ -240,4 +236,4 @@ def parse_args():
print("Creating %s to store plots and tables" % args.output_dir)
os.makedirs(args.output_dir)

create_fg_plots(args.result_dir, args.output_dir)
create_fg_plots(args.result_dir, args.output_dir)