Skip to content

Sample weights merged #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mverzett wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Sample weights merged #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e3a7211
Squash 4 commits to allow easier merging
May 14, 2018
2189fc0
making datasets compatible again
May 24, 2018
e485ec3
add uncertainties on the loss plots for MC-based trainings
May 30, 2018
2d308ef
introduced stretching
May 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 172 additions & 71 deletions AdaptMeDelphes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
from sklearn.model_selection import train_test_split
from keras.models import Model
from keras.layers import Input
from keras.metrics import binary_accuracy
from Layers import *
from weightedobjectives import binary_crossentropy_labelweights, binary_accuracy_labelweights
global_loss_list={}

parser = ArgumentParser()
Expand Down Expand Up @@ -39,6 +41,12 @@
parser.add_argument("--nepochscorr", help="number of epochs for MC correction", type=int, default=100)
parser.add_argument("--batchsize", help="batch size", type=int, default=10000)
parser.add_argument("--runIC", action='store_true')
parser.add_argument("--skewfraction", help='fraction to skew data samples, >0 dowsamples lights, <0 downsamples bs',
type=float, default=None)
parser.add_argument("--pretrainbtag", help='number of epochs to train the b-tagging part', type=int, default=0)
parser.add_argument("--pretraindatamc", help='number of epochs to train the data/mc part', type=int, default=0)
parser.add_argument("--fixweights" , action='store_true', help='fix label fractions weights')
parser.add_argument("--rightweights", action='store_true', help='set starting label fractions weights to the right value')

args = parser.parse_args()

Expand All @@ -52,7 +60,6 @@
print "Using GPU: ",os.environ['CUDA_VISIBLE_DEVICES']
except ImportError:
pass

else:
print "LXPLUS settings"
if args.gpu<0:
Expand All @@ -67,7 +74,6 @@
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
print('running on GPU '+str(args.gpu))


if args.gpufraction>0 and args.gpufraction<1:
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpufraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
Expand All @@ -79,17 +85,13 @@
from make_samples import make_sample
import pandas as pd
from keras.optimizers import Adam, SGD
from utils import save, fix_layers, open_layers, set_trainable


def save(df, fname):
dname = os.path.dirname(fname)
if not os.path.isdir(dname):
os.makedirs(dname)
records = df.to_records(index=False)
records.dtype.names = [str(i) for i in records.dtype.names]
np.save(fname, records)


def get_weight(model):
session = K.get_session()
sub_models = [i for i in model.layers if hasattr(i, 'layers')]
weight_layer = [i for m in sub_models for i in m.layers if i.name == 'weight_layer'][0]
return weight_layer.weights[0].eval(session=session).ravel()[0]

def schedule(x):
lr=args.lr
Expand All @@ -104,17 +106,26 @@ def schedule(x):


def run_model(outdir, training_method=1):

nepochstotal=args.nepochs
nepochscorr=args.nepochscorr
batchsize=args.batchsize

metrics_dict = {
'btag_discriminator' : binary_accuracy,
'datamc_discriminator' : [binary_accuracy_labelweights, binary_accuracy],
}

Inputs = Input((10,))
Inputs = [Input((10,)), Input((1,))]

global_loss_list['GradientReversal']=GradientReversal()
X_traintest, isB_traintest , isMC_traintest = make_sample(args.indir, args.addsv)
X_all, X_test, isB_all, isB_test, isMC_all, isMC_test = train_test_split(X_traintest, isB_traintest , isMC_traintest, test_size=0.1, random_state=42)

X_traintest, isB_traintest , isMC_traintest, weight_in_traintest, exp_weight = make_sample(
args.indir, args.addsv, args.skewfraction
)
get_expected_weight = lambda x: exp_weight #dummy function

X_all, X_test, isB_all, isB_test, isMC_all, isMC_test, weight_in_all, weight_in_test = train_test_split(
X_traintest, isB_traintest , isMC_traintest, weight_in_traintest, test_size=0.1, random_state=42
)


onesarray=np.ones(isB_all.shape[0])
Expand All @@ -125,42 +136,44 @@ def run_model(outdir, training_method=1):
#for compatibility
def modelIverseGrad(Inputs,rev_grad):
#build the models
feat = common_features(Inputs)
feat_outshape = Input(feat.get_layer(index=-1).output_shape)
feat = common_features(Inputs[0])
feat_outshape = Input(shape=(feat.get_layer(index=-1).output_shape[1:]))
btag = btag_discriminator(feat_outshape,dropoutrate=0.03)
datamc = datamc_discriminator(feat_outshape,rev_grad=rev_grad,dropoutrate=0.03)
datamc = datamc_discriminator([feat_outshape, Inputs[1]],rev_grad=rev_grad,dropoutrate=0.03)

return Model(inputs=Inputs, outputs=btag(feat(Inputs))+datamc(feat(Inputs)),name='fullModel')
return Model(inputs=Inputs, outputs=btag(feat(Inputs[0]))+datamc([feat(Inputs[0]), Inputs[1]]),name='fullModel')

model = modelIverseGrad(Inputs,rev_grad=args.lmb)



if(training_method == 'MC_training'):
model.compile(
loss=['binary_crossentropy']*4,
optimizer=optimizer,
loss_weights=[1., 0., 0., 0.],
weighted_metrics=['accuracy']
)
loss = ['binary_crossentropy']*2+[binary_crossentropy_labelweights, 'binary_crossentropy'],
optimizer=optimizer,
loss_weights=[1.,0.,0.,0.],
weighted_metrics = metrics_dict
)
history = model.fit(
X_all,
[isB_all, isB_all, isMC_all, isMC_all],
[X_all, weight_in_all],
[isB_all, isB_all, isMC_all, isMC_all],
batch_size=batchsize, epochs=nepochstotal, verbose=2, validation_split=0.2,
sample_weight=[isMC_all.ravel(), 1-isMC_all.ravel(),
1+0.5*isB_all.ravel(), 1-0.5*isB_all.ravel()],
)


sample_weight = [
isMC_all.ravel(),
1-isMC_all.ravel(),
1+0.5*isB_all.ravel(),
1-0.5*isB_all.ravel()],

)
elif(training_method == 'data_training'):
model.compile(
loss=['binary_crossentropy']*4,
loss = ['binary_crossentropy']*2+[binary_crossentropy_labelweights, 'binary_crossentropy'],
optimizer=optimizer,
loss_weights=[0.,1.,0.,0.],
weighted_metrics=['accuracy']
weighted_metrics = metrics_dict
)
history = model.fit(
X_all,
[X_all, weight_in_all],
[isB_all, isB_all, isMC_all, isMC_all],
batch_size=batchsize, epochs=nepochstotal, verbose=2, validation_split=0.2,
sample_weight = [
Expand All @@ -174,62 +187,142 @@ def modelIverseGrad(Inputs,rev_grad):

from block_models import mc_correction

da_epochs = nepochstotal
da_epochs = nepochstotal - args.pretrainbtag - args.pretraindatamc

if training_method == 'corrected_domain_adaptation':
da_epochs=nepochstotal-nepochscorr

#build the models
feat = common_features(Inputs)
feat = common_features(Inputs[0])
feat_outshape = Input(shape=(feat.get_layer(index=-1).output_shape[1:]))
btag = btag_discriminator(feat_outshape,dropoutrate=0.03)
datamc = datamc_discriminator(feat_outshape,rev_grad=args.lmb,dropoutrate=0.03)


modelallopen = Model(inputs=Inputs, outputs=btag(feat(Inputs))+datamc(feat(Inputs)),name='fullModel')
datamc = datamc_discriminator([feat_outshape, Inputs[1]],rev_grad=args.lmb,dropoutrate=0.03)
#fix the data/mc discriminator, leave weights trainable
datamc = set_trainable(datamc, 'ada_*', False)

if args.rightweights:
weight_layer = [i for i in datamc.layers if i.name == 'weight_layer'][0]
sess.run(
weight_layer.weights[0].assign(
np.array([[[exp_weight]]])
)
)

#print(feat.summary())
#print(btag.summary())
#print(datamc.summary())
if args.fixweights:
datamc = set_trainable(datamc, 'weight_*', False)

modelallopen = Model(inputs=Inputs, outputs=btag(feat(Inputs[0]))+datamc([feat(Inputs[0]), Inputs[1]]),name='fullModel')

#pre-train b-tagger
hist_prebtag = {}
if args.pretrainbtag:
print 'pre-training the b-tag classifier'
datamc = set_trainable(datamc, 'weight_*', False)
modelallopen.compile(
loss = ['binary_crossentropy']*2+[binary_crossentropy_labelweights, 'binary_crossentropy'],
optimizer = Adam(lr=args.lr),
loss_weights = [1.,0.,0.,0.],
weighted_metrics = metrics_dict
)

hist_prebtag = modelallopen.fit(
[X_all, weight_in_all],
[isB_all, isB_all, isMC_all, isMC_all],
batch_size=batchsize, epochs=args.pretrainbtag,
verbose=2, validation_split=0.2,
sample_weight = [
isMC_all.ravel(),
1-isMC_all.ravel(),
onesarray,
onesarray],
).history
hist_prebtag['weight'] = [get_weight(modelallopen)]*len(hist_prebtag['loss'])
hist_prebtag['real_weight'] = [get_expected_weight(modelallopen)]*len(hist_prebtag['loss'])
datamc = set_trainable(datamc, 'weight_*', not args.fixweights)

modelallopen.compile(
loss = ['binary_crossentropy']*4,
loss = ['binary_crossentropy']*2+[binary_crossentropy_labelweights, 'binary_crossentropy'],
optimizer = optimizer,
loss_weights = [1.,0.,args.weight,0.],
weighted_metrics=['accuracy']
weighted_metrics = metrics_dict
)


feat.trainable = False
btag.trainable = False
#set data/mc discriminator as trainable, fix weights
datamc = set_trainable(datamc, 'ada_*', True)
datamc = set_trainable(datamc, 'weight_*', False)

modelfixedbtag = Model(inputs=Inputs, outputs=btag(feat(Inputs))+datamc(feat(Inputs)),name='fullModel2')
modelfixedbtag = Model(inputs=Inputs, outputs=btag(feat(Inputs[0]))+datamc([feat(Inputs[0]), Inputs[1]]),name='fullModel2')

#pre-train data/mc
hist_predatamc = {}
if args.pretraindatamc:
print 'pre-training the data/MC classifier'

modelfixedbtag.compile(
loss = ['binary_crossentropy']*2+[binary_crossentropy_labelweights, 'binary_crossentropy'],
optimizer = Adam(lr=args.lr),
loss_weights = [0.,0.,args.weight,0.],
weighted_metrics = metrics_dict
)
hist_predatamc = modelfixedbtag.fit(
[X_all, weight_in_all],
[isB_all, isB_all, isMC_all, isMC_all],
batch_size=batchsize, epochs=args.pretraindatamc,
verbose=2, validation_split=0.2,
sample_weight = [
isMC_all.ravel(),
1-isMC_all.ravel(),
onesarray,
onesarray],
).history
hist_predatamc['weight'] = [get_weight(modelfixedbtag)]*len(hist_predatamc['loss'])
hist_predatamc['real_weight'] = [get_expected_weight(modelfixedbtag)]*len(hist_predatamc['loss'])


modelfixedbtag.compile(
loss = ['binary_crossentropy']*4,
loss = ['binary_crossentropy']*2+[binary_crossentropy_labelweights, 'binary_crossentropy'],
optimizer = optimizer,
loss_weights = [1.,0.,args.weight,0.],
weighted_metrics=['accuracy']
)

weighted_metrics = metrics_dict
)
from training_tools import train_models_adversarial
history=train_models_adversarial([modelallopen,modelfixedbtag],
[X_all],
[isB_all, isB_all, isMC_all, isMC_all],
da_epochs,
sample_weight=[isMC_all.ravel(),
1-isMC_all.ravel(),
onesarray,
onesarray],
batch_size=batchsize,
validation_split=0.2,
nstepspermodel=None,
custombatchgenerators=None,
verbose=1)
history=train_models_adversarial(
[modelallopen,modelfixedbtag],
[X_all, weight_in_all],
[isB_all, isB_all, isMC_all, isMC_all],
da_epochs,
sample_weight=[
isMC_all.ravel(),
1-isMC_all.ravel(),
onesarray,
onesarray],
batch_size=batchsize,
validation_split=0.2,
nstepspermodel=None,
custombatchgenerators=None,
verbose=1,
history_addenda = {
'weight' : get_weight,
'real_weight' : get_expected_weight,
}
)

print ('history type ', type(history))
history=pd.DataFrame(history)

#collate histories
if args.pretraindatamc:
history = pd.concat((
pd.DataFrame(hist_predatamc),
history
))
if args.pretrainbtag:
history = pd.concat((
pd.DataFrame(hist_prebtag),
history
))
model=modelallopen

if(training_method == 'corrected_domain_adaptation'):
Expand Down Expand Up @@ -307,9 +400,7 @@ def generate_noise(nsamples):
model=mcmodel

noise=generate_noise(X_test.shape[0])
X_test=[X_test,isMC_test,noise,isB_all]


X_test=[X_test,isMC_test,noise,isB_all]
else:
raise ValueError('%s is an unknown run option' % training_method)

Expand All @@ -319,13 +410,23 @@ def generate_noise(nsamples):
if args.nopred:
return history

predictions = model.predict(X_test)
predictions = model.predict([X_test, weight_in_test])
preds = pd.DataFrame()
preds['prediction'] = predictions[0].ravel()
preds['isB'] = isB_test
preds['isMC'] = isMC_test
save(preds, '%s/predictions.npy' %outdir)

from shutil import copyfile
import os

copyfile(
os.path.abspath(__file__),
'%s/script.py' % outdir
)

with open('%s/opts.txt' % outdir, 'w') as out:
out.write(args.__str__())

return history

Expand Down
Loading