-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathRAVDESS_CrossNetwork_RAVDESS.py
154 lines (99 loc) · 4.04 KB
/
RAVDESS_CrossNetwork_RAVDESS.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
# -*- coding: utf-8 -*-
"""Experiments with the RAVDESS-Crossmodal using Mel Spectrograms and 1 video frame
More Information: Barros, P., & Wermter, S. (2016). Developing crossmodal expression recognition based on a deep neural model. Adaptive behavior, 24(5), 373-396.
Parameters:
baseDirectory (String): Base directory where the experiment will be saved.
datasetFolderTrain (String): Folder where the audios used for training the model are stored
datasetFolderTest (String): Folder where the audios used for testing the model are stored
experimentName (String): Name of the experiment.
logManager (LogManager):
Author: Pablo Barros
Created on: 02.05.2018
Last Update: 16.06.2018
"""
import matplotlib
matplotlib.use('Agg')
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
from keras import backend as K
def set_keras_backend(backend):
if K.backend() != backend:
os.environ['KERAS_BACKEND'] = backend
reload(K)
assert K.backend() == backend
def runModel():
from KEF.Controllers import ExperimentManager
from KEF.DataLoaders import CrosschannelLoader_RAVDESS_Frame_WithDirectory
from KEF.Implementations import Cross_CNN_RAVDESS
dataDirectory = "/data/barros/RAVDESS_CrossChannel/"
"""
Both audio training and testing data must be stored as:
class1/
file1.wav
file2.wav
...
class2/
file1.wav
file2.wav
...
"""
datasetFolderAudioTrain = "/data/datasets/RAVDESS/Audio/High/train/" # RAVDESS_Audio_Train
datasetFolderAudioTest = "/data/datasets/RAVDESS/Audio/High/test/" # RAVDESS_Audio_test
"""
Both video training and testing data must be stored as:
class1/
video1/
file1.png
file2.png
video2/
file1.png
file2.png
file3.png
...
class2/
video1/
file1.png
file2.png
video2/
file1.png
file2.png
file3.png
...
"""
datasetFolderVisionTrain = "/data/datasets/RAVDESS/Frames/High/train/" # RAVDESS_Audio_train
datasetFolderVisionTest = "/data/datasets/RAVDESS/Frames/High/test/" # RAVDESS_Audio_test
""" Initianize all the loading parameters and modules necessary
image size: 64,64
"""
experimentManager = ExperimentManager.ExperimentManager(dataDirectory,
"RADVDESS_CCCNN",
verbose=True)
grayScale = True
preProcessingProperties = [(64, 64), grayScale]
fps = 90
stride = 30
""" Loading the training and testing data
"""
dataLoader = CrosschannelLoader_RAVDESS_Frame_WithDirectory.CrosschannelLoader_RAVDESS(experimentManager.logManager, preProcessingProperties, fps, stride)
dataLoader.loadTrainData([datasetFolderAudioTrain,datasetFolderVisionTrain])
dataLoader.loadTestData([datasetFolderAudioTest,datasetFolderVisionTest])
""" Building the Network
"""
cnnModel = Cross_CNN_RAVDESS.Cross_CNN_RAVDESS(experimentManager, "Vision_Deep_CNN", experimentManager.plotManager)
cnnModel.buildModel((dataLoader.dataTest.dataXAudio[0].shape,dataLoader.dataTest.dataXVideo[0].shape), len(dataLoader.dataTest.labelDictionary))
""" Training the Network
"""
cnnModel.train(dataLoader.dataTrain, dataLoader.dataTest, False)
cnnModel.save(experimentManager.modelDirectory)
set_keras_backend("tensorflow")
print K.backend
if K.backend == "tensorflow":
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
with tf.device('/gpu:0'):
runModel()
else:
runModel()