Add initial files

Adds files for testing and evaluation on benchmark datasets (NYU, Make3D), including the required 'Interleaving' dagnn-layer

matlab/+dagnn/Combine.m

@@ -0,0 +1,34 @@
+classdef Combine < dagnn.ElementWise
+
+    methods
+        function outputs = forward(self, inputs, params)
+            %double the size of feature maps, combining four responses
+            Y = zeros(size(inputs{1},1)*2, size(inputs{1},2)*2, size(inputs{1},3), size(inputs{1},4), 'like', inputs{1});
+            Y(1:2:end, 1:2:end, :, :) = inputs{1};  %A
+            Y(2:2:end, 1:2:end, :, :) = inputs{2};  %C
+            Y(1:2:end, 2:2:end, :, :) = inputs{3};  %B
+            Y(2:2:end, 2:2:end, :, :) = inputs{4};  %D
+            outputs{1} = Y;
+        end
+
+        function [derInputs, derParams] = backward(self, inputs, params, derOutputs)
+            %split the feature map into four feature maps of half size
+            derInputs{1} = derOutputs{1}(1:2:end, 1:2:end, :, :);
+            derInputs{2} = derOutputs{1}(2:2:end, 1:2:end, :, :);
+            derInputs{3} = derOutputs{1}(1:2:end, 2:2:end, :, :);
+            derInputs{4} = derOutputs{1}(2:2:end, 2:2:end, :, :);
+            derParams = {} ;
+        end
+
+        function outputSizes = getOutputSizes(obj, inputSizes)
+            outputSizes{1}(1) = 2*inputSizes{1}(1);
+            outputSizes{1}(2) = 2*inputSizes{1}(2);
+            outputSizes{1}(3) = inputSizes{1}(3);
+            outputSizes{1}(4) = inputSizes{1}(4);
+        end
+
+        function obj = Combine(varargin)
+            obj.load(varargin) ;
+        end
+    end
+end

matlab/DepthMapPrediction.m

@@ -0,0 +1,78 @@
+function pred = DepthMapPrediction(imdb, net, varargin)
+
+% Depth prediction (inference) using a trained model.
+% Inputs (imdb) can be either from the NYUDepth_v2 or Make3D dataset, along
+% with the corresponding trained model (net). Additionally, the evaluation
+% can be run for any single image. MatConvNet library has to be already
+% setup for this function to work properly.
+% -------------------------------------------------------------------------
+% Inputs:
+%   - imdb: a structure with fields 'images' and 'depths' in the case of
+%           the benchmark datasets with known ground truth. imdb could
+%           alternatively be any single RGB image of size NxMx3 in [0,255]
+%           or a tensor of D input images NxMx3xD.
+%   - net:  a trained model of type struct (suitable to be converted to a 
+%           DagNN object and successively processed using the DagNN 
+%           wrapper). For testing on arbitrary images, use NYU model for 
+%           indoor and Make3D model for outdoor scenes respectively.
+% -------------------------------------------------------------------------
+
+opts.gpu = false;           % Set to true (false) for GPU (CPU only) support 
+opts.plot = false;          % Set to true to visualize the predictions during inference
+opts = vl_argparse(opts, varargin);
+
+% Set network properties
+net = dagnn.DagNN.loadobj(net);
+net.mode = 'test';
+out = net.getVarIndex('prediction');
+if opts.gpu
+    net.move('gpu');
+end
+
+% Check input
+if isa(imdb, 'struct')
+    % case of benchmark datasets (NYU, Make3D)
+    images = imdb.images;
+    groundTruth = imdb.depths;
+else
+    % case of arbitrary image(s)
+    images = imdb;
+    images = imresize(images, net.meta.normalization.imageSize(1:2));
+    groundTruth = [];
+end
+
+% Get output size for initialization
+varSizes = net.getVarSizes({'data', net.meta.normalization.imageSize});  % get variable sizes
+pred = zeros(varSizes{out}(1), varSizes{out}(2), varSizes{out}(3), size(images, 4));    % initiliaze 
+
+if opts.plot, figure(); end
+
+fprintf('predicting...\n');
+for i = 1:size(images, 4)
+    % get input image
+    im = single(images(:,:,:,i));
+    if opts.gpu
+        im = gpuArray(im);
+    end
+
+    % run the CNN
+    inputs = {'data', im};
+    net.eval(inputs) ;
+
+    % obtain prediction
+    pred(:,:,i) = gather(net.vars(out).value);
+
+    % visualize results
+    if opts.plot
+        colormap jet
+        if ~isempty(groundTruth)
+            subplot(1,3,1), imagesc(uint8(images(:,:,:,i))), title('RGB Input'), axis off
+            subplot(1,3,2), imagesc(groundTruth(:,:,i)), title('Depth Ground Truth'), axis off
+            subplot(1,3,3), imagesc(pred(:,:,i)), title('Depth Prediction'), axis off
+        else
+            subplot(1,2,1), imagesc(uint8(images(:,:,:,i))), title('RGB Input'), axis off
+            subplot(1,2,2), imagesc(pred(:,:,i)), title('Depth Prediction'), axis off
+        end
+        drawnow;
+    end
+end

matlab/error_metrics.m

@@ -0,0 +1,40 @@
+function results = error_metrics(pred, gt, mask)
+
+% Compute error metrics on benchmark datasets
+% -------------------------------------------------------------------------
+
+% make sure predictions and ground truth have same dimensions
+if size(pred) ~= size(gt)
+    pred = imresize(pred, [size(gt,1), size(gt,2)], 'bilinear');  
+end
+
+if isempty(mask)
+    n_pxls = numel(gt);
+else
+    n_pxls = sum(mask(:));  % average over valid pixels only
+end
+
+fprintf('\n Errors computed over the entire test set \n');
+fprintf('------------------------------------------\n');
+
+% Mean Absolute Relative Error
+rel = abs(gt(:) - pred(:)) ./ gt(:);    % compute errors
+rel(~mask) = 0;                         % mask out invalid ground truth pixels
+rel = sum(rel) / n_pxls;                % average over all pixels
+fprintf('Mean Absolute Relative Error: %4f\n', rel);
+
+% Root Mean Squared Error
+rms = (gt(:) - pred(:)).^2;
+rms(~mask) = 0;
+rms = sqrt(sum(rms) / n_pxls);
+fprintf('Root Mean Squared Error: %4f\n', rms);
+
+% LOG10 Error
+lg10 = abs(log10(gt(:)) - log10(pred(:)));
+lg10(~mask) = 0;
+lg10 = sum(lg10) / n_pxls ; 
+fprintf('Mean Log10 Error: %4f\n', lg10);
+
+results.rel = rel;
+results.rms = rms;
+results.log10 = lg10;

matlab/evaluateMake3D.m

@@ -0,0 +1,117 @@
+function evaluateMake3D
+
+% Evaluation of depth prediction on Make3D dataset.
+
+% -------------------------------------------------------------------------
+% Setup MatConvNet
+% -------------------------------------------------------------------------
+
+% Set your matconvnet path here:
+matconvnet_path = '../../matconvnet-1.0-beta20';
+setupMatConvNet(matconvnet_path);
+
+% -------------------------------------------------------------------------
+% Options
+% -------------------------------------------------------------------------
+
+opts.dataDir = fullfile(pwd, 'Make3D');     % working directory
+opts.interp = 'nearest';    % interpolation method applied during resizing
+opts.imageSize = [460,345]; % desired image size for evaluation
+
+netOpts.gpu = true;     % set to true to enable GPU support
+netOpts.plot = true;    % set to true to visualize the predictions during inference
+
+% -------------------------------------------------------------------------
+% Prepate data
+% -------------------------------------------------------------------------
+
+imdb = get_Make3D(opts);
+net = get_model(opts);
+
+% Test set
+testSet.images = imdb.images(:,:,:, imdb.set == 2);
+testSet.depths = imdb.depths(:,:, imdb.set == 2);
+
+% resize images to input resolution (equal to round(opts.imageSize/2))
+testSet.images = imresize(testSet.images, net.meta.normalization.imageSize(1:2), opts.interp);    
+% resize depth to opts.imageSize resolution
+testSet.depths = imresize(testSet.depths, opts.imageSize, opts.interp);     
+
+% -------------------------------------------------------------------------
+% Evaluate network
+% -------------------------------------------------------------------------
+
+% Get predictions
+predictions = DepthMapPrediction(testSet, net, netOpts);
+predictions = squeeze(predictions); % remove singleton dimensions
+predictions = imresize(predictions, [size(testSet.depths,1), size(testSet.depths,2)], 'bilinear'); %rescale
+
+% Error calculation
+c1_mask = testSet.depths > 0 & testSet.depths < 70;
+errors = error_metrics(predictions, testSet.depths, c1_mask);
+
+% Save results
+fprintf('\nsaving predictions...');
+save(fullfile(opts.dataDir, 'results.mat'), 'predictions', 'errors', '-v7.3');
+fprintf('done!\n');
+
+
+function imdb = get_Make3D(opts)
+% -------------------------------------------------------------------------
+% Download required data (test only)
+% -------------------------------------------------------------------------
+
+opts.dataDirImages = fullfile(opts.dataDir, 'data', 'Test134');
+opts.dataDirDepths = fullfile(opts.dataDir, 'data', 'Gridlaserdata');
+
+% Download test set
+if ~exist(opts.dataDirImages, 'dir')
+    fprintf('downloading Make3D testing images (~190 MB)...');
+    mkdir(opts.dataDirImages);
+    untar('http://www.cs.cornell.edu/~asaxena/learningdepth/Test134.tar.gz', fileparts(opts.dataDirImages));
+    fprintf('done.\n');
+end
+
+if ~exist(opts.dataDirDepths, 'dir')
+    fprintf('downloading Make3D testing depth maps (~22 MB)...');
+    mkdir(opts.dataDirDepths);   
+    untar('http://www.cs.cornell.edu/~asaxena/learningdepth/Test134Depth.tar.gz', fileparts(opts.dataDirDepths));
+    fprintf('done.\n');
+end
+
+fprintf('preparing testing data...');
+img_files = dir(fullfile(opts.dataDirImages, 'img-*.jpg'));
+depth_files = dir(fullfile(opts.dataDirDepths, 'depth_sph_corr-*.mat'));
+
+% Verify that the correct number of files has been found
+assert(numel(img_files)==134, 'Incorrect number of Make3D test images. \n');
+assert(numel(depth_files)==134, 'Incorrect number of Make3D test depths. \n');
+
+% Read dataset files and store necessary information to imdb structure
+for i = 1:numel(img_files)
+    imdb.images(:,:,:,i) = single(imread(fullfile(opts.dataDirImages, img_files(i).name)));   % get RGB image
+    gt = load(fullfile(opts.dataDirDepths, depth_files(i).name));
+    imdb.depths(:,:,i) = single(gt.Position3DGrid(:,:,4));  % get depth channel
+    imdb.set(i) = 2;
+end
+fprintf(' done!\n');
+
+
+
+function net = get_model(opts)
+% -------------------------------------------------------------------------
+% Download trained models
+% -------------------------------------------------------------------------
+
+opts.dataDir = fullfile(opts.dataDir, 'models');
+if ~exist(opts.dataDir, 'dir'), mkdir(opts.dataDir); end
+
+filename = fullfile(opts.dataDir, 'Make3D_ResNet-UpProj.mat');
+if ~exist(filename, 'file')
+    url = 'http://campar.in.tum.de/files/rupprecht/depthpred/Make3D_ResNet-UpProj.zip';
+    fprintf('downloading trained model: %s\n', url);
+    unzip(url, opts.dataDir);
+end
+
+net = load(filename);
+

matlab/evaluateNYU.m

@@ -0,0 +1,115 @@
+function evaluateNYU
+
+% Evaluation of depth prediction on NYU Depth v2 dataset. 
+
+% -------------------------------------------------------------------------
+% Setup MatConvNet
+% -------------------------------------------------------------------------
+
+% Set your matconvnet path here:
+matconvnet_path = '../../matconvnet-1.0-beta20';
+setupMatConvNet(matconvnet_path);
+
+% -------------------------------------------------------------------------
+% Options
+% -------------------------------------------------------------------------
+
+opts.dataDir = fullfile(pwd, 'NYU');    % working directory
+opts.interp = 'nearest';                % interpolation method applied during resizing
+
+netOpts.gpu = true;     % set to true to enable GPU support      
+netOpts.plot = true;    % set to true to visualize the predictions during inference          
+
+% -------------------------------------------------------------------------
+% Prepate data
+% -------------------------------------------------------------------------
+
+imdb = get_NYUDepth_v2(opts);
+net = get_model(opts);
+
+% Test set
+testSet.images = imdb.images(:,:,:, imdb.set == 2);
+testSet.depths = imdb.depths(:,:, imdb.set == 2);
+
+% Prepare input for evaluation through the network, in accordance to the
+% way the model was trained for the NYU dataset. No processing is applied
+% to the ground truth. 
+meta = net.meta.normalization;  % information about input
+res = meta.imageSize(1:2) + 2*meta.border; 
+testSet.images = imresize(testSet.images, res, opts.interp);  % resize
+testSet.images = testSet.images(1+meta.border(1):end-meta.border(1), 1+meta.border(2):end-meta.border(2), :, :);  % center crop
+
+% -------------------------------------------------------------------------
+% Evaluate network 
+% -------------------------------------------------------------------------
+
+% Get predictions
+predictions = DepthMapPrediction(testSet, net, netOpts);
+predictions = squeeze(predictions); % remove singleton dimensions
+predictions = imresize(predictions, [size(testSet.depths,1), size(testSet.depths,2)], 'bilinear'); %rescale
+
+% Error calculation 
+errors = error_metrics(predictions, testSet.depths, []);
+
+% Save results
+fprintf('\nsaving predictions...');
+save(fullfile(opts.dataDir, 'results.mat'), 'predictions', 'errors', '-v7.3');
+fprintf('done!\n');
+
+
+
+function imdb = get_NYUDepth_v2(opts)
+% -------------------------------------------------------------------------
+% Download required data 
+% -------------------------------------------------------------------------
+
+opts.dataDir = fullfile(opts.dataDir, 'data');
+if ~exist(opts.dataDir, 'dir'), mkdir(opts.dataDir); end
+
+% Download dataset
+filename = fullfile(opts.dataDir, 'nyu_depth_v2_labeled.mat');
+if ~exist(filename, 'file')
+    url = 'http://horatio.cs.nyu.edu/mit/silberman/nyu_depth_v2/nyu_depth_v2_labeled.mat';
+    fprintf('downloading dataset (~2.8 GB): %s\n', url);
+    websave(filename, url);
+end
+
+% Download official train/test split
+filename_splits = fullfile(opts.dataDir, 'splits.mat');
+if ~exist(filename_splits, 'file')
+    url_split = 'http://horatio.cs.nyu.edu/mit/silberman/indoor_seg_sup/splits.mat';
+    fprintf('downloading train/test split: %s\n', url_split);
+    websave(filename_splits, url_split);
+end
+
+% Load dataset and splits
+fprintf('loading data to workspace...');
+data = load(filename);
+splits = load(filename_splits);
+
+% Store necessary information to imdb structure
+imdb.images = single(data.images);  %(no mean subtraction has been performed)
+imdb.depths = single(data.depths);  %depth filled-in values
+imdb.set(splits.trainNdxs) = 1;     %training indices (ignored for inference)
+imdb.set(splits.testNdxs) = 2;      %testing indices (on which evaluation is performed)
+fprintf(' done!\n');
+
+
+
+function net = get_model(opts)
+% -------------------------------------------------------------------------
+% Download trained models
+% -------------------------------------------------------------------------
+
+opts.dataDir = fullfile(opts.dataDir, 'models');
+if ~exist(opts.dataDir, 'dir'), mkdir(opts.dataDir); end
+
+filename = fullfile(opts.dataDir, 'NYU_ResNet-UpProj.mat');
+if ~exist(filename, 'file')
+    url = 'http://campar.in.tum.de/files/rupprecht/depthpred/NYU_ResNet-UpProj.zip';
+    fprintf('downloading trained model: %s\n', url);
+    unzip(url, opts.dataDir);
+end
+
+net = load(filename);
+