/
modd2_evaluate_all_sequences_raw.m
134 lines (117 loc) · 6 KB
/
modd2_evaluate_all_sequences_raw.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
%% MODD2 EVALUATION SCRIPT
%
% This function evaluates segmentation outputs of the given method on raw
% sequences from the Multi-modal Marine Obstacle Detection Dataset 2 (MODD2)
%
% Input parameters:
% dataset_path - a path to the MODD2 dataset root folder
% output_path - a path to the output root folder
% method_name - name of the given method
% segmentation_colors (optional) - a 3x3 matrix of colors used to
% represent labels in the output mask.
% In first row should be the RGB
% representation of the sky component,
% second row corresponds to the RGB
% representation of the
% obstacles/environment component, while
% the last row corresponds to the RGB
% representation of the water component.
%
function modd2_evaluate_all_sequences_raw(dataset_path, output_path, method_name, segmentation_colors)
if(nargin < 3)
error('Not enough input parameters');
end
if(nargin == 3)
segmentation_colors = [ 0, 255, 0; ... % Sky represented with green
0, 0, 0; ... % obstacles with black
255, 0, 0]; % and water with red color.
end
%% Evaluation parameters
eval_params = get_eval_params(segmentation_colors);
%% Extreme conditions information
extreme = get_extreme_conditions();
%% Makedir for storing interim results and posprocessed segmentation masks
if(~exist(fullfile('results', method_name, 'postprocessing'), 'dir'))
mkdir(fullfile('results', method_name, 'postprocessing'));
end
if(~exist(fullfile('results', method_name, 'eval_results'), 'dir'))
mkdir(fullfile('results', method_name, 'eval_results'), 'dir');
end
%% Initializations
% Evaluation results cell. Each cell represents each own sequence
eval_results = cell(28,2);
% For storing special sequence tag
seq.special = [];
% Evaluation measures
total_rmse = [];
total_tp = 0;
total_fp = 0;
total_fn = 0;
% are segmentation masks rectified?
seq.is_rectified = 0;
%% Loop through all sequences...
for seq_num_id = 1 : 28
seq.id = seq_num_id;
%% Fill in sequence information/details
% Add special sequences tag if necessary
if(ismember(seq.id, extreme.sequences_sudden_movement))
seq.special = [seq.special, 1];
end
if(ismember(seq.id, extreme.sequences_sun_glitter))
seq.special = [seq.special, 2];
end
if(ismember(seq.id, extreme.sequences_env_reflections))
seq.special = [seq.special, 3];
end
% Get sequence details (name, start frame, end frame)
[seq.name, seq.start_frame, seq.end_frame] = get_seq_details(seq.id);
%% Set paths accordingly...
paths.dataset_path = fullfile(dataset_path);
% Path to frames of the current sequence
paths.frames = fullfile(dataset_path, 'video_data', seq.name, 'frames');
% Path to ground-truth of the current sequence
paths.ground_truth = fullfile(dataset_path, 'annotationsV2', seq.name, 'ground_truth');
% Path to segmentation results of the current sequence
paths.output = fullfile(output_path, sprintf('seq%02d', seq.id), method_name);
% Path to USV masks
paths.USV_parts_masks = fullfile(dataset_path, 'USV_parts_masks');
%% Perform evaluation on sequence seq_num_id
% Results for each frame are written in format:
% 1 2 3 4 5
% RMSE TP FP FN Special sequence tag
fprintf('Evaluation on sequence %02d started...\n', seq.id);
[output_results, output_detections] = perform_evaluation_on_sequence(paths, seq, eval_params);
%% Save interim results
% Save detections information for the processed sequence...
save(fullfile('results', method_name, 'postprocessing', sprintf('seq%02d_%s.mat', seq.id, method_name)), 'output_detections');
% Save output results of the processed sequence...
eval_results{seq.id, 1} = output_results;
eval_results{seq.id, 2} = seq.special;
save(fullfile('results', method_name, 'eval_results', sprintf('seq%02d_%s.mat', seq.id, method_name)), 'eval_results');
%% Print interim results
fprintf('Seq %02d done\n', seq.id);
tmp_rmse = mean(cell2mat(output_results(:,1)));
tmp_tp = sum(cell2mat(output_results(:,2)));
tmp_fp = sum(cell2mat(output_results(:,3)));
tmp_fn = sum(cell2mat(output_results(:,4)));
fprintf('RMSE: %f\nTotal TP: %d\nTotal FP: %d\nTotal FN: %d\n', tmp_rmse, tmp_tp, tmp_fp, tmp_fn);
%% Update total results
total_rmse = [total_rmse; cell2mat(output_results(:,1))];
total_tp = total_tp + tmp_tp;
total_fp = total_fp + tmp_fp;
total_fn = total_fn + tmp_fn;
end
fprintf('--- all done ---\n');
fprintf('**********************************\n');
fprintf('* Evaluation on RAW images *\n');
fprintf('* Method: %*.*s *\n', 22, 22, method_name);
fprintf('**********************************\n');
fprintf('* Total RMSE: %04.01f px *\n', mean(total_rmse(:)) * 958);
fprintf('* Total STD: %04.01f px *\n', std(total_rmse(:)) * 958);
fprintf('* Total STE: %04.01f px *\n', std(total_rmse(:)) / sqrt(length(total_rmse(:))) * 958);
fprintf('* Total TP: %04d *\n', total_tp);
fprintf('* Total FP: %05d *\n', total_fp);
fprintf('* Total FN: %04d *\n', total_fn);
fprintf('* F-measure: %04.01f %% *\n', ((2 * total_tp) / (2 * total_tp + total_fp + total_fn) * 100));
fprintf('**********************************\n');
end