testdriver.h
Go to the documentation of this file.
00001 /** 00002 * @file testdriver.h 00003 * @brief Default test driver include file. 00004 * 00005 * This file is specified as INCLUDE argument to the create_test_sourcelist() 00006 * command of CMake which generates the code of the test driver. Such test 00007 * driver is used, in particular, to run a test which generates an output image. 00008 * The resulting image can then be compared by the test driver to one or more 00009 * baseline images. Note the difference to plain non-image processing based 00010 * unit tests. These shall make use of the unit testing frameworks included 00011 * with BASIS instead (see test.h for a C++ unit testing framework). 00012 * 00013 * This file in particular declares the functions which are used to parse 00014 * the command-line arguments of the test driver and those which are used by 00015 * the code fragments defined in the files testdriver-before-test.inc and 00016 * testdriver-after-test.inc. 00017 * 00018 * Currently available test driver implementations included by this file are: 00019 * - testdriver.hxx 00020 * - testdriver-itk.hxx 00021 * 00022 * This file is in parts a modified version of the itkTestDriverInclude.h 00023 * file which is part of the TestKernel module of the ITK 4 project. 00024 * 00025 * Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen<br /> 00026 * Copyright Insight Software Consortium.<br /> 00027 * Copyright (c) 2011, 2012 University of Pennsylvania. 00028 * 00029 * Portions of this file are subject to the VTK Toolkit Version 3 copyright. 00030 * 00031 * For complete copyright, license and disclaimer of warranty information 00032 * please refer to the COPYRIGHT file. 00033 * 00034 * Contact: SBIA Group <sbia-software at uphs.upenn.edu> 00035 */ 00036 00037 /*========================================================================= 00038 * 00039 * Copyright Insight Software Consortium 00040 * 00041 * Licensed under the Apache License, Version 2.0 (the "License"); 00042 * you may not use this file except in compliance with the License. 00043 * You may obtain a copy of the License at 00044 * 00045 * http://www.apache.org/licenses/LICENSE-2.0.txt 00046 * 00047 * Unless required by applicable law or agreed to in writing, software 00048 * distributed under the License is distributed on an "AS IS" BASIS, 00049 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00050 * See the License for the specific language governing permissions and 00051 * limitations under the License. 00052 * 00053 *=========================================================================*/ 00054 /*========================================================================= 00055 * 00056 * Portions of this file are subject to the VTK Toolkit Version 3 copyright. 00057 * 00058 * Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen 00059 * 00060 * For complete copyright, license and disclaimer of warranty information 00061 * please refer to the NOTICE file at the top of the ITK source tree. 00062 * 00063 *=========================================================================*/ 00064 00065 #pragma once 00066 #ifndef _BASIS_TESTDRIVER_H 00067 #define _BASIS_TESTDRIVER_H 00068 00069 00070 #include <string> 00071 #include <map> 00072 #include <vector> 00073 #include <iostream> 00074 #include <fstream> 00075 #include <cstdio> // remove() - removes a file 00076 #include <limits> // used in basistest-after-test.inc 00077 00078 #include <basis/basis.h> 00079 00080 00081 // acceptable in test driver includes 00082 using namespace std; 00083 using namespace basis; 00084 00085 00086 // =========================================================================== 00087 // arguments 00088 // =========================================================================== 00089 00090 /// @brief Maximum dimension of images used for testing. 00091 const unsigned int BASIS_MAX_TEST_IMAGE_DIMENSION = 6; 00092 00093 // --------------------------------------------------------------------------- 00094 // environment 00095 MultiStringArg add_before_libpath( 00096 "", "add-before-libpath", 00097 "Add a path to the library path environment. This option takes" 00098 " care of choosing the right environment variable for your system.", 00099 false, "<dir>"); 00100 00101 MultiStringArg add_before_env( 00102 "", "add-before-env", 00103 "Add an environment variable named <name> with the given value." 00104 " The seperator used is the default one on the system.", 00105 false, "<name> <value>", 2); 00106 00107 MultiStringArg add_before_env_with_sep( 00108 "", "add-before-env-with-sep", 00109 "Add an environment variable named <name> with the given value.", 00110 false, "<name> <value> <sep>", 3); 00111 00112 // --------------------------------------------------------------------------- 00113 // regression testing 00114 00115 enum TestMethod 00116 { 00117 METHOD_UNKNOWN, 00118 COMPARE_IMAGES, 00119 BINARY_DIFF, 00120 DIFF_LINES 00121 }; 00122 00123 /// @brief Structure holding arguments to regression test options and currently 00124 /// set tolerances to be used for the regression test. 00125 struct RegressionTest { 00126 string test_file; 00127 string baseline_file; 00128 double intensity_tolerance; 00129 unsigned int max_number_of_differences; 00130 unsigned int tolerance_radius; 00131 bool orientation_insensitive; 00132 TestMethod method; 00133 }; 00134 00135 /// @brief Container storing added regression tests. 00136 vector<RegressionTest> regression_tests; 00137 00138 /// @brief Visitor used to handle --diff option. 00139 class BinaryDiffVisitor : public TCLAP::Visitor 00140 { 00141 public: 00142 BinaryDiffVisitor() {} 00143 ~BinaryDiffVisitor() {} 00144 void visit(); 00145 }; 00146 00147 /// @brief Visitor used to handle --diff-lines option. 00148 class LineDiffVisitor : public TCLAP::Visitor 00149 { 00150 public: 00151 LineDiffVisitor() {} 00152 ~LineDiffVisitor() {} 00153 void visit(); 00154 }; 00155 00156 /// @brief Visitor used to handle --compare option. 00157 class CompareVisitor : public TCLAP::Visitor 00158 { 00159 public: 00160 CompareVisitor() {} 00161 ~CompareVisitor() {} 00162 void visit(); 00163 }; 00164 00165 CompareVisitor compare_visitor; 00166 BinaryDiffVisitor diff_visitor; 00167 LineDiffVisitor diff_lines_visitor; 00168 00169 MultiStringArg diff( 00170 "", "diff", 00171 "Compare the <test> file to the <baseline> file byte by byte." 00172 " Can by used to compare any files including text files." 00173 " For images, the --compare option should be used instead.", 00174 false, "<test> <baseline>", 2, false, &diff_visitor); 00175 00176 MultiStringArg diff_lines( 00177 "", "diff-lines", 00178 "Compare the <test> file to the <baseline> file line by line." 00179 " Can by used to compare text files. The current --max-number-of-differences" 00180 " setting determines the number of lines which may differ between the files." 00181 " For binary files, consider the --diff option instead.", 00182 false, "<test> <baseline>", 2, false, &diff_lines_visitor); 00183 00184 MultiStringArg compare( 00185 "", "compare", 00186 "Compare the <test> image to the <baseline> image using the" 00187 " current tolerances. If the test image should be compared to" 00188 " to more than one baseline image, specify the file name of" 00189 " the main baseline image and name the other baseline images" 00190 " similarly with only a numerical suffix appended to the" 00191 " basename of the image file path using a dot (.) as separator." 00192 " For example, name your baseline images baseline.nii," 00193 " baseline.1.nii, baseline.2.nii,..., and specify baseline.nii" 00194 " second argument value.", 00195 false, "<test> <baseline>", 2, false, &compare_visitor); 00196 00197 DoubleArg intensity_tolerance( 00198 "", "intensity-tolerance", 00199 "The accepted maximum difference between image intensities" 00200 " to use for the following regression tests." 00201 // default should be printed automatically 00202 " (default: 2.0)", 00203 false, 2.0, "<float>", true); 00204 00205 UIntArg max_number_of_differences( 00206 "", "max-number-of-differences", 00207 "When comparing images specified with the following --compare option(s)," 00208 " allow the given number of image elements to differ.", 00209 false, 0, "<n>", true); 00210 00211 UIntArg tolerance_radius( 00212 "", "tolerance-radius", 00213 "At most one image element in the neighborhood specified by the" 00214 " given radius has to fulfill the criteria of the following" 00215 " regression tests", 00216 false, 0, "<int>", true); 00217 00218 SwitchArg orientation_insensitive( 00219 "", "orientation-insensitive", 00220 "Allow the test and baseline images to have different orientation." 00221 " When this option is given, the orientation of both images is made" 00222 " identical before they are compared. It is suitable if the test" 00223 " and baseline images are simply stored with different orientation," 00224 " but with proper orientation information in the file header."); 00225 00226 // --------------------------------------------------------------------------- 00227 // test execution 00228 StringArg redirect_output( 00229 "", "redirect-output", 00230 "Redirects the test output to the specified file.", 00231 false, "", "<file>"); 00232 00233 UIntArg max_number_of_threads( 00234 "", "max-number-of-threads", 00235 "Use at most <n> threads. Set explicitly to n=1 to disable" 00236 " multi-threading. Note that the test itself still may use" 00237 " more threads, but the regression tests will not.", 00238 false, 0, "<n>"); 00239 00240 SwitchArg full_output( 00241 "", "full-output", 00242 "Causes the full output of the test to be passed to CDash.", 00243 false); 00244 00245 MultiSwitchArg verbose( 00246 "v", "verbose", 00247 "Increase verbosity of output messages.", 00248 false); 00249 00250 // --------------------------------------------------------------------------- 00251 // test / test command 00252 SwitchArg clean_cwd_before_test( 00253 "", "clean-cwd-before", 00254 "Request the removal of all files and directories from the current" 00255 " working directory before the execution of the test. This option is" 00256 " in particular useful if the test writes any results to the current" 00257 " working directory.", 00258 false); 00259 00260 SwitchArg clean_cwd_after_test( 00261 "", "clean-cwd-after", 00262 "Request the removal of all files and directories from the current" 00263 " working directory after the successful execution of the test." 00264 " This option is in particular useful if the test writes any results" 00265 " to the current working directory.", 00266 false); 00267 00268 #ifdef BASIS_STANDALONE_TESTDRIVER 00269 00270 PositionalArgs testcmd( 00271 "testcmd", 00272 "The external test command and its command-line arguments." 00273 " This command is executed by the test driver after altering the" 00274 " environment as subprocess. After the subprocess finished, the" 00275 " requested regression tests are performed by the test driver." 00276 " Note that if the -- option is not given before the test command," 00277 " labeled arguments following the test command will be considered" 00278 " to be options of the test driver if known by the test driver.", 00279 true, "[--] <test command> <arg>..."); 00280 00281 SwitchArg noprocess( 00282 "", "noprocess", 00283 "Do not run any test subprocess but only perform the regression tests.", 00284 true); 00285 00286 #else // defined(BASIS_STANDALONE_TESTDRIVER) 00287 00288 PositionalArgs testcmd( 00289 "testcmd", 00290 "The name of the test to run and optional arguments." 00291 " Displays a list of available tests if this argument is omitted" 00292 " and waits for the user to input the number of the test to run." 00293 " Exist with error if an invalid test was specified." 00294 " Note that if the -- option is not given before the test name," 00295 " labeled arguments following the test name will be considered" 00296 " to be options of the test driver if known by the test driver." 00297 " Otherwise, if the option is unknown to the test driver or the" 00298 " -- option has been given before the test name, the remaining" 00299 " arguments are passed on to the test.", 00300 false, "", "[--] [<test name> [<arg>...]]"); 00301 00302 #endif // defined(BASIS_STANDALONE_TESTDRIVER) 00303 00304 // =========================================================================== 00305 // initialization 00306 // =========================================================================== 00307 00308 /** 00309 * @brief Parse command-line arguments and initialize test driver. 00310 * 00311 * 00312 * @param [in] argc Number of arguments. 00313 * @param [in] argv Command-line arguments. 00314 */ 00315 void testdriversetup(int* argc, char** argv[]); 00316 00317 // =========================================================================== 00318 // low-level file comparison 00319 // =========================================================================== 00320 00321 /** 00322 * @brief Compare two files byte by byte. 00323 * 00324 * @param [in] testfile File generated by test. 00325 * @param [in] baseline Baseline file. 00326 * 00327 * @retval -1 if the test file could not be read 00328 * @retval -2 if the baseline file could not be read 00329 * @retval 0 if the two files are identical 00330 * @retval 1 if the two files differ 00331 */ 00332 int binary_diff(const char* testfile, const char* baseline); 00333 00334 /** 00335 * @brief Compare two text files line by line. 00336 * 00337 * @param [in] testfile File generated by test. 00338 * @param [in] baseline Baseline file. 00339 * @param [in] max_number_of_differences Number of lines that may differ at most. 00340 * 00341 * @retval -1 if the test file could not be read 00342 * @retval -2 if the baseline file could not be read 00343 * @retval 0 if the two files differ in no more than @p max_number_of_differences lines 00344 * @retval 1 if the two files differ in more than the allowed number of lines 00345 */ 00346 int text_diff_lines(const char* testfile, const char* baseline, unsigned int max_number_of_differences = 0); 00347 00348 // =========================================================================== 00349 // image regression testing 00350 // =========================================================================== 00351 00352 /** 00353 * @brief Generate list of names of baseline files from a given template filename. 00354 * 00355 * The list of baseline file names is generated from the template filename using 00356 * the following algorithm: 00357 * -# Strip the file name suffix. 00358 * -# Append a suffix containing of a dot (.) and a digit, i.e., .x 00359 * -# Append the original file name suffix. 00360 * It the file exists, increment x and continue. 00361 * 00362 * Additionally, if a file @p filename_template exists, it is the first 00363 * element in the resulting list. 00364 * 00365 * @param [in] filename_template File path template. 00366 * 00367 * @return List of baseline filenames or empty list if no such files exist. 00368 */ 00369 vector<string> get_baseline_filenames(string filename_template); 00370 00371 /** 00372 * @brief Compare output image to baseline image. 00373 * 00374 * This function compares a given image to a baseline image and returns a 00375 * regression test result depending on how well the output image matches the 00376 * baseline image given the provided tolerance arguments. 00377 * 00378 * @param [in] imagefile Output image file of test run. 00379 * @param [in] baseline Baseline image file. 00380 * @param [in] intensity_tolerance Maximum tolerable intensity difference. 00381 * @param [in] max_number_of_differences Maximum number of differing pixels. 00382 * @param [in] tolerance_radius Tolerance radius. 00383 * @param [in] orientation_insensitive Change orientation of both images to 00384 * a common coordinate orientation before 00385 * comparing them. 00386 * @param [in] report Level of test report to generate. 00387 * If zero, no report is generated. 00388 * If greater than zero, a report is 00389 * generated. Similar to the verbosity of 00390 * a program, is this parameter used to 00391 * set the verbosity of the report. Most 00392 * implementations yet only either 00393 * generate a (full) report or none. 00394 * 00395 * @returns Number of voxels with a difference above the set @p intensity_tolerance. 00396 */ 00397 int image_regression_test(const char* imagefile, 00398 const char* baseline, 00399 double intensity_tolerance = 2.0, 00400 unsigned int max_number_of_differences = 0, 00401 unsigned int tolerance_radius = 0, 00402 bool orientation_insensitive = false, 00403 int report = 0); 00404 00405 00406 // inline definitions 00407 #include "testdriver.hxx" 00408 00409 00410 #endif // _BASIS_TESTDRIVER_H