tests/utils/tap.c

   1 /*-
   2  * Copyright (c) 2004 Nik Clayton
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #define _GNU_SOURCE
  28 #include <ctype.h>
  29 #include <stdarg.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32
  33 #include "tap.h"
  34
  35 static int no_plan = 0;
  36 static int skip_all = 0;
  37 static int have_plan = 0;
  38 static unsigned int test_count = 0; /* Number of tests that have been run */
  39 static unsigned int e_tests = 0; /* Expected number of tests to run */
  40 static unsigned int failures = 0; /* Number of tests that failed */
  41 static char *todo_msg = NULL;
  42 static char *todo_msg_fixed = "libtap malloc issue";
  43 static int todo = 0;
  44 static int test_died = 0;
  45
  46 /* Encapsulate the pthread code in a conditional.  In the absence of
  47    libpthread the code does nothing */
  48 #ifdef HAVE_LIBPTHREAD
  49 #include <pthread.h>
  50 static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER;
  51 # define LOCK pthread_mutex_lock(&M);
  52 # define UNLOCK pthread_mutex_unlock(&M);
  53 #else
  54 # define LOCK
  55 # define UNLOCK
  56 #endif
  57
  58 static void _expected_tests(unsigned int);
  59 static void _tap_init(void);
  60 static void _cleanup(void);
  61
  62 /*
  63  * Generate a test result.
  64  *
  65  * ok -- boolean, indicates whether or not the test passed.
  66  * test_name -- the name of the test, may be NULL
  67  * test_comment -- a comment to print afterwards, may be NULL
  68  */
  69 unsigned int
  70 _gen_result(int ok, const char *func, char *file, unsigned int line,
  71             char *test_name, ...)
  72 {
  73         va_list ap;
  74         char *local_test_name = NULL;
  75         char *c;
  76         int name_is_digits;
  77
  78         LOCK;
  79
  80         test_count++;
  81
  82         /* Start by taking the test name and performing any printf()
  83            expansions on it */
  84         if(test_name != NULL) {
  85                 va_start(ap, test_name);
  86                 if (vasprintf(&local_test_name, test_name, ap) == -1) {
  87                         local_test_name = NULL;
  88                 }
  89                 va_end(ap);
  90
  91                 /* Make sure the test name contains more than digits
  92                    and spaces.  Emit an error message and exit if it
  93                    does */
  94                 if(local_test_name) {
  95                         name_is_digits = 1;
  96                         for(c = local_test_name; *c != '\0'; c++) {
  97                                 if(!isdigit(*c) && !isspace(*c)) {
  98                                         name_is_digits = 0;
  99                                         break;
 100                                 }
 101                         }
 102
 103                         if(name_is_digits) {
 104                                 diag("    You named your test '%s'.  You shouldn't use numbers for your test names.", local_test_name);
 105                                 diag("    Very confusing.");
 106                         }
 107                 }
 108         }
 109
 110         if(!ok) {
 111                 printf("not ");
 112                 failures++;
 113         }
 114
 115         printf("ok %d", test_count);
 116
 117         if(test_name != NULL) {
 118                 printf(" - ");
 119
 120                 /* Print the test name, escaping any '#' characters it
 121                    might contain */
 122                 if(local_test_name != NULL) {
 123                         flockfile(stdout);
 124                         for(c = local_test_name; *c != '\0'; c++) {
 125                                 if(*c == '#')
 126                                         fputc('\\', stdout);
 127                                 fputc((int)*c, stdout);
 128                         }
 129                         funlockfile(stdout);
 130                 } else {        /* vasprintf() failed, use a fixed message */
 131                         printf("%s", todo_msg_fixed);
 132                 }
 133         }
 134
 135         /* If we're in a todo_start() block then flag the test as being
 136            TODO.  todo_msg should contain the message to print at this
 137            point.  If it's NULL then asprintf() failed, and we should
 138            use the fixed message.
 139
 140            This is not counted as a failure, so decrement the counter if
 141            the test failed. */
 142         if(todo) {
 143                 printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed);
 144                 if(!ok)
 145                         failures--;
 146         }
 147
 148         printf("\n");
 149
 150         if(!ok) {
 151                 if(getenv("HARNESS_ACTIVE") != NULL)
 152                         fputs("\n", stderr);
 153
 154                 diag("    Failed %stest (%s:%s() at line %d)",
 155                      todo ? "(TODO) " : "", file, func, line);
 156         }
 157         free(local_test_name);
 158
 159         UNLOCK;
 160
 161         /* We only care (when testing) that ok is positive, but here we
 162            specifically only want to return 1 or 0 */
 163         return ok ? 1 : 0;
 164 }
 165
 166 /*
 167  * Initialise the TAP library.  Will only do so once, however many times it's
 168  * called.
 169  */
 170 void
 171 _tap_init(void)
 172 {
 173         static int run_once = 0;
 174
 175         if(!run_once) {
 176                 atexit(_cleanup);
 177
 178                 /* stdout needs to be unbuffered so that the output appears
 179                    in the same place relative to stderr output as it does
 180                    with Test::Harness */
 181                 setbuf(stdout, 0);
 182                 run_once = 1;
 183         }
 184 }
 185
 186 /*
 187  * Note that there's no plan.
 188  */
 189 int
 190 plan_no_plan(void)
 191 {
 192
 193         LOCK;
 194
 195         _tap_init();
 196
 197         if(have_plan != 0) {
 198                 fprintf(stderr, "You tried to plan twice!\n");
 199                 test_died = 1;
 200                 UNLOCK;
 201                 exit(255);
 202         }
 203
 204         have_plan = 1;
 205         no_plan = 1;
 206
 207         UNLOCK;
 208
 209         return 1;
 210 }
 211
 212 /*
 213  * Note that the plan is to skip all tests
 214  */
 215 int
 216 plan_skip_all(char *reason)
 217 {
 218
 219         LOCK;
 220
 221         _tap_init();
 222
 223         skip_all = 1;
 224
 225         printf("1..0");
 226
 227         if(reason != NULL)
 228                 printf(" # Skip %s", reason);
 229
 230         printf("\n");
 231
 232         UNLOCK;
 233
 234         exit(0);
 235 }
 236
 237 /*
 238  * Note the number of tests that will be run.
 239  */
 240 int
 241 plan_tests(unsigned int tests)
 242 {
 243
 244         LOCK;
 245
 246         _tap_init();
 247
 248         if(have_plan != 0) {
 249                 fprintf(stderr, "You tried to plan twice!\n");
 250                 test_died = 1;
 251                 UNLOCK;
 252                 exit(255);
 253         }
 254
 255         if(tests == 0) {
 256                 fprintf(stderr, "You said to run 0 tests!  You've got to run something.\n");
 257                 test_died = 1;
 258                 UNLOCK;
 259                 exit(255);
 260         }
 261
 262         have_plan = 1;
 263
 264         _expected_tests(tests);
 265
 266         UNLOCK;
 267
 268         return e_tests;
 269 }
 270
 271 unsigned int
 272 diag(char *fmt, ...)
 273 {
 274         va_list ap;
 275
 276         fputs("# ", stderr);
 277
 278         va_start(ap, fmt);
 279         vfprintf(stderr, fmt, ap);
 280         va_end(ap);
 281
 282         fputs("\n", stderr);
 283
 284         return 0;
 285 }
 286
 287 void
 288 _expected_tests(unsigned int tests)
 289 {
 290
 291         printf("1..%d\n", tests);
 292         e_tests = tests;
 293 }
 294
 295 int
 296 skip(unsigned int n, char *fmt, ...)
 297 {
 298         va_list ap;
 299         char *skip_msg = NULL;
 300
 301         LOCK;
 302
 303         va_start(ap, fmt);
 304         if (asprintf(&skip_msg, fmt, ap) == -1) {
 305                 skip_msg = NULL;
 306         }
 307         va_end(ap);
 308
 309         while(n-- > 0) {
 310                 test_count++;
 311                 printf("ok %d # skip %s\n", test_count,
 312                        skip_msg != NULL ?
 313                        skip_msg : "libtap():malloc() failed");
 314         }
 315
 316         free(skip_msg);
 317
 318         UNLOCK;
 319
 320         return 1;
 321 }
 322
 323 void
 324 todo_start(char *fmt, ...)
 325 {
 326         va_list ap;
 327
 328         LOCK;
 329
 330         va_start(ap, fmt);
 331         if (vasprintf(&todo_msg, fmt, ap) == -1) {
 332                 todo_msg = NULL;
 333         }
 334         va_end(ap);
 335
 336         todo = 1;
 337
 338         UNLOCK;
 339 }
 340
 341 void
 342 todo_end(void)
 343 {
 344
 345         LOCK;
 346
 347         todo = 0;
 348         free(todo_msg);
 349
 350         UNLOCK;
 351 }
 352
 353 int
 354 exit_status(void)
 355 {
 356         int r;
 357
 358         LOCK;
 359
 360         /* If there's no plan, just return the number of failures */
 361         if(no_plan || !have_plan) {
 362                 UNLOCK;
 363                 return failures;
 364         }
 365
 366         /* Ran too many tests?  Return the number of tests that were run
 367            that shouldn't have been */
 368         if(e_tests < test_count) {
 369                 r = test_count - e_tests;
 370                 UNLOCK;
 371                 return r;
 372         }
 373
 374         /* Return the number of tests that failed + the number of tests
 375            that weren't run */
 376         r = failures + e_tests - test_count;
 377         UNLOCK;
 378
 379         return r;
 380 }
 381
 382 /*
 383  * Cleanup at the end of the run, produce any final output that might be
 384  * required.
 385  */
 386 void
 387 _cleanup(void)
 388 {
 389
 390         LOCK;
 391
 392         /* If plan_no_plan() wasn't called, and we don't have a plan,
 393            and we're not skipping everything, then something happened
 394            before we could produce any output */
 395         if(!no_plan && !have_plan && !skip_all) {
 396                 diag("Looks like your test died before it could output anything.");
 397                 UNLOCK;
 398                 return;
 399         }
 400
 401         if(test_died) {
 402                 diag("Looks like your test died just after %d.", test_count);
 403                 UNLOCK;
 404                 return;
 405         }
 406
 407
 408         /* No plan provided, but now we know how many tests were run, and can
 409            print the header at the end */
 410         if(!skip_all && (no_plan || !have_plan)) {
 411                 printf("1..%d\n", test_count);
 412         }
 413
 414         if((have_plan && !no_plan) && e_tests < test_count) {
 415                 diag("Looks like you planned %d %s but ran %d extra.",
 416                      e_tests, e_tests == 1 ? "test" : "tests", test_count - e_tests);
 417                 UNLOCK;
 418                 return;
 419         }
 420
 421         if((have_plan || !no_plan) && e_tests > test_count) {
 422                 diag("Looks like you planned %d %s but only ran %d.",
 423                      e_tests, e_tests == 1 ? "test" : "tests", test_count);
 424                 UNLOCK;
 425                 return;
 426         }
 427
 428         if(failures)
 429                 diag("Looks like you failed %d %s of %d.",
 430                      failures, failures == 1 ? "test" : "tests", test_count);
 431
 432         UNLOCK;
 433 }