tests/weston-test-runner.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining
   5  * a copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sublicense, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial
  14  * portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  17  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19  * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  20  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  21  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  22  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  */
  25
  26 #include "config.h"
  27
  28 #include <unistd.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <sys/wait.h>
  32 #include <string.h>
  33 #include <assert.h>
  34 #include <errno.h>
  35 #include <signal.h>
  36 #include <getopt.h>
  37
  38 #include "weston-test-runner.h"
  39 #include "weston-testsuite-data.h"
  40 #include "shared/string-helpers.h"
  41
  42 /**
  43  * \defgroup testharness Test harness
  44  * \defgroup testharness_private Test harness private
  45  */
  46
  47 extern const struct weston_test_entry __start_test_section, __stop_test_section;
  48
  49 struct weston_test_run_info {
  50         char name[512];
  51         int fixture_nr;
  52 };
  53
  54 static const struct weston_test_run_info *test_run_info_;
  55
  56 /** Get the test name string with counter
  57  *
  58  * \return The test name with fixture number \c -f%%d added. For an array
  59  * driven test, e.g. defined with TEST_P(), the name has also a \c -e%%d
  60  * suffix to indicate the array element number.
  61  *
  62  * This is only usable from code paths inside TEST(), TEST_P(), PLUGIN_TEST()
  63  * etc. defined functions.
  64  *
  65  * \ingroup testharness
  66  */
  67 const char *
  68 get_test_name(void)
  69 {
  70         return test_run_info_->name;
  71 }
  72
  73 /** Get the current fixture index
  74  *
  75  * Returns the current fixture index which can be used directly as an index
  76  * into the array passed as an argument to DECLARE_FIXTURE_SETUP_WITH_ARG().
  77  *
  78  * This is only usable from code paths inside TEST(), TEST_P(), PLUGIN_TEST()
  79  * etc. defined functions.
  80  *
  81  * \ingroup testharness
  82  */
  83 int
  84 get_test_fixture_index(void)
  85 {
  86         return test_run_info_->fixture_nr - 1;
  87 }
  88
  89 /** Print into test log
  90  *
  91  * This is exactly like printf() except the output goes to the test log,
  92  * which is at stderr.
  93  *
  94  * \param fmt printf format string
  95  *
  96  * \ingroup testharness
  97  */
  98 void
  99 testlog(const char *fmt, ...)
 100 {
 101         va_list argp;
 102
 103         va_start(argp, fmt);
 104         vfprintf(stderr, fmt, argp);
 105         va_end(argp);
 106 }
 107
 108 static const struct weston_test_entry *
 109 find_test(const char *name)
 110 {
 111         const struct weston_test_entry *t;
 112
 113         for (t = &__start_test_section; t < &__stop_test_section; t++)
 114                 if (strcmp(t->name, name) == 0)
 115                         return t;
 116
 117         return NULL;
 118 }
 119
 120 static enum test_result_code
 121 run_test(int fixture_nr, const struct weston_test_entry *t, void *data,
 122          int iteration)
 123 {
 124         struct weston_test_run_info info;
 125
 126         if (data) {
 127                 snprintf(info.name, sizeof(info.name), "%s-f%02d-e%02d",
 128                          t->name, fixture_nr, iteration);
 129         } else {
 130                 snprintf(info.name, sizeof(info.name), "%s-f%02d",
 131                          t->name, fixture_nr);
 132         }
 133         info.fixture_nr = fixture_nr;
 134
 135         test_run_info_ = &info;
 136         t->run(data);
 137         test_run_info_ = NULL;
 138
 139         /*
 140          * XXX: We should return t->run(data); but that requires changing
 141          * the function signature and stop using assert() in tests.
 142          * https://gitlab.freedesktop.org/wayland/weston/issues/311
 143          */
 144         return RESULT_OK;
 145 }
 146
 147 static void
 148 list_tests(void)
 149 {
 150         const struct fixture_setup_array *fsa;
 151         const struct weston_test_entry *t;
 152
 153         fsa = fixture_setup_array_get_();
 154
 155         printf("Fixture setups: %d\n", fsa->n_elements);
 156
 157         for (t = &__start_test_section; t < &__stop_test_section; t++) {
 158                 printf("  %s\n", t->name);
 159                 if (t->n_elements > 1)
 160                         printf("    with array of %d cases\n", t->n_elements);
 161         }
 162 }
 163
 164 struct weston_test_harness {
 165         int32_t fixt_ind;
 166         char *chosen_testname;
 167         int32_t case_ind;
 168
 169         struct wet_testsuite_data data;
 170 };
 171
 172 typedef void (*weston_test_cb)(struct wet_testsuite_data *suite_data,
 173                                const struct weston_test_entry *t,
 174                                const void *test_data,
 175                                int iteration);
 176
 177 static void
 178 for_each_test_case(struct wet_testsuite_data *data, weston_test_cb cb)
 179 {
 180         unsigned i;
 181
 182         for (i = 0; i < data->tests_count; i++) {
 183                 const struct weston_test_entry *t = &data->tests[i];
 184                 const void *current_test_data = t->table_data;
 185                 int elem;
 186                 int elem_end;
 187
 188                 if (data->case_index == -1) {
 189                         elem = 0;
 190                         elem_end = t->n_elements;
 191                 } else {
 192                         elem = data->case_index;
 193                         elem_end = elem + 1;
 194                 }
 195
 196                 for (; elem < elem_end; elem++) {
 197                         current_test_data = (char *)t->table_data +
 198                                             elem * t->element_size;
 199                         cb(data, t, current_test_data, elem);
 200                  }
 201         }
 202 }
 203
 204 static const char *
 205 result_to_str(enum test_result_code ret)
 206 {
 207         static const char *names[] = {
 208                 [RESULT_FAIL] = "fail",
 209                 [RESULT_HARD_ERROR] = "hard error",
 210                 [RESULT_OK] = "ok",
 211                 [RESULT_SKIP] = "skip",
 212         };
 213
 214         assert(ret >= 0 && ret < ARRAY_LENGTH(names));
 215         return names[ret];
 216 }
 217
 218 static void
 219 run_case(struct wet_testsuite_data *suite_data,
 220          const struct weston_test_entry *t,
 221          const void *test_data,
 222          int iteration)
 223 {
 224         enum test_result_code ret;
 225         const char *fail = "";
 226         const char *skip = "";
 227         int fixture_nr = suite_data->fixture_iteration + 1;
 228         int iteration_nr = iteration + 1;
 229
 230         testlog("*** Run fixture %d, %s/%d\n",
 231                 fixture_nr, t->name, iteration_nr);
 232
 233         if (suite_data->type == TEST_TYPE_PLUGIN) {
 234                 ret = run_test(fixture_nr, t, suite_data->compositor,
 235                                iteration);
 236         } else {
 237                 ret = run_test(fixture_nr, t, (void *)test_data, iteration);
 238         }
 239
 240         switch (ret) {
 241         case RESULT_OK:
 242                 suite_data->passed++;
 243                 break;
 244         case RESULT_FAIL:
 245         case RESULT_HARD_ERROR:
 246                 suite_data->failed++;
 247                 fail = "not ";
 248                 break;
 249         case RESULT_SKIP:
 250                 suite_data->skipped++;
 251                 skip = " # SKIP";
 252                 break;
 253         }
 254
 255         testlog("*** Result fixture %d, %s/%d: %s\n",
 256                 fixture_nr, t->name, iteration_nr, result_to_str(ret));
 257
 258         suite_data->counter++;
 259         printf("%sok %d fixture %d %s/%d%s\n", fail, suite_data->counter,
 260                fixture_nr, t->name, iteration_nr, skip);
 261 }
 262
 263 /* This function might run in a new thread */
 264 static void
 265 testsuite_run(struct wet_testsuite_data *data)
 266 {
 267         for_each_test_case(data, run_case);
 268 }
 269
 270 static void
 271 count_case(struct wet_testsuite_data *suite_data,
 272            const struct weston_test_entry *t,
 273            const void *test_data,
 274            int iteration)
 275 {
 276         suite_data->total++;
 277 }
 278
 279 static void
 280 tap_plan(struct wet_testsuite_data *data, int count_fixtures)
 281 {
 282         data->total = 0;
 283         for_each_test_case(data, count_case);
 284
 285         printf("1..%d\n", data->total * count_fixtures);
 286 }
 287
 288 static void
 289 skip_case(struct wet_testsuite_data *suite_data,
 290           const struct weston_test_entry *t,
 291           const void *test_data,
 292           int iteration)
 293 {
 294         int fixture_nr = suite_data->fixture_iteration + 1;
 295         int iteration_nr = iteration + 1;
 296
 297         suite_data->counter++;
 298         printf("ok %d fixture %d %s/%d # SKIP fixture\n", suite_data->counter,
 299                fixture_nr, t->name, iteration_nr);
 300 }
 301
 302 static void
 303 tap_skip_fixture(struct wet_testsuite_data *data)
 304 {
 305         for_each_test_case(data, skip_case);
 306 }
 307
 308 static void
 309 help(const char *exe)
 310 {
 311         printf(
 312                 "Usage: %s [options] [testname [index]]\n"
 313                 "\n"
 314                 "This is a Weston test suite executable that runs some tests.\n"
 315                 "Options:\n"
 316                 "  -f, --fixture N  Run only fixture index N. Indices start from 1.\n"
 317                 "  -h, --help       Print this help and exit with success.\n"
 318                 "  -l, --list       List all tests in this executable and exit with success.\n"
 319                 "testname:          Optional; name of the test to execute instead of all tests.\n"
 320                 "index:             Optional; for a multi-case test, run the given case only.\n",
 321                 exe);
 322 }
 323
 324 static void
 325 parse_command_line(struct weston_test_harness *harness, int argc, char **argv)
 326 {
 327         int c;
 328         static const struct option opts[] = {
 329                 { "fixture", required_argument, NULL,      'f' },
 330                 { "help",    no_argument,       NULL,      'h' },
 331                 { "list",    no_argument,       NULL,      'l' },
 332                 { 0,         0,                 NULL,      0  }
 333         };
 334
 335         while ((c = getopt_long(argc, argv, "f:hl", opts, NULL)) != -1) {
 336                 switch (c) {
 337                 case 'f':
 338                         if (!safe_strtoint(optarg, &harness->fixt_ind)) {
 339                                 fprintf(stderr,
 340                                         "Error: '%s' does not look like a number (command line).\n",
 341                                         optarg);
 342                                 exit(RESULT_HARD_ERROR);
 343                         }
 344                         harness->fixt_ind--; /* convert base-1 to base 0 */
 345                         break;
 346                 case 'h':
 347                         help(argv[0]);
 348                         exit(RESULT_OK);
 349                 case 'l':
 350                         list_tests();
 351                         exit(RESULT_OK);
 352                 case 0:
 353                         break;
 354                 default:
 355                         exit(RESULT_HARD_ERROR);
 356                 }
 357         }
 358
 359         if (optind < argc)
 360                 harness->chosen_testname = argv[optind++];
 361
 362         if (optind < argc) {
 363                 if (!safe_strtoint(argv[optind], &harness->case_ind)) {
 364                         fprintf(stderr,
 365                                 "Error: '%s' does not look like a number (command line).\n",
 366                                 argv[optind]);
 367                         exit(RESULT_HARD_ERROR);
 368                 }
 369                 harness->case_ind--; /* convert base-1 to base 0 */
 370                 optind++;
 371         }
 372
 373         if (optind < argc) {
 374                 fprintf(stderr, "Unexpected extra arguments given (command line).\n\n");
 375                 help(argv[0]);
 376                 exit(RESULT_HARD_ERROR);
 377         }
 378 }
 379
 380 static struct weston_test_harness *
 381 weston_test_harness_create(int argc, char **argv)
 382 {
 383         const struct fixture_setup_array *fsa;
 384         struct weston_test_harness *harness;
 385
 386         harness = zalloc(sizeof(*harness));
 387         assert(harness);
 388
 389         harness->fixt_ind = -1;
 390         harness->case_ind = -1;
 391         parse_command_line(harness, argc, argv);
 392
 393         fsa = fixture_setup_array_get_();
 394         if (harness->fixt_ind < -1 || harness->fixt_ind >= fsa->n_elements) {
 395                 fprintf(stderr,
 396                         "Error: fixture index %d (command line) is invalid for this program.\n",
 397                         harness->fixt_ind + 1);
 398                 exit(RESULT_HARD_ERROR);
 399         }
 400
 401         if (harness->chosen_testname) {
 402                 const struct weston_test_entry *t;
 403
 404                 t = find_test(harness->chosen_testname);
 405                 if (!t) {
 406                         fprintf(stderr,
 407                                 "Error: test '%s' not found (command line).\n",
 408                                 harness->chosen_testname);
 409                         exit(RESULT_HARD_ERROR);
 410                 }
 411
 412                 if (harness->case_ind < -1 ||
 413                     harness->case_ind >= t->n_elements) {
 414                         fprintf(stderr,
 415                                 "Error: case index %d (command line) is invalid for this test.\n",
 416                                 harness->case_ind + 1);
 417                         exit(RESULT_HARD_ERROR);
 418                 }
 419
 420                 harness->data.tests = t;
 421                 harness->data.tests_count = 1;
 422                 harness->data.case_index = harness->case_ind;
 423         } else {
 424                 harness->data.tests = &__start_test_section;
 425                 harness->data.tests_count =
 426                         &__stop_test_section - &__start_test_section;
 427                 harness->data.case_index = -1;
 428         }
 429
 430         harness->data.run = testsuite_run;
 431
 432         return harness;
 433 }
 434
 435 static void
 436 weston_test_harness_destroy(struct weston_test_harness *harness)
 437 {
 438         free(harness);
 439 }
 440
 441 static enum test_result_code
 442 counts_to_result(const struct wet_testsuite_data *data)
 443 {
 444         /* RESULT_SKIP is reserved for fixture setup itself skipping everything */
 445         if (data->total == data->passed + data->skipped)
 446                 return RESULT_OK;
 447         return RESULT_FAIL;
 448 }
 449
 450 /** Execute all tests as client tests
 451  *
 452  * \param harness The test harness context.
 453  * \param setup The compositor configuration.
 454  *
 455  * Initializes the compositor with the given setup and executes the compositor.
 456  * The compositor creates a new thread where all tests in the test program are
 457  * serially executed. Once the thread finishes, the compositor returns from its
 458  * event loop and cleans up.
 459  *
 460  * Returns RESULT_SKIP if the requested compositor features, e.g. GL-renderer,
 461  * are not built.
 462  *
 463  * \sa DECLARE_FIXTURE_SETUP(), DECLARE_FIXTURE_SETUP_WITH_ARG()
 464  * \ingroup testharness
 465  */
 466 enum test_result_code
 467 weston_test_harness_execute_as_client(struct weston_test_harness *harness,
 468                                       const struct compositor_setup *setup)
 469 {
 470         struct wet_testsuite_data *data = &harness->data;
 471
 472         data->type = TEST_TYPE_CLIENT;
 473         return execute_compositor(setup, data);
 474 }
 475
 476 /** Execute all tests as plugin tests
 477  *
 478  * \param harness The test harness context.
 479  * \param setup The compositor configuration.
 480  *
 481  * Initializes the compositor with the given setup and executes the compositor.
 482  * The compositor executes all tests in the test program serially from an idle
 483  * handler, then returns from its event loop and cleans up.
 484  *
 485  * Returns RESULT_SKIP if the requested compositor features, e.g. GL-renderer,
 486  * are not built.
 487  *
 488  * \sa DECLARE_FIXTURE_SETUP(), DECLARE_FIXTURE_SETUP_WITH_ARG()
 489  * \ingroup testharness
 490  */
 491 enum test_result_code
 492 weston_test_harness_execute_as_plugin(struct weston_test_harness *harness,
 493                                       const struct compositor_setup *setup)
 494 {
 495         struct wet_testsuite_data *data = &harness->data;
 496
 497         data->type = TEST_TYPE_PLUGIN;
 498         return execute_compositor(setup, data);
 499 }
 500
 501 /** Execute all tests as standalone tests
 502  *
 503  * \param harness The test harness context.
 504  *
 505  * Executes all tests in the test program serially without any further setup,
 506  * particularly without any compositor instance created.
 507  *
 508  * \sa DECLARE_FIXTURE_SETUP(), DECLARE_FIXTURE_SETUP_WITH_ARG()
 509  * \ingroup testharness
 510  */
 511 enum test_result_code
 512 weston_test_harness_execute_standalone(struct weston_test_harness *harness)
 513 {
 514         struct wet_testsuite_data *data = &harness->data;
 515
 516         data->type = TEST_TYPE_STANDALONE;
 517         data->run(data);
 518
 519         return RESULT_OK;
 520 }
 521
 522 /** Fixture data array getter method
 523  *
 524  * DECLARE_FIXTURE_SETUP_WITH_ARG() overrides this in test programs.
 525  * The default implementation has no data and makes the tests run once.
 526  *
 527  * \ingroup testharness
 528  */
 529 __attribute__((weak)) const struct fixture_setup_array *
 530 fixture_setup_array_get_(void)
 531 {
 532         /* A dummy fixture without a data array. */
 533         static const struct fixture_setup_array default_fsa = {
 534                 .array = NULL,
 535                 .element_size = 0,
 536                 .n_elements = 1,
 537         };
 538
 539         return &default_fsa;
 540 }
 541
 542 /** Fixture setup function
 543  *
 544  * DECLARE_FIXTURE_SETUP() and DECLARE_FIXTURE_SETUP_WITH_ARG() override
 545  * this in test programs.
 546  * The default implementation just calls
 547  * weston_test_harness_execute_standalone().
 548  *
 549  * \ingroup testharness
 550  */
 551 __attribute__((weak)) enum test_result_code
 552 fixture_setup_run_(struct weston_test_harness *harness, const void *arg_)
 553 {
 554         return weston_test_harness_execute_standalone(harness);
 555 }
 556
 557 static void
 558 fixture_report(const struct wet_testsuite_data *d, enum test_result_code ret)
 559 {
 560         int fixture_nr = d->fixture_iteration + 1;
 561
 562         testlog("--- Fixture %d %s: passed %d, skipped %d, failed %d, total %d\n",
 563                 fixture_nr, result_to_str(ret),
 564                 d->passed, d->skipped, d->failed, d->total);
 565 }
 566
 567 int
 568 main(int argc, char *argv[])
 569 {
 570         struct weston_test_harness *harness;
 571         enum test_result_code ret;
 572         enum test_result_code result = RESULT_OK;
 573         const struct fixture_setup_array *fsa;
 574         const char *array_data;
 575         int fi;
 576         int fi_end;
 577
 578         harness = weston_test_harness_create(argc, argv);
 579
 580         fsa = fixture_setup_array_get_();
 581         array_data = fsa->array;
 582
 583         if (harness->fixt_ind == -1) {
 584                 fi = 0;
 585                 fi_end = fsa->n_elements;
 586         } else {
 587                 fi = harness->fixt_ind;
 588                 fi_end = fi + 1;
 589         }
 590
 591         tap_plan(&harness->data, fi_end - fi);
 592         testlog("Iterating through %d fixtures.\n", fi_end - fi);
 593
 594         for (; fi < fi_end; fi++) {
 595                 const void *arg = array_data + fi * fsa->element_size;
 596
 597                 testlog("--- Fixture %d...\n", fi + 1);
 598                 harness->data.fixture_iteration = fi;
 599                 harness->data.passed = 0;
 600                 harness->data.skipped = 0;
 601                 harness->data.failed = 0;
 602
 603                 ret = fixture_setup_run_(harness, arg);
 604                 fixture_report(&harness->data, ret);
 605
 606                 if (ret == RESULT_SKIP) {
 607                         tap_skip_fixture(&harness->data);
 608                         continue;
 609                 }
 610
 611                 if (ret != RESULT_OK && result != RESULT_HARD_ERROR)
 612                         result = ret;
 613                 else if (counts_to_result(&harness->data) != RESULT_OK)
 614                         result = RESULT_FAIL;
 615         }
 616
 617         weston_test_harness_destroy(harness);
 618
 619         return result;
 620 }