tap-xfail-tests.sh 1.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263
  1. #! /bin/sh
  2. # Copyright (C) 2011-2017 Free Software Foundation, Inc.
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation; either version 2, or (at your option)
  7. # any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. # TAP and $(XFAIL_TESTS): test results without directives are turned from
  17. # PASS to XPASS and from FAIL to XFAIL; other results are unchanged.
  18. . test-init.sh
  19. . tap-setup.sh
  20. echo 'XFAIL_TESTS = $(TESTS)' >> Makefile
  21. cat > all.test <<END
  22. 1..6
  23. ok 1
  24. not ok 2
  25. ok 3 # TODO
  26. not ok 4 # TODO
  27. ok 5 # SKIP
  28. # The next should count as a failure (and thus will be
  29. # rendered into an expected failure by XFAIL_TESTS).
  30. not ok 6 # SKIP
  31. Bail out!
  32. END
  33. run_make -O -e FAIL check
  34. count_test_results total=7 pass=0 fail=0 xpass=2 xfail=3 skip=1 error=1
  35. grep '^XPASS: all\.test 1$' stdout
  36. grep '^XFAIL: all\.test 2$' stdout
  37. grep '^XPASS: all\.test 3 # TODO' stdout
  38. grep '^XFAIL: all\.test 4 # TODO' stdout
  39. grep '^SKIP: all\.test 5 # SKIP' stdout
  40. grep '^XFAIL: all\.test 6 # SKIP' stdout
  41. grep '^ERROR: all\.test - Bail out!' stdout
  42. # Check that the exit status of the testsuite is indeed 0 when we
  43. # would expect success.
  44. cat > all.test <<END
  45. 1..3
  46. not ok 1
  47. ok 2 # SKIP
  48. not ok 3 # TODO
  49. END
  50. run_make -O check
  51. count_test_results total=3 pass=0 fail=0 xpass=0 xfail=2 skip=1 error=0
  52. :