tap-plan-errors.sh 2.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112
  1. #! /bin/sh
  2. # Copyright (C) 2011-2017 Free Software Foundation, Inc.
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation; either version 2, or (at your option)
  7. # any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. # TAP support: the following situations should be flagged as errors:
  17. # - unmatched test plan (too few tests run)
  18. # - multiple test plans
  19. # - missing test plan
  20. # - misplaced test plan (tests run after a late plan)
  21. # Checks about unplanned tests are performed in 'tap-unplanned.sh'
  22. # and 'tap-skip-whole-badcount.sh'. More checks about corner-cases
  23. # in TAP plans are performed in 'tap-plan-corner.sh'.
  24. . test-init.sh
  25. . tap-setup.sh
  26. my_check ()
  27. {
  28. cat > all.test
  29. test -n "$err" || fatal_ "\$err not set before calling my_check"
  30. cat all.test # For debugging.
  31. run_make -O -e FAIL check
  32. count_test_results "$@"
  33. grep "^ERROR: all\\.test $err$" stdout
  34. unset err
  35. }
  36. err='- too few tests run (expected 2, got 1)'
  37. my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  38. 1..2
  39. ok 1
  40. END
  41. err='- too few tests run (expected 12, got 3)'
  42. my_check total=4 pass=2 fail=0 xpass=0 xfail=1 skip=0 error=1 <<END
  43. ok 1
  44. ok 2
  45. not ok 3 # TODO
  46. 1..12
  47. END
  48. err='- too few tests run (expected 1, got 0)'
  49. my_check total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  50. 1..1
  51. END
  52. err='2 # AFTER LATE PLAN'
  53. my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  54. ok 1
  55. 1..2
  56. ok 2
  57. END
  58. err='5 # AFTER LATE PLAN'
  59. my_check total=5 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  60. ok 1
  61. ok 2
  62. ok 3
  63. ok 4
  64. 1..5
  65. ok 5
  66. END
  67. err='- missing test plan'
  68. my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  69. ok 1
  70. END
  71. # The two test plans here are deliberately equal.
  72. err='- multiple test plans'
  73. my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  74. 1..2
  75. ok 1
  76. ok 2
  77. 1..2
  78. END
  79. # The second plan is diagnosed as extra, and only the first one is
  80. # relevant w.r.t. the number of the expected test.
  81. err='- multiple test plans'
  82. my_check total=4 pass=3 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  83. 1..3
  84. ok 1
  85. ok 2
  86. 1..2
  87. ok 3
  88. END
  89. # As above, in a slightly different situation.
  90. err='- multiple test plans'
  91. my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  92. 1..2
  93. ok 1
  94. ok 2
  95. 1..4
  96. END
  97. :