test-log.sh 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176
  1. #! /bin/sh
  2. # Copyright (C) 2011-2017 Free Software Foundation, Inc.
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation; either version 2, or (at your option)
  7. # any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. # Check parallel-tests features:
  17. # - log file creation
  18. # - log file removal
  19. # - stdout and stderr of a test script go in its log file
  20. # - TEST_SUITE_LOG redefinition, at either automake or make time
  21. # - VERBOSE environment variable support
  22. # Keep in sync with 'tap-log.sh'.
  23. . test-init.sh
  24. cat >> configure.ac <<END
  25. AC_OUTPUT
  26. END
  27. cat > Makefile.am << 'END'
  28. TESTS = pass.test skip.test xfail.test fail.test xpass.test error.test
  29. XFAIL_TESTS = xpass.test xfail.test
  30. TEST_SUITE_LOG = global.log
  31. END
  32. # Custom markers, for use in grepping checks.
  33. cmarker=::: # comment marker
  34. pmarker=%%% # plain maker
  35. cat > pass.test <<END
  36. #! /bin/sh
  37. echo "$pmarker pass $pmarker" >&2
  38. echo "# $cmarker pass $cmarker" >&2
  39. exit 0
  40. END
  41. cat > skip.test <<END
  42. #! /bin/sh
  43. echo "$pmarker skip $pmarker"
  44. echo "# $cmarker skip $cmarker"
  45. exit 77
  46. END
  47. cat > xfail.test <<END
  48. #! /bin/sh
  49. echo "$pmarker xfail $pmarker" >&2
  50. echo "# $cmarker xfail $cmarker" >&2
  51. exit 1
  52. END
  53. cat > fail.test <<END
  54. #! /bin/sh
  55. echo "$pmarker fail $pmarker"
  56. echo "# $cmarker fail $cmarker"
  57. exit 1
  58. END
  59. cat > xpass.test <<END
  60. #! /bin/sh
  61. echo "$pmarker xpass $pmarker" >&2
  62. echo "# $cmarker xpass $cmarker" >&2
  63. exit 0
  64. END
  65. cat > error.test <<END
  66. #! /bin/sh
  67. echo "$pmarker error $pmarker"
  68. echo "# $cmarker error $cmarker"
  69. exit 99
  70. END
  71. chmod a+x *.test
  72. $ACLOCAL
  73. $AUTOCONF
  74. $AUTOMAKE -a
  75. ./configure
  76. run_make -e FAIL TEST_SUITE_LOG=my.log check
  77. ls -l # For debugging.
  78. test ! -e test-suite.log
  79. test ! -e global.log
  80. test -f my.log
  81. st=0
  82. for result in pass fail xfail xpass skip error; do
  83. cat $result.log # For debugging.
  84. $FGREP "$pmarker $result $pmarker" $result.log || st=1
  85. $FGREP "$cmarker $result $cmarker" $result.log || st=1
  86. done
  87. test $st -eq 0 || exit 1
  88. cat my.log # For debugging.
  89. for result in xfail fail xpass skip error; do
  90. cat $result.log # For debugging.
  91. $FGREP "$pmarker $result $pmarker" my.log || st=1
  92. $FGREP "$cmarker $result $cmarker" my.log || st=1
  93. done
  94. test $($FGREP -c "$pmarker" my.log) -eq 5
  95. test $($FGREP -c "$cmarker" my.log) -eq 5
  96. have_rst_section ()
  97. {
  98. eqeq=$(echo "$1" | sed 's/./=/g')
  99. # Assume $1 contains no RE metacharacters.
  100. sed -n "/^$1$/,/^$eqeq$/p" $2 > got
  101. (echo "$1" && echo "$eqeq") > exp
  102. cat exp
  103. cat got
  104. diff exp got
  105. }
  106. # Passed test scripts shouldn't be mentioned in the global log.
  107. $EGREP ':.*[^x]pass' my.log && exit 1
  108. # But failing (expectedly or not) and skipped ones should.
  109. have_rst_section 'SKIP: skip' my.log
  110. have_rst_section 'FAIL: fail' my.log
  111. have_rst_section 'XFAIL: xfail' my.log
  112. have_rst_section 'XPASS: xpass' my.log
  113. have_rst_section 'ERROR: error' my.log
  114. touch error2.log test-suite.log global.log
  115. run_make TEST_SUITE_LOG=my.log mostlyclean
  116. ls -l # For debugging.
  117. test ! -e my.log
  118. test ! -e pass.log
  119. test ! -e fail.log
  120. test ! -e xfail.log
  121. test ! -e xpass.log
  122. test ! -e skip.log
  123. test ! -e error.log
  124. # "make mostlyclean" shouldn't remove unrelated log files.
  125. test -f error2.log
  126. test -f test-suite.log
  127. test -f global.log
  128. rm -f *.log
  129. run_make -O -e FAIL check VERBOSE=yes
  130. cat global.log
  131. test ! -e my.log
  132. test ! -e test-suite.log
  133. # Check that VERBOSE causes the global testsuite log to be
  134. # emitted on stdout.
  135. out=$(cat stdout)
  136. log=$(cat global.log)
  137. case $out in *"$log"*) ;; *) exit 1;; esac
  138. touch error2.log test-suite.log my.log
  139. $MAKE clean
  140. ls -l # For debugging.
  141. test ! -e global.log
  142. test ! -e pass.log
  143. test ! -e fail.log
  144. test ! -e xfail.log
  145. test ! -e xpass.log
  146. test ! -e skip.log
  147. test ! -e error.log
  148. # "make clean" shouldn't remove unrelated log files.
  149. test -f error2.log
  150. test -f test-suite.log
  151. test -f my.log
  152. rm -f *.log
  153. :