tests

2026-08-02 15:26:07 +02:00
parent 336b36c8e1
commit b2eb145a6f
415 changed files with 24264 additions and 36 deletions
@@ -0,0 +1,220 @@
+#!/usr/bin/env bats
+
+bats_require_minimum_version 1.5.0
+
+load test_helper
+fixtures parallel
+
+# shellcheck disable=SC2034
+BATS_TEST_TIMEOUT=10 # only intended for the "short form ..."" test
+
+setup() {
+  (type -p "${BATS_PARALLEL_BINARY_NAME:-"parallel"}" &>/dev/null && "${BATS_PARALLEL_BINARY_NAME:-"parallel"}" --version &>/dev/null) || skip "--jobs requires GNU parallel"
+  (type -p flock &>/dev/null || type -p shlock &>/dev/null) || skip "--jobs requires flock/shlock"
+}
+
+check_parallel_tests() { # <expected maximum parallelity>
+  local expected_maximum_parallelity="$1"
+  local expected_number_of_lines="${2:-$((2 * expected_maximum_parallelity))}"
+
+  max_parallel_tests=0
+  started_tests=0
+  read_lines=0
+  while IFS= read -r line; do
+    ((++read_lines))
+    case "$line" in
+    "start "*)
+      if ((++started_tests > max_parallel_tests)); then
+        max_parallel_tests="$started_tests"
+      fi
+      ;;
+    "stop "*)
+      ((started_tests--))
+      ;;
+    esac
+  done <"$FILE_MARKER"
+
+  echo "max_parallel_tests: $max_parallel_tests"
+  [[ $max_parallel_tests -eq $expected_maximum_parallelity ]]
+
+  echo "read_lines: $read_lines"
+  [[ $read_lines -eq $expected_number_of_lines ]]
+}
+
+@test "parallel test execution with --jobs" {
+  # shellcheck disable=SC2031,SC2030
+  export FILE_MARKER
+  # shellcheck disable=SC2030
+  FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX")
+  # shellcheck disable=SC2030
+  export PARALLELITY=3
+  reentrant_run bats --jobs $PARALLELITY "$FIXTURE_ROOT/parallel.bats"
+
+  [ "$status" -eq 0 ]
+  # Make sure the lines are in-order.
+  [[ "${lines[0]}" == "1..3" ]]
+  for t in {1..3}; do
+    [[ "${lines[$t]}" == "ok $t slow test $t" ]]
+  done
+
+  check_parallel_tests $PARALLELITY
+}
+
+@test "parallel can preserve environment variables" {
+  export TEST_ENV_VARIABLE='test-value'
+  reentrant_run bats --jobs 2 "$FIXTURE_ROOT/parallel-preserve-environment.bats"
+  echo "$output"
+  [[ "$status" -eq 0 ]]
+}
+
+@test "parallel suite execution with --jobs" {
+  # shellcheck disable=SC2031,SC2030
+  export FILE_MARKER
+  # shellcheck disable=SC2030
+  FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX")
+  # shellcheck disable=SC2031,SC2030
+  export PARALLELITY=12
+
+  # file parallelization is needed for maximum parallelity!
+  # If we got over the skip (if no GNU parallel) in setup() we can re-enable it safely!
+  unset BATS_NO_PARALLELIZE_ACROSS_FILES
+  reentrant_run bash -c "bats --jobs $PARALLELITY \"${FIXTURE_ROOT}/suite/\" 2> >(grep -v '^parallel: Warning: ')"
+
+  echo "$output"
+  [ "$status" -eq 0 ]
+
+  # Make sure the lines are in-order.
+  [[ "${lines[0]}" == "1..$PARALLELITY" ]]
+  i=0
+  for _ in {1..4}; do
+    for t in {1..3}; do
+      ((++i))
+      [[ "${lines[$i]}" == "ok $i slow test $t" ]]
+    done
+  done
+
+  check_parallel_tests $PARALLELITY
+}
+
+@test "setup_file is not over parallelized" {
+  #shellcheck disable=SC2031
+  export FILE_MARKER
+  FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX")
+  #shellcheck disable=SC2031,SC2030
+  export PARALLELITY=2
+
+  # file parallelization is needed for this test!
+  # If we got over the skip (if no GNU parallel) in setup() we can re-enable it safely!
+  unset BATS_NO_PARALLELIZE_ACROSS_FILES
+  # run 4 files with parallelity of 2 -> serialize 2
+  reentrant_run bats --jobs $PARALLELITY "$FIXTURE_ROOT/setup_file"
+
+  [[ $status -eq 0 ]] || (
+    echo "$output"
+    false
+  )
+
+  cat "$FILE_MARKER"
+
+  [[ $(grep -c "start " "$FILE_MARKER") -eq 4 ]] # beware of grepping the filename as well!
+  [[ $(grep -c "stop " "$FILE_MARKER") -eq 4 ]]
+
+  check_parallel_tests $PARALLELITY 8
+}
+
+@test "running the same file twice runs its tests twice without errors" {
+  reentrant_run bats --jobs 2 "$FIXTURE_ROOT/../bats/passing.bats" "$FIXTURE_ROOT/../bats/passing.bats"
+  echo "$output"
+  [[ $status -eq 0 ]]
+  [[ "${lines[0]}" == "1..2" ]] # got 2x1 tests
+  [[ "${lines[1]}" == "ok 1 "* ]]
+  [[ "${lines[2]}" == "ok 2 "* ]]
+  [[ "${#lines[@]}" -eq 3 ]]
+}
+
+@test "parallelity factor is met exactly" {
+  # shellcheck disable=SC2031
+  export MARKER_FILE="${BATS_TEST_TMPDIR}/marker" PARALLELITY=5 # run the 10 tests in 2 batches with 5 test each
+  bats --jobs $PARALLELITY "$FIXTURE_ROOT/parallel_factor.bats"
+  local current_parallel_count=0 maximum_parallel_count=0 total_count=0
+  while read -r line; do
+    case "$line" in
+    setup*)
+      ((++current_parallel_count))
+      ((++total_count))
+      ;;
+    teardown*)
+      ((current_parallel_count--))
+      ;;
+    esac
+    if ((current_parallel_count > maximum_parallel_count)); then
+      maximum_parallel_count=$current_parallel_count
+    fi
+  done <"$MARKER_FILE"
+
+  cat "$MARKER_FILE" # for debugging purposes
+  [[ "$maximum_parallel_count" -eq $PARALLELITY ]]
+  [[ "$current_parallel_count" -eq 0 ]]
+  [[ "$total_count" -eq 10 ]]
+}
+
+@test "parallel mode correctly forwards failure return code" {
+  reentrant_run bats --jobs 2 "$FIXTURE_ROOT/../bats/failing.bats"
+  [[ "$status" -eq 1 ]]
+}
+
+@test "--no-parallelize-across-files test file detects parallel execution" {
+  # ensure that we really run parallelization across files!
+  # (setup should have skipped already, if there was no GNU parallel)
+  unset BATS_NO_PARALLELIZE_ACROSS_FILES
+  FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") \
+    reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_across_files/"
+}
+
+@test "--no-parallelize-across-files prevents parallelization across files" {
+  FILE_MARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") \
+    bats --jobs 2 --no-parallelize-across-files "$FIXTURE_ROOT/must_not_parallelize_across_files/"
+}
+
+@test "--no-parallelize-across-files does not prevent parallelization within files" {
+  reentrant_run ! bats --jobs 2 --no-parallelize-across-files "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "--no-parallelize-within-files test file detects parallel execution" {
+  reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "--no-parallelize-within-files prevents parallelization within files" {
+  bats --jobs 2 --no-parallelize-within-files "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "--no-parallelize-within-files does not prevent parallelization across files" {
+  # ensure that we really run parallelization across files!
+  # (setup should have skipped already, if there was no GNU parallel)
+  unset BATS_NO_PARALLELIZE_ACROSS_FILES
+  FILEMARKER=$(mktemp "${BATS_RUN_TMPDIR}/file_marker.XXXXXX") \
+    reentrant_run ! bats --jobs 2 --no-parallelize-within-files "$FIXTURE_ROOT/must_not_parallelize_across_files/"
+}
+
+@test "BATS_NO_PARALLELIZE_WITHIN_FILE works from inside setup_file()" {
+  DISABLE_IN_SETUP_FILE_FUNCTION=1 bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "BATS_NO_PARALLELIZE_WITHIN_FILE works from outside all functions" {
+  DISABLE_OUTSIDE_ALL_FUNCTIONS=1 bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "BATS_NO_PARALLELIZE_WITHIN_FILE does not work from inside setup()" {
+  DISABLE_IN_SETUP_FUNCTION=1 reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "BATS_NO_PARALLELIZE_WITHIN_FILE does not work from inside test function" {
+  DISABLE_IN_TEST_FUNCTION=1 reentrant_run ! bats --jobs 2 "$FIXTURE_ROOT/must_not_parallelize_within_file.bats"
+}
+
+@test "Negative jobs number does not run endlessly" {
+  unset BATS_NO_PARALLELIZE_ACROSS_FILES
+  run bats -j -3 "$FIXTURE_ROOT/../bats/passing.bats"
+  (( SECONDS < 5 ))
+  [ "${lines[1]}" = 'Invalid number of jobs: -3' ]
+}