find_flaky_selenium_tests

#!/usr/bin/env bash
#
# Run ./run_selenium_tests RUNS times (default 1) against the current install and report
# flaky tests. A test that passes in some runs and fails in others is flaky; a test that
# fails in every run is consistently failing (likely a real failure, not flakiness).
# Skips installation (assumes ./fresh_install or ./install has been run first).
#
# The report is built from the JUnit XML result files WDIO writes into this run's log
# directory (log/<timestamp>/WDIO.xunit-*.xml — the directory the container mounts at
# /workspace/log), not from console output. After each run, the result files that run
# produced are moved to log/<timestamp>/find_flaky_selenium_tests/run-N/ and parsed;
# result files from other invocations are never touched (each has its own run directory,
# so concurrent Selenium runs in other terminals can't pollute this report). Failure
# screenshots stay in the run directory. Exits 0 only when every run passed.
#
# Components whose selenium-test script runs Cypress instead of WDIO (e.g.
# extensions/Wikibase) write no XML result files into the run directory (only artifacts
# like videos in its <Component>/ subdirectory) — for those, every run shows
# "(no result files)", the per-test sections stay empty, and flakiness is still detected
# per run from exit codes.
#
# All arguments pass through unchanged to ./run_selenium_tests (component, --spec,
# --mochaOpts.grep, ...). To check whether ONE suspected test is flaky, combine --spec
# and --mochaOpts.grep and raise RUNS — far faster than repeating a whole suite:
#   RUNS=100 ./find_flaky_selenium_tests extensions/Echo --spec tests/selenium/specs/notifications.js --mochaOpts.grep "alerts and notices are visible"
# (each run still pays the fixed container and wdio startup overhead, so 100 targeted
# runs take on the order of an hour)
#
# WARNING: This script inhibits sleep to prevent the machine from suspending.
#
# Usage: ./find_flaky_selenium_tests --spec tests/selenium/wdio-mediawiki/specs/BlankPage.js
#        ./find_flaky_selenium_tests extensions/Echo
#        RUNS=2 ./find_flaky_selenium_tests extensions/Echo --spec tests/selenium/specs/notifications.js
#        RUNS=5 ./find_flaky_selenium_tests extensions/Echo
# RUNS=N: how many times to run the tests (positive integer, default 1). The default is
# the minimal behavior (like PARALLEL); set RUNS=2 or higher to detect flakiness.
#
# See: https://www.mediawiki.org/wiki/Selenium/How-to/Run_tests_targeting_Quibble
#
# No set -euo pipefail: we want to continue after failed runs to collect all results

# Validate RUNS before sourcing lib/batch_setup so this error path has no side effects
# (batch_setup creates log directories and starts the sleep inhibitor).
runs="${RUNS:-1}" # how many times to run; ${RUNS:-1} defaults to 1 when RUNS is unset or empty (matches the ${PARALLEL:-0} style)
case "$runs" in # validate that the value is all digits
  *[!0-9]*) # the pattern matches when any character is NOT a digit (letters, '-', '.', spaces)
    echo "Error: RUNS must be a positive integer, got '$runs'" >&2 # >&2 prints to stderr
    exit 1 # invalid value; nothing has been set up yet, safe to just exit
    ;;
esac
if [ "$runs" -lt 1 ]; then # all digits but still zero (e.g. RUNS=0 or RUNS=00)
  echo "Error: RUNS must be at least 1, got '$runs'" >&2 # >&2 prints to stderr
  exit 1 # invalid value; nothing has been set up yet, safe to just exit
fi

# Parse optional component (extensions/X or skins/X) and extra arguments (e.g. --spec).
# Sets: component, zuul_project (unused here), extra_args
. "$(dirname "$0")"/lib/parse_component_args

# Bail out early if the target has no Selenium tests: N runs would all fail the same way.
# ${component:+"$component"}: pass the component argument only when one was given
# (bash 3.2-safe way to pass zero or one argument).
if ! ./selenium_tests_exist ${component:+"$component"}; then
  echo "Error: no Selenium tests found for ${component:-core}. Nothing to run." >&2 # ${component:-core}: show "core" when no component was given
  exit 1 # nothing to run; exit before any setup
fi

. "$(dirname "$0")"/lib/batch_setup # shared setup: silent output, log dir, run_step, passed/failed result tracking (also sources lib/inhibit_sleep, lib/print_results, lib/utc_timestamp, lib/print_header, lib/record_passed)
. "$(dirname "$0")"/lib/pluralize   # provides pluralize function ("1 time" vs "2 times")

# Directory for this script's per-run result files and report data, inside this run's
# log directory — each invocation gets its own report next to the logs and XML it was
# built from, and concurrent invocations (other terminals, generate_examples pool
# workers with per-slot dirs) can't collide.
report_dir="$QUIBBLE_LOG_DIR/find_flaky_selenium_tests"
rm -rf "$report_dir" # no-op on a fresh run dir; clears a previous item's report when a pool worker reuses its slot dir. run-N dirs are host-created, so removal works even when the XML files inside are owned by the container's root user (unlink needs write permission on the parent directory, not file ownership)
mkdir -p "$report_dir" 2>/dev/null || : # create the report directory; errors are handled by the check below (mirrors lib/batch_setup)
if [ ! -d "$report_dir" ]; then # fail loudly if the directory could not be created
  echo "Error: cannot create $report_dir. log/ is likely owned by root from a previous Docker run." >&2
  echo "Fix: run ./remove_all to wipe all generated directories (uses Docker-as-root to delete root-owned files), then start fresh." >&2
  exit 1 # cannot store per-run results; continuing would produce a useless report
fi
results_tsv="$report_dir/results.tsv" # accumulates run<TAB>status<TAB>test lines across all runs
: > "$results_tsv" # ':' is a no-op; the '>' redirect creates/truncates the file

printf "Running ./run_selenium_tests %d %s\n" "$runs" "$(pluralize "$runs" time)" # state the effective RUNS (including the default) so the user sees what was picked up

# List the WDIO JUnit XML result files currently in this run's log directory (the
# container mounts it at /workspace/log), one per line, sorted (comm below requires
# sorted input). Uses a for-loop over the glob instead of ls: when the glob matches
# nothing it stays literal and fails the -e test, so "no files" prints nothing instead
# of erroring.
list_result_files() {
  local f # loop variable, local so it doesn't leak into the caller
  for f in "$QUIBBLE_LOG_DIR"/WDIO.xunit-*.xml; do # every WDIO result file in the run dir
    [ -e "$f" ] && printf '%s\n' "$f" # print only if it really exists (guards the unmatched-glob case)
  done | sort # sorted output for comm
}

# --- Run loop: run the tests RUNS times, collecting each run's result files ---
run_number=1 # current run, 1..RUNS
while [ "$run_number" -le "$runs" ]; do # -le: numeric less-than-or-equal
  label="run-$run_number" # user-facing label; hyphenated so the step log gets a clean filename ($QUIBBLE_LOG_DIR/run-1--run_selenium_tests.log)
  run_dir="$report_dir/$label" # this run's result files are moved here
  mkdir -p "$run_dir" # created even when a run produces no XML, as a visible marker of the attempt

  print_header "$label" # print run label (dots follow on the same line)
  before=$(list_result_files) # snapshot the result files BEFORE the run (pre-existing files are never touched)

  # Run the tests once via run_step (handles output capture, logging, and failure
  # tracking). ${component:+"$component"}: pass component only when given.
  # ${extra_args[@]+"${extra_args[@]}"}: bash 3.2 workaround to expand the array only when
  # non-empty (an empty array with set -u in a child would error; also avoids passing "").
  if run_step "$label" ./run_selenium_tests ${component:+"$component"} ${extra_args[@]+"${extra_args[@]}"}; then
    record_passed "$label" # record only on success; on failure run_step already appended to $failed
  fi

  after=$(list_result_files) # snapshot the result files AFTER the run

  # Move the files this run produced (= new since the "before" snapshot) into run_dir.
  # comm -13 prints lines only in the second input (-1 and -3 suppress "only in first"
  # and "in both"). <(...) is process substitution: each snapshot becomes a file-like input.
  # IFS= preserves whitespace; -r prevents backslash interpretation; <&3 reads from file
  # descriptor 3 (3< <(...) connects it), the idiom batch scripts use for read loops.
  moved=0 # how many result files this run produced
  while IFS= read -r file <&3; do
    [ -n "$file" ] || continue # skip the empty line comm emits when both snapshots were empty
    mv "$file" "$run_dir"/ # mv is a rename: needs write permission on the run dir (chmod 777 by lib/log_dir), not ownership of the container-root-owned file
    moved=$((moved + 1)) # count the moved files
  done 3< <(comm -13 <(printf '%s\n' "$before") <(printf '%s\n' "$after"))

  if [ "$moved" -gt 0 ]; then # the run produced result files: parse them
    # Parse this run's XML files into run<TAB>status<TAB>test lines and append them to the
    # combined results file. -v run=N tags every line with the run number.
    awk -v run="$run_number" -f "$(dirname "$0")"/lib/parse_junit_xml.awk "$run_dir"/*.xml >> "$results_tsv"
  else
    printf "(no result files) " # e.g. the run crashed before WDIO wrote any XML; the run still counts as passed/failed by exit code
  fi
  printf " %s\n" "$(utc_timestamp)" # end the run's output line (UTC timestamp only when TIME_UTC=1)
  run_number=$((run_number + 1)) # next run
done

# --- Per-test report, printed BEFORE print_results because print_results exits 1 when
#     any run failed (same ordering as find_dependencies_minimal_gated) ---
printf "\n========================================\n"
printf "= Test results across %d %s\n" "$runs" "$(pluralize "$runs" run)" # "across 1 run" / "across 5 runs"
printf "========================================\n"
if [ -s "$results_tsv" ]; then # -s: file exists and is not empty
  awk -f "$(dirname "$0")"/lib/aggregate_flaky_tests.awk "$results_tsv" # aggregate all runs into the four report sections
else
  printf "\nNo test result files were produced by any run.\n" # nothing to aggregate (e.g. every run crashed before writing XML)
fi
printf "\nPer-run result files: %s/run-N/\n" "$report_dir" # where to find each run's raw XML for deeper digging

# Print the per-run pass/fail summary and exit 1 if any run failed (0 otherwise)
print_results