test_integration

#!/usr/bin/env bash
#
# Run fast integration tests and report which ones passed or failed.
# Useful for detecting regressions after changes.
# Each entry is annotated with its approximate runtime in minutes (or <1m).
# Total runtime: ~20 minutes.
# WARNING: This script inhibits sleep to prevent the machine from suspending.
#
# Usage: ./test_integration
#

# No set -euo pipefail: we want to continue after failures to collect all results

. "$(dirname "$0")"/lib/batch_setup # shared setup: silent output, log dir, result tracking
. "$(dirname "$0")"/lib/run_test    # provides run_test function and test_counter

# --- Development and CI ---
run_test ./help                        # <1m: test that help output works
run_test ./help install                # <1m: test detailed help for a specific script
run_test ./lint                        # <1m: test that all scripts pass shellcheck
run_test ./ci                          # <1m: test that CI lint passes in Docker
run_test ./generate_example log/generate_example_test.txt './help'  # <1m: test capturing a command's output to a file
run_test env PREVIEW=1 ./generate_examples  # <1m: test the bulk regenerator's quick-subset iteration logic (PREVIEW to avoid re-running the scripts)
run_test env PREVIEW=1 ./generate_examples_slow  # <1m: test the full-set variant previews every Usage line, slow scripts included
run_test env PREVIEW=1 DRY_RUN=1 ./generate_examples  # <1m: test the DRY_RUN plan targets examples/dry_run/ (PREVIEW so nothing runs or is written)

# --- Setup and cleanup ---
run_test ./prepare                     # 6m: test that prepare pulls image and clones repos
run_test env PARALLEL=1 ./prepare      # <1m: test prepare parallel cap (re-runs 1 job at a time; image+repos already present)
run_test ./fetch                       # <1m: test fetching all bare repos (update stale repos before use)
run_test ./fetch ref/mediawiki/core.git  # <1m: test fetching a specific repo
run_test env PARALLEL=1 ./fetch        # <1m: test parallel fetch
run_test ./fresh_install               # 1m: test fresh core install
run_test env BRANCH=master ./fresh_install  # 1m: test BRANCH installs and checks out a branch (fails loudly if the re-attach can't)
run_test env DRY_RUN=1 ./fresh_install  # <1m: test DRY_RUN passes --dry-run to Quibble and leaves src/ untouched (the ./save below saves the install these dry runs must not wipe)
run_test env DRY_RUN=1 BRANCH=master ./fresh_install  # <1m: test BRANCH+DRY_RUN skips the re-attach (no real install)
run_test ./save                        # <1m: test saving fresh_install state — deliberately BEFORE the GERRIT_PATCHES entry, so the saved state is a clean master core (and AFTER the DRY_RUN entries, so a dry run that wiped src/ fails here instead of poisoning the save)
# The GERRIT_PATCHES entry checks core out at the change's base commit, which is old (and
# only gets older). It must sit between ./save and ./restore: letting the patched tree leak
# into the save or the entries below breaks them — an old core against the current vendor/
# fails Quibble's wiki install with PHP fatals (e.g. the TransactionProfiler::setLogger
# signature mismatch that made every core Selenium entry fail).
run_test env GERRIT_PATCHES=refs/changes/94/1146994/6 ./fresh_install  # 1m: test GERRIT_PATCHES fetches and checks out a core patch
run_test ./restore                     # <1m: test restoring saved state — over the gerrit-patched src, proving restore replaces a divergent tree (and healing core for the entries below)
run_test env ENVIRONMENT=0 ./fresh_install  # 1m: test environment isolation
run_test env ENVIRONMENT=0 ./save           # 1m: test saving with environment
run_test env ENVIRONMENT=0 ./restore        # 1m: test restoring with environment
run_test env ENVIRONMENT=0 ./remove          # <1m: test cleaning with environment

# --- Core tests ---
run_test ./selenium_tests_exist        # <1m: test that core has Selenium tests
run_test ./run_selenium_tests          # 1m: test running all core Selenium tests
run_test ./run_selenium_tests --spec tests/selenium/wdio-mediawiki/specs/BlankPage.js  # <1m: test running a specific test file
run_test ./run_selenium_tests --spec tests/selenium/specs/user.js --mochaOpts.grep "should be able to create account"  # <1m: test filtering by grep pattern
run_test env RUNS=2 ./find_flaky_selenium_tests --spec tests/selenium/wdio-mediawiki/specs/BlankPage.js  # 2m: test the flaky finder's run loop, per-run XML attribution, and report on a fast spec

# --- Extension tests (extensions/Echo) ---
run_test ./install extensions/Echo     # 1m: test installing an extension
run_test env BRANCH=master ./install extensions/Echo  # 1m: test BRANCH on an extension install (fails loudly if the re-attach can't)
run_test env GERRIT_PATCHES=refs/changes/20/1306020/1 ./install extensions/Echo  # 1m: test GERRIT_PATCHES fetches and checks out a component patch
run_test env DRY_RUN=1 ./install extensions/Echo  # <1m: test DRY_RUN passes --dry-run to Quibble
run_test env RESOLVE_REQUIRES=0 ./install extensions/Echo  # 1m: test RESOLVE_REQUIRES=0 omits --resolve-requires
run_test env QUIBBLE_DEPS="" ./install extensions/Echo  # 1m: test installing with no dependencies
run_test env QUIBBLE_DEPS="EventLogging" ./install extensions/Echo  # 1m: test installing with specific dependencies
run_test ./selenium_tests_exist extensions/Echo  # <1m: test that extension has Selenium tests
run_test ./run_selenium_tests extensions/Echo  # 1m: test running extension Selenium tests
run_test ./run_selenium_tests extensions/Echo --spec tests/selenium/specs/notifications.js  # <1m: test running a specific extension test file
run_test ./run_selenium_tests extensions/Echo --spec tests/selenium/specs/notifications.js --mochaOpts.grep "alerts and notices are visible"  # <1m: test filtering extension tests by grep
run_test ./run_php_unit_tests extensions/Echo  # 2m: test running extension PHPUnit tests
run_test ./run_php_unit_tests extensions/Echo --filter testNotificationCount  # <1m: test filtering PHPUnit tests
run_test env DRY_RUN=1 ./run_php_unit_tests extensions/Echo  # <1m: test DRY_RUN passes --dry-run to Quibble

# --- Skin tests (skins/MinervaNeue) ---
run_test ./install skins/MinervaNeue  # 1m: test installing a skin (exercises skins/ code path)
run_test ./selenium_tests_exist skins/MinervaNeue  # <1m: test that skin has Selenium tests
run_test ./run_selenium_tests skins/MinervaNeue  # 1m: test running skin Selenium tests

# --- Data queries ---
run_test ./list_dependencies extensions/Echo  # <1m: test listing extension dependencies
run_test ./list_dependencies_required extensions/Echo  # <1m: test listing required dependencies
run_test ./list_dependencies_optional extensions/Echo  # <1m: test listing optional dependencies
run_test ./list_dependencies_combinations extensions/Echo  # <1m: test listing dependency combinations
run_test ./list_dependencies skins/MinervaNeue  # <1m: test listing skin dependencies
run_test ./list_dependencies_required skins/MinervaNeue  # <1m: test listing required skin dependencies
run_test ./list_dependencies_optional skins/MinervaNeue  # <1m: test listing optional skin dependencies
run_test ./list_dependencies_combinations skins/MinervaNeue  # <1m: test listing skin dependency combinations
run_test ./list_gated                  # <1m: test listing gated repos
run_test ./suggest_parallel            # <1m: test suggesting number of parallel workers

# --- Cleanup ---
run_test ./remove                       # <1m: test cleaning up src/

# Print pass/fail summary and exit with error if any script failed
print_results