toolchain.eclass: rework tests more

Rework src_test around contrib/testsuite-management/validate_failures.py
in gcc.git.

This script is being used by the new Linaro CI effort and it appears well-suited
to us, as it allows marking expected failures easily.

Followup to 9ac3f1cf62.

Bug: https://bugs.gentoo.org/214345
Bug: https://bugs.gentoo.org/253926
Signed-off-by: Sam James <sam@gentoo.org>
This commit is contained in:
Sam James
2024-03-23 15:40:51 +00:00
parent abf8e2ee55
commit 1d93a49109
32 changed files with 141 additions and 38 deletions
+110 -38
View File
@@ -22,7 +22,7 @@ _TOOLCHAIN_ECLASS=1
DESCRIPTION="The GNU Compiler Collection"
HOMEPAGE="https://gcc.gnu.org/"
inherit edo flag-o-matic gnuconfig libtool multilib pax-utils toolchain-funcs prefix
inherit edo flag-o-matic gnuconfig libtool multilib pax-utils python-any-r1 toolchain-funcs prefix
tc_is_live() {
[[ ${PV} == *9999* ]]
@@ -84,6 +84,11 @@ tc_version_is_between() {
# Used to override GCC version. Useful for e.g. live ebuilds or snapshots.
# Defaults to ${PV}.
# @ECLASS_VARIABLE: TOOLCHAIN_GCC_VALIDATE_FAILURES_VERSION
# @DESCRIPTION:
# Version of test comparison script (validate_fails.py) to use.
: "${GCC_VALIDATE_FAILURES_VERSION:=7bbfb01a32b73842f8908de028703510a0e12057}"
# @ECLASS_VARIABLE: TOOLCHAIN_USE_GIT_PATCHES
# @DEFAULT_UNSET
# @DESCRIPTION:
@@ -93,6 +98,39 @@ tc_version_is_between() {
# for 13.0, we don't want to create new patchsets for every single 13.0 snapshot,
# so just grab patches from git each time if this variable is set).
# @ECLASS_VARIABLE: GCC_TESTS_COMPARISON_DIR
# @USER_VARIABLE
# @DESCRIPTION:
# Source of previous GCC test results and location to store new results.
: "${GCC_TESTS_COMPARISON_DIR:=${BROOT}/var/cache/gcc/testresults/${CHOST}}"
# @ECLASS_VARIABLE: GCC_TESTS_COMPARISON_SLOT
# @USER_VARIABLE
# @DESCRIPTION:
# Slot to compare test results with. Defaults to current slot.
: "${GCC_TESTS_COMPARISON_SLOT:=${SLOT}}"
# @ECLASS_VARIABLE: GCC_TESTS_IGNORE_NO_BASELINE
# @DEFAULT_UNSET
# @USER_VARIABLE
# @DESCRIPTION:
# Ignore missing baseline/reference data and create new baseline.
: "${GCC_TESTS_IGNORE_NO_BASELINE:=}"
# @ECLASS_VARIABLE: GCC_TESTS_CHECK_TARGET
# @USER_VARIABLE
# @DESCRIPTION:
# Defaults to 'check'. Allows choosing a different test target, e.g.
# 'test-gcc' (https://gcc.gnu.org/install/test.html).
: "${GCC_TESTS_CHECK_TARGET:=check}"
# @ECLASS_VARIABLE: GCC_TESTS_RUNTESTFLAGS
# @DEFAULT_UNSET
# @USER_VARIABLE
# @DESCRIPTION:
# Extra options to pass to DejaGnu as RUNTESTFLAGS.
: "${GCC_TESTS_RUNTESTFLAGS:=}"
# @ECLASS_VARIABLE: TOOLCHAIN_PATCH_DEV
# @DEFAULT_UNSET
# @DESCRIPTION:
@@ -311,6 +349,7 @@ BDEPEND="
>=sys-devel/flex-2.5.4
nls? ( sys-devel/gettext )
test? (
${PYTHON_DEPS}
>=dev-util/dejagnu-1.4.4
>=sys-devel/autogen-5.5.4
)
@@ -484,6 +523,8 @@ get_gcc_src_uri() {
[[ -n ${MUSL_VER} ]] && \
GCC_SRC_URI+=" $(gentoo_urls gcc-${MUSL_GCC_VER}-musl-patches-${MUSL_VER}.tar.${TOOLCHAIN_PATCH_SUFFIX})"
GCC_SRC_URI+=" test? ( https://gitweb.gentoo.org/proj/gcc-patches.git/plain/scripts/testsuite-management/validate_failures.py?id=${GCC_VALIDATE_FAILURES_VERSION} -> ${PN}-validate-failures-${GCC_VALIDATE_FAILURES_VERSION}.py )"
echo "${GCC_SRC_URI}"
}
@@ -513,6 +554,8 @@ toolchain_pkg_setup() {
# Avoid really confusing logs from subconfigure spam, makes logs far
# more legible.
MAKEOPTS="--output-sync=line ${MAKEOPTS}"
use test && python-any-r1_pkg_setup
}
#---->> src_unpack <<----
@@ -576,6 +619,11 @@ toolchain_src_prepare() {
tc_enable_hardened_gcc
fi
if use test ; then
cp "${DISTDIR}"/${PN}-validate-failures-${GCC_VALIDATE_FAILURES_VERSION}.py "${T}"/validate_failures.py || die
chmod +x "${T}"/validate_failures.py || die
fi
# Make sure the pkg-config files install into multilib dirs.
# Since we configure with just one --libdir, we can't use that
# (as gcc itself takes care of building multilibs). bug #435728
@@ -1835,48 +1883,68 @@ gcc_do_make() {
#---->> src_test <<----
toolchain_src_test() {
cd "${WORKDIR}"/build || die
# GCC's testsuite is a special case.
#
# * Generally, people work off comparisons rather than a full set of
# passing tests.
#
# * The guality (sic) tests are for debug info quality and are especially
# unreliable.
#
# * The execute torture tests are hopefully a good way for us to smoketest
# and find critical regresions.
# From opensuse's spec file:
# "asan needs a whole shadow address space"
# From opensuse's spec file: "asan needs a whole shadow address space"
ulimit -v unlimited
# 'asan' wants to be preloaded first, so does 'sandbox'.
# To make asan tests work disable sandbox for all of test suite.
# 'backtrace' tests also does not like 'libsandbox.so' presence.
#
# Nonfatal here as we die if compare_tests failed
SANDBOX_ON=0 LD_PRELOAD= nonfatal emake -k check
local success_tests=$?
# To make asan tests work, we disable sandbox for all of test suite.
# The 'backtrace' tests also do not like the presence of 'libsandbox.so'.
local -x SANDBOX_ON=0
local -x LD_PRELOAD=
# Controls running expensive tests in e.g. the torture testsuite.
local -x GCC_TEST_RUN_EXPENSIVE=1
# nonfatal here as we die if the comparison below fails. Also, note that
# the exit code of targets other than 'check' may be unreliable.
nonfatal emake -C "${WORKDIR}"/build -k "${GCC_TESTS_CHECK_TARGET}" RUNTESTFLAGS="${GCC_TESTS_RUNTESTFLAGS}"
if [[ -z ${GCC_TESTS_IGNORE_NO_BASELINE} && -f "${GCC_TESTS_COMPARISON_DIR}/${GCC_TESTS_COMPARISON_SLOT}/${CHOST}.xfail" ]] ; then
# TODO: Distribute some baseline results in e.g. gcc-patches.git?
# validate_failures.py manifest files support include directives.
einfo "Comparing with previous cached results at GCC_TESTS_COMPARISON_DIR=${GCC_TESTS_COMPARISON_DIR}/${GCC_TESTS_COMPARISON_SLOT}/${CHOST}.xfail"
edo "${T}"/validate_failures.py \
--srcpath="${S}" \
--build_dir="${WORKDIR}"/build \
--manifest="${GCC_TESTS_COMPARISON_DIR}/${GCC_TESTS_COMPARISON_SLOT}/${CHOST}.xfail"
else
# nonfatal first because we want to run again with comparison data if available.
nonfatal edo "${T}"/validate_failures.py \
--srcpath="${S}" \
--build_dir="${WORKDIR}"/build
ret=$?
if [[ ! -d "${BROOT}"/var/cache/gcc/${CHOST}/${SLOT} ]] && ! [[ ${success_tests} -eq 0 ]] ; then
# We have no reference data saved from a previous run to know if
# the failures are tolerable or not, so we bail out.
eerror "Reference test data does NOT exist at ${BROOT}/var/cache/gcc/${CHOST}/${SLOT}"
eerror "Tests failed and nothing to compare with, so this is a fatal error."
eerror "(Set GCC_TESTS_IGNORE_NO_BASELINE=1 to make this non-fatal for initial run.)"
eerror "No reference test data at GCC_TESTS_COMPARISON_DIR=${GCC_TESTS_COMPARISON_DIR}/${GCC_TESTS_COMPARISON_SLOT}/${CHOST}.xfail!"
eerror "GCC's tests require a baseline to compare with for any reasonable interpretation of results."
if [[ -z ${GCC_TESTS_IGNORE_NO_BASELINE} ]] ; then
if [[ -n ${GCC_TESTS_IGNORE_NO_BASELINE} ]] ; then
eerror "GCC_TESTS_IGNORE_NO_BASELINE is set, creating new baseline manifest..."
elif [[ ${ret} != 0 ]]; then
eerror "(Set GCC_TESTS_IGNORE_NO_BASELINE=1 to make this non-fatal for initial run.)"
die "Tests failed (failures occurred with no reference data)"
fi
fi
einfo "Testing complete! Review the following output to check for success or failure."
einfo "Please ignore any 'mail' lines in the summary output below (no mail is sent)."
einfo "Summary:"
"${S}"/contrib/test_summary
# If previous results exist on the system, compare with it
# TODO: Distribute some baseline results in e.g. gcc-patches.git?
if [[ -d "${BROOT}"/var/cache/gcc/${CHOST}/${SLOT} ]] ; then
einfo "Comparing with previous cached results at ${BROOT}/var/cache/gcc/${CHOST}/${SLOT}"
# Exit with the following values:
# 0 if there is nothing of interest
# 1 if there are errors when comparing single test case files
# N for the number of errors found when comparing directories
"${S}"/contrib/compare_tests "${BROOT}"/var/cache/gcc/${CHOST}/${SLOT}/ . || die "Comparison for tests results failed, error code: $?"
fi
# Produce an updated set of expected results
edo "${T}"/validate_failures.py \
--srcpath="${S}" \
--build_dir="${WORKDIR}"/build \
--manifest="${T}"/${CHOST}.xfail \
--produce_manifest &> /dev/null
}
#---->> src_install <<----
@@ -2259,15 +2327,19 @@ create_revdep_rebuild_entry() {
toolchain_pkg_preinst() {
if use test ; then
# Install as orphaned to allow comparison across more
# versions even after unmerged. Also useful for historical records
# and tracking down regressions a while after they first appeared,
# but were only just reported.
einfo "Copying test results to ${BROOT}/var/cache/gcc/${CHOST}/${SLOT} for future comparison"
# Install as orphaned to allow comparison across more versions even
# after unmerged. Also useful for historical records and tracking
# down regressions a while after they first appeared, but were only
# just reported.
einfo "Copying test results to ${GCC_TESTS_COMPARISON_DIR}/${SLOT}/${CHOST}.xfail for future comparison"
(
mkdir -p "${BROOT}"/var/cache/gcc/${CHOST}/${SLOT} || die
mkdir -p "${GCC_TESTS_COMPARISON_DIR}/${SLOT}" || die
cd "${T}"/test-results || die
find . -name \*.sum -exec cp --parents -v {} "${BROOT}"/var/cache/gcc/${CHOST}/${SLOT} \;
# May not exist with test-fail-continue
if [[ -f "${T}"/${CHOST}.xfail ]] ; then
cp -v "${T}"/${CHOST}.xfail "${GCC_TESTS_COMPARISON_DIR}/${SLOT}" || die
fi
find . -name \*.sum -exec cp --parents -v {} "${GCC_TESTS_COMPARISON_DIR}/${SLOT}" \;
)
fi
}
+1
View File
@@ -40,3 +40,4 @@ DIST gcc-8.5.0-patches-4.tar.xz 18616 BLAKE2B 7594cbaea5daecf1a0b3d526c06aeba0a8
DIST gcc-8.5.0.tar.xz 63841008 BLAKE2B aa81a1a730fd7371360f6abed6ba78b5843fd18c58d5de5687acc320741b9e430e85df3535a1ef7a26051409be8d2f0945f503e5968480d919103123a99d4b12 SHA512 92f599680e6b7fbce88bcdda810f468777d541e5fddfbb287f7977d51093de2a5178bd0e6a08dfe37090ea10a0508a43ccd00220041abbbec33f1179bfc174d8
DIST gcc-9.5.0-patches-2.tar.xz 14324 BLAKE2B 1d3bf3e673108ad2fcdcb86daefb4603d1c9f6aa815e34d1591054d466b76d6f50cce124ab4760528c4951ea16eb00888a3e6485417f81ebf250548732df4a8e SHA512 a6d6f650aaf0bd193a17ac279649f0544a282cea295824384c59341c3000eac4b574b60e00dd427585d928f9768166abf85168b47772132790ad98854acacbbf
DIST gcc-9.5.0.tar.xz 72462752 BLAKE2B 69e39314ee6bb46e3be491ba9c3cbc4914cb716f732c6d3f2b14d9382750b40c4f14b5d3051225b81f936ef8297b0ba8cb5439512e961dae5815d3bee2b0bfdb SHA512 cdd144ce4f747e051480410afc8506c90a57cb45da89071ddae377b1453bca3014422455ade9fe4563ebe51f4b6348cbc0c61905a9b8801cd597d51ad0ec62b3
DIST gcc-validate-failures-7bbfb01a32b73842f8908de028703510a0e12057.py 27533 BLAKE2B 58917750d247d3a9e7078c62fd4412027a7d02d06c659a9c0743697529754218257c00dc6e10dff467e78b4e019225c39b1627b48a5298d13481a7072748a15f SHA512 b74d6358538d1b076f3d5b6632ea405b99627d5b5621b6f5101cb3453be13ba41e11d590039535b0c74b1624b69885a7093600d5395cc60bc63cea15ae7472b3
+1
View File
@@ -8,6 +8,7 @@ PATCH_GCC_VER="10.5.0"
PATCH_VER="6"
MUSL_VER="2"
MUSL_GCC_VER="10.5.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="11.4.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="11.4.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="11.4.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="11.4.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="11.4.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="11.4.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
+1
View File
@@ -6,6 +6,7 @@ EAPI=8
TOOLCHAIN_PATCH_DEV="sam"
PATCH_GCC_VER="11.4.0"
MUSL_GCC_VER="11.4.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="12.3.0"
PATCH_VER="2"
MUSL_VER="1"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="12.3.0"
PATCH_VER="3"
MUSL_VER="1"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="12.3.0"
PATCH_VER="3"
MUSL_VER="1"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="12.3.0"
PATCH_VER="3"
MUSL_VER="1"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="12.3.0"
PATCH_VER="3"
MUSL_VER="1"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="12.3.0"
PATCH_VER="3"
MUSL_VER="1"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
+1
View File
@@ -6,6 +6,7 @@ EAPI=8
TOOLCHAIN_PATCH_DEV="sam"
PATCH_GCC_VER="12.3.0"
MUSL_GCC_VER="12.3.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="13.2.0"
PATCH_VER="12"
MUSL_VER="2"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="13.2.0"
PATCH_VER="14"
MUSL_VER="2"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="13.2.0"
PATCH_VER="14"
MUSL_VER="2"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="13.2.0"
PATCH_VER="14"
MUSL_VER="2"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="13.2.0"
PATCH_VER="14"
MUSL_VER="2"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="13.2.0"
PATCH_VER="14"
MUSL_VER="2"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
+1
View File
@@ -6,6 +6,7 @@ EAPI=8
TOOLCHAIN_PATCH_DEV="sam"
PATCH_GCC_VER="13.2.0"
MUSL_GCC_VER="13.2.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="14.0.0"
PATCH_VER="23"
MUSL_VER="1"
MUSL_GCC_VER="14.0.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="14.0.0"
PATCH_VER="23"
MUSL_VER="1"
MUSL_GCC_VER="14.0.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="14.0.0"
PATCH_VER="24"
MUSL_VER="1"
MUSL_GCC_VER="14.0.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="14.0.0"
PATCH_VER="25"
MUSL_VER="1"
MUSL_GCC_VER="14.0.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
@@ -8,6 +8,7 @@ PATCH_GCC_VER="14.0.0"
PATCH_VER="25"
MUSL_VER="1"
MUSL_GCC_VER="14.0.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)
+1
View File
@@ -6,6 +6,7 @@ EAPI=8
TOOLCHAIN_PATCH_DEV="sam"
PATCH_GCC_VER="14.0.0"
MUSL_GCC_VER="14.0.0"
PYTHON_COMPAT=( python3_{10..11} )
if [[ -n ${TOOLCHAIN_GCC_RC} ]] ; then
# Cheesy hack for RCs
+1
View File
@@ -6,6 +6,7 @@ EAPI="7"
TOOLCHAIN_PATCH_SUFFIX="xz"
TOOLCHAIN_PATCH_DEV="sam"
PATCH_VER="4"
PYTHON_COMPAT=( python3_{10..11} )
inherit toolchain
+1
View File
@@ -6,6 +6,7 @@ EAPI=8
TOOLCHAIN_PATCH_DEV="sam"
PATCH_GCC_VER="9.5.0"
PATCH_VER="2"
PYTHON_COMPAT=( python3_{10..11} )
if [[ ${PV} == *.9999 ]] ; then
MY_PV_2=$(ver_cut 2)