diff --git a/notify/fixes.sum b/notify/fixes.sum index e6fe5c7..87db459 100644 --- a/notify/fixes.sum +++ b/notify/fixes.sum @@ -1,6 +1,49 @@ + === test-suite tests === + +Running test-suite:Fujitsu/C/0056 ... +FAIL: test-suite :: Fujitsu/C/0056/Fujitsu-C-0056_0011.test + +Running test-suite:Fujitsu/C/0072 ... +NOEXE: test-suite :: Fujitsu/C/0072/Fujitsu-C-0072_0004.test +NOEXE: test-suite :: Fujitsu/C/0072/Fujitsu-C-0072_0008.test + +Running test-suite:Fujitsu/C/0119 ... +NOEXE: test-suite :: Fujitsu/C/0119/Fujitsu-C-0119_0228.test + +Running test-suite:Fujitsu/C/0128 ... +NOEXE: test-suite :: Fujitsu/C/0128/Fujitsu-C-0128_0004.test +NOEXE: test-suite :: Fujitsu/C/0128/Fujitsu-C-0128_0008.test + +Running test-suite:Fujitsu/C/0139 ... +NOEXE: test-suite :: Fujitsu/C/0139/Fujitsu-C-0139_0039.test + +Running test-suite:Fujitsu/Fortran/0204 ... +FAIL: test-suite :: Fujitsu/Fortran/0204/Fujitsu-Fortran-0204_0018.test +FAIL: test-suite :: Fujitsu/Fortran/0204/Fujitsu-Fortran-0204_0019.test +FAIL: test-suite :: Fujitsu/Fortran/0204/Fujitsu-Fortran-0204_0016.test +FAIL: test-suite :: Fujitsu/Fortran/0204/Fujitsu-Fortran-0204_0017.test + +Running test-suite:Fujitsu/Fortran/0352 ... +FAIL: test-suite :: Fujitsu/Fortran/0352/Fujitsu-Fortran-0352_0059.test + +Running test-suite:Fujitsu/Fortran/0360 ... +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0323.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0319.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0321.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0282.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0324.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0283.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0322.test +FAIL: test-suite :: Fujitsu/Fortran/0360/Fujitsu-Fortran-0360_0320.test + +Running test-suite:Fujitsu/Fortran/0670 ... +FAIL: test-suite :: Fujitsu/Fortran/0670/Fujitsu-Fortran-0670_0064.test + +Running test-suite:Fujitsu/Fortran/1052 ... +FAIL: test-suite :: Fujitsu/Fortran/1052/Fujitsu-Fortran-1052_0112.test === Results Summary === -# of stable fails 113 -# of flaky entries 186 +# of stable fails 108 +# of flaky entries 25 # of all tests 89090 diff --git a/notify/jira/comment-template.txt b/notify/jira/comment-template.txt index e7ff5e9..76d92a4 100644 --- a/notify/jira/comment-template.txt +++ b/notify/jira/comment-template.txt @@ -1,3 +1,3 @@ [LLVM-1290] -no regression/no fix +29 regressions 22 fixes Details: https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1304/artifact/artifacts/notify/mail-body.txt/*view*/ diff --git a/notify/lnt_report.json b/notify/lnt_report.json index 15bf9f2..8bca49c 100644 --- a/notify/lnt_report.json +++ b/notify/lnt_report.json @@ -7,18 +7,18 @@ "Info": { "tag": "tcwg_flang_test", "test_url": "https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1304/", - "run_order": "1304 (llvmorg-22-init-04760-g2b46f31ee332)", - "git_llvm": "https://github.com/llvm/llvm-project/commit/2b46f31ee332b7ff3d22b4fffdba12e231b3f796 (+ 24 commits)", - "git_llvm_testsuite": "https://github.com/llvm/llvm-test-suite/commit/2cedd3edd9e4e2188b33a048826a9d2d859e7c03 (unchanged)", - "git_fujitsu_testsuite": "https://github.com/fujitsu/compiler-test-suite/commit/34fbbf72b903545437f9b5fa1c1d83635d34b8da (unchanged)", + "run_order": "1304 (3-changed-components)", + "git_llvm": "https://github.com/llvm/llvm-project/commit/2b46f31ee332b7ff3d22b4fffdba12e231b3f796 (+ 599 commits)", + "git_llvm_testsuite": "https://github.com/llvm/llvm-test-suite/commit/2cedd3edd9e4e2188b33a048826a9d2d859e7c03 (+ 2 commits)", + "git_fujitsu_testsuite": "https://github.com/fujitsu/compiler-test-suite/commit/34fbbf72b903545437f9b5fa1c1d83635d34b8da (+ 2 commits)", "__report_version__": "1" }, - "Start Time": "2025-08-22 04:03:49" + "Start Time": "2025-08-20 07:34:01" }, "Tests": [ { "Data": [ - 113 + 115 ], "Info": {}, "Name": "tcwg_flang_test.TOTAL.fails.nb_fails" @@ -26,7 +26,7 @@ , { "Data": [ - 186 + 25 ], "Info": {}, "Name": "tcwg_flang_test.TOTAL.flaky.nb_flaky" diff --git a/notify/mail-body.txt b/notify/mail-body.txt index f91ce8d..22681c8 100644 --- a/notify/mail-body.txt +++ b/notify/mail-body.txt @@ -1,16 +1,33 @@ Dear contributor, -Our automatic CI successfully passed with your patch(es). Please find some details below. +Our automatic CI has detected problems related to your patch(es). Please find some details below. In tcwg_flang_test/main-aarch64-O0-debug, after: - | 24 commits in llvm + | 599 commits in llvm | 2b46f31ee332 AMDGPU: Sign extend immediates for 32-bit subregister extracts (#154870) | e0945dfa3090 [AMDGPU] Add test to show failure with SRC_*_HI registers. NFC. (#154828) | cf5243619ab2 [AMDGPU] Common up two local memory size calculations. NFCI. (#154784) | 50f7c6a5b94e Default to GLIBCXX_USE_CXX11_ABI=ON | 945a186089f7 [DAGCombiner] Remove most `UnsafeFPMath` references (#146295) - | ... and 19 more commits in llvm - + | ... and 594 more commits in llvm + +Produces 29 regressions 22 fixes: + | + | regressions.sum: + | Running test-suite:Fujitsu/Fortran/0141 ... + | FAIL: test-suite :: Fujitsu/Fortran/0141/Fujitsu-Fortran-0141_0064.test + | Running test-suite:Fujitsu/Fortran/0145 ... + | FAIL: test-suite :: Fujitsu/Fortran/0145/Fujitsu-Fortran-0145_0011.test + | Running test-suite:Fujitsu/Fortran/0155 ... + | ... and 41 more + | + | fixes.sum: + | Running test-suite:Fujitsu/C/0056 ... + | FAIL: test-suite :: Fujitsu/C/0056/Fujitsu-C-0056_0011.test + | Running test-suite:Fujitsu/C/0072 ... + | NOEXE: test-suite :: Fujitsu/C/0072/Fujitsu-C-0072_0004.test + | NOEXE: test-suite :: Fujitsu/C/0072/Fujitsu-C-0072_0008.test + | ... and 27 more Used configuration : * Toolchain : cmake -G Ninja ../llvm/llvm "-DLLVM_ENABLE_PROJECTS=clang;lld;flang;clang-tools-extra" "-DLLVM_ENABLE_RUNTIMES=openmp" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=True -DCMAKE_INSTALL_PREFIX=../llvm-install "-DLLVM_TARGETS_TO_BUILD=AArch64" -DCLANG_DEFAULT_LINKER=lld @@ -23,8 +40,8 @@ If you have any questions regarding this report, please ask on linaro-toolchain@ The information below contains the details of the failures, and the ways to reproduce a debug environment: -You can find the failure logs in *.log.1.xz files in - * https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1304/artifact/artifacts/00-sumfiles/ +You can find the failure logs in + * https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1304/artifact/artifacts/ The full lists of regressions and improvements as well as configure and make commands are in * https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1304/artifact/artifacts/notify/ The list of [ignored] baseline and flaky failures are in @@ -33,5 +50,5 @@ The list of [ignored] baseline and flaky failures are in Fujitsu testsuite : https://github.com/fujitsu/compiler-test-suite/ Current build : https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1304/artifact/artifacts -Reference build : https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1301/artifact/artifacts +Reference build : https://ci.linaro.org/job/tcwg_flang_test--main-aarch64-O0-debug-build/1281/artifact/artifacts diff --git a/notify/mail-recipients.txt b/notify/mail-recipients.txt index a59549b..7920443 100644 --- a/notify/mail-recipients.txt +++ b/notify/mail-recipients.txt @@ -1 +1 @@ -cc:jmolenda@apple.com,cc:jay.foad@amd.com,cc:craig.topper@sifive.com,ohno.yasuyuki@fujitsu.com,bcc:tcwg-validation@linaro.org,itou.tetsuya@fujitsu.com,cc:137622562+ampandey-1995@users.noreply.github.com,cc:Stanislav.Mekhanoshin@amd.com,cc:frederik.harwath@amd.com,cc:djordje.todorovic@htecgroup.com,cc:lhames@gmail.com,cc:durgadossr@nvidia.com,cc:maxim.kuvyrkov@linaro.org,sguelton@mozilla.com,t-kawashima@fujitsu.com,cc:brad@comstyle.com,i@maskray.me,cc:ismail@bennani.ma,cc:rbajpai@nvidia.com,cc:mh@glandium.org,cc:liujunchang97@outlook.com,cc:Matthew.Arsenault@amd.com +cc:jmolenda@apple.com,cc:wrotki@msn.com,cc:shafik.yaghmour@intel.com,cc:ramkumar.ramachandra@codasip.com,jay.foad@amd.com,cc:yamamoto@midokura.com,cc:gfunni234@gmail.com,cc:james.newling@gmail.com,cc:136391709+charithaintc@users.noreply.github.com,cc:kpandey81930@gmail.com,cc:wenju.he@intel.com,cc:kerry.mclaughlin@arm.com,cc:86881367+codefaber@users.noreply.github.com,stephen.tozer@sony.com,craig.topper@sifive.com,cc:nikolasklauser@berlin.de,cc:sbc@chromium.org,cc:kareem.ergawy@amd.com,cc:JanPatrick.Lehr@amd.com,cc:git@mythreya.dev,cc:aditimedhane73@gmail.com,cc:mikhail@igalia.com,npopov@redhat.com,cc:petergrind3@gmail.com,huberjn@outlook.com,barannikov88@gmail.com,yedeng.yd@linux.alibaba.com,cc:michael.klemm@amd.com,cc:devnexen@gmail.com,ohno.yasuyuki@fujitsu.com,cc:jonas@devlieghere.com,cc:scott.linder@amd.com,cc:xuqian@spacemit.com,cc:62861466+nuudlman@users.noreply.github.com,cc:macurtis@amd.com,cc:lyk_03@hotmail.com,cc:matze@braunis.de,cc:maks@fb.com,ekeane@nvidia.com,cc:gchatelet@google.com,kazu@google.com,david.majnemer@gmail.com,llvm-dev@redking.me.uk,srastogi22@apple.com,cc:janek.vanoirschot@amd.com,cc:leandrolcampos@yahoo.com.br,cc:barsolo@meta.com,cc:stevenperron@google.com,ibiryukov@google.com,cc:michaelbuch12@gmail.com,cc:zyn7109@gmail.com,bcc:tcwg-validation@linaro.org,itou.tetsuya@fujitsu.com,cc:101849050+tinnamchoi@users.noreply.github.com,cc:49036880+hankluo6@users.noreply.github.com,cc:dthorn@google.com,cc:yashasvi.khatavkar@intel.com,aidengrossman@google.com,cc:ldionne.2@gmail.com,cc:baiyang0132@gmail.com,cc:szakharin@nvidia.com,cc:sindhu.chittireddy@intel.com,cc:shihpo.hung@sifive.com,cc:20868632+Tomahawkd@users.noreply.github.com,cc:stl@nuwen.net,cc:dkersnar@nvidia.com,cc:asudhaku@amd.com,cc:quic_svs@quicinc.com,cc:ckoparkar@gmail.com,cc:sivan.shani@arm.com,cc:andrealebarbaritos@gmail.com,cc:137622562+ampandey-1995@users.noreply.github.com,cc:simeon@schaub.rocks,cc:guochen2@amd.com,cc:min.hsu@sifive.com,michael.hliao@gmail.com,cc:yifei.xu@utexas.edu,cc:wangpengcheng.pp@bytedance.com,yikong@google.com,flo@fhahn.com,bjorn.a.pettersson@ericsson.com,i@tianshilei.me,cc:daltenty@ibm.com,cc:spop@nvidia.com,cc:32487576+vinay-deshmukh@users.noreply.github.com,cc:lforg37@users.noreply.github.com,cc:Stanislav.Mekhanoshin@amd.com,cc:yronglin777@gmail.com,aeubanks@google.com,cc:mmha@users.noreply.github.com,cc:amaclean@nvidia.com,cc:danpao@microsoft.com,cc:dan@teamsalvato.com,cc:hnrklssn@gmail.com,cc:hekotas@microsoft.com,cc:paul.walker@arm.com,cc:me@m-sp.org,cc:bazuzi@google.com,cc:truber@mozilla.com,cc:frederik.harwath@amd.com,cc:tim@gymni.ch,cc:thurston@google.com,cc:mail@inbelic.dev,david.green@arm.com,cc:u_hameed@apple.com,cc:djordje.todorovic@htecgroup.com,cc:dan@su-root.co.uk,cc:kasuga.ryotaro@fujitsu.com,cc:akash.banerjee@amd.com,cc:andersonbhat491@gmail.com,cc:mary.kassayova@arm.com,cc:c_zablit@apple.com,jim@andestech.com,cc:pierre.vanhoutryve@amd.com,peter@pcc.me.uk,lhames@gmail.com,cc:preames@rivosinc.com,cc:stevenwu@apple.com,cc:Krzysztof.Parzyszek@amd.com,cc:tomer.shafir8@gmail.com,cc:rupprecht@google.com,cc:durgadossr@nvidia.com,cc:rengolin@systemcall.eu,cc:maxim.kuvyrkov@linaro.org,cc:arghnews@hotmail.co.uk,cc:platonov.aleksandr@huawei.com,cc:chao.chen@intel.com,cc:akaylor@nvidia.com,cc:orlando.hyams@sony.com,cc:andre.kuhlenschmidt@gmail.com,cc:jiangzhaoxuan94@gmail.com,cc:jesse@jvfam.ca,cc:joker.eph@gmail.com,cc:zwuis@outlook.com,cc:yu.li@arm.com,cc:naveen.hanig@outlook.com,cc:0.0@owo.li,cc:ayzhao@google.com,cc:davelee.com@gmail.com,cc:bar.victor.2002@gmail.com,cc:luke@igalia.com,cc:53339291+slavek-kucera@users.noreply.github.com,cc:jean-didier.pailleux@sipearl.com,cc:leandro.lupori@linaro.org,cc:tbaeder@redhat.com,cc:ross@codeplay.com,cc:kkleine@redhat.com,cc:mizvekov@gmail.com,cc:harshilsolanki1912@gmail.com,sguelton@mozilla.com,cc:elyronnen@gmail.com,cc:64162836+DanilaZhebryakov@users.noreply.github.com,cc:fraser@codeplay.com,t-kawashima@fujitsu.com,cc:nuno.lopes@tecnico.ulisboa.pt,cc:benjamin.maxwell@arm.com,cc:William.Huynh@arm.com,cc:harjohn@google.com,cc:ianwood2024@u.northwestern.edu,cc:guray.ozen@gmail.com,cc:i@7li.moe,cc:resistor@mac.com,cc:brad@comstyle.com,cc:pklausler@nvidia.com,cc:douglas.yung@sony.com,cc:hehiralbi@gmail.com,cc:rolf.morel@intel.com,cc:rafal.bielski@codeplay.com,cc:usx@google.com,cc:mel.chen@sifive.com,cc:caiqihan021@hotmail.com,i@maskray.me,cc:aeternalmail@gmail.com,cc:dan.palermo@amd.com,cc:kmcafee@nvidia.com,cc:dan_blackwell@apple.com,cc:tangyan01@loongson.cn,aganea@havenstudios.com,cc:abhishek.kaushik@intel.com,cc:david.sherwood@arm.com,cc:amr96@programmer.net,cc:Mirko.Brkusanin@amd.com,cc:oleksandr.tarasiuk@outlook.com,cc:ymand@users.noreply.github.com,jiefu@tencent.com,cc:clementval@gmail.com,cc:elver@google.com,cc:c8ef@outlook.com,cc:MichaelGerald.Halkenhauser@amd.com,cc:ismail@bennani.ma,arsenm2@gmail.com,cc:rbajpai@nvidia.com,cc:jacquesguan@me.com,cc:dtellenbach@apple.com,cc:yongzhu@fb.com,cc:mh@glandium.org,cc:6019989+PiJoules@users.noreply.github.com,cc:rjoshi@nvidia.com,cc:peter.smith@arm.com,cc:saugustine@google.com,cc:939808194@qq.com,cc:liujunchang97@outlook.com,cc:rkauffmann@nvidia.com,cc:msrlancern@gmail.com,cc:elvis.wang@sifive.com,cc:zhaoqi01@loongson.cn,cc:dominik.adamski@amd.com,cc:michaelrj@google.com,cc:enh@google.com,cc:mgorny@gentoo.org,cc:md.asghar.ahmad.shahid@intel.com,cc:aaron@aaronballman.com,cc:yln@users.noreply.github.com,cc:58375877+Temperz87@users.noreply.github.com,cc:linuxrocks123@netscape.net,cc:57025411+harrisonGPU@users.noreply.github.com,cc:aqamoss3fan2010@gmail.com,cc:tom.honermann@intel.com,cc:carlosgalvezp@gmail.com,cc:chenxunyu1993@gmail.com,cc:phoebe.wang@intel.com,cc:gangc@amd.com,cc:oliver@apple.com,cc:songwu0813@gmail.com,cc:Matthew.Arsenault@amd.com,cc:60100307+bassiounix@users.noreply.github.com,cc:alex_toresh@yahoo.fr,cc:fmayer@google.com diff --git a/notify/mail-subject.txt b/notify/mail-subject.txt index 195fee4..36a363a 100644 --- a/notify/mail-subject.txt +++ b/notify/mail-subject.txt @@ -1 +1 @@ -[Linaro-TCWG-CI] 24 commits in llvm: no regression/no fix on aarch64 +[Linaro-TCWG-CI] 599 commits in llvm: 29 regressions 22 fixes on aarch64 diff --git a/notify/regressions.sum b/notify/regressions.sum index e6fe5c7..07b6ef9 100644 --- a/notify/regressions.sum +++ b/notify/regressions.sum @@ -1,6 +1,70 @@ + === test-suite tests === + +Running test-suite:Fujitsu/Fortran/0141 ... +FAIL: test-suite :: Fujitsu/Fortran/0141/Fujitsu-Fortran-0141_0064.test + +Running test-suite:Fujitsu/Fortran/0145 ... +FAIL: test-suite :: Fujitsu/Fortran/0145/Fujitsu-Fortran-0145_0011.test + +Running test-suite:Fujitsu/Fortran/0155 ... +FAIL: test-suite :: Fujitsu/Fortran/0155/Fujitsu-Fortran-0155_0003.test + +Running test-suite:Fujitsu/Fortran/0157 ... +FAIL: test-suite :: Fujitsu/Fortran/0157/Fujitsu-Fortran-0157_0003.test +FAIL: test-suite :: Fujitsu/Fortran/0157/Fujitsu-Fortran-0157_0004.test +FAIL: test-suite :: Fujitsu/Fortran/0157/Fujitsu-Fortran-0157_0005.test + +Running test-suite:Fujitsu/Fortran/0311 ... +FAIL: test-suite :: Fujitsu/Fortran/0311/Fujitsu-Fortran-0311_0039.test + +Running test-suite:Fujitsu/Fortran/0380 ... +FAIL: test-suite :: Fujitsu/Fortran/0380/Fujitsu-Fortran-0380_0002.test +FAIL: test-suite :: Fujitsu/Fortran/0380/Fujitsu-Fortran-0380_0004.test + +Running test-suite:Fujitsu/Fortran/0396 ... +FAIL: test-suite :: Fujitsu/Fortran/0396/Fujitsu-Fortran-0396_0001.test +FAIL: test-suite :: Fujitsu/Fortran/0396/Fujitsu-Fortran-0396_0002.test +FAIL: test-suite :: Fujitsu/Fortran/0396/Fujitsu-Fortran-0396_0003.test +FAIL: test-suite :: Fujitsu/Fortran/0396/Fujitsu-Fortran-0396_0004.test +FAIL: test-suite :: Fujitsu/Fortran/0396/Fujitsu-Fortran-0396_0005.test + +Running test-suite:Fujitsu/Fortran/0415 ... +FAIL: test-suite :: Fujitsu/Fortran/0415/Fujitsu-Fortran-0415_0019.test + +Running test-suite:Fujitsu/Fortran/0449 ... +FAIL: test-suite :: Fujitsu/Fortran/0449/Fujitsu-Fortran-0449_0033.test + +Running test-suite:Fujitsu/Fortran/0461 ... +FAIL: test-suite :: Fujitsu/Fortran/0461/Fujitsu-Fortran-0461_0024.test + +Running test-suite:Fujitsu/Fortran/0468 ... +FAIL: test-suite :: Fujitsu/Fortran/0468/Fujitsu-Fortran-0468_0007.test + +Running test-suite:Fujitsu/Fortran/0489 ... +FAIL: test-suite :: Fujitsu/Fortran/0489/Fujitsu-Fortran-0489_0031.test + +Running test-suite:Fujitsu/Fortran/0539 ... +FAIL: test-suite :: Fujitsu/Fortran/0539/Fujitsu-Fortran-0539_0052.test +FAIL: test-suite :: Fujitsu/Fortran/0539/Fujitsu-Fortran-0539_0053.test + +Running test-suite:Fujitsu/Fortran/0583 ... +FAIL: test-suite :: Fujitsu/Fortran/0583/Fujitsu-Fortran-0583_0037.test + +Running test-suite:Fujitsu/Fortran/0598 ... +FAIL: test-suite :: Fujitsu/Fortran/0598/Fujitsu-Fortran-0598_0001.test + +Running test-suite:Fujitsu/Fortran/0660 ... +NOEXE: test-suite :: Fujitsu/Fortran/0660/Fujitsu-Fortran-0660_0104.test + +Running test-suite:Fujitsu/Fortran/0983 ... +FAIL: test-suite :: Fujitsu/Fortran/0983/Fujitsu-Fortran-0983_0001.test +FAIL: test-suite :: Fujitsu/Fortran/0983/Fujitsu-Fortran-0983_0002.test +FAIL: test-suite :: Fujitsu/Fortran/0983/Fujitsu-Fortran-0983_0003.test +FAIL: test-suite :: Fujitsu/Fortran/0983/Fujitsu-Fortran-0983_0004.test +FAIL: test-suite :: Fujitsu/Fortran/0983/Fujitsu-Fortran-0983_0008.test === Results Summary === -# of stable fails 113 -# of flaky entries 186 +# of stable fails 115 +# of flaky entries 25 # of all tests 89090 diff --git a/notify/results-summary.txt b/notify/results-summary.txt index 4f4c871..cc0d072 100644 --- a/notify/results-summary.txt +++ b/notify/results-summary.txt @@ -1,5 +1,5 @@ === Results Summary === -# of stable fails 113 -# of flaky entries 186 +# of stable fails 115 +# of flaky entries 25 # of all tests 89090