From 69f92838255656c91930a010f2feb0a9cd01b39f Mon Sep 17 00:00:00 2001
From: Dhruv <dhruvmanila@gmail.com>
Date: Mon, 12 Oct 2020 11:29:39 +0530
Subject: [PATCH] Start running validate_solutions script for Travis CI (#3215)

* Removed print error_msgs at the end of test:

This was done only to reduce the message clutter produced by 60
failing tests. As that is fixed, we can produce the traceback in
short form and allow pytest to print the captured error message
output at the end of test.

* Start validate_solutions script for Travis CI

I am separating out the solution testing and doctest as validating
the solutions for the current number of solutions present is
taking 2 minutes to run.
---
 .travis.yml                         |  9 ++++++---
 project_euler/validate_solutions.py | 19 +++----------------
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index bda0fc31c..ff59af2db 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,11 +12,14 @@ jobs:
         - pytest --doctest-modules --ignore=project_euler/ --durations=10 --cov-report=term-missing:skip-covered --cov=. .
     - name: Project Euler
       install:
-        - pip install pytest-cov pytest-subtests
-      before_script:
-        - pytest --tb=no --no-summary --capture=no project_euler/validate_solutions.py || true  # fail fast on wrong solution
+        - pip install pytest-cov
       script:
         - pytest --doctest-modules --durations=10 --cov-report=term-missing:skip-covered --cov=project_euler/ project_euler/
+    - name: Project Euler Solution
+      install:
+        - pip install pytest-subtests
+      script:
+        - pytest --tb=short project_euler/validate_solutions.py
 after_success:
   - scripts/build_directory_md.py 2>&1 | tee DIRECTORY.md
 notifications:
diff --git a/project_euler/validate_solutions.py b/project_euler/validate_solutions.py
index 01d70721e..f3ae9fbef 100755
--- a/project_euler/validate_solutions.py
+++ b/project_euler/validate_solutions.py
@@ -15,8 +15,6 @@ PROJECT_EULER_ANSWERS_PATH = PROJECT_EULER_DIR_PATH.joinpath(
 with open(PROJECT_EULER_ANSWERS_PATH) as file_handle:
     PROBLEM_ANSWERS = json.load(file_handle)
 
-error_msgs = []
-
 
 def generate_solution_modules(
     dir_path: pathlib.Path,
@@ -48,20 +46,9 @@ def test_project_euler(subtests, problem_number: int, expected: str):
                     answer = str(solution_module.solution())
                     assert answer == expected, f"Expected {expected} but got {answer}"
                 except (AssertionError, AttributeError, TypeError) as err:
-                    error_msgs.append(
-                        f"problem_{problem_number}/{solution_module.__name__}: {err}"
+                    print(
+                        f"problem_{problem_number:02}/{solution_module.__name__}: {err}"
                     )
-                    raise  # We still want pytest to know that this test failed
+                    raise
     else:
         pytest.skip(f"Solution {problem_number} does not exist yet.")
-
-
-# Run this function at the end of all the tests
-# https://stackoverflow.com/a/52873379
-@pytest.fixture(scope="session", autouse=True)
-def custom_print_message(request):
-    def print_error_messages():
-        if error_msgs:
-            print("\n" + "\n".join(error_msgs))
-
-    request.addfinalizer(print_error_messages)