18: Upgrade runner to meet version 2 spec (exercism#104)

* Run tests with custom hspec formatter which outputs result.json v2

* Update all expected results.json files in tests

* Added extra pre-compiled packages needed by injected code

* Improved comment

Co-authored-by: Erik Schierboom <[email protected]>

* Fixed success results.json + improved run.sh

- If all tests pass, hspec formatter now correctly outputs top-level
  success status
- Implemented bash LSP hints in bin/run.sh
- Added newline at end of code injection to package.yaml files

* Fixed expected results in example-empty-file and example-syntax-error

* Precompile bin/setup-tests executable instead of using stack runghc

* Fixed bin/run.sh to run setup-tests executable correctly

* Automatically cleanup code injections when running tests

* Copy results.json to desired output path

* Improved cleanup process

---------

Co-authored-by: Erik Schierboom <[email protected]>

Author: cdimitroulas

.gitignore

@@ -1,4 +1,6 @@
-tests/**/results.json
-tests/**/*.cabal
-tests/**/.stack-work
-tests/**/stack.yaml.lock
+results.json
+*.cabal
+.stack-work
+stack.yaml.lock
+bin/setup-tests
+tests/**/HspecFormatter.hs

Dockerfile

@@ -14,5 +14,8 @@ COPY pre-compiled/ .
 RUN stack build --resolver lts-20.18 --no-terminal --test --no-run-tests
 
 COPY . .
+
+RUN cd ./test-setup/ && stack build setup-tests --copy-bins --local-bin-path /opt/test-runner/bin/ && cd ..
+
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]
 

bin/build-setup-tests.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# This script is just a quick way to build the setup-tests binary for local development, similar to what is done
+# in the Dockerfile.
+# It outputs the resulting executable in bin/setup-tests
+
+pushd ./test-setup/ && stack build setup-tests --copy-bins --local-bin-path ../bin/ && popd 

bin/run-tests.sh

@@ -21,7 +21,7 @@ for test_dir in tests/*; do
     test_dir_path=$(realpath "${test_dir}")
     results_file_path="${test_dir_path}/results.json"
     expected_results_file_path="${test_dir_path}/expected_results.json"
-    stack_root=$(stack path --stack-root)
+    stack_root=$(stack --resolver lts-20.18 path --stack-root)
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 

bin/run.sh

@@ -27,16 +27,30 @@ slug="$1"
 input_dir="${2%/}"
 output_dir="${3%/}"
 results_file="${output_dir}/results.json"
+setup_tests_executable="bin/setup-tests"
 
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
 
-file_contents=$(< "${input_dir}/stack.yaml")
+stack_yml_file_contents=$(< "${input_dir}/stack.yaml")
+package_yml_file_contents=$(< "${input_dir}/package.yaml")
+tests_file_contents=$(< "${input_dir}/test/Tests.hs")
 
 echo "system-ghc: true" >> "${input_dir}/stack.yaml"
 
+# Run our test setup which does some code injection to modify how the tests
+# will run to use our custom hspec formatter that outputs results.json automatically.
+# We expect the setup-tests executable to be pre-built in Docker, but fallback to using runghc in case it isn't
+# found so that developers can continue to easily run `bin/run.sh` locally.
+if [ -f "${setup_tests_executable}" ]; then
+  ${setup_tests_executable} "${input_dir}"
+else
+  echo "Did not find bin/setup-tests executable - using stack runghc ./test-setup/src/Main.hs instead"
+  stack --resolver lts-20.18 runghc ./test-setup/src/Main.hs "$input_dir"
+fi
+
 pushd "${input_dir}" > /dev/null
 
 # disable -e since we expect some tests to fail
@@ -46,7 +60,12 @@ set +e
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
 test_output=$(stack build --resolver lts-20.18 --test --allow-different-user 2>&1)
-exit_code=$?
+
+# Copy results.json to the output directory (only if output directory is different from
+# the input directory)
+if [ "${input_dir}/results.json" != "${results_file}" ]; then
+  mv "${input_dir}/results.json" "${results_file}"
+fi
 
 # re-enable original options
 set -$old_opts
@@ -56,27 +75,29 @@ rm -rf .stack-work
 
 popd
 
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
-if [ $exit_code -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
+# If the results.json file does not exist, it means that the tests failed to run
+# (usually this would be a compiler error)
+if ! [ -f "${results_file}" ]; then
     # Sanitize the output
     if grep -q "Registering library for " <<< "${test_output}" ; then
-        sanitized_test_output=$(printf "${test_output}" | sed -n -E -e '1,/^Registering library for/!p')
+	sanitized_test_output=$(printf "%s" "${test_output}" | sed -n -E -e '1,/^Registering library for/!p')
     elif grep -q "Building library for " <<< "${test_output}" ; then
-        sanitized_test_output=$(printf "${test_output}" | sed -n -E -e '1,/^Building library for/!p')
+	sanitized_test_output=$(printf "%s" "${test_output}" | sed -n -E -e '1,/^Building library for/!p')
     else
-        sanitized_test_output="${test_output}"
+	sanitized_test_output="${test_output}"
     fi
 
     # Manually add colors to the output to help scanning the output for errors
     colorized_test_output=$(echo "${sanitized_test_output}" \
-         | GREP_COLOR='01;31' grep --color=always -E -e '.*FAILED \[[0-9]+\]$|$')
+	| GREP_COLOR='01;31' grep --color=always -E -e '.*FAILED \[[0-9]+\]$|$')
 
-    jq -n --arg output "${colorized_test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    jq -n --arg output "${colorized_test_output}" '{version: 2, status: "error", message: $output}' > "${results_file}"
 fi
 
-echo "$file_contents" > "${input_dir}/stack.yaml"
+# Revert input directory code to it's initial state
+echo "$stack_yml_file_contents" > "${input_dir}/stack.yaml"
+echo "$package_yml_file_contents" > "${input_dir}/package.yaml"
+echo "$tests_file_contents" > "${input_dir}/test/Tests.hs"
+rm -f "${input_dir}/test/HspecFormatter.hs"
 
 echo "${slug}: done"

pre-compiled/package.yaml

@@ -31,4 +31,9 @@ tests:
     source-dirs: test
     dependencies:
       - leap
+      - aeson
+      - aeson-pretty
+      - bytestring
       - hspec
+      - hspec-core
+      - stm

pre-compiled/test/HspecFormatter.hs

@@ -0,0 +1,91 @@
+-- NOTE: This file is used by the setup-tests executable (built from the test-setup/ directory).
+-- It's copied into the target project and is configured to be the hspec formatter using code injection.
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE OverloadedRecordDot #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+module HspecFormatter (formatter) where
+
+import Data.Aeson (ToJSON, toJSON, object, encode, (.=))
+import Data.Aeson.Encode.Pretty (encodePretty)
+import qualified Data.ByteString.Lazy as BS
+import Data.Maybe (fromMaybe)
+import Control.Monad (forM_)
+import Control.Concurrent.STM
+import GHC.IO.Unsafe (unsafePerformIO)
+import Test.Hspec
+import Test.Hspec.Core.Formatters.V2
+import Test.Hspec.Core.Format (Format, FormatConfig, Path, Event(ItemDone, Done), FailureReason(..))
+import GHC.Generics (Generic)
+
+data TestResultStatus = Pass | Fail | Err deriving (Eq, Show)
+
+instance ToJSON TestResultStatus where
+  toJSON Pass = "pass"
+  toJSON Fail = "fail"
+  toJSON Err = "error"
+
+data TestResult = TestResult {
+  name :: String,
+  status :: TestResultStatus,
+  message :: Maybe String
+} deriving (Generic, Show)
+
+instance ToJSON TestResult where
+
+data TestResults = TestResults {
+  resultsStatus :: TestResultStatus,
+  tests :: [TestResult],
+  resultsMessage :: Maybe String,
+  version :: Int
+} deriving (Generic, Show)
+
+instance ToJSON TestResults where
+  toJSON t = object [
+      "version" .= t.version
+      , "status" .= t.resultsStatus
+      , "message" .= t.resultsMessage
+      , "tests" .= t.tests
+    ]
+
+results :: TVar TestResults
+{-# NOINLINE results #-}
+results = unsafePerformIO $ newTVarIO (TestResults Fail [] Nothing 2)
+
+format :: Format
+format event = case event of
+  ItemDone path item -> handleItemDone path item
+  Done _ -> handleDone
+  _ -> return ()
+  where
+    handleItemDone :: Path -> Item -> IO ()
+    handleItemDone (_, requirement) item =
+        case itemResult item of
+          Success ->
+            addTestResult TestResult { name = requirement, status = Pass, message = Nothing }
+          -- NOTE: We don't expect pending tests in Exercism exercises
+          Pending _ _ -> return ()
+          Failure _ failureReason ->
+            let baseResult = TestResult { name = requirement, status = Fail, message = Just "" }
+                result = case failureReason of
+                  NoReason -> baseResult { message = Just "No reason" }
+                  Reason reason -> baseResult { message = Just reason }
+                  ExpectedButGot _ expected got -> 
+                    baseResult { 
+                      message = Just $ "Expected '" ++ expected ++ "' but got '" ++ got ++ "'"
+                    }
+                  Error _ exception -> baseResult { message = Just $ show exception }
+            in addTestResult result
+      where
+        addTestResult tr = atomically $ modifyTVar' results (\r -> r { tests = r.tests <> [tr] })
+
+    handleDone :: IO ()
+    handleDone = do
+      resultsVal <- readTVarIO results
+      let finalResults = if all (\t -> t.status == Pass) resultsVal.tests then resultsVal { resultsStatus = Pass } else resultsVal
+      BS.writeFile "results.json" (encodePretty finalResults)
+      return ()
+
+
+formatter :: FormatConfig -> IO Format
+formatter _config = return format

test-setup/package.yaml

@@ -0,0 +1,13 @@
+name: setup-tests
+version: 1.0.0.0
+
+dependencies:
+  - base
+
+executables:
+  setup-tests:
+    main: Main.hs
+    source-dirs: src
+    ghc-options: -Wall
+    dependencies:
+      - directory

test-setup/src/Main.hs

@@ -0,0 +1,77 @@
+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
+module Main (main) where
+import Data.List (findIndex, isInfixOf)
+import Data.IORef
+import System.Environment (getArgs)
+import Control.Arrow ((>>>))
+import Control.Monad (when)
+import System.Directory (copyFile)
+
+main :: IO ()
+main = do
+  args <- getArgs
+  case args of 
+    [] -> error "setup-tests expects one argument - the project directory whose code should be modified"
+    xs -> modifyTests (head xs)
+
+hspecFormatterPath :: String
+hspecFormatterPath = "pre-compiled/test/HspecFormatter.hs"
+
+modifyTests :: String -> IO ()
+modifyTests inputDir = do
+  let testFile = inputDir ++ "/test/Tests.hs"
+      packageFile = inputDir ++ "/package.yaml"
+
+  testCodeRef <- readFile testFile >>= newIORef . lines
+
+  readIORef testCodeRef >>=
+    (updateHspecRunnerImport >>> updateMainFunc >>> writeIORef testCodeRef)
+
+  -- Update the test/Tests.hs file with the new contents
+  -- We use `when (length newTestFileData > 0)` as a trick to strictly evaluate the
+  -- file data before trying to write to the file otherwise we get a "resouce busy (file is locked)" error
+  newTestFileData <- readIORef testCodeRef
+  {-# HLINT ignore "Use null" #-}
+  when (length newTestFileData > 0) $ writeFile testFile (unlines newTestFileData)
+
+  -- Add aeson, aeson-pretty, bytestring, hspec-core, stm and text packages to `tests` section of 
+  -- package.yaml.
+  -- (assumes that the tests.test.dependencies is the last item in package.yaml!)
+  appendFile packageFile "      - aeson\n      - aeson-pretty\n      - bytestring\n      - hspec-core\n      - stm\n      - text\n"
+
+  -- Copy our custom hspec formatter into the input code directory so it can be used
+  copyFile hspecFormatterPath (inputDir ++ "/test/HspecFormatter.hs")
+
+  where
+    -- Update Test.Hspec.Runner import to add the `configFormat` import that we need
+    -- and also add the import HspecFormatter line
+    updateHspecRunnerImport = 
+      updateLineOfCode 
+        isHspecRunnerImport 
+        "import Test.Hspec.Runner (configFailFast, defaultConfig, hspecWith, configFormat)\nimport HspecFormatter" 
+
+    -- Update the main function to add the configFormat option to hspec to use our custom
+    -- formatter that outputs results.json in the necessary format.
+    -- It also removes the configFailFast option so that we run ALL tests rather than stopping
+    -- at the first failing.
+    updateMainFunc =
+      updateLineOfCode 
+        isMainFunc 
+        "main = hspecWith defaultConfig {configFormat = Just formatter} specs"
+
+updateLineOfCode :: (String -> Bool) -> String -> [String] -> [String]
+updateLineOfCode isLineToUpdate newLine fileContents =
+  case findIndex isLineToUpdate fileContents of
+    Just idx -> replaceNth idx newLine fileContents
+    Nothing -> fileContents
+
+isHspecRunnerImport :: String -> Bool
+isHspecRunnerImport = isInfixOf "import Test.Hspec.Runner"
+
+isMainFunc :: String -> Bool
+isMainFunc = isInfixOf "main = hspecWith"
+
+replaceNth :: Int -> a -> [a] -> [a]
+replaceNth idx newVal list = 
+  let (first, second) = splitAt idx list
+  in first <> (newVal : tail second)

test-setup/stack.yaml

@@ -0,0 +1,3 @@
+system-ghc: true
+
+resolver: lts-20.18

tests/example-all-fail/expected_results.json

@@ -1,5 +1,52 @@
 {
-  "version": 1,
-  "status": "fail",
-  "message": "leap> test (suite: test)\n\n\nisLeapYear\n  2015 - year not divisible by 4 in common year [✘]\n\nFailures:\n\n  test/Tests.hs:20:55: \n  1) isLeapYear 2015 - year not divisible by 4 in common year\n       expected: False\n        but got: True\n\n  To rerun use: --match \"/isLeapYear/2015 - year not divisible by 4 in common year/\"\n\n\n1 example, 1 failure\n\nleap> Test suite test failed\n\nError: [S-7282]\n       Stack failed to execute the build plan.\n       \n       While executing the build plan, Stack encountered the following errors:\n       \n       TestSuiteFailure (PackageIdentifier {pkgName = PackageName \"leap\", pkgVersion = mkVersion [1,6,0,10]}) (fromList [(\"test\",Just (ExitFailure 1))]) Nothing \"\""
-}
+    "message": null,
+    "status": "fail",
+    "tests": [
+        {
+            "message": "Expected 'False' but got 'True'",
+            "name": "2015 - year not divisible by 4 in common year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'False' but got 'True'",
+            "name": "1970 - year divisible by 2, not divisible by 4 in common year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'True' but got 'False'",
+            "name": "1996 - year divisible by 4, not divisible by 100 in leap year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'True' but got 'False'",
+            "name": "1960 - year divisible by 4 and 5 is still a leap year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'False' but got 'True'",
+            "name": "2100 - year divisible by 100, not divisible by 400 in common year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'False' but got 'True'",
+            "name": "1900 - year divisible by 100 but not by 3 is still not a leap year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'True' but got 'False'",
+            "name": "2000 - year divisible by 400 in leap year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'True' but got 'False'",
+            "name": "2400 - year divisible by 400 but not by 125 is still a leap year",
+            "status": "fail"
+        },
+        {
+            "message": "Expected 'False' but got 'True'",
+            "name": "1800 - year divisible by 200, not divisible by 400 in common year",
+            "status": "fail"
+        }
+    ],
+    "version": 2
+}

tests/example-empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
-  "version": 1,
-  "status": "fail",
+  "version": 2,
+  "status": "error",
   "message": "\n/solution/src/LeapYear.hs:1:1: error:\n    File name does not match module name:\n    Saw: ‘Main’\n    Expected: ‘LeapYear’\n\nError: [S-7282]\n       Stack failed to execute the build plan.\n       \n       While executing the build plan, Stack encountered the following errors:\n       \n       [S-7011]\n       While building package leap-1.6.0.10 (scroll up to its section to see the error) using:\n       --verbose=1 build lib:leap test:test --ghc-options \"\"\n       Process exited with code: ExitFailure 1 "
 }