Merge pull request #32 from BintLopez/add_option_to_display_file_path_vs_classname

Add optional input to use file path vs classname in annotation summary

Author: toolmantim

README.md

@@ -27,12 +27,22 @@ The artifact glob path to find the JUnit XML files.
 Example: `tmp/junit-*.xml`
 
 ### `job-uuid-file-pattern` (optional)
+Default: `-(.*).xml`
 
 The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation.
 
 To use this, configure your test reporter to embed the `$BUILDKITE_JOB_ID` environment variable into your junit file names. For example `"junit-buildkite-job-$BUILDKITE_JOB_ID.xml"`.
 
-Default: `-(.*).xml`
+### `failure-format` (optional)
+Default: `classname`
+
+This setting controls the format of your failed test in the main annotation summary.
+
+There are two options for this:
+* `classname`
+  * displays: `MyClass::UnderTest text of the failed expectation in path.to.my_class.under_test`
+* `file`
+  * displays: `MyClass::UnderTest text of the failed expectation in path/to/my_class/under_test.file_ext`
 
 ## Developing
 

plugin.yml

@@ -9,6 +9,11 @@ configuration:
       type: string
     job-uuid-file-pattern:
       type: string
+    failure-format:
+      type: string
+      enum:
+        - classname
+        - file
   required:
     - artifacts
-  additionalProperties: false
+  additionalProperties: false

ruby/bin/annotate

@@ -12,7 +12,15 @@ abort("#{junits_dir} does not exist") unless Dir.exist?(junits_dir)
 job_pattern = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN']
 job_pattern = '-(.*).xml' if !job_pattern || job_pattern.empty?
 
-class Failure < Struct.new(:name, :classname, :body, :job, :type)
+failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT']
+failure_format = 'classname' if !failure_format || failure_format.empty?
+
+# TODO -- remove this validation once https://github.com/buildkite/agent/pull/748 is merged
+if !%w(classname file).include?(failure_format)
+  abort("Invalid failure-format #{failure_format} provided -- leave this setting blank or specify `file` or `classname`")
+end
+
+class Failure < Struct.new(:name, :failed_test, :body, :job, :type)
 end
 
 junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"))
@@ -28,12 +36,12 @@ junit_report_files.sort.each do |file|
 
   REXML::XPath.each(doc, '//testsuite//testcase') do |testcase|
     name = testcase.attributes['name'].to_s
-    classname = testcase.attributes['classname'].to_s
+    failed_test = testcase.attributes[failure_format].to_s
     testcase.elements.each("failure") do |failure|
-      failures << Failure.new(name, classname, failure.text, job, :failure)
+      failures << Failure.new(name, failed_test, failure.text, job, :failure)
     end
     testcase.elements.each("error") do |error|
-      failures << Failure.new(name, classname, error.text, job, :error)
+      failures << Failure.new(name, failed_test, error.text, job, :error)
     end
   end
 end
@@ -59,7 +67,7 @@ puts [
 
 failures.each do |failure|
   puts "<details>"
-  puts "<summary><code>#{failure.name} in #{failure.classname}</code></summary>\n\n"
+  puts "<summary><code>#{failure.name} in #{failure.failed_test}</code></summary>\n\n"
   if failure.body
     puts "<code><pre>#{failure.body.chomp.strip}</pre></code>\n\n"
   end

ruby/tests/annotate_test.rb

@@ -215,6 +215,95 @@
     assert_equal 0, status.exitstatus
   end
 
+  it "uses the file path instead of classname for annotation content when specified" do
+    output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=file", "#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
+
+    assert_equal <<~OUTPUT, output
+      Parsing junit-1.xml
+      Parsing junit-2.xml
+      Parsing junit-3.xml
+      --- ❓ Checking failures
+      There are 4 failures/errors 😭
+      --- ✍️ Preparing annotation
+      2 failures and 2 errors:
+
+      <details>
+      <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb</code></summary>
+
+      <code><pre>Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+
+        expected: 250
+             got: 500
+
+        (compared using eql?)
+      ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
+      ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
+      ./spec/support/log.rb:17:in `run'
+      ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'</pre></code>
+
+      in <a href="#1">Job #1</a>
+      </details>
+
+      <details>
+      <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in ./spec/models/account_spec.rb</code></summary>
+
+      <code><pre>Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+
+        expected: 700
+             got: 500
+
+        (compared using eql?)
+      ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
+      ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
+      ./spec/support/log.rb:17:in `run'
+      ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'</pre></code>
+
+      in <a href="#2">Job #2</a>
+      </details>
+
+      <details>
+      <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in ./spec/models/account_spec.rb</code></summary>
+
+      <code><pre>Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+
+        expected: 700
+             got: 500
+
+        (compared using eql?)
+      ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
+      ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
+      ./spec/support/log.rb:17:in `run'
+      ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'</pre></code>
+
+      in <a href="#3">Job #3</a>
+      </details>
+
+      <details>
+      <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in ./spec/models/account_spec.rb</code></summary>
+
+      <code><pre>Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+
+        expected: 250
+             got: 500
+
+        (compared using eql?)
+      ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
+      ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
+      ./spec/support/log.rb:17:in `run'
+      ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'</pre></code>
+
+      in <a href="#3">Job #3</a>
+      </details>
+    OUTPUT
+
+    assert_equal 0, status.exitstatus
+  end
+
+  it "raises an error when an invalid failure-format is provided" do
+    output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=kittens", "#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
+    assert_equal false, status.success?
+  end
+
   it "handles failures across multiple files in sub dirs" do
     output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/tests-in-sub-dirs/")