JSON ABI listTests stopped reporting suites unless they're explicitly tagged with @Suite

[allevato]

Description

When using the JSON Xcode 16.0, the listTests request responded with entries for suites whether they had the @Suite attribute or if they were just a regular struct that contained @Test methods. For example, given this code:

struct ImplicitSuite {
  @Test func test1() {}
}

@Suite struct ExplicitSuite {
  @Test func test2() {}
}

listTests would return 4 JSON objects: 2 functions and 2 suites:

{
    kind = test;
    payload =     {
        id = "Tests_16.ExplicitSuite/test2()/Tests.swift:10:4";
        isParameterized = 0;
        kind = function;
        name = "test2()";
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 4;
            fileID = "Tests_16/Tests.swift";
            line = 10;
        };
    };
    version = 0;
}
{
    kind = test;
    payload =     {
        id = "Tests_16.ImplicitSuite/test1()/Tests.swift:6:4";
        isParameterized = 0;
        kind = function;
        name = "test1()";
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 4;
            fileID = "Tests_16/Tests.swift";
            line = 6;
        };
    };
    version = 0;
}
{
    kind = test;
    payload =     {
        id = "Tests_16.ImplicitSuite";
        kind = suite;
        name = ImplicitSuite;
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 4;
            fileID = "Tests_16/Tests.swift";
            line = 6;
        };
    };
    version = 0;
}
{
    kind = test;
    payload =     {
        id = "Tests_16.ExplicitSuite";
        kind = suite;
        name = ExplicitSuite;
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 2;
            fileID = "Tests_16/Tests.swift";
            line = 9;
        };
    };
    version = 0;
}

In Xcode 16.3 and 16.4 (these are the ones I checked), structs that didn't have the @Suite attribute stopped being reported:

{
    kind = test;
    payload =     {
        id = "Tests_16.ImplicitSuite/test1()/Tests.swift:6:4";
        isParameterized = 0;
        kind = function;
        name = "test1()";
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 4;
            fileID = "Tests_16/Tests.swift";
            line = 6;
        };
    };
    version = 0;
}
{
    kind = test;
    payload =     {
        id = "Tests_16.ExplicitSuite/test2()/Tests.swift:10:4";
        isParameterized = 0;
        kind = function;
        name = "test2()";
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 4;
            fileID = "Tests_16/Tests.swift";
            line = 10;
        };
    };
    version = 0;
}
{
    kind = test;
    payload =     {
        id = "Tests_16.ExplicitSuite";
        kind = suite;
        name = ExplicitSuite;
        sourceLocation =         {
            "_filePath" = "Tests.swift";
            column = 2;
            fileID = "Tests_16/Tests.swift";
            line = 9;
        };
    };
    version = 0;
}

If I run the tests (not shown here), I still get testStarted and testEnded events for the suites. It's only the listTests behavior that changed.

Was this an intentional change or is this a bug?

Reproduction

build.sh

#!/usr/bin/env bash

sudo xcode-select -s /Applications/Xcode_16.0.0.app
xcrun swiftc -emit-executable -parse-as-library \
    -F $(xcrun -sdk macosx --show-sdk-path)/../../Library/Frameworks \
    -Xlinker -rpath -Xlinker $(xcrun -sdk macosx --show-sdk-path)/../../Library/Frameworks \
    -o Tests_16.0 Tests.swift
./Tests_16.0

sudo xcode-select -s /Applications/Xcode_16.4.0b1.app
xcrun swiftc -emit-executable -parse-as-library \
    -F $(xcrun -sdk macosx --show-sdk-path)/../../Library/Frameworks \
    -Xlinker -rpath -Xlinker $(xcrun -sdk macosx --show-sdk-path)/../../Library/Frameworks \
    -o Tests_16.4 Tests.swift
./Tests_16.4

Tests.swift

import Darwin
import Foundation
import Testing

struct ImplicitSuite {
  @Test func test1() {}
}

@Suite struct ExplicitSuite {
  @Test func test2() {}
}

private typealias ABIv0EntryPoint = @convention(thin) @Sendable (
  _ configurationJSON: UnsafeRawBufferPointer?,
  _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void
) async throws -> Bool

@main
struct Main {
  static func main() async throws {
    let entryPointRaw = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "swt_abiv0_getEntryPoint")!
    let abiv0_getEntryPoint = unsafeBitCast(
      entryPointRaw, to: (@convention(c) () -> UnsafeRawPointer).self)
    let entryPoint = unsafeBitCast(abiv0_getEntryPoint(), to: ABIv0EntryPoint.self)

    let request: [String: AnyHashable] = ["listTests": true]
    let requestBytes = try JSONSerialization.data(withJSONObject: request).withUnsafeBytes { bytes in
      let result = UnsafeMutableRawBufferPointer.allocate(byteCount: bytes.count, alignment: 1)
      result.copyMemory(from: bytes)
      return result
    }
    defer { requestBytes.deallocate() }

    try await entryPoint(UnsafeRawBufferPointer(requestBytes)) { responseBytes in
      let response = try! JSONSerialization.jsonObject(with: Data(bytes: responseBytes.baseAddress!, count: responseBytes.count))
      print(response)
    }
  }
}

Expected behavior

I would expect the Xcode 16.0 behavior, that suites are supported whether or not they're explicitly tagged with @Suite, since they still show up as testStarted/testEnded events.

Environment

macOS 15.4.1
Tested with Xcode 16.0, 16.3, and 16.4 beta 1

Additional information

No response

[grynspan]

@stmontgomery I vaguely recall this might have been intentional... do you know when/why this changed?