diff --git a/apple/testing/default_runner/BUILD b/apple/testing/default_runner/BUILD index 99eb35c449..d3b4324971 100644 --- a/apple/testing/default_runner/BUILD +++ b/apple/testing/default_runner/BUILD @@ -1,5 +1,6 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_python//python:py_binary.bzl", "py_binary") +load("@rules_python//python:py_library.bzl", "py_library") load( "//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner", @@ -90,6 +91,12 @@ exports_files([ "xctrunner_entitlements.template.plist", ]) +py_library( + name = "simulator_utils", + srcs = ["simulator_utils.py"], + visibility = ["//apple:__subpackages__"], +) + py_binary( name = "simulator_creator", srcs = ["simulator_creator.py"], @@ -99,6 +106,9 @@ py_binary( # should be considered an implementation detail of the rules and # not used by other things. visibility = ["//visibility:public"], + deps = [ + ":simulator_utils", + ], ) ios_test_runner( diff --git a/apple/testing/default_runner/ios_xctestrun_runner.bzl b/apple/testing/default_runner/ios_xctestrun_runner.bzl index 342115b853..fdba766d77 100644 --- a/apple/testing/default_runner/ios_xctestrun_runner.bzl +++ b/apple/testing/default_runner/ios_xctestrun_runner.bzl @@ -25,7 +25,9 @@ def _get_template_substitutions( xctrunner_entitlements_template, pre_action_binary, post_action_binary, - post_action_determines_exit_code): + post_action_determines_exit_code, + simulator_pool_server_port, + simulator_pool_client): substitutions = { "device_type": device_type, "os_version": os_version, @@ -43,6 +45,8 @@ def _get_template_substitutions( "pre_action_binary": pre_action_binary, "post_action_binary": post_action_binary, "post_action_determines_exit_code": post_action_determines_exit_code, + "simulator_pool_server_port": simulator_pool_server_port, + "simulator_pool_client.py": simulator_pool_client, } return {"%({})s".format(key): value for key, value in substitutions.items()} @@ -71,6 +75,8 @@ def _ios_xctestrun_runner_impl(ctx): ctx.file._xctrunner_entitlements_template, ]).merge(ctx.attr._simulator_creator[DefaultInfo].default_runfiles) + runfiles = runfiles.merge(ctx.attr._simulator_pool_client[DefaultInfo].default_runfiles) + default_action_binary = "/usr/bin/true" pre_action_binary = default_action_binary @@ -105,6 +111,8 @@ def _ios_xctestrun_runner_impl(ctx): pre_action_binary = pre_action_binary, post_action_binary = post_action_binary, post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false", + simulator_pool_server_port = "" if ctx.attr.simulator_pool_server_port else str(ctx.attr.simulator_pool_server_port), + simulator_pool_client = ctx.executable._simulator_pool_client.short_path, ), ) @@ -204,6 +212,14 @@ A binary to run following test execution. Runs after testing but before test res When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria. """, ), + "simulator_pool_server_port": attr.int( + doc = "The port of a running simulator pool server. If set, the test runner will connect to the simulator pool server and use the simulators from the pool instead of creating new ones.", + ), + "_simulator_pool_client": attr.label( + default = "//apple/testing/simulator_pool:simulator_pool_client", + executable = True, + cfg = "exec", + ), "_simulator_creator": attr.label( default = Label( "//apple/testing/default_runner:simulator_creator", diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index e44a94033d..31127ab249 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -62,10 +62,30 @@ basename_without_extension() { echo "${filename%.*}" } +simulator_pool_client_path="%(simulator_pool_client.py)s" +simulator_pool_server_port="%(simulator_pool_server_port)s" +simulator_pool_enabled=false +if [[ -n "$simulator_pool_server_port" ]] && [[ -n "$device_id" ]] && [[ -n "$simulator_pool_client_path" ]]; then + simulator_pool_enabled=true +fi +simulator_id="" + +return_simulator_to_pool() { + if [[ "$simulator_pool_enabled" == true ]]; then + "$simulator_pool_client_path" return --udid="$simulator_id" --port "$simulator_pool_server_port" + fi +} + +teardown() { + return_simulator_to_pool + rm -rf "${test_tmp_dir}" +} + test_tmp_dir="$(mktemp -d "${TEST_TMPDIR:-${TMPDIR:-/tmp}}/test_tmp_dir.XXXXXX")" if [[ -z "${NO_CLEAN:-}" ]]; then - trap 'rm -rf "${test_tmp_dir}"' EXIT + trap 'teardown' EXIT else + return_simulator_to_pool test_tmp_dir="${TMPDIR:-/tmp}/test_tmp_dir" rm -rf "$test_tmp_dir" mkdir -p "$test_tmp_dir" @@ -404,8 +424,18 @@ else simulator_creator_args+=(--no-reuse-simulator) fi -simulator_id="unused" -if [[ "$build_for_device" == false ]]; then +if [[ "$simulator_pool_enabled" == true ]]; then + request_simulator_args=( + --port "$simulator_pool_server_port" \ + --test-target "$test_bundle_name" + --device-type "%(device_type)s" + --os-version "%(os_version)s" + ) + if [[ -n "${test_host_path:-}" ]]; then + request_simulator_args+=(--test-host "$(basename_without_extension "$test_host_path")") + fi + simulator_id=$("$simulator_pool_client_path" request "${request_simulator_args[@]}") +else simulator_id="$("./%(simulator_creator.py)s" \ "${simulator_creator_args[@]}" )" diff --git a/apple/testing/default_runner/simulator_creator.py b/apple/testing/default_runner/simulator_creator.py index aff31a971b..89a90243ed 100755 --- a/apple/testing/default_runner/simulator_creator.py +++ b/apple/testing/default_runner/simulator_creator.py @@ -20,67 +20,9 @@ import subprocess import sys import time +import apple.testing.default_runner.simulator_utils as simulator_utils from typing import List, Optional - -def _simctl(extra_args: List[str]) -> str: - return subprocess.check_output(["xcrun", "simctl"] + extra_args).decode() - - -def _boot_simulator(simulator_id: str) -> None: - # This private command boots the simulator if it isn't already, and waits - # for the appropriate amount of time until we can actually run tests - try: - output = _simctl(["bootstatus", simulator_id, "-b"]) - print(output, file=sys.stderr) - except subprocess.CalledProcessError as e: - exit_code = e.returncode - - # When reusing simulators we may encounter the error: - # 'Unable to boot device in current state: Booted'. - # - # This is because the simulator is already booted, and we can ignore it - # if we check and the simulator is in fact booted. - if exit_code == 149: - devices = json.loads( - _simctl(["list", "devices", "-j", simulator_id]), - )["devices"] - device = next( - ( - blob - for devices_for_os in devices.values() - for blob in devices_for_os - if blob["udid"] == simulator_id - ), - None - ) - if device and device["state"].lower() == "booted": - print( - f"Simulator '{device['name']}' ({simulator_id}) is already booted", - file=sys.stderr, - ) - exit_code = 0 - - # Both of these errors translate to strange simulator states that may - # end up causing issues, but attempting to actually use the simulator - # instead of failing at this point might still succeed - # - # 164: EBADDEVICE - # 165: EBADDEVICESTATE - if exit_code in (164, 165): - print( - f"Ignoring 'simctl bootstatus' exit code {exit_code}", - file=sys.stderr, - ) - elif exit_code != 0: - print(f"'simctl bootstatus' exit code {exit_code}", file=sys.stderr) - raise - - # Add more arbitrary delay before tests run. Even bootstatus doesn't wait - # long enough and tests can still fail because the simulator isn't ready - time.sleep(3) - - def _device_name(device_type: str, os_version: str) -> str: return f"BAZEL_TEST_{device_type}_{os_version}" @@ -109,7 +51,7 @@ def _build_parser() -> argparse.ArgumentParser: def _main(os_version: str, device_type: str, name: Optional[str], reuse_simulator: bool) -> None: - devices = json.loads(_simctl(["list", "devices", "-j"]))["devices"] + devices = json.loads(simulator_utils.simctl(["list", "devices", "-j"]))["devices"] device_name = name or _device_name(device_type, os_version) runtime_identifier = "com.apple.CoreSimulator.SimRuntime.iOS-{}".format( os_version.replace(".", "-") @@ -132,7 +74,7 @@ def _main(os_version: str, device_type: str, name: Optional[str], reuse_simulato state = existing_device["state"].lower() print(f"Existing simulator '{name}' ({simulator_id}) state is: {state}", file=sys.stderr) if state != "booted": - _boot_simulator(simulator_id) + simulator_utils.boot_simulator(simulator_id) else: if not reuse_simulator: # Simulator reuse is based on device name, therefore we must generate a unique name to @@ -140,11 +82,11 @@ def _main(os_version: str, device_type: str, name: Optional[str], reuse_simulato device_name_suffix = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) device_name += f"_{device_name_suffix}" - simulator_id = _simctl( + simulator_id = simulator_utils.simctl( ["create", device_name, device_type, runtime_identifier] ).strip() print(f"Created new simulator '{device_name}' ({simulator_id})", file=sys.stderr) - _boot_simulator(simulator_id) + simulator_utils.boot_simulator(simulator_id) print(simulator_id.strip()) diff --git a/apple/testing/default_runner/simulator_utils.py b/apple/testing/default_runner/simulator_utils.py new file mode 100644 index 0000000000..5836e60d33 --- /dev/null +++ b/apple/testing/default_runner/simulator_utils.py @@ -0,0 +1,88 @@ +#!/usr/bin/python3 +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import sys +import time +import subprocess +from typing import List + + +def simctl(extra_args: List[str]) -> str: + """Execute simctl command with the given arguments. + + Args: + extra_args: List of additional arguments to pass to simctl + + Returns: + The decoded output from the simctl command + + Raises: + subprocess.CalledProcessError: If the simctl command fails + """ + return subprocess.check_output(["xcrun", "simctl"] + extra_args).decode() + +def boot_simulator(simulator_id: str) -> None: + # This private command boots the simulator if it isn't already, and waits + # for the appropriate amount of time until we can actually run tests + try: + output = simctl(["bootstatus", simulator_id, "-b"]) + print(output, file=sys.stderr) + except subprocess.CalledProcessError as e: + exit_code = e.returncode + + # When reusing simulators we may encounter the error: + # 'Unable to boot device in current state: Booted'. + # + # This is because the simulator is already booted, and we can ignore it + # if we check and the simulator is in fact booted. + if exit_code == 149: + devices = json.loads( + simctl(["list", "devices", "-j", simulator_id]), + )["devices"] + device = next( + ( + blob + for devices_for_os in devices.values() + for blob in devices_for_os + if blob["udid"] == simulator_id + ), + None + ) + if device and device["state"].lower() == "booted": + print( + f"Simulator '{device['name']}' ({simulator_id}) is already booted", + file=sys.stderr, + ) + exit_code = 0 + + # Both of these errors translate to strange simulator states that may + # end up causing issues, but attempting to actually use the simulator + # instead of failing at this point might still succeed + # + # 164: EBADDEVICE + # 165: EBADDEVICESTATE + if exit_code in (164, 165): + print( + f"Ignoring 'simctl bootstatus' exit code {exit_code}", + file=sys.stderr, + ) + elif exit_code != 0: + print(f"'simctl bootstatus' exit code {exit_code}", file=sys.stderr) + raise + + # Add more arbitrary delay before tests run. Even bootstatus doesn't wait + # long enough and tests can still fail because the simulator isn't ready + time.sleep(3) diff --git a/apple/testing/simulator_pool/BUILD b/apple/testing/simulator_pool/BUILD new file mode 100644 index 0000000000..12c2df3b3f --- /dev/null +++ b/apple/testing/simulator_pool/BUILD @@ -0,0 +1,46 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") +load("//apple/testing/simulator_pool:create_simulator_pool.bzl", "create_simulator_pool") + +exports_files(["create_simulator_pool.template.sh"]) + +create_simulator_pool( + name = "create_simulator_pool", + device_type = "iPhone 15 Pro", + os_version = "18.3", + pool_size = 3, + server_port = 50051, +) + +py_binary( + name = "create_simulator_pool_tool", + srcs = [ + "create_simulator_pool_tool.py", + ], + python_version = "PY3", + srcs_version = "PY3", + deps = [ + "//apple/testing/default_runner:simulator_utils", + ], +) + +py_binary( + name = "simulator_pool_server", + srcs = [ + "simulator_pool_server.py", + ], + python_version = "PY3", + srcs_version = "PY3", + deps = [ + "//apple/testing/default_runner:simulator_utils", + ], +) + +py_binary( + name = "simulator_pool_client", + srcs = [ + "simulator_pool_client.py", + ], + python_version = "PY3", + srcs_version = "PY3", + visibility = ["//visibility:public"], +) diff --git a/apple/testing/simulator_pool/create_simulator_pool.bzl b/apple/testing/simulator_pool/create_simulator_pool.bzl new file mode 100644 index 0000000000..09d1fbd89a --- /dev/null +++ b/apple/testing/simulator_pool/create_simulator_pool.bzl @@ -0,0 +1,58 @@ +""" Creates a simulator pool for a given OS version and device type. """ + +def _create_simulator_pool_impl(ctx): + executable = ctx.actions.declare_file(ctx.label.name + ".sh") + ctx.actions.expand_template( + template = ctx.file._template, + output = executable, + is_executable = True, + substitutions = { + "%create_simulator_pool%": ctx.executable._create_simulator_pool_tool.short_path, + "%simulator_pool_server%": ctx.executable._simulator_pool_server.short_path, + "%simulator_pool_port%": str(ctx.attr.server_port), + "%os_version%": ctx.attr.os_version, + "%device_type%": ctx.attr.device_type, + "%pool_size%": str(ctx.attr.pool_size), + }, + ) + runfiles = ctx.runfiles(files = [ctx.executable._create_simulator_pool_tool, ctx.executable._simulator_pool_server]) + runfiles = runfiles.merge(ctx.attr._create_simulator_pool_tool[DefaultInfo].default_runfiles) + runfiles = runfiles.merge(ctx.attr._simulator_pool_server[DefaultInfo].default_runfiles) + return [DefaultInfo(executable = executable, runfiles = runfiles)] + +create_simulator_pool = rule( + implementation = _create_simulator_pool_impl, + executable = True, + attrs = { + "_create_simulator_pool_tool": attr.label( + default = "//apple/testing/simulator_pool:create_simulator_pool_tool", + executable = True, + cfg = "exec", + ), + "_simulator_pool_server": attr.label( + default = "//apple/testing/simulator_pool:simulator_pool_server", + executable = True, + cfg = "exec", + ), + "_template": attr.label( + allow_single_file = True, + default = "//apple/testing/simulator_pool:create_simulator_pool.template.sh", + ), + "server_port": attr.int( + mandatory = True, + doc = "The port to run the simulator pool server on, this value must match the value set in your test runner otherwise the test runner will not be able to connect to the simulator pool server.", + ), + "os_version": attr.string( + mandatory = True, + doc = "The OS version to create the simulator pool for.", + ), + "device_type": attr.string( + mandatory = True, + doc = "The device type to create the simulator pool for.", + ), + "pool_size": attr.int( + mandatory = True, + doc = "The number of simulators to create in the pool.", + ), + }, +) diff --git a/apple/testing/simulator_pool/create_simulator_pool.template.sh b/apple/testing/simulator_pool/create_simulator_pool.template.sh new file mode 100644 index 0000000000..bfd3a76819 --- /dev/null +++ b/apple/testing/simulator_pool/create_simulator_pool.template.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +simulator_pool_config_output_path="$PWD/simulator_pool_config.json" + +"%create_simulator_pool%" \ + --os-version "%os_version%" \ + --device-type "%device_type%" \ + --pool-size "%pool_size%" \ + --simulator-pool-config-output-path "$simulator_pool_config_output_path" + +"%simulator_pool_server%" \ + --simulator-pool-config-path "$simulator_pool_config_output_path" \ + --port "%simulator_pool_port%" > "$PWD/simulator_pool_server.log" 2>&1 & diff --git a/apple/testing/simulator_pool/create_simulator_pool_tool.py b/apple/testing/simulator_pool/create_simulator_pool_tool.py new file mode 100644 index 0000000000..29beab8c46 --- /dev/null +++ b/apple/testing/simulator_pool/create_simulator_pool_tool.py @@ -0,0 +1,124 @@ +#!/usr/bin/python3 +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import apple.testing.default_runner.simulator_utils as simulator_utils +import random +import string +import sys +import json +import os +import subprocess + +def _golden_device_name(device_type: str, os_version: str) -> str: + return f"RULES_APPLE_GOLDEN_SIMULATOR_{device_type}_{os_version}" + +def _clone_simulator_name(device_type: str, os_version: str) -> str: + device_name_suffix = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) + return f"{_cloned_simulator_prefix()}{device_type}_{os_version}_{device_name_suffix}" + +def _cloned_simulator_prefix() -> str: + return "RULES_APPLE_CLONED_GOLDEN_SIMULATOR_" + +def _clone_simulator(simulator_id: str, device_type: str, os_version: str) -> str: + return simulator_utils.simctl(["clone", simulator_id, _clone_simulator_name(device_type, os_version)]).strip() + +def _shutdown_simulator(simulator_id: str) -> None: + simulator_utils.simctl(["shutdown", simulator_id]) + +def _delete_simulator(simulator_id: str) -> None: + simulator_utils.simctl(["delete", simulator_id]) + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument( + "--os-version", help="The iOS version to use for simulators created in the pool, ex: 12.1" + ) + parser.add_argument( + "--device-type", help="The iOS device to use for simulators created in the pool, ex: iPhone 12" + ) + parser.add_argument( + "--pool-size", help="The number of simulators to create in the pool, ex: 3", type=int + ) + parser.add_argument( + "--simulator-pool-config-output-path", help="The path to the simulator pool config output file", + ) + return parser + + +def _main(os_version: str, device_type: str, pool_size: int, simulator_pool_config_output_path: str) -> None: + devices = json.loads(simulator_utils.simctl(["list", "devices", "-j"]))["devices"] + device_name = _golden_device_name(device_type, os_version) + runtime_identifier = "com.apple.CoreSimulator.SimRuntime.iOS-{}".format( + os_version.replace(".", "-") + ) + + devices_for_os = devices.get(runtime_identifier) or [] + existing_golden_device = next( + (blob for blob in devices_for_os if blob["name"] == device_name), None + ) + + for device in devices_for_os: + if device["name"].startswith(_cloned_simulator_prefix()): + _delete_simulator(device["udid"]) + + simulator_udids = [] + + if existing_golden_device: + simulator_id = existing_golden_device["udid"] + simulator_udids.append(simulator_id) + name = existing_golden_device["name"] + # If the device is already booted assume that it was created with this + # script and bootstatus has already waited for it to be in a good state + # once + state = existing_golden_device["state"].lower() + print(f"Existing simulator '{name}' ({simulator_id}) state is: {state}", file=sys.stderr) + if state == "booted": + _shutdown_simulator(simulator_id) + for _ in range(pool_size): + cloned_simulator_id = _clone_simulator(simulator_id, device_type, os_version) + simulator_udids.append(cloned_simulator_id) + print(f"Cloned simulator '{name}' ({simulator_id}) -> '{cloned_simulator_id}'", file=sys.stderr) + else: + simulator_id = simulator_utils.simctl( + ["create", device_name, device_type, runtime_identifier] + ).strip() + simulator_utils.boot_simulator(simulator_id) + _shutdown_simulator(simulator_id) + print(f"Created new simulator '{device_name}' ({simulator_id})", file=sys.stderr) + for _ in range(pool_size): + cloned_simulator_id = _clone_simulator(simulator_id, device_type, os_version) + simulator_udids.append(cloned_simulator_id) + print(f"Cloned simulator '{device_name}' ({simulator_id}) -> '{cloned_simulator_id}'", file=sys.stderr) + for simulator in simulator_udids: + simulator_utils.boot_simulator(simulator) + simulator_pool_config = { + "simulators": [ + { + "device_type": device_type, + "os_version": os_version, + "udid": simulator_udid + } + for simulator_udid in simulator_udids + ] + } + with open(simulator_pool_config_output_path, "w") as f: + json.dump(simulator_pool_config, f) + print(f"Simulator pool config written to {simulator_pool_config_output_path}", file=sys.stderr) + +if __name__ == "__main__": + args = _build_parser().parse_args() + _main(args.os_version, args.device_type, args.pool_size, args.simulator_pool_config_output_path) diff --git a/apple/testing/simulator_pool/simulator_pool_client.py b/apple/testing/simulator_pool/simulator_pool_client.py new file mode 100644 index 0000000000..7fc6d34d40 --- /dev/null +++ b/apple/testing/simulator_pool/simulator_pool_client.py @@ -0,0 +1,235 @@ +#!/usr/bin/python3 +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import sys +import urllib.request +import urllib.parse +from typing import Optional + + +class SimulatorPoolClient: + """Client for interacting with the simulator pool server.""" + + def __init__(self, port: int): + """Initialize the client with the server URL. + + Args: + port: Port of the simulator pool server (e.g., 8080) + """ + self.port = port + + def request_simulator(self, device_type: str, os_version: str, test_target: str, test_host: Optional[str]) -> Optional[str]: + """Request a simulator from the pool. + + Args: + device_type: Type of device (e.g., 'iPhone 14') + os_version: iOS version (e.g., '16.0') + test_target: Test target (e.g., 'MyAppTests') + test_host: Test host (e.g., 'MyApp') + + Returns: + Simulator UDID if available, None otherwise + """ + try: + # Build query parameters + params = { + 'device_type': device_type, + 'os_version': os_version, + 'test_target': test_target, + 'test_host': test_host if test_host else '', + } + query_string = urllib.parse.urlencode(params) + url = f"http://localhost:{self.port}/request?{query_string}" + + # Make the request + with urllib.request.urlopen(url) as response: + if response.status == 200: + data = json.loads(response.read().decode()) + if data.get('success'): + return data.get('udid') + else: + print(f"No simulator available: {data.get('udid', '')}", file=sys.stderr) + return None + else: + print(f"Request failed with status {response.status}", file=sys.stderr) + return None + + except urllib.error.HTTPError as e: + if e.code == 400: + error_data = json.loads(e.read().decode()) + print(f"Bad request: {error_data.get('error', 'Unknown error')}", file=sys.stderr) + else: + print(f"HTTP error {e.code}: {e.reason}", file=sys.stderr) + return None + except urllib.error.URLError as e: + print(f"Connection error: {e.reason}", file=sys.stderr) + return None + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + return None + + def return_simulator(self, simulator_udid: str) -> bool: + """Return a simulator to the pool. + + Args: + simulator_udid: UDID of the simulator to return + + Returns: + True if successful, False otherwise + """ + try: + # Prepare the request data + data = { + 'udid': simulator_udid + } + json_data = json.dumps(data).encode('utf-8') + + # Create the request + url = f"http://localhost:{self.port}/return" + req = urllib.request.Request(url, data=json_data, method='POST') + req.add_header('Content-Type', 'application/json') + + # Make the request + with urllib.request.urlopen(req) as response: + if response.status == 200: + response_data = json.loads(response.read().decode()) + success = response_data.get('success', False) + message = response_data.get('message', 'Unknown response') + + if success: + print(f"Success: {message}", file=sys.stderr) + else: + print(f"Failed: {message}", file=sys.stderr) + + return success + else: + print(f"Return failed with status {response.status}", file=sys.stderr) + return False + + except urllib.error.HTTPError as e: + if e.code == 400: + error_data = json.loads(e.read().decode()) + print(f"Bad request: {error_data.get('error', 'Unknown error')}", file=sys.stderr) + else: + print(f"HTTP error {e.code}: {e.reason}", file=sys.stderr) + return False + except urllib.error.URLError as e: + print(f"Connection error: {e.reason}", file=sys.stderr) + return False + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + return False + + +def main(): + """Main entry point for the command line tool.""" + parser = argparse.ArgumentParser( + description='Simulator Pool Client - Interact with the simulator pool server', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Request a simulator + %(prog)s request --device-type "iPhone 14" --os-version "16.0" + + # Return a simulator + %(prog)s return --udid "12345678-1234-1234-1234-123456789012" + + # Use custom port + %(prog)s --port 9000 request --device-type "iPhone 14" --os-version "16.0" + """ + ) + + # Global options + parser.add_argument( + '--port', + help='Simulator pool server port' + ) + + # Subcommands + subparsers = parser.add_subparsers(dest='command', help='Available commands') + + # Request command + request_parser = subparsers.add_parser( + 'request', + help='Request a simulator from the pool' + ) + request_parser.add_argument( + '--device-type', + required=True, + help='Device type (e.g., "iPhone 14", "iPad Pro")' + ) + request_parser.add_argument( + '--os-version', + required=True, + help='iOS version (e.g., "16.0", "15.5")' + ) + request_parser.add_argument( + '--test-target', + required=True, + help='Test target (e.g., "MyAppTests")' + ) + request_parser.add_argument( + '--test-host', + required=False, + help='Test host (e.g., "MyApp")' + ) + + # Return command + return_parser = subparsers.add_parser( + 'return', + help='Return a simulator to the pool' + ) + return_parser.add_argument( + '--udid', + required=True, + help='Simulator UDID to return' + ) + + args = parser.parse_args() + + if not args.command: + parser.print_help() + sys.exit(1) + + # Create client + client = SimulatorPoolClient(args.port) + + try: + if args.command == 'request': + # Request a simulator + simulator_udid = client.request_simulator(args.device_type, args.os_version, args.test_target, args.test_host) + while not simulator_udid: + time.sleep(1) + simulator_udid = client.request_simulator(args.device_type, args.os_version, args.test_target, args.test_host) + print(simulator_udid) + sys.exit(0) + + elif args.command == 'return': + # Return a simulator + success = client.return_simulator(args.udid) + sys.exit(0 if success else 1) + + except KeyboardInterrupt: + print("\nOperation cancelled by user", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/apple/testing/simulator_pool/simulator_pool_server.py b/apple/testing/simulator_pool/simulator_pool_server.py new file mode 100644 index 0000000000..54e4b642fe --- /dev/null +++ b/apple/testing/simulator_pool/simulator_pool_server.py @@ -0,0 +1,347 @@ +#!/usr/bin/python3 +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import signal +import socket +import subprocess +import sys +import time +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse, parse_qs +from typing import Dict, Set, Optional, List +import apple.testing.default_runner.simulator_utils as simulator_utils +import random +import string + +class Simulator: + def __init__(self, udid: str, device_type: str, os_version: str): + self.udid = udid + self.device_type = device_type + self.os_version = os_version + + def __hash__(self): + return hash((self.udid, self.device_type, self.os_version)) + +class SimulatorPoolHandler(BaseHTTPRequestHandler): + """HTTP request handler for simulator pool management.""" + + # Class-level variables to store the pool + available_simulators = set() + all_simulators = set() + + @classmethod + def set_simulator_pool(cls, simulator_pool: List[Simulator]): + """Set the simulator pool for the handler class.""" + cls.available_simulators = set(simulator_pool) + cls.all_simulators = set(simulator_pool) + + def do_GET(self): + """Handle GET requests for requesting a simulator.""" + parsed_url = urlparse(self.path) + + if parsed_url.path == '/request': + # Request a simulator UDID + # Get device_type and os_version from query parameters + query_params = parse_qs(parsed_url.query) + device_type = query_params.get('device_type', [''])[0] + os_version = query_params.get('os_version', [''])[0] + + if not device_type or not os_version: + self._send_error_response(400, "Missing device_type or os_version query parameters") + return + + simulator = self._get_available_simulator(device_type, os_version) + + response_data = { + 'udid': simulator.udid if simulator else '', + 'success': bool(simulator) + } + + self._send_json_response(response_data) + elif parsed_url.path == '/status': + # Status endpoint + response_data = { + 'status': 'running', + 'available_simulators': len(self.available_simulators), + 'total_simulators': len(self.all_simulators), + 'timestamp': time.time() + } + self._send_json_response(response_data) + elif parsed_url.path == '/shutdown': + # Graceful shutdown endpoint + response_data = { + 'success': True, + 'message': 'Server shutdown initiated' + } + self._send_json_response(response_data) + + # Schedule shutdown after response is sent + import threading + def delayed_shutdown(): + time.sleep(0.1) # Small delay to ensure response is sent + os.kill(os.getpid(), signal.SIGTERM) + + threading.Thread(target=delayed_shutdown, daemon=True).start() + else: + self._send_error_response(404, "Endpoint not found") + + def do_POST(self): + """Handle POST requests for returning a simulator.""" + parsed_url = urlparse(self.path) + + if parsed_url.path == '/return': + # Return a simulator UDID + content_length = int(self.headers.get('Content-Length', 0)) + if content_length == 0: + self._send_error_response(400, "Missing request body") + return + + try: + request_data = json.loads(self.rfile.read(content_length)) + simulator_udid = request_data.get('udid') + + if not simulator_udid: + self._send_error_response(400, "Missing 'udid' in request body") + return + + success = self._return_simulator(simulator_udid) + + response_data = { + 'success': success, + 'message': f"Simulator {simulator_udid} {'returned to pool' if success else 'not found in pool'}" + } + + self._send_json_response(response_data) + + except json.JSONDecodeError: + self._send_error_response(400, "Invalid JSON in request body") + else: + self._send_error_response(404, "Endpoint not found") + + def _get_available_simulator(self, device_type: str, os_version: str) -> Optional[Simulator]: + """Get an available simulator UDID from the pool. If there are no available simulators, return None.""" + if not self.available_simulators: + return None + for simulator in self.available_simulators: + if simulator.device_type == device_type and simulator.os_version == os_version: + self.available_simulators.remove(simulator) + return simulator + return None + + def _return_simulator(self, simulator_udid: str) -> bool: + """Return a simulator UDID to the available pool.""" + for simulator in self.all_simulators: + if simulator.udid == simulator_udid: + self.available_simulators.add(simulator) + return True + return False + + def _send_json_response(self, data: Dict) -> None: + """Send a JSON response.""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + + def _send_error_response(self, status_code: int, message: str) -> None: + """Send an error response.""" + self.send_response(status_code) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + error_data = { + 'error': message, + 'status_code': status_code + } + self.wfile.write(json.dumps(error_data).encode()) + + def log_message(self, format, *args): + """Override to use stderr for logging.""" + print(f"[{self.log_date_time_string()}] {format % args}", file=sys.stderr) + +def parse_simulators(simulator_pool_config_path: str): + with open(simulator_pool_config_path, 'r') as f: + simulator_pool_config = json.load(f) + return [Simulator(simulator['udid'], simulator['device_type'], simulator['os_version']) for simulator in simulator_pool_config['simulators']] + +def is_port_in_use(host: str, port: int) -> bool: + """Check if a port is already in use.""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(1) + result = s.connect_ex((host, port)) + return result == 0 + except Exception: + return False + +def kill_processes_on_port(host: str, port: int) -> bool: + """Kill any processes using the specified port.""" + try: + # Use lsof to find processes using the port + if host == 'localhost' or host == '127.0.0.1': + # For localhost, we can use lsof to find the process + cmd = ['lsof', '-ti', f':{port}'] + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0 and result.stdout.strip(): + pids = result.stdout.strip().split('\n') + killed_count = 0 + + for pid in pids: + if pid.strip(): + try: + # Try graceful shutdown first + os.kill(int(pid), signal.SIGTERM) + time.sleep(1) + + # Check if process is still running + try: + os.kill(int(pid), 0) # Signal 0 doesn't kill, just checks if process exists + # Process still running, force kill + os.kill(int(pid), signal.SIGKILL) + time.sleep(0.5) + except OSError: + pass # Process is dead + + killed_count += 1 + print(f"Killed process {pid} using port {port}", file=sys.stderr) + + except (OSError, ValueError) as e: + print(f"Error killing process {pid}: {e}", file=sys.stderr) + + # Wait a bit for port to be released + time.sleep(2) + return killed_count > 0 + else: + # For non-localhost, we can't easily kill remote processes + print(f"Warning: Cannot kill processes on remote host {host}", file=sys.stderr) + return False + + except Exception as e: + print(f"Error checking/killing processes on port {port}: {e}", file=sys.stderr) + return False + + return False + +def ensure_port_available(host: str, port: int, force_kill: bool = True) -> bool: + """Ensure the port is available, optionally killing existing processes.""" + if not is_port_in_use(host, port): + return True + + # Check if there's already a simulator pool server running + server_status = check_server_status(host, port) + if server_status['running']: + print(f"Port {port} is in use by another simulator pool server:", file=sys.stderr) + if server_status['data']: + print(f" Available simulators: {server_status['data'].get('available_simulators', 'unknown')}", file=sys.stderr) + print(f" Total simulators: {server_status['data'].get('total_simulators', 'unknown')}", file=sys.stderr) + + if not force_kill: + print(f"Port {port} is already in use. Use --force-kill to kill existing processes.", file=sys.stderr) + return False + + print(f"Port {port} is in use. Attempting to kill existing processes...", file=sys.stderr) + if kill_processes_on_port(host, port): + # Check again if port is now available + time.sleep(1) + if not is_port_in_use(host, port): + print(f"Port {port} is now available", file=sys.stderr) + return True + else: + print(f"Port {port} is still in use after killing processes", file=sys.stderr) + return False + else: + print(f"Failed to free port {port}", file=sys.stderr) + return False + +def check_server_status(host: str, port: int) -> Dict[str, any]: + """Check if there's already a simulator pool server running on the port.""" + try: + import urllib.request + import urllib.error + + url = f"http://{host}:{port}/status" + with urllib.request.urlopen(url, timeout=5) as response: + if response.status == 200: + data = json.loads(response.read().decode()) + return { + 'running': True, + 'data': data + } + except (urllib.error.URLError, urllib.error.HTTPError): + pass + except Exception: + pass + + return {'running': False, 'data': None} + +def signal_handler(signum, frame): + """Handle shutdown signals gracefully.""" + print(f"\nReceived signal {signum}, shutting down server...", file=sys.stderr) + sys.exit(0) + +def run_server(host: str, port: int, simulator_pool: List[Simulator], force_kill: bool = True): + """Run the simulator pool server.""" + # Set up signal handlers for graceful shutdown + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + # Ensure port is available before starting + if not ensure_port_available(host, port, force_kill): + print(f"Failed to start server: port {port} is not available", file=sys.stderr) + sys.exit(1) + + # Set the pool on the handler class before creating the server + SimulatorPoolHandler.set_simulator_pool(simulator_pool) + + server_address = (host, port) + httpd = HTTPServer(server_address, SimulatorPoolHandler) + + print(f"Simulator pool server starting on {host}:{port}", file=sys.stderr) + print("Available endpoints:", file=sys.stderr) + print(" GET /request?device_type=&os_version= - Request a simulator UDID", file=sys.stderr) + print(" POST /return - Return a simulator UDID (body: {\"udid\": \"\"})", file=sys.stderr) + print(" GET /status - Get server status and pool information", file=sys.stderr) + print(" GET /shutdown - Gracefully shutdown the server", file=sys.stderr) + + try: + httpd.serve_forever() + + except KeyboardInterrupt: + print("\nShutting down server...", file=sys.stderr) + httpd.shutdown() + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description='Simulator Pool HTTP Server') + parser.add_argument('--host', default='localhost', help='Host to bind to (default: localhost)') + parser.add_argument('--port', type=int, help='Port to bind to') + parser.add_argument('--simulator-pool-config-path', help='Path to the simulator pool config file') + parser.add_argument('--force-kill', action='store_true', default=True, + help='Kill existing processes using the port (default: True)') + parser.add_argument('--no-force-kill', dest='force_kill', action='store_false', + help='Do not kill existing processes using the port') + + args = parser.parse_args() + + simulator_pool = parse_simulators(args.simulator_pool_config_path) + + run_server(args.host, args.port, simulator_pool, args.force_kill) + +if __name__ == '__main__': + main() diff --git a/doc/rules-ios.md b/doc/rules-ios.md index bbef7e5ebb..0aeca86955 100644 --- a/doc/rules-ios.md +++ b/doc/rules-ios.md @@ -696,7 +696,7 @@ load("@rules_apple//apple:ios.doc.bzl", "ios_xctestrun_runner") ios_xctestrun_runner(name, attachment_lifetime, command_line_args, create_xcresult_bundle, destination_timeout, device_type, os_version, post_action, post_action_determines_exit_code, pre_action, random, reuse_simulator, - xcodebuild_args) + simulator_pool_server_port, xcodebuild_args) This rule creates a test runner for iOS tests that uses xctestrun files to run @@ -753,6 +753,7 @@ in Xcode. | pre_action | A binary to run prior to test execution. Runs after simulator creation. Sets the `$SIMULATOR_UDID` environment variable, in addition to any other variables available to the test runner. | Label | optional | `None` | | random | Whether to run the tests in random order to identify unintended state dependencies. | Boolean | optional | `False` | | reuse_simulator | Toggle simulator reuse. The default behavior is to reuse an existing device of the same type and OS version. When disabled, a new simulator is created before testing starts and shutdown when the runner completes. | Boolean | optional | `True` | +| simulator_pool_server_port | The port of a running simulator pool server. If set, the test runner will connect to the simulator pool server and use the simulators from the pool instead of creating new ones. | Integer | optional | `0` | | xcodebuild_args | Arguments to pass to `xcodebuild` when running the test bundle. This means it will always use `xcodebuild test-without-building` to run the test bundle. | List of strings | optional | `[]` | diff --git a/test/ios_xctestrun_runner_unit_test.sh b/test/ios_xctestrun_runner_unit_test.sh index 511ed5782b..c72110a152 100755 --- a/test/ios_xctestrun_runner_unit_test.sh +++ b/test/ios_xctestrun_runner_unit_test.sh @@ -38,6 +38,10 @@ load( "@build_bazel_rules_apple//apple/testing/default_runner:ios_xctestrun_runner.bzl", "ios_xctestrun_runner" ) +load( + "@build_bazel_rules_apple//apple/testing/simulator_pool:create_simulator_pool.bzl", + "create_simulator_pool" +) load("@rules_shell//shell:sh_binary.bzl", "sh_binary") ios_xctestrun_runner( @@ -45,6 +49,20 @@ ios_xctestrun_runner( device_type = "iPhone Xs", ) +create_simulator_pool( + name = "create_simulator_pool", + device_type = "iPhone Xs", + pool_size = 2, + os_version = "${MIN_OS_IOS}", + server_port = 50051, +) + +ios_xctestrun_runner( + name = "ios_x86_64_sim_runner_using_simulator_pool", + device_type = "iPhone Xs", + simulator_pool_server_port = 50051, +) + ios_xctestrun_runner( name = "ios_x86_64_sim_reuse_disabled_runner", device_type = "iPhone Xs", @@ -109,6 +127,15 @@ ios_xctestrun_runner( EOF } +function setup_simulator_pool_server() { + bazel run //ios:create_simulator_pool +} + +function teardown_simulator_pool_server() { + curl -X GET http://localhost:50051/shutdown --fail > /dev/null 2>&1 + xcrun simctl delete all +} + function create_test_host_app() { if [[ ! -f ios/BUILD ]]; then fail "create_sim_runners must be called first." @@ -321,6 +348,15 @@ ios_unit_test( runner = ":ios_x86_64_sim_runner", ) +ios_unit_test( + name = "SmallUnitTestUsingSimulatorPool", + infoplists = ["SmallUnitTest-Info.plist"], + deps = [":small_unit_test_lib"], + minimum_os_version = "${MIN_OS_IOS}", + env = test_env, + runner = ":ios_x86_64_sim_runner_using_simulator_pool", +) + objc_library( name = "pass_unit_test_lib", srcs = ["pass_unit_test.m"], @@ -731,6 +767,19 @@ function test_ios_unit_test_small_pass() { expect_log "Executed 2 tests, with 0 failures" } +function test_ios_unit_test_small_pass_using_simulator_pool() { + create_sim_runners + create_ios_unit_tests + setup_simulator_pool_server + do_ios_test //ios:SmallUnitTestUsingSimulatorPool || fail "should pass" + teardown_simulator_pool_server + + expect_log "Test Suite 'SmallUnitTest1' passed" + expect_log "Test Suite 'SmallUnitTest2' passed" + expect_log "Test Suite 'SmallUnitTestUsingSimulatorPool.xctest' passed" + expect_log "Executed 2 tests, with 0 failures" +} + # Test bundle has tests with one test class with all tests filtered. function test_ios_unit_test_small_empty_test_class_filter_pass() { create_sim_runners