123 lines
4 KiB
Python
123 lines
4 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# Copyright Catch2 Authors
|
|
# Distributed under the Boost Software License, Version 1.0.
|
|
# (See accompanying file LICENSE.txt or copy at
|
|
# https://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
# SPDX-License-Identifier: BSL-1.0
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def build_project(sources_dir, output_base_path, catch2_path):
|
|
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
|
|
config_cmd = ['cmake',
|
|
'-B', build_dir,
|
|
'-S', sources_dir,
|
|
f'-DCATCH2_PATH={catch2_path}',
|
|
'-DCMAKE_BUILD_TYPE=Debug']
|
|
|
|
build_cmd = ['cmake',
|
|
'--build', build_dir,
|
|
'--config', 'Debug']
|
|
|
|
try:
|
|
subprocess.run(config_cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
subprocess.run(build_cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
except subprocess.CalledProcessError as err:
|
|
print('Error when building the test project')
|
|
print(f'cmd: {err.cmd}')
|
|
print(f'stderr: {err.stderr}')
|
|
print(f'stdout: {err.stdout}')
|
|
exit(3)
|
|
|
|
return build_dir
|
|
|
|
|
|
|
|
def get_test_names(build_path):
|
|
# For now we assume that Windows builds are done using MSBuild under
|
|
# Debug configuration. This means that we need to add "Debug" folder
|
|
# to the path when constructing it. On Linux, we don't add anything.
|
|
config_path = "Debug" if os.name == 'nt' else ""
|
|
full_path = os.path.join(build_path, config_path, 'tests')
|
|
|
|
|
|
cmd = [full_path, '--reporter', 'xml', '--list-tests']
|
|
result = subprocess.run(cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
|
|
import xml.etree.ElementTree as ET
|
|
root = ET.fromstring(result.stdout)
|
|
return [tc.text for tc in root.findall('TestCase/Name')]
|
|
|
|
|
|
def list_ctest_tests(build_path):
|
|
old_path = os.getcwd()
|
|
os.chdir(build_path)
|
|
|
|
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
|
|
result = subprocess.run(cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
os.chdir(old_path)
|
|
|
|
import json
|
|
|
|
ctest_response = json.loads(result.stdout)
|
|
tests = ctest_response['tests']
|
|
test_names = []
|
|
for test in tests:
|
|
test_command = test['command']
|
|
# First part of the command is the binary, second is the filter.
|
|
# If there are less, registration has failed. If there are more,
|
|
# registration has changed and the script needs updating.
|
|
assert len(test_command) == 2
|
|
test_names.append(test_command[1])
|
|
test_name = test_command[1]
|
|
|
|
return test_names
|
|
|
|
def escape_catch2_test_name(name):
|
|
for char in ('\\', ',', '[', ']'):
|
|
name = name.replace(char, f"\\{char}")
|
|
return name
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) != 3:
|
|
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
|
|
exit(2)
|
|
catch2_path = sys.argv[1]
|
|
output_base_path = sys.argv[2]
|
|
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
|
|
|
|
build_path = build_project(sources_dir, output_base_path, catch2_path)
|
|
|
|
catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
|
|
ctest_test_names = list_ctest_tests(build_path)
|
|
|
|
mismatched = 0
|
|
for catch_test in catch_test_names:
|
|
if catch_test not in ctest_test_names:
|
|
print(f"Catch2 test '{catch_test}' not found in CTest")
|
|
mismatched += 1
|
|
for ctest_test in ctest_test_names:
|
|
if ctest_test not in catch_test_names:
|
|
print(f"CTest test '{ctest_test}' not found in Catch2")
|
|
mismatched += 1
|
|
|
|
if mismatched:
|
|
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
|
|
exit(1)
|