ci: Run cross tests on Windows.

A driver program is introduced to coordinate test running on Windows,
similarly to what "make test" does on Linux and macOS.
This commit is contained in:
Giovanni Mascellani 2023-09-26 11:20:39 +02:00 committed by Alexandre Julliard
parent f75bdd6e21
commit dbc5e7d07c
Notes: Alexandre Julliard 2023-10-31 22:37:34 +01:00
Approved-by: Henri Verbeet (@hverbeet)
Approved-by: Alexandre Julliard (@julliard)
Merge-Request: https://gitlab.winehq.org/wine/vkd3d/-/merge_requests/413
7 changed files with 278 additions and 6 deletions

1
.gitignore vendored
View File

@ -23,6 +23,7 @@ vkd3d-*.tar.xz
*.tab.c
*.tab.h
*.trs
*.txt
*.yy.c
*~

View File

@ -1,7 +1,9 @@
stages:
- image
- build
- test
include:
- local: "/gitlab/image.yml"
- local: "/gitlab/build.yml"
- local: "/gitlab/test.yml"

View File

@ -480,6 +480,9 @@ shader_runner_cross_sources = \
$(srcdir)/tests/shader_runner_d3d11.c \
$(srcdir)/tests/shader_runner_d3d12.c
driver_cross_sources = \
$(srcdir)/tests/driver.c
if HAVE_CROSSTARGET32
CROSS32_CC = @CROSSCC32@
CROSS32_DLLTOOL = @CROSSTARGET32@-dlltool
@ -495,6 +498,7 @@ endif
CROSS32_FILES = $(CROSS32_EXEFILES)
if BUILD_TESTS
CROSS32_FILES += tests/shader_runner.cross32.exe
CROSS32_FILES += tests/driver.cross32.exe
endif
CLEANFILES += $(CROSS32_IMPLIBS) $(CROSS32_FILES)
@ -517,6 +521,11 @@ tests/shader_runner.cross32.exe: $(shader_runner_cross_sources) $(CROSS32_IMPLIB
$(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS32_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \
$(am__mv) $$depbase.Tpo $$depbase.Po
tests/driver.cross32.exe: $(driver_cross_sources)
$(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!\.exe$$!!'`; \
$(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \
$(am__mv) $$depbase.Tpo $$depbase.Po
else
crosstest32:
endif
@ -536,6 +545,7 @@ endif
CROSS64_FILES = $(CROSS64_EXEFILES)
if BUILD_TESTS
CROSS64_FILES += tests/shader_runner.cross64.exe
CROSS64_FILES += tests/driver.cross64.exe
endif
CLEANFILES += $(CROSS64_IMPLIBS) $(CROSS64_FILES)
@ -558,12 +568,25 @@ tests/shader_runner.cross64.exe: $(shader_runner_cross_sources) $(CROSS64_IMPLIB
$(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS64_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \
$(am__mv) $$depbase.Tpo $$depbase.Po
tests/driver.cross64.exe: $(driver_cross_sources)
$(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!\.exe$$!!'`; \
$(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \
$(am__mv) $$depbase.Tpo $$depbase.Po
else
crosstest64:
endif
.PHONY: crosstest crosstest32 crosstest64
crosstest: crosstest32 crosstest64
tests/crosstests.txt: FORCE
$(AM_V_GEN) for i in $(vkd3d_cross_tests) ; do echo $$i ; done > $@
crosstest-lists: tests/crosstests.txt
CLEANFILES += tests/crosstests.txt
FORCE:
.PHONY: crosstest crosstest32 crosstest64 crosstest-lists FORCE
crosstest: crosstest32 crosstest64 crosstest-lists
if BUILD_DOC
@DX_RULES@

View File

@ -19,9 +19,10 @@ MoltenVK as the Vulkan driver. The llvmpipe and macOS jobs are
currently allowed to fail.
Additionally, MinGW is used to build PE binaries for both vkd3d and
its crosstests, for both 32 and 64 bit. These builds are not currently
tested (but the pipeline still fails if the compilation is not
successful).
its crosstests, for both 32 and 64 bit. The PE crosstests are executed
on Windows 10 to check that behavior imposed by the tests corresponds
to Microsoft's D3D12 implementation. The rendering backend is
currently Window's WARP software implementation.
The testing logs are available as CI artifacts, as well as the PE
modules built by the crosstest and MinGW jobs.
@ -58,3 +59,6 @@ environment for running the tests. All the software required to
compile and run the tests will therefore have to be installed directly
on the host system. Complete instructions to setup the macOS are
currently not available.
Finally, a runner tagged with `win10-21h2' must be available and
submit jobs to a Windows 10 virtual machine.

View File

@ -19,6 +19,9 @@ cd build
touch ../pipeline_failed
mkdir -p ../artifacts/$COMMIT
rsync -Rr config.log tests/*.exe ../artifacts/$COMMIT
rsync -Rr config.log tests/*.txt tests/*.exe ../artifacts/$COMMIT
# Make the driver easily available to the Windows CI job
cp tests/driver.cross64.exe ../artifacts
git reset --hard

35
gitlab/test.yml Normal file
View File

@ -0,0 +1,35 @@
test-win-64:
stage: test
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
interruptible: true
needs:
- job: build-crosstest
tags:
- win10-21h2
script:
- ./artifacts/driver.cross64.exe
variables:
TEST_ARCH: "64"
artifacts:
when: always
paths:
- artifacts
test-win-32:
stage: test
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
interruptible: true
needs:
- job: build-crosstest
tags:
- win10-21h2
script:
- ./artifacts/driver.cross64.exe
variables:
TEST_ARCH: "32"
artifacts:
when: always
paths:
- artifacts

204
tests/driver.c Normal file
View File

@ -0,0 +1,204 @@
/*
* Copyright 2023 Giovanni Mascellani for CodeWeavers
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
*/
#include <stdio.h>
#include <stdbool.h>
#include <windows.h>
#define TIMEOUT_MS (10 * 1000)
#define MAX_TIMEOUT_COUNT 3
enum program_result
{
PROGRAM_RESULT_SUCCESS,
PROGRAM_RESULT_TIMEOUT,
PROGRAM_RESULT_FAILURE,
};
static enum program_result run_program(const char *cmdline, const char *log_filename)
{
enum program_result ret = PROGRAM_RESULT_SUCCESS;
HANDLE log = INVALID_HANDLE_VALUE;
SECURITY_ATTRIBUTES attrs = {0};
PROCESS_INFORMATION info = {0};
DWORD exit_code, wait_result;
STARTUPINFOA startup = {0};
char cmdline2[1024];
strcpy(cmdline2, cmdline);
attrs.nLength = sizeof(attrs);
attrs.bInheritHandle = TRUE;
log = CreateFileA(log_filename, GENERIC_WRITE, 0, &attrs, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (log == INVALID_HANDLE_VALUE)
{
fprintf(stderr, "Cannot create log file %s, last error %ld.\n", log_filename, GetLastError());
ret = PROGRAM_RESULT_FAILURE;
goto out;
}
startup.cb = sizeof(startup);
startup.dwFlags = STARTF_USESTDHANDLES;
startup.hStdInput = INVALID_HANDLE_VALUE;
startup.hStdOutput = log;
startup.hStdError = log;
if (!CreateProcessA(NULL, cmdline2, NULL, NULL, TRUE, 0, NULL, NULL, &startup, &info))
{
fprintf(stderr, "Cannot create process %s, last error %ld.\n", cmdline2, GetLastError());
ret = PROGRAM_RESULT_FAILURE;
goto out;
}
wait_result = WaitForSingleObject(info.hProcess, TIMEOUT_MS);
if (wait_result == WAIT_TIMEOUT)
{
fprintf(stderr, "Process timed out, terminating it.\n");
ret = PROGRAM_RESULT_TIMEOUT;
if (!TerminateProcess(info.hProcess, 1))
{
fprintf(stderr, "Cannot terminate process, last error %ld.\n", GetLastError());
goto out;
}
wait_result = WaitForSingleObject(info.hProcess, INFINITE);
}
if (wait_result != WAIT_OBJECT_0)
{
fprintf(stderr, "Cannot wait for process termination, last error %ld.\n", GetLastError());
ret = PROGRAM_RESULT_FAILURE;
goto out;
}
if (!GetExitCodeProcess(info.hProcess, &exit_code))
{
fprintf(stderr, "Cannot retrive the process exit code, last error %ld.\n", GetLastError());
ret = PROGRAM_RESULT_FAILURE;
goto out;
}
ret = exit_code == 0 ? PROGRAM_RESULT_SUCCESS : PROGRAM_RESULT_FAILURE;
printf("%s: %s\n", ret == PROGRAM_RESULT_SUCCESS ? "PASS" : "FAIL", cmdline);
out:
if (info.hProcess && !CloseHandle(info.hProcess))
fprintf(stderr, "Cannot close process, last error %ld.\n", GetLastError());
if (info.hThread && !CloseHandle(info.hThread))
fprintf(stderr, "Cannot close thread, last error %ld.\n", GetLastError());
if (log != INVALID_HANDLE_VALUE && !CloseHandle(log))
fprintf(stderr, "Cannot close log file, last error %ld.\n", GetLastError());
return ret;
}
static bool run_tests_for_directory(const char *commit_dir)
{
char cmdline[1024], log_filename[1024], list_filename[1024], line[1024];
unsigned int success_count = 0, test_count = 0, timeout_count = 0;
const char *test_arch = getenv("TEST_ARCH");
enum program_result result;
FILE *list_file;
bool ret = true;
if (!test_arch)
test_arch = "64";
printf("Building %s\n", commit_dir);
printf("---\n");
sprintf(list_filename, "artifacts/%s/tests/crosstests.txt", commit_dir);
list_file = fopen(list_filename, "r");
if (!list_file)
{
fprintf(stderr, "Cannot open list file %s, errno %d.\n", list_filename, errno);
ret = false;
}
else
{
while (fgets(line, sizeof(line), list_file) && timeout_count < MAX_TIMEOUT_COUNT)
{
size_t len = strlen(line);
if (line[len - 1] == '\n')
line[len - 1] = '\0';
sprintf(cmdline, "artifacts/%s/%s.cross%s.exe", commit_dir, line, test_arch);
sprintf(log_filename, "artifacts/%s/%s.log", commit_dir, line);
++test_count;
result = run_program(cmdline, log_filename);
success_count += result == PROGRAM_RESULT_SUCCESS;
timeout_count += result == PROGRAM_RESULT_TIMEOUT;
}
fclose(list_file);
}
if (timeout_count >= MAX_TIMEOUT_COUNT)
fprintf(stderr, "Too many timeouts, aborting tests.\n");
printf("=======\n");
printf("Summary\n");
printf("=======\n");
printf("# TOTAL: %u\n", test_count);
printf("# PASS: %u\n", success_count);
printf("# FAIL: %u\n", test_count - success_count);
if (test_count != success_count)
ret = false;
return ret;
}
int wmain(void)
{
WIN32_FIND_DATAA find_data;
HANDLE find_handle;
bool ret = true;
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
find_handle = FindFirstFileA("artifacts/*-*", &find_data);
if (find_handle == INVALID_HANDLE_VALUE)
{
fprintf(stderr, "Cannot list commits, last error %ld.\n", GetLastError());
ret = false;
}
else
{
do
{
ret &= run_tests_for_directory(find_data.cFileName);
} while (FindNextFileA(find_handle, &find_data));
if (GetLastError() != ERROR_NO_MORE_FILES)
{
fprintf(stderr, "Cannot list tests, last error %ld.\n", GetLastError());
ret = false;
}
FindClose(find_handle);
}
return !ret;
}