diff --git a/cmake/common.cmake b/cmake/common.cmake index 9836fa7666..2de6dc758f 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1142,6 +1142,7 @@ option(NSC_DEBUG_EDIF_SOURCE_BIT "Add \"-fspv-debug=source\" to NSC Debug CLI" O option(NSC_DEBUG_EDIF_LINE_BIT "Add \"-fspv-debug=line\" to NSC Debug CLI" OFF) option(NSC_DEBUG_EDIF_TOOL_BIT "Add \"-fspv-debug=tool\" to NSC Debug CLI" ON) option(NSC_DEBUG_EDIF_NON_SEMANTIC_BIT "Add \"-fspv-debug=vulkan-with-source\" to NSC Debug CLI" OFF) +option(NSC_USE_DEPFILE "Generate depfiles for NSC custom commands" ON) function(NBL_CREATE_NSC_COMPILE_RULES) set(COMMENT "this code has been autogenerated with Nabla CMake NBL_CREATE_HLSL_COMPILE_RULES utility") @@ -1180,6 +1181,7 @@ struct DeviceConfigCaps -fspv-target-env=vulkan1.3 -Wshadow -Wconversion + -Wno-local-type-template-args $<$:-O0> $<$:-O3> $<$:-O3> @@ -1207,6 +1209,7 @@ struct DeviceConfigCaps if(NOT NBL_EMBED_BUILTIN_RESOURCES) list(APPEND REQUIRED_OPTIONS + -no-nbl-builtins -I "${NBL_ROOT_PATH}/include" -I "${NBL_ROOT_PATH}/3rdparty/dxc/dxc/external/SPIRV-Headers/include" -I "${NBL_ROOT_PATH}/3rdparty/boost/superproject/libs/preprocessor/include" @@ -1215,9 +1218,20 @@ struct DeviceConfigCaps endif() set(REQUIRED_SINGLE_ARGS TARGET BINARY_DIR OUTPUT_VAR INPUTS INCLUDE NAMESPACE MOUNT_POINT_DEFINE) - cmake_parse_arguments(IMPL "" "${REQUIRED_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) + set(OPTIONAL_SINGLE_ARGS GLOB_DIR) + cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) NBL_PARSE_REQUIRED(IMPL ${REQUIRED_SINGLE_ARGS}) + set(IMPL_HLSL_GLOB "") + if(NOT IMPL_DISCARD_DEFAULT_GLOB) + set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + if(IMPL_GLOB_DIR) + set(GLOB_ROOT "${IMPL_GLOB_DIR}") + endif() + get_filename_component(GLOB_ROOT "${GLOB_ROOT}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + file(GLOB_RECURSE IMPL_HLSL_GLOB CONFIGURE_DEPENDS "${GLOB_ROOT}/*.hlsl") + endif() + if(NOT TARGET ${IMPL_TARGET}) add_library(${IMPL_TARGET} INTERFACE) endif() @@ -1293,6 +1307,10 @@ namespace @IMPL_NAMESPACE@ { list(APPEND MP_DEFINES ${IMPL_MOUNT_POINT_DEFINE}="${IMPL_BINARY_DIR}") set_target_properties(${IMPL_TARGET} PROPERTIES NBL_MOUNT_POINT_DEFINES "${MP_DEFINES}") + set(RTE "NSC Rules") + set(IN "${RTE}/In") + set(OUT "${RTE}/Out") + string(JSON JSON_LENGTH LENGTH "${IMPL_INPUTS}") math(EXPR LAST_INDEX "${JSON_LENGTH} - 1") @@ -1479,30 +1497,73 @@ namespace @IMPL_NAMESPACE@ { # generate keys and commands for compiling shaders set(FINAL_KEY_REL_PATH "$/${FINAL_KEY}") set(TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY_REL_PATH}") + set(DEPFILE_PATH "${TARGET_OUTPUT}.d") + set(NBL_NSC_LOG_PATH "${TARGET_OUTPUT}.log") + + set(NBL_NSC_DEPFILE_ARGS "") + if(NSC_USE_DEPFILE) + set(NBL_NSC_DEPFILE_ARGS -MD -MF "${DEPFILE_PATH}") + endif() set(NBL_NSC_COMPILE_COMMAND "$" -Fc "${TARGET_OUTPUT}" ${COMPILE_OPTIONS} ${REQUIRED_OPTIONS} ${IMPL_COMMON_OPTIONS} + ${NBL_NSC_DEPFILE_ARGS} "${CONFIG_FILE}" ) - add_custom_command(OUTPUT "${TARGET_OUTPUT}" + get_filename_component(NBL_NSC_INPUT_NAME "${TARGET_INPUT}" NAME) + get_filename_component(NBL_NSC_CONFIG_NAME "${CONFIG_FILE}" NAME) + set(NBL_NSC_BYPRODUCTS "${NBL_NSC_LOG_PATH}") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_BYPRODUCTS "${DEPFILE_PATH}") + endif() + + set(NBL_NSC_CUSTOM_COMMAND_ARGS + OUTPUT "${TARGET_OUTPUT}" + BYPRODUCTS ${NBL_NSC_BYPRODUCTS} COMMAND ${NBL_NSC_COMPILE_COMMAND} DEPENDS ${DEPENDS_ON} - COMMENT "Creating \"${TARGET_OUTPUT}\"" + COMMENT "${NBL_NSC_CONFIG_NAME} (${NBL_NSC_INPUT_NAME})" VERBATIM COMMAND_EXPAND_LISTS ) - set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES GENERATED TRUE) + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_CUSTOM_COMMAND_ARGS DEPFILE "${DEPFILE_PATH}") + endif() + add_custom_command(${NBL_NSC_CUSTOM_COMMAND_ARGS}) + set(NBL_NSC_OUT_FILES "${TARGET_OUTPUT}" "${NBL_NSC_LOG_PATH}") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES "${DEPFILE_PATH}") + endif() - set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" "${TARGET_OUTPUT}") + set_source_files_properties(${NBL_NSC_OUT_FILES} PROPERTIES GENERATED TRUE) + + set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" ${NBL_NSC_OUT_FILES}) target_sources(${IMPL_TARGET} PRIVATE ${HEADER_ONLY_LIKE}) set_source_files_properties(${HEADER_ONLY_LIKE} PROPERTIES HEADER_FILE_ONLY ON VS_TOOL_OVERRIDE None ) + if(CMAKE_CONFIGURATION_TYPES) + foreach(_CFG IN LISTS CMAKE_CONFIGURATION_TYPES) + set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${_CFG}/${FINAL_KEY}") + set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d") + endif() + source_group("${OUT}/${_CFG}" FILES ${NBL_NSC_OUT_FILES_IDE}) + endforeach() + else() + set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${FINAL_KEY}") + set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d") + endif() + source_group("${OUT}" FILES ${NBL_NSC_OUT_FILES_IDE}) + endif() set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES NBL_SPIRV_REGISTERED_INPUT "${TARGET_INPUT}" @@ -1544,12 +1605,15 @@ namespace @IMPL_NAMESPACE@ { list(APPEND KEYS ${ACCESS_KEY}) endforeach() - set(RTE "NSC Rules") - set(IN "${RTE}/In") - set(OUT "${RTE}/Out") - source_group("${IN}" FILES ${CONFIGS} ${INPUTS}) - source_group("${OUT}" FILES ${SPIRVs}) + if(IMPL_HLSL_GLOB) + target_sources(${IMPL_TARGET} PRIVATE ${IMPL_HLSL_GLOB}) + set_source_files_properties(${IMPL_HLSL_GLOB} PROPERTIES + HEADER_FILE_ONLY ON + VS_TOOL_OVERRIDE None + ) + source_group("HLSL Files" FILES ${IMPL_HLSL_GLOB}) + endif() set(${IMPL_OUTPUT_VAR} ${KEYS} PARENT_SCOPE) endfunction() diff --git a/docs/nsc-prebuilds.md b/docs/nsc-prebuilds.md index 4d57d7a8de..400aff5eb7 100644 --- a/docs/nsc-prebuilds.md +++ b/docs/nsc-prebuilds.md @@ -60,7 +60,7 @@ Keys are strings that match the output layout: - `INPUT` (string, required): path to `.hlsl` (relative to `CMAKE_CURRENT_SOURCE_DIR` or absolute). - `KEY` (string, required): base key (prefer without `.spv`; it is always appended, so using `foo.spv` will result in `foo.spv.spv`). - `COMPILE_OPTIONS` (array of strings, optional): per-input extra options (e.g. `["-T","cs_6_8"]`). -- `DEPENDS` (array of strings, optional): per-input dependencies (extra files that should trigger rebuild). +- `DEPENDS` (array of strings, optional): extra per-input dependencies that are not discovered via `#include` (see below). - `CAPS` (array, optional): permutation caps (see below). You can register many rules in a single call, and you can call the function multiple times to append rules to the same `TARGET`. @@ -87,18 +87,14 @@ The helper also exposes CMake options that append NSC debug flags **only for Deb ## Source files and rebuild dependencies (important) -Make sure shader inputs and includes are: +NSC supports depfiles and the CMake custom commands consume them, so **changes in any `#include`d HLSL file automatically trigger recompilation of the affected `.spv` outputs**. In most cases you no longer need to list includes manually. -1. Marked as header-only on your target (so the IDE shows them, but the build system doesn't try to compile them with default HLSL rules like `fxc`): +Use `DEPENDS` only for **extra** inputs that are not discovered via `#include` (e.g. a generated header that is not included, a config file read by a custom include generator, or any non-HLSL file that should trigger a rebuild). You can register those extra dependencies if you need them, but in most projects `DEPENDS` should stay empty. -```cmake -target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS}) -set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON) -``` +By default `NBL_CREATE_NSC_COMPILE_RULES` also collects `*.hlsl` files for IDE visibility. It recursively scans the current source directory (or `GLOB_DIR` if provided), adds those files as header-only, and groups them under `HLSL Files`. If you do not want this behavior, pass `DISCARD_DEFAULT_GLOB`. -2. Listed as dependencies of the NSC custom commands (so editing any of them triggers a rebuild of the `.spv` outputs). - -This is what the `DEPENDS` argument of `NBL_CREATE_NSC_COMPILE_RULES` (and/or per-input JSON `DEPENDS`) is for. Always include the main `INPUT` file itself and any files it includes; otherwise the build system might not re-run `nsc` when you change them. +- `GLOB_DIR` (optional): root directory for the default `*.hlsl` scan. +- `DISCARD_DEFAULT_GLOB` (flag): disables the default scan and IDE grouping. ## Minimal usage (no permutations) @@ -106,12 +102,6 @@ Example pattern (as in `examples_tests/27_MPMCScheduler/CMakeLists.txt`): ```cmake set(OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auto-gen") -set(DEPENDS - app_resources/common.hlsl - app_resources/shader.comp.hlsl -) -target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS}) -set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON) set(JSON [=[ [ @@ -128,7 +118,6 @@ set(JSON [=[ NBL_CREATE_NSC_COMPILE_RULES( TARGET ${EXECUTABLE_NAME}SPIRV LINK_TO ${EXECUTABLE_NAME} - DEPENDS ${DEPENDS} BINARY_DIR ${OUTPUT_DIRECTORY} MOUNT_POINT_DEFINE NBL_THIS_EXAMPLE_BUILD_MOUNT_POINT COMMON_OPTIONS -I ${CMAKE_CURRENT_SOURCE_DIR} @@ -298,6 +287,8 @@ If the error looks like a preprocessing issue, note that we use Boost.Wave as th
NSC rules + archive + runtime key usage +NSC emits depfiles and the custom commands consume them, so changes in `#include`d HLSL files automatically trigger recompilation of the affected outputs. In most cases you do not need to list includes manually. Use `DEPENDS` only for extra inputs that are not discovered via `#include`. + ### CMake (`CMakeLists.txt`) ```cmake @@ -306,12 +297,6 @@ include(common) nbl_create_executable_project("" "" "" "") set(OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auto-gen") -set(DEPENDS - app_resources/common.hlsl - app_resources/shader.hlsl -) -target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS}) -set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON) set(JSON [=[ [ @@ -319,7 +304,6 @@ set(JSON [=[ "INPUT": "app_resources/shader.hlsl", "KEY": "shader", "COMPILE_OPTIONS": ["-T", "lib_6_8"], - "DEPENDS": [], "CAPS": [ { "kind": "limits", @@ -341,7 +325,6 @@ set(JSON [=[ NBL_CREATE_NSC_COMPILE_RULES( TARGET ${EXECUTABLE_NAME}SPIRV LINK_TO ${EXECUTABLE_NAME} - DEPENDS ${DEPENDS} BINARY_DIR ${OUTPUT_DIRECTORY} MOUNT_POINT_DEFINE NBL_THIS_EXAMPLE_BUILD_MOUNT_POINT COMMON_OPTIONS -I ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/examples_tests b/examples_tests index 5df217517f..58b42cfc87 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 5df217517fd5af0964b6d170afb68d5194daf60d +Subproject commit 58b42cfc87274db606d593d5baec787b626bb945 diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 30d37f36c7..9fd4eee833 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -137,6 +137,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted const CIncludeFinder* includeFinder = nullptr; std::span extraDefines = {}; E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6; + bool depfile = false; + system::path depfilePath = {}; }; // https://github.com/microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#debugging @@ -215,6 +217,10 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Needed for json vector serialization. Making it private and declaring from_json(_, SEntry&) as friend didn't work inline SPreprocessingDependency() {} + inline const system::path& getRequestingSourceDir() const { return requestingSourceDir; } + inline std::string_view getIdentifier() const { return identifier; } + inline bool isStandardInclude() const { return standardInclude; } + private: friend void to_json(nlohmann::json& j, const SEntry::SPreprocessingDependency& dependency); friend void from_json(const nlohmann::json& j, SEntry::SPreprocessingDependency& dependency); @@ -447,6 +453,17 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const; }; + struct DepfileWriteParams + { + system::ISystem* system = nullptr; + std::string_view depfilePath = {}; + std::string_view outputPath = {}; + std::string_view sourceIdentifier = {}; + system::path workingDirectory = {}; + }; + + static bool writeDepfile(const DepfileWriteParams& params, const CCache::SEntry::dependency_container_t& dependencies, const CIncludeFinder* includeFinder = nullptr, system::logger_opt_ptr logger = nullptr); + core::smart_refctd_ptr compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const; inline core::smart_refctd_ptr compileToSPIRV(const char* code, const SCompilerOptions& options) const diff --git a/include/nbl/system/IFileBase.h b/include/nbl/system/IFileBase.h index cb0170157e..c9ceb13a04 100644 --- a/include/nbl/system/IFileBase.h +++ b/include/nbl/system/IFileBase.h @@ -25,7 +25,9 @@ class IFileBase : public core::IReferenceCounted ECF_READ_WRITE = 0b0011, ECF_MAPPABLE = 0b0100, //! Implies ECF_MAPPABLE - ECF_COHERENT = 0b1100 + ECF_COHERENT = 0b1100, + ECF_SHARE_READ_WRITE = 0b100000, + ECF_SHARE_DELETE = 0b1000000 }; //! Get size of file. diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index d36ecfa1cb..1020fa9446 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -111,12 +112,23 @@ static bool fixup_spirv_target_ver(std::vector& arguments, system: for (auto targetEnvArgumentPos=arguments.begin(); targetEnvArgumentPos!=arguments.end(); targetEnvArgumentPos++) if (targetEnvArgumentPos->find(Prefix)==0) { - const auto suffix = targetEnvArgumentPos->substr(Prefix.length()); + auto suffix = targetEnvArgumentPos->substr(Prefix.length()); + auto trim = [](std::wstring& value) { + auto isTrimChar = [](wchar_t c) { + return c == L' ' || c == L'\t' || c == L'\r' || c == L'\n' || c == L'"'; + }; + while (!value.empty() && isTrimChar(value.front())) + value.erase(value.begin()); + while (!value.empty() && isTrimChar(value.back())) + value.pop_back(); + }; + trim(suffix); const auto found = AllowedSuffices.find(suffix); if (found!=AllowedSuffices.end()) return true; - logger.log("Compile flag warning: Required compile flag not found -fspv-target-env=. Force enabling -fspv-target-env= found but with unsupported value `%s`.", system::ILogger::ELL_ERROR, "TODO: write wchar to char convert usage"); - return false; + logger.log("Compile flag warning: Required compile flag not found -fspv-target-env=. Force enabling -fspv-target-env=vulkan1.3, previous value `%ls` is unsupported.", system::ILogger::ELL_WARNING, suffix.c_str()); + *targetEnvArgumentPos = L"-fspv-target-env=vulkan1.3"; + return true; } logger.log("Compile flag warning: Required compile flag not found -fspv-target-env=. Force enabling -fspv-target-env=vulkan1.3, as it is required by Nabla.", system::ILogger::ELL_WARNING); @@ -353,6 +365,21 @@ namespace nbl::wave std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector& dxc_compile_flags_override, std::vector* dependencies) const { + const bool depfileEnabled = preprocessOptions.depfile; + if (depfileEnabled) + { + if (preprocessOptions.depfilePath.empty()) + { + preprocessOptions.logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR); + return {}; + } + } + + std::vector localDependencies; + auto* dependenciesOut = dependencies; + if (depfileEnabled && !dependenciesOut) + dependenciesOut = &localDependencies; + // HACK: we do a pre-pre-process here to add \n after every #pragma to neutralize boost::wave's actions // See https://github.com/Devsh-Graphics-Programming/Nabla/issues/746 size_t line_index = 0; @@ -367,8 +394,8 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE } // preprocess - core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependencies) /* if dependencies were passed, we assume we want caching*/, - [&dxc_compile_flags_override, &stage, &dependencies](nbl::wave::context& context) -> void + core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependenciesOut), + [&dxc_compile_flags_override, &stage, &dependenciesOut](nbl::wave::context& context) -> void { if (context.get_hooks().m_dxc_compile_flags_override.size() != 0) dxc_compile_flags_override = context.get_hooks().m_dxc_compile_flags_override; @@ -377,9 +404,8 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE if (context.get_hooks().m_pragmaStage != IShader::E_SHADER_STAGE::ESS_UNKNOWN) stage = context.get_hooks().m_pragmaStage; - if (dependencies) { - *dependencies = std::move(context.get_dependencies()); - } + if (dependenciesOut) + *dependenciesOut = std::move(context.get_dependencies()); } ); @@ -396,13 +422,29 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE } } + if (resolvedString.empty()) + return resolvedString; + + if (depfileEnabled) + { + IShaderCompiler::DepfileWriteParams params = {}; + const std::string depfilePathString = preprocessOptions.depfilePath.generic_string(); + params.depfilePath = depfilePathString; + params.sourceIdentifier = preprocessOptions.sourceIdentifier; + if (!params.sourceIdentifier.empty()) + params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path(); + params.system = m_system.get(); + if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, preprocessOptions.includeFinder, preprocessOptions.logger)) + return {}; + } + return resolvedString; } std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector* dependencies) const { std::vector extra_dxc_compile_flags = {}; - return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags); + return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags, dependencies); } core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::string_view code, const IShaderCompiler::SCompilerOptions& options, std::vector* dependencies) const diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index e60bf31b5c..a6cd95b441 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -21,40 +23,287 @@ IShaderCompiler::IShaderCompiler(core::smart_refctd_ptr&& syste m_defaultIncludeFinder = core::make_smart_refctd_ptr(core::smart_refctd_ptr(m_system)); } -core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const +bool IShaderCompiler::writeDepfile( + const DepfileWriteParams& params, + const CCache::SEntry::dependency_container_t& dependencies, + const CIncludeFinder* includeFinder, + system::logger_opt_ptr logger) { - CCache::SEntry entry; - if (options.readCache || options.writeCache) - entry = CCache::SEntry(code, options); - - if (options.readCache) - { - auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); - if (found != options.readCache->m_container.end()) - { - if (options.writeCache) - { - CCache::SEntry writeEntry = *found; - options.writeCache->insert(std::move(writeEntry)); - } - return found->decompressShader(); - } - } - - auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &entry.dependencies:nullptr); - // compute the SPIR-V shader content hash - if (retVal) - { - auto backingBuffer = retVal->getContent(); - const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); - } + std::string depfilePathString; + if (!params.depfilePath.empty()) + depfilePathString = std::string(params.depfilePath); + else + depfilePathString = std::string(params.outputPath) + ".d"; + + if (depfilePathString.empty()) + { + logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR); + return false; + } + + const auto parentDirectory = std::filesystem::path(depfilePathString).parent_path(); + if (!parentDirectory.empty() && !std::filesystem::exists(parentDirectory)) + { + if (!std::filesystem::create_directories(parentDirectory)) + { + logger.log("Failed to create parent directory for depfile.", system::ILogger::ELL_ERROR); + return false; + } + } + + std::vector depPaths; + depPaths.reserve(dependencies.size() + 1); + + auto addDepPath = [&depPaths, ¶ms](std::filesystem::path path) + { + if (path.empty()) + return; + if (path.is_relative()) + { + if (params.workingDirectory.empty()) + return; + path = std::filesystem::path(params.workingDirectory) / path; + } + std::error_code ec; + std::filesystem::path normalized = std::filesystem::weakly_canonical(path, ec); + if (ec) + { + normalized = std::filesystem::absolute(path, ec); + if (ec) + return; + } + if (normalized.empty() || !std::filesystem::exists(normalized)) + return; + auto normalizedString = normalized.generic_string(); + if (normalizedString.find_first_of("\r\n") != std::string::npos) + return; + depPaths.emplace_back(std::move(normalizedString)); + }; + + if (!params.sourceIdentifier.empty()) + { + std::filesystem::path rootPath{std::string(params.sourceIdentifier)}; + if (rootPath.is_relative()) + { + if (!params.workingDirectory.empty()) + rootPath = std::filesystem::absolute(std::filesystem::path(params.workingDirectory) / rootPath); + else + rootPath = std::filesystem::absolute(rootPath); + } + addDepPath(rootPath); + } + + for (const auto& dep : dependencies) + { + if (includeFinder) + { + IShaderCompiler::IIncludeLoader::found_t header = dep.isStandardInclude() ? + includeFinder->getIncludeStandard(dep.getRequestingSourceDir(), std::string(dep.getIdentifier())) : + includeFinder->getIncludeRelative(dep.getRequestingSourceDir(), std::string(dep.getIdentifier())); + + if (!header) + continue; + addDepPath(header.absolutePath); + } + else + { + std::filesystem::path candidate = dep.isStandardInclude() ? std::filesystem::path(std::string(dep.getIdentifier())) : (dep.getRequestingSourceDir() / std::string(dep.getIdentifier())); + if (candidate.is_relative()) + { + if (!params.workingDirectory.empty()) + candidate = std::filesystem::absolute(std::filesystem::path(params.workingDirectory) / candidate); + else + candidate = std::filesystem::absolute(candidate); + } + addDepPath(candidate); + } + } + + std::sort(depPaths.begin(), depPaths.end()); + depPaths.erase(std::unique(depPaths.begin(), depPaths.end()), depPaths.end()); + + auto escapeDepPath = [](const std::string& path) -> std::string + { + std::string normalized = path; + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + std::string out; + out.reserve(normalized.size()); + for (const char c : normalized) + { + if (c == ' ' || c == '#') + out.push_back('\\'); + if (c == '$') + { + out.push_back('$'); + out.push_back('$'); + continue; + } + out.push_back(c); + } + return out; + }; + + if (!params.system) + { + logger.log("Depfile system is null.", system::ILogger::ELL_ERROR); + return false; + } + + const auto depfilePath = std::filesystem::path(depfilePathString); + auto tempPath = depfilePath; + tempPath += ".tmp"; + params.system->deleteFile(tempPath); + + core::smart_refctd_ptr depfile; + { + system::ISystem::future_t> future; + params.system->createFile(future, tempPath, system::IFileBase::ECF_WRITE); + if (!future.wait()) + { + logger.log("Failed to open depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + future.acquire().move_into(depfile); + } + if (!depfile) + { + logger.log("Failed to open depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + + std::string targetPathString; + if (params.outputPath.empty()) + { + std::filesystem::path targetPath = depfilePathString; + if (targetPath.extension() == ".d") + targetPath.replace_extension(); + targetPathString = targetPath.generic_string(); + } + else + { + targetPathString = std::string(params.outputPath); + } + if (targetPathString.empty()) + { + logger.log("Depfile target path is empty.", system::ILogger::ELL_ERROR); + return false; + } + const std::string target = escapeDepPath(std::filesystem::path(targetPathString).generic_string()); + std::vector escapedDeps; + escapedDeps.reserve(depPaths.size()); + for (const auto& depPath : depPaths) + escapedDeps.emplace_back(escapeDepPath(depPath)); + + std::string depfileContents; + depfileContents.append(target); + depfileContents.append(":"); + if (!escapedDeps.empty()) + { + depfileContents.append(" \\\n"); + for (size_t index = 0; index < escapedDeps.size(); ++index) + { + depfileContents.append(" "); + depfileContents.append(escapedDeps[index]); + if (index + 1 < escapedDeps.size()) + depfileContents.append(" \\\n"); + } + } + depfileContents.append("\n"); + + system::IFile::success_t success; + depfile->write(success, depfileContents.data(), 0, depfileContents.size()); + if (!success) + { + logger.log("Failed to write depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + depfile = nullptr; + + params.system->deleteFile(depfilePath); + const std::error_code moveError = params.system->moveFileOrDirectory(tempPath, depfilePath); + if (moveError) + { + logger.log("Failed to replace depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + return true; +} - if (options.writeCache) - { - if (entry.setContent(retVal->getContent())) - options.writeCache->insert(std::move(entry)); - } - return retVal; +core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const +{ + const bool depfileEnabled = options.preprocessorOptions.depfile; + const bool supportsDependencies = options.getCodeContentType() == IShader::E_CONTENT_TYPE::ECT_HLSL; + + auto writeDepfileFromDependencies = [&](const CCache::SEntry::dependency_container_t& dependencies) -> bool + { + if (!depfileEnabled) + return true; + + if (options.preprocessorOptions.depfilePath.empty()) + { + options.preprocessorOptions.logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR); + return false; + } + + IShaderCompiler::DepfileWriteParams params = {}; + const std::string depfilePathString = options.preprocessorOptions.depfilePath.generic_string(); + params.depfilePath = depfilePathString; + params.sourceIdentifier = options.preprocessorOptions.sourceIdentifier; + if (!params.sourceIdentifier.empty()) + params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path(); + params.system = m_system.get(); + return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger); + }; + + CCache::SEntry entry; + if (options.readCache || options.writeCache) + entry = CCache::SEntry(code, options); + + if (options.readCache) + { + auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); + if (found != options.readCache->m_container.end()) + { + if (options.writeCache) + { + CCache::SEntry writeEntry = *found; + options.writeCache->insert(std::move(writeEntry)); + } + auto shader = found->decompressShader(); + if (depfileEnabled && !writeDepfileFromDependencies(found->dependencies)) + return nullptr; + return shader; + } + } + + CCache::SEntry::dependency_container_t depfileDependencies; + CCache::SEntry::dependency_container_t* dependenciesPtr = nullptr; + if (options.writeCache) + dependenciesPtr = &entry.dependencies; + else if (depfileEnabled && supportsDependencies) + dependenciesPtr = &depfileDependencies; + + auto retVal = compileToSPIRV_impl(code, options, dependenciesPtr); + if (retVal) + { + auto backingBuffer = retVal->getContent(); + const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); + } + + if (retVal && depfileEnabled && supportsDependencies) + { + const auto* deps = options.writeCache ? &entry.dependencies : &depfileDependencies; + if (!writeDepfileFromDependencies(*deps)) + return nullptr; + } + + if (options.writeCache) + { + if (entry.setContent(retVal->getContent())) + options.writeCache->insert(std::move(entry)); + } + + return retVal; } std::string IShaderCompiler::preprocessShader( @@ -72,7 +321,6 @@ std::string IShaderCompiler::preprocessShader( return preprocessShader(std::move(code), stage, preprocessOptions, dependencies); } - auto IShaderCompiler::IIncludeGenerator::getInclude(const std::string& includeName) const -> IIncludeLoader::found_t { core::vector> builtinNames = getBuiltinNamesToFunctionMapping(); @@ -97,7 +345,7 @@ core::vector IShaderCompiler::IIncludeGenerator::parseArgumentsFrom std::stringstream ss{ _path }; std::string arg; while (std::getline(ss, arg, '/')) - args.push_back(std::move(arg)); + args.emplace_back(std::move(arg)); return args; } @@ -178,7 +426,7 @@ void IShaderCompiler::CIncludeFinder::addSearchPath(const std::string& searchPat { if (!loader) return; - m_loaders.push_back(LoaderSearchPath{ loader, searchPath }); + m_loaders.emplace_back(LoaderSearchPath{ loader, searchPath }); } void IShaderCompiler::CIncludeFinder::addGenerator(const core::smart_refctd_ptr& generatorToAdd) @@ -301,7 +549,7 @@ core::smart_refctd_ptr IShaderCompiler::CCache::serialize() const size_t i = 0u; for (auto& entry : m_container) { // Add the entry as a json array - entries.push_back(entry); + entries.emplace_back(entry); // We keep a copy of the offsets and the sizes of each shader. This is so that later on, when we add the shaders to the buffer after json creation // (where the params array has been moved) we don't have to read the json to get the offsets again diff --git a/src/nbl/system/CSystemWin32.cpp b/src/nbl/system/CSystemWin32.cpp index cab809c145..2798b4fb27 100644 --- a/src/nbl/system/CSystemWin32.cpp +++ b/src/nbl/system/CSystemWin32.cpp @@ -43,6 +43,11 @@ core::smart_refctd_ptr CSystemWin32::CCaller::createFile(const std: { const bool writeAccess = flags.value&IFile::ECF_WRITE; const DWORD fileAccess = ((flags.value&IFile::ECF_READ) ? FILE_GENERIC_READ:0)|(writeAccess ? FILE_GENERIC_WRITE:0); + DWORD shareMode = FILE_SHARE_READ; + if (flags.value & IFile::ECF_SHARE_READ_WRITE) + shareMode |= FILE_SHARE_WRITE; + if (flags.value & IFile::ECF_SHARE_DELETE) + shareMode |= FILE_SHARE_DELETE; SECURITY_ATTRIBUTES secAttribs{ sizeof(SECURITY_ATTRIBUTES), nullptr, FALSE }; @@ -51,8 +56,8 @@ core::smart_refctd_ptr CSystemWin32::CCaller::createFile(const std: p.make_preferred(); // Replace "/" separators with "\" // only write access should create new files if they don't exist - const auto creationDisposition = writeAccess ? OPEN_ALWAYS:OPEN_EXISTING; - HANDLE _native = CreateFileA(p.string().data(), fileAccess, FILE_SHARE_READ, &secAttribs, creationDisposition, FILE_ATTRIBUTE_NORMAL, nullptr); + const auto creationDisposition = writeAccess ? OPEN_ALWAYS : OPEN_EXISTING; + HANDLE _native = CreateFileA(p.string().data(), fileAccess, shareMode, &secAttribs, creationDisposition, FILE_ATTRIBUTE_NORMAL, nullptr); if (_native==INVALID_HANDLE_VALUE) { auto e = GetLastError(); @@ -107,4 +112,4 @@ bool isDebuggerAttached() return IsDebuggerPresent(); } -#endif \ No newline at end of file +#endif diff --git a/tools/nsc/CMakeLists.txt b/tools/nsc/CMakeLists.txt index bcdcbca531..2765f02fa5 100644 --- a/tools/nsc/CMakeLists.txt +++ b/tools/nsc/CMakeLists.txt @@ -2,6 +2,9 @@ nbl_create_executable_project("" "" "" "") enable_testing() +add_dependencies(${EXECUTABLE_NAME} argparse) +target_include_directories(${EXECUTABLE_NAME} PRIVATE $) + set(GODBOLT_BINARY_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/compiler-explorer") set(GODBOLT_BINARY_PRETEST_DIRECTORY "${GODBOLT_BINARY_DIRECTORY}/pre-test") set(NBL_NSC_COMPILE_DIRECTORY "${GODBOLT_BINARY_PRETEST_DIRECTORY}/.compile/$") @@ -359,4 +362,4 @@ add_custom_target(run-compiler-explorer ALL add_dependencies(run-compiler-explorer nsc) set_target_properties(run-compiler-explorer PROPERTIES FOLDER "Godbolt") -endif() \ No newline at end of file +endif() diff --git a/tools/nsc/main.cpp b/tools/nsc/main.cpp index edc56de84c..5ab01d72e5 100644 --- a/tools/nsc/main.cpp +++ b/tools/nsc/main.cpp @@ -1,403 +1,554 @@ #include "nabla.h" #include "nbl/system/IApplicationFramework.h" - #include #include +#include #include #include - +#include +#include +#include +#include +#include +#include +#include +#include #include "nbl/asset/metadata/CHLSLMetadata.h" #include "nlohmann/json.hpp" -using json = nlohmann::json; +using json = nlohmann::json; using namespace nbl; using namespace nbl::system; using namespace nbl::core; using namespace nbl::asset; -class ShaderCompiler final : public system::IApplicationFramework +class TrimStdoutLogger final : public CStdoutLogger { - using base_t = system::IApplicationFramework; - public: - using base_t::base_t; - - bool onAppInitialized(smart_refctd_ptr&& system) override - { - const auto argc = argv.size(); - const bool insufficientArguments = argc < 2; - - if (not insufficientArguments) - { - // 1) NOTE: imo each example should be able to dump build info & have such mode, maybe it could go straight to IApplicationFramework main - // 2) TODO: this whole "serialize" logic should go to the GitInfo struct and be static or something, it should be standardized - - if (argv[1] == "--dump-build-info") - { - json j; - - auto& modules = j["modules"]; - - auto serialize = [&](const gtml::GitInfo& info, std::string_view target) -> void - { - auto& s = modules[target.data()]; - - s["isPopulated"] = info.isPopulated; - if (info.hasUncommittedChanges.has_value()) - s["hasUncommittedChanges"] = info.hasUncommittedChanges.value(); - else - s["hasUncommittedChanges"] = "UNKNOWN, BUILT WITHOUT DIRTY-CHANGES CAPTURE"; - - s["commitAuthorName"] = info.commitAuthorName; - s["commitAuthorEmail"] = info.commitAuthorEmail; - s["commitHash"] = info.commitHash; - s["commitShortHash"] = info.commitShortHash; - s["commitDate"] = info.commitDate; - s["commitSubject"] = info.commitSubject; - s["commitBody"] = info.commitBody; - s["describe"] = info.describe; - s["branchName"] = info.branchName; - s["latestTag"] = info.latestTag; - s["latestTagName"] = info.latestTagName; - }; - - serialize(gtml::nabla_git_info, "nabla"); - serialize(gtml::dxc_git_info, "dxc"); - - const auto pretty = j.dump(4); - std::cout << pretty << std::endl; - - std::filesystem::path oPath = "build-info.json"; - - // TOOD: use argparse for it - if (argc > 3 && argv[2] == "--file") - oPath = argv[3]; - - std::ofstream outFile(oPath); - if (outFile.is_open()) - { - outFile << pretty; - outFile.close(); - printf("Saved \"%s\"\n", oPath.string().c_str()); - } - else - { - printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); - exit(-1); - } - - // in this mode terminate with 0 if all good - exit(0); - } - } - - if (not isAPILoaded()) - { - std::cerr << "Could not load Nabla API, terminating!"; - return false; - } - - if (system) - m_system = std::move(system); - else - m_system = system::IApplicationFramework::createSystem(); - - if (!m_system) - return false; - - m_logger = make_smart_refctd_ptr(core::bitflag(ILogger::ELL_DEBUG) | ILogger::ELL_INFO | ILogger::ELL_WARNING | ILogger::ELL_PERFORMANCE | ILogger::ELL_ERROR); - - if (insufficientArguments) - { - m_logger->log("Insufficient arguments.", ILogger::ELL_ERROR); - return false; - } - - m_arguments = std::vector(argv.begin() + 1, argv.end()-1); // turn argv into vector for convenience - - std::string file_to_compile = argv.back(); - - if (!m_system->exists(file_to_compile, IFileBase::ECF_READ)) { - m_logger->log("Incorrect arguments. Expecting last argument to be filename of the shader intended to compile.", ILogger::ELL_ERROR); - return false; - } - std::string output_filepath = ""; - - auto builtin_flag_pos = std::find(m_arguments.begin(), m_arguments.end(), "-no-nbl-builtins"); - if (builtin_flag_pos != m_arguments.end()) { - m_logger->log("Unmounting builtins."); - m_system->unmountBuiltins(); - no_nbl_builtins = true; - m_arguments.erase(builtin_flag_pos); - } - - auto split = [&](const std::string& str, char delim) - { - std::vector strings; - size_t start, end = 0; - - while ((start = str.find_first_not_of(delim, end)) != std::string::npos) - { - end = str.find(delim, start); - strings.push_back(str.substr(start, end - start)); - } - - return strings; - }; - - auto findOutputFlag = [&](const std::string_view& outputFlag) - { - return std::find_if(m_arguments.begin(), m_arguments.end(), [&](const std::string& argument) - { - return argument.find(outputFlag.data()) != std::string::npos; - }); - }; - - auto preprocessOnly = findOutputFlag("-P") != m_arguments.end(); - auto output_flag_pos_fc = findOutputFlag("-Fc"); - auto output_flag_pos_fo = findOutputFlag("-Fo"); - if (output_flag_pos_fc != m_arguments.end() && output_flag_pos_fo != m_arguments.end()) { - m_logger->log("Invalid arguments. Passed both -Fo and -Fc.", ILogger::ELL_ERROR); - return false; - } - auto output_flag_pos = output_flag_pos_fc != m_arguments.end() ? output_flag_pos_fc : output_flag_pos_fo; - if (output_flag_pos == m_arguments.end()) - { - m_logger->log("Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.", ILogger::ELL_ERROR); - return false; - } - else - { - // we need to assume -Fc may be passed with output file name quoted together with "", so we split it (DXC does it) - const auto& outputFlag = *output_flag_pos; - auto outputFlagVector = split(outputFlag, ' '); - - if(outputFlag == "-Fc" || outputFlag == "-Fo") - { - if (output_flag_pos + 1 != m_arguments.end()) - { - output_filepath = *(output_flag_pos + 1); - } - else - { - m_logger->log("Incorrect arguments. Expecting filename after %s.", ILogger::ELL_ERROR, outputFlag); - return false; - } - } - else - { - output_filepath = outputFlagVector[1]; - } - m_arguments.erase(output_flag_pos, output_flag_pos+2); - - if (output_filepath.empty()) - { - m_logger->log("Invalid output file path!" + output_filepath, ILogger::ELL_ERROR); - return false; - } - - std::string outputType = preprocessOnly ? "Preprocessed" : "Compiled"; - m_logger->log(outputType + " shader code will be saved to " + output_filepath, ILogger::ELL_INFO); - } + TrimStdoutLogger(const bitflag logLevelMask) : CStdoutLogger(logLevelMask) {} + +protected: + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto str = constructLogString(fmt, logLevel, args); + size_t size = str.size(); + while (size && str[size - 1] == '\0') + --size; + if (!size) + return; + std::fwrite(str.data(), 1, size, stdout); + std::fflush(stdout); + } +}; -#ifndef NBL_EMBED_BUILTIN_RESOURCES - if (!no_nbl_builtins) { - m_system->unmountBuiltins(); - no_nbl_builtins = true; - m_logger->log("nsc.exe was compiled with builtin resources disabled. Force enabling -no-nbl-builtins.", ILogger::ELL_WARNING); - } -#endif - if (std::find(m_arguments.begin(), m_arguments.end(), "-E") == m_arguments.end()) - { - //Insert '-E main' into arguments if no entry point is specified - m_arguments.push_back("-E"); - m_arguments.push_back("main"); - } - - for (size_t i = 0; i < m_arguments.size() - 1; ++i) // -I must be given with second arg, no need to include iteration over last one - { - const auto& arg = m_arguments[i]; - if (arg == "-I") - m_include_search_paths.emplace_back(m_arguments[i + 1]); - } - - auto [shader, shaderStage] = open_shader_file(file_to_compile); - if (shader->getContentType() != IShader::E_CONTENT_TYPE::ECT_HLSL) - { - m_logger->log("Error. Loaded shader file content is not HLSL.", ILogger::ELL_ERROR); - return false; - } - - auto start = std::chrono::high_resolution_clock::now(); - smart_refctd_ptr compilation_result; - std::string preprocessing_result; - std::string_view result_view; - if (preprocessOnly) - { - preprocessing_result = preprocess_shader(shader.get(), shaderStage, file_to_compile); - result_view = preprocessing_result; - } - else - { - compilation_result = compile_shader(shader.get(), shaderStage, file_to_compile); - result_view = { (const char*)compilation_result->getContent()->getPointer(), compilation_result->getContent()->getSize() }; - } - auto end = std::chrono::high_resolution_clock::now(); - - // write compiled/preprocessed shader to file as bytes - std::string operationType = preprocessOnly ? "preprocessing" : "compilation"; - const bool success = preprocessOnly ? preprocessing_result != std::string{} : bool(compilation_result); - if (success) - { - m_logger->log("Shader " + operationType + " successful.", ILogger::ELL_INFO); - const auto took = std::to_string(std::chrono::duration_cast(end - start).count()); - m_logger->log("Took %s ms.", ILogger::ELL_PERFORMANCE, took.c_str()); - { - const auto location = std::filesystem::path(output_filepath); - const auto parentDirectory = location.parent_path(); - - if (!std::filesystem::exists(parentDirectory)) - { - if (!std::filesystem::create_directories(parentDirectory)) - { - m_logger->log("Failed to create parent directory for the " + output_filepath + "output!", ILogger::ELL_ERROR); - return false; - } - } - } - - std::fstream output_file(output_filepath, std::ios::out | std::ios::binary); - - if (!output_file.is_open()) - { - m_logger->log("Failed to open output file: " + output_filepath, ILogger::ELL_ERROR); - return false; - } - - output_file.write(result_view.data(), result_view.size()); - - if (output_file.fail()) - { - m_logger->log("Failed to write to output file: " + output_filepath, ILogger::ELL_ERROR); - output_file.close(); - return false; - } - - output_file.close(); - - if (output_file.fail()) - { - m_logger->log("Failed to close output file: " + output_filepath, ILogger::ELL_ERROR); - return false; - } - - return true; - } - else - { - m_logger->log("Shader " + operationType + " failed.", ILogger::ELL_ERROR); - return false; - } - } - - void workLoopBody() override {} - - bool keepRunning() override { return false; } +class TrimFileLogger final : public CFileLogger +{ +public: + using CFileLogger::CFileLogger; + +protected: + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto str = constructLogString(fmt, logLevel, args); + size_t size = str.size(); + while (size && str[size - 1] == '\0') + --size; + if (!size) + return; + IFile::success_t succ; + m_file->write(succ, str.data(), m_pos, size); + m_pos += succ.getBytesProcessed(); + } +}; +class ShaderLogger final : public IThreadsafeLogger +{ +public: + ShaderLogger(smart_refctd_ptr system, path logPath, const bitflag fileMask, const bitflag consoleMask, const bool noLog) + : IThreadsafeLogger(fileMask | consoleMask), m_system(std::move(system)), m_logPath(std::move(logPath)), m_fileMask(fileMask), m_consoleMask(consoleMask), m_noLog(noLog) + { + m_stdoutLogger = make_smart_refctd_ptr(m_consoleMask); + beginBuild(); + } + + void beginBuild() + { + m_fileLogger = nullptr; + m_file = nullptr; + + if (m_noLog) + return; + if (!m_system || m_logPath.empty()) + return; + + const auto parent = std::filesystem::path(m_logPath).parent_path(); + if (!parent.empty() && !std::filesystem::exists(parent)) + std::filesystem::create_directories(parent); + + for (auto attempt = 0u; attempt < kDeleteRetries; ++attempt) + { + if (m_system->deleteFile(m_logPath)) + break; + std::this_thread::sleep_for(kDeleteDelay); + } + + ISystem::future_t> fut; + m_system->createFile(fut, m_logPath, kLogFlags); + + if (fut.wait()) + { + auto lk = fut.acquire(); + if (lk) + lk.move_into(m_file); + } + + if (!m_file) + return; + + m_fileLogger = make_smart_refctd_ptr(smart_refctd_ptr(m_file), true, m_fileMask); + } private: + static constexpr auto kDeleteRetries = 3u; + static constexpr auto kDeleteDelay = std::chrono::milliseconds(100); + static constexpr auto kLogFlags = bitflag(IFileBase::ECF_WRITE) | IFileBase::ECF_SHARE_READ_WRITE | IFileBase::ECF_SHARE_DELETE; + + static inline std::string formatMessageOnly(const std::string_view& fmt, va_list args) + { + va_list a; + va_copy(a, args); + const int n = std::vsnprintf(nullptr, 0, fmt.data(), a); + va_end(a); + if (n <= 0) + return {}; + std::string s(size_t(n) + 1u, '\0'); + std::vsnprintf(s.data(), s.size(), fmt.data(), args); + s.resize(size_t(n)); + return s; + } + + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto msg = formatMessageOnly(fmt, args); + if (msg.empty()) + return; + + if (m_stdoutLogger && (logLevel & m_consoleMask.value)) + m_stdoutLogger->log("%s", logLevel, msg.c_str()); + + if (m_noLog || !(logLevel & m_fileMask.value) || !m_fileLogger) + return; + + m_fileLogger->log("%s", logLevel, msg.c_str()); + } + + smart_refctd_ptr m_system; + smart_refctd_ptr m_file; + smart_refctd_ptr m_stdoutLogger; + smart_refctd_ptr m_fileLogger; + path m_logPath; + bitflag m_fileMask; + bitflag m_consoleMask; + bool m_noLog = false; +}; - std::string preprocess_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier) { - smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - - CHLSLCompiler::SPreprocessorOptions options = {}; - options.sourceIdentifier = sourceIdentifier; - options.logger = m_logger.get(); - - auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - auto includeLoader = includeFinder->getDefaultFileSystemLoader(); - - // because before real compilation we do preprocess the input it doesn't really matter we proxy include search direcotries further with dxcOptions since at the end all includes are resolved to single file - for (const auto& it : m_include_search_paths) - includeFinder->addSearchPath(it, includeLoader); - - options.includeFinder = includeFinder.get(); - - const char* code_ptr = (const char*)shader->getContent()->getPointer(); - std::string_view code({ code_ptr, strlen(code_ptr)}); - - return hlslcompiler->preprocessShader(std::string(code), shaderStage, options, nullptr); - } - - core::smart_refctd_ptr compile_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier) { - smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - - CHLSLCompiler::SOptions options = {}; - options.stage = shaderStage; - options.preprocessorOptions.sourceIdentifier = sourceIdentifier; - options.preprocessorOptions.logger = m_logger.get(); - - options.debugInfoFlags = core::bitflag(asset::IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT); - options.dxcOptions = std::span(m_arguments); - - auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - auto includeLoader = includeFinder->getDefaultFileSystemLoader(); - - // because before real compilation we do preprocess the input it doesn't really matter we proxy include search direcotries further with dxcOptions since at the end all includes are resolved to single file - for(const auto& it : m_include_search_paths) - includeFinder->addSearchPath(it, includeLoader); - - options.preprocessorOptions.includeFinder = includeFinder.get(); - - return hlslcompiler->compileToSPIRV((const char*)shader->getContent()->getPointer(), options); - } - - - std::tuple, hlsl::ShaderStage> open_shader_file(std::string filepath) { - - m_assetMgr = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - - IAssetLoader::SAssetLoadParams lp = {}; - lp.logger = m_logger.get(); - lp.workingDirectory = localInputCWD; - auto assetBundle = m_assetMgr->getAsset(filepath, lp); - const auto assets = assetBundle.getContents(); - const auto* metadata = assetBundle.getMetadata(); - if (assets.empty()) { - m_logger->log("Could not load shader %s", ILogger::ELL_ERROR, filepath); - return {nullptr, hlsl::ShaderStage::ESS_UNKNOWN}; - } - assert(assets.size() == 1); - - // could happen when the file is missing an extension and we can't deduce its a shader - if (assetBundle.getAssetType() == IAsset::ET_BUFFER) - { - auto buf = IAsset::castDown(assets[0]); - std::string source; source.resize(buf->getSize()+1); - memcpy(source.data(),buf->getPointer(),buf->getSize()); - return { core::make_smart_refctd_ptr(source.data(), IShader::E_CONTENT_TYPE::ECT_HLSL, std::move(filepath)), hlsl::ShaderStage::ESS_UNKNOWN}; - } - else if (assetBundle.getAssetType() == IAsset::ET_SHADER) - { - const auto hlslMetadata = static_cast(metadata); - return { smart_refctd_ptr_static_cast(assets[0]), hlslMetadata->shaderStages->front()}; - } - else - { - m_logger->log("file '%s' is an asset that is neither a buffer or a shader.", ILogger::ELL_ERROR, filepath); - } - - return {nullptr, hlsl::ShaderStage::ESS_UNKNOWN}; - } +class ShaderCompiler final : public IApplicationFramework +{ + using base_t = IApplicationFramework; +public: + using base_t::base_t; + + bool onAppInitialized(smart_refctd_ptr&& system) override + { + const auto rawArgs = std::vector(argv.begin(), argv.end()); + const auto expandedArgs = expandJoinedArgs(rawArgs); + + argparse::ArgumentParser program("nsc"); + program.add_argument("--dump-build-info").default_value(false).implicit_value(true); + program.add_argument("--file").default_value(std::string{}); + program.add_argument("-P").default_value(false).implicit_value(true); + program.add_argument("-no-nbl-builtins").default_value(false).implicit_value(true); + program.add_argument("-MD").default_value(false).implicit_value(true); + program.add_argument("-M").default_value(false).implicit_value(true); + program.add_argument("-MF").default_value(std::string{}); + program.add_argument("-Fo").default_value(std::string{}); + program.add_argument("-Fc").default_value(std::string{}); + program.add_argument("-log").default_value(std::string{}); + program.add_argument("-nolog").default_value(false).implicit_value(true); + program.add_argument("-quiet").default_value(false).implicit_value(true); + program.add_argument("-verbose").default_value(false).implicit_value(true); + + std::vector unknownArgs; + try + { + unknownArgs = program.parse_known_args(expandedArgs); + } + catch (const std::runtime_error& err) + { + std::cerr << err.what() << std::endl << program; + return false; + } + + if (program.get("--dump-build-info")) + { + dumpBuildInfo(program); + std::exit(0); + } + + if (!isAPILoaded()) + { + std::cerr << "Could not load Nabla API, terminating!"; + return false; + } + + m_system = system ? std::move(system) : IApplicationFramework::createSystem(); + if (!m_system) + return false; + + if (rawArgs.size() < 2) + { + std::cerr << "Insufficient arguments.\n"; + return false; + } + + const std::string fileToCompile = rawArgs.back(); + if (!m_system->exists(fileToCompile, IFileBase::ECF_READ)) + { + std::cerr << "Input shader file does not exist: " << fileToCompile << "\n"; + return false; + } + + const bool preprocessOnly = program.get("-P"); + const bool hasFc = program.is_used("-Fc"); + const bool hasFo = program.is_used("-Fo"); + + if (hasFc == hasFo) + { + if (hasFc) + std::cerr << "Invalid arguments. Passed both -Fo and -Fc.\n"; + else + std::cerr << "Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.\n"; + return false; + } + + const std::string outputFilepath = hasFc ? program.get("-Fc") : program.get("-Fo"); + if (outputFilepath.empty()) + { + std::cerr << "Invalid output file path.\n"; + return false; + } + + const bool quiet = program.get("-quiet"); + const bool verbose = program.get("-verbose"); + if (quiet && verbose) + { + std::cerr << "Invalid arguments. Passed both -quiet and -verbose.\n"; + return false; + } + + const bool noLog = program.get("-nolog"); + const std::string logPathOverride = program.is_used("-log") ? program.get("-log") : std::string{}; + if (noLog && !logPathOverride.empty()) + { + std::cerr << "Invalid arguments. Passed both -nolog and -log.\n"; + return false; + } + + const auto logPath = logPathOverride.empty() ? std::filesystem::path(outputFilepath).concat(".log") : std::filesystem::path(logPathOverride); + const auto fileMask = bitflag(ILogger::ELL_ALL); + const auto consoleMask = bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR; + + m_logger = make_smart_refctd_ptr(m_system, logPath, fileMask, consoleMask, noLog); + + m_arguments = std::move(unknownArgs); + if (!m_arguments.empty() && m_arguments.back() == fileToCompile) + m_arguments.pop_back(); + + bool noNblBuiltins = program.get("-no-nbl-builtins"); + if (noNblBuiltins) + { + m_logger->log("Unmounting builtins."); + m_system->unmountBuiltins(); + } + + DepfileConfig dep; + if (program.get("-MD") || program.get("-M") || program.is_used("-MF")) + dep.enabled = true; + if (program.is_used("-MF")) + dep.path = program.get("-MF"); + if (dep.enabled && dep.path.empty()) + dep.path = outputFilepath + ".d"; + if (dep.enabled) + m_logger->log("Dependency file will be saved to %s", ILogger::ELL_INFO, dep.path.c_str()); - bool no_nbl_builtins{ false }; - smart_refctd_ptr m_system; - smart_refctd_ptr m_logger; - std::vector m_arguments, m_include_search_paths; - core::smart_refctd_ptr m_assetMgr; +#ifndef NBL_EMBED_BUILTIN_RESOURCES + if (!noNblBuiltins) + { + m_system->unmountBuiltins(); + noNblBuiltins = true; + m_logger->log("nsc.exe was compiled with builtin resources disabled. Force enabling -no-nbl-builtins.", ILogger::ELL_WARNING); + } +#endif + if (std::find(m_arguments.begin(), m_arguments.end(), "-E") == m_arguments.end()) + { + m_arguments.push_back("-E"); + m_arguments.push_back("main"); + } + + for (size_t i = 0; i + 1 < m_arguments.size(); ++i) + { + if (m_arguments[i] == "-I") + m_include_search_paths.emplace_back(m_arguments[i + 1]); + } + + const char* const action = preprocessOnly ? "Preprocessing" : "Compiling"; + const char* const outType = preprocessOnly ? "Preprocessed" : "Compiled"; + m_logger->log("%s %s", ILogger::ELL_INFO, action, fileToCompile.c_str()); + m_logger->log("%s shader code will be saved to %s", ILogger::ELL_INFO, outType, outputFilepath.c_str()); + + auto [shader, shaderStage] = open_shader_file(fileToCompile); + if (!shader || shader->getContentType() != IShader::E_CONTENT_TYPE::ECT_HLSL) + { + m_logger->log("Error. Loaded shader file content is not HLSL.", ILogger::ELL_ERROR); + return false; + } + + const auto start = std::chrono::high_resolution_clock::now(); + const auto job = runShaderJob(shader.get(), shaderStage, fileToCompile, dep, preprocessOnly); + const auto end = std::chrono::high_resolution_clock::now(); + + const char* const op = preprocessOnly ? "preprocessing" : "compilation"; + if (!job.ok) + { + m_logger->log("Shader %s failed.", ILogger::ELL_ERROR, op); + return false; + } + + const auto took = std::to_string(std::chrono::duration_cast(end - start).count()); + m_logger->log("Shader %s successful.", ILogger::ELL_INFO, op); + m_logger->log("Took %s ms.", ILogger::ELL_PERFORMANCE, took.c_str()); + + const auto outParent = std::filesystem::path(outputFilepath).parent_path(); + if (!outParent.empty() && !std::filesystem::exists(outParent)) + { + if (!std::filesystem::create_directories(outParent)) + { + m_logger->log("Failed to create parent directory for output %s.", ILogger::ELL_ERROR, outputFilepath.c_str()); + return false; + } + } + + std::fstream out(outputFilepath, std::ios::out | std::ios::binary); + if (!out.is_open()) + { + m_logger->log("Failed to open output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + return false; + } + + out.write(job.view.data(), job.view.size()); + if (out.fail()) + { + m_logger->log("Failed to write to output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + out.close(); + return false; + } + + out.close(); + if (out.fail()) + { + m_logger->log("Failed to close output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + return false; + } + + if (dep.enabled) + m_logger->log("Dependency file written to %s", ILogger::ELL_INFO, dep.path.c_str()); + + return true; + } + + void workLoopBody() override {} + bool keepRunning() override { return false; } +private: + struct DepfileConfig + { + bool enabled = false; + std::string path; + }; + + struct RunResult + { + bool ok = false; + std::string text; + smart_refctd_ptr compiled; + std::string_view view; + }; + + static std::vector expandJoinedArgs(const std::vector& args) + { + std::vector out; + out.reserve(args.size()); + + auto split = [&](const std::string& a, const char* p) + { + const size_t n = std::strlen(p); + if (a.rfind(p, 0) == 0 && a.size() > n) + { + out.emplace_back(p); + out.emplace_back(a.substr(n)); + return true; + } + return false; + }; + + for (const auto& a : args) + { + if (split(a, "-MF")) continue; + if (split(a, "-Fo")) continue; + if (split(a, "-Fc")) continue; + out.push_back(a); + } + + return out; + } + + static void dumpBuildInfo(const argparse::ArgumentParser& program) + { + json j; + auto& modules = j["modules"]; + + auto serialize = [&](const gtml::GitInfo& info, std::string_view target) + { + auto& s = modules[target.data()]; + s["isPopulated"] = info.isPopulated; + s["hasUncommittedChanges"] = info.hasUncommittedChanges.has_value() ? json(info.hasUncommittedChanges.value()) : json("UNKNOWN, BUILT WITHOUT DIRTY-CHANGES CAPTURE"); + s["commitAuthorName"] = info.commitAuthorName; + s["commitAuthorEmail"] = info.commitAuthorEmail; + s["commitHash"] = info.commitHash; + s["commitShortHash"] = info.commitShortHash; + s["commitDate"] = info.commitDate; + s["commitSubject"] = info.commitSubject; + s["commitBody"] = info.commitBody; + s["describe"] = info.describe; + s["branchName"] = info.branchName; + s["latestTag"] = info.latestTag; + s["latestTagName"] = info.latestTagName; + }; + + serialize(gtml::nabla_git_info, "nabla"); + serialize(gtml::dxc_git_info, "dxc"); + + const auto pretty = j.dump(4); + std::cout << pretty << std::endl; + + std::filesystem::path oPath = "build-info.json"; + if (program.is_used("--file")) + { + const auto filePath = program.get("--file"); + if (!filePath.empty()) + oPath = filePath; + } + + std::ofstream outFile(oPath); + if (!outFile.is_open()) + { + std::printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); + std::exit(-1); + } + + outFile << pretty; + std::printf("Saved \"%s\"\n", oPath.string().c_str()); + } + + RunResult runShaderJob(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& dep, const bool preprocessOnly) + { + RunResult r; + auto hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + + auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + auto includeLoader = includeFinder->getDefaultFileSystemLoader(); + for (const auto& p : m_include_search_paths) + includeFinder->addSearchPath(p, includeLoader); + + if (preprocessOnly) + { + CHLSLCompiler::SPreprocessorOptions opt = {}; + opt.sourceIdentifier = sourceIdentifier; + opt.logger = m_logger.get(); + opt.includeFinder = includeFinder.get(); + opt.depfile = dep.enabled; + opt.depfilePath = dep.path; + + const char* codePtr = (const char*)shader->getContent()->getPointer(); + std::string_view code(codePtr, std::strlen(codePtr)); + + r.text = hlslcompiler->preprocessShader(std::string(code), shaderStage, opt, nullptr); + r.ok = !r.text.empty(); + r.view = r.text; + return r; + } + + CHLSLCompiler::SOptions opt = {}; + opt.stage = shaderStage; + opt.preprocessorOptions.sourceIdentifier = sourceIdentifier; + opt.preprocessorOptions.logger = m_logger.get(); + opt.preprocessorOptions.includeFinder = includeFinder.get(); + opt.preprocessorOptions.depfile = dep.enabled; + opt.preprocessorOptions.depfilePath = dep.path; + opt.debugInfoFlags = bitflag(IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT); + opt.dxcOptions = std::span(m_arguments); + + r.compiled = hlslcompiler->compileToSPIRV((const char*)shader->getContent()->getPointer(), opt); + r.ok = bool(r.compiled); + if (r.ok) + r.view = { (const char*)r.compiled->getContent()->getPointer(), r.compiled->getContent()->getSize() }; + + return r; + } + + std::tuple, hlsl::ShaderStage> open_shader_file(std::string filepath) + { + m_assetMgr = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + + IAssetLoader::SAssetLoadParams lp = {}; + lp.logger = m_logger.get(); + lp.workingDirectory = localInputCWD; + + auto bundle = m_assetMgr->getAsset(filepath, lp); + const auto assets = bundle.getContents(); + const auto* metadata = bundle.getMetadata(); + + if (assets.empty()) + { + m_logger->log("Could not load shader %s", ILogger::ELL_ERROR, filepath.c_str()); + return { nullptr, hlsl::ShaderStage::ESS_UNKNOWN }; + } + + if (bundle.getAssetType() == IAsset::ET_BUFFER) + { + auto buf = IAsset::castDown(assets[0]); + std::string source; + source.resize(buf->getSize() + 1); + std::memcpy(source.data(), buf->getPointer(), buf->getSize()); + return { make_smart_refctd_ptr(source.data(), IShader::E_CONTENT_TYPE::ECT_HLSL, std::move(filepath)), hlsl::ShaderStage::ESS_UNKNOWN }; + } + + if (bundle.getAssetType() == IAsset::ET_SHADER) + { + const auto hlslMetadata = static_cast(metadata); + return { smart_refctd_ptr_static_cast(assets[0]), hlslMetadata->shaderStages->front() }; + } + + m_logger->log("file '%s' is an asset that is neither a buffer or a shader.", ILogger::ELL_ERROR, filepath.c_str()); + return { nullptr, hlsl::ShaderStage::ESS_UNKNOWN }; + } + + smart_refctd_ptr m_system; + smart_refctd_ptr m_logger; + std::vector m_arguments, m_include_search_paths; + smart_refctd_ptr m_assetMgr; }; NBL_MAIN_FUNC(ShaderCompiler)