From 6101e5b37a55835a981170576e6256bdf9e642b8 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Fri, 2 Jan 2026 21:33:03 +0100 Subject: [PATCH 1/8] add support for depfiles & use with NSC --- cmake/common.cmake | 15 +- include/nbl/asset/utils/IShaderCompiler.h | 17 ++ src/nbl/asset/utils/CHLSLCompiler.cpp | 61 ++++- src/nbl/asset/utils/IShaderCompiler.cpp | 291 +++++++++++++++++++--- tools/nsc/main.cpp | 63 ++++- 5 files changed, 397 insertions(+), 50 deletions(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index 9836fa7666..0d8b583f18 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1142,6 +1142,7 @@ option(NSC_DEBUG_EDIF_SOURCE_BIT "Add \"-fspv-debug=source\" to NSC Debug CLI" O option(NSC_DEBUG_EDIF_LINE_BIT "Add \"-fspv-debug=line\" to NSC Debug CLI" OFF) option(NSC_DEBUG_EDIF_TOOL_BIT "Add \"-fspv-debug=tool\" to NSC Debug CLI" ON) option(NSC_DEBUG_EDIF_NON_SEMANTIC_BIT "Add \"-fspv-debug=vulkan-with-source\" to NSC Debug CLI" OFF) +option(NSC_USE_DEPFILE "Generate depfiles for NSC custom commands" ON) function(NBL_CREATE_NSC_COMPILE_RULES) set(COMMENT "this code has been autogenerated with Nabla CMake NBL_CREATE_HLSL_COMPILE_RULES utility") @@ -1479,21 +1480,33 @@ namespace @IMPL_NAMESPACE@ { # generate keys and commands for compiling shaders set(FINAL_KEY_REL_PATH "$/${FINAL_KEY}") set(TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY_REL_PATH}") + set(DEPFILE_PATH "${TARGET_OUTPUT}.d") + + set(NBL_NSC_DEPFILE_ARGS "") + if(NSC_USE_DEPFILE) + set(NBL_NSC_DEPFILE_ARGS -MD -MF "${DEPFILE_PATH}") + endif() set(NBL_NSC_COMPILE_COMMAND "$" -Fc "${TARGET_OUTPUT}" ${COMPILE_OPTIONS} ${REQUIRED_OPTIONS} ${IMPL_COMMON_OPTIONS} + ${NBL_NSC_DEPFILE_ARGS} "${CONFIG_FILE}" ) - add_custom_command(OUTPUT "${TARGET_OUTPUT}" + set(NBL_NSC_CUSTOM_COMMAND_ARGS + OUTPUT "${TARGET_OUTPUT}" COMMAND ${NBL_NSC_COMPILE_COMMAND} DEPENDS ${DEPENDS_ON} COMMENT "Creating \"${TARGET_OUTPUT}\"" VERBATIM COMMAND_EXPAND_LISTS ) + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_CUSTOM_COMMAND_ARGS DEPFILE "${DEPFILE_PATH}") + endif() + add_custom_command(${NBL_NSC_CUSTOM_COMMAND_ARGS}) set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES GENERATED TRUE) set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" "${TARGET_OUTPUT}") diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 30d37f36c7..9fd4eee833 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -137,6 +137,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted const CIncludeFinder* includeFinder = nullptr; std::span extraDefines = {}; E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6; + bool depfile = false; + system::path depfilePath = {}; }; // https://github.com/microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#debugging @@ -215,6 +217,10 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Needed for json vector serialization. Making it private and declaring from_json(_, SEntry&) as friend didn't work inline SPreprocessingDependency() {} + inline const system::path& getRequestingSourceDir() const { return requestingSourceDir; } + inline std::string_view getIdentifier() const { return identifier; } + inline bool isStandardInclude() const { return standardInclude; } + private: friend void to_json(nlohmann::json& j, const SEntry::SPreprocessingDependency& dependency); friend void from_json(const nlohmann::json& j, SEntry::SPreprocessingDependency& dependency); @@ -447,6 +453,17 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const; }; + struct DepfileWriteParams + { + system::ISystem* system = nullptr; + std::string_view depfilePath = {}; + std::string_view outputPath = {}; + std::string_view sourceIdentifier = {}; + system::path workingDirectory = {}; + }; + + static bool writeDepfile(const DepfileWriteParams& params, const CCache::SEntry::dependency_container_t& dependencies, const CIncludeFinder* includeFinder = nullptr, system::logger_opt_ptr logger = nullptr); + core::smart_refctd_ptr compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const; inline core::smart_refctd_ptr compileToSPIRV(const char* code, const SCompilerOptions& options) const diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index d36ecfa1cb..e551d3c72b 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -111,12 +111,23 @@ static bool fixup_spirv_target_ver(std::vector& arguments, system: for (auto targetEnvArgumentPos=arguments.begin(); targetEnvArgumentPos!=arguments.end(); targetEnvArgumentPos++) if (targetEnvArgumentPos->find(Prefix)==0) { - const auto suffix = targetEnvArgumentPos->substr(Prefix.length()); + auto suffix = targetEnvArgumentPos->substr(Prefix.length()); + auto trim = [](std::wstring& value) { + auto isTrimChar = [](wchar_t c) { + return c == L' ' || c == L'\t' || c == L'\r' || c == L'\n' || c == L'"'; + }; + while (!value.empty() && isTrimChar(value.front())) + value.erase(value.begin()); + while (!value.empty() && isTrimChar(value.back())) + value.pop_back(); + }; + trim(suffix); const auto found = AllowedSuffices.find(suffix); if (found!=AllowedSuffices.end()) return true; - logger.log("Compile flag warning: Required compile flag not found -fspv-target-env=. Force enabling -fspv-target-env= found but with unsupported value `%s`.", system::ILogger::ELL_ERROR, "TODO: write wchar to char convert usage"); - return false; + logger.log("Compile flag warning: Required compile flag not found -fspv-target-env=. Force enabling -fspv-target-env=vulkan1.3, previous value `%ls` is unsupported.", system::ILogger::ELL_WARNING, suffix.c_str()); + *targetEnvArgumentPos = L"-fspv-target-env=vulkan1.3"; + return true; } logger.log("Compile flag warning: Required compile flag not found -fspv-target-env=. Force enabling -fspv-target-env=vulkan1.3, as it is required by Nabla.", system::ILogger::ELL_WARNING); @@ -353,6 +364,21 @@ namespace nbl::wave std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector& dxc_compile_flags_override, std::vector* dependencies) const { + const bool depfileEnabled = preprocessOptions.depfile; + if (depfileEnabled) + { + if (preprocessOptions.depfilePath.empty()) + { + preprocessOptions.logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR); + return {}; + } + } + + std::vector localDependencies; + auto* dependenciesOut = dependencies; + if (depfileEnabled && !dependenciesOut) + dependenciesOut = &localDependencies; + // HACK: we do a pre-pre-process here to add \n after every #pragma to neutralize boost::wave's actions // See https://github.com/Devsh-Graphics-Programming/Nabla/issues/746 size_t line_index = 0; @@ -367,8 +393,8 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE } // preprocess - core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependencies) /* if dependencies were passed, we assume we want caching*/, - [&dxc_compile_flags_override, &stage, &dependencies](nbl::wave::context& context) -> void + core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependenciesOut), + [&dxc_compile_flags_override, &stage, &dependenciesOut](nbl::wave::context& context) -> void { if (context.get_hooks().m_dxc_compile_flags_override.size() != 0) dxc_compile_flags_override = context.get_hooks().m_dxc_compile_flags_override; @@ -377,9 +403,8 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE if (context.get_hooks().m_pragmaStage != IShader::E_SHADER_STAGE::ESS_UNKNOWN) stage = context.get_hooks().m_pragmaStage; - if (dependencies) { - *dependencies = std::move(context.get_dependencies()); - } + if (dependenciesOut) + *dependenciesOut = std::move(context.get_dependencies()); } ); @@ -396,13 +421,31 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE } } + if (resolvedString.empty()) + return resolvedString; + + if (depfileEnabled) + { + IShaderCompiler::DepfileWriteParams params = {}; + const std::string depfilePathString = preprocessOptions.depfilePath.generic_string(); + std::filesystem::path targetPath = preprocessOptions.depfilePath; + if (targetPath.extension() == ".d") + targetPath.replace_extension(); + params.depfilePath = depfilePathString; + params.outputPath = targetPath.generic_string(); + params.sourceIdentifier = preprocessOptions.sourceIdentifier; + params.system = m_system.get(); + if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, preprocessOptions.includeFinder, preprocessOptions.logger)) + return {}; + } + return resolvedString; } std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector* dependencies) const { std::vector extra_dxc_compile_flags = {}; - return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags); + return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags, dependencies); } core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::string_view code, const IShaderCompiler::SCompilerOptions& options, std::vector* dependencies) const diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index e60bf31b5c..1d0083f818 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -21,40 +23,258 @@ IShaderCompiler::IShaderCompiler(core::smart_refctd_ptr&& syste m_defaultIncludeFinder = core::make_smart_refctd_ptr(core::smart_refctd_ptr(m_system)); } -core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const +bool IShaderCompiler::writeDepfile( + const DepfileWriteParams& params, + const CCache::SEntry::dependency_container_t& dependencies, + const CIncludeFinder* includeFinder, + system::logger_opt_ptr logger) { - CCache::SEntry entry; - if (options.readCache || options.writeCache) - entry = CCache::SEntry(code, options); - - if (options.readCache) - { - auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); - if (found != options.readCache->m_container.end()) - { - if (options.writeCache) - { - CCache::SEntry writeEntry = *found; - options.writeCache->insert(std::move(writeEntry)); - } - return found->decompressShader(); - } - } - - auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &entry.dependencies:nullptr); - // compute the SPIR-V shader content hash - if (retVal) - { - auto backingBuffer = retVal->getContent(); - const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); - } + std::string depfilePathString; + if (!params.depfilePath.empty()) + depfilePathString = std::string(params.depfilePath); + else + depfilePathString = std::string(params.outputPath) + ".d"; + + if (depfilePathString.empty()) + { + logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR); + return false; + } + + const auto parentDirectory = std::filesystem::path(depfilePathString).parent_path(); + if (!parentDirectory.empty() && !std::filesystem::exists(parentDirectory)) + { + if (!std::filesystem::create_directories(parentDirectory)) + { + logger.log("Failed to create parent directory for depfile.", system::ILogger::ELL_ERROR); + return false; + } + } + + std::vector depPaths; + depPaths.reserve(dependencies.size() + 1); + + auto addDepPath = [&depPaths](const std::filesystem::path& path) + { + if (path.empty()) + return; + if (!std::filesystem::exists(path)) + return; + depPaths.emplace_back(path.generic_string()); + }; + + if (!params.sourceIdentifier.empty()) + { + std::filesystem::path rootPath{std::string(params.sourceIdentifier)}; + if (rootPath.is_relative()) + { + if (!params.workingDirectory.empty()) + rootPath = std::filesystem::absolute(std::filesystem::path(params.workingDirectory) / rootPath); + else + rootPath = std::filesystem::absolute(rootPath); + } + addDepPath(rootPath); + } + + for (const auto& dep : dependencies) + { + if (includeFinder) + { + IShaderCompiler::IIncludeLoader::found_t header = dep.isStandardInclude() ? + includeFinder->getIncludeStandard(dep.getRequestingSourceDir(), std::string(dep.getIdentifier())) : + includeFinder->getIncludeRelative(dep.getRequestingSourceDir(), std::string(dep.getIdentifier())); + + if (!header) + continue; + addDepPath(header.absolutePath); + } + else + { + std::filesystem::path candidate = dep.isStandardInclude() ? std::filesystem::path(std::string(dep.getIdentifier())) : (dep.getRequestingSourceDir() / std::string(dep.getIdentifier())); + if (candidate.is_relative()) + { + if (!params.workingDirectory.empty()) + candidate = std::filesystem::absolute(std::filesystem::path(params.workingDirectory) / candidate); + else + candidate = std::filesystem::absolute(candidate); + } + addDepPath(candidate); + } + } + + std::sort(depPaths.begin(), depPaths.end()); + depPaths.erase(std::unique(depPaths.begin(), depPaths.end()), depPaths.end()); + + auto escapeDepPath = [](const std::string& path) -> std::string + { + std::string normalized = path; + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + std::string out; + out.reserve(normalized.size()); + for (const char c : normalized) + { + if (c == ' ' || c == '#') + out.emplace_back('\\'); + if (c == '$') + { + out.emplace_back('$'); + out.emplace_back('$'); + continue; + } + out.emplace_back(c); + } + return out; + }; + + if (!params.system) + { + logger.log("Depfile system is null.", system::ILogger::ELL_ERROR); + return false; + } + + core::smart_refctd_ptr depfile; + { + system::ISystem::future_t> future; + params.system->createFile(future, system::path(depfilePathString), system::IFileBase::ECF_WRITE); + if (!future.wait()) + { + logger.log("Failed to open depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + future.acquire().move_into(depfile); + } + if (!depfile) + { + logger.log("Failed to open depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + + std::string targetPathString; + if (params.outputPath.empty()) + { + std::filesystem::path targetPath = depfilePathString; + if (targetPath.extension() == ".d") + targetPath.replace_extension(); + targetPathString = targetPath.generic_string(); + } + else + { + targetPathString = std::string(params.outputPath); + } + if (targetPathString.empty()) + { + logger.log("Depfile target path is empty.", system::ILogger::ELL_ERROR); + return false; + } + const std::string target = escapeDepPath(std::filesystem::path(targetPathString).generic_string()); + std::vector escapedDeps; + escapedDeps.reserve(depPaths.size()); + for (const auto& depPath : depPaths) + escapedDeps.emplace_back(escapeDepPath(depPath)); + + std::string depfileContents; + depfileContents.append(target); + depfileContents.append(":"); + if (!escapedDeps.empty()) + { + depfileContents.append(" \\\n"); + for (size_t index = 0; index < escapedDeps.size(); ++index) + { + depfileContents.append(" "); + depfileContents.append(escapedDeps[index]); + if (index + 1 < escapedDeps.size()) + depfileContents.append(" \\\n"); + } + } + depfileContents.append("\n"); + + system::IFile::success_t success; + depfile->write(success, depfileContents.data(), 0, depfileContents.size()); + if (!success) + { + logger.log("Failed to write depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } + return true; +} - if (options.writeCache) - { - if (entry.setContent(retVal->getContent())) - options.writeCache->insert(std::move(entry)); - } - return retVal; +core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const +{ + const bool depfileEnabled = options.preprocessorOptions.depfile; + const bool supportsDependencies = options.getCodeContentType() == IShader::E_CONTENT_TYPE::ECT_HLSL; + + auto writeDepfileFromDependencies = [&](const CCache::SEntry::dependency_container_t& dependencies) -> bool + { + if (!depfileEnabled) + return true; + + if (options.preprocessorOptions.depfilePath.empty()) + { + options.preprocessorOptions.logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR); + return false; + } + + IShaderCompiler::DepfileWriteParams params = {}; + const std::string depfilePathString = options.preprocessorOptions.depfilePath.generic_string(); + params.depfilePath = depfilePathString; + auto targetPath = options.preprocessorOptions.depfilePath; + if (targetPath.extension() == ".d") + targetPath.replace_extension(); + params.outputPath = targetPath.generic_string(); + params.sourceIdentifier = options.preprocessorOptions.sourceIdentifier; + params.system = m_system.get(); + return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger); + }; + + CCache::SEntry entry; + if (options.readCache || options.writeCache) + entry = CCache::SEntry(code, options); + + if (options.readCache) + { + auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); + if (found != options.readCache->m_container.end()) + { + if (options.writeCache) + { + CCache::SEntry writeEntry = *found; + options.writeCache->insert(std::move(writeEntry)); + } + auto shader = found->decompressShader(); + if (depfileEnabled && !writeDepfileFromDependencies(found->dependencies)) + return nullptr; + return shader; + } + } + + CCache::SEntry::dependency_container_t depfileDependencies; + CCache::SEntry::dependency_container_t* dependenciesPtr = nullptr; + if (options.writeCache) + dependenciesPtr = &entry.dependencies; + else if (depfileEnabled && supportsDependencies) + dependenciesPtr = &depfileDependencies; + + auto retVal = compileToSPIRV_impl(code, options, dependenciesPtr); + if (retVal) + { + auto backingBuffer = retVal->getContent(); + const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); + } + + if (retVal && depfileEnabled && supportsDependencies) + { + const auto* deps = options.writeCache ? &entry.dependencies : &depfileDependencies; + if (!writeDepfileFromDependencies(*deps)) + return nullptr; + } + + if (options.writeCache) + { + if (entry.setContent(retVal->getContent())) + options.writeCache->insert(std::move(entry)); + } + + return retVal; } std::string IShaderCompiler::preprocessShader( @@ -72,7 +292,6 @@ std::string IShaderCompiler::preprocessShader( return preprocessShader(std::move(code), stage, preprocessOptions, dependencies); } - auto IShaderCompiler::IIncludeGenerator::getInclude(const std::string& includeName) const -> IIncludeLoader::found_t { core::vector> builtinNames = getBuiltinNamesToFunctionMapping(); @@ -97,7 +316,7 @@ core::vector IShaderCompiler::IIncludeGenerator::parseArgumentsFrom std::stringstream ss{ _path }; std::string arg; while (std::getline(ss, arg, '/')) - args.push_back(std::move(arg)); + args.emplace_back(std::move(arg)); return args; } @@ -178,7 +397,7 @@ void IShaderCompiler::CIncludeFinder::addSearchPath(const std::string& searchPat { if (!loader) return; - m_loaders.push_back(LoaderSearchPath{ loader, searchPath }); + m_loaders.emplace_back(LoaderSearchPath{ loader, searchPath }); } void IShaderCompiler::CIncludeFinder::addGenerator(const core::smart_refctd_ptr& generatorToAdd) @@ -301,7 +520,7 @@ core::smart_refctd_ptr IShaderCompiler::CCache::serialize() const size_t i = 0u; for (auto& entry : m_container) { // Add the entry as a json array - entries.push_back(entry); + entries.emplace_back(entry); // We keep a copy of the offsets and the sizes of each shader. This is so that later on, when we add the shaders to the buffer after json creation // (where the params array has been moved) we don't have to read the json to get the offsets again diff --git a/tools/nsc/main.cpp b/tools/nsc/main.cpp index edc56de84c..3332b8377b 100644 --- a/tools/nsc/main.cpp +++ b/tools/nsc/main.cpp @@ -200,6 +200,12 @@ class ShaderCompiler final : public system::IApplicationFramework m_logger->log(outputType + " shader code will be saved to " + output_filepath, ILogger::ELL_INFO); } + DepfileConfig depfileConfig = parseDepfileArgs(m_arguments); + if (depfileConfig.enabled && depfileConfig.path.empty()) + depfileConfig.path = output_filepath + ".d"; + if (depfileConfig.enabled) + m_logger->log("Dependency file will be saved to %s", ILogger::ELL_INFO, depfileConfig.path.c_str()); + #ifndef NBL_EMBED_BUILTIN_RESOURCES if (!no_nbl_builtins) { m_system->unmountBuiltins(); @@ -234,12 +240,12 @@ class ShaderCompiler final : public system::IApplicationFramework std::string_view result_view; if (preprocessOnly) { - preprocessing_result = preprocess_shader(shader.get(), shaderStage, file_to_compile); + preprocessing_result = preprocess_shader(shader.get(), shaderStage, file_to_compile, depfileConfig); result_view = preprocessing_result; } else { - compilation_result = compile_shader(shader.get(), shaderStage, file_to_compile); + compilation_result = compile_shader(shader.get(), shaderStage, file_to_compile, depfileConfig); result_view = { (const char*)compilation_result->getContent()->getPointer(), compilation_result->getContent()->getSize() }; } auto end = std::chrono::high_resolution_clock::now(); @@ -291,6 +297,9 @@ class ShaderCompiler final : public system::IApplicationFramework return false; } + if (depfileConfig.enabled) + m_logger->log("Dependency file written to %s", ILogger::ELL_INFO, depfileConfig.path.c_str()); + return true; } else @@ -307,7 +316,49 @@ class ShaderCompiler final : public system::IApplicationFramework private: - std::string preprocess_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier) { + struct DepfileConfig + { + bool enabled = false; + std::string path; + }; + + DepfileConfig parseDepfileArgs(std::vector& args) + { + DepfileConfig cfg; + for (auto it = args.begin(); it != args.end();) + { + const std::string& arg = *it; + if (arg == "-MD" || arg == "-M") + { + cfg.enabled = true; + it = args.erase(it); + continue; + } + if (arg == "-MF") + { + if (it + 1 == args.end()) + { + m_logger->log("Incorrect arguments. Expecting filename after -MF.", ILogger::ELL_ERROR); + return cfg; + } + cfg.enabled = true; + cfg.path = *(it + 1); + it = args.erase(it, it + 2); + continue; + } + if (arg.rfind("-MF", 0) == 0 && arg.size() > 3) + { + cfg.enabled = true; + cfg.path = arg.substr(3); + it = args.erase(it); + continue; + } + ++it; + } + return cfg; + } + + std::string preprocess_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& depfileConfig) { smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); CHLSLCompiler::SPreprocessorOptions options = {}; @@ -322,6 +373,8 @@ class ShaderCompiler final : public system::IApplicationFramework includeFinder->addSearchPath(it, includeLoader); options.includeFinder = includeFinder.get(); + options.depfile = depfileConfig.enabled; + options.depfilePath = depfileConfig.path; const char* code_ptr = (const char*)shader->getContent()->getPointer(); std::string_view code({ code_ptr, strlen(code_ptr)}); @@ -329,7 +382,7 @@ class ShaderCompiler final : public system::IApplicationFramework return hlslcompiler->preprocessShader(std::string(code), shaderStage, options, nullptr); } - core::smart_refctd_ptr compile_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier) { + core::smart_refctd_ptr compile_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& depfileConfig) { smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); CHLSLCompiler::SOptions options = {}; @@ -348,6 +401,8 @@ class ShaderCompiler final : public system::IApplicationFramework includeFinder->addSearchPath(it, includeLoader); options.preprocessorOptions.includeFinder = includeFinder.get(); + options.preprocessorOptions.depfile = depfileConfig.enabled; + options.preprocessorOptions.depfilePath = depfileConfig.path; return hlslcompiler->compileToSPIRV((const char*)shader->getContent()->getPointer(), options); } From 682ba00a65dae272ff2e8821db3325c835c92a5b Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Fri, 2 Jan 2026 23:03:21 +0100 Subject: [PATCH 2/8] fix certain issues and update docs --- docs/nsc-prebuilds.md | 22 +++------------------- src/nbl/asset/utils/CHLSLCompiler.cpp | 4 ---- src/nbl/asset/utils/IShaderCompiler.cpp | 12 ++++-------- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/docs/nsc-prebuilds.md b/docs/nsc-prebuilds.md index 4d57d7a8de..1f8c73e2ee 100644 --- a/docs/nsc-prebuilds.md +++ b/docs/nsc-prebuilds.md @@ -60,7 +60,7 @@ Keys are strings that match the output layout: - `INPUT` (string, required): path to `.hlsl` (relative to `CMAKE_CURRENT_SOURCE_DIR` or absolute). - `KEY` (string, required): base key (prefer without `.spv`; it is always appended, so using `foo.spv` will result in `foo.spv.spv`). - `COMPILE_OPTIONS` (array of strings, optional): per-input extra options (e.g. `["-T","cs_6_8"]`). -- `DEPENDS` (array of strings, optional): per-input dependencies (extra files that should trigger rebuild). +- `DEPENDS` (array of strings, optional): extra per-input dependencies that are not discovered via `#include` (see below). - `CAPS` (array, optional): permutation caps (see below). You can register many rules in a single call, and you can call the function multiple times to append rules to the same `TARGET`. @@ -87,18 +87,9 @@ The helper also exposes CMake options that append NSC debug flags **only for Deb ## Source files and rebuild dependencies (important) -Make sure shader inputs and includes are: +NSC supports depfiles and the CMake custom commands consume them, so **changes in any `#include`d HLSL file automatically trigger recompilation of the affected `.spv` outputs**. In most cases you no longer need to list includes manually. -1. Marked as header-only on your target (so the IDE shows them, but the build system doesn't try to compile them with default HLSL rules like `fxc`): - -```cmake -target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS}) -set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON) -``` - -2. Listed as dependencies of the NSC custom commands (so editing any of them triggers a rebuild of the `.spv` outputs). - -This is what the `DEPENDS` argument of `NBL_CREATE_NSC_COMPILE_RULES` (and/or per-input JSON `DEPENDS`) is for. Always include the main `INPUT` file itself and any files it includes; otherwise the build system might not re-run `nsc` when you change them. +Use `DEPENDS` only for **extra** inputs that are not discovered via `#include` (e.g. a generated header that is not included, a config file read by a custom include generator, or any non-HLSL file that should trigger a rebuild). You can register those extra dependencies if you need them, but in most projects `DEPENDS` should stay empty. ## Minimal usage (no permutations) @@ -106,12 +97,6 @@ Example pattern (as in `examples_tests/27_MPMCScheduler/CMakeLists.txt`): ```cmake set(OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auto-gen") -set(DEPENDS - app_resources/common.hlsl - app_resources/shader.comp.hlsl -) -target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS}) -set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON) set(JSON [=[ [ @@ -128,7 +113,6 @@ set(JSON [=[ NBL_CREATE_NSC_COMPILE_RULES( TARGET ${EXECUTABLE_NAME}SPIRV LINK_TO ${EXECUTABLE_NAME} - DEPENDS ${DEPENDS} BINARY_DIR ${OUTPUT_DIRECTORY} MOUNT_POINT_DEFINE NBL_THIS_EXAMPLE_BUILD_MOUNT_POINT COMMON_OPTIONS -I ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index e551d3c72b..2b98f9c192 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -428,11 +428,7 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE { IShaderCompiler::DepfileWriteParams params = {}; const std::string depfilePathString = preprocessOptions.depfilePath.generic_string(); - std::filesystem::path targetPath = preprocessOptions.depfilePath; - if (targetPath.extension() == ".d") - targetPath.replace_extension(); params.depfilePath = depfilePathString; - params.outputPath = targetPath.generic_string(); params.sourceIdentifier = preprocessOptions.sourceIdentifier; params.system = m_system.get(); if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, preprocessOptions.includeFinder, preprocessOptions.logger)) diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 1d0083f818..754229f83b 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -114,14 +114,14 @@ bool IShaderCompiler::writeDepfile( for (const char c : normalized) { if (c == ' ' || c == '#') - out.emplace_back('\\'); + out.push_back('\\'); if (c == '$') { - out.emplace_back('$'); - out.emplace_back('$'); + out.push_back('$'); + out.push_back('$'); continue; } - out.emplace_back(c); + out.push_back(c); } return out; }; @@ -217,10 +217,6 @@ core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(cons IShaderCompiler::DepfileWriteParams params = {}; const std::string depfilePathString = options.preprocessorOptions.depfilePath.generic_string(); params.depfilePath = depfilePathString; - auto targetPath = options.preprocessorOptions.depfilePath; - if (targetPath.extension() == ".d") - targetPath.replace_extension(); - params.outputPath = targetPath.generic_string(); params.sourceIdentifier = options.preprocessorOptions.sourceIdentifier; params.system = m_system.get(); return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger); From f04d3a3692c441caef36cd371663997a0c43ae68 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Fri, 2 Jan 2026 23:10:44 +0100 Subject: [PATCH 3/8] update nsc rules, `-Wno-local-type-template-args` in REQUIRED_OPTIONS --- cmake/common.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/common.cmake b/cmake/common.cmake index 0d8b583f18..488ee7342b 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1181,6 +1181,7 @@ struct DeviceConfigCaps -fspv-target-env=vulkan1.3 -Wshadow -Wconversion + -Wno-local-type-template-args $<$:-O0> $<$:-O3> $<$:-O3> From d66fc339d832acbcf583f0cdbf25bc72299ddd5a Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Sat, 3 Jan 2026 17:13:10 +0100 Subject: [PATCH 4/8] finalize depfile & NSC rules updates --- cmake/common.cmake | 65 ++++++- docs/nsc-prebuilds.md | 15 +- examples_tests | 2 +- tools/nsc/CMakeLists.txt | 5 +- tools/nsc/main.cpp | 407 +++++++++++++++++++++++---------------- 5 files changed, 308 insertions(+), 186 deletions(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index 488ee7342b..c3901c6e42 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1209,6 +1209,7 @@ struct DeviceConfigCaps if(NOT NBL_EMBED_BUILTIN_RESOURCES) list(APPEND REQUIRED_OPTIONS + -no-nbl-builtins -I "${NBL_ROOT_PATH}/include" -I "${NBL_ROOT_PATH}/3rdparty/dxc/dxc/external/SPIRV-Headers/include" -I "${NBL_ROOT_PATH}/3rdparty/boost/superproject/libs/preprocessor/include" @@ -1217,7 +1218,8 @@ struct DeviceConfigCaps endif() set(REQUIRED_SINGLE_ARGS TARGET BINARY_DIR OUTPUT_VAR INPUTS INCLUDE NAMESPACE MOUNT_POINT_DEFINE) - cmake_parse_arguments(IMPL "" "${REQUIRED_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) + set(OPTIONAL_SINGLE_ARGS GLOB_DIR) + cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) NBL_PARSE_REQUIRED(IMPL ${REQUIRED_SINGLE_ARGS}) if(NOT TARGET ${IMPL_TARGET}) @@ -1295,6 +1297,10 @@ namespace @IMPL_NAMESPACE@ { list(APPEND MP_DEFINES ${IMPL_MOUNT_POINT_DEFINE}="${IMPL_BINARY_DIR}") set_target_properties(${IMPL_TARGET} PROPERTIES NBL_MOUNT_POINT_DEFINES "${MP_DEFINES}") + set(RTE "NSC Rules") + set(IN "${RTE}/In") + set(OUT "${RTE}/Out") + string(JSON JSON_LENGTH LENGTH "${IMPL_INPUTS}") math(EXPR LAST_INDEX "${JSON_LENGTH} - 1") @@ -1482,6 +1488,7 @@ namespace @IMPL_NAMESPACE@ { set(FINAL_KEY_REL_PATH "$/${FINAL_KEY}") set(TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY_REL_PATH}") set(DEPFILE_PATH "${TARGET_OUTPUT}.d") + set(NBL_NSC_LOG_PATH "${TARGET_OUTPUT}.log") set(NBL_NSC_DEPFILE_ARGS "") if(NSC_USE_DEPFILE) @@ -1496,11 +1503,18 @@ namespace @IMPL_NAMESPACE@ { "${CONFIG_FILE}" ) + get_filename_component(NBL_NSC_INPUT_NAME "${TARGET_INPUT}" NAME) + set(NBL_NSC_BYPRODUCTS "${NBL_NSC_LOG_PATH}") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_BYPRODUCTS "${DEPFILE_PATH}") + endif() + set(NBL_NSC_CUSTOM_COMMAND_ARGS OUTPUT "${TARGET_OUTPUT}" + BYPRODUCTS ${NBL_NSC_BYPRODUCTS} COMMAND ${NBL_NSC_COMPILE_COMMAND} DEPENDS ${DEPENDS_ON} - COMMENT "Creating \"${TARGET_OUTPUT}\"" + COMMENT "${NBL_NSC_INPUT_NAME}" VERBATIM COMMAND_EXPAND_LISTS ) @@ -1508,15 +1522,37 @@ namespace @IMPL_NAMESPACE@ { list(APPEND NBL_NSC_CUSTOM_COMMAND_ARGS DEPFILE "${DEPFILE_PATH}") endif() add_custom_command(${NBL_NSC_CUSTOM_COMMAND_ARGS}) - set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES GENERATED TRUE) + set(NBL_NSC_OUT_FILES "${TARGET_OUTPUT}" "${NBL_NSC_LOG_PATH}") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES "${DEPFILE_PATH}") + endif() - set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" "${TARGET_OUTPUT}") + set_source_files_properties(${NBL_NSC_OUT_FILES} PROPERTIES GENERATED TRUE) + + set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" ${NBL_NSC_OUT_FILES}) target_sources(${IMPL_TARGET} PRIVATE ${HEADER_ONLY_LIKE}) set_source_files_properties(${HEADER_ONLY_LIKE} PROPERTIES HEADER_FILE_ONLY ON VS_TOOL_OVERRIDE None ) + if(CMAKE_CONFIGURATION_TYPES) + foreach(_CFG IN LISTS CMAKE_CONFIGURATION_TYPES) + set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${_CFG}/${FINAL_KEY}") + set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d") + endif() + source_group("${OUT}/${_CFG}" FILES ${NBL_NSC_OUT_FILES_IDE}) + endforeach() + else() + set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${FINAL_KEY}") + set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d") + endif() + source_group("${OUT}" FILES ${NBL_NSC_OUT_FILES_IDE}) + endif() set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES NBL_SPIRV_REGISTERED_INPUT "${TARGET_INPUT}" @@ -1558,12 +1594,23 @@ namespace @IMPL_NAMESPACE@ { list(APPEND KEYS ${ACCESS_KEY}) endforeach() - set(RTE "NSC Rules") - set(IN "${RTE}/In") - set(OUT "${RTE}/Out") - source_group("${IN}" FILES ${CONFIGS} ${INPUTS}) - source_group("${OUT}" FILES ${SPIRVs}) + if(NOT IMPL_DISCARD_DEFAULT_GLOB) + set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + if(IMPL_GLOB_DIR) + set(GLOB_ROOT "${IMPL_GLOB_DIR}") + endif() + get_filename_component(GLOB_ROOT "${GLOB_ROOT}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + file(GLOB_RECURSE IMPL_HLSL_GLOB CONFIGURE_DEPENDS "${GLOB_ROOT}/*.hlsl") + if(IMPL_HLSL_GLOB) + target_sources(${IMPL_TARGET} PRIVATE ${IMPL_HLSL_GLOB}) + set_source_files_properties(${IMPL_HLSL_GLOB} PROPERTIES + HEADER_FILE_ONLY ON + VS_TOOL_OVERRIDE None + ) + source_group("HLSL Files" FILES ${IMPL_HLSL_GLOB}) + endif() + endif() set(${IMPL_OUTPUT_VAR} ${KEYS} PARENT_SCOPE) endfunction() diff --git a/docs/nsc-prebuilds.md b/docs/nsc-prebuilds.md index 1f8c73e2ee..400aff5eb7 100644 --- a/docs/nsc-prebuilds.md +++ b/docs/nsc-prebuilds.md @@ -91,6 +91,11 @@ NSC supports depfiles and the CMake custom commands consume them, so **changes i Use `DEPENDS` only for **extra** inputs that are not discovered via `#include` (e.g. a generated header that is not included, a config file read by a custom include generator, or any non-HLSL file that should trigger a rebuild). You can register those extra dependencies if you need them, but in most projects `DEPENDS` should stay empty. +By default `NBL_CREATE_NSC_COMPILE_RULES` also collects `*.hlsl` files for IDE visibility. It recursively scans the current source directory (or `GLOB_DIR` if provided), adds those files as header-only, and groups them under `HLSL Files`. If you do not want this behavior, pass `DISCARD_DEFAULT_GLOB`. + +- `GLOB_DIR` (optional): root directory for the default `*.hlsl` scan. +- `DISCARD_DEFAULT_GLOB` (flag): disables the default scan and IDE grouping. + ## Minimal usage (no permutations) Example pattern (as in `examples_tests/27_MPMCScheduler/CMakeLists.txt`): @@ -282,6 +287,8 @@ If the error looks like a preprocessing issue, note that we use Boost.Wave as th
NSC rules + archive + runtime key usage +NSC emits depfiles and the custom commands consume them, so changes in `#include`d HLSL files automatically trigger recompilation of the affected outputs. In most cases you do not need to list includes manually. Use `DEPENDS` only for extra inputs that are not discovered via `#include`. + ### CMake (`CMakeLists.txt`) ```cmake @@ -290,12 +297,6 @@ include(common) nbl_create_executable_project("" "" "" "") set(OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auto-gen") -set(DEPENDS - app_resources/common.hlsl - app_resources/shader.hlsl -) -target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS}) -set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON) set(JSON [=[ [ @@ -303,7 +304,6 @@ set(JSON [=[ "INPUT": "app_resources/shader.hlsl", "KEY": "shader", "COMPILE_OPTIONS": ["-T", "lib_6_8"], - "DEPENDS": [], "CAPS": [ { "kind": "limits", @@ -325,7 +325,6 @@ set(JSON [=[ NBL_CREATE_NSC_COMPILE_RULES( TARGET ${EXECUTABLE_NAME}SPIRV LINK_TO ${EXECUTABLE_NAME} - DEPENDS ${DEPENDS} BINARY_DIR ${OUTPUT_DIRECTORY} MOUNT_POINT_DEFINE NBL_THIS_EXAMPLE_BUILD_MOUNT_POINT COMMON_OPTIONS -I ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/examples_tests b/examples_tests index 5df217517f..58b42cfc87 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 5df217517fd5af0964b6d170afb68d5194daf60d +Subproject commit 58b42cfc87274db606d593d5baec787b626bb945 diff --git a/tools/nsc/CMakeLists.txt b/tools/nsc/CMakeLists.txt index bcdcbca531..2765f02fa5 100644 --- a/tools/nsc/CMakeLists.txt +++ b/tools/nsc/CMakeLists.txt @@ -2,6 +2,9 @@ nbl_create_executable_project("" "" "" "") enable_testing() +add_dependencies(${EXECUTABLE_NAME} argparse) +target_include_directories(${EXECUTABLE_NAME} PRIVATE $) + set(GODBOLT_BINARY_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/compiler-explorer") set(GODBOLT_BINARY_PRETEST_DIRECTORY "${GODBOLT_BINARY_DIRECTORY}/pre-test") set(NBL_NSC_COMPILE_DIRECTORY "${GODBOLT_BINARY_PRETEST_DIRECTORY}/.compile/$") @@ -359,4 +362,4 @@ add_custom_target(run-compiler-explorer ALL add_dependencies(run-compiler-explorer nsc) set_target_properties(run-compiler-explorer PROPERTIES FOLDER "Godbolt") -endif() \ No newline at end of file +endif() diff --git a/tools/nsc/main.cpp b/tools/nsc/main.cpp index 3332b8377b..56152701b3 100644 --- a/tools/nsc/main.cpp +++ b/tools/nsc/main.cpp @@ -3,8 +3,11 @@ #include #include +#include #include #include +#include +#include #include "nbl/asset/metadata/CHLSLMetadata.h" #include "nlohmann/json.hpp" @@ -15,6 +18,41 @@ using namespace nbl::system; using namespace nbl::core; using namespace nbl::asset; +class NscLogger final : public system::IThreadsafeLogger +{ +public: + NscLogger(core::smart_refctd_ptr&& logFile, const core::bitflag logLevelMask, const core::bitflag consoleMask) + : IThreadsafeLogger(logLevelMask), m_logFile(std::move(logFile)), m_logPos(m_logFile ? m_logFile->getSize() : 0ull), m_consoleMask(consoleMask) + { + } + +private: + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto line = constructLogString(fmt, logLevel, args); + size_t lineSize = line.size(); + while (lineSize > 0 && line[lineSize - 1] == '\0') + --lineSize; + if (lineSize == 0) + return; + if (m_logFile) + { + system::IFile::success_t succ; + m_logFile->write(succ, line.data(), m_logPos, lineSize); + m_logPos += succ.getBytesProcessed(); + } + if (logLevel & m_consoleMask.value) + { + std::fwrite(line.data(), 1, lineSize, stdout); + std::fflush(stdout); + } + } + + core::smart_refctd_ptr m_logFile; + size_t m_logPos = 0ull; + core::bitflag m_consoleMask; +}; + class ShaderCompiler final : public system::IApplicationFramework { using base_t = system::IApplicationFramework; @@ -24,71 +62,120 @@ class ShaderCompiler final : public system::IApplicationFramework bool onAppInitialized(smart_refctd_ptr&& system) override { - const auto argc = argv.size(); - const bool insufficientArguments = argc < 2; - - if (not insufficientArguments) + const auto rawArgs = std::vector(argv.begin(), argv.end()); + auto expandArgs = [](const std::vector& args) { - // 1) NOTE: imo each example should be able to dump build info & have such mode, maybe it could go straight to IApplicationFramework main - // 2) TODO: this whole "serialize" logic should go to the GitInfo struct and be static or something, it should be standardized - - if (argv[1] == "--dump-build-info") + std::vector expanded; + expanded.reserve(args.size()); + for (const auto& arg : args) { - json j; - - auto& modules = j["modules"]; - - auto serialize = [&](const gtml::GitInfo& info, std::string_view target) -> void + if (arg.rfind("-MF", 0) == 0 && arg.size() > 3) { - auto& s = modules[target.data()]; - - s["isPopulated"] = info.isPopulated; - if (info.hasUncommittedChanges.has_value()) - s["hasUncommittedChanges"] = info.hasUncommittedChanges.value(); - else - s["hasUncommittedChanges"] = "UNKNOWN, BUILT WITHOUT DIRTY-CHANGES CAPTURE"; - - s["commitAuthorName"] = info.commitAuthorName; - s["commitAuthorEmail"] = info.commitAuthorEmail; - s["commitHash"] = info.commitHash; - s["commitShortHash"] = info.commitShortHash; - s["commitDate"] = info.commitDate; - s["commitSubject"] = info.commitSubject; - s["commitBody"] = info.commitBody; - s["describe"] = info.describe; - s["branchName"] = info.branchName; - s["latestTag"] = info.latestTag; - s["latestTagName"] = info.latestTagName; - }; - - serialize(gtml::nabla_git_info, "nabla"); - serialize(gtml::dxc_git_info, "dxc"); - - const auto pretty = j.dump(4); - std::cout << pretty << std::endl; - - std::filesystem::path oPath = "build-info.json"; - - // TOOD: use argparse for it - if (argc > 3 && argv[2] == "--file") - oPath = argv[3]; - - std::ofstream outFile(oPath); - if (outFile.is_open()) + expanded.push_back("-MF"); + expanded.push_back(arg.substr(3)); + continue; + } + if (arg.rfind("-Fo", 0) == 0 && arg.size() > 3) { - outFile << pretty; - outFile.close(); - printf("Saved \"%s\"\n", oPath.string().c_str()); + expanded.push_back("-Fo"); + expanded.push_back(arg.substr(3)); + continue; } - else + if (arg.rfind("-Fc", 0) == 0 && arg.size() > 3) { - printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); - exit(-1); + expanded.push_back("-Fc"); + expanded.push_back(arg.substr(3)); + continue; } + expanded.push_back(arg); + } + return expanded; + }; - // in this mode terminate with 0 if all good - exit(0); + argparse::ArgumentParser program("nsc"); + program.add_argument("--dump-build-info").default_value(false).implicit_value(true); + program.add_argument("--file").default_value(std::string{}); + program.add_argument("-P").default_value(false).implicit_value(true); + program.add_argument("-no-nbl-builtins").default_value(false).implicit_value(true); + program.add_argument("-MD").default_value(false).implicit_value(true); + program.add_argument("-M").default_value(false).implicit_value(true); + program.add_argument("-MF").default_value(std::string{}); + program.add_argument("-Fo").default_value(std::string{}); + program.add_argument("-Fc").default_value(std::string{}); + program.add_argument("-log").default_value(std::string{}); + program.add_argument("-nolog").default_value(false).implicit_value(true); + program.add_argument("-quiet").default_value(false).implicit_value(true); + program.add_argument("-verbose").default_value(false).implicit_value(true); + + std::vector unknownArgs; + try + { + unknownArgs = program.parse_known_args(expandArgs(rawArgs)); + } + catch (const std::runtime_error& err) + { + std::cerr << err.what() << std::endl << program; + return false; + } + + if (program.get("--dump-build-info")) + { + json j; + + auto& modules = j["modules"]; + + auto serialize = [&](const gtml::GitInfo& info, std::string_view target) -> void + { + auto& s = modules[target.data()]; + + s["isPopulated"] = info.isPopulated; + if (info.hasUncommittedChanges.has_value()) + s["hasUncommittedChanges"] = info.hasUncommittedChanges.value(); + else + s["hasUncommittedChanges"] = "UNKNOWN, BUILT WITHOUT DIRTY-CHANGES CAPTURE"; + + s["commitAuthorName"] = info.commitAuthorName; + s["commitAuthorEmail"] = info.commitAuthorEmail; + s["commitHash"] = info.commitHash; + s["commitShortHash"] = info.commitShortHash; + s["commitDate"] = info.commitDate; + s["commitSubject"] = info.commitSubject; + s["commitBody"] = info.commitBody; + s["describe"] = info.describe; + s["branchName"] = info.branchName; + s["latestTag"] = info.latestTag; + s["latestTagName"] = info.latestTagName; + }; + + serialize(gtml::nabla_git_info, "nabla"); + serialize(gtml::dxc_git_info, "dxc"); + + const auto pretty = j.dump(4); + std::cout << pretty << std::endl; + + std::filesystem::path oPath = "build-info.json"; + + if (program.is_used("--file")) + { + const auto filePath = program.get("--file"); + if (!filePath.empty()) + oPath = filePath; + } + + std::ofstream outFile(oPath); + if (outFile.is_open()) + { + outFile << pretty; + outFile.close(); + printf("Saved \"%s\"\n", oPath.string().c_str()); + } + else + { + printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); + exit(-1); } + + exit(0); } if (not isAPILoaded()) @@ -105,102 +192,119 @@ class ShaderCompiler final : public system::IApplicationFramework if (!m_system) return false; - m_logger = make_smart_refctd_ptr(core::bitflag(ILogger::ELL_DEBUG) | ILogger::ELL_INFO | ILogger::ELL_WARNING | ILogger::ELL_PERFORMANCE | ILogger::ELL_ERROR); + const auto defaultConsoleMask = core::bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR; + m_logger = make_smart_refctd_ptr(defaultConsoleMask); - if (insufficientArguments) + if (rawArgs.size() < 2) { m_logger->log("Insufficient arguments.", ILogger::ELL_ERROR); return false; } + std::string file_to_compile = rawArgs.back(); - m_arguments = std::vector(argv.begin() + 1, argv.end()-1); // turn argv into vector for convenience + if (!m_system->exists(file_to_compile, IFileBase::ECF_READ)) + { + m_logger->log("Input shader file does not exist: %s", ILogger::ELL_ERROR, file_to_compile.c_str()); + return false; + } - std::string file_to_compile = argv.back(); + const bool preprocessOnly = program.get("-P"); + const bool outputFlagFc = program.is_used("-Fc"); + const bool outputFlagFo = program.is_used("-Fo"); + if (outputFlagFc && outputFlagFo) + { + m_logger->log("Invalid arguments. Passed both -Fo and -Fc.", ILogger::ELL_ERROR); + return false; + } + if (!outputFlagFc && !outputFlagFo) + { + m_logger->log("Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.", ILogger::ELL_ERROR); + return false; + } - if (!m_system->exists(file_to_compile, IFileBase::ECF_READ)) { - m_logger->log("Incorrect arguments. Expecting last argument to be filename of the shader intended to compile.", ILogger::ELL_ERROR); + std::string output_filepath = outputFlagFc ? program.get("-Fc") : program.get("-Fo"); + if (output_filepath.empty()) + { + m_logger->log("Invalid output file path.", ILogger::ELL_ERROR); return false; } - std::string output_filepath = ""; - auto builtin_flag_pos = std::find(m_arguments.begin(), m_arguments.end(), "-no-nbl-builtins"); - if (builtin_flag_pos != m_arguments.end()) { - m_logger->log("Unmounting builtins."); - m_system->unmountBuiltins(); - no_nbl_builtins = true; - m_arguments.erase(builtin_flag_pos); + const bool quietFlag = program.get("-quiet"); + const bool verboseFlag = program.get("-verbose"); + if (quietFlag && verboseFlag) + { + m_logger->log("Invalid arguments. Passed both -quiet and -verbose.", ILogger::ELL_ERROR); + return false; } - auto split = [&](const std::string& str, char delim) + LogConfig logConfig; + if (verboseFlag) + logConfig.quiet = false; + if (quietFlag) + logConfig.quiet = true; + + logConfig.noLog = program.get("-nolog"); + if (program.is_used("-log")) { - std::vector strings; - size_t start, end = 0; - - while ((start = str.find_first_not_of(delim, end)) != std::string::npos) + logConfig.path = program.get("-log"); + if (logConfig.path.empty()) { - end = str.find(delim, start); - strings.push_back(str.substr(start, end - start)); + m_logger->log("Incorrect arguments. Expecting filename after -log.", ILogger::ELL_ERROR); + return false; } - - return strings; - }; - - auto findOutputFlag = [&](const std::string_view& outputFlag) - { - return std::find_if(m_arguments.begin(), m_arguments.end(), [&](const std::string& argument) - { - return argument.find(outputFlag.data()) != std::string::npos; - }); - }; - - auto preprocessOnly = findOutputFlag("-P") != m_arguments.end(); - auto output_flag_pos_fc = findOutputFlag("-Fc"); - auto output_flag_pos_fo = findOutputFlag("-Fo"); - if (output_flag_pos_fc != m_arguments.end() && output_flag_pos_fo != m_arguments.end()) { - m_logger->log("Invalid arguments. Passed both -Fo and -Fc.", ILogger::ELL_ERROR); - return false; } - auto output_flag_pos = output_flag_pos_fc != m_arguments.end() ? output_flag_pos_fc : output_flag_pos_fo; - if (output_flag_pos == m_arguments.end()) + + if (logConfig.noLog && !logConfig.path.empty()) { - m_logger->log("Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.", ILogger::ELL_ERROR); + m_logger->log("Invalid arguments. Passed both -nolog and -log.", ILogger::ELL_ERROR); return false; } - else + + const auto consoleMask = logConfig.quiet ? (core::bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR) : core::bitflag(ILogger::ELL_ALL); + m_logger = make_smart_refctd_ptr(consoleMask); + + if (!logConfig.noLog) { - // we need to assume -Fc may be passed with output file name quoted together with "", so we split it (DXC does it) - const auto& outputFlag = *output_flag_pos; - auto outputFlagVector = split(outputFlag, ' '); - - if(outputFlag == "-Fc" || outputFlag == "-Fo") - { - if (output_flag_pos + 1 != m_arguments.end()) - { - output_filepath = *(output_flag_pos + 1); - } - else - { - m_logger->log("Incorrect arguments. Expecting filename after %s.", ILogger::ELL_ERROR, outputFlag); - return false; - } - } + const std::filesystem::path logPath = logConfig.path.empty() ? std::filesystem::path(output_filepath).concat(".log") : std::filesystem::path(logConfig.path); + const auto parentDirectory = logPath.parent_path(); + if (!parentDirectory.empty() && !std::filesystem::exists(parentDirectory)) + std::filesystem::create_directories(parentDirectory); + + m_system->deleteFile(logPath); + + system::ISystem::future_t> future; + m_system->createFile(future, logPath, system::IFileBase::ECF_WRITE); + core::smart_refctd_ptr logFile; + if (future.wait()) + future.acquire().move_into(logFile); + + if (logFile) + m_logger = make_smart_refctd_ptr(std::move(logFile), core::bitflag(ILogger::ELL_ALL), consoleMask); else - { - output_filepath = outputFlagVector[1]; - } - m_arguments.erase(output_flag_pos, output_flag_pos+2); + m_logger->log("Failed to open log file: %s", ILogger::ELL_ERROR, logPath.string().c_str()); + } - if (output_filepath.empty()) - { - m_logger->log("Invalid output file path!" + output_filepath, ILogger::ELL_ERROR); - return false; - } - - std::string outputType = preprocessOnly ? "Preprocessed" : "Compiled"; - m_logger->log(outputType + " shader code will be saved to " + output_filepath, ILogger::ELL_INFO); + const char* action = preprocessOnly ? "Preprocessing" : "Compiling"; + m_logger->log("%s %s", ILogger::ELL_INFO, action, file_to_compile.c_str()); + const char* outputType = preprocessOnly ? "Preprocessed" : "Compiled"; + m_logger->log("%s shader code will be saved to %s", ILogger::ELL_INFO, outputType, output_filepath.c_str()); + + m_arguments = std::move(unknownArgs); + if (!m_arguments.empty() && m_arguments.back() == file_to_compile) + m_arguments.pop_back(); + + no_nbl_builtins = program.get("-no-nbl-builtins"); + if (no_nbl_builtins) + { + m_logger->log("Unmounting builtins."); + m_system->unmountBuiltins(); } - DepfileConfig depfileConfig = parseDepfileArgs(m_arguments); + DepfileConfig depfileConfig; + if (program.get("-MD") || program.get("-M") || program.is_used("-MF")) + depfileConfig.enabled = true; + if (program.is_used("-MF")) + depfileConfig.path = program.get("-MF"); if (depfileConfig.enabled && depfileConfig.path.empty()) depfileConfig.path = output_filepath + ".d"; if (depfileConfig.enabled) @@ -215,12 +319,11 @@ class ShaderCompiler final : public system::IApplicationFramework #endif if (std::find(m_arguments.begin(), m_arguments.end(), "-E") == m_arguments.end()) { - //Insert '-E main' into arguments if no entry point is specified m_arguments.push_back("-E"); m_arguments.push_back("main"); } - for (size_t i = 0; i < m_arguments.size() - 1; ++i) // -I must be given with second arg, no need to include iteration over last one + for (size_t i = 0; i + 1 < m_arguments.size(); ++i) { const auto& arg = m_arguments[i]; if (arg == "-I") @@ -250,7 +353,6 @@ class ShaderCompiler final : public system::IApplicationFramework } auto end = std::chrono::high_resolution_clock::now(); - // write compiled/preprocessed shader to file as bytes std::string operationType = preprocessOnly ? "preprocessing" : "compilation"; const bool success = preprocessOnly ? preprocessing_result != std::string{} : bool(compilation_result); if (success) @@ -316,47 +418,18 @@ class ShaderCompiler final : public system::IApplicationFramework private: - struct DepfileConfig + struct LogConfig { - bool enabled = false; + bool quiet = true; + bool noLog = false; std::string path; }; - DepfileConfig parseDepfileArgs(std::vector& args) + struct DepfileConfig { - DepfileConfig cfg; - for (auto it = args.begin(); it != args.end();) - { - const std::string& arg = *it; - if (arg == "-MD" || arg == "-M") - { - cfg.enabled = true; - it = args.erase(it); - continue; - } - if (arg == "-MF") - { - if (it + 1 == args.end()) - { - m_logger->log("Incorrect arguments. Expecting filename after -MF.", ILogger::ELL_ERROR); - return cfg; - } - cfg.enabled = true; - cfg.path = *(it + 1); - it = args.erase(it, it + 2); - continue; - } - if (arg.rfind("-MF", 0) == 0 && arg.size() > 3) - { - cfg.enabled = true; - cfg.path = arg.substr(3); - it = args.erase(it); - continue; - } - ++it; - } - return cfg; - } + bool enabled = false; + std::string path; + }; std::string preprocess_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& depfileConfig) { smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); @@ -448,7 +521,7 @@ class ShaderCompiler final : public system::IApplicationFramework bool no_nbl_builtins{ false }; smart_refctd_ptr m_system; - smart_refctd_ptr m_logger; + smart_refctd_ptr m_logger; std::vector m_arguments, m_include_search_paths; core::smart_refctd_ptr m_assetMgr; From f4d5fde949d759a0a6ecf20525cde963549c2dde Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Sun, 4 Jan 2026 07:50:24 +0100 Subject: [PATCH 5/8] fix depfile generation normalize paths, have a .tmp before writing final file --- cmake/common.cmake | 32 ++++++++++--------- src/nbl/asset/utils/CHLSLCompiler.cpp | 3 ++ src/nbl/asset/utils/IShaderCompiler.cpp | 41 ++++++++++++++++++++++--- 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index c3901c6e42..a4ce345096 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1222,6 +1222,16 @@ struct DeviceConfigCaps cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) NBL_PARSE_REQUIRED(IMPL ${REQUIRED_SINGLE_ARGS}) + set(IMPL_HLSL_GLOB "") + if(NOT IMPL_DISCARD_DEFAULT_GLOB) + set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + if(IMPL_GLOB_DIR) + set(GLOB_ROOT "${IMPL_GLOB_DIR}") + endif() + get_filename_component(GLOB_ROOT "${GLOB_ROOT}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + file(GLOB_RECURSE IMPL_HLSL_GLOB CONFIGURE_DEPENDS "${GLOB_ROOT}/*.hlsl") + endif() + if(NOT TARGET ${IMPL_TARGET}) add_library(${IMPL_TARGET} INTERFACE) endif() @@ -1595,21 +1605,13 @@ namespace @IMPL_NAMESPACE@ { endforeach() source_group("${IN}" FILES ${CONFIGS} ${INPUTS}) - if(NOT IMPL_DISCARD_DEFAULT_GLOB) - set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") - if(IMPL_GLOB_DIR) - set(GLOB_ROOT "${IMPL_GLOB_DIR}") - endif() - get_filename_component(GLOB_ROOT "${GLOB_ROOT}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") - file(GLOB_RECURSE IMPL_HLSL_GLOB CONFIGURE_DEPENDS "${GLOB_ROOT}/*.hlsl") - if(IMPL_HLSL_GLOB) - target_sources(${IMPL_TARGET} PRIVATE ${IMPL_HLSL_GLOB}) - set_source_files_properties(${IMPL_HLSL_GLOB} PROPERTIES - HEADER_FILE_ONLY ON - VS_TOOL_OVERRIDE None - ) - source_group("HLSL Files" FILES ${IMPL_HLSL_GLOB}) - endif() + if(IMPL_HLSL_GLOB) + target_sources(${IMPL_TARGET} PRIVATE ${IMPL_HLSL_GLOB}) + set_source_files_properties(${IMPL_HLSL_GLOB} PROPERTIES + HEADER_FILE_ONLY ON + VS_TOOL_OVERRIDE None + ) + source_group("HLSL Files" FILES ${IMPL_HLSL_GLOB}) endif() set(${IMPL_OUTPUT_VAR} ${KEYS} PARENT_SCOPE) diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index 2b98f9c192..1020fa9446 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -430,6 +431,8 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE const std::string depfilePathString = preprocessOptions.depfilePath.generic_string(); params.depfilePath = depfilePathString; params.sourceIdentifier = preprocessOptions.sourceIdentifier; + if (!params.sourceIdentifier.empty()) + params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path(); params.system = m_system.get(); if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, preprocessOptions.includeFinder, preprocessOptions.logger)) return {}; diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 754229f83b..a6cd95b441 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -54,13 +54,30 @@ bool IShaderCompiler::writeDepfile( std::vector depPaths; depPaths.reserve(dependencies.size() + 1); - auto addDepPath = [&depPaths](const std::filesystem::path& path) + auto addDepPath = [&depPaths, ¶ms](std::filesystem::path path) { if (path.empty()) return; - if (!std::filesystem::exists(path)) + if (path.is_relative()) + { + if (params.workingDirectory.empty()) + return; + path = std::filesystem::path(params.workingDirectory) / path; + } + std::error_code ec; + std::filesystem::path normalized = std::filesystem::weakly_canonical(path, ec); + if (ec) + { + normalized = std::filesystem::absolute(path, ec); + if (ec) + return; + } + if (normalized.empty() || !std::filesystem::exists(normalized)) + return; + auto normalizedString = normalized.generic_string(); + if (normalizedString.find_first_of("\r\n") != std::string::npos) return; - depPaths.emplace_back(path.generic_string()); + depPaths.emplace_back(std::move(normalizedString)); }; if (!params.sourceIdentifier.empty()) @@ -132,10 +149,15 @@ bool IShaderCompiler::writeDepfile( return false; } + const auto depfilePath = std::filesystem::path(depfilePathString); + auto tempPath = depfilePath; + tempPath += ".tmp"; + params.system->deleteFile(tempPath); + core::smart_refctd_ptr depfile; { system::ISystem::future_t> future; - params.system->createFile(future, system::path(depfilePathString), system::IFileBase::ECF_WRITE); + params.system->createFile(future, tempPath, system::IFileBase::ECF_WRITE); if (!future.wait()) { logger.log("Failed to open depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); @@ -195,6 +217,15 @@ bool IShaderCompiler::writeDepfile( logger.log("Failed to write depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); return false; } + depfile = nullptr; + + params.system->deleteFile(depfilePath); + const std::error_code moveError = params.system->moveFileOrDirectory(tempPath, depfilePath); + if (moveError) + { + logger.log("Failed to replace depfile: %s", system::ILogger::ELL_ERROR, depfilePathString.c_str()); + return false; + } return true; } @@ -218,6 +249,8 @@ core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(cons const std::string depfilePathString = options.preprocessorOptions.depfilePath.generic_string(); params.depfilePath = depfilePathString; params.sourceIdentifier = options.preprocessorOptions.sourceIdentifier; + if (!params.sourceIdentifier.empty()) + params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path(); params.system = m_system.get(); return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger); }; From 67b9a40ba404eff91b976bbd13d6342977f43245 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 5 Jan 2026 00:12:39 +0100 Subject: [PATCH 6/8] add new flags to IFileBase, CSystemWin32 & polish NSC code --- include/nbl/system/IFileBase.h | 5 +- src/nbl/system/CSystemWin32.cpp | 14 +- tools/nsc/main.cpp | 1015 ++++++++++++++++--------------- 3 files changed, 534 insertions(+), 500 deletions(-) diff --git a/include/nbl/system/IFileBase.h b/include/nbl/system/IFileBase.h index cb0170157e..58ab34fac5 100644 --- a/include/nbl/system/IFileBase.h +++ b/include/nbl/system/IFileBase.h @@ -25,7 +25,10 @@ class IFileBase : public core::IReferenceCounted ECF_READ_WRITE = 0b0011, ECF_MAPPABLE = 0b0100, //! Implies ECF_MAPPABLE - ECF_COHERENT = 0b1100 + ECF_COHERENT = 0b1100, + ECF_SHARE_READ = 0b10000, + ECF_SHARE_WRITE = 0b100000, + ECF_SHARE_DELETE = 0b1000000 }; //! Get size of file. diff --git a/src/nbl/system/CSystemWin32.cpp b/src/nbl/system/CSystemWin32.cpp index cab809c145..af33e1460d 100644 --- a/src/nbl/system/CSystemWin32.cpp +++ b/src/nbl/system/CSystemWin32.cpp @@ -43,6 +43,14 @@ core::smart_refctd_ptr CSystemWin32::CCaller::createFile(const std: { const bool writeAccess = flags.value&IFile::ECF_WRITE; const DWORD fileAccess = ((flags.value&IFile::ECF_READ) ? FILE_GENERIC_READ:0)|(writeAccess ? FILE_GENERIC_WRITE:0); + const bool hasShareFlags = flags.value & (IFile::ECF_SHARE_READ | IFile::ECF_SHARE_WRITE | IFile::ECF_SHARE_DELETE); + DWORD shareMode = hasShareFlags ? 0 : FILE_SHARE_READ; + if (flags.value & IFile::ECF_SHARE_READ) + shareMode |= FILE_SHARE_READ; + if (flags.value & IFile::ECF_SHARE_WRITE) + shareMode |= FILE_SHARE_WRITE; + if (flags.value & IFile::ECF_SHARE_DELETE) + shareMode |= FILE_SHARE_DELETE; SECURITY_ATTRIBUTES secAttribs{ sizeof(SECURITY_ATTRIBUTES), nullptr, FALSE }; @@ -51,8 +59,8 @@ core::smart_refctd_ptr CSystemWin32::CCaller::createFile(const std: p.make_preferred(); // Replace "/" separators with "\" // only write access should create new files if they don't exist - const auto creationDisposition = writeAccess ? OPEN_ALWAYS:OPEN_EXISTING; - HANDLE _native = CreateFileA(p.string().data(), fileAccess, FILE_SHARE_READ, &secAttribs, creationDisposition, FILE_ATTRIBUTE_NORMAL, nullptr); + const auto creationDisposition = writeAccess ? OPEN_ALWAYS : OPEN_EXISTING; + HANDLE _native = CreateFileA(p.string().data(), fileAccess, shareMode, &secAttribs, creationDisposition, FILE_ATTRIBUTE_NORMAL, nullptr); if (_native==INVALID_HANDLE_VALUE) { auto e = GetLastError(); @@ -107,4 +115,4 @@ bool isDebuggerAttached() return IsDebuggerPresent(); } -#endif \ No newline at end of file +#endif diff --git a/tools/nsc/main.cpp b/tools/nsc/main.cpp index 56152701b3..12738c038e 100644 --- a/tools/nsc/main.cpp +++ b/tools/nsc/main.cpp @@ -1,531 +1,554 @@ #include "nabla.h" #include "nbl/system/IApplicationFramework.h" - #include #include #include #include #include #include +#include +#include +#include +#include +#include +#include #include - #include "nbl/asset/metadata/CHLSLMetadata.h" #include "nlohmann/json.hpp" -using json = nlohmann::json; +using json = nlohmann::json; using namespace nbl; using namespace nbl::system; using namespace nbl::core; using namespace nbl::asset; -class NscLogger final : public system::IThreadsafeLogger +class TrimStdoutLogger final : public CStdoutLogger { public: - NscLogger(core::smart_refctd_ptr&& logFile, const core::bitflag logLevelMask, const core::bitflag consoleMask) - : IThreadsafeLogger(logLevelMask), m_logFile(std::move(logFile)), m_logPos(m_logFile ? m_logFile->getSize() : 0ull), m_consoleMask(consoleMask) - { - } + TrimStdoutLogger(const bitflag logLevelMask) : CStdoutLogger(logLevelMask) {} + +protected: + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto str = constructLogString(fmt, logLevel, args); + size_t size = str.size(); + while (size && str[size - 1] == '\0') + --size; + if (!size) + return; + std::fwrite(str.data(), 1, size, stdout); + std::fflush(stdout); + } +}; + +class TrimFileLogger final : public CFileLogger +{ +public: + using CFileLogger::CFileLogger; + +protected: + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto str = constructLogString(fmt, logLevel, args); + size_t size = str.size(); + while (size && str[size - 1] == '\0') + --size; + if (!size) + return; + IFile::success_t succ; + m_file->write(succ, str.data(), m_pos, size); + m_pos += succ.getBytesProcessed(); + } +}; + +class ShaderLogger final : public IThreadsafeLogger +{ +public: + ShaderLogger(smart_refctd_ptr system, path logPath, const bitflag fileMask, const bitflag consoleMask, const bool noLog) + : IThreadsafeLogger(fileMask | consoleMask), m_system(std::move(system)), m_logPath(std::move(logPath)), m_fileMask(fileMask), m_consoleMask(consoleMask), m_noLog(noLog) + { + m_stdoutLogger = make_smart_refctd_ptr(m_consoleMask); + beginBuild(); + } + + void beginBuild() + { + m_fileLogger = nullptr; + m_file = nullptr; + + if (m_noLog) + return; + if (!m_system || m_logPath.empty()) + return; + + const auto parent = std::filesystem::path(m_logPath).parent_path(); + if (!parent.empty() && !std::filesystem::exists(parent)) + std::filesystem::create_directories(parent); + + for (auto attempt = 0u; attempt < kDeleteRetries; ++attempt) + { + if (m_system->deleteFile(m_logPath)) + break; + std::this_thread::sleep_for(kDeleteDelay); + } + + ISystem::future_t> fut; + m_system->createFile(fut, m_logPath, kLogFlags); + + if (fut.wait()) + { + auto lk = fut.acquire(); + if (lk) + lk.move_into(m_file); + } + + if (!m_file) + return; + + m_fileLogger = make_smart_refctd_ptr(smart_refctd_ptr(m_file), true, m_fileMask); + } private: - void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override - { - const auto line = constructLogString(fmt, logLevel, args); - size_t lineSize = line.size(); - while (lineSize > 0 && line[lineSize - 1] == '\0') - --lineSize; - if (lineSize == 0) - return; - if (m_logFile) - { - system::IFile::success_t succ; - m_logFile->write(succ, line.data(), m_logPos, lineSize); - m_logPos += succ.getBytesProcessed(); - } - if (logLevel & m_consoleMask.value) - { - std::fwrite(line.data(), 1, lineSize, stdout); - std::fflush(stdout); - } - } - - core::smart_refctd_ptr m_logFile; - size_t m_logPos = 0ull; - core::bitflag m_consoleMask; + static constexpr auto kDeleteRetries = 3u; + static constexpr auto kDeleteDelay = std::chrono::milliseconds(100); + static constexpr auto kLogFlags = bitflag(IFileBase::ECF_WRITE) | IFileBase::ECF_SHARE_READ | IFileBase::ECF_SHARE_WRITE | IFileBase::ECF_SHARE_DELETE; + + static inline std::string formatMessageOnly(const std::string_view& fmt, va_list args) + { + va_list a; + va_copy(a, args); + const int n = std::vsnprintf(nullptr, 0, fmt.data(), a); + va_end(a); + if (n <= 0) + return {}; + std::string s(size_t(n) + 1u, '\0'); + std::vsnprintf(s.data(), s.size(), fmt.data(), args); + s.resize(size_t(n)); + return s; + } + + void threadsafeLog_impl(const std::string_view& fmt, E_LOG_LEVEL logLevel, va_list args) override + { + const auto msg = formatMessageOnly(fmt, args); + if (msg.empty()) + return; + + if (m_stdoutLogger && (logLevel & m_consoleMask.value)) + m_stdoutLogger->log("%s", logLevel, msg.c_str()); + + if (m_noLog || !(logLevel & m_fileMask.value) || !m_fileLogger) + return; + + m_fileLogger->log("%s", logLevel, msg.c_str()); + } + + smart_refctd_ptr m_system; + smart_refctd_ptr m_file; + smart_refctd_ptr m_stdoutLogger; + smart_refctd_ptr m_fileLogger; + path m_logPath; + bitflag m_fileMask; + bitflag m_consoleMask; + bool m_noLog = false; }; -class ShaderCompiler final : public system::IApplicationFramework +class ShaderCompiler final : public IApplicationFramework { - using base_t = system::IApplicationFramework; + using base_t = IApplicationFramework; public: - using base_t::base_t; - - bool onAppInitialized(smart_refctd_ptr&& system) override - { - const auto rawArgs = std::vector(argv.begin(), argv.end()); - auto expandArgs = [](const std::vector& args) - { - std::vector expanded; - expanded.reserve(args.size()); - for (const auto& arg : args) - { - if (arg.rfind("-MF", 0) == 0 && arg.size() > 3) - { - expanded.push_back("-MF"); - expanded.push_back(arg.substr(3)); - continue; - } - if (arg.rfind("-Fo", 0) == 0 && arg.size() > 3) - { - expanded.push_back("-Fo"); - expanded.push_back(arg.substr(3)); - continue; - } - if (arg.rfind("-Fc", 0) == 0 && arg.size() > 3) - { - expanded.push_back("-Fc"); - expanded.push_back(arg.substr(3)); - continue; - } - expanded.push_back(arg); - } - return expanded; - }; - - argparse::ArgumentParser program("nsc"); - program.add_argument("--dump-build-info").default_value(false).implicit_value(true); - program.add_argument("--file").default_value(std::string{}); - program.add_argument("-P").default_value(false).implicit_value(true); - program.add_argument("-no-nbl-builtins").default_value(false).implicit_value(true); - program.add_argument("-MD").default_value(false).implicit_value(true); - program.add_argument("-M").default_value(false).implicit_value(true); - program.add_argument("-MF").default_value(std::string{}); - program.add_argument("-Fo").default_value(std::string{}); - program.add_argument("-Fc").default_value(std::string{}); - program.add_argument("-log").default_value(std::string{}); - program.add_argument("-nolog").default_value(false).implicit_value(true); - program.add_argument("-quiet").default_value(false).implicit_value(true); - program.add_argument("-verbose").default_value(false).implicit_value(true); - - std::vector unknownArgs; - try - { - unknownArgs = program.parse_known_args(expandArgs(rawArgs)); - } - catch (const std::runtime_error& err) - { - std::cerr << err.what() << std::endl << program; - return false; - } - - if (program.get("--dump-build-info")) - { - json j; - - auto& modules = j["modules"]; - - auto serialize = [&](const gtml::GitInfo& info, std::string_view target) -> void - { - auto& s = modules[target.data()]; - - s["isPopulated"] = info.isPopulated; - if (info.hasUncommittedChanges.has_value()) - s["hasUncommittedChanges"] = info.hasUncommittedChanges.value(); - else - s["hasUncommittedChanges"] = "UNKNOWN, BUILT WITHOUT DIRTY-CHANGES CAPTURE"; - - s["commitAuthorName"] = info.commitAuthorName; - s["commitAuthorEmail"] = info.commitAuthorEmail; - s["commitHash"] = info.commitHash; - s["commitShortHash"] = info.commitShortHash; - s["commitDate"] = info.commitDate; - s["commitSubject"] = info.commitSubject; - s["commitBody"] = info.commitBody; - s["describe"] = info.describe; - s["branchName"] = info.branchName; - s["latestTag"] = info.latestTag; - s["latestTagName"] = info.latestTagName; - }; - - serialize(gtml::nabla_git_info, "nabla"); - serialize(gtml::dxc_git_info, "dxc"); - - const auto pretty = j.dump(4); - std::cout << pretty << std::endl; - - std::filesystem::path oPath = "build-info.json"; - - if (program.is_used("--file")) - { - const auto filePath = program.get("--file"); - if (!filePath.empty()) - oPath = filePath; - } - - std::ofstream outFile(oPath); - if (outFile.is_open()) - { - outFile << pretty; - outFile.close(); - printf("Saved \"%s\"\n", oPath.string().c_str()); - } - else - { - printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); - exit(-1); - } - - exit(0); - } - - if (not isAPILoaded()) - { - std::cerr << "Could not load Nabla API, terminating!"; - return false; - } - - if (system) - m_system = std::move(system); - else - m_system = system::IApplicationFramework::createSystem(); - - if (!m_system) - return false; - - const auto defaultConsoleMask = core::bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR; - m_logger = make_smart_refctd_ptr(defaultConsoleMask); - - if (rawArgs.size() < 2) - { - m_logger->log("Insufficient arguments.", ILogger::ELL_ERROR); - return false; - } - std::string file_to_compile = rawArgs.back(); - - if (!m_system->exists(file_to_compile, IFileBase::ECF_READ)) - { - m_logger->log("Input shader file does not exist: %s", ILogger::ELL_ERROR, file_to_compile.c_str()); - return false; - } - - const bool preprocessOnly = program.get("-P"); - const bool outputFlagFc = program.is_used("-Fc"); - const bool outputFlagFo = program.is_used("-Fo"); - if (outputFlagFc && outputFlagFo) - { - m_logger->log("Invalid arguments. Passed both -Fo and -Fc.", ILogger::ELL_ERROR); - return false; - } - if (!outputFlagFc && !outputFlagFo) - { - m_logger->log("Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.", ILogger::ELL_ERROR); - return false; - } - - std::string output_filepath = outputFlagFc ? program.get("-Fc") : program.get("-Fo"); - if (output_filepath.empty()) - { - m_logger->log("Invalid output file path.", ILogger::ELL_ERROR); - return false; - } - - const bool quietFlag = program.get("-quiet"); - const bool verboseFlag = program.get("-verbose"); - if (quietFlag && verboseFlag) - { - m_logger->log("Invalid arguments. Passed both -quiet and -verbose.", ILogger::ELL_ERROR); - return false; - } - - LogConfig logConfig; - if (verboseFlag) - logConfig.quiet = false; - if (quietFlag) - logConfig.quiet = true; - - logConfig.noLog = program.get("-nolog"); - if (program.is_used("-log")) - { - logConfig.path = program.get("-log"); - if (logConfig.path.empty()) - { - m_logger->log("Incorrect arguments. Expecting filename after -log.", ILogger::ELL_ERROR); - return false; - } - } - - if (logConfig.noLog && !logConfig.path.empty()) - { - m_logger->log("Invalid arguments. Passed both -nolog and -log.", ILogger::ELL_ERROR); - return false; - } - - const auto consoleMask = logConfig.quiet ? (core::bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR) : core::bitflag(ILogger::ELL_ALL); - m_logger = make_smart_refctd_ptr(consoleMask); - - if (!logConfig.noLog) - { - const std::filesystem::path logPath = logConfig.path.empty() ? std::filesystem::path(output_filepath).concat(".log") : std::filesystem::path(logConfig.path); - const auto parentDirectory = logPath.parent_path(); - if (!parentDirectory.empty() && !std::filesystem::exists(parentDirectory)) - std::filesystem::create_directories(parentDirectory); - - m_system->deleteFile(logPath); - - system::ISystem::future_t> future; - m_system->createFile(future, logPath, system::IFileBase::ECF_WRITE); - core::smart_refctd_ptr logFile; - if (future.wait()) - future.acquire().move_into(logFile); - - if (logFile) - m_logger = make_smart_refctd_ptr(std::move(logFile), core::bitflag(ILogger::ELL_ALL), consoleMask); - else - m_logger->log("Failed to open log file: %s", ILogger::ELL_ERROR, logPath.string().c_str()); - } - - const char* action = preprocessOnly ? "Preprocessing" : "Compiling"; - m_logger->log("%s %s", ILogger::ELL_INFO, action, file_to_compile.c_str()); - const char* outputType = preprocessOnly ? "Preprocessed" : "Compiled"; - m_logger->log("%s shader code will be saved to %s", ILogger::ELL_INFO, outputType, output_filepath.c_str()); - - m_arguments = std::move(unknownArgs); - if (!m_arguments.empty() && m_arguments.back() == file_to_compile) - m_arguments.pop_back(); - - no_nbl_builtins = program.get("-no-nbl-builtins"); - if (no_nbl_builtins) - { - m_logger->log("Unmounting builtins."); - m_system->unmountBuiltins(); - } - - DepfileConfig depfileConfig; - if (program.get("-MD") || program.get("-M") || program.is_used("-MF")) - depfileConfig.enabled = true; - if (program.is_used("-MF")) - depfileConfig.path = program.get("-MF"); - if (depfileConfig.enabled && depfileConfig.path.empty()) - depfileConfig.path = output_filepath + ".d"; - if (depfileConfig.enabled) - m_logger->log("Dependency file will be saved to %s", ILogger::ELL_INFO, depfileConfig.path.c_str()); + using base_t::base_t; + + bool onAppInitialized(smart_refctd_ptr&& system) override + { + const auto rawArgs = std::vector(argv.begin(), argv.end()); + const auto expandedArgs = expandJoinedArgs(rawArgs); + + argparse::ArgumentParser program("nsc"); + program.add_argument("--dump-build-info").default_value(false).implicit_value(true); + program.add_argument("--file").default_value(std::string{}); + program.add_argument("-P").default_value(false).implicit_value(true); + program.add_argument("-no-nbl-builtins").default_value(false).implicit_value(true); + program.add_argument("-MD").default_value(false).implicit_value(true); + program.add_argument("-M").default_value(false).implicit_value(true); + program.add_argument("-MF").default_value(std::string{}); + program.add_argument("-Fo").default_value(std::string{}); + program.add_argument("-Fc").default_value(std::string{}); + program.add_argument("-log").default_value(std::string{}); + program.add_argument("-nolog").default_value(false).implicit_value(true); + program.add_argument("-quiet").default_value(false).implicit_value(true); + program.add_argument("-verbose").default_value(false).implicit_value(true); + + std::vector unknownArgs; + try + { + unknownArgs = program.parse_known_args(expandedArgs); + } + catch (const std::runtime_error& err) + { + std::cerr << err.what() << std::endl << program; + return false; + } + + if (program.get("--dump-build-info")) + { + dumpBuildInfo(program); + std::exit(0); + } + + if (!isAPILoaded()) + { + std::cerr << "Could not load Nabla API, terminating!"; + return false; + } + + m_system = system ? std::move(system) : IApplicationFramework::createSystem(); + if (!m_system) + return false; + + if (rawArgs.size() < 2) + { + std::cerr << "Insufficient arguments.\n"; + return false; + } + + const std::string fileToCompile = rawArgs.back(); + if (!m_system->exists(fileToCompile, IFileBase::ECF_READ)) + { + std::cerr << "Input shader file does not exist: " << fileToCompile << "\n"; + return false; + } + + const bool preprocessOnly = program.get("-P"); + const bool hasFc = program.is_used("-Fc"); + const bool hasFo = program.is_used("-Fo"); + + if (hasFc == hasFo) + { + if (hasFc) + std::cerr << "Invalid arguments. Passed both -Fo and -Fc.\n"; + else + std::cerr << "Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.\n"; + return false; + } + + const std::string outputFilepath = hasFc ? program.get("-Fc") : program.get("-Fo"); + if (outputFilepath.empty()) + { + std::cerr << "Invalid output file path.\n"; + return false; + } + + const bool quiet = program.get("-quiet"); + const bool verbose = program.get("-verbose"); + if (quiet && verbose) + { + std::cerr << "Invalid arguments. Passed both -quiet and -verbose.\n"; + return false; + } + + const bool noLog = program.get("-nolog"); + const std::string logPathOverride = program.is_used("-log") ? program.get("-log") : std::string{}; + if (noLog && !logPathOverride.empty()) + { + std::cerr << "Invalid arguments. Passed both -nolog and -log.\n"; + return false; + } + + const auto logPath = logPathOverride.empty() ? std::filesystem::path(outputFilepath).concat(".log") : std::filesystem::path(logPathOverride); + const auto fileMask = bitflag(ILogger::ELL_ALL); + const auto consoleMask = bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR; + + m_logger = make_smart_refctd_ptr(m_system, logPath, fileMask, consoleMask, noLog); + + m_arguments = std::move(unknownArgs); + if (!m_arguments.empty() && m_arguments.back() == fileToCompile) + m_arguments.pop_back(); + + bool noNblBuiltins = program.get("-no-nbl-builtins"); + if (noNblBuiltins) + { + m_logger->log("Unmounting builtins."); + m_system->unmountBuiltins(); + } + + DepfileConfig dep; + if (program.get("-MD") || program.get("-M") || program.is_used("-MF")) + dep.enabled = true; + if (program.is_used("-MF")) + dep.path = program.get("-MF"); + if (dep.enabled && dep.path.empty()) + dep.path = outputFilepath + ".d"; + if (dep.enabled) + m_logger->log("Dependency file will be saved to %s", ILogger::ELL_INFO, dep.path.c_str()); #ifndef NBL_EMBED_BUILTIN_RESOURCES - if (!no_nbl_builtins) { - m_system->unmountBuiltins(); - no_nbl_builtins = true; - m_logger->log("nsc.exe was compiled with builtin resources disabled. Force enabling -no-nbl-builtins.", ILogger::ELL_WARNING); - } + if (!noNblBuiltins) + { + m_system->unmountBuiltins(); + noNblBuiltins = true; + m_logger->log("nsc.exe was compiled with builtin resources disabled. Force enabling -no-nbl-builtins.", ILogger::ELL_WARNING); + } #endif - if (std::find(m_arguments.begin(), m_arguments.end(), "-E") == m_arguments.end()) - { - m_arguments.push_back("-E"); - m_arguments.push_back("main"); - } - - for (size_t i = 0; i + 1 < m_arguments.size(); ++i) - { - const auto& arg = m_arguments[i]; - if (arg == "-I") - m_include_search_paths.emplace_back(m_arguments[i + 1]); - } - - auto [shader, shaderStage] = open_shader_file(file_to_compile); - if (shader->getContentType() != IShader::E_CONTENT_TYPE::ECT_HLSL) - { - m_logger->log("Error. Loaded shader file content is not HLSL.", ILogger::ELL_ERROR); - return false; - } - - auto start = std::chrono::high_resolution_clock::now(); - smart_refctd_ptr compilation_result; - std::string preprocessing_result; - std::string_view result_view; - if (preprocessOnly) - { - preprocessing_result = preprocess_shader(shader.get(), shaderStage, file_to_compile, depfileConfig); - result_view = preprocessing_result; - } - else - { - compilation_result = compile_shader(shader.get(), shaderStage, file_to_compile, depfileConfig); - result_view = { (const char*)compilation_result->getContent()->getPointer(), compilation_result->getContent()->getSize() }; - } - auto end = std::chrono::high_resolution_clock::now(); - - std::string operationType = preprocessOnly ? "preprocessing" : "compilation"; - const bool success = preprocessOnly ? preprocessing_result != std::string{} : bool(compilation_result); - if (success) - { - m_logger->log("Shader " + operationType + " successful.", ILogger::ELL_INFO); - const auto took = std::to_string(std::chrono::duration_cast(end - start).count()); - m_logger->log("Took %s ms.", ILogger::ELL_PERFORMANCE, took.c_str()); - { - const auto location = std::filesystem::path(output_filepath); - const auto parentDirectory = location.parent_path(); - - if (!std::filesystem::exists(parentDirectory)) - { - if (!std::filesystem::create_directories(parentDirectory)) - { - m_logger->log("Failed to create parent directory for the " + output_filepath + "output!", ILogger::ELL_ERROR); - return false; - } - } - } - - std::fstream output_file(output_filepath, std::ios::out | std::ios::binary); - - if (!output_file.is_open()) - { - m_logger->log("Failed to open output file: " + output_filepath, ILogger::ELL_ERROR); - return false; - } - - output_file.write(result_view.data(), result_view.size()); - - if (output_file.fail()) - { - m_logger->log("Failed to write to output file: " + output_filepath, ILogger::ELL_ERROR); - output_file.close(); - return false; - } - - output_file.close(); - - if (output_file.fail()) - { - m_logger->log("Failed to close output file: " + output_filepath, ILogger::ELL_ERROR); - return false; - } - - if (depfileConfig.enabled) - m_logger->log("Dependency file written to %s", ILogger::ELL_INFO, depfileConfig.path.c_str()); - - return true; - } - else - { - m_logger->log("Shader " + operationType + " failed.", ILogger::ELL_ERROR); - return false; - } - } - - void workLoopBody() override {} - - bool keepRunning() override { return false; } + if (std::find(m_arguments.begin(), m_arguments.end(), "-E") == m_arguments.end()) + { + m_arguments.push_back("-E"); + m_arguments.push_back("main"); + } + + for (size_t i = 0; i + 1 < m_arguments.size(); ++i) + { + if (m_arguments[i] == "-I") + m_include_search_paths.emplace_back(m_arguments[i + 1]); + } + + const char* const action = preprocessOnly ? "Preprocessing" : "Compiling"; + const char* const outType = preprocessOnly ? "Preprocessed" : "Compiled"; + m_logger->log("%s %s", ILogger::ELL_INFO, action, fileToCompile.c_str()); + m_logger->log("%s shader code will be saved to %s", ILogger::ELL_INFO, outType, outputFilepath.c_str()); + + auto [shader, shaderStage] = open_shader_file(fileToCompile); + if (!shader || shader->getContentType() != IShader::E_CONTENT_TYPE::ECT_HLSL) + { + m_logger->log("Error. Loaded shader file content is not HLSL.", ILogger::ELL_ERROR); + return false; + } + + const auto start = std::chrono::high_resolution_clock::now(); + const auto job = runShaderJob(shader.get(), shaderStage, fileToCompile, dep, preprocessOnly); + const auto end = std::chrono::high_resolution_clock::now(); + + const char* const op = preprocessOnly ? "preprocessing" : "compilation"; + if (!job.ok) + { + m_logger->log("Shader %s failed.", ILogger::ELL_ERROR, op); + return false; + } + + const auto took = std::to_string(std::chrono::duration_cast(end - start).count()); + m_logger->log("Shader %s successful.", ILogger::ELL_INFO, op); + m_logger->log("Took %s ms.", ILogger::ELL_PERFORMANCE, took.c_str()); + + const auto outParent = std::filesystem::path(outputFilepath).parent_path(); + if (!outParent.empty() && !std::filesystem::exists(outParent)) + { + if (!std::filesystem::create_directories(outParent)) + { + m_logger->log("Failed to create parent directory for output %s.", ILogger::ELL_ERROR, outputFilepath.c_str()); + return false; + } + } + + std::fstream out(outputFilepath, std::ios::out | std::ios::binary); + if (!out.is_open()) + { + m_logger->log("Failed to open output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + return false; + } + + out.write(job.view.data(), job.view.size()); + if (out.fail()) + { + m_logger->log("Failed to write to output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + out.close(); + return false; + } + + out.close(); + if (out.fail()) + { + m_logger->log("Failed to close output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + return false; + } + + if (dep.enabled) + m_logger->log("Dependency file written to %s", ILogger::ELL_INFO, dep.path.c_str()); + + return true; + } + + void workLoopBody() override {} + bool keepRunning() override { return false; } private: - - struct LogConfig - { - bool quiet = true; - bool noLog = false; - std::string path; - }; - - struct DepfileConfig - { - bool enabled = false; - std::string path; - }; - - std::string preprocess_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& depfileConfig) { - smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - - CHLSLCompiler::SPreprocessorOptions options = {}; - options.sourceIdentifier = sourceIdentifier; - options.logger = m_logger.get(); - - auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - auto includeLoader = includeFinder->getDefaultFileSystemLoader(); - - // because before real compilation we do preprocess the input it doesn't really matter we proxy include search direcotries further with dxcOptions since at the end all includes are resolved to single file - for (const auto& it : m_include_search_paths) - includeFinder->addSearchPath(it, includeLoader); - - options.includeFinder = includeFinder.get(); - options.depfile = depfileConfig.enabled; - options.depfilePath = depfileConfig.path; - - const char* code_ptr = (const char*)shader->getContent()->getPointer(); - std::string_view code({ code_ptr, strlen(code_ptr)}); - - return hlslcompiler->preprocessShader(std::string(code), shaderStage, options, nullptr); - } - - core::smart_refctd_ptr compile_shader(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& depfileConfig) { - smart_refctd_ptr hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - - CHLSLCompiler::SOptions options = {}; - options.stage = shaderStage; - options.preprocessorOptions.sourceIdentifier = sourceIdentifier; - options.preprocessorOptions.logger = m_logger.get(); - - options.debugInfoFlags = core::bitflag(asset::IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT); - options.dxcOptions = std::span(m_arguments); - - auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - auto includeLoader = includeFinder->getDefaultFileSystemLoader(); - - // because before real compilation we do preprocess the input it doesn't really matter we proxy include search direcotries further with dxcOptions since at the end all includes are resolved to single file - for(const auto& it : m_include_search_paths) - includeFinder->addSearchPath(it, includeLoader); - - options.preprocessorOptions.includeFinder = includeFinder.get(); - options.preprocessorOptions.depfile = depfileConfig.enabled; - options.preprocessorOptions.depfilePath = depfileConfig.path; - - return hlslcompiler->compileToSPIRV((const char*)shader->getContent()->getPointer(), options); - } - - - std::tuple, hlsl::ShaderStage> open_shader_file(std::string filepath) { - - m_assetMgr = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - - IAssetLoader::SAssetLoadParams lp = {}; - lp.logger = m_logger.get(); - lp.workingDirectory = localInputCWD; - auto assetBundle = m_assetMgr->getAsset(filepath, lp); - const auto assets = assetBundle.getContents(); - const auto* metadata = assetBundle.getMetadata(); - if (assets.empty()) { - m_logger->log("Could not load shader %s", ILogger::ELL_ERROR, filepath); - return {nullptr, hlsl::ShaderStage::ESS_UNKNOWN}; - } - assert(assets.size() == 1); - - // could happen when the file is missing an extension and we can't deduce its a shader - if (assetBundle.getAssetType() == IAsset::ET_BUFFER) - { - auto buf = IAsset::castDown(assets[0]); - std::string source; source.resize(buf->getSize()+1); - memcpy(source.data(),buf->getPointer(),buf->getSize()); - return { core::make_smart_refctd_ptr(source.data(), IShader::E_CONTENT_TYPE::ECT_HLSL, std::move(filepath)), hlsl::ShaderStage::ESS_UNKNOWN}; - } - else if (assetBundle.getAssetType() == IAsset::ET_SHADER) - { - const auto hlslMetadata = static_cast(metadata); - return { smart_refctd_ptr_static_cast(assets[0]), hlslMetadata->shaderStages->front()}; - } - else - { - m_logger->log("file '%s' is an asset that is neither a buffer or a shader.", ILogger::ELL_ERROR, filepath); - } - - return {nullptr, hlsl::ShaderStage::ESS_UNKNOWN}; - } - - - bool no_nbl_builtins{ false }; - smart_refctd_ptr m_system; - smart_refctd_ptr m_logger; - std::vector m_arguments, m_include_search_paths; - core::smart_refctd_ptr m_assetMgr; - - + struct DepfileConfig + { + bool enabled = false; + std::string path; + }; + + struct RunResult + { + bool ok = false; + std::string text; + smart_refctd_ptr compiled; + std::string_view view; + }; + + static std::vector expandJoinedArgs(const std::vector& args) + { + std::vector out; + out.reserve(args.size()); + + auto split = [&](const std::string& a, const char* p) + { + const size_t n = std::strlen(p); + if (a.rfind(p, 0) == 0 && a.size() > n) + { + out.emplace_back(p); + out.emplace_back(a.substr(n)); + return true; + } + return false; + }; + + for (const auto& a : args) + { + if (split(a, "-MF")) continue; + if (split(a, "-Fo")) continue; + if (split(a, "-Fc")) continue; + out.push_back(a); + } + + return out; + } + + static void dumpBuildInfo(const argparse::ArgumentParser& program) + { + json j; + auto& modules = j["modules"]; + + auto serialize = [&](const gtml::GitInfo& info, std::string_view target) + { + auto& s = modules[target.data()]; + s["isPopulated"] = info.isPopulated; + s["hasUncommittedChanges"] = info.hasUncommittedChanges.has_value() ? json(info.hasUncommittedChanges.value()) : json("UNKNOWN, BUILT WITHOUT DIRTY-CHANGES CAPTURE"); + s["commitAuthorName"] = info.commitAuthorName; + s["commitAuthorEmail"] = info.commitAuthorEmail; + s["commitHash"] = info.commitHash; + s["commitShortHash"] = info.commitShortHash; + s["commitDate"] = info.commitDate; + s["commitSubject"] = info.commitSubject; + s["commitBody"] = info.commitBody; + s["describe"] = info.describe; + s["branchName"] = info.branchName; + s["latestTag"] = info.latestTag; + s["latestTagName"] = info.latestTagName; + }; + + serialize(gtml::nabla_git_info, "nabla"); + serialize(gtml::dxc_git_info, "dxc"); + + const auto pretty = j.dump(4); + std::cout << pretty << std::endl; + + std::filesystem::path oPath = "build-info.json"; + if (program.is_used("--file")) + { + const auto filePath = program.get("--file"); + if (!filePath.empty()) + oPath = filePath; + } + + std::ofstream outFile(oPath); + if (!outFile.is_open()) + { + std::printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); + std::exit(-1); + } + + outFile << pretty; + std::printf("Saved \"%s\"\n", oPath.string().c_str()); + } + + RunResult runShaderJob(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& dep, const bool preprocessOnly) + { + RunResult r; + auto hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + + auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + auto includeLoader = includeFinder->getDefaultFileSystemLoader(); + for (const auto& p : m_include_search_paths) + includeFinder->addSearchPath(p, includeLoader); + + if (preprocessOnly) + { + CHLSLCompiler::SPreprocessorOptions opt = {}; + opt.sourceIdentifier = sourceIdentifier; + opt.logger = m_logger.get(); + opt.includeFinder = includeFinder.get(); + opt.depfile = dep.enabled; + opt.depfilePath = dep.path; + + const char* codePtr = (const char*)shader->getContent()->getPointer(); + std::string_view code(codePtr, std::strlen(codePtr)); + + r.text = hlslcompiler->preprocessShader(std::string(code), shaderStage, opt, nullptr); + r.ok = !r.text.empty(); + r.view = r.text; + return r; + } + + CHLSLCompiler::SOptions opt = {}; + opt.stage = shaderStage; + opt.preprocessorOptions.sourceIdentifier = sourceIdentifier; + opt.preprocessorOptions.logger = m_logger.get(); + opt.preprocessorOptions.includeFinder = includeFinder.get(); + opt.preprocessorOptions.depfile = dep.enabled; + opt.preprocessorOptions.depfilePath = dep.path; + opt.debugInfoFlags = bitflag(IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT); + opt.dxcOptions = std::span(m_arguments); + + r.compiled = hlslcompiler->compileToSPIRV((const char*)shader->getContent()->getPointer(), opt); + r.ok = bool(r.compiled); + if (r.ok) + r.view = { (const char*)r.compiled->getContent()->getPointer(), r.compiled->getContent()->getSize() }; + + return r; + } + + std::tuple, hlsl::ShaderStage> open_shader_file(std::string filepath) + { + m_assetMgr = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + + IAssetLoader::SAssetLoadParams lp = {}; + lp.logger = m_logger.get(); + lp.workingDirectory = localInputCWD; + + auto bundle = m_assetMgr->getAsset(filepath, lp); + const auto assets = bundle.getContents(); + const auto* metadata = bundle.getMetadata(); + + if (assets.empty()) + { + m_logger->log("Could not load shader %s", ILogger::ELL_ERROR, filepath.c_str()); + return { nullptr, hlsl::ShaderStage::ESS_UNKNOWN }; + } + + if (bundle.getAssetType() == IAsset::ET_BUFFER) + { + auto buf = IAsset::castDown(assets[0]); + std::string source; + source.resize(buf->getSize() + 1); + std::memcpy(source.data(), buf->getPointer(), buf->getSize()); + return { make_smart_refctd_ptr(source.data(), IShader::E_CONTENT_TYPE::ECT_HLSL, std::move(filepath)), hlsl::ShaderStage::ESS_UNKNOWN }; + } + + if (bundle.getAssetType() == IAsset::ET_SHADER) + { + const auto hlslMetadata = static_cast(metadata); + return { smart_refctd_ptr_static_cast(assets[0]), hlslMetadata->shaderStages->front() }; + } + + m_logger->log("file '%s' is an asset that is neither a buffer or a shader.", ILogger::ELL_ERROR, filepath.c_str()); + return { nullptr, hlsl::ShaderStage::ESS_UNKNOWN }; + } + + smart_refctd_ptr m_system; + smart_refctd_ptr m_logger; + std::vector m_arguments, m_include_search_paths; + smart_refctd_ptr m_assetMgr; }; NBL_MAIN_FUNC(ShaderCompiler) From 00a613d1166f919254a03d56fce5dfab2b946f9a Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 5 Jan 2026 01:04:38 +0100 Subject: [PATCH 7/8] cmake: show config name in NSC comment --- cmake/common.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index a4ce345096..2de6dc758f 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1514,6 +1514,7 @@ namespace @IMPL_NAMESPACE@ { ) get_filename_component(NBL_NSC_INPUT_NAME "${TARGET_INPUT}" NAME) + get_filename_component(NBL_NSC_CONFIG_NAME "${CONFIG_FILE}" NAME) set(NBL_NSC_BYPRODUCTS "${NBL_NSC_LOG_PATH}") if(NSC_USE_DEPFILE) list(APPEND NBL_NSC_BYPRODUCTS "${DEPFILE_PATH}") @@ -1524,7 +1525,7 @@ namespace @IMPL_NAMESPACE@ { BYPRODUCTS ${NBL_NSC_BYPRODUCTS} COMMAND ${NBL_NSC_COMPILE_COMMAND} DEPENDS ${DEPENDS_ON} - COMMENT "${NBL_NSC_INPUT_NAME}" + COMMENT "${NBL_NSC_CONFIG_NAME} (${NBL_NSC_INPUT_NAME})" VERBATIM COMMAND_EXPAND_LISTS ) From dc13c4544e74d89a2807fa20a954d45e21a7124d Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 5 Jan 2026 07:19:12 +0100 Subject: [PATCH 8/8] ECF_SHARE_READ_WRITE for backwards compatibility --- include/nbl/system/IFileBase.h | 3 +-- src/nbl/system/CSystemWin32.cpp | 7 ++----- tools/nsc/main.cpp | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/include/nbl/system/IFileBase.h b/include/nbl/system/IFileBase.h index 58ab34fac5..c9ceb13a04 100644 --- a/include/nbl/system/IFileBase.h +++ b/include/nbl/system/IFileBase.h @@ -26,8 +26,7 @@ class IFileBase : public core::IReferenceCounted ECF_MAPPABLE = 0b0100, //! Implies ECF_MAPPABLE ECF_COHERENT = 0b1100, - ECF_SHARE_READ = 0b10000, - ECF_SHARE_WRITE = 0b100000, + ECF_SHARE_READ_WRITE = 0b100000, ECF_SHARE_DELETE = 0b1000000 }; diff --git a/src/nbl/system/CSystemWin32.cpp b/src/nbl/system/CSystemWin32.cpp index af33e1460d..2798b4fb27 100644 --- a/src/nbl/system/CSystemWin32.cpp +++ b/src/nbl/system/CSystemWin32.cpp @@ -43,11 +43,8 @@ core::smart_refctd_ptr CSystemWin32::CCaller::createFile(const std: { const bool writeAccess = flags.value&IFile::ECF_WRITE; const DWORD fileAccess = ((flags.value&IFile::ECF_READ) ? FILE_GENERIC_READ:0)|(writeAccess ? FILE_GENERIC_WRITE:0); - const bool hasShareFlags = flags.value & (IFile::ECF_SHARE_READ | IFile::ECF_SHARE_WRITE | IFile::ECF_SHARE_DELETE); - DWORD shareMode = hasShareFlags ? 0 : FILE_SHARE_READ; - if (flags.value & IFile::ECF_SHARE_READ) - shareMode |= FILE_SHARE_READ; - if (flags.value & IFile::ECF_SHARE_WRITE) + DWORD shareMode = FILE_SHARE_READ; + if (flags.value & IFile::ECF_SHARE_READ_WRITE) shareMode |= FILE_SHARE_WRITE; if (flags.value & IFile::ECF_SHARE_DELETE) shareMode |= FILE_SHARE_DELETE; diff --git a/tools/nsc/main.cpp b/tools/nsc/main.cpp index 12738c038e..5ab01d72e5 100644 --- a/tools/nsc/main.cpp +++ b/tools/nsc/main.cpp @@ -111,7 +111,7 @@ class ShaderLogger final : public IThreadsafeLogger private: static constexpr auto kDeleteRetries = 3u; static constexpr auto kDeleteDelay = std::chrono::milliseconds(100); - static constexpr auto kLogFlags = bitflag(IFileBase::ECF_WRITE) | IFileBase::ECF_SHARE_READ | IFileBase::ECF_SHARE_WRITE | IFileBase::ECF_SHARE_DELETE; + static constexpr auto kLogFlags = bitflag(IFileBase::ECF_WRITE) | IFileBase::ECF_SHARE_READ_WRITE | IFileBase::ECF_SHARE_DELETE; static inline std::string formatMessageOnly(const std::string_view& fmt, va_list args) {