Merge pull request #35 from spacemeshos/long-run-test-app

Long run test app

Author: avive

.github/workflows/build.yml

@@ -16,6 +16,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Install CUDA
       env:
@@ -60,6 +62,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Install CUDA
       run: choco install cuda --version=${{ matrix.cuda }} --checksum ${{ matrix.cuda_hash }}
@@ -110,11 +114,13 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
 
     - name: Install Vulkan
       run: .github/tools/macos-install-vulkan.sh
 
-    - run: cmake -DSPACEMESHCL=OFF -DSPACEMESHCUDA=OFF .
+    - run: cmake -DSPACEMESHCUDA=OFF .
 
     - run: make
 
@@ -137,6 +143,24 @@ jobs:
         name: macos
         path: include/api.h
 
+    - name: Upload MoltenVK
+      uses: actions/upload-artifact@v2
+      with:
+        name: macos
+        path: ${{ env.VULKAN_ROOT_LOCATION }}/MoltenVK/dylib/macOS/libMoltenVK.dylib
+
+    - name: Upload MoltenVK config
+      uses: actions/upload-artifact@v2
+      with:
+        name: macos
+        path: ${{ env.VULKAN_ROOT_LOCATION }}/MoltenVK/dylib/macOS/MoltenVK_icd.json
+
+    - name: Upload Vulkan
+      uses: actions/upload-artifact@v2
+      with:
+        name: macos
+        path: ${{ env.VULKAN_ROOT_LOCATION }}/macOS/lib/libvulkan.*.dylib
+
   release:
     runs-on: ubuntu-latest
     needs: [linux, windows, macos]

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "zlib"]
+	path = src/vulkan/zlib
+	url = https://github.com/madler/zlib

gpu-setup-test

@@ -26,6 +26,9 @@ if (MSVC)
     )
     include_directories(compat)
     add_definitions(-D_CRT_SECURE_NO_WARNINGS)
+else()
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 endif()
 
 include_directories(.)
@@ -92,6 +95,13 @@ if (SPACEMESHCUDA)
     )
 endif()
 
+add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/vulkan-shaders-vault.inl
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vulkan/gen/shaders-gen
+    COMMENT "Generating Vulkan Kernels Byte Array"
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/vulkan/gen/scrypt-chacha.comp ${CMAKE_CURRENT_BINARY_DIR}/vulkan/gen/shaders-gen
+)
+
 if (SPACEMESHVULKAN)
     # Look for vulkan SDK
     # For a Windows build on Linux, install the Vulkan SDK with wine, and set VULKAN_SDK to ..../.wine/drive_c/VulkanSDK/1.x.y
@@ -119,8 +129,15 @@ if (SPACEMESHVULKAN)
         vulkan/driver-vulkan.h
         vulkan/vulkan-helpers.c
         vulkan/vulkan-helpers.h
+
+		${CMAKE_CURRENT_BINARY_DIR}/vulkan-shaders-vault.inl
     )
 
+    include_directories(vulkan/zlib)
+	include_directories(${CMAKE_CURRENT_BINARY_DIR})
+	include_directories(${CMAKE_CURRENT_BINARY_DIR}/vulkan/zlib)
+
+    add_subdirectory(vulkan/zlib)
     add_subdirectory(vulkan/gen)
 endif()
 
@@ -139,6 +156,10 @@ if (SPACEMESH_LIB_SHARED)
             target_link_libraries(gpu-setup -L${CUDA_LIB_PATH} -lcuda)
         endif()
     endif()
+
+	if (SPACEMESHVULKAN)
+	    target_link_libraries(gpu-setup zlibstatic)
+    endif()
 else()
     if (SPACEMESHCUDA)
         cuda_add_library(gpu-setup ${SOURCES})

macOS/.env

@@ -229,9 +229,9 @@ static _vulkanState *initVulkan(struct cgpu_info *cgpu, char *name, size_t nameS
 
 	state->pipeline = compileShader(state->vkDevice, state->pipelineLayout, &state->shaderModule, scrypt_chacha_comp, options, (int)cgpu->work_size, hash_len_bits, copy_only);
 #else
-	char filename[64];
-	snprintf(filename, sizeof(filename), "kernel-%02d-%03d.spirv", (int)cgpu->work_size, hash_len_bits);
-	state->pipeline = loadShader(state->vkDevice, state->pipelineLayout, &state->shaderModule, filename);
+//	char filename[64];
+//	snprintf(filename, sizeof(filename), "kernel-%02d-%03d.spirv", (int)cgpu->work_size, hash_len_bits);
+	state->pipeline = loadShader(state->vkDevice, state->pipelineLayout, &state->shaderModule, cgpu->work_size, hash_len_bits);
 #endif
 	if (!state->pipeline) {
 		return NULL;

macOS/MoltenVK_icd.json

@@ -19,6 +19,9 @@ set(SOURCES
     ${CMAKE_CURRENT_BINARY_DIR}/scrypt-chacha-vulkan.inl
 )
 
+include_directories(vulkan/zlib)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/vulkan/zlib)
+
 add_executable(shaders-gen ${SOURCES})
 
 # Add Include/Lib (Windows) and include/lib (Linux)
@@ -29,3 +32,5 @@ elseif(APPLE)
 elseif(UNIX)
     target_link_libraries(shaders-gen -L$ENV{VULKAN_SDK}/lib -lvulkan shaderc_combined)
 endif()
+
+target_link_libraries(shaders-gen zlibstatic)

macOS/kernel-64-001.spirv

@@ -20,6 +20,8 @@
 #include <vector>
 #include <stdarg.h>
 
+#include "zlib.h"
+
 #define	VK_NO_PROTOTYPES 1
 #include <vulkan/vulkan.h>
 #include <vulkan/vulkan_core.h>
@@ -457,19 +459,56 @@ int main(int argc, char **argv)
 {
 	init_glslang();
 
+	std::vector<uint8_t> out((4 + 4 * 256) * sizeof(uint32_t));
+	uint32_t *header = (uint32_t *)out.data();
+
+	header[0] = 1;
+	header[1] = 64;
+	header[2] = 12;
+	header[3] = 256;
+
 	for (int hash_len_bits = 1; hash_len_bits <= 256; hash_len_bits++) {
 		char options[256];
 		std::vector<uint32_t> spirv;
 
 		snprintf(options, sizeof(options), "#version 450\n#define LOOKUP_GAP %d\n#define WORKSIZE %d\n#define LABEL_SIZE %d\n",	4, 64, hash_len_bits);
 
 		if (compileShaderToSpirV(scrypt_chacha_comp, options, 64, hash_len_bits, false, spirv)) {
-			char filename[128];
-			snprintf(filename, sizeof(filename), "kernel-%02d-%03d.spirv", 64, hash_len_bits);
-			saveToFile(filename, spirv.data(), spirv.size() * sizeof(uint32_t));
+//			char filename[128];
+//			snprintf(filename, sizeof(filename), "kernel-%02d-%03d.spirv", 64, hash_len_bits);
+//			saveToFile(filename, spirv.data(), spirv.size() * sizeof(uint32_t));
+			uLongf destLen = spirv.size() * sizeof(uint32_t);
+			std::vector<Bytef> dst(destLen);
+			if (Z_OK != compress2(dst.data(), &destLen, (Bytef*)spirv.data(), destLen, 9)) {
+				return 1;
+			}
+			header[hash_len_bits * 4 + 0] = hash_len_bits;
+			header[hash_len_bits * 4 + 1] = spirv.size() * sizeof(uint32_t);
+			header[hash_len_bits * 4 + 2] = destLen;
+			header[hash_len_bits * 4 + 3] = out.size();
+
+			printf("64:%03u %u -> %u\n", header[hash_len_bits * 4 + 0], header[hash_len_bits * 4 + 1], header[hash_len_bits * 4 + 2]);
+
+			out.insert(out.end(), dst.begin(), dst.begin() + destLen);
+			header = (uint32_t *)out.data();
 		}
 	}
 
+	if (FILE *h = fopen("vulkan-shaders-vault.inl", "wb")) {
+		fprintf(h, "uint8_t vulkan_shaders_vault[] = {\r\n");
+		uint8_t *src = out.data();
+		for (uint32_t length = (uint32_t)out.size(); length > 0; length -= std::min<uint32_t>(32, length)) {
+			for (int i = 0; i < std::min<uint32_t>(32, length); i++) {
+				fprintf(h, "0x%02x, ", *src++);
+			}
+			fprintf(h, "\n");
+		}
+
+		fprintf(h, "};\r\n");
+
+		fclose(h);
+	}
+
 	finalize_glslang();
 
 	return 0;

macOS/kernel-64-002.spirv

@@ -6,6 +6,8 @@
 
 #ifdef HAVE_VULKAN
 #include "vulkan-helpers.h"
+#include "zlib.h"
+#include "vulkan-shaders-vault.inl"
 
 Vulkan gVulkan = { NULL };
 VkInstance gInstance = NULL;
@@ -28,6 +30,10 @@ int initVulkanLibrary()
 			gVulkan.library = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL);
 		}
 #elif defined( __APPLE__ )
+		const char *idc = getenv("VK_ICD_FILENAMES");
+		if (NULL == idc || 0 == idc[0]) {
+			setenv("VK_ICD_FILENAMES", "./MoltenVK_icd.json", 1);
+		}
 		gVulkan.library = dlopen("libvulkan.dylib", RTLD_NOW | RTLD_LOCAL);
 #elif defined( _WIN32 )
 		gVulkan.library = LoadLibraryA("vulkan-1.dll");
@@ -301,7 +307,7 @@ uint64_t getBufferMemoryRequirements(VkDevice vkDevice, VkBuffer b)
 	return req.alignment;
 }
 
-VkPipeline loadShader(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShaderModule *shader_module, const char * spirv_file_name)
+VkPipeline loadShaderFromFile(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShaderModule *shader_module, const char * spirv_file_name)
 {
 	uint32_t *shader;
 	size_t shader_size;
@@ -361,5 +367,86 @@ VkPipeline loadShader(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShad
 	return pipeline;
 }
 
+static uint32_t * getShader(uint32_t workSize, uint32_t labelSize, uint32_t *shader_size)
+{
+	uint32_t *vulkan_shaders_vault_header = (uint32_t *)vulkan_shaders_vault;
+	uint32_t count;
+
+	for (count = *vulkan_shaders_vault_header++; count > 0; count--, vulkan_shaders_vault_header += 2) {
+		if (vulkan_shaders_vault_header[0] == workSize) {
+			vulkan_shaders_vault_header = (uint32_t *)(vulkan_shaders_vault + vulkan_shaders_vault_header[1]);
+			for (count = *vulkan_shaders_vault_header++; count > 0; count--, vulkan_shaders_vault_header += 4) {
+				if (vulkan_shaders_vault_header[0] == labelSize) {
+					uint32_t *shader = (uint32_t*)calloc(1, vulkan_shaders_vault_header[1]);
+					if (NULL == shader) {
+						applog(LOG_ERR, "Failed to allocate shader %u:%u %u\n", workSize, labelSize, vulkan_shaders_vault_header[1]);
+						return NULL;
+					}
+					applog(LOG_INFO, "64:%03u %u -> %u\n", vulkan_shaders_vault_header[0], vulkan_shaders_vault_header[2], vulkan_shaders_vault_header[1]);
+					*shader_size = vulkan_shaders_vault_header[1];
+					uint8_t *src = vulkan_shaders_vault + vulkan_shaders_vault_header[3];
+					uLongf shaderSize = vulkan_shaders_vault_header[1];
+					if (Z_OK != uncompress((uint8_t*)shader, &shaderSize, src, vulkan_shaders_vault_header[2])) {
+						applog(LOG_ERR, "Failed to uncompress shader %u:%u\n", workSize, labelSize);
+						free(shader);
+						return NULL;
+					}
+					return shader;
+				}
+			}
+			break;
+		}
+	}
+	return NULL;
+}
+
+VkPipeline loadShader(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShaderModule *shader_module, uint32_t workSize, uint32_t labelSize)
+{
+	uint32_t shader_size = 0;
+	uint32_t *shader = getShader(workSize, labelSize, &shader_size);
+
+	if (NULL == shader) {
+		applog(LOG_ERR, "SPIR-V program %d:%d not found\n", workSize, labelSize);
+		return NULL;
+	}
+
+	applog(LOG_INFO, "SPIR-V program %u:%u %u bytes\n", workSize, labelSize, shader_size);
+
+	VkShaderModuleCreateInfo shaderModuleCreateInfo = {
+		VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
+		0,
+		0,
+		shader_size,
+		shader
+	};
+
+	CHECK_RESULT(gVulkan.vkCreateShaderModule(vkDevice, &shaderModuleCreateInfo, 0, shader_module), "vkCreateShaderModule", NULL);
+
+	VkComputePipelineCreateInfo computePipelineCreateInfo = {
+		VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+		0,
+		0,
+		{
+			VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+			0,
+			0,
+			VK_SHADER_STAGE_COMPUTE_BIT,
+			*shader_module,
+			"main",
+			0
+		},
+		pipelineLayout,
+		0,
+		0
+	};
+
+	VkPipeline pipeline;
+	CHECK_RESULT(gVulkan.vkCreateComputePipelines(vkDevice, 0, 1, &computePipelineCreateInfo, 0, &pipeline), "vkCreateComputePipelines", NULL);
+
+	free(shader);
+
+	return pipeline;
+}
+
 #endif
 

macOS/kernel-64-003.spirv

@@ -88,10 +88,11 @@ VkDeviceMemory allocateGPUMemory(int index,  VkDevice vkDevice, const VkDeviceSi
 VkBuffer createBuffer(VkDevice vkDevice, uint32_t computeQueueFamillyIndex, VkDeviceMemory memory, VkDeviceSize bufferSize, VkDeviceSize offset);
 VkPipelineLayout bindBuffers(VkDevice vkDevice, VkDescriptorSet *descriptorSet, VkDescriptorPool *descriptorPool, VkDescriptorSetLayout *descriptorSetLayout, VkBuffer b0, VkBuffer b1, VkBuffer b2, VkBuffer b3, VkBuffer b4, VkBuffer b5);
 uint64_t getBufferMemoryRequirements(VkDevice vkDevice, VkBuffer b);
-VkPipeline loadShader(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShaderModule *shader_module, const char * spirv_file_name);
+VkPipeline loadShaderFromFile(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShaderModule *shader_module, const char * spirv_file_name);
+VkPipeline loadShader(VkDevice vkDevice, VkPipelineLayout pipelineLayout, VkShaderModule *shader_module, uint32_t workSize, uint32_t labelSize);
 
 #ifdef __cplusplus
-}
+} /* extern "C" */
 #endif
 
 #endif	// _SPACEMESH_VULKAN_VULKAN_HELPERS_H__

macOS/kernel-64-004.spirv

@@ -287,18 +287,6 @@ void test_core(int aLabelsCount, unsigned aDiff, unsigned aSeed, int labelSize)
 
 						switch(status) {
 							case SPACEMESH_API_POW_SOLUTION_FOUND:
-								printf("Pow solution at %u\n", (uint32_t)idx_solution);
-
-								// compute 256 hash at solution index:
-								uint8_t hash[32];
-								scryptPositions(cpu_id, id, idx_solution, idx_solution, 256, salt, SPACEMESH_API_COMPUTE_LEAFS, hash, 512, 1, 1, NULL, NULL, &hashes_computed, &hashes_per_sec);
-
-								printf("D: ");
-								print_hex32(D);
-								printf("\n");
-								printf("H: ");
-								print_hex32(hash);
-								printf("\n");
 								break;
 
 							case SPACEMESH_API_ERROR_NONE:
@@ -314,6 +302,25 @@ void test_core(int aLabelsCount, unsigned aDiff, unsigned aSeed, int labelSize)
 						// set index for next iteration
 						idx += labels_per_iter;
 					}
+
+				 	if (idx_solution != -1ull) {
+
+						printf("Pow solution at %u\n", (uint32_t)idx_solution);
+
+						// compute 256 hash at solution index:
+						uint8_t hash[32];
+						scryptPositions(cpu_id, id, idx_solution, idx_solution, 256, salt, SPACEMESH_API_COMPUTE_LEAFS, hash, 512, 1, 1, NULL, NULL, &hashes_computed, &hashes_per_sec);
+
+						printf("D: ");
+						print_hex32(D);
+						printf("\n");
+						printf("H: ");
+						print_hex32(hash);
+						printf("\n");
+					} else {
+						printf("error: no pow solution found");
+					}
+
 				}
 			}
 		}