Merge pull request #100 from pknu-wap/#91/enhancement/jsp/app_enhance…

…ment

#91/enhancement/jsp/app enhancement

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/benchmark.h

@@ -24,6 +24,9 @@ namespace ncnn {
 // get now timestamp in ms
 NCNN_EXPORT double get_current_time();
 
+// sleep milliseconds
+NCNN_EXPORT void sleep(unsigned long long int milliseconds = 1000);
+
 #if NCNN_BENCHMARK
 
 NCNN_EXPORT void benchmark(const Layer* layer, double start, double end);

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/cpu.h

@@ -60,6 +60,8 @@ NCNN_EXPORT int cpu_support_arm_neon();
 NCNN_EXPORT int cpu_support_arm_vfpv4();
 // asimdhp = aarch64 asimd half precision
 NCNN_EXPORT int cpu_support_arm_asimdhp();
+// cpuid = aarch64 cpuid info
+NCNN_EXPORT int cpu_support_arm_cpuid();
 // asimddp = aarch64 asimd dot product
 NCNN_EXPORT int cpu_support_arm_asimddp();
 // asimdfhm = aarch64 asimd fhm
@@ -147,6 +149,9 @@ NCNN_EXPORT const CpuSet& get_cpu_thread_affinity_mask(int powersave);
 // set explicit thread affinity
 NCNN_EXPORT int set_cpu_thread_affinity(const CpuSet& thread_affinity_mask);
 
+// runtime thread affinity info
+NCNN_EXPORT int is_current_thread_running_on_a53_a55();
+
 // misc function wrapper for openmp routines
 NCNN_EXPORT int get_omp_num_threads();
 NCNN_EXPORT void set_omp_num_threads(int num_threads);

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/gpu.h

@@ -28,7 +28,19 @@
 namespace ncnn {
 
 // instance
+
+// Create VkInstance and initialize some objects that need to be calculated by GPU
+// Creates a VkInstance object, Checks the extended attributes supported by the Vulkan instance concerned,
+// Initializes, and creates Vulkan validation layers (if ENABLE_VALIDATION_LAYER is enabled),
+// Iterates over all supported physical devices, etc.
 NCNN_EXPORT int create_gpu_instance();
+
+// Get global VkInstance variable
+// Must be called after create_gpu_instance() and before destroy_gpu_instance()
+NCNN_EXPORT VkInstance get_gpu_instance();
+
+// Destroy VkInstance object and free the memory of the associated object
+// Usually called in the destructor of the main program exit
 NCNN_EXPORT void destroy_gpu_instance();
 
 // instance extension capability
@@ -37,6 +49,8 @@ extern int support_VK_KHR_get_physical_device_properties2;
 extern int support_VK_KHR_get_surface_capabilities2;
 extern int support_VK_KHR_surface;
 extern int support_VK_EXT_debug_utils;
+extern int support_VK_EXT_validation_features;
+extern int support_VK_EXT_validation_flags;
 #if __ANDROID_API__ >= 26
 extern int support_VK_KHR_android_surface;
 #endif // __ANDROID_API__ >= 26
@@ -167,6 +181,7 @@ class NCNN_EXPORT GpuInfo
     int support_VK_KHR_8bit_storage() const;
     int support_VK_KHR_16bit_storage() const;
     int support_VK_KHR_bind_memory2() const;
+    int support_VK_KHR_buffer_device_address() const;
     int support_VK_KHR_create_renderpass2() const;
     int support_VK_KHR_dedicated_allocation() const;
     int support_VK_KHR_descriptor_update_template() const;
@@ -183,9 +198,12 @@ class NCNN_EXPORT GpuInfo
     int support_VK_KHR_shader_float_controls() const;
     int support_VK_KHR_storage_buffer_storage_class() const;
     int support_VK_KHR_swapchain() const;
+    int support_VK_EXT_buffer_device_address() const;
     int support_VK_EXT_descriptor_indexing() const;
     int support_VK_EXT_memory_budget() const;
+    int support_VK_EXT_memory_priority() const;
     int support_VK_EXT_queue_family_foreign() const;
+    int support_VK_AMD_device_coherent_memory() const;
 #if __ANDROID_API__ >= 26
     int support_VK_ANDROID_external_memory_android_hardware_buffer() const;
 #endif // __ANDROID_API__ >= 26
@@ -269,6 +287,11 @@ class NCNN_EXPORT VulkanDevice
     PFN_vkBindBufferMemory2KHR vkBindBufferMemory2KHR;
     PFN_vkBindImageMemory2KHR vkBindImageMemory2KHR;
 
+    // VK_KHR_buffer_device_address
+    PFN_vkGetBufferDeviceAddressKHR vkGetBufferDeviceAddressKHR;
+    PFN_vkGetBufferOpaqueCaptureAddressKHR vkGetBufferOpaqueCaptureAddressKHR;
+    PFN_vkGetDeviceMemoryOpaqueCaptureAddressKHR vkGetDeviceMemoryOpaqueCaptureAddressKHR;
+
     // VK_KHR_create_renderpass2
     PFN_vkCmdBeginRenderPass2KHR vkCmdBeginRenderPass2KHR;
     PFN_vkCmdEndRenderPass2KHR vkCmdEndRenderPass2KHR;
@@ -306,6 +329,9 @@ class NCNN_EXPORT VulkanDevice
     PFN_vkAcquireNextImageKHR vkAcquireNextImageKHR;
     PFN_vkQueuePresentKHR vkQueuePresentKHR;
 
+    // VK_EXT_buffer_device_address
+    PFN_vkGetBufferDeviceAddressEXT vkGetBufferDeviceAddressEXT;
+
 #if __ANDROID_API__ >= 26
     // VK_ANDROID_external_memory_android_hardware_buffer
     PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID;

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/layer.h

@@ -188,6 +188,16 @@ struct custom_layer_registry_entry
     void* userdata;
 };
 
+struct overwrite_builtin_layer_registry_entry
+{
+    // layer type index
+    int typeindex;
+    // layer factory entry
+    layer_creator_func creator;
+    layer_destroyer_func destroyer;
+    void* userdata;
+};
+
 #if NCNN_STRING
 // get layer type from type name
 NCNN_EXPORT int layer_to_index(const char* type);

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/layer_shader_type_enum.h

@@ -364,6 +364,15 @@ mish_pack8 = 358,
 swish = 359,
 swish_pack4 = 360,
 swish_pack8 = 361,
-convert_ycbcr = 362,
-vulkan_activation = 363,
+gemm = 362,
+multiheadattention_qk_cross = 363,
+multiheadattention_qk_cross_pack1to4 = 364,
+multiheadattention_qk_cross_pack4 = 365,
+multiheadattention_qk_cross_pack4to1 = 366,
+multiheadattention_qkv_cross = 367,
+multiheadattention_qkv_cross_pack1to4 = 368,
+multiheadattention_qkv_cross_pack4 = 369,
+multiheadattention_qkv_cross_pack4to1 = 370,
+convert_ycbcr = 371,
+vulkan_activation = 372,
 

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/modelbin.h

@@ -31,7 +31,7 @@ class NCNN_EXPORT ModelBin
     // 2 = float16
     // 3 = int8
     // load vec
-    virtual Mat load(int w, int type) const = 0;
+    virtual Mat load(int w, int type) const;
     // load image
     virtual Mat load(int w, int h, int type) const;
     // load dim

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/net.h

@@ -58,12 +58,12 @@ class NCNN_EXPORT Net
 #endif // NCNN_VULKAN
 
 #if NCNN_STRING
-    // register custom layer by layer type name
+    // register custom layer or overwrite built-in layer by layer type name
     // return 0 if success
     int register_custom_layer(const char* type, layer_creator_func creator, layer_destroyer_func destroyer = 0, void* userdata = 0);
     virtual int custom_layer_to_index(const char* type);
 #endif // NCNN_STRING
-    // register custom layer by layer type
+    // register custom layer or overwrite built-in layer by layer type
     // return 0 if success
     int register_custom_layer(int index, layer_creator_func creator, layer_destroyer_func destroyer = 0, void* userdata = 0);
 
@@ -149,8 +149,10 @@ class NCNN_EXPORT Net
     int find_blob_index_by_name(const char* name) const;
     int find_layer_index_by_name(const char* name) const;
     virtual Layer* create_custom_layer(const char* type);
+    virtual Layer* create_overwrite_builtin_layer(const char* type);
 #endif // NCNN_STRING
     virtual Layer* create_custom_layer(int index);
+    virtual Layer* create_overwrite_builtin_layer(int typeindex);
 
 private:
     Net(const Net&);

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/option.h

@@ -140,7 +140,10 @@ class NCNN_EXPORT Option
     bool use_winograd43_convolution;
     bool use_winograd63_convolution;
 
-    bool use_reserved_6;
+    // this option is turned on for A53/A55 automatically
+    // but you can force this on/off if you wish
+    bool use_a53_a55_optimized_kernel;
+
     bool use_reserved_7;
     bool use_reserved_8;
     bool use_reserved_9;

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/platform.h

@@ -42,8 +42,7 @@
 #define NCNN_AVX512VNNI 0
 #define NCNN_AVX512BF16 0
 #define NCNN_AVX512FP16 0
-#define NCNN_VFPV4 0
-#if __aarch64__
+#define NCNN_VFPV4 1
 #define NCNN_ARM82 1
 #define NCNN_ARM82DOT 1
 #define NCNN_ARM82FP16FML 1
@@ -54,7 +53,6 @@
 #define NCNN_ARM86SVEBF16 1
 #define NCNN_ARM86SVEI8MM 1
 #define NCNN_ARM86SVEF32MM 1
-#endif // __aarch64__
 #define NCNN_MSA 0
 #define NCNN_LSX 0
 #define NCNN_MMI 0
@@ -63,7 +61,7 @@
 #define NCNN_BF16 1
 #define NCNN_FORCE_INLINE 1
 
-#define NCNN_VERSION_STRING "1.0.20230223"
+#define NCNN_VERSION_STRING "1.0.20230517"
 
 #include "ncnn_export.h"
 

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/simpleocv.h

@@ -448,6 +448,8 @@ enum ImreadModes
 
 NCNN_EXPORT Mat imread(const std::string& path, int flags = IMREAD_COLOR);
 
+NCNN_EXPORT Mat imdecode(const std::vector<uchar>& buf, int flags = IMREAD_COLOR);
+
 enum ImwriteFlags
 {
     IMWRITE_JPEG_QUALITY = 1
@@ -498,4 +500,4 @@ NCNN_EXPORT Size getTextSize(const std::string& text, int fontFace, double fontS
 
 #endif // NCNN_SIMPLEOCV
 
-#endif // NCNN_SIMPLEOCV_H
+#endif // NCNN_SIMPLEOCV_H

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/include/ncnn/vulkan_header_fix.h

@@ -177,13 +177,87 @@ typedef struct VkPhysicalDeviceFloat16Int8FeaturesKHR
 
 #if VK_HEADER_VERSION < 97
 #define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT (VkStructureType)1000237000
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT (VkStructureType)1000238000
+#define VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT            (VkStructureType)1000238001
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_ADDRESS_FEATURES_EXT  (VkStructureType)1000244000
+#define VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO_EXT               (VkStructureType)1000244001
+#define VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_CREATE_INFO_EXT        (VkStructureType)1000244002
+#define VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT                      (VkStructureType)1000247000
+#define VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT_EXT         (VkBufferCreateFlagBits)0x00020000
+#define VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_EXT                  (VkBufferUsageFlagBits)0x00020000
+typedef uint64_t VkDeviceAddress;
 typedef struct VkPhysicalDeviceMemoryBudgetPropertiesEXT
 {
     VkStructureType sType;
     void* pNext;
     VkDeviceSize heapBudget[VK_MAX_MEMORY_HEAPS];
     VkDeviceSize heapUsage[VK_MAX_MEMORY_HEAPS];
 } VkPhysicalDeviceMemoryBudgetPropertiesEXT;
+typedef struct VkPhysicalDeviceMemoryPriorityFeaturesEXT
+{
+    VkStructureType sType;
+    void* pNext;
+    VkBool32 memoryPriority;
+} VkPhysicalDeviceMemoryPriorityFeaturesEXT;
+typedef struct VkMemoryPriorityAllocateInfoEXT
+{
+    VkStructureType sType;
+    const void* pNext;
+    float priority;
+} VkMemoryPriorityAllocateInfoEXT;
+typedef struct VkPhysicalDeviceBufferAddressFeaturesEXT
+{
+    VkStructureType sType;
+    void* pNext;
+    VkBool32 bufferDeviceAddress;
+    VkBool32 bufferDeviceAddressCaptureReplay;
+    VkBool32 bufferDeviceAddressMultiDevice;
+} VkPhysicalDeviceBufferAddressFeaturesEXT;
+typedef struct VkBufferDeviceAddressInfoEXT
+{
+    VkStructureType sType;
+    const void* pNext;
+    VkBuffer buffer;
+} VkBufferDeviceAddressInfoEXT;
+typedef struct VkBufferDeviceAddressCreateInfoEXT
+{
+    VkStructureType sType;
+    const void* pNext;
+    VkDeviceSize deviceAddress;
+} VkBufferDeviceAddressCreateInfoEXT;
+typedef VkDeviceAddress(VKAPI_PTR* PFN_vkGetBufferDeviceAddressEXT)(VkDevice device, const VkBufferDeviceAddressInfoEXT* pInfo);
+typedef enum VkValidationFeatureEnableEXT
+{
+    VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT = 0,
+    VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT = 1,
+    VK_VALIDATION_FEATURE_ENABLE_BEGIN_RANGE_EXT = VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
+    VK_VALIDATION_FEATURE_ENABLE_END_RANGE_EXT = VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
+    VK_VALIDATION_FEATURE_ENABLE_RANGE_SIZE_EXT = (VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT - VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT + 1),
+    VK_VALIDATION_FEATURE_ENABLE_MAX_ENUM_EXT = 0x7FFFFFFF
+} VkValidationFeatureEnableEXT;
+typedef enum VkValidationFeatureDisableEXT
+{
+    VK_VALIDATION_FEATURE_DISABLE_ALL_EXT = 0,
+    VK_VALIDATION_FEATURE_DISABLE_SHADERS_EXT = 1,
+    VK_VALIDATION_FEATURE_DISABLE_THREAD_SAFETY_EXT = 2,
+    VK_VALIDATION_FEATURE_DISABLE_API_PARAMETERS_EXT = 3,
+    VK_VALIDATION_FEATURE_DISABLE_OBJECT_LIFETIMES_EXT = 4,
+    VK_VALIDATION_FEATURE_DISABLE_CORE_CHECKS_EXT = 5,
+    VK_VALIDATION_FEATURE_DISABLE_UNIQUE_HANDLES_EXT = 6,
+    VK_VALIDATION_FEATURE_DISABLE_BEGIN_RANGE_EXT = VK_VALIDATION_FEATURE_DISABLE_ALL_EXT,
+    VK_VALIDATION_FEATURE_DISABLE_END_RANGE_EXT = VK_VALIDATION_FEATURE_DISABLE_UNIQUE_HANDLES_EXT,
+    VK_VALIDATION_FEATURE_DISABLE_RANGE_SIZE_EXT = (VK_VALIDATION_FEATURE_DISABLE_UNIQUE_HANDLES_EXT - VK_VALIDATION_FEATURE_DISABLE_ALL_EXT + 1),
+    VK_VALIDATION_FEATURE_DISABLE_MAX_ENUM_EXT = 0x7FFFFFFF
+} VkValidationFeatureDisableEXT;
+typedef struct VkValidationFeaturesEXT
+{
+    VkStructureType sType;
+    const void* pNext;
+    uint32_t enabledValidationFeatureCount;
+    const VkValidationFeatureEnableEXT* pEnabledValidationFeatures;
+    uint32_t disabledValidationFeatureCount;
+    const VkValidationFeatureDisableEXT* pDisabledValidationFeatures;
+} VkValidationFeaturesEXT;
 #endif // VK_HEADER_VERSION < 97
 
 #if VK_HEADER_VERSION < 101
@@ -248,6 +322,65 @@ typedef struct VkPhysicalDeviceCooperativeMatrixPropertiesNV
 typedef VkResult(VKAPI_PTR* PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesNV)(VkPhysicalDevice physicalDevice, uint32_t* pPropertyCount, VkCooperativeMatrixPropertiesNV* pProperties);
 #endif // VK_HEADER_VERSION < 101
 
+#if VK_HEADER_VERSION < 121
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COHERENT_MEMORY_FEATURES_AMD (VkStructureType)1000229000
+#define VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD                     (VkMemoryPropertyFlagBits)0x00000040
+#define VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD                     (VkMemoryPropertyFlagBits)0x00000040
+typedef struct VkPhysicalDeviceCoherentMemoryFeaturesAMD
+{
+    VkStructureType sType;
+    void* pNext;
+    VkBool32 deviceCoherentMemory;
+} VkPhysicalDeviceCoherentMemoryFeaturesAMD;
+#endif // VK_HEADER_VERSION < 121
+
+#if VK_HEADER_VERSION < 129
+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_KHR (VkStructureType)1000257000
+#define VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO_KHR                     (VkStructureType)1000244001
+#define VK_STRUCTURE_TYPE_BUFFER_OPAQUE_CAPTURE_ADDRESS_CREATE_INFO_KHR      (VkStructureType)1000257002
+#define VK_STRUCTURE_TYPE_MEMORY_OPAQUE_CAPTURE_ADDRESS_ALLOCATE_INFO_KHR    (VkStructureType)1000257003
+#define VK_STRUCTURE_TYPE_DEVICE_MEMORY_OPAQUE_CAPTURE_ADDRESS_INFO_KHR      (VkStructureType)1000257004
+#define VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT_KHR               (VkBufferCreateFlagBits)0x00020000
+#define VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_KHR                        (VkBufferUsageFlagBits)0x00020000
+#define VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR                            (VkMemoryAllocateFlagBits)0x00000002
+#define VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT_KHR             (VkMemoryAllocateFlagBits)0x00000004
+typedef struct VkPhysicalDeviceBufferDeviceAddressFeaturesKHR
+{
+    VkStructureType sType;
+    void* pNext;
+    VkBool32 bufferDeviceAddress;
+    VkBool32 bufferDeviceAddressCaptureReplay;
+    VkBool32 bufferDeviceAddressMultiDevice;
+} VkPhysicalDeviceBufferDeviceAddressFeaturesKHR;
+typedef struct VkBufferDeviceAddressInfoKHR
+{
+    VkStructureType sType;
+    const void* pNext;
+    VkBuffer buffer;
+} VkBufferDeviceAddressInfoKHR;
+typedef struct VkBufferOpaqueCaptureAddressCreateInfoKHR
+{
+    VkStructureType sType;
+    const void* pNext;
+    uint64_t opaqueCaptureAddress;
+} VkBufferOpaqueCaptureAddressCreateInfoKHR;
+typedef struct VkMemoryOpaqueCaptureAddressAllocateInfoKHR
+{
+    VkStructureType sType;
+    const void* pNext;
+    uint64_t opaqueCaptureAddress;
+} VkMemoryOpaqueCaptureAddressAllocateInfoKHR;
+typedef struct VkDeviceMemoryOpaqueCaptureAddressInfoKHR
+{
+    VkStructureType sType;
+    const void* pNext;
+    VkDeviceMemory memory;
+} VkDeviceMemoryOpaqueCaptureAddressInfoKHR;
+typedef VkDeviceAddress(VKAPI_PTR* PFN_vkGetBufferDeviceAddressKHR)(VkDevice device, const VkBufferDeviceAddressInfoKHR* pInfo);
+typedef uint64_t(VKAPI_PTR* PFN_vkGetBufferOpaqueCaptureAddressKHR)(VkDevice device, const VkBufferDeviceAddressInfoKHR* pInfo);
+typedef uint64_t(VKAPI_PTR* PFN_vkGetDeviceMemoryOpaqueCaptureAddressKHR)(VkDevice device, const VkDeviceMemoryOpaqueCaptureAddressInfoKHR* pInfo);
+#endif // VK_HEADER_VERSION < 129
+
 #if VK_HEADER_VERSION < 208
 typedef enum VkInstanceCreateFlagBits
 {

feature/camera/src/main/jni/ncnn_android_vulkan/arm64-v8a/lib/cmake/glslang/glslang-config-version.cmake

@@ -52,11 +52,6 @@ else()
 endif()
 
 
-# if the installed project requested no architecture check, don't perform the check
-if("FALSE")
-  return()
-endif()
-
 # if the installed or the using project don't have CMAKE_SIZEOF_VOID_P set, ignore it:
 if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "" OR "8" STREQUAL "")
   return()