Merge pull request #10 from libxengine/develop

V2.2 Merge

README.en.md

@@ -1,4 +1,6 @@
+[中文](README.md) ||  [English](README.en.md)  
 # XEngine_Storage
+This repository has a development and master branch. If you want to use it, please use the master branch  
 
 ## Introduction
 c c++ 存储服务 c c++ 文件存储服务  
@@ -10,7 +12,6 @@ best storage service for http,batter than nginx and more convenient to manage. O
 
 ## Software feature
 The purpose of development and implementation based on libXEngine is a cross-platform network storage service  
-This repository has a development and master branch. If you want to use it, please use the master branch  
 
 feature list:  
 1. support file http upload and download(use put and get method)  
@@ -25,9 +26,10 @@ feature list:
 10. support p2p  
 11. bt(planning)  
 12. data distributed  
-13. support second pass  
+13. support second pass and Resumable   
 14. support nginx upload module proxy_pass  
 15. support upload and download Redirect  
+16. dynamic rate of the download
 
 ## install
 
@@ -85,6 +87,10 @@ The Second pass is not realized by the server, it is by the client
 upload file second pass is first check the HASH file is on the server, if has file on the server, it will directly prompt the upload is successful.  
 The realization of downloading second transmission is to first query the local file save path through HASH, and download it directly if it exists.
 
+## 关于P2P
+P2P distributed download has been supported, but currently only in the lan,cross-network segment is not supported for the time being, you need to wait for the development to be completed  
+P2P distributed download is the same as the hyper-threaded download of other download tools. The principle is to use the HTTP RANGE field. You can implement this function through libraries such as libcurl.  
+
 ## directory struct
 - XEngine_Docment   docment directory
 - XEngine_Release   install directory
@@ -94,7 +100,6 @@ The realization of downloading second transmission is to first query the local f
 
 ## now task
 expand management interface  
-P2P
 
 ## other problems   
 You can refer to the document under the docment directory. It contains API protocol and service description.  

README.md

@@ -1,4 +1,6 @@
+[中文](README.md) ||  [English](README.en.md)  
 # XEngine_Storage
+本仓库有开发和主分支,如果要使用,请使用master分支下的代码  
 
 ## 介绍
 c c++ 存储服务 c c++ 文件存储服务  
@@ -10,7 +12,6 @@ c c++ file storage service
 
 ## 软件特性
 基于libXEngine开发并实现的一套简洁高性能跨平台网络存储服务  
-本仓库有开发和主分支,如果要使用,请使用master分支下的代码  
 软件特性:  
 1. 支持HTTP协议上传和下载(采用PUT和GET)  
 2. 支持HTTP API接口事件通知与管理  
@@ -24,9 +25,10 @@ c c++ file storage service
 10. 支持P2P  
 11. BT(规划中)  
 12. 数据分发  
-13. 支持秒传  
+13. 支持秒传和断点续传  
 14. 支持NGINX UPLOAD MODULE上传代理  
 15. 支持上传和下载重定向  
+16. 下载动态速率
 
 ## 安装教程
 
@@ -90,9 +92,12 @@ make FLAGS=CleanAll 清理编译
 上传秒传的实现是先通过HASH查询文件是否在服务器,如果存在就不上传直接提示客户端上传成功.  
 下载秒传的实现是先通过HASH查询本地文件保存路径,如果存在就直接下载完成.  
 
+## 关于P2P
+P2P分布式下载已经支持,不过目前只能在局域网中,暂时不支持跨网段,需要等待开发完毕  
+P2P分布式下载与其他下载工具的超线程下载一样,原理是使用HTTP RANGE字段实现.各位可以通过libcurl等库实现此功能.  
+
 ## 当前任务
 扩展管理接口  
-实现P2P  
 
 ## 其他问题  
 你可以参考docment目录下的文档.里面包含了API协议和服务说明.

XEngine_APPClient/APPClient_Download/APPClient_Download.cpp

@@ -1,19 +1,149 @@
-#ifdef _WINDOWS
+//P2P 分布式(超线程下载示例代码),目前仅支持局域网,示例代码仅仅是演示如何实现,正式使用需要做调整
+//P2P Distributed(Hyper-Threading download sample code),only supports LAN,The example code is just to demonstrate how to implement it, and it needs to be adjusted for business
+//你可以使用libcurl实现HTTP下载功能,主要是利用了Range字段实现分块下载
+//You can use libcurl to implement the HTTP download, mainly using the Range field to achieve block download
+#ifdef _WINDOWS
 #include <windows.h>
 #include <tchar.h>
 #else
 #endif
+#include <list>
 #include <json/json.h>
 #include <XEngine_Include/XEngine_CommHdr.h>
+#include <XEngine_Include/XEngine_Types.h>
+#include <XEngine_Include/XEngine_ProtocolHdr.h>
 #include <XEngine_Include/XEngine_BaseLib/BaseLib_Define.h>
 #include <XEngine_Include/XEngine_BaseLib/BaseLib_Error.h>
 #include <XEngine_Include/XEngine_NetHelp/APIHelp_Define.h>
 #include <XEngine_Include/XEngine_NetHelp/APIHelp_Error.h>
+#include <XEngine_Include/XEngine_DownLoad/DownLoad_Define.h>
+#include <XEngine_Include/XEngine_DownLoad/DownLoad_Error.h>
+#include <XEngine_Include/XEngine_SystemSdk/ProcFile_Define.h>
+#include <XEngine_Include/XEngine_SystemSdk/SystemApi_Define.h>
+#include <XEngine_Include/XEngine_SystemSdk/SystemApi_Error.h>
+using namespace std;
 
 #pragma comment(lib,"x86/XEngine_BaseLib/XEngine_BaseLib")
 #pragma comment(lib,"x86/XEngine_NetHelp/NetHelp_APIHelp")
+#pragma comment(lib,"x86/XEngine_DownLoad/XEngine_DownLoad")
+#pragma comment(lib,"x86/XEngine_SystemSdk/XEngine_SystemApi")
 #pragma comment(lib,"Ws2_32")
 
+typedef struct 
+{
+	XENGINE_PROTOCOLFILE st_ProtocolFile;
+	CHAR tszIPAddr[64];                                     
+}P2PFILE_INFO;
+
+//解析局域网中所有文件
+void P2PParse_List(LPCTSTR lpszMsgBuffer, int nMsgLen, list<P2PFILE_INFO>* pStl_ListFile)
+{
+	Json::Value st_JsonRoot;
+	Json::CharReaderBuilder st_JsonBuild;
+	Json::CharReader* pSt_JsonReader(st_JsonBuild.newCharReader());
+
+	JSONCPP_STRING st_JsonError;
+	//解析JSON
+	if (!pSt_JsonReader->parse(lpszMsgBuffer, lpszMsgBuffer + nMsgLen, &st_JsonRoot, &st_JsonError))
+	{
+		return;
+	}
+	delete pSt_JsonReader;
+	pSt_JsonReader = NULL;
+
+	int nCount = st_JsonRoot["Count"].asInt();
+	Json::Value st_JsonArray = st_JsonRoot["List"];
+	for (int i = 0; i < nCount; i++)
+	{
+		P2PFILE_INFO st_P2PFile;
+		memset(&st_P2PFile, '\0', sizeof(P2PFILE_INFO));
+
+		st_P2PFile.st_ProtocolFile.nFileSize = st_JsonArray[i]["nFileSize"].asInt();
+		_tcscpy(st_P2PFile.st_ProtocolFile.tszFileHash, st_JsonArray[i]["tszFileHash"].asCString());
+		_tcscpy(st_P2PFile.st_ProtocolFile.tszFileName, st_JsonArray[i]["tszFileName"].asCString());
+		_tcscpy(st_P2PFile.st_ProtocolFile.tszFilePath, st_JsonArray[i]["tszFilePath"].asCString());
+		_tcscpy(st_P2PFile.st_ProtocolFile.tszFileTime, st_JsonArray[i]["tszFileTime"].asCString());
+		_tcscpy(st_P2PFile.st_ProtocolFile.tszFileUser, st_JsonArray[i]["tszFileUser"].asCString());
+		_tcscpy(st_P2PFile.tszIPAddr, st_JsonArray[i]["tszTableName"].asCString());
+
+		pStl_ListFile->push_back(st_P2PFile);
+	}
+}
+//创建分布式文件下载器
+typedef struct  
+{
+	XNETHANDLE xhToken;            //下载句柄
+	__int64x nPosStart;
+	__int64x nPosEnd;
+}P2PFILE_PIECE;
+void P2PFile_Create(list<P2PFILE_INFO>* pStl_ListFile, LPCTSTR lpszFile)
+{
+	P2PFILE_PIECE* pSt_P2PFile = new P2PFILE_PIECE[pStl_ListFile->size()];
+
+	int nPos = 0;
+	//得到每个块大小
+	int nPiece = pStl_ListFile->front().st_ProtocolFile.nFileSize / pStl_ListFile->size();
+	//这是一个简单的分布式块算法示例.怎么做分布式,可以根据自己需求做算法拆解
+	list<P2PFILE_INFO>::const_iterator stl_ListIterator = pStl_ListFile->begin();
+	for (int i = 0; stl_ListIterator != pStl_ListFile->end(); stl_ListIterator++, i++)
+	{
+		TCHAR tszDLUrl[MAX_PATH];
+		TCHAR tszRange[128];
+
+		memset(tszDLUrl, '\0', MAX_PATH);
+		memset(tszRange, '\0', sizeof(tszRange));
+
+		_stprintf(tszDLUrl, _T("%s/%s/%s"), stl_ListIterator->tszIPAddr, stl_ListIterator->st_ProtocolFile.tszFilePath + 2, stl_ListIterator->st_ProtocolFile.tszFileName);
+		//是否是最后一块
+		if (pStl_ListFile->size() == (i + 1))
+		{
+			pSt_P2PFile[i].nPosStart = nPos;
+			pSt_P2PFile[i].nPosEnd = 0;
+			_stprintf(tszRange, _T("%lld-"), pSt_P2PFile[i].nPosStart);
+		}
+		else
+		{
+			pSt_P2PFile[i].nPosStart = nPos;
+			pSt_P2PFile[i].nPosEnd = nPiece;
+			nPos += nPiece;
+			_stprintf(tszRange, _T("%lld-%lld"), pSt_P2PFile[i].nPosStart, pSt_P2PFile[i].nPosEnd);
+		}
+
+		if (!DownLoad_Http_Create(&pSt_P2PFile[i].xhToken, tszDLUrl, lpszFile, tszRange, NULL, NULL, NULL))
+		{
+			printf("create download task is failed:%lX\n", Download_GetLastError());
+		}
+	}
+	//直到所有完成
+	while (1)
+	{
+		BOOL bComplete = TRUE;
+		for (unsigned int i = 0; i < pStl_ListFile->size(); i++)
+		{
+			NETDOWNLOAD_TASKINFO st_TaskInfo;
+			memset(&st_TaskInfo, '\0', sizeof(NETDOWNLOAD_TASKINFO));
+
+			DownLoad_Http_Query(pSt_P2PFile[i].xhToken, &st_TaskInfo);
+			if (ENUM_XENGINE_DOWNLOAD_STATUS_COMPLETE != st_TaskInfo.en_DownStatus)
+			{
+				bComplete = FALSE;
+			}
+			printf("DLToken:%lld DLTotal:%lf DLNow:%lf DLStatus:%d\n", pSt_P2PFile[i].xhToken, st_TaskInfo.dlTotal, st_TaskInfo.dlNow, st_TaskInfo.en_DownStatus);
+		}
+		if (bComplete)
+		{
+			break;
+		}
+		Sleep(500);
+	}
+
+	for (unsigned int i = 0; i < pStl_ListFile->size(); i++)
+	{
+		DownLoad_Http_Delete(pSt_P2PFile[i].xhToken);
+	}
+	delete[] pSt_P2PFile;
+	pSt_P2PFile = NULL;
+}
 
 int main()
 {
@@ -24,13 +154,25 @@ int main()
 	int nBodyLen = 2048;
 	TCHAR *ptszMsgBody = NULL;
 	//请求分布式存储文件所有位置
-	LPCTSTR lpszUrl = _T("http://192.168.1.7:5100/EC9B9B75A04F3B323EFD348F9B795539");
+	LPCTSTR lpszUrl = _T("http://127.0.0.1:5100/7D54D6E40367F2763B8C8056EADC517F");
+	LPCTSTR lpszFile = _T("D:\\XEngine_Storage\\XEngine_APPClient\\Debug\\qq.exe");
+
 	if (!APIHelp_HttpRequest_Get(lpszUrl, &ptszMsgBody, &nBodyLen, &nHTTPCode))
 	{
 		return -1;
 	}
 	printf("%s\n", ptszMsgBody);
 
+	list<P2PFILE_INFO> stl_ListFile;
+	P2PParse_List(ptszMsgBody, nBodyLen, &stl_ListFile);
+
+	//创建稀疏文件(一个空白的文件)
+	if (!SystemApi_File_CreateSparseFile(lpszFile, stl_ListFile.front().st_ProtocolFile.nFileSize))
+	{
+		return -1;
+	}
+	P2PFile_Create(&stl_ListFile, lpszFile);
+
 	BaseLib_OperatorMemory_FreeCStyle((VOID**)&ptszMsgBody);
 	WSACleanup();
 	return 0;

XEngine_APPClient/APPClient_Download/APPClient_Download.vcxproj

@@ -88,7 +88,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_WINDOWS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
     </ClCompile>
     <Link>

XEngine_Release/XEngine_Config/XEngine_Config.json

@@ -1,5 +1,5 @@
 {
-  "tszIPAddr": "192.168.1.7",
+  "tszIPAddr": "192.168.1.9",
   "bDeamon": 0,
   "nCenterPort": 5100,
   "nStorageDLPort": 5101,
@@ -38,7 +38,7 @@
   "XStorage": {
     "nHashMode": 2,
     "bRename": 0,
-    "tszFileDir": "./XEngine_File"
+    "tszFileDir": "/home/ubuntu/桌面/XEngine_Storage/XEngine_Source/Debug/XEngine_File"
   },
   "XProxy": {
     "XProxyAuth": {
@@ -54,18 +54,21 @@
     }
   },
   "XLimit": {
+    "nDLTry": 100,
+    "nDLError": 10,
     "nMaxUPLoad": 0,
-    "nMaxDNLoad": 1024000
+    "nMaxDNLoad": 10240000
   },
   "XP2xp": {
-    "nMode": 0,
+    "nMode": 2,
     "nTime": 2,
     "nSDPort": 15000,
     "nRVPort": 15001,
     "tszQQWryFile": "./XEngine_SQL/qqwry.dat"
   },
   "XVer": {
     "StorageVersion": [
+      "2.2.0.1001 Build20210820",
       "2.1.0.1001 Build20210805",
       "2.0.0.1001 Build20210723",
       "1.5.0.1001 Build20210716",
@@ -76,4 +79,4 @@
       "1.0.0.1001 Build20210501"
     ]
   }
-}
+}

XEngine_Source/StorageModule_Config/Config_Define.h

@@ -84,6 +84,8 @@ typedef struct tag_XEngine_ServerConfig
     }st_XProxy;
 	struct
 	{
+        int nDLTry;
+        int nDLError;
         __int64x nMaxUPLoader;
         __int64x nMaxDNLoader;
 	}st_XLimit;

XEngine_Source/StorageModule_Config/Config_Json/Config_Json.cpp

@@ -155,13 +155,15 @@ BOOL CConfig_Json::Config_Json_File(LPCTSTR lpszConfigFile,XENGINE_SERVERCONFIG
     _tcscpy(pSt_ServerConfig->st_XProxy.st_XProxyPass.tszDLPass, st_JsonXProxyPass["tszDLPass"].asCString());
     _tcscpy(pSt_ServerConfig->st_XProxy.st_XProxyPass.tszUPPass, st_JsonXProxyPass["tszUPPass"].asCString());
 
-	if (st_JsonRoot["XLimit"].empty() || (2 != st_JsonRoot["XLimit"].size()))
+	if (st_JsonRoot["XLimit"].empty() || (4 != st_JsonRoot["XLimit"].size()))
 	{
 		Config_IsErrorOccur = TRUE;
 		Config_dwErrorCode = ERROR_XENGINE_BLOGIC_CONFIG_JSON_XSTORAGE;
 		return FALSE;
 	}
 	Json::Value st_JsonXLimit = st_JsonRoot["XLimit"];
+    pSt_ServerConfig->st_XLimit.nDLTry = st_JsonXLimit["nDLTry"].asUInt();
+    pSt_ServerConfig->st_XLimit.nDLError = st_JsonXLimit["nDLError"].asUInt();
 	pSt_ServerConfig->st_XLimit.nMaxDNLoader = st_JsonXLimit["nMaxDNLoad"].asInt64();
     pSt_ServerConfig->st_XLimit.nMaxUPLoader = st_JsonXLimit["nMaxUPLoad"].asInt64();
 

XEngine_Source/StorageModule_Session/Makefile

@@ -2,8 +2,8 @@ CC = g++ -Wall -std=c++17 -fPIC
 RELEASE = 0
 UNICODE = 0
 INCLUDE = -I ./
-LOADBIN = 
-LIB = 
+LOADBIN = -L /usr/local/lib/XEngine_Release/XEngine_BaseLib
+LIB = -lXEngine_BaseLib
 LIBEX = -static-libgcc -lrt -ldl -lpthread
 LOADSO = -Wl,-rpath=
 LIBINCLUDE = Session_DLStroage.o Session_UPStroage.o Session_User.o pch.o