diff --git a/.gitignore b/.gitignore index da71e17a..6fa3768d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,13 @@ grive.kdev4 .project .cproject build/ +/CMakeCache.txt +CMakeFiles +moc_*.cxx* +bgrive/ui_MainWindow.h +Makefile +*.a +bgrive/bgrive +grive/grive +libgrive/btest +*.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 92c50b4b..95443aa7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,11 @@ cmake_minimum_required(VERSION 2.8) -set( GRIVE_VERSION "0.2.0" ) +# Grive version. remember to update it for every new release! +set( GRIVE_VERSION "0.5.1-dev" ) + +# common compile options +add_definitions( -DVERSION="${GRIVE_VERSION}" ) +add_definitions( -D_FILE_OFFSET_BITS=64 -std=c++0x ) add_subdirectory( libgrive ) add_subdirectory( grive ) diff --git a/README b/README deleted file mode 100644 index a15f1c9b..00000000 --- a/README +++ /dev/null @@ -1,50 +0,0 @@ -Grive 0.2.0 -9 June 2012 - -http://www.lbreda.com/grive/ - -Grive is still considered experimental. It just downloads all the files in your google drive -into the current directory. After you make some changes to the local files, run grive and -it will upload your changes back to your google drive. New files created in local or google -drive will be uploaded or downloaded respectively. Deleted files will also be "removed". -Currently Grive will NOT destroy any of your files: it will only move the files to a -directory named .trash, or put it in the google drive trash. You can always recover them. - -There are a few things that grive does not do at the moment: -- wait for changes in file system to occur and upload. Grive only sync when you run it. -- symbolic links support -- support for Google documents -- support for files >2GB - -Of course these will be done in future, possibly the next release. - -You need the following libraries: - -- json-c -- libcurl -- libstdc++ -- libgcrypt -- Boost (Boost filesystem and program_option are required) - -There are also some optional dependencies: -- CppUnit (for unit tests) -- libbfd (for backtrace) -- binutils (for libiberty, required for compilation in OpenSUSE & ubuntu) - -Grive uses cmake to build, see the instructions in: - -http://www.lbreda.com/grive/installation - -for detailed procedures to compile Grive. - -When grive is ran for the first time, you should use the "-a" argument to grant -permission to grive to access to your Google Drive. An URL should be printed. -Go to the link. You will need to login to your google account if you haven't -done so. After granting the permission to grive, the browser will show you -an authenication code. Copy-and-paste that to the standard input of grive. - -If everything works fine, grive will create a .grive and a .grive_state file in your -current directory. It will also start downloading files from your Google Drive to -your current directory. - -Enjoy! diff --git a/README.md b/README.md new file mode 100644 index 00000000..f71efa1c --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +GRIVE NO LONGER MAINTAINED +PLEASE USE GRIVE2 + +Grive is DEPRECATED. Please refer to the newest fork Grive2. + +Grive2, WHICH IS ACTIVELY MAINTAINED, is here: https://github.com/vitalif/grive2 + + diff --git a/cmake/Modules/FindBFD.cmake b/cmake/Modules/FindBFD.cmake index 785e29b6..35dfd5d6 100644 --- a/cmake/Modules/FindBFD.cmake +++ b/cmake/Modules/FindBFD.cmake @@ -1,5 +1,5 @@ -find_library( DL_LIBRARY NAMES dl PATH /usr/lib /usr/lib64 ) -find_library( BFD_LIBRARY NAMES bfd PATH /usr/lib /usr/lib64 ) +find_library( DL_LIBRARY NAMES dl PATH ${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES} ) +find_library( BFD_LIBRARY NAMES bfd PATH ${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES} ) if ( DL_LIBRARY AND BFD_LIBRARY ) set( BFD_FOUND TRUE ) @@ -9,5 +9,4 @@ if ( BFD_FOUND ) message( STATUS "Found libbfd: ${BFD_LIBRARY}") - endif ( BFD_FOUND ) diff --git a/cmake/Modules/FindGDBM.cmake b/cmake/Modules/FindGDBM.cmake deleted file mode 100644 index e1ba00a2..00000000 --- a/cmake/Modules/FindGDBM.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# - Find gdbm -# Find the native GDBM includes and library -# -# GDBM_INCLUDE_DIR - where to find gdbm.h, etc. -# GDBM_LIBRARIES - List of libraries when using gdbm. -# GDBM_FOUND - True if gdbm found. - - -IF (GDBM_INCLUDE_DIR) - # Already in cache, be silent - SET(GDBM_FIND_QUIETLY TRUE) -ENDIF (GDBM_INCLUDE_DIR) - -FIND_PATH(GDBM_INCLUDE_DIR gdbm.h - /usr/local/include - /usr/include - /opt/local/include -) - -SET(GDBM_NAMES gdbm) -FIND_LIBRARY(GDBM_LIBRARY - NAMES ${GDBM_NAMES} - PATHS /usr/lib /usr/local/lib /opt/local/lib -) - -IF (GDBM_INCLUDE_DIR AND GDBM_LIBRARY) - SET(GDBM_FOUND TRUE) - SET( GDBM_LIBRARIES ${GDBM_LIBRARY} ) -ELSE (GDBM_INCLUDE_DIR AND GDBM_LIBRARY) - SET(GDBM_FOUND FALSE) - SET( GDBM_LIBRARIES ) -ENDIF (GDBM_INCLUDE_DIR AND GDBM_LIBRARY) - -IF (GDBM_FOUND) - IF (NOT GDBM_FIND_QUIETLY) - MESSAGE(STATUS "Found GDBM: ${GDBM_LIBRARY}") - ENDIF (NOT GDBM_FIND_QUIETLY) -ELSE (GDBM_FOUND) - IF (GDBM_FIND_REQUIRED) - MESSAGE(STATUS "Looked for gdbm libraries named ${GDBMS_NAMES}.") - MESSAGE(FATAL_ERROR "Could NOT find gdbm library") - ENDIF (GDBM_FIND_REQUIRED) -ENDIF (GDBM_FOUND) - -MARK_AS_ADVANCED( - GDBM_LIBRARY - GDBM_INCLUDE_DIR -) diff --git a/cmake/Modules/FindIberty.cmake b/cmake/Modules/FindIberty.cmake index a14ba45b..6562419a 100644 --- a/cmake/Modules/FindIberty.cmake +++ b/cmake/Modules/FindIberty.cmake @@ -2,13 +2,13 @@ # This module finds libiberty. # # It sets the following variables: -# IBERTY_LIBRARY - The JSON-C library to link against. +# IBERTY_LIBRARY - The library to link against. FIND_LIBRARY( IBERTY_LIBRARY NAMES iberty ) IF (IBERTY_LIBRARY) - # show which JSON-C was found only if not quiet + # show which library was found only if not quiet MESSAGE( STATUS "Found libiberty: ${IBERTY_LIBRARY}") SET(IBERTY_FOUND TRUE) diff --git a/cmake/Modules/FindJSONC.cmake b/cmake/Modules/FindJSONC.cmake deleted file mode 100644 index f72e8ea1..00000000 --- a/cmake/Modules/FindJSONC.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# - Find JSON-C -# This module finds an installed JSON-C package. -# -# It sets the following variables: -# JSONC_FOUND - Set to false, or undefined, if JSON-C isn't found. -# JSONC_INCLUDE_DIR - The JSON-C include directory. -# JSONC_LIBRARY - The JSON-C library to link against. - -FIND_PATH(JSONC_INCLUDE_DIR json/json.h) -FIND_LIBRARY(JSONC_LIBRARY NAMES json) - -IF (JSONC_INCLUDE_DIR AND JSONC_LIBRARY) - SET(JSONC_FOUND TRUE) -ENDIF (JSONC_INCLUDE_DIR AND JSONC_LIBRARY) - -IF (JSONC_FOUND) - - # show which JSON-C was found only if not quiet - IF (NOT JSONC_FIND_QUIETLY) - MESSAGE(STATUS "Found JSON-C: ${JSONC_LIBRARY}") - ENDIF (NOT JSONC_FIND_QUIETLY) - -ELSE (JSONC_FOUND) - - # fatal error if JSON-C is required but not found - IF (JSONC_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find JSON-C") - ENDIF (JSONC_FIND_REQUIRED) - -ENDIF (JSONC_FOUND) diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000..db47f06e --- /dev/null +++ b/debian/changelog @@ -0,0 +1,90 @@ +grive2 (0.5.1+git20160731) unstable; urgency=medium + + * Newer dev version + + -- Vitaliy Filippov Wed, 31 Jul 2016 22:04:53 +0300 + +grive2 (0.5+git20160114) unstable; urgency=medium + + * Newer release, with support for faster sync and rename detection + + -- Vitaliy Filippov Sun, 03 Jan 2016 12:51:55 +0300 + +grive2 (0.4.1+git20151011) unstable; urgency=medium + + * Add Debian packaging scripts to the official repository + + -- Vitaliy Filippov Sun, 11 Oct 2015 15:03:55 +0300 + +grive2 (0.4.1-1+git20151008~webupd8~wily0) wily; urgency=medium + + * new git pull + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Thu, 08 Oct 2015 11:50:55 +0200 + +grive2 (0.4.1-1+git20151007~webupd8~wily0) wily; urgency=medium + + * new git pull + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Wed, 07 Oct 2015 13:46:29 +0200 + +grive2 (0.4.1-1+git20151001~webupd8~precise0) precise; urgency=medium + + * new git pull + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Thu, 01 Oct 2015 12:54:44 +0200 + +grive2 (0.4.0-1+git20150928~webupd8~wily1) wily; urgency=medium + + * new git pull + * build-depend on libboost-regex-dev + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Mon, 28 Sep 2015 11:36:19 +0200 + +grive2 (0.4.0-1+git20150810~webupd8~wily0) wily; urgency=medium + + * upload for Wily + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Sat, 04 Jul 2015 12:26:14 +0200 + +grive2 (0.4.0-1+git20150810~webupd8~vivid0) vivid; urgency=medium + + * new git pull + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Sat, 04 Jul 2015 12:26:14 +0200 + +grive2 (0.4.0-1+git20150629~webupd8~vivid0) vivid; urgency=medium + + * new git pull + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Mon, 29 Jun 2015 12:27:21 +0200 + +grive2 (0.4.0-1~webupd8~precise3) precise; urgency=medium + + * Initial packaging (based on grive) + * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 + because libcurl4-openssl-dev depends on it (and for Precise, + there's no libgcrypt20) + + -- Alin Andrei Mon, 25 May 2015 15:31:24 +0200 diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000..7f8f011e --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000..edad2557 --- /dev/null +++ b/debian/control @@ -0,0 +1,25 @@ +Source: grive2 +Section: net +Priority: optional +Maintainer: Vitaliy Filippov +Build-Depends: debhelper, cmake, pkg-config, zlib1g-dev, libcurl4-openssl-dev | libcurl4-gnutls-dev, libstdc++-6-dev | libstdc++6-4.4-dev | libstdc++-4.9-dev | libstdc++-5-dev, libboost-filesystem-dev, libboost-program-options-dev, libboost-test-dev, libboost-regex-dev, libexpat1-dev, binutils-dev, libgcrypt-dev, libyajl-dev +Standards-Version: 3.9.6 +Homepage: https://yourcmc.ru/wiki/Grive2 + +Package: grive +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends} +Description: Grive2: an open source Linux client for Google Drive + . + This is the up-to-date fork of the original "Grive" (https://github.com/Grive/grive) + Google Drive client with the support for the new Drive REST API and partial + sync. + . + For the first time running grive, you should use the "-a" argument to grant + permission to grive to access to your Google Drive. An URL should be printed. + Go to the page. You will need to login to your google account if you haven't + done so. After granting the permission to grive, the browser will show you + an authenication code. Copy-and-paste that to the standard input of grive. + If everything works fine, grive will create .grive and .grive_state inside the + synchronized directory. It will also start downloading files from your Google + Drive to that directory. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000..e3628400 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,20 @@ +Grive2 + + https://github.com/vitalif/grive2 + +Current developers: + + Vitaliy Filippov + +Previous developers: + + Nestal Wan (me@nestal.net) 16.05.2012 — 03.05.2013 + Matchman Green (match065@gmail.com) 26.04.2012 — 20.06.2012 + +Contributors: + + See full list here https://yourcmc.ru/wiki/Grive2#Full_list_of_contributors + +License: + + GPL 2.0+ diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000..81484573 --- /dev/null +++ b/debian/rules @@ -0,0 +1,4 @@ +#!/usr/bin/make -f + +%: + dh $@ --buildsystem=cmake --parallel diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 00000000..89ae9db8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/grive/CMakeLists.txt b/grive/CMakeLists.txt index cde35784..837a36d5 100644 --- a/grive/CMakeLists.txt +++ b/grive/CMakeLists.txt @@ -5,12 +5,11 @@ find_package(Boost COMPONENTS program_options REQUIRED) include_directories( ${grive_SOURCE_DIR}/../libgrive/src ${OPT_INCS} + ${Boost_INCLUDE_DIRS} ) -add_definitions( -DVERSION="${GRIVE_VERSION}" ) - file (GLOB GRIVE_EXE_SRC - ${grive_SOURCE_DIR}/src/*.cc + ${grive_SOURCE_DIR}/src/*.cc ) add_executable( grive_executable @@ -19,11 +18,11 @@ add_executable( grive_executable target_link_libraries( grive_executable ${Boost_LIBRARIES} - grive + grive ) set_target_properties( grive_executable - PROPERTIES OUTPUT_NAME grive + PROPERTIES OUTPUT_NAME grive ) install(TARGETS grive_executable RUNTIME DESTINATION bin) diff --git a/grive/doc/grive.1 b/grive/doc/grive.1 index 57b2e9cc..cf0bc26d 100644 --- a/grive/doc/grive.1 +++ b/grive/doc/grive.1 @@ -2,7 +2,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH "GRIVE" 1 "June 19, 2012" +.TH "GRIVE" 1 "January 3, 2016" .SH NAME grive \- Google Drive client for GNU/Linux @@ -26,33 +26,72 @@ Requests authorization token from Google Enable debug level messages. Implies \-V .TP \fB\-\-dry-run\fR -Only detects which files are needed for download or upload without doing it +Only detect which files need to be uploaded/downloaded, without actually performing changes .TP \fB\-f, \-\-force\fR Forces .I grive to always download a file from Google Drive instead uploading it .TP +\fB\-u, \-\-upload\-only\fR +Forces +.I grive +to not download anything from Google Drive and only upload local changes to server instead +.TP +\fB\-n, \-\-no\-remote\-new\fR +Forces +.I grive +to download only files that are changed in Google Drive and already exist locally +.TP \fB\-h\fR, \fB\-\-help\fR Produces help message .TP -\fB\-l\fR filename, \fB\-\-log\fR filename -Set log output to -.I filename +\fB\-\-ignore\fR +Ignore files with relative paths matching this Perl Regular Expression. +Value is remembered for next runs. +.TP +\fB\-l\fR , \fB\-\-log\fR +Write log output to +.I +.TP +\fB\-\-log\-http\fR +Log all HTTP responses in files named +.I YYYY-MM-DD.HHMMSS.txt +for debugging +.TP +\fB\-\-new\-rev\fR +Create new revisions in server for updated files +.TP +\fB\-p\fR , \fB\-\-path\fR +Use +.I +as the working copy root directory +.TP +\fB\-s\fR , \fB\-\-dir\fR +Sync a single +.I +subdirectory. Internally converted to an ignore regexp, remembered for next runs. .TP \fB\-v\fR, \fB\-\-version\fR Displays program version .TP +\fB\-P\fR, \fB\-\-progress-bar\fR +Print ASCII progress bar for each downloaded/uploaded file. +.TP \fB\-V\fR, \fB\-\-verbose\fR Verbose mode. Enables more messages than usual. -.SH AUTHOR +.SH AUTHORS .PP -The software was developed by Nestal Wan. +Current maintainer is Vitaliy Filippov. .PP +Original author was Nestal Wan. This manpage was written by José Luis Segura Lucas (josel.segura@gmx.es) +.PP +The full list of contributors may be found here +.I http://yourcmc.ru/wiki/Grive2#Full_list_of_contributors .SH REPORT BUGS .PP -.I https://github.com/Grive/grive +.I https://github.com/vitalif/grive2/issues .I https://groups.google.com/forum/?fromgroups#!forum/grive-devel diff --git a/grive/src/main.cc b/grive/src/main.cc index d51e0d92..c87a8543 100644 --- a/grive/src/main.cc +++ b/grive/src/main.cc @@ -17,12 +17,16 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "Config.hh" +#include "util/Config.hh" +#include "util/ProgressBar.hh" -#include "drive/Drive.hh" +#include "base/Drive.hh" +#include "drive2/Syncer2.hh" +#include "http/CurlAgent.hh" +#include "protocol/AuthAgent.hh" #include "protocol/OAuth2.hh" -#include "protocol/Json.hh" +#include "json/Val.hh" #include "bfd/Backtrace.hh" #include "util/Exception.hh" @@ -46,12 +50,13 @@ const std::string client_id = "22314510474.apps.googleusercontent.com" ; const std::string client_secret = "bl4ufi89h-9MkFlypcI7R785" ; using namespace gr ; +namespace po = boost::program_options; // libgcrypt insist this to be done in application, not library void InitGCrypt() { if ( !gcry_check_version(GCRYPT_VERSION) ) - throw Exception() << expt::ErrMsg( "libgcrypt version mismatch" ) ; + throw std::runtime_error( "libgcrypt version mismatch" ) ; // disable secure memory gcry_control(GCRYCTL_DISABLE_SECMEM, 0); @@ -60,18 +65,45 @@ void InitGCrypt() gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); } -int Main( int argc, char **argv ) +void InitLog( const po::variables_map& vm ) { - InitGCrypt() ; - - Config config ; - - std::auto_ptr comp_log(new log::CompositeLog) ; - LogBase* console_log = comp_log->Add( std::auto_ptr( new log::DefaultLog ) ) ; + std::unique_ptr comp_log( new log::CompositeLog ) ; + std::unique_ptr def_log( new log::DefaultLog ); + LogBase* console_log = comp_log->Add( def_log ) ; + + if ( vm.count( "log" ) ) + { + std::unique_ptr file_log( new log::DefaultLog( vm["log"].as() ) ) ; + file_log->Enable( log::debug ) ; + file_log->Enable( log::verbose ) ; + file_log->Enable( log::info ) ; + file_log->Enable( log::warning ) ; + file_log->Enable( log::error ) ; + file_log->Enable( log::critical ) ; + + // log grive version to log file + file_log->Log( log::Fmt("grive version " VERSION " " __DATE__ " " __TIME__), log::verbose ) ; + file_log->Log( log::Fmt("current time: %1%") % DateTime::Now(), log::verbose ) ; + + comp_log->Add( file_log ) ; + } - Json options ; + if ( vm.count( "verbose" ) ) + { + console_log->Enable( log::verbose ) ; + } - namespace po = boost::program_options; + if ( vm.count( "debug" ) ) + { + console_log->Enable( log::verbose ) ; + console_log->Enable( log::debug ) ; + } + LogBase::Inst( comp_log.release() ) ; +} + +int Main( int argc, char **argv ) +{ + InitGCrypt() ; // construct the program options po::options_description desc( "Grive options" ); @@ -79,30 +111,68 @@ int Main( int argc, char **argv ) ( "help,h", "Produce help message" ) ( "version,v", "Display Grive version" ) ( "auth,a", "Request authorization token" ) + ( "path,p", po::value(), "Path to working copy root") + ( "dir,s", po::value(), "Single subdirectory to sync") ( "verbose,V", "Verbose mode. Enable more messages than normal.") + ( "log-http", po::value(), "Log all HTTP responses in this file for debugging.") + ( "new-rev", "Create new revisions in server for updated files.") ( "debug,d", "Enable debug level messages. Implies -v.") ( "log,l", po::value(), "Set log output filename." ) ( "force,f", "Force grive to always download a file from Google Drive " "instead of uploading it." ) + ( "upload-only,u", "Do not download anything from Google Drive, only upload local changes" ) + ( "no-remote-new,n", "Download only files that are changed in Google Drive and already exist locally" ) ( "dry-run", "Only detect which files need to be uploaded/downloaded, " "without actually performing them." ) + ( "ignore", po::value(), "Perl RegExp to ignore files (matched against relative paths)." ) + ( "upload-speed,U", po::value(), "Limit upload speed in kbytes per second" ) + ( "download-speed,D", po::value(), "Limit download speed in kbytes per second" ) + ( "progress-bar,P", "Enable progress bar for upload/download of files") ; po::variables_map vm; po::store(po::parse_command_line( argc, argv, desc), vm ); po::notify(vm); + // simple commands that doesn't require log or config if ( vm.count("help") ) { std::cout << desc << std::endl ; return 0 ; } + else if ( vm.count( "version" ) ) + { + std::cout + << "grive version " << VERSION << ' ' << __DATE__ << ' ' << __TIME__ << std::endl ; + return 0 ; + } + + // initialize logging + InitLog(vm) ; + + Config config(vm) ; + + Log( "config file name %1%", config.Filename(), log::verbose ); + + std::unique_ptr http( new http::CurlAgent ); + if ( vm.count( "log-http" ) ) + http->SetLog( new http::ResponseLog( vm["log-http"].as(), ".txt" ) ); + + std::unique_ptr pb; + if ( vm.count( "progress-bar" ) ) + { + pb.reset( new ProgressBar() ); + http->SetProgressReporter( pb.get() ); + } + if ( vm.count( "auth" ) ) { + OAuth2 token( http.get(), client_id, client_secret ) ; + std::cout << "-----------------------\n" << "Please go to this URL and get an authentication code:\n\n" - << OAuth2::MakeAuthURL( client_id ) + << token.MakeAuthURL() << std::endl ; std::cout @@ -111,55 +181,17 @@ int Main( int argc, char **argv ) std::string code ; std::cin >> code ; - OAuth2 token( client_id, client_secret ) ; token.Auth( code ) ; // save to config - config.Get().Add( "refresh_token", Json( token.RefreshToken() ) ) ; + config.Set( "refresh_token", Val( token.RefreshToken() ) ) ; config.Save() ; } - if ( vm.count( "log" ) ) - { - std::auto_ptr file_log(new log::DefaultLog( vm["log"].as() )) ; - file_log->Enable( log::debug ) ; - file_log->Enable( log::verbose ) ; - file_log->Enable( log::info ) ; - file_log->Enable( log::warning ) ; - file_log->Enable( log::error ) ; - file_log->Enable( log::critical ) ; - - // log grive version to log file - file_log->Log( log::Fmt("grive version " VERSION " " __DATE__ " " __TIME__), log::verbose ) ; - file_log->Log( log::Fmt("current time: %1%") % DateTime::Now(), log::verbose ) ; - - comp_log->Add( file_log ) ; - } - if ( vm.count( "version" ) ) - { - std::cout - << "grive version " << VERSION << ' ' << __DATE__ << ' ' << __TIME__ << std::endl ; - return 0 ; - } - if ( vm.count( "verbose" ) ) - { - console_log->Enable( log::verbose ) ; - } - if ( vm.count( "debug" ) ) - { - console_log->Enable( log::verbose ) ; - console_log->Enable( log::debug ) ; - } - if ( vm.count( "force" ) ) - { - options.Add( "force", Json(true) ) ; - } - - LogBase::Inst( std::auto_ptr(comp_log.release()) ) ; std::string refresh_token ; try { - refresh_token = config.Get()["refresh_token"].Str() ; + refresh_token = config.Get("refresh_token").Str() ; } catch ( Exception& e ) { @@ -171,18 +203,32 @@ int Main( int argc, char **argv ) return -1; } - OAuth2 token( refresh_token, client_id, client_secret ) ; - Drive drive( token, options ) ; + OAuth2 token( http.get(), refresh_token, client_id, client_secret ) ; + AuthAgent agent( token, http.get() ) ; + v2::Syncer2 syncer( &agent ); + + if ( vm.count( "upload-speed" ) > 0 ) + agent.SetUploadSpeed( vm["upload-speed"].as() * 1000 ); + if ( vm.count( "download-speed" ) > 0 ) + agent.SetDownloadSpeed( vm["download-speed"].as() * 1000 ); + + Drive drive( &syncer, config.GetAll() ) ; drive.DetectChanges() ; if ( vm.count( "dry-run" ) == 0 ) { + // The progress bar should just be enabled when actual file transfers take place + if ( pb ) + pb->setShowProgressBar( true ) ; drive.Update() ; + if ( pb ) + pb->setShowProgressBar( false ) ; + drive.SaveState() ; } else drive.DryRun() ; - + config.Save() ; Log( "Finished!", log::info ) ; return 0 ; diff --git a/libgrive/CMakeLists.txt b/libgrive/CMakeLists.txt index 000d898b..54776bd1 100644 --- a/libgrive/CMakeLists.txt +++ b/libgrive/CMakeLists.txt @@ -3,15 +3,19 @@ project(libgrive) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") find_package(LibGcrypt REQUIRED) -find_package(JSONC REQUIRED) find_package(CURL REQUIRED) find_package(EXPAT REQUIRED) -find_package(Boost 1.40.0 COMPONENTS filesystem system REQUIRED) +find_package(Boost 1.40.0 COMPONENTS program_options filesystem unit_test_framework regex system REQUIRED) find_package(BFD) find_package(CppUnit) find_package(Iberty) find_package(ZLIB) +find_package(PkgConfig) +pkg_check_modules(YAJL REQUIRED yajl) + +add_definitions(-Wall) + # additional headers if build unit tests IF ( CPPUNIT_FOUND ) set( OPT_INCS ${CPPUNIT_INCLUDE_DIR} ) @@ -29,6 +33,8 @@ endif ( BFD_FOUND ) if ( IBERTY_FOUND ) set( OPT_LIBS ${OPT_LIBS} ${IBERTY_LIBRARY} ) +else ( IBERTY_FOUND ) + set( IBERTY_LIBRARY "" ) endif ( IBERTY_FOUND ) if ( ZLIB_FOUND ) @@ -38,12 +44,9 @@ endif ( ZLIB_FOUND ) include_directories( ${libgrive_SOURCE_DIR}/src ${libgrive_SOURCE_DIR}/test - ${GDBM_INCLUDE_DIR} + ${Boost_INCLUDE_DIRS} ${OPT_INCS} -) - -file(GLOB DRIVE_HEADERS - ${libgrive_SOURCE_DIR}/src/drive/*.hh + ${YAJL_INCLUDE_DIRS} ) file (GLOB PROTOCOL_HEADERS @@ -59,16 +62,16 @@ file (GLOB XML_HEADERS ) file (GLOB LIBGRIVE_SRC - src/drive/*.cc + src/base/*.cc + src/drive2/*.cc src/http/*.cc src/protocol/*.cc + src/json/*.cc src/util/*.cc src/util/log/*.cc - src/xml/*.cc ) add_definitions( - -DVERSION="${GRIVE_VERSION}" -DTEST_DATA="${libgrive_SOURCE_DIR}/test/data/" -DSRC_DIR="${libgrive_SOURCE_DIR}/src" ) @@ -77,10 +80,9 @@ add_definitions( add_library( grive STATIC ${LIBGRIVE_SRC} ${OPT_SRC} ) target_link_libraries( grive + ${YAJL_LIBRARIES} ${CURL_LIBRARIES} - ${JSONC_LIBRARY} ${LIBGCRYPT_LIBRARIES} - ${GDBM_LIBRARIES} ${Boost_LIBRARIES} ${IBERTY_LIBRARY} ${EXPAT_LIBRARY} @@ -112,9 +114,8 @@ IF ( CPPUNIT_FOUND ) # list of test source files here file(GLOB TEST_SRC - test/drive/*.cc + test/base/*.cc test/util/*.cc - test/xml/*.cc ) add_executable( unittest @@ -125,6 +126,24 @@ IF ( CPPUNIT_FOUND ) target_link_libraries( unittest grive ${CPPUNIT_LIBRARY} + ${Boost_LIBRARIES} ) ENDIF ( CPPUNIT_FOUND ) + +file(GLOB BTEST_SRC + test/btest/*.cc +) + +add_executable( btest ${BTEST_SRC} ) + +target_link_libraries( btest + grive + ${Boost_LIBRARIES} +) + +if ( WIN32 ) +else ( WIN32 ) + set_target_properties( btest + PROPERTIES COMPILE_FLAGS -DBOOST_TEST_DYN_LINK ) +endif (WIN32) diff --git a/libgrive/src/base/Drive.cc b/libgrive/src/base/Drive.cc new file mode 100644 index 00000000..fd26225c --- /dev/null +++ b/libgrive/src/base/Drive.cc @@ -0,0 +1,135 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Drive.hh" + +#include "Entry.hh" +#include "Feed.hh" +#include "Syncer.hh" + +#include "http/Agent.hh" +#include "util/Destroy.hh" +#include "util/log/Log.hh" + +#include + +// standard C++ library +#include +#include +#include +#include +#include + +// for debugging only +#include + +namespace gr { + +namespace +{ + const std::string state_file = ".grive_state" ; +} + +Drive::Drive( Syncer *syncer, const Val& options ) : + m_syncer ( syncer ), + m_root ( options["path"].Str() ), + m_state ( m_root / state_file, options ), + m_options ( options ) +{ + assert( m_syncer ) ; +} + +void Drive::FromRemote( const Entry& entry ) +{ + m_state.FromRemote( entry ) ; +} + +void Drive::FromChange( const Entry& entry ) +{ + if ( entry.IsRemoved() ) + Log( "file \"%1%\" represents a deletion, ignored", entry.Title(), log::verbose ) ; + + // folders go directly + else + m_state.FromRemote( entry ) ; +} + +void Drive::SaveState() +{ + m_state.Write( m_root / state_file ) ; +} + +void Drive::DetectChanges() +{ + Log( "Reading local directories", log::info ) ; + m_state.FromLocal( m_root ) ; + + Log( "Reading remote server file list", log::info ) ; + std::unique_ptr feed = m_syncer->GetAll() ; + + while ( feed->GetNext( m_syncer->Agent() ) ) + { + std::for_each( + feed->begin(), feed->end(), + boost::bind( &Drive::FromRemote, this, _1 ) ) ; + } + m_state.ResolveEntry() ; +} + +// pull the changes feed +// FIXME: unused until Grive will use the feed-based sync instead of reading full tree +void Drive::ReadChanges() +{ + long prev_stamp = m_state.ChangeStamp() ; + if ( prev_stamp != -1 ) + { + Trace( "previous change stamp is %1%", prev_stamp ) ; + Log( "Detecting changes from last sync", log::info ) ; + std::unique_ptr feed = m_syncer->GetChanges( prev_stamp+1 ) ; + while ( feed->GetNext( m_syncer->Agent() ) ) + { + std::for_each( + feed->begin(), feed->end(), + boost::bind( &Drive::FromChange, this, _1 ) ) ; + } + } +} + +void Drive::Update() +{ + Log( "Synchronizing files", log::info ) ; + m_state.Sync( m_syncer, m_options ) ; + + UpdateChangeStamp( ) ; +} + +void Drive::DryRun() +{ + Log( "Synchronizing files (dry-run)", log::info ) ; + m_state.Sync( NULL, m_options ) ; +} + +void Drive::UpdateChangeStamp( ) +{ + // FIXME: we should go through the changes to see if it was really Grive to made that change + // maybe by recording the updated timestamp and compare it? + m_state.ChangeStamp( m_syncer->GetChangeStamp( m_state.ChangeStamp()+1 ) ); +} + +} // end of namespace gr diff --git a/libgrive/src/drive/Drive.hh b/libgrive/src/base/Drive.hh similarity index 76% rename from libgrive/src/drive/Drive.hh rename to libgrive/src/base/Drive.hh index 9a6227c2..5f99be24 100644 --- a/libgrive/src/drive/Drive.hh +++ b/libgrive/src/base/Drive.hh @@ -19,9 +19,9 @@ #pragma once -#include "State.hh" +#include "base/State.hh" -#include "http/Header.hh" +#include "json/Val.hh" #include "util/Exception.hh" #include @@ -29,19 +29,16 @@ namespace gr { -namespace http -{ - class Agent ; -} +class Syncer ; class Entry ; -class OAuth2 ; -class Json ; + +class State ; class Drive { public : - Drive( OAuth2& auth, const Json& options ) ; + Drive( Syncer *syncer, const Val& options ) ; void DetectChanges() ; void Update() ; @@ -51,18 +48,16 @@ public : struct Error : virtual Exception {} ; private : - void SyncFolders( http::Agent *http ) ; - void file(); + void ReadChanges() ; void FromRemote( const Entry& entry ) ; void FromChange( const Entry& entry ) ; - void UpdateChangeStamp( http::Agent *http ) ; + void UpdateChangeStamp( ) ; private : - OAuth2& m_auth ; - http::Header m_http_hdr ; - - std::string m_resume_link ; + Syncer *m_syncer ; + fs::path m_root ; State m_state ; + Val m_options ; } ; -} // end of namespace +} // end of namespace gr diff --git a/libgrive/src/base/Entry.cc b/libgrive/src/base/Entry.cc new file mode 100644 index 00000000..05f346bc --- /dev/null +++ b/libgrive/src/base/Entry.cc @@ -0,0 +1,129 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Entry.hh" + +#include "util/Crypt.hh" +#include "util/log/Log.hh" +#include "util/OS.hh" +#include "xml/Node.hh" +#include "xml/NodeSet.hh" + +#include +#include + +namespace gr { + +/// construct an entry for the root folder +Entry::Entry( ) : + m_title ( "." ), + m_is_dir ( true ), + m_resource_id ( "folder:root" ), + m_change_stamp ( -1 ), + m_is_removed ( false ), + m_size ( 0 ) +{ +} + +const std::vector& Entry::ParentHrefs() const +{ + return m_parent_hrefs ; +} + +std::string Entry::Title() const +{ + return m_title ; +} + +std::string Entry::Filename() const +{ + return m_filename ; +} + +bool Entry::IsDir() const +{ + return m_is_dir ; +} + +std::string Entry::MD5() const +{ + return m_md5 ; +} + +u64_t Entry::Size() const +{ + return m_size ; +} + +DateTime Entry::MTime() const +{ + return m_mtime ; +} + +std::string Entry::SelfHref() const +{ + return m_self_href ; +} + +std::string Entry::ParentHref() const +{ + return m_parent_hrefs.empty() ? "" : m_parent_hrefs.front() ; +} + +std::string Entry::ResourceID() const +{ + return m_resource_id ; +} + +std::string Entry::ETag() const +{ + return m_etag ; +} + +std::string Entry::ContentSrc() const +{ + return m_content_src ; +} + +bool Entry::IsEditable() const +{ + return m_is_editable ; +} + +long Entry::ChangeStamp() const +{ + return m_change_stamp ; +} + +bool Entry::IsChange() const +{ + return m_change_stamp != -1 ; +} + +bool Entry::IsRemoved() const +{ + return m_is_removed ; +} + +std::string Entry::Name() const +{ + return !m_filename.empty() ? m_filename : m_title ; +} + +} // end of namespace gr diff --git a/libgrive/src/drive/Entry.hh b/libgrive/src/base/Entry.hh similarity index 82% rename from libgrive/src/drive/Entry.hh rename to libgrive/src/base/Entry.hh index 8a9b8cc7..8324f6db 100644 --- a/libgrive/src/drive/Entry.hh +++ b/libgrive/src/base/Entry.hh @@ -19,6 +19,7 @@ #pragma once +#include "util/Types.hh" #include "util/DateTime.hh" #include "util/FileSystem.hh" @@ -28,11 +29,6 @@ namespace gr { -namespace xml -{ - class Node ; -} - /*! \brief corresponds to an "entry" in the resource feed This class is decodes an entry in the resource feed. It will stored the properties like @@ -43,13 +39,13 @@ class Entry { public : Entry( ) ; - explicit Entry( const xml::Node& n ) ; std::string Title() const ; std::string Filename() const ; - std::string Kind() const ; + bool IsDir() const ; std::string MD5() const ; DateTime MTime() const ; + u64_t Size() const ; std::string Name() const ; @@ -57,11 +53,9 @@ public : std::string ETag() const ; std::string SelfHref() const ; - std::string AltSelf() const ; std::string ParentHref() const ; std::string ContentSrc() const ; - std::string EditLink() const ; - std::string CreateLink() const ; + bool IsEditable() const ; long ChangeStamp() const ; bool IsChange() const ; @@ -69,30 +63,26 @@ public : const std::vector& ParentHrefs() const ; - void Swap( Entry& e ) ; - - void Update( const xml::Node& entry ) ; - -private : +protected : std::string m_title ; std::string m_filename ; - std::string m_kind ; + bool m_is_dir ; std::string m_md5 ; std::string m_etag ; std::string m_resource_id ; std::vector m_parent_hrefs ; - + std::string m_self_href ; - std::string m_alt_self ; std::string m_content_src ; - std::string m_edit_link ; - std::string m_create_link ; + + bool m_is_editable ; long m_change_stamp ; - + DateTime m_mtime ; bool m_is_removed ; + u64_t m_size ; } ; -} // end of namespace +} // end of namespace gr diff --git a/libgrive/src/drive/CommonUri.cc b/libgrive/src/base/Feed.cc similarity index 73% rename from libgrive/src/drive/CommonUri.cc rename to libgrive/src/base/Feed.cc index dc3e0558..0ea834d3 100644 --- a/libgrive/src/drive/CommonUri.cc +++ b/libgrive/src/base/Feed.cc @@ -17,15 +17,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "CommonUri.hh" -#include +#include "Feed.hh" + +#include "Entry.hh" + +#include "http/Agent.hh" namespace gr { -std::string ChangesFeed( int changestamp ) +Feed::Feed( const std::string &url ): + m_next( url ) +{ +} + +Feed::iterator Feed::begin() const +{ + return m_entries.begin() ; +} + +Feed::iterator Feed::end() const { - boost::format feed( feed_changes + "?start-index=%1%" ) ; - return changestamp > 0 ? (feed%changestamp).str() : feed_changes ; + return m_entries.end() ; } -} \ No newline at end of file +} // end of namespace gr::v1 diff --git a/libgrive/src/base/Feed.hh b/libgrive/src/base/Feed.hh new file mode 100644 index 00000000..d43c68ce --- /dev/null +++ b/libgrive/src/base/Feed.hh @@ -0,0 +1,52 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "base/Entry.hh" + +#include + +#include + +namespace gr { + +namespace http +{ + class Agent ; +} + +class Feed +{ +public : + typedef std::vector Entries; + typedef std::vector::const_iterator iterator; + +public : + Feed( const std::string& url ); + virtual bool GetNext( http::Agent *http ) = 0 ; + iterator begin() const ; + iterator end() const ; + +protected : + Entries m_entries ; + std::string m_next ; +} ; + +} // end of namespace gr diff --git a/libgrive/src/base/Resource.cc b/libgrive/src/base/Resource.cc new file mode 100644 index 00000000..23ce335d --- /dev/null +++ b/libgrive/src/base/Resource.cc @@ -0,0 +1,793 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Resource.hh" +#include "ResourceTree.hh" +#include "Entry.hh" +#include "Syncer.hh" + +#include "json/Val.hh" +#include "util/CArray.hh" +#include "util/Crypt.hh" +#include "util/log/Log.hh" +#include "util/OS.hh" +#include "util/File.hh" +#include "http/Error.hh" + +#include +#include +#include + +#include + +#include + +// for debugging +#include + +namespace gr { + +/// default constructor creates the root folder +Resource::Resource( const fs::path& root_folder ) : + m_name ( root_folder.string() ), + m_kind ( "folder" ), + m_size ( 0 ), + m_id ( "folder:root" ), + m_href ( "root" ), + m_is_editable( true ), + m_parent ( 0 ), + m_state ( sync ), + m_json ( NULL ), + m_local_exists( true ) +{ +} + +Resource::Resource( const std::string& name, const std::string& kind ) : + m_name ( name ), + m_kind ( kind ), + m_size ( 0 ), + m_is_editable( true ), + m_parent ( 0 ), + m_state ( unknown ), + m_json ( NULL ), + m_local_exists( false ) +{ +} + +void Resource::SetState( State new_state ) +{ + // only the new and delete states need to be set recursively + assert( + new_state == remote_new || new_state == remote_deleted || + new_state == local_new || new_state == local_deleted + ) ; + + m_state = new_state ; + std::for_each( m_child.begin(), m_child.end(), + boost::bind( &Resource::SetState, _1, new_state ) ) ; +} + +void Resource::FromRemoteFolder( const Entry& remote ) +{ + fs::path path = Path() ; + + if ( !remote.IsEditable() ) + Log( "folder %1% is read-only", path, log::verbose ) ; + + // already sync + if ( m_local_exists && m_kind == "folder" ) + { + Log( "folder %1% is in sync", path, log::verbose ) ; + m_state = sync ; + } + else if ( m_local_exists && m_kind == "file" ) + { + // TODO: handle type change + Log( "%1% changed from folder to file", path, log::verbose ) ; + m_state = sync ; + } + else if ( m_local_exists && m_kind == "bad" ) + { + Log( "%1% inaccessible", path, log::verbose ) ; + m_state = sync ; + } + else if ( remote.MTime().Sec() > m_mtime.Sec() ) // FIXME only seconds are stored in local index + { + // remote folder created after last sync, so remote is newer + Log( "folder %1% is created in remote", path, log::verbose ) ; + SetState( remote_new ) ; + } + else + { + Log( "folder %1% is deleted in local", path, log::verbose ) ; + SetState( local_deleted ) ; + } +} + +/// Update the state according to information (i.e. Entry) from remote. This function +/// compares the modification time and checksum of both copies and determine which +/// one is newer. +void Resource::FromRemote( const Entry& remote ) +{ + // sync folder + if ( remote.IsDir() && IsFolder() ) + FromRemoteFolder( remote ) ; + else + FromRemoteFile( remote ) ; + + AssignIDs( remote ) ; + + assert( m_state != unknown ) ; + + if ( m_state == remote_new || m_state == remote_changed ) + m_md5 = remote.MD5() ; + + m_mtime = remote.MTime() ; +} + +void Resource::AssignIDs( const Entry& remote ) +{ + // the IDs from change feed entries are different + if ( !remote.IsChange() ) + { + m_id = remote.ResourceID() ; + m_href = remote.SelfHref() ; + m_content = remote.ContentSrc() ; + m_is_editable = remote.IsEditable() ; + m_etag = remote.ETag() ; + m_md5 = remote.MD5() ; + } +} + +void Resource::FromRemoteFile( const Entry& remote ) +{ + assert( m_parent != 0 ) ; + + fs::path path = Path() ; + + // recursively create/delete folder + if ( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted || + m_parent->m_state == local_new || m_parent->m_state == local_deleted ) + { + Log( "file %1% parent %2% recursively in %3% (%4%)", path, + ( m_parent->m_state == remote_new || m_parent->m_state == local_new ) ? "created" : "deleted", + ( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted ) ? "remote" : "local", + m_parent->m_state, log::verbose ) ; + + m_state = m_parent->m_state ; + } + + else if ( m_kind == "bad" ) + { + m_state = sync; + } + + // local not exists + else if ( !m_local_exists ) + { + Trace( "file %1% change stamp = %2%", Path(), remote.ChangeStamp() ) ; + + if ( remote.MTime().Sec() > m_mtime.Sec() || remote.MD5() != m_md5 || remote.ChangeStamp() > 0 ) + { + Log( "file %1% is created in remote (change %2%)", path, + remote.ChangeStamp(), log::verbose ) ; + m_size = remote.Size(); + m_state = remote_new ; + } + else + { + Log( "file %1% is deleted in local", path, log::verbose ) ; + m_state = local_deleted ; + } + } + + // remote checksum unknown, assume the file is not changed in remote + else if ( remote.MD5().empty() ) + { + Log( "file %1% has unknown checksum in remote. assumed in sync", + Path(), log::verbose ) ; + m_state = sync ; + } + + // use mtime to check which one is more recent + else if ( remote.Size() != m_size || remote.MD5() != GetMD5() ) + { + assert( m_state != unknown ) ; + + // if remote is modified + if ( remote.MTime().Sec() > m_mtime.Sec() ) + { + Log( "file %1% is changed in remote", path, log::verbose ) ; + m_size = remote.Size(); + m_state = remote_changed ; + } + + // remote also has the file, so it's not new in local + else if ( m_state == local_new || m_state == remote_deleted ) + { + Log( "file %1% is changed in local", path, log::verbose ) ; + m_state = local_changed ; + } + else + Trace( "file %1% state is %2%", m_name, m_state ) ; + } + + // if checksum is equal, no need to compare the mtime + else + { + Log( "file %1% is already in sync", Path(), log::verbose ) ; + m_state = sync ; + } +} + +void Resource::FromDeleted( Val& state ) +{ + assert( !m_json ); + m_json = &state; + if ( state.Has( "ctime" ) ) + m_ctime.Assign( state["ctime"].U64(), 0 ); + if ( state.Has( "md5" ) ) + m_md5 = state["md5"]; + if ( state.Has( "srv_time" ) ) + m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ; + if ( state.Has( "size" ) ) + m_size = state[ "size" ].U64(); + m_state = both_deleted; +} + +/// Update the resource with the attributes of local file or directory. This +/// function will propulate the fields in m_entry. +void Resource::FromLocal( Val& state ) +{ + assert( !m_json ); + m_json = &state; + + // root folder is always in sync + if ( !IsRoot() ) + { + fs::path path = Path() ; + bool is_dir; + try + { + os::Stat( path, &m_ctime, (off64_t*)&m_size, &is_dir ) ; + } + catch ( os::Error &e ) + { + // invalid symlink, unreadable file or something else + int const* eno = boost::get_error_info< boost::errinfo_errno >(e); + Log( "Error accessing %1%: %2%; skipping file", path.string(), strerror( *eno ), log::warning ); + m_state = sync; + m_kind = "bad"; + return; + } + + m_name = path.filename().string() ; + m_kind = is_dir ? "folder" : "file"; + m_local_exists = true; + + bool is_changed; + if ( state.Has( "ctime" ) && (u64_t) m_ctime.Sec() <= state["ctime"].U64() && + ( is_dir || state.Has( "md5" ) ) ) + { + if ( !is_dir ) + m_md5 = state["md5"]; + is_changed = false; + } + else + { + if ( !is_dir ) + { + // File is changed locally. TODO: Detect conflicts + is_changed = ( state.Has( "size" ) && m_size != state["size"].U64() ) || + !state.Has( "md5" ) || GetMD5() != state["md5"].Str(); + } + else + is_changed = true; + } + if ( state.Has( "srv_time" ) ) + m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ; + + // Upload file if it is changed and remove if not. + // State will be updated to sync/remote_changed in FromRemote() + m_state = is_changed ? local_new : remote_deleted; + if ( m_state == local_new ) + { + // local_new means this file is changed in local. + // this means we can't delete any of its parents. + // make sure their state is also set to local_new. + Resource *p = m_parent; + while ( p && p->m_state == remote_deleted ) + { + p->m_state = local_new; + p = p->m_parent; + } + } + } + + assert( m_state != unknown ) ; +} + +std::string Resource::SelfHref() const +{ + return m_href ; +} + +std::string Resource::ContentSrc() const +{ + return m_content ; +} + +std::string Resource::ETag() const +{ + return m_etag ; +} + +std::string Resource::Name() const +{ + return m_name ; +} + +std::string Resource::Kind() const +{ + return m_kind ; +} + +DateTime Resource::ServerTime() const +{ + return m_mtime ; +} + +std::string Resource::ResourceID() const +{ + return m_id ; +} + +Resource::State Resource::GetState() const +{ + return m_state ; +} + +const Resource* Resource::Parent() const +{ + assert( m_parent == 0 || m_parent->IsFolder() ) ; + return m_parent ; +} + +Resource* Resource::Parent() +{ + assert( m_parent == 0 || m_parent->IsFolder() ) ; + return m_parent ; +} + +void Resource::AddChild( Resource *child ) +{ + assert( child != 0 ) ; + assert( child->m_parent == 0 || child->m_parent == this ) ; + assert( child != this ) ; + + child->m_parent = this ; + m_child.push_back( child ) ; +} + +bool Resource::IsFolder() const +{ + return m_kind == "folder" ; +} + +bool Resource::IsEditable() const +{ + return m_is_editable ; +} + +fs::path Resource::Path() const +{ + assert( m_parent != this ) ; + assert( m_parent == 0 || m_parent->IsFolder() ) ; + + return m_parent != 0 ? (m_parent->Path() / m_name) : m_name ; +} + +// Path relative to the root directory +fs::path Resource::RelPath() const +{ + assert( m_parent != this ) ; + assert( m_parent == 0 || m_parent->IsFolder() ) ; + + return m_parent != 0 && !m_parent->IsRoot() ? (m_parent->RelPath() / m_name) : m_name ; +} + +bool Resource::IsInRootTree() const +{ + assert( m_parent == 0 || m_parent->IsFolder() ) ; + return m_parent == 0 ? IsRoot() : m_parent->IsInRootTree() ; +} + +Resource* Resource::FindChild( const std::string& name ) +{ + for ( std::vector::iterator i = m_child.begin() ; i != m_child.end() ; ++i ) + { + assert( (*i)->m_parent == this ) ; + if ( (*i)->m_name == name ) + return *i ; + } + return 0 ; +} + +// try to change the state to "sync" +void Resource::Sync( Syncer *syncer, ResourceTree *res_tree, const Val& options ) +{ + assert( m_state != unknown ) ; + assert( !IsRoot() || m_state == sync ) ; // root folder is already synced + + try + { + SyncSelf( syncer, res_tree, options ) ; + } + catch ( File::Error &e ) + { + int *en = boost::get_error_info< boost::errinfo_errno > ( e ) ; + Log( "Error syncing %1%: %2%", Path(), en ? strerror( *en ) : "", log::error ); + return; + } + catch ( boost::filesystem::filesystem_error &e ) + { + Log( "Error syncing %1%: %2%", Path(), e.what(), log::error ); + return; + } + catch ( http::Error &e ) + { + int *curlcode = boost::get_error_info< http::CurlCode > ( e ) ; + int *httpcode = boost::get_error_info< http::HttpResponseCode > ( e ) ; + std::string msg; + if ( curlcode ) + msg = *( boost::get_error_info< http::CurlErrMsg > ( e ) ); + else if ( httpcode ) + msg = "HTTP " + boost::to_string( *httpcode ); + else + msg = e.what(); + Log( "Error syncing %1%: %2%", Path(), msg, log::error ); + std::string *url = boost::get_error_info< http::Url > ( e ); + std::string *resp_hdr = boost::get_error_info< http::HttpResponseHeaders > ( e ); + std::string *resp_txt = boost::get_error_info< http::HttpResponseText > ( e ); + http::Header *req_hdr = boost::get_error_info< http::HttpRequestHeaders > ( e ); + if ( url ) + Log( "Request URL: %1%", *url, log::verbose ); + if ( req_hdr ) + Log( "Request headers: %1%", req_hdr->Str(), log::verbose ); + if ( resp_hdr ) + Log( "Response headers: %1%", *resp_hdr, log::verbose ); + if ( resp_txt ) + Log( "Response text: %1%", *resp_txt, log::verbose ); + return; + } + + // if myself is deleted, no need to do the childrens + if ( m_state != local_deleted && m_state != remote_deleted ) + { + std::for_each( m_child.begin(), m_child.end(), + boost::bind( &Resource::Sync, _1, syncer, res_tree, options ) ) ; + } +} + +bool Resource::CheckRename( Syncer* syncer, ResourceTree *res_tree ) +{ + if ( !IsFolder() && ( m_state == local_new || m_state == remote_new ) ) + { + bool is_local = m_state == local_new; + State other = is_local ? local_deleted : remote_deleted; + if ( is_local ) + { + // First check size index for locally added files + details::SizeRange moved = res_tree->FindBySize( m_size ); + bool found = false; + for ( details::SizeMap::iterator i = moved.first ; i != moved.second; i++ ) + { + Resource *m = *i; + if ( m->m_state == other ) + { + found = true; + break; + } + } + if ( !found ) + { + // Don't check md5 sums if there are no deleted files with same size + return false; + } + } + details::MD5Range moved = res_tree->FindByMD5( GetMD5() ); + for ( details::MD5Map::iterator i = moved.first ; i != moved.second; i++ ) + { + Resource *m = *i; + if ( m->m_state == other ) + { + Resource* from = m_state == local_new || m_state == remote_new ? m : this; + Resource* to = m_state == local_new || m_state == remote_new ? this : m; + Log( "sync %1% moved to %2%. moving %3%", from->Path(), to->Path(), + is_local ? "remote" : "local", log::info ); + if ( syncer ) + { + if ( is_local ) + { + syncer->Move( from, to->Parent(), to->Name() ); + to->SetIndex( false ); + } + else + { + fs::rename( from->Path(), to->Path() ); + to->SetIndex( true ); + } + to->m_mtime = from->m_mtime; + to->m_json->Set( "srv_time", Val( from->m_mtime.Sec() ) ); + from->DeleteIndex(); + } + from->m_state = both_deleted; + to->m_state = sync; + return true; + } + } + } + return false; +} + +void Resource::SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) +{ + assert( !IsRoot() || m_state == sync ) ; // root is always sync + assert( IsRoot() || !syncer || m_parent->IsFolder() ) ; + assert( IsRoot() || m_parent->m_state != remote_deleted ) ; + assert( IsRoot() || m_parent->m_state != local_deleted ) ; + + const fs::path path = Path() ; + + // Detect renames + if ( CheckRename( syncer, res_tree ) ) + return; + + switch ( m_state ) + { + case local_new : + Log( "sync %1% doesn't exist in server, uploading", path, log::info ) ; + + if ( syncer && syncer->Create( this ) ) + { + m_state = sync ; + SetIndex( false ); + } + break ; + + case local_deleted : + Log( "sync %1% deleted in local. deleting remote", path, log::info ) ; + if ( syncer && !options["no-delete-remote"].Bool() ) + { + syncer->DeleteRemote( this ) ; + DeleteIndex() ; + } + break ; + + case local_changed : + Log( "sync %1% changed in local. uploading", path, log::info ) ; + if ( syncer && syncer->EditContent( this, options["new-rev"].Bool() ) ) + { + m_state = sync ; + SetIndex( false ); + } + break ; + + case remote_new : + if ( options["no-remote-new"].Bool() ) + Log( "sync %1% created in remote. skipping", path, log::info ) ; + else + { + Log( "sync %1% created in remote. creating local", path, log::info ) ; + if ( syncer ) + { + if ( IsFolder() ) + fs::create_directories( path ) ; + else + syncer->Download( this, path ) ; + SetIndex( true ) ; + m_state = sync ; + } + } + break ; + + case remote_changed : + assert( !IsFolder() ) ; + if ( options["upload-only"].Bool() ) + Log( "sync %1% changed in remote. skipping", path, log::info ) ; + else + { + Log( "sync %1% changed in remote. downloading", path, log::info ) ; + if ( syncer ) + { + syncer->Download( this, path ) ; + SetIndex( true ) ; + m_state = sync ; + } + } + break ; + + case remote_deleted : + Log( "sync %1% deleted in remote. deleting local", path, log::info ) ; + if ( syncer ) + { + DeleteLocal() ; + DeleteIndex() ; + } + break ; + + case both_deleted : + if ( syncer ) + DeleteIndex() ; + break ; + + case sync : + Log( "sync %1% already in sync", path, log::verbose ) ; + if ( !IsRoot() ) + SetIndex( false ) ; + break ; + + // shouldn't go here + case unknown : + default : + assert( false ) ; + break ; + } + + if ( syncer && m_json ) + { + // Update server time of this file + m_json->Set( "srv_time", Val( m_mtime.Sec() ) ); + } +} + +void Resource::SetServerTime( const DateTime& time ) +{ + m_mtime = time ; +} + +/// this function doesn't really remove the local file. it renames it. +void Resource::DeleteLocal() +{ + static const boost::format trash_file( "%1%-%2%" ) ; + + assert( m_parent != NULL ) ; + Resource* p = m_parent; + fs::path destdir; + while ( !p->IsRoot() ) + { + destdir = p->Name() / destdir; + p = p->Parent(); + } + destdir = p->Path() / ".trash" / destdir; + + fs::path dest = destdir / Name(); + std::size_t idx = 1 ; + while ( fs::exists( dest ) && idx != 0 ) + dest = destdir / (boost::format(trash_file) % Name() % idx++).str() ; + + // wrap around! just remove the file + if ( idx == 0 ) + fs::remove_all( Path() ) ; + else + { + fs::create_directories( dest.parent_path() ) ; + fs::rename( Path(), dest ) ; + } +} + +void Resource::DeleteIndex() +{ + (*m_parent->m_json)["tree"].Del( Name() ); + m_json = NULL; +} + +void Resource::SetIndex( bool re_stat ) +{ + assert( m_parent && m_parent->m_json != NULL ); + if ( !m_json ) + m_json = &((*m_parent->m_json)["tree"]).Item( Name() ); + bool is_dir; + if ( re_stat ) + os::Stat( Path(), &m_ctime, NULL, &is_dir ); + else + is_dir = IsFolder(); + m_json->Set( "ctime", Val( m_ctime.Sec() ) ); + if ( !is_dir ) + { + m_json->Set( "md5", Val( m_md5 ) ); + m_json->Set( "size", Val( m_size ) ); + m_json->Del( "tree" ); + } + else + { + // add tree item if it does not exist + m_json->Item( "tree" ); + m_json->Del( "md5" ); + m_json->Del( "size" ); + } +} + +Resource::iterator Resource::begin() const +{ + return m_child.begin() ; +} + +Resource::iterator Resource::end() const +{ + return m_child.end() ; +} + +std::size_t Resource::size() const +{ + return m_child.size() ; +} + +std::ostream& operator<<( std::ostream& os, Resource::State s ) +{ + static const char *state[] = + { + "sync", "local_new", "local_changed", "local_deleted", "remote_new", + "remote_changed", "remote_deleted", "both_deleted" + } ; + assert( s >= 0 && s < Count(state) ) ; + return os << state[s] ; +} + +std::string Resource::StateStr() const +{ + std::ostringstream ss ; + ss << m_state ; + return ss.str() ; +} + +u64_t Resource::Size() const +{ + return m_size ; +} + +std::string Resource::MD5() const +{ + return m_md5 ; +} + +std::string Resource::GetMD5() +{ + if ( m_md5.empty() && !IsFolder() && m_local_exists ) + { + // MD5 checksum is calculated lazily and only when really needed: + // 1) when a local rename is supposed (when there are a new file and a deleted file of the same size) + // 2) when local ctime is changed, but file size isn't + m_md5 = crypt::MD5::Get( Path() ); + } + return m_md5 ; +} + +bool Resource::IsRoot() const +{ + // Root entry does not show up in file feeds, so we check for empty parent (and self-href) + return !m_parent ; +} + +bool Resource::HasID() const +{ + return !m_href.empty() && !m_id.empty() ; +} + +} // end of namespace diff --git a/libgrive/src/drive/Resource.hh b/libgrive/src/base/Resource.hh similarity index 74% rename from libgrive/src/drive/Resource.hh rename to libgrive/src/base/Resource.hh index b14e7ee7..57d7319c 100644 --- a/libgrive/src/drive/Resource.hh +++ b/libgrive/src/base/Resource.hh @@ -19,6 +19,7 @@ #pragma once +#include "util/Types.hh" #include "util/DateTime.hh" #include "util/Exception.hh" #include "util/FileSystem.hh" @@ -29,11 +30,11 @@ namespace gr { -namespace http -{ - class Agent ; - class Header ; -} +class ResourceTree ; + +class Syncer ; + +class Val ; class Entry ; @@ -45,48 +46,9 @@ class Entry ; class Resource { public : - struct Error : virtual Exception {} ; - typedef std::vector Children ; typedef Children::const_iterator iterator ; -public : - Resource() ; - Resource( const std::string& name, const std::string& kind ) ; - - // default copy ctor & op= are fine - void Swap( Resource& coll ) ; - - bool IsFolder() const ; - - std::string Name() const ; - std::string SelfHref() const ; - std::string ResourceID() const ; - - const Resource* Parent() const ; - Resource* Parent() ; - void AddChild( Resource *child ) ; - Resource* FindChild( const std::string& title ) ; - - fs::path Path() const ; - bool IsInRootTree() const ; - bool IsRoot() const ; - bool HasID() const ; - std::string MD5() const ; - - void FromRemote( const Entry& remote, const DateTime& last_sync ) ; - void FromLocal( const DateTime& last_sync ) ; - - void Sync( http::Agent* http, const http::Header& auth ) ; - - // children access - iterator begin() const ; - iterator end() const ; - std::size_t size() const ; - - std::string StateStr() const ; - -private : /// State of the resource. indicating what to do with the resource enum State { @@ -108,59 +70,104 @@ private : /// We should download the file. remote_new, - /// Resource exists in both local & remote, but remote is newer. + /// Resource exists in both local & remote, but remote is newer. remote_changed, /// Resource delete in remote, need to delete in local remote_deleted, + /// Both deleted. State is used to remove leftover files from the index after sync. + both_deleted, /// invalid value unknown } ; - friend std::ostream& operator<<( std::ostream& os, State s ) ; +public : + Resource(const fs::path& root_folder) ; + Resource( const std::string& name, const std::string& kind ) ; -private : - void SetState( State new_state ) ; + bool IsFolder() const ; + bool IsEditable() const ; - void Download( http::Agent* http, const fs::path& file, const http::Header& auth ) const ; - bool EditContent( http::Agent* http, const http::Header& auth ) ; - bool Create( http::Agent* http, const http::Header& auth ) ; - bool Upload( http::Agent* http, const std::string& link, const http::Header& auth, bool post ) ; + std::string Name() const ; + std::string Kind() const ; + DateTime ServerTime() const ; + std::string SelfHref() const ; + std::string ContentSrc() const ; + std::string ETag() const ; + std::string ResourceID() const ; + State GetState() const; - void FromRemoteFolder( const Entry& remote, const DateTime& last_sync ) ; - void FromRemoteFile( const Entry& remote, const DateTime& last_sync ) ; + const Resource* Parent() const ; + Resource* Parent() ; + void AddChild( Resource *child ) ; + Resource* FindChild( const std::string& title ) ; - void DeleteLocal() ; - void DeleteRemote( http::Agent* http, const http::Header& auth ) ; + fs::path Path() const ; + fs::path RelPath() const ; + bool IsInRootTree() const ; + bool IsRoot() const ; + bool HasID() const ; + u64_t Size() const; + std::string MD5() const ; + std::string GetMD5() ; + + void FromRemote( const Entry& remote ) ; + void FromDeleted( Val& state ) ; + void FromLocal( Val& state ) ; + + void Sync( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ; + void SetServerTime( const DateTime& time ) ; + + // children access + iterator begin() const ; + iterator end() const ; + std::size_t size() const ; + std::string StateStr() const ; + +private : + void AssignIDs( const Entry& remote ) ; - void SyncSelf( http::Agent* http, const http::Header& auth ) ; + + friend std::ostream& operator<<( std::ostream& os, State s ) ; + friend class Syncer ; + +private : + void SetState( State new_state ) ; + void FromRemoteFolder( const Entry& remote ) ; + void FromRemoteFile( const Entry& remote ) ; + + void DeleteLocal() ; + void DeleteIndex() ; + void SetIndex( bool ) ; + + bool CheckRename( Syncer* syncer, ResourceTree *res_tree ) ; + void SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ; + private : std::string m_name ; std::string m_kind ; std::string m_md5 ; DateTime m_mtime ; - + DateTime m_ctime ; + u64_t m_size ; + std::string m_id ; std::string m_href ; - std::string m_edit ; - std::string m_create ; std::string m_content ; std::string m_etag ; + bool m_is_editable ; // not owned Resource *m_parent ; std::vector m_child ; - + State m_state ; + Val* m_json ; + bool m_local_exists ; } ; -} // end of namespace - -namespace std -{ - void swap( gr::Resource& c1, gr::Resource& c2 ) ; -} +} // end of namespace gr::v1 diff --git a/libgrive/src/drive/ResourceTree.cc b/libgrive/src/base/ResourceTree.cc similarity index 72% rename from libgrive/src/drive/ResourceTree.cc rename to libgrive/src/base/ResourceTree.cc index aedbd238..e6eb6190 100644 --- a/libgrive/src/drive/ResourceTree.cc +++ b/libgrive/src/base/ResourceTree.cc @@ -18,9 +18,7 @@ */ #include "ResourceTree.hh" -#include "CommonUri.hh" -#include "protocol/Json.hh" #include "util/Destroy.hh" #include "util/log/Log.hh" @@ -31,8 +29,8 @@ namespace gr { using namespace details ; -ResourceTree::ResourceTree( ) : - m_root( new Resource ) +ResourceTree::ResourceTree( const fs::path& rootFolder ) : + m_root( new Resource( rootFolder ) ) { m_set.insert( m_root ) ; } @@ -44,7 +42,7 @@ ResourceTree::ResourceTree( const ResourceTree& fs ) : for ( Set::const_iterator i = s.begin() ; i != s.end() ; ++i ) { Resource *c = new Resource( **i ) ; - if ( c->SelfHref() == root_href ) + if ( c->IsRoot() ) m_root = c ; m_set.insert( c ) ; @@ -80,18 +78,6 @@ const Resource* ResourceTree::Root() const return m_root ; } -void ResourceTree::Swap( ResourceTree& fs ) -{ - m_set.swap( fs.m_set ) ; -} - -ResourceTree& ResourceTree::operator=( const ResourceTree& fs ) -{ - ResourceTree tmp( fs ) ; - Swap( tmp ) ; - return *this ; -} - Resource* ResourceTree::FindByHref( const std::string& href ) { // for the resource that not yet have href (e.g. not yet fetched from server) @@ -111,30 +97,21 @@ const Resource* ResourceTree::FindByHref( const std::string& href ) const return i != map.end() ? *i : 0 ; } -/// Unlike other search functions, this one does not depend on the multi-index -/// container. It traverses the tree instead. -Resource* ResourceTree::FindByPath( const fs::path& path ) +MD5Range ResourceTree::FindByMD5( const std::string& md5 ) { - Resource *current = m_root ; - for ( fs::path::iterator i = path.begin() ; i != path.end() && current != 0 ; ++i ) - { - Trace( "path it = %1%", *i ) ; - - // current directory - if ( *i == "." ) - continue ; - - else if ( *i == ".." ) - current = current->Parent() ; - - else - current = current->FindChild( Path2Str(*i) ) ; - } - - return current ; + MD5Map& map = m_set.get() ; + if ( !md5.empty() ) + return map.equal_range( md5 ); + return MD5Range( map.end(), map.end() ) ; +} + +SizeRange ResourceTree::FindBySize( u64_t size ) +{ + SizeMap& map = m_set.get() ; + return map.equal_range( size ); } -/// Reinsert should be called when the ID/HREF were updated +/// Reinsert should be called when the ID/HREF/MD5 were updated bool ResourceTree::ReInsert( Resource *coll ) { Set& s = m_set.get() ; @@ -160,11 +137,11 @@ void ResourceTree::Erase( Resource *coll ) s.erase( s.find( coll ) ) ; } -void ResourceTree::Update( Resource *coll, const Entry& e, const DateTime& last_sync ) +void ResourceTree::Update( Resource *coll, const Entry& e ) { assert( coll != 0 ) ; - coll->FromRemote( e, last_sync ) ; + coll->FromRemote( e ) ; ReInsert( coll ) ; } @@ -178,4 +155,4 @@ ResourceTree::iterator ResourceTree::end() return m_set.get().end() ; } -} // end of namespace +} // end of namespace gr diff --git a/libgrive/src/drive/ResourceTree.hh b/libgrive/src/base/ResourceTree.hh similarity index 76% rename from libgrive/src/drive/ResourceTree.hh rename to libgrive/src/base/ResourceTree.hh index b7a2344e..b28c4a3b 100644 --- a/libgrive/src/drive/ResourceTree.hh +++ b/libgrive/src/base/ResourceTree.hh @@ -30,27 +30,30 @@ namespace gr { -class Json ; - namespace details { using namespace boost::multi_index ; - struct ByID {} ; + struct ByMD5 {} ; struct ByHref {} ; struct ByIdentity {} ; + struct BySize {} ; typedef multi_index_container< Resource*, indexed_by< hashed_non_unique, const_mem_fun >, - hashed_non_unique, const_mem_fun >, + hashed_non_unique, const_mem_fun >, + hashed_non_unique, const_mem_fun >, hashed_unique, identity > > > Folders ; - typedef Folders::index::type IDMap ; + typedef Folders::index::type MD5Map ; typedef Folders::index::type HrefMap ; + typedef Folders::index::type SizeMap ; typedef Folders::index::type Set ; + typedef std::pair SizeRange ; + typedef std::pair MD5Range ; } /*! \brief A simple container for storing folders @@ -64,24 +67,20 @@ public : typedef details::Set::iterator iterator ; public : - ResourceTree( ) ; + ResourceTree( const fs::path& rootFolder ) ; ResourceTree( const ResourceTree& fs ) ; ~ResourceTree( ) ; - void Swap( ResourceTree& fs ) ; - ResourceTree& operator=( const ResourceTree& fs ) ; - Resource* FindByHref( const std::string& href ) ; const Resource* FindByHref( const std::string& href ) const ; + details::MD5Range FindByMD5( const std::string& md5 ) ; + details::SizeRange FindBySize( u64_t size ) ; - Resource* FindByPath( const fs::path& path ) ; - Resource* FindByID( const std::string& id ) ; - bool ReInsert( Resource *coll ) ; void Insert( Resource *coll ) ; void Erase( Resource *coll ) ; - void Update( Resource *coll, const Entry& e, const DateTime& last_sync ) ; + void Update( Resource *coll, const Entry& e ) ; Resource* Root() ; const Resource* Root() const ; @@ -97,4 +96,4 @@ private : Resource* m_root ; } ; -} // end of namespace +} // end of namespace gr diff --git a/libgrive/src/base/State.cc b/libgrive/src/base/State.cc new file mode 100644 index 00000000..9fddf4d1 --- /dev/null +++ b/libgrive/src/base/State.cc @@ -0,0 +1,322 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "State.hh" + +#include "Entry.hh" +#include "Resource.hh" +#include "Syncer.hh" + +#include "util/Crypt.hh" +#include "util/File.hh" +#include "util/log/Log.hh" +#include "json/JsonParser.hh" + +#include + +namespace gr { + +State::State( const fs::path& filename, const Val& options ) : + m_res ( options["path"].Str() ), + m_cstamp ( -1 ) +{ + Read( filename ) ; + + // the "-f" option will make grive always think remote is newer + m_force = options.Has( "force" ) ? options["force"].Bool() : false ; + + std::string m_orig_ign = m_ign; + m_ign = ""; + if ( options.Has( "ignore" ) && options["ignore"].Str() != m_ign ) + m_ign = options["ignore"].Str(); + else if ( options.Has( "dir" ) ) + { + const boost::regex trim_path( "^/+|/+$" ); + std::string m_dir = regex_replace( options["dir"].Str(), trim_path, "" ); + if ( !m_dir.empty() ) + { + // "-s" is internally converted to an ignore regexp + const boost::regex esc( "[.^$|()\\[\\]{}*+?\\\\]" ); + m_dir = regex_replace( m_dir, esc, "\\\\&", boost::format_sed ); + size_t pos = 0; + while ( ( pos = m_dir.find( '/', pos ) ) != std::string::npos ) + { + m_dir = m_dir.substr( 0, pos ) + "$|" + m_dir; + pos = pos*2 + 3; + } + std::string ign = "^(?!"+m_dir+"(/|$))"; + m_ign = ign; + } + } + + m_ign_changed = m_orig_ign != "" && m_orig_ign != m_ign; + m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive|grive_state|trash)" : ( m_ign+"|^\\.(grive|grive_state|trash)" ) ); +} + +State::~State() +{ +} + +/// Synchronize local directory. Build up the resource tree from files and folders +/// of local directory. +void State::FromLocal( const fs::path& p ) +{ + m_res.Root()->FromLocal( m_st ) ; + FromLocal( p, m_res.Root(), m_st.Item( "tree" ) ) ; +} + +bool State::IsIgnore( const std::string& filename ) +{ + return regex_search( filename.c_str(), m_ign_re ); +} + +void State::FromLocal( const fs::path& p, Resource* folder, Val& tree ) +{ + assert( folder != 0 ) ; + assert( folder->IsFolder() ) ; + + Val::Object leftover = tree.AsObject(); + + for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i ) + { + std::string fname = i->path().filename().string() ; + std::string path = folder->IsRoot() ? fname : ( folder->RelPath() / fname ).string(); + + if ( IsIgnore( path ) ) + Log( "file %1% is ignored by grive", path, log::verbose ) ; + else + { + // if the Resource object of the child already exists, it should + // have been so no need to do anything here + Resource *c = folder->FindChild( fname ), *c2 = c ; + if ( !c ) + { + c2 = new Resource( fname, "" ) ; + folder->AddChild( c2 ) ; + } + leftover.erase( fname ); + Val& rec = tree.Item( fname ); + if ( m_force ) + rec.Del( "srv_time" ); + c2->FromLocal( rec ) ; + if ( !c ) + m_res.Insert( c2 ) ; + if ( c2->IsFolder() ) + FromLocal( *i, c2, rec.Item( "tree" ) ) ; + } + } + + for( Val::Object::iterator i = leftover.begin(); i != leftover.end(); i++ ) + { + std::string path = folder->IsRoot() ? i->first : ( folder->RelPath() / i->first ).string(); + if ( IsIgnore( path ) ) + Log( "file %1% is ignored by grive", path, log::verbose ) ; + else + { + // Restore state of locally deleted files + Resource *c = folder->FindChild( i->first ), *c2 ; + if ( !c ) + { + c2 = new Resource( i->first, i->second.Has( "tree" ) ? "folder" : "file" ) ; + folder->AddChild( c2 ) ; + } + Val& rec = tree.Item( i->first ); + if ( m_force || m_ign_changed ) + rec.Del( "srv_time" ); + c2->FromDeleted( rec ); + if ( !c ) + m_res.Insert( c2 ) ; + } + } +} + +void State::FromRemote( const Entry& e ) +{ + std::string fn = e.Filename() ; + std::string k = e.IsDir() ? "folder" : "file"; + + // common checkings + if ( !e.IsDir() && ( fn.empty() || e.ContentSrc().empty() ) ) + Log( "%1% \"%2%\" is a google document, ignored", k, e.Name(), log::verbose ) ; + + else if ( fn.find('/') != fn.npos ) + Log( "%1% \"%2%\" contains a slash in its name, ignored", k, e.Name(), log::verbose ) ; + + else if ( !e.IsChange() && e.ParentHrefs().size() != 1 ) + Log( "%1% \"%2%\" has multiple parents, ignored", k, e.Name(), log::verbose ) ; + + else if ( e.IsChange() ) + FromChange( e ) ; + + else if ( !Update( e ) ) + m_unresolved.push_back( e ) ; +} + +void State::ResolveEntry() +{ + while ( !m_unresolved.empty() ) + { + if ( TryResolveEntry() == 0 ) + break ; + } +} + +std::size_t State::TryResolveEntry() +{ + assert( !m_unresolved.empty() ) ; + + std::size_t count = 0 ; + std::list& en = m_unresolved ; + + for ( std::list::iterator i = en.begin() ; i != en.end() ; ) + { + if ( Update( *i ) ) + { + i = en.erase( i ) ; + count++ ; + } + else + ++i ; + } + return count ; +} + +void State::FromChange( const Entry& e ) +{ + assert( e.IsChange() ) ; + + // entries in the change feed is always treated as newer in remote, + // so we override the last sync time to 0 + if ( Resource *res = m_res.FindByHref( e.SelfHref() ) ) + m_res.Update( res, e ) ; +} + +bool State::Update( const Entry& e ) +{ + assert( !e.IsChange() ) ; + assert( !e.ParentHref().empty() ) ; + + if ( Resource *res = m_res.FindByHref( e.SelfHref() ) ) + { + std::string path = res->RelPath().string(); + if ( IsIgnore( path ) ) + { + Log( "%1% is ignored by grive", path, log::verbose ) ; + return true; + } + m_res.Update( res, e ) ; + return true; + } + else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) ) + { + assert( parent->IsFolder() ) ; + + std::string path = parent->IsRoot() ? e.Name() : ( parent->RelPath() / e.Name() ).string(); + if ( IsIgnore( path ) ) + { + Log( "%1% is ignored by grive", path, log::verbose ) ; + return true; + } + + // see if the entry already exist in local + std::string name = e.Name() ; + Resource *child = parent->FindChild( name ) ; + if ( child ) + { + // since we are updating the ID and Href, we need to remove it and re-add it. + m_res.Update( child, e ) ; + } + + // folder entry exist in google drive, but not local. we should create + // the directory + else if ( e.IsDir() || !e.Filename().empty() ) + { + // first create a dummy resource and update it later + child = new Resource( name, e.IsDir() ? "folder" : "file" ) ; + parent->AddChild( child ) ; + m_res.Insert( child ) ; + + // update the state of the resource + m_res.Update( child, e ) ; + } + + return true ; + } + else + return false ; +} + +Resource* State::FindByHref( const std::string& href ) +{ + return m_res.FindByHref( href ) ; +} + +State::iterator State::begin() +{ + return m_res.begin() ; +} + +State::iterator State::end() +{ + return m_res.end() ; +} + +void State::Read( const fs::path& filename ) +{ + try + { + File file( filename ) ; + + m_st = ParseJson( file ); + m_ign = m_st.Has( "ignore_regexp" ) ? m_st["ignore_regexp"].Str() : std::string(); + + m_cstamp = m_st["change_stamp"].Int() ; + } + catch ( Exception& ) + { + } +} + +void State::Write( const fs::path& filename ) +{ + m_st.Set( "change_stamp", Val( m_cstamp ) ) ; + m_st.Set( "ignore_regexp", Val( m_ign ) ) ; + + std::ofstream fs( filename.string().c_str() ) ; + fs << m_st ; +} + +void State::Sync( Syncer *syncer, const Val& options ) +{ + // set the last sync time to the time on the client + m_res.Root()->Sync( syncer, &m_res, options ) ; +} + +long State::ChangeStamp() const +{ + return m_cstamp ; +} + +void State::ChangeStamp( long cstamp ) +{ + Log( "change stamp is set to %1%", cstamp, log::verbose ) ; + m_cstamp = cstamp ; +} + +} // end of namespace gr diff --git a/libgrive/src/drive/State.hh b/libgrive/src/base/State.hh similarity index 74% rename from libgrive/src/drive/State.hh rename to libgrive/src/base/State.hh index 563abe56..68b6df88 100644 --- a/libgrive/src/drive/State.hh +++ b/libgrive/src/base/State.hh @@ -23,20 +23,18 @@ #include "util/DateTime.hh" #include "util/FileSystem.hh" +#include "json/Val.hh" #include +#include namespace gr { -namespace http -{ - class Agent ; - class Header ; -} +class Entry ; + +class Syncer ; -class Json ; class Resource ; -class Entry ; class State { @@ -44,7 +42,7 @@ public : typedef ResourceTree::iterator iterator ; public : - explicit State( const fs::path& filename, const Json& options ) ; + explicit State( const fs::path& filename, const Val& options ) ; ~State() ; void FromLocal( const fs::path& p ) ; @@ -52,34 +50,37 @@ public : void ResolveEntry() ; void Read( const fs::path& filename ) ; - void Write( const fs::path& filename ) const ; + void Write( const fs::path& filename ) ; Resource* FindByHref( const std::string& href ) ; Resource* FindByID( const std::string& id ) ; - Resource* Find( const fs::path& path ) ; - void Sync( http::Agent *http, const http::Header& auth ) ; + void Sync( Syncer *syncer, const Val& options ) ; iterator begin() ; iterator end() ; long ChangeStamp() const ; void ChangeStamp( long cstamp ) ; - + private : - void FromLocal( const fs::path& p, Resource *folder ) ; + void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ; void FromChange( const Entry& e ) ; bool Update( const Entry& e ) ; std::size_t TryResolveEntry() ; - static bool IsIgnore( const std::string& filename ) ; + bool IsIgnore( const std::string& filename ) ; private : ResourceTree m_res ; - DateTime m_last_sync ; - long m_cstamp ; + int m_cstamp ; + std::string m_ign ; + boost::regex m_ign_re ; + Val m_st ; + bool m_force ; + bool m_ign_changed ; - std::vector m_unresolved ; + std::list m_unresolved ; } ; -} // end of namespace +} // end of namespace gr diff --git a/libgrive/src/base/Syncer.cc b/libgrive/src/base/Syncer.cc new file mode 100644 index 00000000..f68d3090 --- /dev/null +++ b/libgrive/src/base/Syncer.cc @@ -0,0 +1,59 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Syncer.hh" +#include "Resource.hh" +#include "Entry.hh" +#include "http/Agent.hh" +#include "http/Header.hh" +#include "http/Download.hh" +#include "util/OS.hh" +#include "util/log/Log.hh" + +namespace gr { + +Syncer::Syncer( http::Agent *http ): + m_http( http ) +{ +} + +http::Agent* Syncer::Agent() const +{ + return m_http; +} + +void Syncer::Download( Resource *res, const fs::path& file ) +{ + http::Download dl( file.string(), http::Download::NoChecksum() ) ; + long r = m_http->Get( res->ContentSrc(), &dl, http::Header(), res->Size() ) ; + if ( r <= 400 ) + { + if ( res->ServerTime() != DateTime() ) + os::SetFileTime( file, res->ServerTime() ) ; + else + Log( "encountered zero date time after downloading %1%", file, log::warning ) ; + } +} + +void Syncer::AssignIDs( Resource *res, const Entry& remote ) +{ + res->AssignIDs( remote ); +} + +} // end of namespace gr diff --git a/libgrive/src/base/Syncer.hh b/libgrive/src/base/Syncer.hh new file mode 100644 index 00000000..237b8949 --- /dev/null +++ b/libgrive/src/base/Syncer.hh @@ -0,0 +1,71 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "util/FileSystem.hh" + +#include +#include +#include + +namespace gr { + +namespace http +{ + class Agent ; +} + +class DateTime ; + +class Resource ; + +class Entry ; + +class Feed ; + +/*! \brief A Syncer incapsulates all resource-related upload/download/edit methods */ +class Syncer +{ +public : + + Syncer( http::Agent *http ); + + http::Agent* Agent() const; + + virtual void DeleteRemote( Resource *res ) = 0; + virtual void Download( Resource *res, const fs::path& file ); + virtual bool EditContent( Resource *res, bool new_rev ) = 0; + virtual bool Create( Resource *res ) = 0; + virtual bool Move( Resource* res, Resource* newParent, std::string newFilename ) = 0; + + virtual std::unique_ptr GetFolders() = 0; + virtual std::unique_ptr GetAll() = 0; + virtual std::unique_ptr GetChanges( long min_cstamp ) = 0; + virtual long GetChangeStamp( long min_cstamp ) = 0; + +protected: + + http::Agent *m_http; + + void AssignIDs( Resource *res, const Entry& remote ); + +} ; + +} // end of namespace gr diff --git a/libgrive/src/bfd/SymbolInfo.cc b/libgrive/src/bfd/SymbolInfo.cc index afd3ba14..5876cc00 100644 --- a/libgrive/src/bfd/SymbolInfo.cc +++ b/libgrive/src/bfd/SymbolInfo.cc @@ -22,6 +22,7 @@ #include +#define PACKAGE "libgrive" #include #include #include diff --git a/libgrive/src/bfd/SymbolInfo.hh b/libgrive/src/bfd/SymbolInfo.hh index bcbc3b14..9284612d 100644 --- a/libgrive/src/bfd/SymbolInfo.hh +++ b/libgrive/src/bfd/SymbolInfo.hh @@ -54,7 +54,7 @@ public : private : struct Impl ; - const std::auto_ptr m_impl ; + const std::unique_ptr m_impl ; struct BacktraceInfo ; friend struct BacktraceInfo ; diff --git a/libgrive/src/drive/CommonUri.hh b/libgrive/src/drive/CommonUri.hh deleted file mode 100644 index 7be31b28..00000000 --- a/libgrive/src/drive/CommonUri.hh +++ /dev/null @@ -1,36 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#pragma once - -#include - -namespace gr -{ - const std::string feed_base = "https://docs.google.com/feeds/default/private/full" ; - const std::string feed_changes = "https://docs.google.com/feeds/default/private/changes" ; - const std::string feed_metadata = "https://docs.google.com/feeds/metadata/default" ; - - const std::string root_href = - "https://docs.google.com/feeds/default/private/full/folder%3Aroot" ; - const std::string root_create = - "https://docs.google.com/feeds/upload/create-session/default/private/full" ; - - std::string ChangesFeed( int changestamp ) ; -} diff --git a/libgrive/src/drive/Drive.cc b/libgrive/src/drive/Drive.cc deleted file mode 100644 index 258ce94d..00000000 --- a/libgrive/src/drive/Drive.cc +++ /dev/null @@ -1,196 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "Drive.hh" - -#include "CommonUri.hh" -#include "Entry.hh" -#include "Feed.hh" - -#include "http/CurlAgent.hh" -#include "http/ResponseLog.hh" -#include "http/XmlResponse.hh" -#include "protocol/Json.hh" -#include "protocol/OAuth2.hh" -#include "util/Destroy.hh" -#include "util/log/Log.hh" -#include "xml/Node.hh" -#include "xml/NodeSet.hh" - -#include - -// standard C++ library -#include -#include -#include -#include -#include -#include - -// for debugging only -#include - -namespace gr { - -namespace -{ - const std::string state_file = ".grive_state" ; -} - -Drive::Drive( OAuth2& auth, const Json& options ) : - m_auth( auth ), - m_state( state_file, options ) -{ - m_http_hdr.Add( "Authorization: Bearer " + m_auth.AccessToken() ) ; - m_http_hdr.Add( "GData-Version: 3.0" ) ; -} - -void Drive::FromRemote( const Entry& entry ) -{ - // entries from change feed does not have the parent HREF, - // so these checkings are done in normal entries only - Resource *parent = m_state.FindByHref( entry.ParentHref() ) ; - - if ( parent != 0 && !parent->IsFolder() ) - Log( "warning: entry %1% has parent %2% which is not a folder, ignored", - entry.Title(), parent->Name(), log::verbose ) ; - - else if ( parent == 0 || !parent->IsInRootTree() ) - Log( "file \"%1%\" parent doesn't exist, ignored", entry.Title(), log::verbose ) ; - - else - m_state.FromRemote( entry ) ; -} - -void Drive::FromChange( const Entry& entry ) -{ - if ( entry.IsRemoved() ) - Log( "file \"%1%\" represents a deletion, ignored", entry.Title(), log::verbose ) ; - - // folders go directly - else - m_state.FromRemote( entry ) ; -} - -void Drive::SaveState() -{ - m_state.Write( state_file ) ; -} - -void Drive::SyncFolders( http::Agent *http ) -{ - assert( http != 0 ) ; - - Log( "Synchronizing folders", log::info ) ; - - http::XmlResponse xml ; - http->Get( feed_base + "/-/folder?max-results=50&showroot=true", &xml, m_http_hdr ) ; - - Feed feed( xml.Response() ) ; - do - { - // first, get all collections from the query result - for ( Feed::iterator i = feed.begin() ; i != feed.end() ; ++i ) - { - Entry e( *i ) ; - if ( e.Kind() == "folder" ) - { - if ( e.ParentHrefs().size() != 1 ) - Log( "folder \"%1%\" has multiple parents, ignored", e.Title(), log::verbose ) ; - - else if ( e.Title().find('/') != std::string::npos ) - Log( "folder \"%1%\" contains a slash in its name, ignored", e.Title(), log::verbose ) ; - - else - m_state.FromRemote( e ) ; - } - } - } while ( feed.GetNext( http, m_http_hdr ) ) ; - - m_state.ResolveEntry() ; -} - -void Drive::DetectChanges() -{ - Log( "Reading local directories", log::info ) ; - m_state.FromLocal( "." ) ; - - http::CurlAgent http ; - - long prev_stamp = m_state.ChangeStamp() ; - Trace( "previous change stamp is %1%", prev_stamp ) ; - - SyncFolders( &http ) ; - - Log( "Reading remote server file list", log::info ) ; - http::XmlResponse xrsp ; - http.Get( feed_base + "?showfolders=true&showroot=true", &xrsp, m_http_hdr ) ; - xml::Node resp = xrsp.Response() ; - - m_resume_link = resp["link"]. - Find( "@rel", "http://schemas.google.com/g/2005#resumable-create-media" )["@href"] ; - - Feed feed( resp ) ; - do - { - std::for_each( feed.begin(), feed.end(), boost::bind( &Drive::FromRemote, this, _1 ) ) ; - } while ( feed.GetNext( &http, m_http_hdr ) ) ; - - // pull the changes feed - if ( prev_stamp != -1 ) - { - http::ResponseLog log( "/tmp/changes-", ".xml", &xrsp ) ; - Log( "Detecting changes from last sync", log::info ) ; - http.Get( ChangesFeed(prev_stamp+1), &log, m_http_hdr ) ; - - Feed changes( xrsp.Response() ) ; - std::for_each( changes.begin(), changes.end(), boost::bind( &Drive::FromChange, this, _1 ) ) ; - } -} - -void Drive::Update() -{ - Log( "Synchronizing files", log::info ) ; - http::CurlAgent http ; - m_state.Sync( &http, m_http_hdr ) ; - - UpdateChangeStamp( &http ) ; -} - -void Drive::DryRun() -{ - Log( "Synchronizing files (dry-run)", log::info ) ; - m_state.Sync( 0, m_http_hdr ) ; -} - -void Drive::UpdateChangeStamp( http::Agent *http ) -{ - assert( http != 0 ) ; - - // get changed feed - http::XmlResponse xrsp ; - http->Get( ChangesFeed(m_state.ChangeStamp()+1), &xrsp, m_http_hdr ) ; - - // we should go through the changes to see if it was really Grive to made that change - // maybe by recording the updated timestamp and compare it? - m_state.ChangeStamp( - std::atoi(xrsp.Response()["docs:largestChangestamp"]["@value"].front().Value().c_str()) ) ; -} - -} // end of namespace diff --git a/libgrive/src/drive/Entry.cc b/libgrive/src/drive/Entry.cc deleted file mode 100644 index a04ca28b..00000000 --- a/libgrive/src/drive/Entry.cc +++ /dev/null @@ -1,198 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "Entry.hh" -#include "CommonUri.hh" - -#include "util/Crypt.hh" -#include "util/log/Log.hh" -#include "util/OS.hh" -#include "xml/Node.hh" -#include "xml/NodeSet.hh" - -#include -#include - -namespace gr { - -/// construct an entry for the root folder -Entry::Entry( ) : - m_title ( "." ), - m_kind ( "folder" ), - m_resource_id ( "folder:root" ), - m_self_href ( root_href ), - m_create_link ( root_create ), - m_change_stamp ( -1 ), - m_is_removed ( false ) -{ -} - -/// construct an entry for remote -Entry::Entry( const xml::Node& n ) : - m_change_stamp( -1 ), - m_is_removed( false ) -{ - Update( n ) ; -} - -void Entry::Update( const xml::Node& n ) -{ - m_title = n["title"] ; - m_etag = n["@gd:etag"] ; - m_filename = n["docs:suggestedFilename"] ; - m_content_src = n["content"]["@src"] ; - m_self_href = n["link"].Find( "@rel", "self" )["@href"] ; - m_alt_self = n["link"].Find( "@rel", "http://schemas.google.com/docs/2007#alt-self" )["@href"] ; - m_mtime = DateTime( n["updated"] ) ; - - m_resource_id = n["gd:resourceId"] ; - m_md5 = n["docs:md5Checksum"] ; - m_kind = n["category"].Find( "@scheme", "http://schemas.google.com/g/2005#kind" )["@label"] ; - m_edit_link = n["link"].Find( "@rel", "http://schemas.google.com/g/2005#resumable-edit-media")["@href"] ; - m_create_link = n["link"].Find( "@rel", "http://schemas.google.com/g/2005#resumable-create-media")["@href"] ; - - // changestamp only appear in change feed entries - xml::NodeSet cs = n["docs:changestamp"]["@value"] ; - m_change_stamp = cs.empty() ? -1 : std::atoi( cs.front().Value().c_str() ) ; - - m_parent_hrefs.clear( ) ; - xml::NodeSet parents = n["link"].Find( "@rel", "http://schemas.google.com/docs/2007#parent" ) ; - for ( xml::NodeSet::iterator i = parents.begin() ; i != parents.end() ; ++i ) - m_parent_hrefs.push_back( (*i)["@href"] ) ; - - // convert to lower case for easy comparison - std::transform( m_md5.begin(), m_md5.end(), m_md5.begin(), tolower ) ; - - m_is_removed = !n["gd:deleted"].empty() || !n["docs:removed"].empty() ; -} - -const std::vector& Entry::ParentHrefs() const -{ - return m_parent_hrefs ; -} - -std::string Entry::Title() const -{ - return m_title ; -} - -std::string Entry::Filename() const -{ - return m_filename ; -} - -std::string Entry::Kind() const -{ - return m_kind ; -} - -std::string Entry::MD5() const -{ - return m_md5 ; -} - -DateTime Entry::MTime() const -{ - return m_mtime ; -} - -std::string Entry::SelfHref() const -{ - return m_self_href ; -} - -std::string Entry::AltSelf() const -{ - return m_alt_self ; -} - -std::string Entry::ParentHref() const -{ - return m_parent_hrefs.empty() ? "" : m_parent_hrefs.front() ; -} - -std::string Entry::ResourceID() const -{ - return m_resource_id ; -} - -std::string Entry::ETag() const -{ - return m_etag ; -} - -std::string Entry::ContentSrc() const -{ - return m_content_src ; -} - -std::string Entry::EditLink() const -{ - return m_edit_link ; -} - -std::string Entry::CreateLink() const -{ - return m_create_link ; -} - -void Entry::Swap( Entry& e ) -{ - m_title.swap( e.m_title ) ; - m_filename.swap( e.m_filename ) ; - m_kind.swap( e.m_kind ) ; - m_md5.swap( e.m_md5 ) ; - m_etag.swap( e.m_etag ) ; - m_resource_id.swap( e.m_resource_id ) ; - - m_parent_hrefs.swap( e.m_parent_hrefs ) ; - - m_self_href.swap( e.m_self_href ) ; - m_alt_self.swap( e.m_alt_self ) ; - m_content_src.swap( e.m_content_src ) ; - m_edit_link.swap( e.m_edit_link ) ; - m_create_link.swap( e.m_create_link ) ; - - m_mtime.Swap( e.m_mtime ) ; - - std::swap( m_change_stamp, e.m_change_stamp ) ; - std::swap( m_is_removed, e.m_is_removed ) ; -} - -long Entry::ChangeStamp() const -{ - return m_change_stamp ; -} - -bool Entry::IsChange() const -{ - return m_change_stamp != -1 ; -} - -bool Entry::IsRemoved() const -{ - return m_is_removed ; -} - -std::string Entry::Name() const -{ - return m_kind == "file" ? m_filename : m_title ; -} - -} // end of namespace diff --git a/libgrive/src/drive/Feed.cc b/libgrive/src/drive/Feed.cc deleted file mode 100644 index bbb1b7f6..00000000 --- a/libgrive/src/drive/Feed.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "Feed.hh" - -#include "http/Agent.hh" -#include "http/XmlResponse.hh" -#include "xml/NodeSet.hh" - -#include - -namespace gr { - -Feed::Feed( const xml::Node& root ) : - m_root ( root ), - m_entries ( m_root["entry"] ) -{ -} - -void Feed::Assign( const xml::Node& root ) -{ - m_root = root ; - m_entries = m_root["entry"] ; -} - -Feed::iterator Feed::begin() const -{ - return iterator( m_entries.begin() ) ; -} - -Feed::iterator Feed::end() const -{ - return iterator( m_entries.end() ) ; -} - -std::string Feed::Next() const -{ - xml::NodeSet nss = m_root["link"].Find( "@rel", "next" ) ; - return nss.empty() ? "" : std::string(nss["@href"]) ; -} - -bool Feed::GetNext( http::Agent *http, const http::Header& auth ) -{ - assert( http != 0 ) ; - - xml::NodeSet nss = m_root["link"].Find( "@rel", "next" ) ; - if ( !nss.empty() ) - { - http::XmlResponse xrsp ; - http->Get( nss["@href"], &xrsp, auth ) ; - - m_root = xrsp.Response() ; - m_entries = m_root["entry"] ; - - return true ; - } - else - return false ; -} - -Feed::iterator::iterator( ) -{ -} - -Feed::iterator::iterator( xml::Node::iterator i ) -{ - // for some reason, gcc 4.4.4 doesn't allow me to initialize the base class - // in the initializer. I have no choice but to initialize here. - base_reference() = i ; -} - -Feed::iterator::reference Feed::iterator::dereference() const -{ - return Entry( *base_reference() ) ; -} - -} // end of namespace diff --git a/libgrive/src/drive/Feed.hh b/libgrive/src/drive/Feed.hh deleted file mode 100644 index 321498cc..00000000 --- a/libgrive/src/drive/Feed.hh +++ /dev/null @@ -1,77 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#pragma once - -#include "Entry.hh" - -#include "xml/Node.hh" -#include "xml/NodeSet.hh" - -#include - -#include - -namespace gr { - -namespace http -{ - class Agent ; - class Header ; -} - -class Feed -{ -public : - class iterator ; - -public : - explicit Feed( const xml::Node& root ) ; - void Assign( const xml::Node& root ) ; - - iterator begin() const ; - iterator end() const ; - - std::string Next() const ; - bool GetNext( http::Agent *http, const http::Header& auth ) ; - -private : - xml::Node m_root ; - xml::NodeSet m_entries ; -} ; - -class Feed::iterator : public boost::iterator_adaptor< - Feed::iterator, - xml::Node::iterator, - Entry, - boost::random_access_traversal_tag, - Entry -> -{ -public : - iterator() ; - explicit iterator( xml::Node::iterator i ) ; - -private : - friend class boost::iterator_core_access; - - reference dereference() const ; -} ; - -} // end of namespace diff --git a/libgrive/src/drive/Resource.cc b/libgrive/src/drive/Resource.cc deleted file mode 100644 index a4e85013..00000000 --- a/libgrive/src/drive/Resource.cc +++ /dev/null @@ -1,671 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "Resource.hh" -#include "CommonUri.hh" -#include "Entry.hh" - -#include "http/Agent.hh" -#include "http/Download.hh" -#include "http/Header.hh" -// #include "http/ResponseLog.hh" -#include "http/StringResponse.hh" -#include "http/XmlResponse.hh" -#include "protocol/Json.hh" -#include "util/CArray.hh" -#include "util/Crypt.hh" -#include "util/log/Log.hh" -#include "util/OS.hh" -#include "util/StdioFile.hh" -#include "xml/Node.hh" -#include "xml/NodeSet.hh" -#include "xml/String.hh" - -#include -#include - -#include - -// for debugging -#include - -namespace gr { - -// hard coded XML file -const std::string xml_meta = - "\n" - "" - "" - "%2%" - "" ; - - -/// default constructor creates the root folder -Resource::Resource() : - m_name ( "." ), - m_kind ( "folder" ), - m_id ( "folder:root" ), - m_href ( root_href ), - m_create ( root_create ), - m_parent ( 0 ), - m_state ( sync ) -{ -} - -Resource::Resource( const std::string& name, const std::string& kind ) : - m_name ( name ), - m_kind ( kind ), - m_parent ( 0 ), - m_state ( unknown ) -{ -} - -void Resource::SetState( State new_state ) -{ - // only the new and delete states need to be set recursively - assert( - new_state == remote_new || new_state == remote_deleted || - new_state == local_new || new_state == local_deleted - ) ; - - m_state = new_state ; - std::for_each( m_child.begin(), m_child.end(), - boost::bind( &Resource::SetState, _1, new_state ) ) ; -} - -void Resource::FromRemoteFolder( const Entry& remote, const DateTime& last_sync ) -{ - fs::path path = Path() ; - - if ( remote.CreateLink().empty() ) - Log( "folder %1% is read-only", path, log::verbose ) ; - - // already sync - if ( fs::is_directory( path ) ) - { - Log( "folder %1% is in sync", path, log::verbose ) ; - m_state = sync ; - } - - // remote file created after last sync, so remote is newer - else if ( remote.MTime() > last_sync ) - { - if ( fs::exists( path ) ) - { - // TODO: handle type change - Log( "%1% changed from folder to file", path, log::verbose ) ; - m_state = sync ; - } - else - { - // make all children as remote_new, if any - Log( "folder %1% is created in remote", path, log::verbose ) ; - SetState( remote_new ) ; - } - } - else - { - if ( fs::exists( path ) ) - { - // TODO: handle type chage - Log( "%1% changed from file to folder", path, log::verbose ) ; - m_state = sync ; - } - else - { - Log( "folder %1% is deleted in local", path, log::verbose ) ; - SetState( local_deleted ) ; - } - } -} - -/// Update the state according to information (i.e. Entry) from remote. This function -/// compares the modification time and checksum of both copies and determine which -/// one is newer. -void Resource::FromRemote( const Entry& remote, const DateTime& last_sync ) -{ - // sync folder - if ( remote.Kind() == "folder" && IsFolder() ) - FromRemoteFolder( remote, last_sync ) ; - else - FromRemoteFile( remote, last_sync ) ; - - AssignIDs( remote ) ; - - assert( m_state != unknown ) ; - - if ( m_state == remote_new || m_state == remote_changed ) - { - m_md5 = remote.MD5() ; - m_mtime = remote.MTime() ; - } -} - -void Resource::AssignIDs( const Entry& remote ) -{ - // the IDs from change feed entries are different - if ( !remote.IsChange() ) - { - m_id = remote.ResourceID() ; - m_href = remote.SelfHref() ; - m_edit = remote.EditLink() ; - m_create = remote.CreateLink() ; - m_content = remote.ContentSrc() ; - m_etag = remote.ETag() ; - } -} - -void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_sync ) -{ - assert( m_parent != 0 ) ; - - fs::path path = Path() ; - - // recursively create/delete folder - if ( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted || - m_parent->m_state == local_new || m_parent->m_state == local_deleted ) - { - Log( "file %1% parent %2% recursively in %3% (%4%)", path, - ( m_parent->m_state == remote_new || m_parent->m_state == local_new ) ? "created" : "deleted", - ( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted ) ? "remote" : "local", - m_parent->m_state, log::verbose ) ; - - m_state = m_parent->m_state ; - } - - // local not exists - else if ( !fs::exists( path ) ) - { - Trace( "file %1% change stamp = %2%", Path(), remote.ChangeStamp() ) ; - - if ( remote.MTime() > last_sync || remote.ChangeStamp() > 0 ) - { - Log( "file %1% is created in remote (change %2%)", path, - remote.ChangeStamp(), log::verbose ) ; - - m_state = remote_new ; - } - else - { - Log( "file %1% is deleted in local", path, log::verbose ) ; - m_state = local_deleted ; - } - } - - // if checksum is equal, no need to compare the mtime - else if ( remote.MD5() == m_md5 ) - { - Log( "file %1% is already in sync", Path(), log::verbose ) ; - m_state = sync ; - } - - // use mtime to check which one is more recent - else - { - assert( m_state != unknown ) ; - - // if remote is modified - if ( remote.MTime() > m_mtime ) - { - Log( "file %1% is changed in remote", path, log::verbose ) ; - m_state = remote_changed ; - } - - // remote also has the file, so it's not new in local - else if ( m_state == local_new || m_state == remote_deleted ) - { - Log( "file %1% is changed in local", path, log::verbose ) ; - m_state = local_changed ; - } - else - Trace( "file %1% state is %2%", m_name, m_state ) ; - } -} - -/// Update the resource with the attributes of local file or directory. This -/// function will propulate the fields in m_entry. -void Resource::FromLocal( const DateTime& last_sync ) -{ - fs::path path = Path() ; - assert( fs::exists( path ) ) ; - - // root folder is always in sync - if ( !IsRoot() ) - { - m_mtime = os::FileCTime( path ) ; - - // follow parent recursively - if ( m_parent->m_state == local_new || m_parent->m_state == local_deleted ) - m_state = local_new ; - - // if the file is not created after last sync, assume file is - // remote_deleted first, it will be updated to sync/remote_changed - // in FromRemote() - else - m_state = ( m_mtime > last_sync ? local_new : remote_deleted ) ; - - m_name = Path2Str( path.filename() ) ; - m_kind = fs::is_directory(path) ? "folder" : "file" ; - m_md5 = fs::is_directory(path) ? "" : crypt::MD5::Get( path ) ; - } - - assert( m_state != unknown ) ; -} - -std::string Resource::SelfHref() const -{ - return m_href ; -} - -std::string Resource::Name() const -{ - return m_name ; -} - -std::string Resource::ResourceID() const -{ - return m_id ; -} - -const Resource* Resource::Parent() const -{ - assert( m_parent == 0 || m_parent->IsFolder() ) ; - return m_parent ; -} - -Resource* Resource::Parent() -{ - assert( m_parent == 0 || m_parent->IsFolder() ) ; - return m_parent ; -} - -void Resource::AddChild( Resource *child ) -{ - assert( child != 0 ) ; - assert( child->m_parent == 0 || child->m_parent == this ) ; - assert( child != this ) ; - - child->m_parent = this ; - m_child.push_back( child ) ; -} - -void Resource::Swap( Resource& coll ) -{ - m_name.swap( coll.m_name ) ; - m_kind.swap( coll.m_kind ) ; - m_md5.swap( coll.m_md5 ) ; - m_etag.swap( coll.m_etag ) ; - m_id.swap( coll.m_id ) ; - - m_href.swap( coll.m_href ) ; - m_content.swap( coll.m_content ) ; - m_edit.swap( coll.m_edit ) ; - m_create.swap( coll.m_create ) ; - - m_mtime.Swap( coll.m_mtime ) ; - - std::swap( m_parent, coll.m_parent ) ; - m_child.swap( coll.m_child ) ; - std::swap( m_state, coll.m_state ) ; -} - -bool Resource::IsFolder() const -{ - return m_kind == "folder" ; -} - -fs::path Resource::Path() const -{ - assert( m_parent != this ) ; - assert( m_parent == 0 || m_parent->IsFolder() ) ; - - return m_parent != 0 ? (m_parent->Path() / m_name) : m_name ; -} - -bool Resource::IsInRootTree() const -{ - assert( m_parent == 0 || m_parent->IsFolder() ) ; - return m_parent == 0 ? (SelfHref() == root_href) : m_parent->IsInRootTree() ; -} - -Resource* Resource::FindChild( const std::string& name ) -{ - for ( std::vector::iterator i = m_child.begin() ; i != m_child.end() ; ++i ) - { - assert( (*i)->m_parent == this ) ; - if ( (*i)->m_name == name ) - return *i ; - } - return 0 ; -} - -// try to change the state to "sync" -void Resource::Sync( http::Agent *http, const http::Header& auth ) -{ - assert( m_state != unknown ) ; - assert( !IsRoot() || m_state == sync ) ; // root folder is already synced - - SyncSelf( http, auth ) ; - - // if myself is deleted, no need to do the childrens - if ( m_state != local_deleted && m_state != remote_deleted ) - std::for_each( m_child.begin(), m_child.end(), - boost::bind( &Resource::Sync, _1, http, auth ) ) ; -} - -void Resource::SyncSelf( http::Agent* http, const http::Header& auth ) -{ - assert( !IsRoot() || m_state == sync ) ; // root is always sync - assert( IsRoot() || http == 0 || fs::is_directory( m_parent->Path() ) ) ; - assert( IsRoot() || m_parent->m_state != remote_deleted ) ; - assert( IsRoot() || m_parent->m_state != local_deleted ) ; - - const fs::path path = Path() ; - - switch ( m_state ) - { - case local_new : - Log( "sync %1% doesn't exist in server, uploading", path, log::info ) ; - - if ( http != 0 && Create( http, auth ) ) - m_state = sync ; - break ; - - case local_deleted : - Log( "sync %1% deleted in local. deleting remote", path, log::info ) ; - if ( http != 0 ) - DeleteRemote( http, auth ) ; - break ; - - case local_changed : - Log( "sync %1% changed in local. uploading", path, log::info ) ; - if ( http != 0 && EditContent( http, auth ) ) - m_state = sync ; - break ; - - case remote_new : - Log( "sync %1% created in remote. creating local", path, log::info ) ; - if ( http != 0 ) - { - if ( IsFolder() ) - fs::create_directories( path ) ; - else - Download( http, path, auth ) ; - - m_state = sync ; - } - break ; - - case remote_changed : - assert( !IsFolder() ) ; - Log( "sync %1% changed in remote. downloading", path, log::info ) ; - if ( http != 0 ) - { - Download( http, path, auth ) ; - m_state = sync ; - } - break ; - - case remote_deleted : - Log( "sync %1% deleted in remote. deleting local", path, log::info ) ; - if ( http != 0 ) - DeleteLocal() ; - break ; - - case sync : - Log( "sync %1% already in sync", path, log::verbose ) ; - break ; - - // shouldn't go here - case unknown : - assert( false ) ; - break ; - - default : - break ; - } -} - -/// this function doesn't really remove the local file. it renames it. -void Resource::DeleteLocal() -{ - static const boost::format trash_file( "%1%-%2%" ) ; - - assert( m_parent != 0 ) ; - fs::path parent = m_parent->Path() ; - fs::path dest = ".trash" / parent / Name() ; - - std::size_t idx = 1 ; - while ( fs::exists( dest ) && idx != 0 ) - dest = ".trash" / parent / (boost::format(trash_file) % Name() % idx++).str() ; - - // wrap around! just remove the file - if ( idx == 0 ) - fs::remove_all( Path() ) ; - else - { - fs::create_directories( dest.parent_path() ) ; - fs::rename( Path(), dest ) ; - } -} - -void Resource::DeleteRemote( http::Agent *http, const http::Header& auth ) -{ - assert( http != 0 ) ; - http::StringResponse str ; - - try - { - http::Header hdr( auth ) ; - hdr.Add( "If-Match: " + m_etag ) ; - - // doesn't know why, but an update before deleting seems to work always - http::XmlResponse xml ; - http->Get( m_href, &xml, hdr ) ; - AssignIDs( Entry( xml.Response() ) ) ; - - http->Custom( "DELETE", m_href, &str, hdr ) ; - } - catch ( Exception& e ) - { - // don't rethrow here. there are some cases that I don't know why - // the delete will fail. - Trace( "Exception %1% %2%", - boost::diagnostic_information(e), - str.Response() ) ; - } -} - - -void Resource::Download( http::Agent* http, const fs::path& file, const http::Header& auth ) const -{ - assert( http != 0 ) ; - - http::Download dl( file.string(), http::Download::NoChecksum() ) ; - long r = http->Get( m_content, &dl, auth ) ; - if ( r <= 400 ) - { - if ( m_mtime != DateTime() ) - os::SetFileTime( file, m_mtime ) ; - else - Log( "encountered zero date time after downloading %1%", file, log::warning ) ; - } -} - -bool Resource::EditContent( http::Agent* http, const http::Header& auth ) -{ - assert( http != 0 ) ; - assert( m_parent != 0 ) ; - assert( m_parent->m_state == sync ) ; - - // upload link missing means that file is read only - if ( m_edit.empty() ) - { - Log( "Cannot upload %1%: file read-only. %2%", m_name, m_state, log::warning ) ; - return false ; - } - - return Upload( http, m_edit, auth, false ) ; -} - -bool Resource::Create( http::Agent* http, const http::Header& auth ) -{ - assert( http != 0 ) ; - assert( m_parent != 0 ) ; - assert( m_parent->IsFolder() ) ; - assert( m_parent->m_state == sync ) ; - - if ( IsFolder() ) - { - std::string uri = feed_base ; - if ( !m_parent->IsRoot() ) - uri += ("/" + http->Escape(m_parent->m_id) + "/contents") ; - - std::string meta = (boost::format( xml_meta ) - % "folder" - % xml::Escape(m_name) - ).str() ; - - http::Header hdr( auth ) ; - hdr.Add( "Content-Type: application/atom+xml" ) ; - - http::XmlResponse xml ; -// http::ResponseLog log( "create", ".xml", &xml ) ; - http->Post( uri, meta, &xml, hdr ) ; - AssignIDs( Entry( xml.Response() ) ) ; - - return true ; - } - else if ( !m_parent->m_create.empty() ) - { - return Upload( http, m_parent->m_create + "?convert=false", auth, true ) ; - } - else - { - Log( "parent of %1% does not exist: cannot upload", Name(), log::warning ) ; - return false ; - } -} - -bool Resource::Upload( http::Agent* http, const std::string& link, const http::Header& auth, bool post ) -{ - assert( http != 0 ) ; - - StdioFile file( Path() ) ; - - // TODO: upload in chunks - std::string data ; - char buf[4096] ; - std::size_t count = 0 ; - while ( (count = file.Read( buf, sizeof(buf) )) > 0 ) - data.append( buf, count ) ; - - std::ostringstream xcontent_len ; - xcontent_len << "X-Upload-Content-Length: " << data.size() ; - - http::Header hdr( auth ) ; - hdr.Add( "Content-Type: application/atom+xml" ) ; - hdr.Add( "X-Upload-Content-Type: application/octet-stream" ) ; - hdr.Add( xcontent_len.str() ) ; - hdr.Add( "If-Match: " + m_etag ) ; - hdr.Add( "Expect:" ) ; - - std::string meta = (boost::format( xml_meta ) - % m_kind - % xml::Escape(m_name) - ).str() ; - - http::StringResponse str ; - if ( post ) - http->Post( link, meta, &str, hdr ) ; - else - http->Put( link, meta, &str, hdr ) ; - - http::Header uphdr ; - uphdr.Add( "Expect:" ) ; - uphdr.Add( "Accept:" ) ; - - // the content upload URL is in the "Location" HTTP header - std::string uplink = http->RedirLocation() ; - http::XmlResponse xml ; - - http->Put( uplink, data, &xml, uphdr ) ; - AssignIDs( Entry( xml.Response() ) ) ; - - return true ; -} - -Resource::iterator Resource::begin() const -{ - return m_child.begin() ; -} - -Resource::iterator Resource::end() const -{ - return m_child.end() ; -} - -std::size_t Resource::size() const -{ - return m_child.size() ; -} - -std::ostream& operator<<( std::ostream& os, Resource::State s ) -{ - static const char *state[] = - { - "sync", "local_new", "local_changed", "local_deleted", "remote_new", - "remote_changed", "remote_deleted" - } ; - assert( s >= 0 && s < Count(state) ) ; - return os << state[s] ; -} - -std::string Resource::StateStr() const -{ - std::ostringstream ss ; - ss << m_state ; - return ss.str() ; -} - -std::string Resource::MD5() const -{ - return m_md5 ; -} - -bool Resource::IsRoot() const -{ - return m_parent == 0 ; -} - -bool Resource::HasID() const -{ - return !m_href.empty() && !m_id.empty() ; -} - -} // end of namespace - -namespace std -{ - void swap( gr::Resource& c1, gr::Resource& c2 ) - { - c1.Swap( c2 ) ; - } -} diff --git a/libgrive/src/drive/State.cc b/libgrive/src/drive/State.cc deleted file mode 100644 index dddb0128..00000000 --- a/libgrive/src/drive/State.cc +++ /dev/null @@ -1,283 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "State.hh" - -#include "Entry.hh" -#include "Resource.hh" -#include "CommonUri.hh" - -#include "http/Agent.hh" -#include "http/Header.hh" -#include "util/Crypt.hh" -#include "util/log/Log.hh" -#include "protocol/Json.hh" - -#include - -namespace gr { - -State::State( const fs::path& filename, const Json& options ) : - m_cstamp( -1 ) -{ - Read( filename ) ; - - // the "-f" option will make grive always thinks remote is newer - Json force ; - if ( options.Get("force", force) && force.Bool() ) - m_last_sync = DateTime() ; - - Log( "last sync time: %1%", m_last_sync, log::verbose ) ; -} - -State::~State() -{ -} - -/// Synchronize local directory. Build up the resource tree from files and folders -/// of local directory. -void State::FromLocal( const fs::path& p ) -{ - FromLocal( p, m_res.Root() ) ; -} - -bool State::IsIgnore( const std::string& filename ) -{ - return filename[0] == '.' ; -} - -void State::FromLocal( const fs::path& p, gr::Resource* folder ) -{ - assert( folder != 0 ) ; - assert( folder->IsFolder() ) ; - - // sync the folder itself - folder->FromLocal( m_last_sync ) ; - - for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i ) - { - std::string fname = Path2Str(i->path().filename()) ; -// Trace( "found file %1%", i->path() ) ; - - if ( IsIgnore(fname) ) - Log( "file %1% is ignored by grive", fname, log::verbose ) ; - - // check for broken symblic links - else if ( !fs::exists( i->path() ) ) - Log( "file %1% doesn't exist (broken link?), ignored", i->path(), log::verbose ) ; - - else - { - // if the Resource object of the child already exists, it should - // have been so no need to do anything here - Resource *c = folder->FindChild( fname ) ; - if ( c == 0 ) - { - c = new Resource( fname, fs::is_directory(i->path()) ? "folder" : "file" ) ; - folder->AddChild( c ) ; - m_res.Insert( c ) ; - } - - c->FromLocal( m_last_sync ) ; - - if ( fs::is_directory( i->path() ) ) - FromLocal( *i, c ) ; - } - } -} - -void State::FromRemote( const Entry& e ) -{ - std::string fn = e.Filename() ; - - if ( IsIgnore( e.Name() ) ) - Log( "%1% %2% is ignored by grive", e.Kind(), e.Name(), log::verbose ) ; - - // common checkings - else if ( e.Kind() != "folder" && (fn.empty() || e.ContentSrc().empty()) ) - Log( "%1% \"%2%\" is a google document, ignored", e.Kind(), e.Name(), log::verbose ) ; - - else if ( fn.find('/') != fn.npos ) - Log( "%1% \"%2%\" contains a slash in its name, ignored", e.Kind(), e.Name(), log::verbose ) ; - - else if ( !e.IsChange() && e.ParentHrefs().size() != 1 ) - Log( "%1% \"%2%\" has multiple parents, ignored", e.Kind(), e.Name(), log::verbose ) ; - - else if ( e.IsChange() ) - FromChange( e ) ; - - else if ( !Update( e ) ) - { - m_unresolved.push_back( e ) ; - } -} - -void State::ResolveEntry() -{ - while ( !m_unresolved.empty() ) - { - if ( TryResolveEntry() == 0 ) - break ; - } -} - -std::size_t State::TryResolveEntry() -{ - assert( !m_unresolved.empty() ) ; - - std::size_t count = 0 ; - std::vector& en = m_unresolved ; - - for ( std::vector::iterator i = en.begin() ; i != en.end() ; ) - { - if ( Update( *i ) ) - { - i = en.erase( i ) ; - count++ ; - } - else - ++i ; - } - return count ; -} - -void State::FromChange( const Entry& e ) -{ - assert( e.IsChange() ) ; - assert( !IsIgnore( e.Name() ) ) ; - - // entries in the change feed is always treated as newer in remote, - // so we override the last sync time to 0 - if ( Resource *res = m_res.FindByHref( e.AltSelf() ) ) - m_res.Update( res, e, DateTime() ) ; -} - -bool State::Update( const Entry& e ) -{ - assert( !e.IsChange() ) ; - assert( !e.ParentHref().empty() ) ; - - if ( Resource *res = m_res.FindByHref( e.SelfHref() ) ) - { - m_res.Update( res, e, m_last_sync ) ; - return true ; - } - else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) ) - { - assert( parent->IsFolder() ) ; - - // see if the entry already exist in local - std::string name = e.Name() ; - Resource *child = parent->FindChild( name ) ; - if ( child != 0 ) - { - // since we are updating the ID and Href, we need to remove it and re-add it. - m_res.Update( child, e, m_last_sync ) ; - } - - // folder entry exist in google drive, but not local. we should create - // the directory - else if ( e.Kind() == "folder" || !e.Filename().empty() ) - { - // first create a dummy resource and update it later - child = new Resource( name, e.Kind() ) ; - parent->AddChild( child ) ; - m_res.Insert( child ) ; - - // update the state of the resource - m_res.Update( child, e, m_last_sync ) ; - } - - return true ; - } - else - return false ; -} - -Resource* State::FindByHref( const std::string& href ) -{ - return m_res.FindByHref( href ) ; -} - -Resource* State::Find( const fs::path& path ) -{ - return m_res.FindByPath( path ) ; -} - -State::iterator State::begin() -{ - return m_res.begin() ; -} - -State::iterator State::end() -{ - return m_res.end() ; -} - -void State::Read( const fs::path& filename ) -{ - try - { - Json json = Json::ParseFile( filename.string() ) ; - - Json last_sync = json["last_sync"] ; - m_last_sync.Assign( - last_sync["sec"].Int(), - last_sync["nsec"].Int() ) ; - - m_cstamp = json["change_stamp"].Int() ; - } - catch ( Exception& ) - { - m_last_sync.Assign(0) ; - } -} - -void State::Write( const fs::path& filename ) const -{ - Json last_sync ; - last_sync.Add( "sec", Json(m_last_sync.Sec() ) ); - last_sync.Add( "nsec", Json(m_last_sync.NanoSec() ) ); - - Json result ; - result.Add( "last_sync", last_sync ) ; - result.Add( "change_stamp", Json(m_cstamp) ) ; - - std::ofstream fs( filename.string().c_str() ) ; - fs << result ; -} - -void State::Sync( http::Agent *http, const http::Header& auth ) -{ - m_res.Root()->Sync( http, auth ) ; - m_last_sync = DateTime::Now() ; -} - -long State::ChangeStamp() const -{ - return m_cstamp ; -} - -void State::ChangeStamp( long cstamp ) -{ - Log( "change stamp is set to %1%", cstamp, log::verbose ) ; - m_cstamp = cstamp ; -} - -} // end of namespace diff --git a/libgrive/src/drive2/CommonUri.hh b/libgrive/src/drive2/CommonUri.hh new file mode 100644 index 00000000..9cef07b6 --- /dev/null +++ b/libgrive/src/drive2/CommonUri.hh @@ -0,0 +1,39 @@ +/* + Common URIs for REST API + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include + +namespace gr { namespace v2 { + +const std::string upload_base = "https://www.googleapis.com/upload/drive/v2/files" ; + +namespace feeds +{ + const std::string files = "https://www.googleapis.com/drive/v2/files" ; + const std::string changes = "https://www.googleapis.com/drive/v2/changes" ; +} + +namespace mime_types +{ + const std::string folder = "application/vnd.google-apps.folder" ; +} + +} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Entry2.cc b/libgrive/src/drive2/Entry2.cc new file mode 100644 index 00000000..b445c1b8 --- /dev/null +++ b/libgrive/src/drive2/Entry2.cc @@ -0,0 +1,96 @@ +/* + REST API item class implementation + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Entry2.hh" +#include "CommonUri.hh" + +#include "util/Crypt.hh" +#include "util/log/Log.hh" +#include "util/OS.hh" + +#include "json/Val.hh" + +#include +#include + +namespace gr { namespace v2 { + +/// construct an entry for remote, from "file" JSON object - Drive REST API +Entry2::Entry2( const Val& item ) +{ + Update( item ) ; +} + +void Entry2::Update( const Val& item ) +{ + bool is_chg = item["kind"].Str() == "drive#change"; + + // changestamp only appears in change feed entries + m_change_stamp = is_chg ? item["id"].Int() : -1 ; + m_is_removed = is_chg && item["deleted"].Bool() ; + m_size = 0 ; + + const Val& file = is_chg && !m_is_removed ? item["file"] : item; + + if ( m_is_removed ) + { + m_resource_id = item["fileId"]; + } + else + { + m_title = file["title"] ; + m_etag = file["etag"] ; + Val fn; + m_filename = file.Get( "title", fn ) ? fn.Str() : std::string(); + m_self_href = file["selfLink"] ; + m_mtime = DateTime( file["modifiedDate"] ) ; + + m_resource_id = file["id"]; + m_is_dir = file["mimeType"].Str() == mime_types::folder ; + m_is_editable = file["editable"].Bool() ; + m_is_removed = file["labels"]["trashed"].Bool() ; + if ( !m_is_dir ) + { + if ( !file.Has( "md5Checksum" ) || !file.Has("downloadUrl") ) + { + // This is either a google docs document or a not-yet-uploaded file. Ignore it. + // FIXME: We'll need to compare timestamps to support google docs. + m_is_removed = true; + } + else + { + m_md5 = file["md5Checksum"] ; + m_size = file["fileSize"].U64() ; + m_content_src = file["downloadUrl"] ; + // convert to lower case for easy comparison + std::transform( m_md5.begin(), m_md5.end(), m_md5.begin(), tolower ) ; + } + } + + m_parent_hrefs.clear( ) ; + + Val::Array parents = file["parents"].AsArray() ; + for ( Val::Array::iterator i = parents.begin() ; i != parents.end() ; ++i ) + { + m_parent_hrefs.push_back( (*i)["isRoot"].Bool() ? std::string( "root" ) : (*i)["parentLink"] ) ; + } + } +} + +} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Entry2.hh b/libgrive/src/drive2/Entry2.hh new file mode 100644 index 00000000..2c848d2c --- /dev/null +++ b/libgrive/src/drive2/Entry2.hh @@ -0,0 +1,38 @@ +/* + REST API item class implementation + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "base/Entry.hh" + +namespace gr { + +class Val ; + +namespace v2 { + +class Entry2: public Entry +{ +public : + explicit Entry2( const Val& item ) ; +private : + void Update( const Val& item ) ; +} ; + +} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Feed2.cc b/libgrive/src/drive2/Feed2.cc new file mode 100644 index 00000000..8f3d2344 --- /dev/null +++ b/libgrive/src/drive2/Feed2.cc @@ -0,0 +1,58 @@ +/* + REST API item list ("Feed") implementation + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "CommonUri.hh" +#include "Feed2.hh" +#include "Entry2.hh" + +#include "http/Agent.hh" +#include "http/Header.hh" +#include "json/Val.hh" +#include "json/ValResponse.hh" + +#include +#include + +namespace gr { namespace v2 { + +Feed2::Feed2( const std::string& url ): + Feed( url ) +{ +} + +bool Feed2::GetNext( http::Agent *http ) +{ + if ( m_next.empty() ) + return false ; + + http::ValResponse out ; + http->Get( m_next, &out, http::Header(), 0 ) ; + Val m_content = out.Response() ; + + Val::Array items = m_content["items"].AsArray() ; + m_entries.clear() ; + for ( Val::Array::iterator i = items.begin() ; i != items.end() ; ++i ) + m_entries.push_back( Entry2( *i ) ); + + Val url ; + m_next = m_content.Get( "nextLink", url ) ? url : std::string( "" ) ; + return true ; +} + +} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Feed2.hh b/libgrive/src/drive2/Feed2.hh new file mode 100644 index 00000000..4de411e9 --- /dev/null +++ b/libgrive/src/drive2/Feed2.hh @@ -0,0 +1,37 @@ +/* + REST API item list ("Feed") implementation + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "base/Feed.hh" + +#include "util/Exception.hh" + +#include + +namespace gr { namespace v2 { + +class Feed2: public Feed +{ +public : + Feed2( const std::string& url ) ; + bool GetNext( http::Agent *http ) ; +} ; + +} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Syncer2.cc b/libgrive/src/drive2/Syncer2.cc new file mode 100644 index 00000000..fa5e05dd --- /dev/null +++ b/libgrive/src/drive2/Syncer2.cc @@ -0,0 +1,243 @@ +/* + REST API Syncer implementation + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "base/Resource.hh" +#include "CommonUri.hh" +#include "Entry2.hh" +#include "Feed2.hh" +#include "Syncer2.hh" + +#include "http/Agent.hh" +#include "http/Download.hh" +#include "http/Header.hh" +#include "http/StringResponse.hh" +#include "json/ValResponse.hh" +#include "json/JsonWriter.hh" + +#include "util/OS.hh" +#include "util/log/Log.hh" +#include "util/StringStream.hh" +#include "util/ConcatStream.hh" + +#include + +#include + +// for debugging +#include + +namespace gr { namespace v2 { + +Syncer2::Syncer2( http::Agent *http ): + Syncer( http ) +{ + assert( http != 0 ) ; +} + +void Syncer2::DeleteRemote( Resource *res ) +{ + http::StringResponse str ; + http::Header hdr ; + hdr.Add( "If-Match: " + res->ETag() ) ; + m_http->Post( res->SelfHref() + "/trash", "", &str, hdr ) ; +} + +bool Syncer2::EditContent( Resource *res, bool new_rev ) +{ + assert( res->Parent() ) ; + assert( !res->ResourceID().empty() ) ; + assert( res->Parent()->GetState() == Resource::sync ) ; + + if ( !res->IsEditable() ) + { + Log( "Cannot upload %1%: file read-only. %2%", res->Name(), res->StateStr(), log::warning ) ; + return false ; + } + + return Upload( res, new_rev ) ; +} + +bool Syncer2::Create( Resource *res ) +{ + assert( res->Parent() ) ; + assert( res->Parent()->IsFolder() ) ; + assert( res->Parent()->GetState() == Resource::sync ) ; + assert( res->ResourceID().empty() ) ; + + if ( !res->Parent()->IsEditable() ) + { + Log( "Cannot upload %1%: parent directory read-only. %2%", res->Name(), res->StateStr(), log::warning ) ; + return false ; + } + + return Upload( res, false ); +} + +bool Syncer2::Move( Resource* res, Resource* newParentRes, std::string newFilename ) +{ + if ( res->ResourceID().empty() ) + { + Log("Can't rename file %1%, no server id found", res->Name()); + return false; + } + + Val meta; + meta.Add( "title", Val(newFilename) ); + if ( res->IsFolder() ) + { + meta.Add( "mimeType", Val( mime_types::folder ) ); + } + std::string json_meta = WriteJson( meta ); + + Val valr ; + + // Issue metadata update request + { + std::string addRemoveParents(""); + if (res->Parent()->IsRoot() ) + addRemoveParents += "&removeParents=root"; + else + addRemoveParents += "&removeParents=" + res->Parent()->ResourceID(); + if ( newParentRes->IsRoot() ) + addRemoveParents += "&addParents=root"; + else + addRemoveParents += "&addParents=" + newParentRes->ResourceID(); + http::Header hdr2 ; + hdr2.Add( "Content-Type: application/json" ); + http::ValResponse vrsp ; + // Don't change modified date because we're only moving + long http_code = m_http->Put( + feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents, + json_meta, &vrsp, hdr2 + ) ; + valr = vrsp.Response(); + assert( http_code == 200 && !( valr["id"].Str().empty() ) ); + } + + return true; +} + +std::string to_string( uint64_t n ) +{ + std::ostringstream s; + s << n; + return s.str(); +} + +bool Syncer2::Upload( Resource *res, bool new_rev ) +{ + Val meta; + meta.Add( "title", Val( res->Name() ) ); + if ( res->IsFolder() ) + meta.Add( "mimeType", Val( mime_types::folder ) ); + if ( !res->Parent()->IsRoot() ) + { + Val parent; + parent.Add( "id", Val( res->Parent()->ResourceID() ) ); + Val parents( Val::array_type ); + parents.Add( parent ); + meta.Add( "parents", parents ); + } + std::string json_meta = WriteJson( meta ); + + Val valr ; + + if ( res->IsFolder() ) + { + // Only issue metadata update request + http::Header hdr2 ; + hdr2.Add( "Content-Type: application/json" ); + http::ValResponse vrsp ; + long http_code = 0; + if ( res->ResourceID().empty() ) + http_code = m_http->Post( feeds::files, json_meta, &vrsp, hdr2 ) ; + else + http_code = m_http->Put( feeds::files + "/" + res->ResourceID(), json_meta, &vrsp, hdr2 ) ; + valr = vrsp.Response(); + assert( http_code == 200 && !( valr["id"].Str().empty() ) ); + } + else + { + File file( res->Path() ) ; + uint64_t size = file.Size() ; + ConcatStream multipart ; + StringStream p1( + "--file_contents\r\nContent-Type: application/json; charset=utf-8\r\n\r\n" + json_meta + + "\r\n--file_contents\r\nContent-Type: application/octet-stream\r\nContent-Length: " + to_string( size ) + + "\r\n\r\n" + ); + StringStream p2("\r\n--file_contents--\r\n"); + multipart.Append( &p1 ); + multipart.Append( &file ); + multipart.Append( &p2 ); + + http::Header hdr ; + if ( !res->ETag().empty() ) + hdr.Add( "If-Match: " + res->ETag() ) ; + hdr.Add( "Content-Type: multipart/related; boundary=\"file_contents\"" ); + hdr.Add( "Content-Length: " + to_string( multipart.Size() ) ); + + http::ValResponse vrsp; + m_http->Request( + res->ResourceID().empty() ? "POST" : "PUT", + upload_base + ( res->ResourceID().empty() ? "" : "/" + res->ResourceID() ) + + "?uploadType=multipart&newRevision=" + ( new_rev ? "true" : "false" ), + &multipart, &vrsp, hdr + ) ; + valr = vrsp.Response() ; + assert( !( valr["id"].Str().empty() ) ); + } + + Entry2 responseEntry = Entry2( valr ) ; + AssignIDs( res, responseEntry ) ; + res->SetServerTime( responseEntry.MTime() ); + + return true ; +} + +std::unique_ptr Syncer2::GetFolders() +{ + return std::unique_ptr( new Feed2( feeds::files + "?maxResults=1000&q=trashed%3dfalse+and+mimeType%3d%27" + mime_types::folder + "%27" ) ); +} + +std::unique_ptr Syncer2::GetAll() +{ + return std::unique_ptr( new Feed2( feeds::files + "?maxResults=1000&q=trashed%3dfalse" ) ); +} + +std::string ChangesFeed( long changestamp, int maxResults = 1000 ) +{ + boost::format feed( feeds::changes + "?maxResults=%1%&includeSubscribed=false" + ( changestamp > 0 ? "&startChangeId=%2%" : "" ) ) ; + return ( changestamp > 0 ? feed % maxResults % changestamp : feed % maxResults ).str() ; +} + +std::unique_ptr Syncer2::GetChanges( long min_cstamp ) +{ + return std::unique_ptr( new Feed2( ChangesFeed( min_cstamp ) ) ); +} + +long Syncer2::GetChangeStamp( long min_cstamp ) +{ + http::ValResponse res ; + m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header(), 0 ) ; + + return std::atoi( res.Response()["largestChangeId"].Str().c_str() ); +} + +} } // end of namespace gr::v1 diff --git a/libgrive/src/drive2/Syncer2.hh b/libgrive/src/drive2/Syncer2.hh new file mode 100644 index 00000000..e62d8b55 --- /dev/null +++ b/libgrive/src/drive2/Syncer2.hh @@ -0,0 +1,53 @@ +/* + REST API Syncer implementation + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "base/Syncer.hh" + +namespace gr { + +class Feed; + +namespace v2 { + +class Syncer2: public Syncer +{ + +public : + + Syncer2( http::Agent *http ); + + void DeleteRemote( Resource *res ); + bool EditContent( Resource *res, bool new_rev ); + bool Create( Resource *res ); + bool Move( Resource* res, Resource* newParent, std::string newFilename ); + + std::unique_ptr GetFolders(); + std::unique_ptr GetAll(); + std::unique_ptr GetChanges( long min_cstamp ); + long GetChangeStamp( long min_cstamp ); + +private : + + bool Upload( Resource *res, bool new_rev ); + +} ; + +} } // end of namespace gr::v2 diff --git a/libgrive/src/http/Agent.cc b/libgrive/src/http/Agent.cc new file mode 100644 index 00000000..7cfc99fa --- /dev/null +++ b/libgrive/src/http/Agent.cc @@ -0,0 +1,83 @@ +/* + Convenience wrapper methods for various kinds of HTTP requests + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Agent.hh" +#include "Header.hh" +#include "util/StringStream.hh" + +namespace gr { + +namespace http { + +Agent::Agent() +{ + mMaxUpload = mMaxDownload = 0; +} + +long Agent::Put( + const std::string& url, + const std::string& data, + DataStream *dest, + const Header& hdr ) +{ + StringStream s( data ); + return Request( "PUT", url, &s, dest, hdr ); +} + +long Agent::Put( + const std::string& url, + File *file, + DataStream *dest, + const Header& hdr ) +{ + return Request( "PUT", url, (SeekStream*)file, dest, hdr ); +} + +long Agent::Get( + const std::string& url, + DataStream *dest, + const Header& hdr, + u64_t downloadFileBytes ) +{ + return Request( "GET", url, NULL, dest, hdr, downloadFileBytes ); +} + +long Agent::Post( + const std::string& url, + const std::string& data, + DataStream *dest, + const Header& hdr ) +{ + Header h( hdr ) ; + StringStream s( data ); + h.Add( "Content-Type: application/x-www-form-urlencoded" ); + return Request( "POST", url, &s, dest, h ); +} + +void Agent::SetUploadSpeed( unsigned kbytes ) +{ + mMaxUpload = kbytes; +} + +void Agent::SetDownloadSpeed( unsigned kbytes ) +{ + mMaxDownload = kbytes; +} + +} } // end of namespace diff --git a/libgrive/src/http/Agent.hh b/libgrive/src/http/Agent.hh index b7b4779c..3ca7c118 100644 --- a/libgrive/src/http/Agent.hh +++ b/libgrive/src/http/Agent.hh @@ -20,42 +20,75 @@ #pragma once #include +#include "ResponseLog.hh" +#include "util/Types.hh" +#include "util/Progress.hh" -namespace gr { namespace http { +namespace gr { + +class SeekStream ; +class File ; + +namespace http { class Header ; -class Receivable ; class Agent { +protected: + unsigned mMaxUpload, mMaxDownload ; + public : + Agent() ; + virtual ~Agent() {} + + virtual ResponseLog* GetLog() const = 0 ; + virtual void SetLog( ResponseLog* ) = 0 ; + virtual long Put( const std::string& url, const std::string& data, - Receivable *dest, - const Header& hdr ) = 0 ; + DataStream *dest, + const Header& hdr ) ; + virtual long Put( + const std::string& url, + File *file, + DataStream *dest, + const Header& hdr ) ; + virtual long Get( const std::string& url, - Receivable *dest, - const Header& hdr ) = 0 ; + DataStream *dest, + const Header& hdr, + u64_t downloadFileBytes = 0 ) ; virtual long Post( const std::string& url, const std::string& data, - Receivable *dest, - const Header& hdr ) = 0 ; + DataStream *dest, + const Header& hdr ) ; - virtual long Custom( + virtual long Request( const std::string& method, const std::string& url, - Receivable *dest, - const Header& hdr ) = 0 ; + SeekStream *in, + DataStream *dest, + const Header& hdr, + u64_t downloadFileBytes = 0 ) = 0 ; + + virtual void SetUploadSpeed( unsigned kbytes ) ; + virtual void SetDownloadSpeed( unsigned kbytes ) ; + + virtual std::string LastError() const = 0 ; + virtual std::string LastErrorHeaders() const = 0 ; virtual std::string RedirLocation() const = 0 ; virtual std::string Escape( const std::string& str ) = 0 ; virtual std::string Unescape( const std::string& str ) = 0 ; + + virtual void SetProgressReporter( Progress* ) = 0; } ; } } // end of namespace diff --git a/libgrive/src/http/CurlAgent.cc b/libgrive/src/http/CurlAgent.cc index f61f3cb2..eee9a8d9 100644 --- a/libgrive/src/http/CurlAgent.cc +++ b/libgrive/src/http/CurlAgent.cc @@ -19,22 +19,19 @@ #include "CurlAgent.hh" -#include "Download.hh" #include "Error.hh" #include "Header.hh" -#include "Receivable.hh" #include "util/log/Log.hh" +#include "util/DataStream.hh" +#include "util/File.hh" #include -// dependent libraries -#include - #include #include #include -#include +#include #include #include @@ -43,20 +40,17 @@ namespace { using namespace gr::http ; +using namespace gr ; -size_t ReadCallback( void *ptr, std::size_t size, std::size_t nmemb, std::string *data ) +std::size_t ReadFileCallback( void *ptr, std::size_t size, std::size_t nmemb, SeekStream *file ) { assert( ptr != 0 ) ; - assert( data != 0 ) ; + assert( file != 0 ) ; - std::size_t count = std::min( size * nmemb, data->size() ) ; - if ( count > 0 ) - { - std::memcpy( ptr, &(*data)[0], count ) ; - data->erase( 0, count ) ; - } - - return count ; + if ( size*nmemb > 0 ) + return file->Read( static_cast(ptr), size*nmemb ) ; + + return 0 ; } } // end of local namespace @@ -67,10 +61,17 @@ struct CurlAgent::Impl { CURL *curl ; std::string location ; + bool error ; + std::string error_headers ; + std::string error_data ; + DataStream *dest ; + u64_t total_download, total_upload ; } ; -CurlAgent::CurlAgent() : - m_pimpl( new Impl ) +static struct curl_slist* SetHeader( CURL* handle, const Header& hdr ); + +CurlAgent::CurlAgent() : Agent(), + m_pimpl( new Impl ), m_pb( 0 ) { m_pimpl->curl = ::curl_easy_init(); } @@ -78,11 +79,20 @@ CurlAgent::CurlAgent() : void CurlAgent::Init() { ::curl_easy_reset( m_pimpl->curl ) ; - ::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYPEER, 0L ) ; + ::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYPEER, 0L ) ; ::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYHOST, 0L ) ; ::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADERFUNCTION, &CurlAgent::HeaderCallback ) ; - ::curl_easy_setopt( m_pimpl->curl, CURLOPT_WRITEHEADER , this ) ; - ::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADER, 0L ) ; + ::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADERDATA, this ) ; + ::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADER, 0L ) ; + if ( mMaxUpload > 0 ) + ::curl_easy_setopt( m_pimpl->curl, CURLOPT_MAX_SEND_SPEED_LARGE, mMaxUpload ) ; + if ( mMaxDownload > 0 ) + ::curl_easy_setopt( m_pimpl->curl, CURLOPT_MAX_RECV_SPEED_LARGE, mMaxDownload ) ; + m_pimpl->error = false; + m_pimpl->error_headers = ""; + m_pimpl->error_data = ""; + m_pimpl->dest = NULL; + m_pimpl->total_download = m_pimpl->total_upload = 0; } CurlAgent::~CurlAgent() @@ -90,31 +100,81 @@ CurlAgent::~CurlAgent() ::curl_easy_cleanup( m_pimpl->curl ); } +ResponseLog* CurlAgent::GetLog() const +{ + return m_log.get(); +} + +void CurlAgent::SetLog(ResponseLog *log) +{ + m_log.reset( log ); +} + +void CurlAgent::SetProgressReporter(Progress *progress) +{ + m_pb = progress; +} + std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) { - char *str = reinterpret_cast(ptr) ; + char *str = static_cast(ptr) ; std::string line( str, str + size*nmemb ) ; + // Check for error (HTTP 400 and above) + if ( line.substr( 0, 5 ) == "HTTP/" && line[9] >= '4' ) + pthis->m_pimpl->error = true; + + if ( pthis->m_pimpl->error ) + pthis->m_pimpl->error_headers += line; + + if ( pthis->m_log.get() ) + pthis->m_log->Write( str, size*nmemb ); + static const std::string loc = "Location: " ; std::size_t pos = line.find( loc ) ; if ( pos != line.npos ) { std::size_t end_pos = line.find( "\r\n", pos ) ; - pthis->m_pimpl->location = line.substr( loc.size(), end_pos - loc.size() ) ; + pthis->m_pimpl->location = line.substr( pos+loc.size(), end_pos - loc.size() ) ; } return size*nmemb ; } -std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, Receivable *recv ) +std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) { - assert( recv != 0 ) ; - return recv->OnData( ptr, size * nmemb ) ; + assert( pthis != 0 ) ; + if ( pthis->m_log.get() ) + pthis->m_log->Write( (const char*)ptr, size*nmemb ); + + if ( pthis->m_pimpl->error && pthis->m_pimpl->error_data.size() < 65536 ) + { + // Do not feed error responses to destination stream + pthis->m_pimpl->error_data.append( static_cast(ptr), size * nmemb ) ; + return size * nmemb ; + } + return pthis->m_pimpl->dest->Write( static_cast(ptr), size * nmemb ) ; +} + +int CurlAgent::progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload ) +{ + // Only report download progress when set explicitly + if ( pthis->m_pb ) + { + totalDownload = pthis->m_pimpl->total_download; + if ( !totalUpload ) + totalUpload = pthis->m_pimpl->total_upload; + pthis->m_pb->reportProgress( + totalDownload > 0 ? totalDownload : totalUpload, + totalDownload > 0 ? finishedDownload : finishedUpload + ); + } + return 0; } long CurlAgent::ExecCurl( const std::string& url, - Receivable *dest, + DataStream *dest, const http::Header& hdr ) { CURL *curl = m_pimpl->curl ; @@ -124,116 +184,90 @@ long CurlAgent::ExecCurl( ::curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error ) ; ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlAgent::Receive ) ; - ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, dest ) ; + ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, this ) ; + m_pimpl->dest = dest ; - SetHeader( hdr ) ; + struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ; + + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); - dest->Clear() ; CURLcode curl_code = ::curl_easy_perform(curl); - long http_code = 0; - ::curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + curl_slist_free_all(slist); + // get the HTTP response code + long http_code = 0; + ::curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); Trace( "HTTP response %1%", http_code ) ; - ::curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, 0 ) ; - - if ( curl_code != CURLE_OK || http_code >= 400 ) + + // reset the curl buffer to prevent it from touching our "error" buffer + ::curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, 0 ) ; + + m_pimpl->dest = NULL; + + // only throw for libcurl errors + if ( curl_code != CURLE_OK ) { BOOST_THROW_EXCEPTION( Error() << CurlCode( curl_code ) - << HttpResponse( http_code ) << Url( url ) - << expt::ErrMsg( error ) - << HttpHeader( hdr ) + << CurlErrMsg( error ) + << HttpRequestHeaders( hdr ) ) ; } return http_code ; } -long CurlAgent::Put( - const std::string& url, - const std::string& data, - Receivable *dest, - const Header& hdr ) +long CurlAgent::Request( + const std::string& method, + const std::string& url, + SeekStream *in, + DataStream *dest, + const Header& hdr, + u64_t downloadFileBytes ) { - Trace("HTTP PUT \"%1%\"", url ) ; - + Trace("HTTP %1% \"%2%\"", method, url ) ; + Init() ; + m_pimpl->total_download = downloadFileBytes ; CURL *curl = m_pimpl->curl ; - std::string put_data = data ; - // set common options - ::curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L ) ; - ::curl_easy_setopt(curl, CURLOPT_READFUNCTION, &ReadCallback ) ; - ::curl_easy_setopt(curl, CURLOPT_READDATA , &put_data ) ; - ::curl_easy_setopt(curl, CURLOPT_INFILESIZE, put_data.size() ) ; - - return ExecCurl( url, dest, hdr ) ; -} - -long CurlAgent::Get( - const std::string& url, - Receivable *dest, - const Header& hdr ) -{ - Trace("HTTP GET \"%1%\"", url ) ; - Init() ; - - // set get specific options - ::curl_easy_setopt(m_pimpl->curl, CURLOPT_HTTPGET, 1L); + ::curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str() ); + if ( in ) + { + ::curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L ) ; + ::curl_easy_setopt(curl, CURLOPT_READFUNCTION, &ReadFileCallback ) ; + ::curl_easy_setopt(curl, CURLOPT_READDATA , in ) ; + ::curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast( in->Size() ) ) ; + } return ExecCurl( url, dest, hdr ) ; } -long CurlAgent::Post( - const std::string& url, - const std::string& data, - Receivable *dest, - const Header& hdr ) +static struct curl_slist* SetHeader( CURL *handle, const Header& hdr ) { - Trace("HTTP POST \"%1%\" with \"%2%\"", url, data ) ; - - Init() ; - CURL *curl = m_pimpl->curl ; - - // make a copy because the parameter is const - std::string post_data = data ; + // set headers + struct curl_slist *curl_hdr = 0 ; + for ( Header::iterator i = hdr.begin() ; i != hdr.end() ; ++i ) + curl_hdr = curl_slist_append( curl_hdr, i->c_str() ) ; - // set post specific options - ::curl_easy_setopt(curl, CURLOPT_POST, 1L); - ::curl_easy_setopt(curl, CURLOPT_POSTFIELDS, &post_data[0] ) ; - ::curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.size() ) ; - - return ExecCurl( url, dest, hdr ) ; + ::curl_easy_setopt( handle, CURLOPT_HTTPHEADER, curl_hdr ) ; + return curl_hdr; } -long CurlAgent::Custom( - const std::string& method, - const std::string& url, - Receivable *dest, - const Header& hdr ) +std::string CurlAgent::LastError() const { - Trace("HTTP %2% \"%1%\"", url, method ) ; - - CURL *curl = m_pimpl->curl ; - - ::curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str() ); -// ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1 ); - - return ExecCurl( url, dest, hdr ) ; + return m_pimpl->error_data ; } -void CurlAgent::SetHeader( const Header& hdr ) +std::string CurlAgent::LastErrorHeaders() const { - // set headers - struct curl_slist *curl_hdr = 0 ; - for ( Header::iterator i = hdr.begin() ; i != hdr.end() ; ++i ) - curl_hdr = curl_slist_append( curl_hdr, i->c_str() ) ; - - ::curl_easy_setopt( m_pimpl->curl, CURLOPT_HTTPHEADER, curl_hdr ) ; + return m_pimpl->error_headers ; } std::string CurlAgent::RedirLocation() const diff --git a/libgrive/src/http/CurlAgent.hh b/libgrive/src/http/CurlAgent.hh index b1756384..b659f501 100644 --- a/libgrive/src/http/CurlAgent.hh +++ b/libgrive/src/http/CurlAgent.hh @@ -24,9 +24,13 @@ #include #include -namespace gr { namespace http { +#include -class Receivable ; +namespace gr { + +class DataStream ; + +namespace http { /*! \brief agent to provide HTTP access @@ -38,50 +42,45 @@ class CurlAgent : public Agent public : CurlAgent() ; ~CurlAgent() ; - - long Put( - const std::string& url, - const std::string& data, - Receivable *dest, - const Header& hdr ) ; - long Get( - const std::string& url, - Receivable *dest, - const Header& hdr ) ; - - long Post( - const std::string& url, - const std::string& data, - Receivable *dest, - const Header& hdr ) ; - - long Custom( + ResponseLog* GetLog() const ; + void SetLog( ResponseLog *log ) ; + void SetProgressReporter( Progress *progress ) ; + + long Request( const std::string& method, const std::string& url, - Receivable *dest, - const Header& hdr ) ; + SeekStream *in, + DataStream *dest, + const Header& hdr, + u64_t downloadFileBytes = 0 ) ; + + std::string LastError() const ; + std::string LastErrorHeaders() const ; std::string RedirLocation() const ; std::string Escape( const std::string& str ) ; std::string Unescape( const std::string& str ) ; + static int progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload ); + private : static std::size_t HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ; - static std::size_t Receive( void* ptr, size_t size, size_t nmemb, Receivable *recv ) ; - - void SetHeader( const Header& hdr ) ; + static std::size_t Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ; + long ExecCurl( const std::string& url, - Receivable *dest, + DataStream *dest, const Header& hdr ) ; void Init() ; - + private : struct Impl ; - std::auto_ptr m_pimpl ; + std::unique_ptr m_pimpl ; + std::unique_ptr m_log ; + Progress* m_pb ; } ; } } // end of namespace diff --git a/libgrive/src/http/Download.cc b/libgrive/src/http/Download.cc index 15171cdf..26804e6f 100644 --- a/libgrive/src/http/Download.cc +++ b/libgrive/src/http/Download.cc @@ -20,7 +20,6 @@ #include "Download.hh" // #include "util/SignalHandler.hh" -#include "Error.hh" #include "util/Crypt.hh" // boost headers @@ -65,7 +64,7 @@ std::string Download::Finish() const return m_crypt.get() != 0 ? m_crypt->Get() : "" ; } -std::size_t Download::OnData( void *data, std::size_t count ) +std::size_t Download::Write( const char *data, std::size_t count ) { assert( data != 0 ) ; @@ -75,4 +74,10 @@ std::size_t Download::OnData( void *data, std::size_t count ) return m_file.Write( data, count ) ; } + +std::size_t Download::Read( char *data, std::size_t count ) +{ + return count ; +} + } } // end of namespace diff --git a/libgrive/src/http/Download.hh b/libgrive/src/http/Download.hh index d3bdc59d..fdb75800 100644 --- a/libgrive/src/http/Download.hh +++ b/libgrive/src/http/Download.hh @@ -19,8 +19,7 @@ #pragma once -#include "Receivable.hh" -#include "util/StdioFile.hh" +#include "util/File.hh" #include @@ -33,7 +32,7 @@ namespace crypt namespace http { -class Download : public http::Receivable +class Download : public DataStream { public : struct NoChecksum {} ; @@ -44,11 +43,12 @@ public : std::string Finish() const ; void Clear() ; - std::size_t OnData( void *data, std::size_t count ) ; + std::size_t Write( const char *data, std::size_t count ) ; + std::size_t Read( char *, std::size_t ) ; private : - StdioFile m_file ; - std::auto_ptr m_crypt ; + File m_file ; + std::unique_ptr m_crypt ; } ; } } // end of namespace diff --git a/libgrive/src/http/Error.hh b/libgrive/src/http/Error.hh index a513f5f3..897b4555 100644 --- a/libgrive/src/http/Error.hh +++ b/libgrive/src/http/Error.hh @@ -27,18 +27,24 @@ namespace gr { namespace http { struct Error : virtual Exception {} ; // CURL error code -typedef boost::error_info CurlCode ; +typedef boost::error_info CurlCode ; -// HTTP response code -typedef boost::error_info HttpResponse ; - -// HTTP response body -typedef boost::error_info HttpResponseText ; +// CURL error message +typedef boost::error_info CurlErrMsg ; // URL -typedef boost::error_info Url ; +typedef boost::error_info Url ; + +// HTTP request headers +typedef boost::error_info HttpRequestHeaders ; -// HTTP headers -typedef boost::error_info HttpHeader ; +// HTTP response code +typedef boost::error_info HttpResponseCode ; + +// HTTP response headers +typedef boost::error_info HttpResponseHeaders ; + +// HTTP response body +typedef boost::error_info HttpResponseText ; } } // end of namespace diff --git a/libgrive/src/http/Header.cc b/libgrive/src/http/Header.cc index 36bd245a..6d875f91 100644 --- a/libgrive/src/http/Header.cc +++ b/libgrive/src/http/Header.cc @@ -22,6 +22,7 @@ #include #include #include +#include namespace gr { namespace http { @@ -34,6 +35,13 @@ void Header::Add( const std::string& str ) m_vec.push_back( str ) ; } +std::string Header::Str() const +{ + std::ostringstream s ; + s << *this ; + return s.str() ; +} + Header::iterator Header::begin() const { return m_vec.begin() ; @@ -50,4 +58,11 @@ std::ostream& operator<<( std::ostream& os, const Header& h ) return os ; } +Header operator+( const Header& header, const std::string& str ) +{ + Header h( header ) ; + h.Add( str ) ; + return h ; +} + } } // end of namespace diff --git a/libgrive/src/http/Header.hh b/libgrive/src/http/Header.hh index c581d7d1..844b6e6a 100644 --- a/libgrive/src/http/Header.hh +++ b/libgrive/src/http/Header.hh @@ -37,6 +37,7 @@ public : Header() ; void Add( const std::string& str ) ; + std::string Str() const ; iterator begin() const ; iterator end() const ; @@ -46,5 +47,6 @@ private : } ; std::ostream& operator<<( std::ostream& os, const Header& h ) ; +Header operator+( const Header& header, const std::string& str ) ; }} // end of namespace diff --git a/libgrive/src/http/ResponseLog.cc b/libgrive/src/http/ResponseLog.cc index 0e7e631e..b26856e5 100644 --- a/libgrive/src/http/ResponseLog.cc +++ b/libgrive/src/http/ResponseLog.cc @@ -19,6 +19,7 @@ #include "ResponseLog.hh" +#include "util/log/Log.hh" #include "util/DateTime.hh" #include @@ -27,28 +28,55 @@ namespace gr { namespace http { ResponseLog::ResponseLog( const std::string& prefix, - const std::string& suffix, - Receivable *next ) : - m_log( Filename(prefix, suffix).c_str() ), - m_next( next ) + const std::string& suffix ) { + Reset( prefix, suffix ) ; } -std::size_t ResponseLog::OnData( void *data, std::size_t count ) +std::size_t ResponseLog::Write( const char *data, std::size_t count ) { - m_log.rdbuf()->sputn( reinterpret_cast(data), count ) ; - return m_next->OnData( data, count ) ; + if ( m_enabled ) + { + assert( m_log.rdbuf() != 0 ) ; + m_log.rdbuf()->sputn( data, count ) ; + m_log.flush(); + } + return count; } -void ResponseLog::Clear() +std::size_t ResponseLog::Read( char *data, std::size_t count ) { - assert( m_next != 0 ) ; - m_next->Clear() ; + return 0 ; } std::string ResponseLog::Filename( const std::string& prefix, const std::string& suffix ) { - return prefix + DateTime::Now().Format( "%H%M%S" ) + suffix ; + return prefix + DateTime::Now().Format( "%F.%H%M%S" ) + suffix ; +} + +void ResponseLog::Reset( const std::string& prefix, const std::string& suffix ) +{ + if ( m_log.is_open() ) + m_log.close() ; + + const std::string fname = Filename( prefix, suffix ) ; + + // reset previous stream state. don't care if file can be opened + // successfully previously + m_log.clear() ; + + // re-open the file + m_log.open( fname.c_str() ) ; + if ( m_log ) + { + Trace( "logging HTTP response: %1%", fname ) ; + m_enabled = true ; + } + else + { + Trace( "cannot open log file %1%", fname ) ; + m_enabled = false ; + } } }} // end of namespace diff --git a/libgrive/src/http/ResponseLog.hh b/libgrive/src/http/ResponseLog.hh index a4273244..9c3b63a9 100644 --- a/libgrive/src/http/ResponseLog.hh +++ b/libgrive/src/http/ResponseLog.hh @@ -19,30 +19,31 @@ #pragma once -#include "Receivable.hh" +#include "util/DataStream.hh" #include #include namespace gr { namespace http { -class ResponseLog : public Receivable +class ResponseLog : public DataStream { public : ResponseLog( const std::string& prefix, - const std::string& suffix, - Receivable *next ) ; - - std::size_t OnData( void *data, std::size_t count ) ; - void Clear() ; - + const std::string& suffix ) ; + + std::size_t Write( const char *data, std::size_t count ) ; + std::size_t Read( char *data, std::size_t count ) ; + + void Reset( const std::string& prefix, const std::string& suffix ) ; + private : static std::string Filename( const std::string& prefix, const std::string& suffix ) ; private : + bool m_enabled ; std::ofstream m_log ; - Receivable *m_next ; } ; } } // end of namespace diff --git a/libgrive/src/http/StringResponse.cc b/libgrive/src/http/StringResponse.cc index d183be3f..c316d6f3 100644 --- a/libgrive/src/http/StringResponse.cc +++ b/libgrive/src/http/StringResponse.cc @@ -30,9 +30,14 @@ void StringResponse::Clear() m_resp.clear() ; } -std::size_t StringResponse::OnData( void *data, std::size_t count ) +std::size_t StringResponse::Write( const char *data, std::size_t count ) +{ + m_resp.append( data, count ) ; + return count ; +} + +std::size_t StringResponse::Read( char *data, std::size_t count ) { - m_resp.append( reinterpret_cast(data), count ) ; return count ; } diff --git a/libgrive/src/http/StringResponse.hh b/libgrive/src/http/StringResponse.hh index e361d01c..3f262a94 100644 --- a/libgrive/src/http/StringResponse.hh +++ b/libgrive/src/http/StringResponse.hh @@ -19,18 +19,19 @@ #pragma once -#include "Receivable.hh" +#include "util/DataStream.hh" #include namespace gr { namespace http { -class StringResponse : public Receivable +class StringResponse : public DataStream { public : StringResponse() ; - std::size_t OnData( void *data, std::size_t count ) ; + std::size_t Write( const char *data, std::size_t count ) ; + std::size_t Read( char *data, std::size_t count ) ; void Clear() ; const std::string& Response() const ; diff --git a/libgrive/src/http/XmlResponse.cc b/libgrive/src/http/XmlResponse.cc index 49395f17..3c9d6f33 100644 --- a/libgrive/src/http/XmlResponse.cc +++ b/libgrive/src/http/XmlResponse.cc @@ -28,15 +28,21 @@ XmlResponse::XmlResponse() : m_tb( new xml::TreeBuilder ) { } -std::size_t XmlResponse::OnData( void *data, std::size_t count ) +void XmlResponse::Clear() { - m_tb->ParseData( reinterpret_cast(data), count ) ; + m_tb.reset(new xml::TreeBuilder); +} + +std::size_t XmlResponse::Write( const char *data, std::size_t count ) +{ + m_tb->ParseData( data, count ) ; return count ; } -void XmlResponse::Clear() +std::size_t XmlResponse::Read( char *, std::size_t ) { - m_tb.reset( new xml::TreeBuilder ) ; + // throw something better + throw -1 ; } void XmlResponse::Finish() diff --git a/libgrive/src/http/XmlResponse.hh b/libgrive/src/http/XmlResponse.hh index 6af2b745..26736e46 100644 --- a/libgrive/src/http/XmlResponse.hh +++ b/libgrive/src/http/XmlResponse.hh @@ -19,31 +19,27 @@ #pragma once -#include "Receivable.hh" +#include "util/DataStream.hh" +#include "xml/TreeBuilder.hh" #include -namespace gr { namespace xml -{ - class Node ; - class TreeBuilder ; -} } - namespace gr { namespace http { -class XmlResponse : public Receivable +class XmlResponse : public DataStream { public : XmlResponse() ; void Clear() ; - std::size_t OnData( void *data, std::size_t count ) ; + std::size_t Write( const char *data, std::size_t count ) ; + std::size_t Read( char *data, std::size_t count ) ; void Finish() ; xml::Node Response() const ; private : - std::auto_ptr m_tb ; + std::unique_ptr m_tb ; } ; } } // end of namespace diff --git a/libgrive/src/json/JsonParser.cc b/libgrive/src/json/JsonParser.cc new file mode 100644 index 00000000..9727b8e8 --- /dev/null +++ b/libgrive/src/json/JsonParser.cc @@ -0,0 +1,196 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include "JsonParser.hh" + +#include "Val.hh" +#include "ValBuilder.hh" + +#include + +namespace gr { + +namespace +{ + int OnNull( void *ctx ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->VisitNull() ; + return true ; + } + + int OnBool( void *ctx, int value ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->Visit( static_cast(value) ) ; + return true ; + } + + int OnInt( void *ctx, long long value ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->Visit(value) ; + return true ; + } + + int OnDouble( void *ctx, double value ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->Visit(value) ; + return true ; + } + + int OnStr( void *ctx, const unsigned char *str, std::size_t len ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->Visit( std::string(reinterpret_cast(str), len) ) ; + return true ; + } + + int StartMap( void *ctx ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->StartObject() ; + return true ; + } + + int OnMapKey( void *ctx, const unsigned char *str, std::size_t len ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->VisitKey( std::string(reinterpret_cast(str), len) ) ; + return true ; + } + + int EndMap( void *ctx ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->EndObject() ; + return true ; + } + + int StartArray( void *ctx ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->StartArray() ; + return true ; + } + + int EndArray( void *ctx ) + { + ValVisitor *b = reinterpret_cast(ctx) ; + b->EndArray() ; + return true ; + } + + const yajl_callbacks callbacks = { + OnNull, + OnBool, + OnInt, + OnDouble, + 0, + OnStr, + StartMap, + OnMapKey, + EndMap, + StartArray, + EndArray, + }; +} + +Val ParseJson( const std::string& json ) +{ + ValBuilder b; + JsonParser parser( &b ) ; + parser.Parse( json.c_str(), json.size() ) ; + parser.Finish() ; + return b.Result(); +} + +Val ParseJson( DataStream &in ) +{ + ValBuilder b; + JsonParser parser( &b ) ; + parser.Parse( in ) ; + parser.Finish() ; + return b.Result(); +} + +struct JsonParser::Impl +{ + ValVisitor *callback ; + yajl_handle hand ; +} ; + +JsonParser::JsonParser( ValVisitor *callback ) : + m_impl( new Impl ) +{ + m_impl->callback = callback ; + m_impl->hand = yajl_alloc( &callbacks, 0, m_impl->callback ) ; +} + +JsonParser::~JsonParser() +{ + yajl_free( m_impl->hand ) ; +} + +void JsonParser::Parse( const char *str, std::size_t size ) +{ + const unsigned char *ustr = reinterpret_cast(str) ; + + yajl_status r = yajl_parse( m_impl->hand, ustr, size ) ; + + if ( r != yajl_status_ok ) + { + unsigned char *msg = yajl_get_error( m_impl->hand, true, ustr, size ) ; + std::string msg_str(reinterpret_cast(msg)) ; + yajl_free_error(m_impl->hand, msg) ; + + BOOST_THROW_EXCEPTION( + Error() + << ParseErr_(msg_str) + << JsonText_(std::string(str,size)) + ); + } +} + +void JsonParser::Parse( DataStream &in ) +{ + char buf[1024] ; + std::size_t count = 0 ; + + while ( (count = in.Read( buf, sizeof(buf) ) ) > 0 ) + { + Parse( buf, count ); + } +} + +void JsonParser::Finish() +{ + if ( yajl_complete_parse(m_impl->hand) != yajl_status_ok ) + { + unsigned char *msg = yajl_get_error( m_impl->hand, false, 0, 0 ) ; + std::string msg_str(reinterpret_cast(msg)) ; + yajl_free_error(m_impl->hand, msg) ; + + BOOST_THROW_EXCEPTION( Error() << ParseErr_(msg_str) ) ; + } +} + +} // end of namespace gr::json diff --git a/libgrive/src/json/JsonParser.hh b/libgrive/src/json/JsonParser.hh new file mode 100644 index 00000000..5842699d --- /dev/null +++ b/libgrive/src/json/JsonParser.hh @@ -0,0 +1,57 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include "Val.hh" +#include "util/Exception.hh" +#include "util/DataStream.hh" + +#include +#include + +namespace gr { + +class ValVisitor ; + +Val ParseJson( const std::string& json ) ; +Val ParseJson( DataStream &in ) ; + +class JsonParser +{ +public : + struct Error : virtual Exception {} ; + typedef boost::error_info ParseErr_ ; + typedef boost::error_info JsonText_ ; + + explicit JsonParser( ValVisitor *callback ) ; + ~JsonParser() ; + + void Parse( const char *str, std::size_t size ) ; + void Parse( DataStream &in ) ; + void Finish() ; + +private : + struct Impl ; + std::unique_ptr m_impl ; +} ; + +} // end of namespace + diff --git a/libgrive/src/json/JsonWriter.cc b/libgrive/src/json/JsonWriter.cc new file mode 100644 index 00000000..700c10c9 --- /dev/null +++ b/libgrive/src/json/JsonWriter.cc @@ -0,0 +1,119 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include "JsonWriter.hh" +#include "util/StringStream.hh" + +#include + +#include + +namespace gr { + +struct JsonWriter::Impl +{ + yajl_gen gen ; + DataStream *out ; +} ; + +JsonWriter::JsonWriter( DataStream *out ) : + m_impl( new Impl ) +{ + assert( out != 0 ) ; + + m_impl->out = out ; + m_impl->gen = yajl_gen_alloc(0) ; + yajl_gen_config( m_impl->gen, yajl_gen_print_callback, &JsonWriter::WriteCallback, this ) ; +} + +JsonWriter::~JsonWriter() +{ + yajl_gen_free( m_impl->gen ) ; +} + +void JsonWriter::Visit( long long t ) +{ + yajl_gen_integer( m_impl->gen, t ) ; +} + +void JsonWriter::Visit( double t ) +{ + yajl_gen_double( m_impl->gen, t ) ; +} + +void JsonWriter::Visit( const std::string& t ) +{ + yajl_gen_string( m_impl->gen, + reinterpret_cast(t.c_str()), t.size() ) ; +} + +void JsonWriter::Visit( bool t ) +{ + yajl_gen_bool( m_impl->gen, t ) ; +} + +void JsonWriter::VisitNull() +{ + yajl_gen_null( m_impl->gen ) ; +} + +void JsonWriter::StartArray() +{ + yajl_gen_array_open( m_impl->gen ) ; +} + +void JsonWriter::EndArray() +{ + yajl_gen_array_close( m_impl->gen ) ; +} + +void JsonWriter::StartObject() +{ + yajl_gen_map_open( m_impl->gen ) ; +} + +void JsonWriter::VisitKey( const std::string& t ) +{ + Visit(t) ; +} + +void JsonWriter::EndObject() +{ + yajl_gen_map_close( m_impl->gen ) ; +} + +void JsonWriter::WriteCallback( void *ctx, const char *str, std::size_t size ) +{ + JsonWriter *pthis = reinterpret_cast(ctx) ; + assert( pthis != 0 ) ; + assert( pthis->m_impl->out != 0 ) ; + + pthis->m_impl->out->Write( str, size ) ; +} + +std::string WriteJson( const Val& val ) +{ + StringStream ss ; + JsonWriter wr( &ss ) ; + val.Visit( &wr ) ; + return ss.Str() ; +} + +} // end of namespace diff --git a/libgrive/src/json/JsonWriter.hh b/libgrive/src/json/JsonWriter.hh new file mode 100644 index 00000000..595f8e06 --- /dev/null +++ b/libgrive/src/json/JsonWriter.hh @@ -0,0 +1,60 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include "Val.hh" +#include "ValVisitor.hh" +#include + +namespace gr { + +class DataStream ; + +class JsonWriter : public ValVisitor +{ +public : + JsonWriter( DataStream *out ) ; + ~JsonWriter() ; + + void Visit( long long t ) ; + void Visit( double t ) ; + void Visit( const std::string& t ) ; + void Visit( bool t ) ; + void VisitNull() ; + + void StartArray() ; + void EndArray() ; + void StartObject() ; + void VisitKey( const std::string& t ) ; + void EndObject() ; + +private : + static void WriteCallback( void *ctx, const char *str, std::size_t size ) ; + +private : + struct Impl ; + std::unique_ptr m_impl ; +} ; + +std::string WriteJson( const Val& val ); + +} // end of namespace + diff --git a/libgrive/src/json/Val.cc b/libgrive/src/json/Val.cc new file mode 100644 index 00000000..2facfffd --- /dev/null +++ b/libgrive/src/json/Val.cc @@ -0,0 +1,315 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include "Val.hh" +#include "JsonWriter.hh" +#include "ValVisitor.hh" +#include "util/StdStream.hh" + +#include + +namespace gr { + +const Val& Val::Null() +{ + static const Val null( null_type ) ; + return null ; +} + +Val::Val( ) : + m_base( new Impl ) +{ +} + +Val::Val( TypeEnum type ) +{ + switch ( type ) + { + case int_type: m_base.reset( new Impl ) ; break ; + case bool_type: m_base.reset( new Impl ) ; break ; + case double_type: m_base.reset( new Impl ) ; break ; + case string_type: m_base.reset( new Impl ) ; break ; + case array_type: m_base.reset( new Impl ) ; break ; + case object_type: m_base.reset( new Impl ) ; break ; + case null_type: + default: m_base.reset( new Impl ) ; break ; + } +} + +Val::Val( const Val& v ) : + m_base( v.m_base->Clone() ) +{ +} + +Val::~Val() +{ +} + +void Val::Swap( Val& val ) +{ + std::swap( m_base, val.m_base ) ; +} + +Val& Val::operator=( const Val& val ) +{ + Val tmp(val) ; + Swap(tmp) ; + return *this ; +} + +Val::TypeEnum Val::Type() const +{ + return m_base->Type() ; +} + +const Val& Val::operator[]( const std::string& key ) const +{ + const Object& obj = As() ; + Object::const_iterator i = obj.find(key) ; + if ( i != obj.end() ) + return i->second ; + + // shut off compiler warning + BOOST_THROW_EXCEPTION(Error() << NoKey_(key)) ; + throw ; +} + +Val& Val::operator[]( const std::string& key ) +{ + Object& obj = As() ; + Object::iterator i = obj.find(key) ; + if ( i != obj.end() ) + return i->second ; + + // shut off compiler warning + BOOST_THROW_EXCEPTION(Error() << NoKey_(key)) ; + throw ; +} + +const Val& Val::operator[]( std::size_t index ) const +{ + const Array& ar = As() ; + if ( index < ar.size() ) + return ar[index] ; + + // shut off compiler warning + BOOST_THROW_EXCEPTION(Error() << OutOfRange_(index)) ; + throw ; +} + +std::string Val::Str() const +{ + if ( Type() == int_type ) + return boost::to_string( As() ); + return As() ; +} + +Val::operator std::string() const +{ + return Str(); +} + +int Val::Int() const +{ + if ( Type() == string_type ) + return std::atoi( As().c_str() ); + return static_cast(As()) ; +} + +unsigned long long Val::U64() const +{ + if ( Type() == string_type ) + return strtoull( As().c_str(), NULL, 10 ); + return static_cast(As()) ; +} + +double Val::Double() const +{ + if ( Type() == string_type ) + return std::atof( As().c_str() ); + return As() ; +} + +bool Val::Bool() const +{ + return As() ; +} + +const Val::Array& Val::AsArray() const +{ + return As() ; +} + +Val::Array& Val::AsArray() +{ + return As() ; +} + +const Val::Object& Val::AsObject() const +{ + return As() ; +} + +Val::Object& Val::AsObject() +{ + return As() ; +} + +bool Val::Has( const std::string& key ) const +{ + const Object& obj = As() ; + return obj.find(key) != obj.end() ; +} + +bool Val::Del( const std::string& key ) +{ + Object& obj = As() ; + return obj.erase(key) > 0 ; +} + +Val& Val::Item( const std::string& key ) +{ + return As()[key]; +} + +bool Val::Get( const std::string& key, Val& val ) const +{ + const Object& obj = As() ; + Object::const_iterator i = obj.find(key) ; + if ( i != obj.end() ) + { + val = i->second ; + return true ; + } + else + return false ; +} + +void Val::Add( const std::string& key, const Val& value ) +{ + As().insert( std::make_pair(key, value) ) ; +} + +void Val::Set( const std::string& key, const Val& value ) +{ + Object& obj = As(); + Object::iterator i = obj.find(key); + if (i == obj.end()) + obj.insert(std::make_pair(key, value)); + else + i->second = value; +} + +void Val::Add( const Val& json ) +{ + As().push_back( json ) ; +} + +void Val::Visit( ValVisitor *visitor ) const +{ + switch ( Type() ) + { + case null_type: visitor->VisitNull() ; break ; + case int_type: visitor->Visit( As() ) ; break ; + case double_type: visitor->Visit( As() ) ; break ; + case string_type: visitor->Visit( As() ) ; break ; + case bool_type: visitor->Visit( As() ) ; break ; + + case object_type: + { + visitor->StartObject() ; + + const Object& obj = As() ; + for ( Object::const_iterator i = obj.begin() ; i != obj.end() ; ++i ) + { + visitor->VisitKey( i->first ) ; + i->second.Visit( visitor ) ; + } + + visitor->EndObject() ; + break ; + } + + case array_type: + { + visitor->StartArray() ; + + const Array& arr = As() ; + for ( Array::const_iterator i = arr.begin() ; i != arr.end() ; ++i ) + i->Visit( visitor ) ; + + visitor->EndArray() ; + break ; + } + } +} + +void Val::Select( const Object& obj, const std::string& key, std::vector& result ) const +{ + Object::const_iterator i = obj.find(key) ; + if ( i != obj.end() ) + result.push_back(i->second) ; +} + +/** If \a this is an array of objects, this function returns all values of + the objects in the array with the key \a key. If \a this is an object, + just return the value with the key \a key. +*/ +std::vector Val::Select( const std::string& key ) const +{ + std::vector result ; + if ( Is() ) + Select( As(), key, result ) ; + + else if ( Is() ) + { + const Array& array = As() ; + for ( Array::const_iterator i = array.begin() ; i != array.end() ; ++i ) + { + if ( i->Is() ) + Select( i->As(), key, result ) ; + } + } + return result ; +} + +std::ostream& operator<<( std::ostream& os, const Val& val ) +{ + StdStream ss( os.rdbuf() ) ; + JsonWriter wr( &ss ) ; + val.Visit( &wr ) ; + + return os ; +} + +} // end of namespace + +namespace std +{ + void swap( gr::Val& v1, gr::Val& v2 ) + { + v1.Swap( v2 ) ; + } + + ostream& operator<<( ostream& os, gr::Val::TypeEnum t ) + { + return os << static_cast(t) ; + } +} diff --git a/libgrive/src/json/Val.hh b/libgrive/src/json/Val.hh new file mode 100644 index 00000000..eace6a29 --- /dev/null +++ b/libgrive/src/json/Val.hh @@ -0,0 +1,234 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include "util/Exception.hh" + +#include +#include +#include +#include +#include +#include + +namespace gr { + +class ValVisitor ; + +class Val +{ +public : + enum TypeEnum { null_type, bool_type, double_type, int_type, object_type, array_type, string_type } ; + + struct Error : virtual Exception {} ; + typedef boost::error_info SrcType_ ; + typedef boost::error_info DestType_ ; + typedef boost::error_info NoKey_ ; + typedef boost::error_info OutOfRange_ ; + +private : + template + struct Type2Enum ; + + template + struct SupportType ; + +public : + typedef std::vector Array ; + typedef std::map Object ; + +public : + Val() ; + Val( const Val& v ) ; + explicit Val( TypeEnum type ) ; + ~Val() ; + + static const Val& Null() ; + + template + explicit Val( const T& t ) + { + Assign(t) ; + } + + template + Val& Assign( const T& t ) ; + + void Swap( Val& val ) ; + Val& operator=( const Val& val ) ; + + template + Val& operator=( const T& t ) + { + return Assign(t) ; + } + + operator std::string() const ; + + template + const T& As() const ; + + template + T& As() ; + + template + bool Is() const ; + + TypeEnum Type() const ; + + // shortcuts for As<>() + std::string Str() const ; + int Int() const ; + unsigned long long U64() const ; + double Double() const ; + bool Bool() const ; + const Array& AsArray() const ; + Array& AsArray() ; + const Object& AsObject() const ; + Object& AsObject() ; + + // shortcuts for objects + Val& operator[]( const std::string& key ) ; // get updatable ref or throw + const Val& operator[]( const std::string& key ) const ; // get const ref or throw + Val& Item( const std::string& key ) ; // insert if not exists and get + bool Has( const std::string& key ) const ; // check if exists + bool Get( const std::string& key, Val& val ) const ; // get or return false + void Add( const std::string& key, const Val& val ) ; // insert or do nothing + void Set( const std::string& key, const Val& val ) ; // insert or update + bool Del( const std::string& key ); // delete or do nothing + + // shortcuts for array (and array of objects) + const Val& operator[]( std::size_t index ) const ; + void Add( const Val& json ) ; + + std::vector Select( const std::string& key ) const ; + + friend std::ostream& operator<<( std::ostream& os, const Val& val ) ; + void Visit( ValVisitor *visitor ) const ; + +private : + struct Base ; + + template + struct Impl ; + + std::unique_ptr m_base ; + +private : + void Select( const Object& obj, const std::string& key, std::vector& result ) const ; +} ; + +template <> struct Val::Type2Enum { static const TypeEnum type = null_type ; } ; +template <> struct Val::Type2Enum { static const TypeEnum type = int_type ; } ; +template <> struct Val::Type2Enum { static const TypeEnum type = bool_type ; } ; +template <> struct Val::Type2Enum { static const TypeEnum type = double_type ;} ; +template <> struct Val::Type2Enum { static const TypeEnum type = string_type ; } ; +template <> struct Val::Type2Enum { static const TypeEnum type = array_type ; } ; +template <> struct Val::Type2Enum { static const TypeEnum type = object_type ; } ; + +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; +template <> struct Val::SupportType { typedef long long Type ; } ; + +template <> struct Val::SupportType { typedef bool Type ; } ; +template <> struct Val::SupportType { typedef double Type ; } ; +template <> struct Val::SupportType { typedef std::string Type ; } ; +template <> struct Val::SupportType { typedef std::string Type ; } ; +template <> struct Val::SupportType { typedef Val::Array Type ; } ; +template <> struct Val::SupportType { typedef Val::Object Type ; } ; + +struct Val::Base +{ + virtual ~Base() {} + virtual Base* Clone() const = 0 ; + virtual TypeEnum Type() const = 0 ; +} ; + +template +struct Val::Impl : public Base +{ + T val ; + Impl( ) : val() {} + Impl( const T& t ) : val(t) {} + Impl* Clone() const { return new Impl(val); } + TypeEnum Type() const { return Type2Enum::type ; } +} ; + +template <> +struct Val::Impl : public Base +{ + Impl* Clone() const { return new Impl; } + TypeEnum Type() const { return null_type ; } +} ; + +template +Val& Val::Assign( const T& t ) +{ + m_base.reset( new Impl::Type>(t) ) ; + return *this ; +} + +template +const T& Val::As() const +{ + const Impl *impl = dynamic_cast *>( m_base.get() ) ; + if ( !impl ) + { + TypeEnum dest = Type2Enum::type ; + BOOST_THROW_EXCEPTION( + Error() << SrcType_( Type() ) << DestType_( dest ) + ) ; + } + return impl->val ; +} + +template +T& Val::As() +{ + Impl *impl = dynamic_cast *>( m_base.get() ) ; + if ( !impl ) + { + TypeEnum dest = Type2Enum::type ; + BOOST_THROW_EXCEPTION( + Error() << SrcType_( Type() ) << DestType_( dest ) + ) ; + } + return impl->val ; +} + +template +bool Val::Is() const +{ + return Type() == Type2Enum::type ; +} + +} // end of namespace + +namespace std +{ + void swap( gr::Val& v1, gr::Val& v2 ) ; + ostream& operator<<( ostream& os, gr::Val::TypeEnum t ) ; +} diff --git a/libgrive/src/json/ValBuilder.cc b/libgrive/src/json/ValBuilder.cc new file mode 100644 index 00000000..e3b1b0ab --- /dev/null +++ b/libgrive/src/json/ValBuilder.cc @@ -0,0 +1,140 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include "ValBuilder.hh" + +namespace gr { + +ValBuilder::ValBuilder( ) +{ +} + +ValBuilder::~ValBuilder() +{ +} + +void ValBuilder::Visit( long long t ) +{ + Build(Val(t)) ; +} + +void ValBuilder::Visit( double t ) +{ + Build(Val(t)) ; +} + +void ValBuilder::Visit( const std::string& t ) +{ + Build(Val(t)) ; +} + +void ValBuilder::Visit( bool t ) +{ + Build(Val(t)) ; +} + +void ValBuilder::VisitNull() +{ + Build(Val()) ; +} + +void ValBuilder::Build( const Val& t ) +{ + if ( m_ctx.empty() ) + { + Level l = { Val::Null(), t } ; + m_ctx.push( l ) ; + } + + else if ( m_ctx.top().val.Is() ) + { + Val::Array& ar = m_ctx.top().val.As() ; + ar.push_back( t ) ; + } + else if ( m_ctx.top().val.Is() ) + { + if ( !m_ctx.top().key.Is() ) + BOOST_THROW_EXCEPTION( Error() << NoKey_(t) ) ; + + else + { + Val::Object& obj = m_ctx.top().val.As() ; + obj.insert( std::make_pair( m_ctx.top().key.Str(), t ) ) ; + m_ctx.top().key = Val::Null() ; + } + } + else + BOOST_THROW_EXCEPTION( Error() << Unexpected_(m_ctx.top().val) ) ; +} + +void ValBuilder::VisitKey( const std::string& t ) +{ + m_ctx.top().key = t ; +} + +void ValBuilder::StartArray() +{ + Level l = { Val::Null(), Val(Val::Array()) } ; + m_ctx.push(l) ; +} + +void ValBuilder::EndArray() +{ + End( Val::array_type ) ; +} + +void ValBuilder::End( Val::TypeEnum type ) +{ + if ( m_ctx.top().val.Type() == type ) + { + if( !m_ctx.top().key.Is() ) + BOOST_THROW_EXCEPTION( Error() << Unexpected_(m_ctx.top().key) ) ; + + // get top Val from stack + Val current ; + current.Swap( m_ctx.top().val ) ; + m_ctx.pop() ; + + Build(current) ; + } +} + +void ValBuilder::StartObject() +{ + Level l = { Val::Null(), Val( Val::Object() ) } ; + m_ctx.push(l) ; +} + +void ValBuilder::EndObject() +{ + End( Val::object_type ) ; +} + +Val ValBuilder::Result() const +{ + if ( !m_ctx.size() ) + BOOST_THROW_EXCEPTION( Error() << NoKey_( Val(std::string("")) ) ) ; + Val r = m_ctx.top().val; + if ( m_ctx.size() > 1 ) + BOOST_THROW_EXCEPTION( Error() << Unexpected_(m_ctx.top().val) ) ; + return r; +} + +} // end of namespace diff --git a/libgrive/src/json/ValBuilder.hh b/libgrive/src/json/ValBuilder.hh new file mode 100644 index 00000000..efbe5b2a --- /dev/null +++ b/libgrive/src/json/ValBuilder.hh @@ -0,0 +1,74 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include "ValVisitor.hh" + +#include "Val.hh" +#include "util/Exception.hh" + +#include +#include + +namespace gr { + +class ValBuilder : public ValVisitor +{ +public : + struct Error : virtual Exception {} ; + typedef boost::error_info Mismatch_ ; + typedef boost::error_info Unexpected_ ; + typedef boost::error_info NoKey_ ; + +public : + ValBuilder( ) ; + ~ValBuilder() ; + + void Visit( long long t ) ; + void Visit( double t ) ; + void Visit( const std::string& t ) ; + void Visit( bool t ) ; + void VisitNull() ; + void Build( const Val& t ) ; + + void StartArray() ; + void EndArray() ; + void StartObject() ; + void VisitKey( const std::string& t ) ; + void EndObject() ; + + Val Result() const ; + +private : + void End( Val::TypeEnum type ) ; + +private : + struct Level + { + Val key ; + Val val ; + } ; + + std::stack m_ctx ; +} ; + +} // end of namespace + diff --git a/libgrive/src/json/ValResponse.cc b/libgrive/src/json/ValResponse.cc new file mode 100644 index 00000000..02c81b48 --- /dev/null +++ b/libgrive/src/json/ValResponse.cc @@ -0,0 +1,53 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include "ValResponse.hh" + +#include "Val.hh" + +namespace gr { namespace http { + +ValResponse::ValResponse( ) : + m_parser( &m_val ) +{ +} + +std::size_t ValResponse::Write( const char *data, std::size_t count ) +{ + m_parser.Parse( data, count ) ; + return count ; +} + +std::size_t ValResponse::Read( char *data, std::size_t count ) +{ + return count ; +} + +Val ValResponse::Response() const +{ + return m_val.Result() ; +} + +void ValResponse::Finish() +{ + m_parser.Finish() ; +} + +} } // end of namespace gr::http diff --git a/libgrive/src/json/ValResponse.hh b/libgrive/src/json/ValResponse.hh new file mode 100644 index 00000000..cdeca6af --- /dev/null +++ b/libgrive/src/json/ValResponse.hh @@ -0,0 +1,52 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include "util/DataStream.hh" + +#include "JsonParser.hh" +#include "ValBuilder.hh" + +namespace gr +{ + class Val ; +} + +namespace gr { namespace http { + +class ValResponse : public DataStream +{ +public : + ValResponse() ; + + std::size_t Write( const char *data, std::size_t count ) ; + std::size_t Read( char *data, std::size_t count ) ; + + void Finish() ; + Val Response() const ; + +private : + ValBuilder m_val ; + JsonParser m_parser ; +} ; + +} } // end of namespace gr::http + diff --git a/libgrive/src/json/ValVisitor.hh b/libgrive/src/json/ValVisitor.hh new file mode 100644 index 00000000..b523c70d --- /dev/null +++ b/libgrive/src/json/ValVisitor.hh @@ -0,0 +1,46 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include + +namespace gr { + +class ValVisitor +{ +public : + virtual ~ValVisitor() {} + + virtual void Visit( long long t ) = 0 ; + virtual void Visit( double t ) = 0 ; + virtual void Visit( const std::string& t ) = 0 ; + virtual void Visit( bool t ) = 0 ; + virtual void VisitNull() = 0 ; + + virtual void StartArray() = 0 ; + virtual void EndArray() = 0 ; + virtual void StartObject() = 0 ; + virtual void VisitKey( const std::string& t ) = 0 ; + virtual void EndObject() = 0 ; +} ; + +} // end of namespace + diff --git a/libgrive/src/protocol/AuthAgent.cc b/libgrive/src/protocol/AuthAgent.cc new file mode 100644 index 00000000..6baf1133 --- /dev/null +++ b/libgrive/src/protocol/AuthAgent.cc @@ -0,0 +1,165 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "AuthAgent.hh" + +#include "http/Error.hh" +#include "http/Header.hh" +#include "util/log/Log.hh" +#include "util/OS.hh" +#include "util/File.hh" + +#include + +namespace gr { + +using namespace http ; + +AuthAgent::AuthAgent( OAuth2& auth, Agent *real_agent ) : + Agent(), + m_auth ( auth ), + m_agent ( real_agent ) +{ +} + +http::ResponseLog* AuthAgent::GetLog() const +{ + return m_agent->GetLog(); +} + +void AuthAgent::SetLog( http::ResponseLog *log ) +{ + return m_agent->SetLog( log ); +} + +void AuthAgent::SetProgressReporter( Progress *progress ) +{ + m_agent->SetProgressReporter( progress ); +} + +void AuthAgent::SetUploadSpeed( unsigned kbytes ) +{ + m_agent->SetUploadSpeed( kbytes ); +} + +void AuthAgent::SetDownloadSpeed( unsigned kbytes ) +{ + m_agent->SetDownloadSpeed( kbytes ); +} + +http::Header AuthAgent::AppendHeader( const http::Header& hdr ) const +{ + http::Header h(hdr) ; + h.Add( "Authorization: Bearer " + m_auth.AccessToken() ) ; + h.Add( "GData-Version: 3.0" ) ; + return h ; +} + +long AuthAgent::Request( + const std::string& method, + const std::string& url, + SeekStream *in, + DataStream *dest, + const http::Header& hdr, + u64_t downloadFileBytes ) +{ + long response; + Header auth; + do + { + auth = AppendHeader( hdr ); + if ( in ) + in->Seek( 0, 0 ); + response = m_agent->Request( method, url, in, dest, auth, downloadFileBytes ); + } while ( CheckRetry( response ) ); + return CheckHttpResponse( response, url, auth ); +} + +std::string AuthAgent::LastError() const +{ + return m_agent->LastError() ; +} + +std::string AuthAgent::LastErrorHeaders() const +{ + return m_agent->LastErrorHeaders() ; +} + +std::string AuthAgent::RedirLocation() const +{ + return m_agent->RedirLocation() ; +} + +std::string AuthAgent::Escape( const std::string& str ) +{ + return m_agent->Escape( str ) ; +} + +std::string AuthAgent::Unescape( const std::string& str ) +{ + return m_agent->Unescape( str ) ; +} + +bool AuthAgent::CheckRetry( long response ) +{ + // HTTP 500 and 503 should be temporary. just wait a bit and retry + if ( response == 500 || response == 503 ) + { + Log( "request failed due to temporary error: %1% (body: %2%). retrying in 5 seconds", + response, m_agent->LastError(), log::warning ) ; + + os::Sleep( 5 ) ; + return true ; + } + + // HTTP 401 Unauthorized. the auth token has been expired. refresh it + else if ( response == 401 ) + { + Log( "request failed due to auth token expired: %1% (body: %2%). refreshing token", + response, m_agent->LastError(), log::warning ) ; + + os::Sleep( 5 ) ; + m_auth.Refresh() ; + return true ; + } + else + return false ; +} + +long AuthAgent::CheckHttpResponse( + long response, + const std::string& url, + const http::Header& hdr ) +{ + // throw for other HTTP errors + if ( response >= 400 ) + { + BOOST_THROW_EXCEPTION( + Error() + << HttpResponseCode( response ) + << HttpResponseHeaders( m_agent->LastErrorHeaders() ) + << HttpResponseText( m_agent->LastError() ) + << Url( url ) + << HttpRequestHeaders( hdr ) ) ; + } + + return response ; +} + +} // end of namespace diff --git a/libgrive/src/protocol/AuthAgent.hh b/libgrive/src/protocol/AuthAgent.hh new file mode 100644 index 00000000..328e8cf4 --- /dev/null +++ b/libgrive/src/protocol/AuthAgent.hh @@ -0,0 +1,76 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "http/Agent.hh" +#include "OAuth2.hh" + +#include + +namespace gr { + +/*! \brief An HTTP agent with support OAuth2 + + This is a HTTP agent that provide support for OAuth2. It will also perform retries on + certain HTTP errors. +*/ +class AuthAgent : public http::Agent +{ +public : + AuthAgent( OAuth2& auth, http::Agent* real_agent ) ; + + http::ResponseLog* GetLog() const ; + void SetLog( http::ResponseLog *log ) ; + + long Request( + const std::string& method, + const std::string& url, + SeekStream *in, + DataStream *dest, + const http::Header& hdr, + u64_t downloadFileBytes = 0 ) ; + + std::string LastError() const ; + std::string LastErrorHeaders() const ; + + std::string RedirLocation() const ; + + std::string Escape( const std::string& str ) ; + std::string Unescape( const std::string& str ) ; + + void SetUploadSpeed( unsigned kbytes ) ; + void SetDownloadSpeed( unsigned kbytes ) ; + + void SetProgressReporter( Progress *progress ) ; + +private : + http::Header AppendHeader( const http::Header& hdr ) const ; + bool CheckRetry( long response ) ; + long CheckHttpResponse( + long response, + const std::string& url, + const http::Header& hdr ) ; + +private : + OAuth2& m_auth ; + http::Agent* m_agent ; +} ; + +} // end of namespace diff --git a/libgrive/src/protocol/Json.cc b/libgrive/src/protocol/Json.cc deleted file mode 100644 index 47ad94cd..00000000 --- a/libgrive/src/protocol/Json.cc +++ /dev/null @@ -1,366 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "Json.hh" - -#include "util/StdioFile.hh" - -#include -#include - -#include -#include -#include -#include - -namespace gr { - -Json::Json( ) : - m_json( ::json_object_new_object() ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json object" ) ) ; -} - -Json::Json( const char *str ) : - m_json( ::json_object_new_string( str ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( - Error() << expt::ErrMsg( "cannot create json string \"" + std::string(str) + "\"" ) ) ; -} - -template <> -Json::Json( const std::string& str ) : - m_json( ::json_object_new_string( str.c_str() ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( - Error() << expt::ErrMsg( "cannot create json string \"" + str + "\"" ) ) ; - - // paranoid check - assert( ::json_object_get_string( m_json ) == str ) ; -} - -template <> -Json::Json( const int& l ) : - m_json( ::json_object_new_int( l ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ; -} - -template <> -Json::Json( const long& l ) : - m_json( ::json_object_new_int( static_cast(l) ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ; -} - -template <> -Json::Json( const unsigned long& l ) : - m_json( ::json_object_new_int( static_cast(l) ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ; -} - -template <> -Json::Json( const std::vector& arr ) : - m_json( ::json_object_new_array( ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ; - - for ( std::vector::const_iterator i = arr.begin() ; i != arr.end() ; ++i ) - Add( *i ) ; -} - -template <> -Json::Json( const bool& b ) : - m_json( ::json_object_new_boolean( b ) ) -{ - if ( m_json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json bool" ) ) ; -} - -Json::Json( struct json_object *json, NotOwned ) : - m_json( json ) -{ - assert( m_json != 0 ) ; -} - -Json::Json( struct json_object *json ) : - m_json( json ) -{ - assert( json != 0 ) ; - ::json_object_get( m_json ) ; -} - -Json::Json( const Json& rhs ) : - m_json( rhs.m_json ) -{ - assert( m_json != 0 ) ; - ::json_object_get( m_json ) ; -} - -Json::~Json( ) -{ - assert( m_json != 0 ) ; - if ( m_json != 0 ) - ::json_object_put( m_json ) ; -} - -Json& Json::operator=( const Json& rhs ) -{ - Json tmp( rhs ) ; - Swap( tmp ) ; - return *this ; -} - -void Json::Swap( Json& other ) -{ - assert( m_json != 0 ) ; - assert( other.m_json != 0 ) ; - std::swap( m_json, other.m_json ) ; -} - -Json Json::operator[]( const std::string& key ) const -{ - assert( m_json != 0 ) ; - - struct json_object *j = ::json_object_object_get( m_json, key.c_str() ) ; - if ( j == 0 ) - BOOST_THROW_EXCEPTION( - Error() - << expt::ErrMsg( "key: " + key + " is not found in object" ) - << JsonInfo( *this ) ) ; - - return Json( j ) ; -} - -Json Json::operator[]( const std::size_t& idx ) const -{ - assert( m_json != 0 ) ; - - struct json_object *j = ::json_object_array_get_idx( m_json, idx ) ; - if ( j == 0 ) - { - std::ostringstream ss ; - ss << "index " << idx << " is not found in array" ; - BOOST_THROW_EXCEPTION( - Error() - << expt::ErrMsg( ss.str() ) - << JsonInfo( *this ) ) ; - } - - return Json( j ) ; -} - -bool Json::Has( const std::string& key ) const -{ - assert( m_json != 0 ) ; - return ::json_object_object_get( m_json, key.c_str() ) != 0 ; -} - -bool Json::Get( const std::string& key, Json& json ) const -{ - assert( m_json != 0 ) ; - struct json_object *j = ::json_object_object_get( m_json, key.c_str() ) ; - if ( j != 0 ) - { - Json tmp( j, NotOwned() ) ; - json.Swap( tmp ) ; - return true ; - } - else - return false ; -} - -void Json::Add( const std::string& key, const Json& json ) -{ - assert( m_json != 0 ) ; - assert( json.m_json != 0 ) ; - - ::json_object_get( json.m_json ) ; - ::json_object_object_add( m_json, key.c_str(), json.m_json ) ; -} - -void Json::Add( const Json& json ) -{ - assert( m_json != 0 ) ; - assert( json.m_json != 0 ) ; - - ::json_object_get( json.m_json ) ; - ::json_object_array_add( m_json, json.m_json ) ; -} - -bool Json::Bool() const -{ - assert( m_json != 0 ) ; - return ::json_object_get_boolean( m_json ) ; -} - -template <> -bool Json::Is() const -{ - assert( m_json != 0 ) ; - return ::json_object_is_type( m_json, json_type_boolean ) ; -} - -std::string Json::Str() const -{ - assert( m_json != 0 ) ; - return ::json_object_get_string( m_json ) ; -} - -template <> -bool Json::Is() const -{ - assert( m_json != 0 ) ; - return ::json_object_is_type( m_json, json_type_string ) ; -} - -int Json::Int() const -{ - assert( m_json != 0 ) ; - return ::json_object_get_int( m_json ) ; -} - -template <> -bool Json::Is() const -{ - assert( m_json != 0 ) ; - return ::json_object_is_type( m_json, json_type_int ) ; -} - -std::ostream& operator<<( std::ostream& os, const Json& json ) -{ - assert( json.m_json != 0 ) ; - return os << ::json_object_to_json_string( json.m_json ) ; -} - -void Json::Write( StdioFile& file ) const -{ - const char *str = ::json_object_to_json_string( m_json ) ; - file.Write( str, std::strlen(str) ) ; -} - -Json::Type Json::DataType() const -{ - assert( m_json != 0 ) ; - return static_cast( ::json_object_get_type( m_json ) ) ; -} - -Json::Object Json::AsObject() const -{ - Object result ; - - json_object_object_foreach( m_json, key, val ) - { - result.insert( Object::value_type( key, Json( val ) ) ) ; - } - - return result ; -} - -template <> -bool Json::Is() const -{ - assert( m_json != 0 ) ; - return ::json_object_is_type( m_json, json_type_object ) ; -} - -Json::Array Json::AsArray() const -{ - std::size_t count = ::json_object_array_length( m_json ) ; - Array result ; - - for ( std::size_t i = 0 ; i < count ; ++i ) - result.push_back( Json( ::json_object_array_get_idx( m_json, i ) ) ) ; - - return result ; -} - -template <> -bool Json::Is() const -{ - assert( m_json != 0 ) ; - return ::json_object_is_type( m_json, json_type_array ) ; -} - -Json Json::FindInArray( const std::string& key, const std::string& value ) const -{ - std::size_t count = ::json_object_array_length( m_json ) ; - - for ( std::size_t i = 0 ; i < count ; ++i ) - { - Json item( ::json_object_array_get_idx( m_json, i ) ) ; - if ( item.Has(key) && item[key].Str() == value ) - return item ; - } - BOOST_THROW_EXCEPTION( - Error() << expt::ErrMsg( "cannot find " + key + " = " + value + " in array" ) ) ; -} - -bool Json::FindInArray( const std::string& key, const std::string& value, Json& result ) const -{ - try - { - result = FindInArray( key, value ) ; - return true ; - } - catch ( Error& ) - { - return false ; - } -} - -Json Json::Parse( const std::string& str ) -{ - struct json_object *json = ::json_tokener_parse( str.c_str() ) ; - if ( json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "json parse error" ) ) ; - - return Json( json, NotOwned() ) ; -} - -Json Json::ParseFile( const std::string& filename ) -{ - StdioFile file( filename ) ; - struct json_tokener *tok = ::json_tokener_new() ; - - struct json_object *json = 0 ; - - char buf[1024] ; - std::size_t count = 0 ; - - while ( (count = file.Read( buf, sizeof(buf) ) ) > 0 ) - json = ::json_tokener_parse_ex( tok, buf, count ) ; - - if ( json == 0 ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( ::json_tokener_errors[tok->err] ) ) ; - - ::json_tokener_free( tok ) ; - - return Json( json, NotOwned() ) ; -} - -} diff --git a/libgrive/src/protocol/Json.hh b/libgrive/src/protocol/Json.hh deleted file mode 100644 index 034a4fc7..00000000 --- a/libgrive/src/protocol/Json.hh +++ /dev/null @@ -1,96 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#pragma once - -#include "util/Exception.hh" - -#include -#include -#include - -struct json_object ; - -namespace gr { - -class StdioFile ; - -class Json -{ -public : - typedef std::map Object ; - typedef std::vector Array ; - - struct Error : virtual Exception {} ; - typedef boost::error_info JsonInfo ; - -public : - template - explicit Json( const T& val ) ; - - Json() ; - Json( const Json& rhs ) ; - Json( const char *str ) ; - ~Json( ) ; - - static Json Parse( const std::string& str ) ; - static Json ParseFile( const std::string& filename ) ; - - Json operator[]( const std::string& key ) const ; - Json operator[]( const std::size_t& idx ) const ; - Json& operator=( const Json& rhs ) ; - - void Swap( Json& other ) ; - - std::string Str() const ; - int Int() const ; - double Double() const ; - bool Bool() const ; - Array AsArray() const ; - Object AsObject() const ; - - template - bool Is() const ; - - bool Has( const std::string& key ) const ; - bool Get( const std::string& key, Json& json ) const ; - void Add( const std::string& key, const Json& json ) ; - void Add( const Json& json ) ; - Json FindInArray( const std::string& key, const std::string& value ) const ; - bool FindInArray( const std::string& key, const std::string& value, Json& result ) const ; - - friend std::ostream& operator<<( std::ostream& os, const Json& json ) ; - void Write( StdioFile& file ) const ; - - enum Type { null_type, bool_type, double_type, int_type, object_type, array_type, string_type } ; - - Type DataType() const ; - -private : - Json( struct json_object *json ) ; - - struct NotOwned {} ; - Json( struct json_object *json, NotOwned ) ; - -private : -public : - struct json_object *m_json ; -} ; - -} \ No newline at end of file diff --git a/libgrive/src/protocol/OAuth2.cc b/libgrive/src/protocol/OAuth2.cc index 5d2c1727..73ae0886 100644 --- a/libgrive/src/protocol/OAuth2.cc +++ b/libgrive/src/protocol/OAuth2.cc @@ -19,8 +19,7 @@ #include "OAuth2.hh" -#include "JsonResponse.hh" -#include "Json.hh" +#include "json/ValResponse.hh" #include "http/CurlAgent.hh" #include "http/Header.hh" @@ -34,10 +33,12 @@ namespace gr { const std::string token_url = "https://accounts.google.com/o/oauth2/token" ; OAuth2::OAuth2( + http::Agent* agent, const std::string& refresh_code, const std::string& client_id, const std::string& client_secret ) : m_refresh( refresh_code ), + m_agent( agent ), m_client_id( client_id ), m_client_secret( client_secret ) { @@ -45,8 +46,10 @@ OAuth2::OAuth2( } OAuth2::OAuth2( + http::Agent* agent, const std::string& client_id, const std::string& client_secret ) : + m_agent( agent ), m_client_id( client_id ), m_client_secret( client_secret ) { @@ -61,33 +64,35 @@ void OAuth2::Auth( const std::string& auth_code ) "&redirect_uri=" + "urn:ietf:wg:oauth:2.0:oob" + "&grant_type=authorization_code" ; - http::JsonResponse resp ; - http::CurlAgent http ; - - DisableLog dlog( log::debug ) ; - http.Post( token_url, post, &resp, http::Header() ) ; - - Json jresp = resp.Response() ; - m_access = jresp["access_token"].Str() ; - m_refresh = jresp["refresh_token"].Str() ; + http::ValResponse resp ; + + long code = m_agent->Post( token_url, post, &resp, http::Header() ) ; + if ( code >= 200 && code < 300 ) + { + Val jresp = resp.Response() ; + m_access = jresp["access_token"].Str() ; + m_refresh = jresp["refresh_token"].Str() ; + } + else + { + Log( "Failed to obtain auth token: HTTP %1%, body: %2%", + code, m_agent->LastError(), log::error ) ; + BOOST_THROW_EXCEPTION( AuthFailed() ); + } } -std::string OAuth2::MakeAuthURL( - const std::string& client_id, - const std::string& state ) +std::string OAuth2::MakeAuthURL() { - http::CurlAgent h ; - return "https://accounts.google.com/o/oauth2/auth" "?scope=" + - h.Escape( "https://www.googleapis.com/auth/userinfo.email" ) + "+" + - h.Escape( "https://www.googleapis.com/auth/userinfo.profile" ) + "+" + - h.Escape( "https://docs.google.com/feeds/" ) + "+" + - h.Escape( "https://docs.googleusercontent.com/" ) + "+" + - h.Escape( "https://spreadsheets.google.com/feeds/" ) + + m_agent->Escape( "https://www.googleapis.com/auth/userinfo.email" ) + "+" + + m_agent->Escape( "https://www.googleapis.com/auth/userinfo.profile" ) + "+" + + m_agent->Escape( "https://docs.google.com/feeds/" ) + "+" + + m_agent->Escape( "https://docs.googleusercontent.com/" ) + "+" + + m_agent->Escape( "https://spreadsheets.google.com/feeds/" ) + "&redirect_uri=urn:ietf:wg:oauth:2.0:oob" "&response_type=code" - "&client_id=" + client_id ; + "&client_id=" + m_client_id ; } void OAuth2::Refresh( ) @@ -98,13 +103,18 @@ void OAuth2::Refresh( ) "&client_secret=" + m_client_secret + "&grant_type=refresh_token" ; - http::JsonResponse resp ; - http::CurlAgent http ; - - DisableLog dlog( log::debug ) ; - http.Post( token_url, post, &resp, http::Header() ) ; + http::ValResponse resp ; + + long code = m_agent->Post( token_url, post, &resp, http::Header() ) ; - m_access = resp.Response()["access_token"].Str() ; + if ( code >= 200 && code < 300 ) + m_access = resp.Response()["access_token"].Str() ; + else + { + Log( "Failed to refresh auth token: HTTP %1%, body: %2%", + code, m_agent->LastError(), log::error ) ; + BOOST_THROW_EXCEPTION( AuthFailed() ); + } } std::string OAuth2::RefreshToken( ) const diff --git a/libgrive/src/protocol/OAuth2.hh b/libgrive/src/protocol/OAuth2.hh index e8713f3e..e9a23dae 100644 --- a/libgrive/src/protocol/OAuth2.hh +++ b/libgrive/src/protocol/OAuth2.hh @@ -19,42 +19,49 @@ #pragma once +#include "http/Agent.hh" +#include "util/Exception.hh" #include +#include namespace gr { class OAuth2 { +public : + struct AuthFailed : virtual Exception {} ; + public : OAuth2( + http::Agent* agent, const std::string& client_id, const std::string& client_secret ) ; OAuth2( + http::Agent* agent, const std::string& refresh_code, const std::string& client_id, const std::string& client_secret ) ; std::string Str() const ; - - static std::string MakeAuthURL( - const std::string& client_id, - const std::string& state = std::string() ) ; + + std::string MakeAuthURL() ; void Auth( const std::string& auth_code ) ; void Refresh( ) ; - + std::string RefreshToken( ) const ; std::string AccessToken( ) const ; - + // adding HTTP auth header std::string HttpHeader( ) const ; - + private : std::string m_access ; std::string m_refresh ; + http::Agent* m_agent ; const std::string m_client_id ; const std::string m_client_secret ; } ; - + } // end of namespace diff --git a/libgrive/src/util/ConcatStream.cc b/libgrive/src/util/ConcatStream.cc new file mode 100644 index 00000000..d93df5e0 --- /dev/null +++ b/libgrive/src/util/ConcatStream.cc @@ -0,0 +1,105 @@ +/* + A stream representing a concatenation of several underlying streams + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include +#include "ConcatStream.hh" + +namespace gr { + +ConcatStream::ConcatStream() : + m_size( 0 ), m_pos( 0 ), m_cur( 0 ) +{ +} + +std::size_t ConcatStream::Read( char *data, std::size_t size ) +{ + std::size_t done = 0, l; + while ( done < size && m_cur < m_streams.size() ) + { + l = m_streams[m_cur]->Read( data+done, ( size-done < m_sizes[m_cur]-m_pos ? size-done : m_sizes[m_cur]-m_pos ) ); + done += l; + m_pos += l; + if ( !l ) + { + // current stream was truncated in the meantime, pad output with zeros + memset( data+done, 0, m_sizes[m_cur]-m_pos ); + done += m_sizes[m_cur]-m_pos; + m_pos = m_sizes[m_cur]; + } + if ( m_pos >= m_sizes[m_cur] ) + { + m_cur++; + if ( m_cur < m_streams.size() ) + m_streams[m_cur]->Seek( 0, 0 ); + } + } + return done ; +} + +std::size_t ConcatStream::Write( const char *data, std::size_t size ) +{ + return 0 ; +} + +off_t ConcatStream::Seek( off_t offset, int whence ) +{ + if ( whence == 1 ) + offset += m_pos; + else if ( whence == 2 ) + offset += Size(); + if ( (u64_t)offset > Size() ) + offset = Size(); + m_cur = 0; + m_pos = offset; + if ( m_streams.size() ) + { + while ( (u64_t)offset > m_sizes[m_cur] ) + m_cur++; + m_streams[m_cur]->Seek( offset - ( m_cur > 0 ? m_sizes[m_cur-1] : 0 ), 0 ); + } + return m_pos ; +} + +off_t ConcatStream::Tell() const +{ + return m_pos ; +} + +u64_t ConcatStream::Size() const +{ + return m_size ; +} + +void ConcatStream::Append( SeekStream *stream ) +{ + if ( stream ) + { + u64_t size = stream->Size(); + if ( size > 0 ) + { + // "fix" stream size at the moment of adding so further changes of underlying files + // don't affect the total size of resulting stream... + m_size += size; + m_streams.push_back( stream ); + m_sizes.push_back( m_size ); + } + } +} + +} // end of namespace diff --git a/libgrive/src/util/ConcatStream.hh b/libgrive/src/util/ConcatStream.hh new file mode 100644 index 00000000..0966f060 --- /dev/null +++ b/libgrive/src/util/ConcatStream.hh @@ -0,0 +1,49 @@ +/* + A stream representing a concatenation of several underlying streams + Copyright (C) 2015 Vitaliy Filippov + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "DataStream.hh" + +#include + +namespace gr { + +class ConcatStream : public SeekStream +{ +public : + ConcatStream() ; + + std::size_t Read( char *data, std::size_t size ) ; + std::size_t Write( const char *data, std::size_t size ) ; + + off_t Seek( off_t offset, int whence ) ; + off_t Tell() const ; + u64_t Size() const ; + + void Append( SeekStream *stream ) ; + +private : + std::vector m_streams ; + std::vector m_sizes ; + u64_t m_size, m_pos ; + size_t m_cur ; +} ; + +} // end of namespace diff --git a/libgrive/src/util/Config.cc b/libgrive/src/util/Config.cc new file mode 100644 index 00000000..ba0c91e3 --- /dev/null +++ b/libgrive/src/util/Config.cc @@ -0,0 +1,115 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Config.hh" + +#include "util/File.hh" +#include "json/JsonWriter.hh" +#include "json/JsonParser.hh" + +#include + +#include +#include + +namespace po = boost::program_options; + +namespace gr { + +const std::string default_filename = ".grive"; +const char *env_name = "GR_CONFIG"; +const std::string default_root_folder = "."; + +Config::Config( const po::variables_map& vm ) +{ + m_cmd.Add( "new-rev", Val(vm.count("new-rev") > 0) ) ; + m_cmd.Add( "force", Val(vm.count("force") > 0 ) ) ; + m_cmd.Add( "path", Val(vm.count("path") > 0 + ? vm["path"].as() + : default_root_folder ) ) ; + m_cmd.Add( "dir", Val(vm.count("dir") > 0 + ? vm["dir"].as() + : "" ) ) ; + if ( vm.count( "ignore" ) > 0 ) + m_cmd.Add( "ignore", Val( vm["ignore"].as() ) ); + m_cmd.Add( "no-remote-new", Val( vm.count( "no-remote-new" ) > 0 || vm.count( "upload-only" ) > 0 ) ); + m_cmd.Add( "upload-only", Val( vm.count( "upload-only" ) > 0 ) ); + m_cmd.Add( "no-delete-remote", Val( vm.count( "no-delete-remote" ) > 0 ) ); + + m_path = GetPath( fs::path(m_cmd["path"].Str()) ) ; + m_file = Read( ) ; +} + +fs::path Config::GetPath( const fs::path& root_path ) +{ + // config file will be (in order of preference) + // value specified in environment string + // value specified in defaultConfigFileName in path from commandline --path + // value specified in defaultConfigFileName in current directory + const char *env = ::getenv( env_name ) ; + return root_path / (env ? env : default_filename) ; +} + +const fs::path Config::Filename() const +{ + return m_path ; +} + +void Config::Save( ) +{ + gr::File file( m_path.string(), 0600 ) ; + JsonWriter wr( &file ) ; + m_file.Visit( &wr ) ; +} + +void Config::Set( const std::string& key, const Val& value ) +{ + m_file.Add( key, value ) ; +} + +Val Config::Get( const std::string& key ) const +{ + return m_cmd.Has(key) ? m_cmd[key] : m_file[key] ; +} + +Val Config::GetAll() const +{ + Val::Object obj = m_file.AsObject() ; + Val::Object cmd_obj = m_cmd.AsObject() ; + + for ( Val::Object::iterator i = cmd_obj.begin() ; i != cmd_obj.end() ; ++i ) + obj[i->first] = i->second ; + + return Val( obj ) ; +} + +Val Config::Read() +{ + try + { + gr::File file(m_path) ; + return ParseJson( file ) ; + } + catch ( Exception& e ) + { + return Val() ; + } +} + +} // end of namespace diff --git a/libgrive/src/util/StdioFile.hh b/libgrive/src/util/Config.hh similarity index 59% rename from libgrive/src/util/StdioFile.hh rename to libgrive/src/util/Config.hh index 30de2fb2..9a654c56 100644 --- a/libgrive/src/util/StdioFile.hh +++ b/libgrive/src/util/Config.hh @@ -21,40 +21,47 @@ #include "Exception.hh" #include "FileSystem.hh" +#include "json/Val.hh" -#include +namespace boost +{ + namespace program_options + { + class variables_map ; + } +} namespace gr { -class StdioFile +class Config { public : struct Error : virtual Exception {} ; + typedef boost::error_info File ; -public : - StdioFile() ; - StdioFile( const fs::path& path ) ; - StdioFile( const fs::path& path, int mode ) ; - ~StdioFile( ) ; - - void OpenForRead( const fs::path& path ) ; - void OpenForWrite( const fs::path& path, int mode = 0600 ) ; - void Close() ; - bool IsOpened() const ; - - std::size_t Read( void *ptr, std::size_t size ) ; - std::size_t Write( const void *ptr, std::size_t size ) ; + Config( const boost::program_options::variables_map& vm ) ; - long Seek( long offset, int whence ) ; - long Tell() const ; + const fs::path Filename() const ; - void Chmod( int mode ) ; + void Set( const std::string& key, const Val& value ) ; + Val Get( const std::string& key ) const ; + + Val GetAll() const ; + void Save() ; private : - void Open( const fs::path& path, int flags, int mode ) ; - + Val Read( ) ; + static fs::path GetPath( const fs::path& root_path ) ; + private : - int m_fd ; + //! config file path + fs::path m_path; + + //! config values loaded from config file + Val m_file ; + + //! config values from command line + Val m_cmd ; } ; } // end of namespace diff --git a/libgrive/src/util/Crypt.cc b/libgrive/src/util/Crypt.cc index 6b9ca329..b34494f9 100644 --- a/libgrive/src/util/Crypt.cc +++ b/libgrive/src/util/Crypt.cc @@ -19,11 +19,11 @@ #include "Crypt.hh" -#include "StdioFile.hh" +#include "File.hh" #include "Exception.hh" +#include "MemMap.hh" #include -#include // dependent libraries #include @@ -31,7 +31,8 @@ namespace gr { namespace crypt { -const std::size_t read_size = 8 * 1024 ; +// map 4MB of data at a time +const u64_t read_size = 1024 * 4096 ; struct MD5::Impl { @@ -43,7 +44,10 @@ MD5::MD5() : m_impl( new Impl ) ::gcry_error_t err = ::gcry_md_open( &m_impl->hd, GCRY_MD_MD5, 0 ) ; if ( err != GPG_ERR_NO_ERROR ) { - BOOST_THROW_EXCEPTION( Exception() << expt::ErrMsg( ::gcry_strerror(err) ) ) ; + BOOST_THROW_EXCEPTION( Exception() + << GCryptErr_( ::gcry_strerror(err) ) + << GCryptApi_( "gcry_md_open" ) + ) ; } } @@ -74,24 +78,25 @@ std::string MD5::Get( const fs::path& file ) { try { - StdioFile sfile( file ) ; + File sfile( file ) ; return Get( sfile ) ; } - catch ( StdioFile::Error& ) + catch ( File::Error& ) { return "" ; } } -std::string MD5::Get( StdioFile& file ) +std::string MD5::Get( File& file ) { - char buf[read_size] ; - MD5 crypt ; - std::size_t count = 0 ; - while ( (count = file.Read( buf, sizeof(buf) )) > 0 ) - crypt.Write( buf, count ) ; + u64_t size = file.Size() ; + for ( u64_t i = 0 ; i < size ; i += read_size ) + { + MemMap map( file, i, static_cast(std::min(read_size, size-i)) ) ; + crypt.Write( map.Addr(), map.Length() ) ; + } return crypt.Get() ; } diff --git a/libgrive/src/util/Crypt.hh b/libgrive/src/util/Crypt.hh index 6126b53c..a9558096 100644 --- a/libgrive/src/util/Crypt.hh +++ b/libgrive/src/util/Crypt.hh @@ -19,6 +19,8 @@ #pragma once +#include "util/Exception.hh" + #include #include @@ -26,17 +28,21 @@ namespace gr { -class StdioFile ; +class File ; namespace crypt { class MD5 { +public : + typedef boost::error_info GCryptErr_ ; + typedef boost::error_info GCryptApi_ ; + public : MD5() ; ~MD5() ; - static std::string Get( StdioFile& file ) ; + static std::string Get( File& file ) ; static std::string Get( const boost::filesystem::path& file ) ; void Write( const void *data, std::size_t size ) ; @@ -44,7 +50,7 @@ public : private : struct Impl ; - std::auto_ptr m_impl ; + std::unique_ptr m_impl ; } ; } } // end of namespace gr diff --git a/libgrive/src/util/DataStream.hh b/libgrive/src/util/DataStream.hh new file mode 100644 index 00000000..1fd06095 --- /dev/null +++ b/libgrive/src/util/DataStream.hh @@ -0,0 +1,60 @@ +/* + webwrite: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include +#include "util/Types.hh" + +namespace gr { + +/** \brief Encapsulation of data streams. Useful for unit tests. + This class provides two functions: Read() and Write(). +*/ +class DataStream +{ +protected : + virtual ~DataStream() {} + +public : + /** Reading from the stream. The caller indicates that it wants + to read `size` bytes and must provide enough space pointed + by `data`. + \param data Buffer to hold the data read from the stream + Must have at least `size` bytes. + \param size Number of bytes the caller wants to read. + \throw wb::Exception In case of any error. + \return The number of byte actually read from the stream. + 0 indicates the end of stream, i.e. you will + still get 0 if you call again. + */ + virtual std::size_t Read( char *data, std::size_t size ) = 0 ; + virtual std::size_t Write( const char *data, std::size_t size ) = 0 ; +} ; + +class SeekStream: public DataStream +{ +public : + virtual off_t Seek( off_t offset, int whence ) = 0 ; + virtual off_t Tell() const = 0 ; + virtual u64_t Size() const = 0 ; +} ; + + +} // end of namespace diff --git a/libgrive/src/util/DateTime.cc b/libgrive/src/util/DateTime.cc index ffb4ec57..2fe79891 100644 --- a/libgrive/src/util/DateTime.cc +++ b/libgrive/src/util/DateTime.cc @@ -33,7 +33,6 @@ #include #include #include -#include #include #include diff --git a/libgrive/src/util/Exception.cc b/libgrive/src/util/Exception.cc index 68738b44..a4652f69 100644 --- a/libgrive/src/util/Exception.cc +++ b/libgrive/src/util/Exception.cc @@ -17,7 +17,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "util/Exception.hh" +#include "Exception.hh" #include "bfd/Backtrace.hh" #include "bfd/Debug.hh" @@ -26,7 +26,6 @@ #include #include -#include namespace gr { @@ -35,8 +34,13 @@ class Backtrace ; Exception::Exception( ) { #ifdef HAVE_BFD - *this << expt::BacktraceInfo( Backtrace() ) ; + *this << expt::Backtrace_( Backtrace() ) ; #endif } +const char* Exception::what() const throw() +{ + return boost::diagnostic_information_what( *this ) ; +} + } // end of namespace diff --git a/libgrive/src/util/Exception.hh b/libgrive/src/util/Exception.hh index fec5af71..f1515841 100644 --- a/libgrive/src/util/Exception.hh +++ b/libgrive/src/util/Exception.hh @@ -1,5 +1,5 @@ /* - grive: an GPL program to sync a local directory with Google Drive + webwrite: an GPL program to sync a local directory with Google Drive Copyright (C) 2012 Wan Wai Ho This program is free software; you can redistribute it and/or @@ -32,47 +32,25 @@ class Backtrace ; /** \defgroup exception Exception Classes */ -/// base class for exception in libpdfdoc -/** \ingroup exception - This class is the base class for all exception class in libpdfdoc. +/** \brief base class for exception in WebWrite + \ingroup exception + This class is the base class for all exception class in WebWrite. + It allows us to catch all WebWrite exception with one catch clause. */ struct Exception : virtual public std::exception, virtual public boost::exception { Exception( ) ; + + virtual const char* what() const throw() ; } ; -struct FileError : virtual Exception {} ; - -/// Parse error exception. -/** \ingroup exception - This exception will be thrown when there is a parse error when reading - a PDF file. -*/ -struct ParseError : virtual Exception {} ; - -/// Invalid type exception. -/** \ingroup exception - This exception will be thrown when the Object cannot convert its - underlying data to a specific type. The what() member function will - describe the expected and actual type of the data. -*/ -struct BadType : virtual Exception {} ; - -struct Unsupported : virtual Exception {} ; - -// Exception informations +/// Exception informations namespace expt { // back-trace information. should be present for all exceptions - typedef boost::error_info BacktraceInfo ; - - // generic error message - typedef boost::error_info ErrMsg ; - - // nested exception - typedef boost::error_info Nested ; + typedef boost::error_info Backtrace_ ; } } // end of namespace diff --git a/libgrive/src/util/File.cc b/libgrive/src/util/File.cc new file mode 100644 index 00000000..375cd625 --- /dev/null +++ b/libgrive/src/util/File.cc @@ -0,0 +1,284 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "File.hh" + +#include + +// boost headers +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef WIN32 + #include + typedef int ssize_t ; +#else + #include +#endif + +// local functions +namespace { + +using namespace gr ; + +off_t LSeek( int fd, off_t offset, int whence ) +{ + assert( fd >= 0 ) ; + + off_t r = ::lseek( fd, offset, whence ) ; + if ( r == static_cast(-1) ) + { + BOOST_THROW_EXCEPTION( + File::Error() + << boost::errinfo_api_function("lseek") + << boost::errinfo_errno(errno) + ) ; + } + + return r ; +} + +struct stat FStat( int fd ) +{ + struct stat s = {} ; + if ( ::fstat( fd, &s ) != 0 ) + { + BOOST_THROW_EXCEPTION( + File::Error() + << boost::errinfo_api_function("fstat") + << boost::errinfo_errno(errno) + ) ; + } + return s ; +} + +} // end of local functions + +namespace gr { + +File::File( ) : m_fd( -1 ) +{ +} + +/** Opens the file for reading. + \param path Path to the file to be opened. + \throw Error When the file cannot be openned. +*/ +File::File( const fs::path& path ) : m_fd( -1 ) +{ + OpenForRead( path ) ; +} + +/** Opens the file for writing. + \param path Path to the file to be opened. + \param mode Mode of the file to be created, e.g. 0600 for user + readable/writable. + \throw Error When the file cannot be opened. +*/ +File::File( const fs::path& path, int mode ) : m_fd( -1 ) +{ + OpenForWrite( path, mode ) ; +} + +/** The destructor will close the file. +*/ +File::~File( ) +{ + Close() ; +} + +void File::Open( const fs::path& path, int flags, int mode ) +{ + if ( IsOpened() ) + Close() ; + + assert( m_fd == -1 ) ; + m_fd = ::open( path.string().c_str(), flags, mode ) ; + if ( m_fd == -1 ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("open") + << boost::errinfo_errno(errno) + << boost::errinfo_file_name(path.string()) + ) ; + } +} + +void File::OpenForRead( const fs::path& path ) +{ + int flags = O_RDONLY ; +#ifdef WIN32 + flags |= O_BINARY ; +#endif + Open( path, flags, 0 ) ; +} + +void File::OpenForWrite( const fs::path& path, int mode ) +{ + int flags = O_CREAT|O_RDWR|O_TRUNC ; +#ifdef WIN32 + flags |= O_BINARY ; +#endif + Open( path, flags, mode ) ; +} + +void File::Close() +{ + if ( IsOpened() ) + { + close( m_fd ) ; + m_fd = -1 ; + } +} + +bool File::IsOpened() const +{ + return m_fd != -1 ; +} + +/** Read bytes from file. See DataStream::Read() for details. + \throw Error In case of any error. +*/ +std::size_t File::Read( char *ptr, std::size_t size ) +{ + assert( IsOpened() ) ; + ssize_t count = ::read( m_fd, ptr, size ) ; + if ( count == -1 ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("read") + << boost::errinfo_errno(errno) + ) ; + } + return count ; +} + +std::size_t File::Write( const char *ptr, std::size_t size ) +{ + assert( IsOpened() ) ; + ssize_t count = ::write( m_fd, ptr, size ) ; + if ( count == -1 ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("write") + << boost::errinfo_errno(errno) + ) ; + } + return count ; +} + +off_t File::Seek( off_t offset, int whence ) +{ + assert( IsOpened() ) ; + return LSeek( m_fd, offset, whence ) ; +} + +off_t File::Tell() const +{ + assert( IsOpened() ) ; + return LSeek( m_fd, 0, SEEK_CUR ) ; +} + +u64_t File::Size() const +{ + assert( IsOpened() ) ; + + struct stat s = FStat(m_fd) ; + + assert( s.st_size >= 0 ) ; + return static_cast( s.st_size ) ; +} + +void File::Chmod( int mode ) +{ + assert( IsOpened() ) ; +#ifndef WIN32 + if ( ::fchmod( m_fd, mode ) != 0 ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("fchmod") + << boost::errinfo_errno(errno) + ) ; + } +#endif +} + +/// This function is not implemented in win32 yet. +void* File::Map( off_t offset, std::size_t length ) +{ + assert( IsOpened() ) ; + +#ifdef WIN32 + assert( false ) ; + return 0 ; +#else + void *addr = ::mmap( 0, length, PROT_READ, MAP_PRIVATE, m_fd, offset ) ; + if ( addr == reinterpret_cast( -1 ) ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("mmap") + << boost::errinfo_errno(errno) + ) ; + } + return addr ; +#endif +} + +void File::UnMap( void *addr, std::size_t length ) +{ +#ifndef WIN32 + if ( ::munmap( addr, length ) != 0 ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("munmap") + << boost::errinfo_errno(errno) + ) ; + } +#endif +} + +struct stat File::Stat() const +{ + struct stat result = {} ; + if ( ::fstat( m_fd, &result ) != 0 ) + { + BOOST_THROW_EXCEPTION( + Error() + << boost::errinfo_api_function("fstat") + << boost::errinfo_errno(errno) + ) ; + } + return result ; +} + +} // end of namespace diff --git a/libgrive/src/util/File.hh b/libgrive/src/util/File.hh new file mode 100644 index 00000000..d756f484 --- /dev/null +++ b/libgrive/src/util/File.hh @@ -0,0 +1,82 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "DataStream.hh" +#include "Exception.hh" +#include "FileSystem.hh" +#include "Types.hh" + +#include + +struct stat ; + +namespace gr { + +/** \brief A wrapper class for file read/write. + + It is a simple wrapper around the UNIX file descriptor. It will + throw exceptions (i.e. Error) when it encounters errors. +*/ +class File : public SeekStream +{ +public : + /// File specific errors. It often includes + /// boost::errinfo_api_function and boost::errinfo_errno for the + /// detail information. + struct Error : virtual Exception {} ; + +public : + File() ; + File( const fs::path& path ) ; + File( const fs::path& path, int mode ) ; + ~File( ) ; + + File( const File& rhs ) ; + File& operator=( const File& rhs ) ; + void Swap( File& other ) ; + + void OpenForRead( const fs::path& path ) ; + void OpenForWrite( const fs::path& path, int mode = 0600 ) ; + void Close() ; + bool IsOpened() const ; + + std::size_t Read( char *ptr, std::size_t size ) ; + std::size_t Write( const char *ptr, std::size_t size ) ; + + off_t Seek( off_t offset, int whence ) ; + off_t Tell() const ; + u64_t Size() const ; + + void Chmod( int mode ) ; + + void* Map( off_t offset, std::size_t length ) ; + static void UnMap( void *addr, std::size_t length ) ; + + struct stat Stat() const ; + +private : + void Open( const fs::path& path, int flags, int mode ) ; + +private : + int m_fd ; +} ; + +} // end of namespace diff --git a/libgrive/src/util/FileSystem.hh b/libgrive/src/util/FileSystem.hh index f6c18acd..9eb02d3b 100644 --- a/libgrive/src/util/FileSystem.hh +++ b/libgrive/src/util/FileSystem.hh @@ -25,17 +25,4 @@ namespace gr { namespace fs = boost::filesystem ; - - // these two functions are for ancient distro which does not have boost v1.44 or later - // will be removed once people upgrade - - inline std::string Path2Str( const fs::path& p ) - { - return p.string() ; - } - - inline std::string Path2Str( const std::string& s ) - { - return s ; - } } diff --git a/libgrive/src/util/Function.hh b/libgrive/src/util/Function.hh index 197418f0..d5d91147 100644 --- a/libgrive/src/util/Function.hh +++ b/libgrive/src/util/Function.hh @@ -178,7 +178,7 @@ public : private : typedef impl::FuncImpl Impl ; - std::auto_ptr m_pimpl ; + std::unique_ptr m_pimpl ; } ; } // end of namespace diff --git a/libgrive/src/protocol/JsonResponse.hh b/libgrive/src/util/MemMap.cc similarity index 67% rename from libgrive/src/protocol/JsonResponse.hh rename to libgrive/src/util/MemMap.cc index 3adae2da..12b1f780 100644 --- a/libgrive/src/protocol/JsonResponse.hh +++ b/libgrive/src/util/MemMap.cc @@ -17,30 +17,30 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#pragma once +#include "MemMap.hh" +#include "File.hh" -#include "http/Receivable.hh" -#include "http/StringResponse.hh" +namespace gr { -namespace gr +MemMap::MemMap( File& file, off_t offset, std::size_t length ) : + m_addr ( file.Map( offset, length ) ), + m_length( length ) { - class Json ; } -namespace gr { namespace http { - -class JsonResponse : public Receivable +MemMap::~MemMap() { -public : - JsonResponse() ; - - std::size_t OnData( void *data, std::size_t count ) ; - void Clear() ; - - Json Response() const ; + File::UnMap( m_addr, m_length ) ; +} -private : - StringResponse m_resp ; -} ; +void* MemMap::Addr() const +{ + return m_addr ; +} + +std::size_t MemMap::Length() const +{ + return m_length ; +} -} } // end of namespace +} // end of namespace diff --git a/grive/src/Config.hh b/libgrive/src/util/MemMap.hh similarity index 75% rename from grive/src/Config.hh rename to libgrive/src/util/MemMap.hh index 11188fa9..b8ca5a26 100644 --- a/grive/src/Config.hh +++ b/libgrive/src/util/MemMap.hh @@ -19,29 +19,29 @@ #pragma once -#include "util/Exception.hh" -#include "protocol/Json.hh" +#include "Exception.hh" +#include "Types.hh" +#include namespace gr { -class Config +class File ; + +class MemMap { public : struct Error : virtual Exception {} ; - typedef boost::error_info File ; - - static const std::string& Filename() ; - Config() ; +public : + MemMap( File& file, off_t offset, std::size_t length ) ; + ~MemMap() ; - Json& Get() ; - void Save() ; + void* Addr() const ; + std::size_t Length() const ; private : - Json Read( const std::string& filename ) ; - -private : - Json m_cfg ; + void *m_addr ; + std::size_t m_length ; } ; - + } // end of namespace diff --git a/libgrive/src/util/OS.cc b/libgrive/src/util/OS.cc index 0d51136b..9095e730 100644 --- a/libgrive/src/util/OS.cc +++ b/libgrive/src/util/OS.cc @@ -39,12 +39,12 @@ namespace gr { namespace os { -DateTime FileCTime( const fs::path& filename ) +void Stat( const fs::path& filename, DateTime *t, off_t *size, bool *is_dir ) { - return FileCTime( filename.string() ) ; + Stat( filename.string(), t, size, is_dir ) ; } -DateTime FileCTime( const std::string& filename ) +void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) { struct stat s = {} ; if ( ::stat( filename.c_str(), &s ) != 0 ) @@ -57,11 +57,18 @@ DateTime FileCTime( const std::string& filename ) ) ; } + if (t) + { #if defined __APPLE__ && defined __DARWIN_64_BIT_INO_T - return DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ; + *t = DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ; #else - return DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec); + *t = DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec); #endif + } + if ( size ) + *size = s.st_size; + if ( is_dir ) + *is_dir = S_ISDIR( s.st_mode ) ? true : false; } void SetFileTime( const fs::path& filename, const DateTime& t ) @@ -81,4 +88,17 @@ void SetFileTime( const std::string& filename, const DateTime& t ) ) ; } +void Sleep( unsigned int sec ) +{ + struct timespec ts = { sec, 0 } ; + + int result = 0 ; + do + { + struct timespec rem ; + nanosleep( &ts, &rem ) ; + ts = rem ; + } while ( result == -1 && errno == EINTR ) ; +} + } } // end of namespaces diff --git a/libgrive/src/util/OS.hh b/libgrive/src/util/OS.hh index 4a0350dc..43497839 100644 --- a/libgrive/src/util/OS.hh +++ b/libgrive/src/util/OS.hh @@ -33,11 +33,13 @@ namespace os { struct Error : virtual Exception {} ; - DateTime FileCTime( const std::string& filename ) ; - DateTime FileCTime( const fs::path& filename ) ; + void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) ; + void Stat( const fs::path& filename, DateTime *t, off64_t *size, bool *is_dir ) ; void SetFileTime( const std::string& filename, const DateTime& t ) ; void SetFileTime( const fs::path& filename, const DateTime& t ) ; + + void Sleep( unsigned int sec ) ; } } // end of namespaces diff --git a/libgrive/src/util/Progress.hh b/libgrive/src/util/Progress.hh new file mode 100644 index 00000000..9e087f90 --- /dev/null +++ b/libgrive/src/util/Progress.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "util/Types.hh" + +namespace gr { + +class Progress +{ +public: + virtual void reportProgress(u64_t total, u64_t processed) = 0; +}; + +} +; diff --git a/libgrive/src/util/ProgressBar.cc b/libgrive/src/util/ProgressBar.cc new file mode 100644 index 00000000..6a26b6e9 --- /dev/null +++ b/libgrive/src/util/ProgressBar.cc @@ -0,0 +1,87 @@ +#include +#include +#include +#include + +#include "ProgressBar.hh" + +namespace gr +{ + +ProgressBar::ProgressBar(): showProgressBar(false), last(1000) +{ +} + +ProgressBar::~ProgressBar() +{ +} + +void ProgressBar::setShowProgressBar(bool showProgressBar) +{ + this->showProgressBar = showProgressBar; +} + +unsigned short int ProgressBar::determineTerminalSize() +{ + struct winsize w; + ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); + return w.ws_col; +} + +void ProgressBar::printBytes(u64_t bytes) +{ + if (bytes >= 1024*1024*1024) + printf("%.3f GB", (double)bytes/1024/1024/1024); + else if (bytes >= 1024*1024) + printf("%.3f MB", (double)bytes/1024/1024); + else + printf("%.3f KB", (double)bytes/1024); +} + +void ProgressBar::reportProgress(u64_t total, u64_t processed) +{ + if (showProgressBar && total) + { + // libcurl seems to process more bytes then the actual file size :) + if (processed > total) + processed = total; + double fraction = (double)processed/total; + + int point = fraction*1000; + if (this->last < 1000 || point != this->last) + { + // do not print 100% progress multiple times (it will duplicate the progressbar) + this->last = point; + + // 10 for prefix of percent and 26 for suffix of file size + int availableSize = determineTerminalSize() - 36; + int totalDots; + if (availableSize > 100) + totalDots = 100; + else if (availableSize < 0) + totalDots = 10; + else + totalDots = availableSize; + + int dotz = round(fraction * totalDots); + int count = 0; + // delete previous output line + printf("\r [%3.0f%%] [", fraction * 100); + for (; count < dotz - 1; count++) + putchar('='); + putchar('>'); + for (; count < totalDots - 1; count++) + putchar(' '); + printf("] "); + printBytes(processed); + putchar('/'); + printBytes(total); + printf("\33[K\r"); + if (point == 1000) + putchar('\n'); + fflush(stdout); + } + } +} + +} diff --git a/libgrive/src/util/ProgressBar.hh b/libgrive/src/util/ProgressBar.hh new file mode 100644 index 00000000..5e10eeea --- /dev/null +++ b/libgrive/src/util/ProgressBar.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "util/Progress.hh" + +namespace gr { + +class ProgressBar: public Progress +{ +public: + ProgressBar(); + virtual ~ProgressBar(); + + void reportProgress(u64_t total, u64_t processed); + void setShowProgressBar(bool showProgressBar); + +private: + static void printBytes(u64_t bytes); + static unsigned short int determineTerminalSize(); + + bool showProgressBar; + int last; +}; + +} +; diff --git a/libgrive/src/protocol/JsonResponse.cc b/libgrive/src/util/StdStream.cc similarity index 56% rename from libgrive/src/protocol/JsonResponse.cc rename to libgrive/src/util/StdStream.cc index f270af6f..eaa10cea 100644 --- a/libgrive/src/protocol/JsonResponse.cc +++ b/libgrive/src/util/StdStream.cc @@ -1,6 +1,6 @@ /* grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho + Copyright (C) 2013 Wan Wai Ho This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -14,32 +14,29 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ -#include "JsonResponse.hh" +#include "StdStream.hh" -#include "protocol/Json.hh" +#include -namespace gr { namespace http { +namespace gr { -JsonResponse::JsonResponse() +StdStream::StdStream( std::streambuf *buf ) : + m_adaptee( buf ) { } -void JsonResponse::Clear() +std::size_t StdStream::Read( char *data, std::size_t size ) { - m_resp.Clear() ; + return m_adaptee == 0 ? 0 : m_adaptee->sgetn( data, size ) ; } -std::size_t JsonResponse::OnData( void *data, std::size_t count ) +std::size_t StdStream::Write( const char *data, std::size_t size ) { - return m_resp.OnData( data, count ) ; + return m_adaptee == 0 ? 0 : m_adaptee->sputn( data, size ) ; } -Json JsonResponse::Response() const -{ - return Json::Parse( m_resp.Response() ) ; -} - -} } // end of namespace +} // end of namespace diff --git a/libgrive/src/util/StdStream.hh b/libgrive/src/util/StdStream.hh new file mode 100644 index 00000000..4751e388 --- /dev/null +++ b/libgrive/src/util/StdStream.hh @@ -0,0 +1,42 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2013 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include "DataStream.hh" + +#include + +namespace gr { + +class StdStream : public DataStream +{ +public : + explicit StdStream( std::streambuf *buf ) ; + + std::size_t Read( char *data, std::size_t size ) ; + std::size_t Write( const char *data, std::size_t size ) ; + +private : + std::streambuf *m_adaptee ; +} ; + +} // end of namespace + diff --git a/libgrive/src/util/StdioFile.cc b/libgrive/src/util/StdioFile.cc deleted file mode 100644 index ae0d41b5..00000000 --- a/libgrive/src/util/StdioFile.cc +++ /dev/null @@ -1,156 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "StdioFile.hh" - -#include - -// boost headers -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace gr { - -StdioFile::StdioFile( ) : m_fd( -1 ) -{ -} - -StdioFile::StdioFile( const fs::path& path ) : m_fd( -1 ) -{ - OpenForRead( path ) ; -} - -StdioFile::StdioFile( const fs::path& path, int mode ) : m_fd( -1 ) -{ - OpenForWrite( path, mode ) ; -} - -StdioFile::~StdioFile( ) -{ - Close() ; -} - -void StdioFile::Open( const fs::path& path, int flags, int mode ) -{ - if ( IsOpened() ) - Close() ; - - assert( m_fd == -1 ) ; - m_fd = ::open( path.string().c_str(), flags, mode ) ; - if ( m_fd == -1 ) - { - BOOST_THROW_EXCEPTION( - Error() - << boost::errinfo_api_function("open") - << boost::errinfo_errno(errno) - << boost::errinfo_file_name(path.string()) - ) ; - } -} - -void StdioFile::OpenForRead( const fs::path& path ) -{ - Open( path, O_RDONLY, 0 ) ; -} - -void StdioFile::OpenForWrite( const fs::path& path, int mode ) -{ - Open( path, O_CREAT|O_RDWR|O_TRUNC, mode ) ; -} - -void StdioFile::Close() -{ - if ( IsOpened() ) - { - close( m_fd ) ; - m_fd = -1 ; - } -} - -bool StdioFile::IsOpened() const -{ - return m_fd != -1 ; -} - -std::size_t StdioFile::Read( void *ptr, std::size_t size ) -{ - assert( IsOpened() ) ; - ssize_t count = ::read( m_fd, ptr, size ) ; - if ( count == -1 ) - { - BOOST_THROW_EXCEPTION( - Error() - << boost::errinfo_api_function("read") - << boost::errinfo_errno(errno) - ) ; - } - return count ; -} - -std::size_t StdioFile::Write( const void *ptr, std::size_t size ) -{ - assert( IsOpened() ) ; - ssize_t count = ::write( m_fd, ptr, size ) ; - if ( count == -1 ) - { - BOOST_THROW_EXCEPTION( - Error() - << boost::errinfo_api_function("read") - << boost::errinfo_errno(errno) - ) ; - } - return count ; -} - -long StdioFile::Seek( long offset, int whence ) -{ - assert( IsOpened() ) ; - return ::lseek( m_fd, offset, whence ) ; -} - -long StdioFile::Tell() const -{ - assert( IsOpened() ) ; - return ::lseek( m_fd, 0, SEEK_CUR ) ; -} - -void StdioFile::Chmod( int mode ) -{ - assert( IsOpened() ) ; - - if ( ::fchmod( m_fd, mode ) != 0 ) - { - BOOST_THROW_EXCEPTION( - Error() - << boost::errinfo_api_function("fchmod") - << boost::errinfo_errno(errno) - ) ; - } -} - -} // end of namespace diff --git a/libgrive/src/util/StringStream.cc b/libgrive/src/util/StringStream.cc new file mode 100644 index 00000000..447f8928 --- /dev/null +++ b/libgrive/src/util/StringStream.cc @@ -0,0 +1,80 @@ +/* + webwrite: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "StringStream.hh" + +#include + +namespace gr { + +StringStream::StringStream( const std::string& init ) : + // FIXME avoid copy + m_str( init ), m_pos( 0 ) +{ +} + +/// Read `size` bytes from the stream. +std::size_t StringStream::Read( char *data, std::size_t size ) +{ + // wow! no need to count count == 0 + std::size_t count = std::min( m_str.size()-m_pos, size ) ; + std::copy( m_str.begin() + m_pos, m_str.begin() + m_pos + count, data ) ; + m_pos += count ; + return count ; +} + +std::size_t StringStream::Write( const char *data, std::size_t size ) +{ + m_str.insert( m_str.size(), data, size ) ; + return size ; +} + +off_t StringStream::Seek( off_t offset, int whence ) +{ + if ( whence == 1 ) + offset += m_pos; + else if ( whence == 2 ) + offset += Size(); + if ( (u64_t)offset > Size() ) + offset = Size(); + m_pos = (size_t)offset; + return m_pos; +} + +off_t StringStream::Tell() const +{ + return m_pos; +} + +u64_t StringStream::Size() const +{ + return m_str.size(); +} + +const std::string& StringStream::Str() const +{ + return m_str ; +} + +void StringStream::Str( const std::string& str ) +{ + m_str = str ; +} + +} // end of namespace diff --git a/libgrive/src/util/StringStream.hh b/libgrive/src/util/StringStream.hh new file mode 100644 index 00000000..39d4fa1f --- /dev/null +++ b/libgrive/src/util/StringStream.hh @@ -0,0 +1,54 @@ +/* + webwrite: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "DataStream.hh" + +#include + +namespace gr { + +/** \brief DataStream base on `std::string`s + + StringStream is a DataStream back-end that uses std::string for storage. + It is useful for unit tests and text parsing, especially when used with + StreamParser. +*/ +class StringStream : public SeekStream +{ +public : + explicit StringStream( const std::string& init = std::string() ) ; + + std::size_t Read( char *data, std::size_t size ) ; + std::size_t Write( const char *data, std::size_t size ) ; + + off_t Seek( off_t offset, int whence ) ; + off_t Tell() const ; + u64_t Size() const ; + + const std::string& Str() const ; + void Str( const std::string& str ) ; + +private : + std::string m_str ; + std::size_t m_pos ; +} ; + +} // end of namespace diff --git a/libgrive/src/http/Receivable.hh b/libgrive/src/util/Types.hh similarity index 79% rename from libgrive/src/http/Receivable.hh rename to libgrive/src/util/Types.hh index 2982aa1a..a981c6d3 100644 --- a/libgrive/src/http/Receivable.hh +++ b/libgrive/src/util/Types.hh @@ -19,15 +19,13 @@ #pragma once -#include +#include -namespace gr { namespace http { - -class Receivable +// import types into the Grive namespace +namespace gr { -public : - virtual std::size_t OnData( void *data, std::size_t count ) = 0 ; - virtual void Clear() = 0 ; -} ; - -} } // end of namespace + using ::off_t ; + + // should use boost/cstdint + typedef unsigned long long u64_t ; +} diff --git a/libgrive/src/util/log/CompositeLog.cc b/libgrive/src/util/log/CompositeLog.cc index 6c9140ab..9de64681 100644 --- a/libgrive/src/util/log/CompositeLog.cc +++ b/libgrive/src/util/log/CompositeLog.cc @@ -39,7 +39,7 @@ CompositeLog::~CompositeLog() std::for_each( m_logs.begin(), m_logs.end(), Destroy() ) ; } -LogBase* CompositeLog::Add( std::auto_ptr log ) +LogBase* CompositeLog::Add( std::unique_ptr& log ) { m_logs.push_back( log.get() ) ; return log.release() ; diff --git a/libgrive/src/util/log/CompositeLog.hh b/libgrive/src/util/log/CompositeLog.hh index 6efe3be0..80b41e41 100644 --- a/libgrive/src/util/log/CompositeLog.hh +++ b/libgrive/src/util/log/CompositeLog.hh @@ -32,7 +32,7 @@ public : CompositeLog() ; ~CompositeLog() ; - LogBase* Add( std::auto_ptr log ) ; + LogBase* Add( std::unique_ptr& log ) ; void Log( const log::Fmt& msg, log::Serverity s ) ; diff --git a/libgrive/src/util/log/Log.cc b/libgrive/src/util/log/Log.cc index b93e1b7f..07bc0860 100644 --- a/libgrive/src/util/log/Log.cc +++ b/libgrive/src/util/log/Log.cc @@ -40,12 +40,12 @@ public : } } ; -LogBase* LogBase::Inst( std::auto_ptr log ) +LogBase* LogBase::Inst( LogBase *log ) { - static std::auto_ptr inst( new MockLog ) ; + static std::unique_ptr inst( new MockLog ) ; - if ( log.get() != 0 ) - inst = log ; + if ( log != 0 ) + inst.reset( log ) ; assert( inst.get() != 0 ) ; return inst.get() ; diff --git a/libgrive/src/util/log/Log.hh b/libgrive/src/util/log/Log.hh index 8bdd6bbb..25760e51 100644 --- a/libgrive/src/util/log/Log.hh +++ b/libgrive/src/util/log/Log.hh @@ -65,8 +65,8 @@ public : virtual bool Enable( log::Serverity s, bool enable = true ) = 0 ; virtual bool IsEnabled( log::Serverity s ) const = 0 ; - static LogBase* Inst( std::auto_ptr log = std::auto_ptr() ) ; - ~LogBase() ; + static LogBase* Inst( LogBase *log = 0 ) ; + virtual ~LogBase() ; protected : LogBase() ; @@ -115,6 +115,12 @@ void Log( LogBase::Inst()->Log( log::Fmt(fmt) % p1 % p2 % p3 % p4, s ) ; } +template +void Log( const std::string& fmt, const P1& p1, const P2& p2, const P3& p3, const P4& p4, const P5& p5, log::Serverity s = log::info ) +{ + LogBase::Inst()->Log( log::Fmt(fmt) % p1 % p2 % p3 % p4 % p5, s ) ; +} + void Trace( const std::string& str ) ; template @@ -135,4 +141,10 @@ void Trace( const std::string& fmt, const P1& p1, const P2& p2, const P3& p3 ) LogBase::Inst()->Log( log::Fmt(fmt) % p1 % p2 % p3, log::debug ) ; } +template +void Trace( const std::string& fmt, const P1& p1, const P2& p2, const P3& p3, const P4& p4 ) +{ + LogBase::Inst()->Log( log::Fmt(fmt) % p1 % p2 % p3 % p4, log::debug ) ; +} + } // end of namespace diff --git a/libgrive/src/xml/Node.cc b/libgrive/src/xml/Node.cc index be12d312..abed0c4e 100644 --- a/libgrive/src/xml/Node.cc +++ b/libgrive/src/xml/Node.cc @@ -92,7 +92,7 @@ public : // cannot allow duplicate attribute nodes if ( child->m_type == attr && p.first != p.second ) - BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "duplicate attribute " + child->m_name ) ) ; + BOOST_THROW_EXCEPTION( Error() << DupAttr_( child->m_name ) ) ; vec.insert( p.second, child ) ; } diff --git a/libgrive/src/xml/Node.hh b/libgrive/src/xml/Node.hh index 85c2a6f8..fb4b6c43 100644 --- a/libgrive/src/xml/Node.hh +++ b/libgrive/src/xml/Node.hh @@ -19,6 +19,8 @@ #pragma once +#include "util/Exception.hh" + #include #include @@ -39,6 +41,8 @@ private : public : class iterator ; + typedef boost::error_info DupAttr_ ; + public : Node() ; Node( const Node& node ) ; diff --git a/libgrive/src/xml/NodeSet.cc b/libgrive/src/xml/NodeSet.cc index 8ccc82fd..42d5c488 100644 --- a/libgrive/src/xml/NodeSet.cc +++ b/libgrive/src/xml/NodeSet.cc @@ -33,6 +33,10 @@ NodeSet::NodeSet() : { } +/** Initialize the node set with an external list of nodes. + With this constructor, the list of nodes pointed by [first,last) will NOT + be deep copied to the node set. +*/ NodeSet::NodeSet( iterator first, iterator last ) : m_first( first ), m_last( last ) @@ -124,7 +128,7 @@ NodeSet NodeSet::operator[]( const std::string& name ) const Node NodeSet::front() const { if ( empty() ) - throw Error() << expt::ErrMsg( "empty node set" ) ; + BOOST_THROW_EXCEPTION( Error() << EmptyNodeSet_(0) ) ; return *m_first ; } diff --git a/libgrive/src/xml/NodeSet.hh b/libgrive/src/xml/NodeSet.hh index 98cb97c9..8820a8fd 100644 --- a/libgrive/src/xml/NodeSet.hh +++ b/libgrive/src/xml/NodeSet.hh @@ -19,6 +19,8 @@ #pragma once +#include "util/Exception.hh" + #include "Node.hh" #include @@ -32,6 +34,8 @@ class NodeSet public : typedef Node::iterator iterator ; + typedef boost::error_info EmptyNodeSet_ ; + public : NodeSet() ; NodeSet( const NodeSet& n ) ; diff --git a/libgrive/src/xml/TreeBuilder.cc b/libgrive/src/xml/TreeBuilder.cc index 66be3344..9d57501b 100644 --- a/libgrive/src/xml/TreeBuilder.cc +++ b/libgrive/src/xml/TreeBuilder.cc @@ -21,6 +21,7 @@ #include "Error.hh" #include "Node.hh" +#include "util/log/Log.hh" #include @@ -44,6 +45,8 @@ TreeBuilder::TreeBuilder() : m_impl( new Impl ) ::XML_SetElementHandler( m_impl->psr, &TreeBuilder::StartElement, &TreeBuilder::EndElement ) ; ::XML_SetCharacterDataHandler( m_impl->psr, &TreeBuilder::OnCharData ) ; ::XML_SetUserData( m_impl->psr , this ) ; + + is_new = true ; } TreeBuilder::~TreeBuilder() @@ -68,8 +71,12 @@ Node TreeBuilder::ParseFile( const std::string& file ) void TreeBuilder::ParseData( const char *data, std::size_t count, bool last ) { - if ( ::XML_Parse( m_impl->psr, data, count, last ) == 0 ) - throw Error() << expt::ErrMsg( "XML parse error" ) ; + is_new = false ; + + if ( ::XML_Parse( m_impl->psr, data, count, last ) == 0 ) { + Log("Error parsing XML: %1%", data, log::error); + BOOST_THROW_EXCEPTION( Error() << ExpatApiError("XML_Parse") ); + } } Node TreeBuilder::Parse( const std::string& xml ) @@ -85,7 +92,7 @@ Node TreeBuilder::Result() const assert( m_impl->stack.size() == 1 ) ; if ( m_impl->stack.front().size() != 1 ) - throw Error() << expt::ErrMsg( "invalid node" ) ; + BOOST_THROW_EXCEPTION( Error() << LogicError(0) ) ; return *m_impl->stack.front().begin() ; } diff --git a/libgrive/src/xml/TreeBuilder.hh b/libgrive/src/xml/TreeBuilder.hh index 184b55f6..2e958e7e 100644 --- a/libgrive/src/xml/TreeBuilder.hh +++ b/libgrive/src/xml/TreeBuilder.hh @@ -19,6 +19,8 @@ #pragma once +#include "util/Exception.hh" + #include #include @@ -28,6 +30,10 @@ class Node ; class TreeBuilder { +public : + typedef boost::error_info ExpatApiError ; + typedef boost::error_info LogicError ; + public : TreeBuilder() ; ~TreeBuilder() ; @@ -39,6 +45,8 @@ public : static Node ParseFile( const std::string& file ) ; static Node Parse( const std::string& xml ) ; + bool is_new ; + private : static void StartElement( void* pvthis, const char* name, const char** attr ) ; @@ -47,7 +55,7 @@ private : private : struct Impl ; - std::auto_ptr m_impl ; + std::unique_ptr m_impl ; } ; } } // end of namespace diff --git a/libgrive/test/UnitTest.cc b/libgrive/test/UnitTest.cc index f599edf5..02121d21 100644 --- a/libgrive/test/UnitTest.cc +++ b/libgrive/test/UnitTest.cc @@ -21,30 +21,30 @@ #include "util/log/DefaultLog.hh" -#include "drive/EntryTest.hh" -#include "drive/ResourceTest.hh" -#include "drive/ResourceTreeTest.hh" -#include "drive/StateTest.hh" +#include "base/ResourceTest.hh" +#include "base/ResourceTreeTest.hh" +#include "base/StateTest.hh" #include "util/DateTimeTest.hh" #include "util/FunctionTest.hh" +#include "util/ConfigTest.hh" #include "util/SignalHandlerTest.hh" -#include "xml/NodeTest.hh" +//#include "xml/NodeTest.hh" int main( int argc, char **argv ) { using namespace grut ; - gr::LogBase::Inst( std::auto_ptr(new gr::log::DefaultLog) ) ; + gr::LogBase::Inst( new gr::log::DefaultLog ) ; CppUnit::TextUi::TestRunner runner; - runner.addTest( EntryTest::suite( ) ) ; runner.addTest( StateTest::suite( ) ) ; runner.addTest( ResourceTest::suite( ) ) ; runner.addTest( ResourceTreeTest::suite( ) ) ; runner.addTest( DateTimeTest::suite( ) ) ; runner.addTest( FunctionTest::suite( ) ) ; + runner.addTest( ConfigTest::suite( ) ) ; runner.addTest( SignalHandlerTest::suite( ) ) ; - runner.addTest( NodeTest::suite( ) ) ; + //runner.addTest( NodeTest::suite( ) ) ; runner.run(); return 0 ; diff --git a/libgrive/test/drive/ResourceTest.cc b/libgrive/test/base/ResourceTest.cc similarity index 65% rename from libgrive/test/drive/ResourceTest.cc rename to libgrive/test/base/ResourceTest.cc index 1199561f..f22a9a53 100644 --- a/libgrive/test/drive/ResourceTest.cc +++ b/libgrive/test/base/ResourceTest.cc @@ -21,16 +21,17 @@ #include "Assert.hh" -#include "drive/Resource.hh" +#include "base/Resource.hh" -#include "drive/Entry.hh" -#include "xml/Node.hh" +#include "drive2/Entry2.hh" +#include "json/Val.hh" #include namespace grut { using namespace gr ; +using namespace gr::v2 ; ResourceTest::ResourceTest( ) { @@ -38,9 +39,10 @@ ResourceTest::ResourceTest( ) void ResourceTest::TestRootPath() { - Resource root ; + std::string rootFolder = "/home/usr/grive/grive"; + Resource root( rootFolder ) ; CPPUNIT_ASSERT( root.IsRoot() ) ; - GRUT_ASSERT_EQUAL( root.Path(), fs::path( "." ) ) ; + GRUT_ASSERT_EQUAL( root.Path(), fs::path( rootFolder ) ) ; } void ResourceTest::TestNormal( ) @@ -49,19 +51,23 @@ void ResourceTest::TestNormal( ) Resource subject( "entry.xml", "file" ) ; root.AddChild( &subject ) ; + GRUT_ASSERT_EQUAL( subject.IsRoot(), false ) ; GRUT_ASSERT_EQUAL( subject.Path(), fs::path( TEST_DATA ) / "entry.xml" ) ; - subject.FromLocal( DateTime() ) ; + Val st; + st.Add( "srv_time", Val( DateTime( "2012-05-09T16:13:22.401Z" ).Sec() ) ); + subject.FromLocal( st ) ; GRUT_ASSERT_EQUAL( subject.MD5(), "c0742c0a32b2c909b6f176d17a6992d0" ) ; GRUT_ASSERT_EQUAL( subject.StateStr(), "local_new" ) ; - xml::Node entry = xml::Node::Element( "entry" ) ; - entry.AddElement( "updated" ).AddText( "2012-05-09T16:13:22.401Z" ) ; + Val entry; + entry.Set( "modifiedDate", Val( std::string( "2012-05-09T16:13:22.401Z" ) ) ); + entry.Set( "md5Checksum", Val( std::string( "DIFFERENT" ) ) ); - Entry remote( entry ) ; - subject.FromRemote( remote, DateTime() ) ; - GRUT_ASSERT_EQUAL( subject.StateStr(), "local_changed" ) ; + Entry2 remote( entry ) ; + GRUT_ASSERT_EQUAL( "different", remote.MD5() ) ; + subject.FromRemote( remote ) ; + GRUT_ASSERT_EQUAL( "local_changed", subject.StateStr() ) ; } - } // end of namespace grut diff --git a/libgrive/test/drive/ResourceTest.hh b/libgrive/test/base/ResourceTest.hh similarity index 100% rename from libgrive/test/drive/ResourceTest.hh rename to libgrive/test/base/ResourceTest.hh diff --git a/libgrive/test/drive/ResourceTreeTest.cc b/libgrive/test/base/ResourceTreeTest.cc similarity index 94% rename from libgrive/test/drive/ResourceTreeTest.cc rename to libgrive/test/base/ResourceTreeTest.cc index 0e07d800..c15df46a 100644 --- a/libgrive/test/drive/ResourceTreeTest.cc +++ b/libgrive/test/base/ResourceTreeTest.cc @@ -21,8 +21,8 @@ #include "Assert.hh" -#include "drive/ResourceTree.hh" -#include "drive/Resource.hh" +#include "base/ResourceTree.hh" +#include "base/Resource.hh" #include diff --git a/libgrive/test/drive/ResourceTreeTest.hh b/libgrive/test/base/ResourceTreeTest.hh similarity index 100% rename from libgrive/test/drive/ResourceTreeTest.hh rename to libgrive/test/base/ResourceTreeTest.hh diff --git a/libgrive/test/drive/StateTest.cc b/libgrive/test/base/StateTest.cc similarity index 94% rename from libgrive/test/drive/StateTest.cc rename to libgrive/test/base/StateTest.cc index 9e33fcfa..6e7aff57 100644 --- a/libgrive/test/drive/StateTest.cc +++ b/libgrive/test/base/StateTest.cc @@ -21,8 +21,8 @@ #include "Assert.hh" -#include "drive/State.hh" -#include "protocol/Json.hh" +#include "base/State.hh" +#include "json/Val.hh" #include "util/log/Log.hh" #include diff --git a/libgrive/test/drive/StateTest.hh b/libgrive/test/base/StateTest.hh similarity index 100% rename from libgrive/test/drive/StateTest.hh rename to libgrive/test/base/StateTest.hh diff --git a/grive/src/Config.cc b/libgrive/test/btest/JsonValTest.cc similarity index 50% rename from grive/src/Config.cc rename to libgrive/test/btest/JsonValTest.cc index 6a66d569..bbf2d9eb 100644 --- a/grive/src/Config.cc +++ b/libgrive/test/btest/JsonValTest.cc @@ -1,5 +1,5 @@ /* - grive: an GPL program to sync a local directory with Google Drive + webwrite: an GPL wiki-like website with in-place editing Copyright (C) 2012 Wan Wai Ho This program is free software; you can redistribute it and/or @@ -17,48 +17,39 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "Config.hh" +#include "json/JsonParser.hh" +#include "json/Val.hh" +#include "json/ValBuilder.hh" +#include "json/JsonWriter.hh" +#include "util/StringStream.hh" -#include "util/StdioFile.hh" +#include -#include +using namespace gr ; -namespace gr { - -const std::string& Config::Filename() +namespace { - static const char *env_cfg = ::getenv( "GR_CONFIG" ) ; - static const std::string filename = (env_cfg != 0) ? env_cfg : ".grive" ; - - return filename ; -} - -Config::Config() : - m_cfg( Read( Filename() ) ) -{ -} - -void Config::Save( ) -{ - StdioFile file( Filename(), 0600 ) ; - m_cfg.Write( file ) ; + struct F + { + } ; } -Json& Config::Get() -{ - return m_cfg ; -} +BOOST_FIXTURE_TEST_SUITE( JsonValTest, F ) -Json Config::Read( const std::string& filename ) +BOOST_AUTO_TEST_CASE( Test ) { - try - { - return Json::ParseFile( filename ) ; - } - catch ( Exception& e ) - { - return Json() ; - } + Val json = ParseJson( "{\"key\": 100 }" ) ; + + BOOST_CHECK( json.Is() ) ; + BOOST_CHECK_EQUAL( json["key"].As(), 100 ) ; + + StringStream ss ; + JsonWriter wr( &ss ) ; + json.Visit( &wr ) ; + + BOOST_CHECK_EQUAL( ss.Str(), "{\"key\":100}" ) ; + +// std::cout << ss.Str() << std::endl ; } -} // end of namespace +BOOST_AUTO_TEST_SUITE_END() diff --git a/libgrive/test/btest/UnitTest.cc b/libgrive/test/btest/UnitTest.cc new file mode 100644 index 00000000..5a824cff --- /dev/null +++ b/libgrive/test/btest/UnitTest.cc @@ -0,0 +1,21 @@ +/* + webwrite: an GPL wiki-like website with in-place editing + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#define BOOST_TEST_MAIN +#include diff --git a/libgrive/test/btest/ValTest.cc b/libgrive/test/btest/ValTest.cc new file mode 100644 index 00000000..31c2cdb1 --- /dev/null +++ b/libgrive/test/btest/ValTest.cc @@ -0,0 +1,54 @@ +/* + webwrite: an GPL wiki-like website with in-place editing + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "json/Val.hh" +#include +#include + +using namespace gr ; + +namespace +{ + struct Fixture + { + } ; +} + +BOOST_FIXTURE_TEST_SUITE( ValTest, Fixture ) + +BOOST_AUTO_TEST_CASE( TestSimpleTypes ) +{ + BOOST_CHECK_EQUAL( Val::Null().Type(), Val::null_type ) ; + BOOST_CHECK( Val::Null().Is() ) ; + + Val i( 100 ) ; + BOOST_CHECK_EQUAL( i.Str(), "100" ); + BOOST_CHECK_EQUAL( i.As(), 100 ) ; + BOOST_CHECK_EQUAL( i.Type(), Val::int_type ) ; +} + +BOOST_AUTO_TEST_CASE( TestMap ) +{ + Val obj(( Val::Object() )) ; + obj.Add( "key", Val( std::string("value") ) ) ; + BOOST_CHECK_EQUAL( obj["key"].Type(), Val::string_type ) ; + BOOST_CHECK_EQUAL( obj["key"].As(), "value" ) ; +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/libgrive/test/data/test_dir1.state b/libgrive/test/data/test_dir1.state deleted file mode 100644 index 67b5d1b9..00000000 --- a/libgrive/test/data/test_dir1.state +++ /dev/null @@ -1 +0,0 @@ -{ "change_stamp": "", "rtree": { "name": ".", "id": "folder:root", "href": "https:\/\/docs.google.com\/feeds\/default\/private\/full\/folder%3Aroot", "md5": "", "kind": "folder", "mtime": { "sec": 0, "nsec": 0 }, "child": [ { "name": "entry.xml", "id": "", "href": "", "md5": "c0742c0a32b2c909b6f176d17a6992d0", "kind": "file", "mtime": { "sec": 1336796872, "nsec": 404985662 }, "child": [ ] } ] } } \ No newline at end of file diff --git a/libgrive/test/drive/EntryTest.cc b/libgrive/test/drive/EntryTest.cc deleted file mode 100644 index 82899617..00000000 --- a/libgrive/test/drive/EntryTest.cc +++ /dev/null @@ -1,58 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2012 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "EntryTest.hh" - -#include "Assert.hh" - -#include "drive/Entry.hh" -#include "xml/Node.hh" -#include "xml/NodeSet.hh" -#include "xml/TreeBuilder.hh" - -#include - -namespace grut { - -using namespace gr ; - -EntryTest::EntryTest( ) -{ -} - -void EntryTest::TestXml( ) -{ - xml::Node root = xml::TreeBuilder::ParseFile( TEST_DATA "entry.xml" ) ; - - CPPUNIT_ASSERT( !root["entry"].empty() ) ; - - Entry subject( root["entry"].front() ) ; - GRUT_ASSERT_EQUAL( "snes", subject.Title() ) ; - GRUT_ASSERT_EQUAL( "\"WxYPGE8CDyt7ImBk\"", subject.ETag() ) ; - GRUT_ASSERT_EQUAL( "https://docs.google.com/feeds/default/private/full/folder%3A0B5KhdsbryVeGMl83OEV1ZVc3cUE", - subject.SelfHref() ) ; - - GRUT_ASSERT_EQUAL( 1U, subject.ParentHrefs().size() ) ; - GRUT_ASSERT_EQUAL( "https://docs.google.com/feeds/default/private/full/folder%3A0B5KhdsbryVeGNEZjdUxzZHl3Sjg", - subject.ParentHrefs().front() ) ; - - GRUT_ASSERT_EQUAL( "folder", subject.Kind() ) ; -} - -} // end of namespace grut diff --git a/libgrive/test/util/ConfigTest.cc b/libgrive/test/util/ConfigTest.cc new file mode 100644 index 00000000..9974dd5e --- /dev/null +++ b/libgrive/test/util/ConfigTest.cc @@ -0,0 +1,65 @@ +/* +grive: an GPL program to sync a local directory with Google Drive +Copyright (C) 2012 Wan Wai Ho + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation version 2 +of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "ConfigTest.hh" + +#include "Assert.hh" + +#include "util/Config.hh" +#include "json/Val.hh" +#include "util/log/Log.hh" + +#include +#include + +using namespace grut; +using namespace gr ; +namespace po = boost::program_options; + +ConfigTest::ConfigTest( ) +{ +} + +void ConfigTest::TestInitialiseWithNoPath( ) +{ + po::variables_map vm; + po::notify(vm); + + Config config(vm); + GRUT_ASSERT_EQUAL( "./.grive", config.Filename().string()) ; +} + +void ConfigTest::TestInitialiseWithPath( ) +{ + char const *argv[] = { "Program", "-p", "/home/grive" }; + int argc = 3; + + po::options_description desc( "Grive options" ); + desc.add_options() + ( "path,p", po::value(), "Path to sync") + ; + + po::variables_map vm; + po::store(po::parse_command_line( argc, argv, desc), vm ); + po::notify(vm); + + Config config(vm); + GRUT_ASSERT_EQUAL( "/home/grive/.grive", config.Filename().string()) ; +} + diff --git a/libgrive/test/drive/EntryTest.hh b/libgrive/test/util/ConfigTest.hh similarity index 76% rename from libgrive/test/drive/EntryTest.hh rename to libgrive/test/util/ConfigTest.hh index 8d4f7f47..636ab8b8 100644 --- a/libgrive/test/drive/EntryTest.hh +++ b/libgrive/test/util/ConfigTest.hh @@ -19,23 +19,26 @@ #pragma once +#include "util/Config.hh" #include #include namespace grut { -class EntryTest : public CppUnit::TestFixture +class ConfigTest : public CppUnit::TestFixture { public : - EntryTest( ) ; + ConfigTest( ) ; - // declare suit function - CPPUNIT_TEST_SUITE( EntryTest ) ; - CPPUNIT_TEST( TestXml ) ; + CPPUNIT_TEST_SUITE( ConfigTest ) ; + CPPUNIT_TEST( TestInitialiseWithPath ) ; + CPPUNIT_TEST( TestInitialiseWithNoPath ) ; CPPUNIT_TEST_SUITE_END(); private : - void TestXml( ) ; + void TestInitialiseWithPath( ); + void TestInitialiseWithNoPath( ); } ; } // end of namespace + diff --git a/package/fedora16/grive.spec b/package/fedora16/grive.spec index 03770b9d..3a7e7675 100644 --- a/package/fedora16/grive.spec +++ b/package/fedora16/grive.spec @@ -31,7 +31,7 @@ Source0: https://github.com/grive/%{name}/tarball/v%{version} BuildRequires: cmake BuildRequires: libstdc++-devel BuildRequires: libcurl-devel -BuildRequires: json-c-devel +BuildRequires: yajl-devel BuildRequires: expat-devel BuildRequires: openssl-devel BuildRequires: boost-devel