diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..34a7f7e9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +* +!cmake +!grive +!libgrive +!CMakeLists.txt diff --git a/.gitignore b/.gitignore index 6fa3768d..de247d2f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,14 @@ bgrive/bgrive grive/grive libgrive/btest *.cmake + +debian/debhelper-build-stamp +debian/files +debian/grive.debhelper.log +debian/grive.substvars +debian/grive/ +debian/.debhelper + +obj-x86_64-linux-gnu/ + +.idea diff --git a/CMakeLists.txt b/CMakeLists.txt index 95443aa7..5bf001a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,27 @@ cmake_minimum_required(VERSION 2.8) +project(grive2) + +include(GNUInstallDirs) # Grive version. remember to update it for every new release! -set( GRIVE_VERSION "0.5.1-dev" ) +set( GRIVE_VERSION "0.5.3" CACHE STRING "Grive version" ) +message(WARNING "Version to build: ${GRIVE_VERSION}") # common compile options add_definitions( -DVERSION="${GRIVE_VERSION}" ) add_definitions( -D_FILE_OFFSET_BITS=64 -std=c++0x ) +if ( APPLE ) + add_definitions( -Doff64_t=off_t ) +endif ( APPLE ) + +find_program( + HAVE_SYSTEMD systemd + PATHS /lib/systemd /usr/lib/systemd + NO_DEFAULT_PATH +) +if ( HAVE_SYSTEMD ) + add_subdirectory( systemd ) +endif( HAVE_SYSTEMD ) add_subdirectory( libgrive ) add_subdirectory( grive ) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ee0ee1b8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM alpine:3.7 as build + +RUN apk add make cmake g++ libgcrypt-dev yajl-dev yajl \ + boost-dev curl-dev expat-dev cppunit-dev binutils-dev \ + pkgconfig + +ADD . /grive2 + +RUN mkdir /grive2/build \ + && cd /grive2/build \ + && cmake .. \ + && make -j4 install + +FROM alpine:3.7 + +RUN apk add yajl libcurl libgcrypt boost-program_options boost-regex libstdc++ boost-system \ + && apk add boost-filesystem --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main + +COPY --from=build /usr/local/bin/grive /bin/grive +RUN chmod 777 /bin/grive \ + && mkdir /data + +VOLUME /data +WORKDIR /data +ENTRYPOINT grive diff --git a/README.md b/README.md index 2e2ee468..a71923d1 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -# Grive2 0.5.1-dev +# Grive2 0.5.3 -28 Sep 2016, Vitaliy Filippov +09 Nov 2022, Vitaliy Filippov http://yourcmc.ru/wiki/Grive2 This is the fork of original "Grive" (https://github.com/Grive/grive) Google Drive client with the support for the new Drive REST API and partial sync. -Grive can be considered still beta or pre-beta quality. It simply downloads all the files in your -Google Drive into the current directory. After you make some changes to the local files, run +Grive simply downloads all the files in your Google Drive into the current directory. +After you make some changes to the local files, run grive again and it will upload your changes back to your Google Drive. New files created locally or in Google Drive will be uploaded or downloaded respectively. Deleted files will also be "removed". Currently Grive will NOT destroy any of your files: it will only move the files to a @@ -16,23 +16,129 @@ directory named .trash or put them in the Google Drive trash. You can always rec There are a few things that Grive does not do at the moment: - continously wait for changes in file system or in Google Drive to occur and upload. - A sync is only performed when you run Grive. + A sync is only performed when you run Grive (there are workarounds for almost + continuous sync. See below). - symbolic links support. - support for Google documents. -These may be added in the future, possibly the next release. +These may be added in the future. + +Enjoy! + +## Usage When Grive is run for the first time, you should use the "-a" argument to grant -permission to Grive to access to your Google Drive. A URL should be printed. -Go to the link. You will need to login to your Google account if you haven't -done so. After granting the permission to Grive, the browser will show you -an authenication code. Copy-and-paste that to the standard input of Grive. +permission to Grive to access to your Google Drive: + +```bash +cd $HOME +mkdir google-drive +cd google-drive +grive -a +``` + +A URL should be printed. Go to the link. You will need to login to your Google +account if you haven't done so. After granting the permission to Grive, the +authorization code will be forwarded to the Grive application and you will be +redirected to a localhost web page confirming the authorization. -If everything works fine, Grive will create .grive and .grive_state files in your +If everything works fine, Grive will create .grive and .grive\_state files in your current directory. It will also start downloading files from your Google Drive to your current directory. -Enjoy! +To resync the direcory, run `grive` in the folder. + +```bash +cd $HOME/google-drive +grive +``` + +### Exclude specific files and folders from sync: .griveignore + +Rules are similar to Git's .gitignore, but may differ slightly due to the different +implementation. + +- lines that start with # are comments +- leading and trailing spaces ignored unless escaped with \ +- non-empty lines without ! in front are treated as "exclude" patterns +- non-empty lines with ! in front are treated as "include" patterns + and have a priority over all "exclude" ones +- patterns are matched against the filenames relative to the grive root +- a/**/b matches any number of subpaths between a and b, including 0 +- **/a matches `a` inside any directory +- b/** matches everything inside `b`, but not b itself +- \* matches any number of any characters except / +- ? matches any character except / +- .griveignore itself isn't ignored by default, but you can include it in itself to ignore + + +### Scheduled syncs and syncs on file change events + +There are tools which you can use to enable both scheduled syncs and syncs +when a file changes. Together these gives you an experience almost like the +Google Drive clients on other platforms (it misses the almost instantious +download of changed files in the google drive). + +Grive installs such a basic solution which uses inotify-tools together with +systemd timer and services. You can enable it for a folder in your `$HOME` +directory (in this case the `$HOME/google-drive`): + +First install the `inotify-tools` (seems to be named like that in all major distros): +test that it works by calling `inotifywait -h`. + +Prepare a Google Drive folder in your $HOME directory with `grive -a`. + +```bash +# 'google-drive' is the name of your Google Drive folder in your $HOME directory +systemctl --user enable grive@$(systemd-escape google-drive).service +systemctl --user start grive@$(systemd-escape google-drive).service +``` + +You can enable and start this unit for multiple folders in your `$HOME` +directory if you need to sync with multiple google accounts. + +You can also only enable the time based syncing or the changes based syncing +by only directly enabling and starting the corresponding unit: +`grive-changes@$(systemd-escape google-drive).service` or +`grive-timer@$(systemd-escape google-drive).timer`. + +### Shared files + +Files and folders which are shared with you don't automatically show up in +your folder. They need to be added explicitly to your Google Drive: go to the +Google Drive website, right click on the file or folder and chose 'Add to My +Drive'. + +### Different OAuth2 client to workaround over quota and google approval issues + +Google recently started to restrict access for unapproved applications: +https://developers.google.com/drive/api/v3/about-auth?hl=ru + +Grive2 is currently awaiting approval but it seems it will take forever. +Also even if they approve it the default Client ID supplied with grive may +exceed quota and grive will then fail to sync. + +You can supply your own OAuth2 client credentials to work around these problems +by following these steps: + +1. Go to https://console.developers.google.com/apis/api/drive.googleapis.com +2. Choose a project (you might need to create one first) +3. Go to https://console.developers.google.com/apis/library/drive.googleapis.com and + "Enable" the Google Drive APIs +4. Go to https://console.cloud.google.com/apis/credentials and click "Create credentials > Help me choose" +5. In the "Find out what credentials you need" dialog, choose: + - Which API are you using: "Google Drive API" + - Where will you be calling the API from: "Other UI (...CLI...)" + - What data will you be accessing: "User Data" +6. In the next steps create a client id (name doesn't matter) and + setup the consent screen (defaults are ok, no need for any URLs) +7. The needed "Client ID" and "Client Secret" are either in the shown download + or can later found by clicking on the created credential on + https://console.developers.google.com/apis/credentials/ +8. When you change client ID/secret in an existing Grive folder you must first delete + the old `.grive` configuration file. +9. Call `grive -a --id --secret ` and follow the steps + to authenticate the OAuth2 client to allow it to access your drive folder. ## Installation @@ -57,8 +163,14 @@ There are also some optional dependencies: On a Debian/Ubuntu/Linux Mint machine just run the following command to install all these packages: - sudo apt-get install git cmake build-essential libgcrypt11-dev libyajl-dev \ - libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev + sudo apt-get install git cmake build-essential libgcrypt20-dev libyajl-dev \ + libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev \ + debhelper zlib1g-dev dpkg-dev pkg-config + +Fedora: + + sudo dnf install git cmake libgcrypt-devel gcc-c++ libstdc++ yajl-devel boost-devel libcurl-devel expat-devel binutils zlib + FreeBSD: @@ -69,7 +181,7 @@ FreeBSD: On a Debian/Ubuntu/Linux Mint you can use `dpkg-buildpackage` utility from `dpkg-dev` package to build grive. Just clone the repository, `cd` into it and run - dpkg-buildpackage -j4 + dpkg-buildpackage -j4 --no-sign ### Manual build @@ -81,17 +193,35 @@ Grive uses cmake to build. Basic install sequence is make -j4 sudo make install +Alternativly you can define your own client_id and client_secret during build + + mkdir build + cd build + cmake .. "-DAPP_ID:STRING=" "-DAPP_SECRET:STRING=" + make -j4 + sudo make install + ## Version History -### Grive2 v0.5.1-dev +### Grive2 v0.5.3 + +- Implement Google OAuth loopback IP redirect flow +- Various small fixes +### Grive2 v0.5.1 + +- Support for .griveignore +- Automatic sync solution based on inotify-tools and systemd - no-remote-new and upload-only modes -- ignore regexp does not persist anymore (note that Grive will still track it to not +- Ignore regexp does not persist anymore (note that Grive will still track it to not accidentally delete remote files when changing ignore regexp) -- added options to limit upload and download speed -- faster upload of new and changed files. now Grive uploads files without first calculating +- Added options to limit upload and download speed +- Faster upload of new and changed files. Now Grive uploads files without first calculating md5 checksum when file is created locally or when its size changes. -- added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin) +- Added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin) +- Added command-line options to specify your own client_id and client_secret +- Now grive2 skips links, sockets, fifos and other unusual files +- Various small build fixes ### Grive2 v0.5 @@ -137,3 +267,4 @@ New features: - #87: support for revisions - #86: ~~partial sync (contributed by justin at tierramedia.com)~~ that's not partial sync, that's only support for specifying local path on command line + diff --git a/cmake/Modules/FindBFD.cmake b/cmake/Modules/FindBFD.cmake index 35dfd5d6..8fd4427b 100644 --- a/cmake/Modules/FindBFD.cmake +++ b/cmake/Modules/FindBFD.cmake @@ -1,12 +1,6 @@ -find_library( DL_LIBRARY NAMES dl PATH ${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES} ) find_library( BFD_LIBRARY NAMES bfd PATH ${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES} ) -if ( DL_LIBRARY AND BFD_LIBRARY ) +if ( BFD_LIBRARY ) set( BFD_FOUND TRUE ) -endif (DL_LIBRARY AND BFD_LIBRARY) - -if ( BFD_FOUND ) - message( STATUS "Found libbfd: ${BFD_LIBRARY}") - -endif ( BFD_FOUND ) +endif ( BFD_LIBRARY ) diff --git a/cmake/Modules/FindLibGcrypt.cmake b/cmake/Modules/FindLibGcrypt.cmake index 0e1727f2..1cbc10a0 100644 --- a/cmake/Modules/FindLibGcrypt.cmake +++ b/cmake/Modules/FindLibGcrypt.cmake @@ -27,6 +27,9 @@ IF(LIBGCRYPTCONFIG_EXECUTABLE) EXEC_PROGRAM(${LIBGCRYPTCONFIG_EXECUTABLE} ARGS --cflags RETURN_VALUE _return_VALUE OUTPUT_VARIABLE LIBGCRYPT_CFLAGS) + string(REPLACE "fgrep: warning: fgrep is obsolescent; using grep -F" "" LIBGCRYPT_LIBRARIES "${LIBGCRYPT_LIBRARIES}") + string(STRIP "${LIBGCRYPT_LIBRARIES}" LIBGCRYPT_LIBRARIES) + IF(${LIBGCRYPT_CFLAGS} MATCHES "\n") SET(LIBGCRYPT_CFLAGS " ") ENDIF(${LIBGCRYPT_CFLAGS} MATCHES "\n") diff --git a/completion.zsh b/completion.zsh new file mode 100644 index 00000000..4d3f8070 --- /dev/null +++ b/completion.zsh @@ -0,0 +1,63 @@ +#compdef grive +# ------------------------------------------------------------------------------ +# Copyright (c) 2015 Github zsh-users - http://github.com/zsh-users +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the zsh-users nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL ZSH-USERS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ------------------------------------------------------------------------------ +# Description +# ----------- +# +# Completion script for Grive (https://github.com/vitalif/grive2) +# +# ------------------------------------------------------------------------------ +# Authors +# ------- +# +# * Doron Behar +# +# ------------------------------------------------------------------------------ +# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*- +# vim: ft=zsh sw=2 ts=2 et +# ------------------------------------------------------------------------------ + +local curcontext="$curcontext" state line ret=1 +typeset -A opt_args + +_arguments -C \ + '(-h --help)'{-h,--help}'[Produce help message]' \ + '(-v --version)'{-v,--version}'[Display Grive version]' \ + '(-a --auth)'{-a,--auth}'[Request authorization token]' \ + '(-p --path)'{-p,--path}'[Root directory to sync]' \ + '(-s --dir)'{-s,--dir}'[Single subdirectory to sync (remembered for next runs)]' \ + '(-V --verbose)'{-V,--verbose}'[Verbose mode. Enable more messages than normal.]' \ + '(--log-http)--log-http[Log all HTTP responses in this file for debugging.]' \ + '(--new-rev)--new-rev[Create,new revisions in server for updated files.]' \ + '(-d --debug)'{-d,--debug}'[Enable debug level messages. Implies -v.]' \ + '(-l --log)'{-l,--log}'[Set log output filename.]' \ + '(-f --force)'{-f,--force}'[Force grive to always download a file from Google Drive instead of uploading it.]' \ + '(--dry-run)--dry-run[Only,detect which files need to be uploaded/downloaded,without actually performing them.]' \ + '(--ignore)--ignore[Perl,RegExp to ignore files (matched against relative paths, remembered for next runs) ]' \ + '*: :_files' && ret=0 + +return ret diff --git a/debian/changelog b/debian/changelog index db47f06e..0910281d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,15 @@ -grive2 (0.5.1+git20160731) unstable; urgency=medium +grive2 (0.5.3) unstable; urgency=medium + + * Implement Google OAuth loopback IP redirect flow + * Various small fixes + + -- Vitaliy Filippov Wed, 09 Nov 2022 12:42:28 +0300 + +grive2 (0.5.2+git20210315) unstable; urgency=medium * Newer dev version + * Add systemd unit files and helper script for automatic syncs + * Add possibility to change client id and secret and save it between runs -- Vitaliy Filippov Wed, 31 Jul 2016 22:04:53 +0300 diff --git a/debian/compat b/debian/compat index 7f8f011e..b4de3947 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -7 +11 diff --git a/debian/control b/debian/control index edad2557..150f6883 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: grive2 Section: net Priority: optional Maintainer: Vitaliy Filippov -Build-Depends: debhelper, cmake, pkg-config, zlib1g-dev, libcurl4-openssl-dev | libcurl4-gnutls-dev, libstdc++-6-dev | libstdc++6-4.4-dev | libstdc++-4.9-dev | libstdc++-5-dev, libboost-filesystem-dev, libboost-program-options-dev, libboost-test-dev, libboost-regex-dev, libexpat1-dev, binutils-dev, libgcrypt-dev, libyajl-dev +Build-Depends: debhelper, cmake, pkg-config, zlib1g-dev, libcurl4-openssl-dev | libcurl4-gnutls-dev, libboost-filesystem-dev, libboost-program-options-dev, libboost-test-dev, libboost-regex-dev, libexpat1-dev, libgcrypt-dev, libyajl-dev Standards-Version: 3.9.6 Homepage: https://yourcmc.ru/wiki/Grive2 diff --git a/debian/rules b/debian/rules index 81484573..6f9c7a78 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,7 @@ #!/usr/bin/make -f +override_dh_auto_configure: + dh_auto_configure -- -DHAVE_SYSTEMD=1 + %: - dh $@ --buildsystem=cmake --parallel + dh $@ --buildsystem=cmake --parallel --builddirectory=build diff --git a/grive/CMakeLists.txt b/grive/CMakeLists.txt index 837a36d5..ac5b781d 100644 --- a/grive/CMakeLists.txt +++ b/grive/CMakeLists.txt @@ -17,13 +17,28 @@ add_executable( grive_executable ) target_link_libraries( grive_executable - ${Boost_LIBRARIES} grive ) +set(DEFAULT_APP_ID "615557989097-i93d4d1ojpen0m0dso18ldr6orjkidgf.apps.googleusercontent.com") +set(DEFAULT_APP_SECRET "xiM8Apu_WuRRdheNelJcNtOD") +set(APP_ID ${DEFAULT_APP_ID} CACHE STRING "Application Id") +set(APP_SECRET ${DEFAULT_APP_SECRET} CACHE STRING "Application Secret") + +target_compile_definitions ( grive_executable + PRIVATE + -DAPP_ID="${APP_ID}" + -DAPP_SECRET="${APP_SECRET}" +) + set_target_properties( grive_executable PROPERTIES OUTPUT_NAME grive ) install(TARGETS grive_executable RUNTIME DESTINATION bin) -install(FILES doc/grive.1 DESTINATION share/man/man1 ) + +if ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD" ) + install(FILES doc/grive.1 DESTINATION man/man1 ) +else ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD" ) + install(FILES doc/grive.1 DESTINATION share/man/man1 ) +endif( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD" ) diff --git a/grive/doc/grive.1 b/grive/doc/grive.1 index cf0bc26d..ccaef40f 100644 --- a/grive/doc/grive.1 +++ b/grive/doc/grive.1 @@ -48,7 +48,6 @@ Produces help message .TP \fB\-\-ignore\fR Ignore files with relative paths matching this Perl Regular Expression. -Value is remembered for next runs. .TP \fB\-l\fR , \fB\-\-log\fR Write log output to @@ -70,7 +69,7 @@ as the working copy root directory \fB\-s\fR , \fB\-\-dir\fR Sync a single .I -subdirectory. Internally converted to an ignore regexp, remembered for next runs. +subdirectory. Internally converted to an ignore regexp. .TP \fB\-v\fR, \fB\-\-version\fR Displays program version @@ -81,6 +80,37 @@ Print ASCII progress bar for each downloaded/uploaded file. \fB\-V\fR, \fB\-\-verbose\fR Verbose mode. Enables more messages than usual. +.SH .griveignore +.PP +You may create .griveignore in your Grive root and use it to setup +exclusion/inclusion rules. +.PP +Rules are similar to Git's .gitignore, but may differ slightly due to the different +implementation. +.IP \[bu] +lines that start with # are comments +.IP \[bu] +leading and trailing spaces ignored unless escaped with \\ +.IP \[bu] +non-empty lines without ! in front are treated as "exclude" patterns +.IP \[bu] +non-empty lines with ! in front are treated as "include" patterns +and have a priority over all "exclude" ones +.IP \[bu] +patterns are matched against the filenames relative to the grive root +.IP \[bu] +a/**/b matches any number of subpaths between a and b, including 0 +.IP \[bu] +**/a matches `a` inside any directory +.IP \[bu] +b/** matches everything inside `b`, but not b itself +.IP \[bu] +* matches any number of any characters except / +.IP \[bu] +? matches any character except / +.IP \[bu] +\[char46]griveignore itself isn't ignored by default, but you can include it in itself to ignore + .SH AUTHORS .PP Current maintainer is Vitaliy Filippov. diff --git a/grive/src/main.cc b/grive/src/main.cc index c87a8543..8009d121 100644 --- a/grive/src/main.cc +++ b/grive/src/main.cc @@ -46,8 +46,8 @@ #include #include -const std::string client_id = "22314510474.apps.googleusercontent.com" ; -const std::string client_secret = "bl4ufi89h-9MkFlypcI7R785" ; +const std::string default_id = APP_ID ; +const std::string default_secret = APP_SECRET ; using namespace gr ; namespace po = boost::program_options; @@ -111,6 +111,9 @@ int Main( int argc, char **argv ) ( "help,h", "Produce help message" ) ( "version,v", "Display Grive version" ) ( "auth,a", "Request authorization token" ) + ( "id,i", po::value(), "Authentication ID") + ( "secret,e", po::value(), "Authentication secret") + ( "print-url", "Only print url for request") ( "path,p", po::value(), "Path to working copy root") ( "dir,s", po::value(), "Single subdirectory to sync") ( "verbose,V", "Verbose mode. Enable more messages than normal.") @@ -124,15 +127,22 @@ int Main( int argc, char **argv ) ( "no-remote-new,n", "Download only files that are changed in Google Drive and already exist locally" ) ( "dry-run", "Only detect which files need to be uploaded/downloaded, " "without actually performing them." ) - ( "ignore", po::value(), "Perl RegExp to ignore files (matched against relative paths)." ) ( "upload-speed,U", po::value(), "Limit upload speed in kbytes per second" ) ( "download-speed,D", po::value(), "Limit download speed in kbytes per second" ) ( "progress-bar,P", "Enable progress bar for upload/download of files") ; po::variables_map vm; - po::store(po::parse_command_line( argc, argv, desc), vm ); - po::notify(vm); + try + { + po::store( po::parse_command_line( argc, argv, desc ), vm ); + } + catch( po::error &e ) + { + std::cerr << "Options are incorrect. Use -h for help\n"; + return -1; + } + po::notify( vm ); // simple commands that doesn't require log or config if ( vm.count("help") ) @@ -148,9 +158,9 @@ int Main( int argc, char **argv ) } // initialize logging - InitLog(vm) ; + InitLog( vm ) ; - Config config(vm) ; + Config config( vm ) ; Log( "config file name %1%", config.Filename(), log::verbose ); @@ -167,31 +177,48 @@ int Main( int argc, char **argv ) if ( vm.count( "auth" ) ) { - OAuth2 token( http.get(), client_id, client_secret ) ; - + std::string id = vm.count( "id" ) > 0 + ? vm["id"].as() + : default_id ; + std::string secret = vm.count( "secret" ) > 0 + ? vm["secret"].as() + : default_secret ; + + OAuth2 token( http.get(), id, secret ) ; + + if ( vm.count("print-url") ) + { + std::cout << token.MakeAuthURL() << std::endl ; + return 0 ; + } + std::cout << "-----------------------\n" - << "Please go to this URL and get an authentication code:\n\n" + << "Please open this URL in your browser to authenticate Grive2:\n\n" << token.MakeAuthURL() << std::endl ; - - std::cout - << "\n-----------------------\n" - << "Please input the authentication code here: " << std::endl ; - std::string code ; - std::cin >> code ; - - token.Auth( code ) ; - + + if ( !token.GetCode() ) + { + std::cout << "Authentication failed\n"; + return -1; + } + // save to config + config.Set( "id", Val( id ) ) ; + config.Set( "secret", Val( secret ) ) ; config.Set( "refresh_token", Val( token.RefreshToken() ) ) ; config.Save() ; } - + std::string refresh_token ; + std::string id ; + std::string secret ; try { refresh_token = config.Get("refresh_token").Str() ; + id = config.Get("id").Str() ; + secret = config.Get("secret").Str() ; } catch ( Exception& e ) { @@ -202,8 +229,8 @@ int Main( int argc, char **argv ) return -1; } - - OAuth2 token( http.get(), refresh_token, client_id, client_secret ) ; + + OAuth2 token( http.get(), refresh_token, id, secret ) ; AuthAgent agent( token, http.get() ) ; v2::Syncer2 syncer( &agent ); diff --git a/libgrive/CMakeLists.txt b/libgrive/CMakeLists.txt index 54776bd1..c56d70cf 100644 --- a/libgrive/CMakeLists.txt +++ b/libgrive/CMakeLists.txt @@ -4,12 +4,11 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") find_package(LibGcrypt REQUIRED) find_package(CURL REQUIRED) -find_package(EXPAT REQUIRED) +find_package(Backtrace) find_package(Boost 1.40.0 COMPONENTS program_options filesystem unit_test_framework regex system REQUIRED) find_package(BFD) find_package(CppUnit) find_package(Iberty) -find_package(ZLIB) find_package(PkgConfig) pkg_check_modules(YAJL REQUIRED yajl) @@ -21,15 +20,15 @@ IF ( CPPUNIT_FOUND ) set( OPT_INCS ${CPPUNIT_INCLUDE_DIR} ) ENDIF ( CPPUNIT_FOUND ) -# build bfd classes if libbfd is found -if ( BFD_FOUND ) - set( OPT_LIBS ${DL_LIBRARY} ${BFD_LIBRARY} ) +# build bfd classes if libbfd and the backtrace library is found +if ( BFD_FOUND AND Backtrace_FOUND ) + set( OPT_LIBS ${BFD_LIBRARY} ${Backtrace_LIBRARY} ) file( GLOB OPT_SRC src/bfd/*.cc ) add_definitions( -DHAVE_BFD ) -endif ( BFD_FOUND ) +endif ( BFD_FOUND AND Backtrace_FOUND ) if ( IBERTY_FOUND ) set( OPT_LIBS ${OPT_LIBS} ${IBERTY_LIBRARY} ) @@ -37,10 +36,6 @@ else ( IBERTY_FOUND ) set( IBERTY_LIBRARY "" ) endif ( IBERTY_FOUND ) -if ( ZLIB_FOUND ) - set( OPT_LIBS ${OPT_LIBS} ${ZLIB_LIBRARIES} ) -endif ( ZLIB_FOUND ) - include_directories( ${libgrive_SOURCE_DIR}/src ${libgrive_SOURCE_DIR}/test @@ -83,9 +78,11 @@ target_link_libraries( grive ${YAJL_LIBRARIES} ${CURL_LIBRARIES} ${LIBGCRYPT_LIBRARIES} - ${Boost_LIBRARIES} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_REGEX_LIBRARY} + ${Boost_SYSTEM_LIBRARY} ${IBERTY_LIBRARY} - ${EXPAT_LIBRARY} ${OPT_LIBS} ) @@ -126,7 +123,6 @@ IF ( CPPUNIT_FOUND ) target_link_libraries( unittest grive ${CPPUNIT_LIBRARY} - ${Boost_LIBRARIES} ) ENDIF ( CPPUNIT_FOUND ) @@ -139,9 +135,13 @@ add_executable( btest ${BTEST_SRC} ) target_link_libraries( btest grive - ${Boost_LIBRARIES} + ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ) +if ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++11-narrowing" ) +endif ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" ) + if ( WIN32 ) else ( WIN32 ) set_target_properties( btest diff --git a/libgrive/src/base/Drive.cc b/libgrive/src/base/Drive.cc index fd26225c..28fd29a1 100644 --- a/libgrive/src/base/Drive.cc +++ b/libgrive/src/base/Drive.cc @@ -41,15 +41,10 @@ namespace gr { -namespace -{ - const std::string state_file = ".grive_state" ; -} - Drive::Drive( Syncer *syncer, const Val& options ) : m_syncer ( syncer ), m_root ( options["path"].Str() ), - m_state ( m_root / state_file, options ), + m_state ( m_root, options ), m_options ( options ) { assert( m_syncer ) ; @@ -72,7 +67,7 @@ void Drive::FromChange( const Entry& entry ) void Drive::SaveState() { - m_state.Write( m_root / state_file ) ; + m_state.Write() ; } void Drive::DetectChanges() diff --git a/libgrive/src/base/Feed.cc b/libgrive/src/base/Feed.cc index 0ea834d3..0a4f51b1 100644 --- a/libgrive/src/base/Feed.cc +++ b/libgrive/src/base/Feed.cc @@ -30,6 +30,10 @@ Feed::Feed( const std::string &url ): { } +Feed::~Feed() +{ +} + Feed::iterator Feed::begin() const { return m_entries.begin() ; diff --git a/libgrive/src/base/Feed.hh b/libgrive/src/base/Feed.hh index d43c68ce..c12f5070 100644 --- a/libgrive/src/base/Feed.hh +++ b/libgrive/src/base/Feed.hh @@ -41,6 +41,7 @@ public : public : Feed( const std::string& url ); virtual bool GetNext( http::Agent *http ) = 0 ; + virtual ~Feed() = 0 ; iterator begin() const ; iterator end() const ; diff --git a/libgrive/src/base/Resource.cc b/libgrive/src/base/Resource.cc index 23ce335d..72c8052a 100644 --- a/libgrive/src/base/Resource.cc +++ b/libgrive/src/base/Resource.cc @@ -262,10 +262,10 @@ void Resource::FromLocal( Val& state ) if ( !IsRoot() ) { fs::path path = Path() ; - bool is_dir; + FileType ft ; try { - os::Stat( path, &m_ctime, (off64_t*)&m_size, &is_dir ) ; + os::Stat( path, &m_ctime, (off64_t*)&m_size, &ft ) ; } catch ( os::Error &e ) { @@ -276,22 +276,30 @@ void Resource::FromLocal( Val& state ) m_kind = "bad"; return; } + if ( ft == FT_UNKNOWN ) + { + // Skip sockets/FIFOs/etc + Log( "File %1% is not a regular file or directory; skipping file", path.string(), log::warning ); + m_state = sync; + m_kind = "bad"; + return; + } m_name = path.filename().string() ; - m_kind = is_dir ? "folder" : "file"; + m_kind = ft == FT_DIR ? "folder" : "file"; m_local_exists = true; bool is_changed; if ( state.Has( "ctime" ) && (u64_t) m_ctime.Sec() <= state["ctime"].U64() && - ( is_dir || state.Has( "md5" ) ) ) + ( ft == FT_DIR || state.Has( "md5" ) ) ) { - if ( !is_dir ) + if ( ft != FT_DIR ) m_md5 = state["md5"]; is_changed = false; } else { - if ( !is_dir ) + if ( ft != FT_DIR ) { // File is changed locally. TODO: Detect conflicts is_changed = ( state.Has( "size" ) && m_size != state["size"].U64() ) || @@ -703,13 +711,13 @@ void Resource::SetIndex( bool re_stat ) assert( m_parent && m_parent->m_json != NULL ); if ( !m_json ) m_json = &((*m_parent->m_json)["tree"]).Item( Name() ); - bool is_dir; + FileType ft; if ( re_stat ) - os::Stat( Path(), &m_ctime, NULL, &is_dir ); + os::Stat( Path(), &m_ctime, NULL, &ft ); else - is_dir = IsFolder(); + ft = IsFolder() ? FT_DIR : FT_FILE; m_json->Set( "ctime", Val( m_ctime.Sec() ) ); - if ( !is_dir ) + if ( ft != FT_DIR ) { m_json->Set( "md5", Val( m_md5 ) ); m_json->Set( "size", Val( m_size ) ); diff --git a/libgrive/src/base/State.cc b/libgrive/src/base/State.cc index 9fddf4d1..9f414bbf 100644 --- a/libgrive/src/base/State.cc +++ b/libgrive/src/base/State.cc @@ -28,21 +28,34 @@ #include "util/log/Log.hh" #include "json/JsonParser.hh" +#include + #include namespace gr { -State::State( const fs::path& filename, const Val& options ) : +const std::string state_file = ".grive_state" ; +const std::string ignore_file = ".griveignore" ; +const int MAX_IGN = 65536 ; +const char* regex_escape_chars = ".^$|()[]{}*+?\\"; +const boost::regex regex_escape_re( "[.^$|()\\[\\]{}*+?\\\\]" ); + +inline std::string regex_escape( std::string s ) +{ + return regex_replace( s, regex_escape_re, "\\\\&", boost::format_sed ); +} + +State::State( const fs::path& root, const Val& options ) : + m_root ( root ), m_res ( options["path"].Str() ), m_cstamp ( -1 ) { - Read( filename ) ; - + Read() ; + // the "-f" option will make grive always think remote is newer m_force = options.Has( "force" ) ? options["force"].Bool() : false ; - + std::string m_orig_ign = m_ign; - m_ign = ""; if ( options.Has( "ignore" ) && options["ignore"].Str() != m_ign ) m_ign = options["ignore"].Str(); else if ( options.Has( "dir" ) ) @@ -52,8 +65,7 @@ State::State( const fs::path& filename, const Val& options ) : if ( !m_dir.empty() ) { // "-s" is internally converted to an ignore regexp - const boost::regex esc( "[.^$|()\\[\\]{}*+?\\\\]" ); - m_dir = regex_replace( m_dir, esc, "\\\\&", boost::format_sed ); + m_dir = regex_escape( m_dir ); size_t pos = 0; while ( ( pos = m_dir.find( '/', pos ) ) != std::string::npos ) { @@ -66,7 +78,7 @@ State::State( const fs::path& filename, const Val& options ) : } m_ign_changed = m_orig_ign != "" && m_orig_ign != m_ign; - m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive|grive_state|trash)" : ( m_ign+"|^\\.(grive|grive_state|trash)" ) ); + m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive$|grive_state$|trash)" : ( m_ign+"|^\\.(grive$|grive_state$|trash)" ) ); } State::~State() @@ -83,7 +95,7 @@ void State::FromLocal( const fs::path& p ) bool State::IsIgnore( const std::string& filename ) { - return regex_search( filename.c_str(), m_ign_re ); + return regex_search( filename.c_str(), m_ign_re, boost::format_perl ); } void State::FromLocal( const fs::path& p, Resource* folder, Val& tree ) @@ -96,7 +108,7 @@ void State::FromLocal( const fs::path& p, Resource* folder, Val& tree ) for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i ) { std::string fname = i->path().filename().string() ; - std::string path = folder->IsRoot() ? fname : ( folder->RelPath() / fname ).string(); + std::string path = ( folder->IsRoot() ? fname : ( folder->RelPath() / fname ).string() ); if ( IsIgnore( path ) ) Log( "file %1% is ignored by grive", path, log::verbose ) ; @@ -130,7 +142,7 @@ void State::FromLocal( const fs::path& p, Resource* folder, Val& tree ) else { // Restore state of locally deleted files - Resource *c = folder->FindChild( i->first ), *c2 ; + Resource *c = folder->FindChild( i->first ), *c2 = c ; if ( !c ) { c2 = new Resource( i->first, i->second.Has( "tree" ) ? "folder" : "file" ) ; @@ -225,6 +237,12 @@ bool State::Update( const Entry& e ) } else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) ) { + if ( !parent->IsFolder() ) + { + // https://github.com/vitalif/grive2/issues/148 + Log( "%1% is owned by something that's not a directory: href=%2% name=%3%", e.Name(), e.ParentHref(), parent->RelPath(), log::error ); + return true; + } assert( parent->IsFolder() ) ; std::string path = parent->IsRoot() ? e.Name() : ( parent->RelPath() / e.Name() ).string(); @@ -277,27 +295,118 @@ State::iterator State::end() return m_res.end() ; } -void State::Read( const fs::path& filename ) +void State::Read() { try { - File file( filename ) ; - - m_st = ParseJson( file ); - m_ign = m_st.Has( "ignore_regexp" ) ? m_st["ignore_regexp"].Str() : std::string(); - + File st_file( m_root / state_file ) ; + m_st = ParseJson( st_file ); m_cstamp = m_st["change_stamp"].Int() ; } catch ( Exception& ) { } + + try + { + File ign_file( m_root / ignore_file ) ; + char ign[MAX_IGN] = { 0 }; + int s = ign_file.Read( ign, MAX_IGN-1 ) ; + ParseIgnoreFile( ign, s ); + } + catch ( Exception& e ) + { + } +} + +std::vector split( const boost::regex& re, const char* str, int len ) +{ + std::vector vec; + boost::cregex_token_iterator i( str, str+len, re, -1, boost::format_perl ); + boost::cregex_token_iterator j; + while ( i != j ) + { + vec.push_back( *i++ ); + } + return vec; +} + +bool State::ParseIgnoreFile( const char* buffer, int size ) +{ + const boost::regex re1( "([^\\\\]|^)[\\t\\r ]+$" ); + const boost::regex re2( "^[\\t\\r ]+" ); + const boost::regex re4( "([^\\\\](\\\\\\\\)*|^)\\\\\\*" ); + const boost::regex re5( "([^\\\\](\\\\\\\\)*|^)\\\\\\?" ); + std::string exclude_re, include_re; + std::vector lines = split( boost::regex( "[\\n\\r]+" ), buffer, size ); + for ( int i = 0; i < (int)lines.size(); i++ ) + { + std::string str = regex_replace( regex_replace( lines[i], re1, "$1" ), re2, "" ); + if ( str[0] == '#' || !str.size() ) + { + continue; + } + bool inc = str[0] == '!'; + if ( inc ) + { + str = str.substr( 1 ); + } + std::vector parts = split( boost::regex( "/+" ), str.c_str(), str.size() ); + for ( int j = 0; j < (int)parts.size(); j++ ) + { + if ( parts[j] == "**" ) + { + parts[j] = ".*"; + } + else if ( parts[j] == "*" ) + { + parts[j] = "[^/]*"; + } + else + { + parts[j] = regex_escape( parts[j] ); + std::string str1; + while (1) + { + str1 = regex_replace( parts[j], re5, "$1[^/]", boost::format_perl ); + str1 = regex_replace( str1, re4, "$1[^/]*", boost::format_perl ); + if ( str1.size() == parts[j].size() ) + break; + parts[j] = str1; + } + } + } + if ( !inc ) + { + str = boost::algorithm::join( parts, "/" ) + "(/|$)"; + exclude_re = exclude_re + ( exclude_re.size() > 0 ? "|" : "" ) + str; + } + else + { + str = ""; + std::string cur; + for ( int j = 0; j < (int)parts.size(); j++ ) + { + cur = cur.size() > 0 ? cur + "/" + parts[j] : "^" + parts[j]; + str = ( str.size() > 0 ? str + "|" + cur : cur ) + ( j < (int)parts.size()-1 ? "$" : "(/|$)" ); + } + include_re = include_re + ( include_re.size() > 0 ? "|" : "" ) + str; + } + } + if ( exclude_re.size() > 0 ) + { + m_ign = "^" + ( include_re.size() > 0 ? "(?!" + include_re + ")" : std::string() ) + "(" + exclude_re + ")$"; + return true; + } + return false; } -void State::Write( const fs::path& filename ) +void State::Write() { m_st.Set( "change_stamp", Val( m_cstamp ) ) ; m_st.Set( "ignore_regexp", Val( m_ign ) ) ; + fs::path filename = m_root / state_file ; std::ofstream fs( filename.string().c_str() ) ; fs << m_st ; } diff --git a/libgrive/src/base/State.hh b/libgrive/src/base/State.hh index 68b6df88..b42a32e7 100644 --- a/libgrive/src/base/State.hh +++ b/libgrive/src/base/State.hh @@ -42,15 +42,15 @@ public : typedef ResourceTree::iterator iterator ; public : - explicit State( const fs::path& filename, const Val& options ) ; + explicit State( const fs::path& root, const Val& options ) ; ~State() ; void FromLocal( const fs::path& p ) ; void FromRemote( const Entry& e ) ; void ResolveEntry() ; - void Read( const fs::path& filename ) ; - void Write( const fs::path& filename ) ; + void Read() ; + void Write() ; Resource* FindByHref( const std::string& href ) ; Resource* FindByID( const std::string& id ) ; @@ -64,6 +64,7 @@ public : void ChangeStamp( long cstamp ) ; private : + bool ParseIgnoreFile( const char* buffer, int size ) ; void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ; void FromChange( const Entry& e ) ; bool Update( const Entry& e ) ; @@ -72,6 +73,7 @@ private : bool IsIgnore( const std::string& filename ) ; private : + fs::path m_root ; ResourceTree m_res ; int m_cstamp ; std::string m_ign ; diff --git a/libgrive/src/base/Syncer.hh b/libgrive/src/base/Syncer.hh index 237b8949..9b6ae4f7 100644 --- a/libgrive/src/base/Syncer.hh +++ b/libgrive/src/base/Syncer.hh @@ -21,6 +21,7 @@ #include "util/FileSystem.hh" +#include #include #include #include diff --git a/libgrive/src/bfd/SymbolInfo.cc b/libgrive/src/bfd/SymbolInfo.cc index 5876cc00..cdb5d7c7 100644 --- a/libgrive/src/bfd/SymbolInfo.cc +++ b/libgrive/src/bfd/SymbolInfo.cc @@ -49,9 +49,9 @@ SymbolInfo::SymbolInfo( ) m_impl->m_bfd = 0 ; m_impl->m_symbols = 0 ; m_impl->m_symbol_count = 0 ; - + bfd_init( ) ; - + // opening itself bfd *b = bfd_openr( "/proc/self/exe", 0 ) ; if ( b == NULL ) @@ -60,13 +60,13 @@ SymbolInfo::SymbolInfo( ) << bfd_errmsg( bfd_get_error() ) << std::endl ; return ; } - + if ( bfd_check_format( b, bfd_archive ) ) { bfd_close( b ) ; return ; } - + char **matching ; if ( !bfd_check_format_matches( b, bfd_object, &matching ) ) { @@ -78,7 +78,7 @@ SymbolInfo::SymbolInfo( ) std::cerr << bfd_get_filename( b ) << ": Matching formats: " ; for ( char **p = matching ; *p != 0 ; p++ ) std::cerr << " " << *p ; - + std::cerr << std::endl ; std::free( matching ) ; } @@ -107,7 +107,7 @@ struct SymbolInfo::BacktraceInfo const char *m_func_name ; unsigned int m_lineno ; unsigned int m_is_found ; - + static void Callback( bfd *abfd, asection *section, void* addr ) ; } ; @@ -117,17 +117,24 @@ void SymbolInfo::BacktraceInfo::Callback( bfd *abfd, asection *section, BacktraceInfo *info = (BacktraceInfo *)data ; if ((section->flags & SEC_ALLOC) == 0) return ; - - bfd_vma vma = bfd_get_section_vma(abfd, section); - + + // bfd_get_section_vma works up to 7b1cfbcf1a27951fb1b3a212995075dd6fdf985b, + // removed in 7c13bc8c91abf291f0206b6608b31955c5ea70d8 (binutils 2.33.1 or so) + // so it's substituted by its implementation to avoid checking for binutils + // version (which at least on Debian SID it's not that easy because the + // version.h is not included with the official package) + bfd_vma vma = section->vma; + unsigned long address = (unsigned long)(info->m_addr); if ( address < vma ) return; - - bfd_size_type size = bfd_section_size(abfd, section); + + // bfd_section_size changed between the two objects described above, + // same rationale applies + bfd_size_type size = section->size; if ( address > (vma + size)) return ; - + const SymbolInfo *pthis = info->m_pthis ; info->m_is_found = bfd_find_nearest_line( abfd, section, pthis->m_impl->m_symbols, @@ -149,7 +156,7 @@ void SymbolInfo::PrintTrace( void *addr, std::ostream& os, std::size_t idx ) { this, addr, 0, 0, 0, false } ; - + Dl_info sym ; bfd_map_over_sections( m_impl->m_bfd, &SymbolInfo::BacktraceInfo::Callback, @@ -165,7 +172,7 @@ if ( btinfo.m_is_found ) filename.erase( pos, std::strlen( SRC_DIR ) ) ; #endif os << "#" << idx << " " << addr << " " - << filename << ":" << btinfo.m_lineno + << filename << ":" << btinfo.m_lineno << " " << (btinfo.m_func_name != 0 ? Demangle(btinfo.m_func_name) : "" ) << std::endl ; diff --git a/libgrive/src/drive2/Feed2.cc b/libgrive/src/drive2/Feed2.cc index 8f3d2344..037df373 100644 --- a/libgrive/src/drive2/Feed2.cc +++ b/libgrive/src/drive2/Feed2.cc @@ -36,6 +36,10 @@ Feed2::Feed2( const std::string& url ): { } +Feed2::~Feed2() +{ +} + bool Feed2::GetNext( http::Agent *http ) { if ( m_next.empty() ) diff --git a/libgrive/src/drive2/Feed2.hh b/libgrive/src/drive2/Feed2.hh index 4de411e9..c5643f6f 100644 --- a/libgrive/src/drive2/Feed2.hh +++ b/libgrive/src/drive2/Feed2.hh @@ -31,6 +31,7 @@ class Feed2: public Feed { public : Feed2( const std::string& url ) ; + ~Feed2() ; bool GetNext( http::Agent *http ) ; } ; diff --git a/libgrive/src/drive2/Syncer2.cc b/libgrive/src/drive2/Syncer2.cc index fa5e05dd..e25632ac 100644 --- a/libgrive/src/drive2/Syncer2.cc +++ b/libgrive/src/drive2/Syncer2.cc @@ -213,12 +213,12 @@ bool Syncer2::Upload( Resource *res, bool new_rev ) std::unique_ptr Syncer2::GetFolders() { - return std::unique_ptr( new Feed2( feeds::files + "?maxResults=1000&q=trashed%3dfalse+and+mimeType%3d%27" + mime_types::folder + "%27" ) ); + return std::unique_ptr( new Feed2( feeds::files + "?maxResults=100000&q=trashed%3dfalse+and+mimeType%3d%27" + mime_types::folder + "%27" ) ); } std::unique_ptr Syncer2::GetAll() { - return std::unique_ptr( new Feed2( feeds::files + "?maxResults=1000&q=trashed%3dfalse" ) ); + return std::unique_ptr( new Feed2( feeds::files + "?maxResults=999999999&q=trashed%3dfalse" ) ); } std::string ChangesFeed( long changestamp, int maxResults = 1000 ) diff --git a/libgrive/src/http/CurlAgent.cc b/libgrive/src/http/CurlAgent.cc index eee9a8d9..8a9d43fc 100644 --- a/libgrive/src/http/CurlAgent.cc +++ b/libgrive/src/http/CurlAgent.cc @@ -190,8 +190,10 @@ long CurlAgent::ExecCurl( struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ; curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback); - curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); + #if LIBCURL_VERSION_NUM >= 0x072000 + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); + #endif CURLcode curl_code = ::curl_easy_perform(curl); diff --git a/libgrive/src/protocol/AuthAgent.cc b/libgrive/src/protocol/AuthAgent.cc index 6baf1133..b15452a3 100644 --- a/libgrive/src/protocol/AuthAgent.cc +++ b/libgrive/src/protocol/AuthAgent.cc @@ -81,6 +81,7 @@ long AuthAgent::Request( { long response; Header auth; + m_interval = 0; do { auth = AppendHeader( hdr ); @@ -127,7 +128,17 @@ bool AuthAgent::CheckRetry( long response ) os::Sleep( 5 ) ; return true ; } - + // HTTP 403 is the result of API rate limiting. attempt exponential backoff and try again + else if ( response == 429 || ( response == 403 && ( + m_agent->LastError().find("\"reason\": \"userRateLimitExceeded\",") != std::string::npos || + m_agent->LastError().find("\"reason\": \"rateLimitExceeded\",") != std::string::npos ) ) ) + { + m_interval = m_interval <= 0 ? 1 : ( m_interval < 64 ? m_interval*2 : 120 ); + Log( "request failed due to rate limiting: %1% (body: %2%). retrying in %3% seconds", + response, m_agent->LastError(), m_interval, log::warning ) ; + os::Sleep( m_interval ) ; + return true ; + } // HTTP 401 Unauthorized. the auth token has been expired. refresh it else if ( response == 401 ) { diff --git a/libgrive/src/protocol/AuthAgent.hh b/libgrive/src/protocol/AuthAgent.hh index 328e8cf4..3068e7d1 100644 --- a/libgrive/src/protocol/AuthAgent.hh +++ b/libgrive/src/protocol/AuthAgent.hh @@ -71,6 +71,7 @@ private : private : OAuth2& m_auth ; http::Agent* m_agent ; + int m_interval ; } ; } // end of namespace diff --git a/libgrive/src/protocol/OAuth2.cc b/libgrive/src/protocol/OAuth2.cc index 73ae0886..478f1082 100644 --- a/libgrive/src/protocol/OAuth2.cc +++ b/libgrive/src/protocol/OAuth2.cc @@ -25,6 +25,13 @@ #include "http/Header.hh" #include "util/log/Log.hh" +#include +#include +#include +#include +#include +#include + // for debugging #include @@ -50,18 +57,29 @@ OAuth2::OAuth2( const std::string& client_id, const std::string& client_secret ) : m_agent( agent ), + m_port( 0 ), + m_socket( -1 ), m_client_id( client_id ), m_client_secret( client_secret ) { } -void OAuth2::Auth( const std::string& auth_code ) +OAuth2::~OAuth2() +{ + if ( m_socket >= 0 ) + { + close( m_socket ); + m_socket = -1; + } +} + +bool OAuth2::Auth( const std::string& auth_code ) { std::string post = "code=" + auth_code + "&client_id=" + m_client_id + "&client_secret=" + m_client_secret + - "&redirect_uri=" + "urn:ietf:wg:oauth:2.0:oob" + + "&redirect_uri=http%3A%2F%2Flocalhost:" + std::to_string( m_port ) + "%2Fauth" + "&grant_type=authorization_code" ; http::ValResponse resp ; @@ -77,24 +95,120 @@ void OAuth2::Auth( const std::string& auth_code ) { Log( "Failed to obtain auth token: HTTP %1%, body: %2%", code, m_agent->LastError(), log::error ) ; - BOOST_THROW_EXCEPTION( AuthFailed() ); + return false; } + + return true; } std::string OAuth2::MakeAuthURL() { + if ( !m_port ) + { + sockaddr_storage addr = { 0 }; + addr.ss_family = AF_INET; + m_socket = socket( AF_INET, SOCK_STREAM, 0 ); + if ( m_socket < 0 ) + throw std::runtime_error( std::string("socket: ") + strerror(errno) ); + if ( bind( m_socket, (sockaddr*)&addr, sizeof( addr ) ) < 0 ) + { + close( m_socket ); + m_socket = -1; + throw std::runtime_error( std::string("bind: ") + strerror(errno) ); + } + socklen_t len = sizeof( addr ); + if ( getsockname( m_socket, (sockaddr *)&addr, &len ) == -1 ) + { + close( m_socket ); + m_socket = -1; + throw std::runtime_error( std::string("getsockname: ") + strerror(errno) ); + } + m_port = ntohs(((sockaddr_in*)&addr)->sin_port); + if ( listen( m_socket, 128 ) < 0 ) + { + close( m_socket ); + m_socket = -1; + m_port = 0; + throw std::runtime_error( std::string("listen: ") + strerror(errno) ); + } + } return "https://accounts.google.com/o/oauth2/auth" - "?scope=" + - m_agent->Escape( "https://www.googleapis.com/auth/userinfo.email" ) + "+" + - m_agent->Escape( "https://www.googleapis.com/auth/userinfo.profile" ) + "+" + - m_agent->Escape( "https://docs.google.com/feeds/" ) + "+" + - m_agent->Escape( "https://docs.googleusercontent.com/" ) + "+" + - m_agent->Escape( "https://spreadsheets.google.com/feeds/" ) + - "&redirect_uri=urn:ietf:wg:oauth:2.0:oob" + "?scope=" + m_agent->Escape( "https://www.googleapis.com/auth/drive" ) + + "&redirect_uri=http%3A%2F%2Flocalhost:" + std::to_string( m_port ) + "%2Fauth" + "&response_type=code" "&client_id=" + m_client_id ; } +bool OAuth2::GetCode( ) +{ + sockaddr_storage addr = { 0 }; + int peer_fd = -1; + while ( peer_fd < 0 ) + { + socklen_t peer_addr_size = sizeof( addr ); + peer_fd = accept( m_socket, (sockaddr*)&addr, &peer_addr_size ); + if ( peer_fd == -1 && errno != EAGAIN && errno != EINTR ) + throw std::runtime_error( std::string("accept: ") + strerror(errno) ); + } + fcntl( peer_fd, F_SETFL, fcntl( peer_fd, F_GETFL, 0 ) | O_NONBLOCK ); + struct pollfd pfd = (struct pollfd){ + .fd = peer_fd, + .events = POLLIN|POLLRDHUP, + }; + char buf[4096]; + std::string request; + while ( true ) + { + pfd.revents = 0; + poll( &pfd, 1, -1 ); + if ( pfd.revents & POLLRDHUP ) + break; + int r = 1; + while ( r > 0 ) + { + r = read( peer_fd, buf, sizeof( buf ) ); + if ( r > 0 ) + request += std::string( buf, r ); + else if ( r == 0 ) + break; + else if ( errno != EAGAIN && errno != EINTR ) + throw std::runtime_error( std::string("read: ") + strerror(errno) ); + } + if ( r == 0 || ( r < 0 && request.find( "\n" ) > 0 ) ) // GET ... HTTP/1.1\r\n + break; + } + bool ok = false; + if ( request.substr( 0, 10 ) == "GET /auth?" ) + { + std::string line = request; + int p = line.find( "\n" ); + if ( p > 0 ) + line = line.substr( 0, p ); + p = line.rfind( " " ); + if ( p > 0 ) + line = line.substr( 0, p ); + p = line.find( "code=" ); + if ( p > 0 ) + line = line.substr( p+5 ); + p = line.find( "&" ); + if ( p > 0 ) + line = line.substr( 0, p ); + ok = Auth( line ); + } + std::string response = ( ok + ? "Authenticated successfully. Please close the page" + : "Authentication error. Please try again" ); + response = "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n"+ + response+ + "\r\n"; + write( peer_fd, response.c_str(), response.size() ); + close( peer_fd ); + return ok; +} + void OAuth2::Refresh( ) { std::string post = diff --git a/libgrive/src/protocol/OAuth2.hh b/libgrive/src/protocol/OAuth2.hh index e9a23dae..ae40db75 100644 --- a/libgrive/src/protocol/OAuth2.hh +++ b/libgrive/src/protocol/OAuth2.hh @@ -41,13 +41,15 @@ public : const std::string& refresh_code, const std::string& client_id, const std::string& client_secret ) ; + ~OAuth2( ) ; std::string Str() const ; std::string MakeAuthURL() ; - void Auth( const std::string& auth_code ) ; + bool Auth( const std::string& auth_code ) ; void Refresh( ) ; + bool GetCode( ) ; std::string RefreshToken( ) const ; std::string AccessToken( ) const ; @@ -59,7 +61,9 @@ private : std::string m_access ; std::string m_refresh ; http::Agent* m_agent ; - + int m_port ; + int m_socket ; + const std::string m_client_id ; const std::string m_client_secret ; } ; diff --git a/libgrive/src/util/Config.cc b/libgrive/src/util/Config.cc index ba0c91e3..7d17a55d 100644 --- a/libgrive/src/util/Config.cc +++ b/libgrive/src/util/Config.cc @@ -38,6 +38,10 @@ const std::string default_root_folder = "."; Config::Config( const po::variables_map& vm ) { + if ( vm.count( "id" ) > 0 ) + m_cmd.Add( "id", Val( vm["id"].as() ) ) ; + if ( vm.count( "secret" ) > 0 ) + m_cmd.Add( "secret", Val( vm["secret"].as() ) ) ; m_cmd.Add( "new-rev", Val(vm.count("new-rev") > 0) ) ; m_cmd.Add( "force", Val(vm.count("force") > 0 ) ) ; m_cmd.Add( "path", Val(vm.count("path") > 0 @@ -80,7 +84,7 @@ void Config::Save( ) void Config::Set( const std::string& key, const Val& value ) { - m_file.Add( key, value ) ; + m_file.Set( key, value ) ; } Val Config::Get( const std::string& key ) const diff --git a/libgrive/src/util/File.cc b/libgrive/src/util/File.cc index 375cd625..c5184ecd 100644 --- a/libgrive/src/util/File.cc +++ b/libgrive/src/util/File.cc @@ -33,6 +33,10 @@ #include #include +#if defined(__FreeBSD__) || defined(__OpenBSD__) +#include +#endif + #ifdef WIN32 #include typedef int ssize_t ; diff --git a/libgrive/src/util/OS.cc b/libgrive/src/util/OS.cc index 9095e730..7c5fa70b 100644 --- a/libgrive/src/util/OS.cc +++ b/libgrive/src/util/OS.cc @@ -39,12 +39,12 @@ namespace gr { namespace os { -void Stat( const fs::path& filename, DateTime *t, off_t *size, bool *is_dir ) +void Stat( const fs::path& filename, DateTime *t, off_t *size, FileType *ft ) { - Stat( filename.string(), t, size, is_dir ) ; + Stat( filename.string(), t, size, ft ) ; } -void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) +void Stat( const std::string& filename, DateTime *t, off64_t *size, FileType *ft ) { struct stat s = {} ; if ( ::stat( filename.c_str(), &s ) != 0 ) @@ -57,9 +57,9 @@ void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) ; } - if (t) + if ( t ) { -#if defined __APPLE__ && defined __DARWIN_64_BIT_INO_T +#if defined __NetBSD__ || ( defined __APPLE__ && defined __DARWIN_64_BIT_INO_T ) *t = DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ; #else *t = DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec); @@ -67,8 +67,8 @@ void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir } if ( size ) *size = s.st_size; - if ( is_dir ) - *is_dir = S_ISDIR( s.st_mode ) ? true : false; + if ( ft ) + *ft = S_ISDIR( s.st_mode ) ? FT_DIR : ( S_ISREG( s.st_mode ) ? FT_FILE : FT_UNKNOWN ) ; } void SetFileTime( const fs::path& filename, const DateTime& t ) diff --git a/libgrive/src/util/OS.hh b/libgrive/src/util/OS.hh index 43497839..3fe89b8b 100644 --- a/libgrive/src/util/OS.hh +++ b/libgrive/src/util/OS.hh @@ -29,12 +29,18 @@ namespace gr { class DateTime ; class Path ; +enum FileType { FT_FILE = 1, FT_DIR = 2, FT_UNKNOWN = 3 } ; + +#ifndef off64_t +#define off64_t off_t +#endif + namespace os { struct Error : virtual Exception {} ; - void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) ; - void Stat( const fs::path& filename, DateTime *t, off64_t *size, bool *is_dir ) ; + void Stat( const std::string& filename, DateTime *t, off64_t *size, FileType *ft ) ; + void Stat( const fs::path& filename, DateTime *t, off64_t *size, FileType *ft ) ; void SetFileTime( const std::string& filename, const DateTime& t ) ; void SetFileTime( const fs::path& filename, const DateTime& t ) ; diff --git a/libgrive/src/xml/TreeBuilder.cc b/libgrive/src/xml/TreeBuilder.cc index 9d57501b..14142557 100644 --- a/libgrive/src/xml/TreeBuilder.cc +++ b/libgrive/src/xml/TreeBuilder.cc @@ -23,7 +23,6 @@ #include "Node.hh" #include "util/log/Log.hh" -#include #include #include diff --git a/systemd/CMakeLists.txt b/systemd/CMakeLists.txt new file mode 100644 index 00000000..36d6d22a --- /dev/null +++ b/systemd/CMakeLists.txt @@ -0,0 +1,27 @@ +SET(GRIVE_SYNC_SH_BINARY "${CMAKE_INSTALL_FULL_LIBEXECDIR}/grive/grive-sync.sh") + +CONFIGURE_FILE(grive-changes@.service.in grive-changes@.service @ONLY) +CONFIGURE_FILE(grive-timer@.service.in grive-timer@.service @ONLY) + +install( + FILES + grive@.service + ${CMAKE_BINARY_DIR}/systemd/grive-changes@.service + ${CMAKE_BINARY_DIR}/systemd/grive-timer@.service + DESTINATION + lib/systemd/user +) + +install( + FILES + grive-timer@.timer + DESTINATION + lib/systemd/user +) + +install( + PROGRAMS + grive-sync.sh + DESTINATION + ${CMAKE_INSTALL_FULL_LIBEXECDIR}/grive +) diff --git a/systemd/grive-changes@.service.in b/systemd/grive-changes@.service.in new file mode 100644 index 00000000..172ab197 --- /dev/null +++ b/systemd/grive-changes@.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Google drive sync (changed files) + +[Service] +ExecStart=@GRIVE_SYNC_SH_BINARY@ listen "%i" +Type=simple +Restart=always +RestartSec=30 + +[Install] +WantedBy=default.target diff --git a/systemd/grive-sync.sh b/systemd/grive-sync.sh new file mode 100755 index 00000000..c80f1de5 --- /dev/null +++ b/systemd/grive-sync.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Copyright (C) 2009 Przemyslaw Pawelczyk +# (C) 2017 Jan Schulz +## +## This script is licensed under the terms of the MIT license. +## https://opensource.org/licenses/MIT + +# Fail on all errors +set -o pipefail + +# We always start in the current users home directory so that names always start there +cd ~ + + +### ARGUMENT PARSING ### +SCRIPT="${0}" +DIRECTORY=$(systemd-escape --unescape -- "$2") + +if [[ -z "$DIRECTORY" ]] || [[ ! -d "$DIRECTORY" ]] ; then + echo "Need a directory name in the current users home directory as second argument. Aborting." + exit 1 +fi + + +if [[ -z "${1}" ]] ; then + echo "Need a command as first argument. Aborting." + exit 1 +else + if [[ "sync" == "${1}" ]] ; then + COMMAND=sync + elif [[ "listen" == "${1}" ]] ; then + COMMAND=listen + else + echo "Unknown command. Aborting." + exit 1 + fi +fi + + +### LOCKFILE BOILERPLATE ### +LOCKFILE="/run/user/"$(id -u)"/"$(basename "$0")"_"${DIRECTORY//\//_}"" +LOCKFD=99 + +# PRIVATE +_lock() { flock -"$1" "$LOCKFD"; } +_no_more_locking() { _lock u; _lock xn && rm -f "$LOCKFILE"; } +_prepare_locking() { eval "exec "$LOCKFD">\""$LOCKFILE"\""; trap _no_more_locking EXIT; } + +# ON START +_prepare_locking + +# PUBLIC +exlock_now() { _lock xn; } # obtain an exclusive lock immediately or fail +exlock() { _lock x; } # obtain an exclusive lock +shlock() { _lock s; } # obtain a shared lock +unlock() { _lock u; } # drop a lock + +### SYNC SCRIPT ### +# Idea: only let one script run, but if the sync script is called a second time +# make sure we sync a second time, too + +sync_directory() { + _directory="${1}" + + reset_timer_and_exit() { echo "Retriggered google drive sync ('${_directory}')" && touch -m $LOCKFILE && exit; } + + exlock_now || reset_timer_and_exit + + if ping -c1 -W1 -q accounts.google.com >/dev/null 2>&1; then + true + # pass + else + echo "Google drive server not reachable, NOT syncing..." + unlock + exit 0 + fi + + TIME_AT_START=0 + TIME_AT_END=1 + while [[ "${TIME_AT_START}" -lt "${TIME_AT_END}" ]]; do + echo "Syncing '${_directory}'..." + TIME_AT_START="$(stat -c %Y "$LOCKFILE")" + grive -p "${_directory}" 2>&1 | grep -v -E "^Reading local directories$|^Reading remote server file list$|^Synchronizing files$|^Finished!$" + TIME_AT_END="$(stat -c %Y "$LOCKFILE")" + echo "Sync of '${_directory}' done." + done + + # always exit ok, so that we never go into a wrong systemd state + unlock + exit 0 +} + +### LISTEN TO CHANGES IN DIRECTORY ### + + +listen_directory() { + _directory="${1}" + + type inotifywait >/dev/null 2>&1 || { echo >&2 "I require inotifywait but it's not installed. Aborting."; exit 1; } + + echo "Listening for changes in '${_directory}'" + + while true #run indefinitely + do + # Use a different call to not need to change exit into return + inotifywait -q -r -e modify,attrib,close_write,move,create,delete --exclude ".grive_state|.grive" "${_directory}" > /dev/null 2>&1 && ${SCRIPT} sync $(systemd-escape "${_directory}") + #echo ${SCRIPT} "${_directory}" + done + + # always exit ok, so that we never go into a wrong systemd state + exit 0 +} + +if [[ "${COMMAND}" == listen ]] ; then + listen_directory "${DIRECTORY}" +else + sync_directory "${DIRECTORY}" +fi + +# always exit ok, so that we never go into a wrong systemd state +exit 0 diff --git a/systemd/grive-timer@.service.in b/systemd/grive-timer@.service.in new file mode 100644 index 00000000..d37070f0 --- /dev/null +++ b/systemd/grive-timer@.service.in @@ -0,0 +1,6 @@ +[Unit] +Description=Google drive sync (executed by timer unit) +After=network-online.target + +[Service] +ExecStart=@GRIVE_SYNC_SH_BINARY@ sync "%i" diff --git a/systemd/grive-timer@.timer b/systemd/grive-timer@.timer new file mode 100644 index 00000000..32f14b9a --- /dev/null +++ b/systemd/grive-timer@.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Google drive sync (fixed intervals) + +[Timer] +OnCalendar=*:0/5 +OnBootSec=3min +OnUnitActiveSec=5min +Unit=grive-timer@%i.service + +[Install] +WantedBy=timers.target diff --git a/systemd/grive@.service b/systemd/grive@.service new file mode 100644 index 00000000..e3372db8 --- /dev/null +++ b/systemd/grive@.service @@ -0,0 +1,13 @@ +[Unit] +Description=Google drive sync (main) +Requires=grive-timer@%i.timer grive-changes@%i.service + +# dummy service +[Service] +Type=oneshot +ExecStart=/bin/true +# This service shall be considered active after start +RemainAfterExit=yes + +[Install] +WantedBy=default.target