diff --git a/.dockerignore b/.dockerignore index 2c23a39..744b479 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,7 @@ docker-compose.yml config.yml config.sample.yml +.git .cache .export last_imported diff --git a/.travis.yml b/.travis.yml index 055a278..f5a91a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: ruby cache: bundler rvm: - 2.4.2 -- 2.5.0 +- 2.5.3 +- 2.6.1 env: - ENV=test before_script: diff --git a/Dockerfile b/Dockerfile index 29ab84e..99c2ec4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:2.5.0 +FROM ruby:2.6 # Set the locale RUN apt-get clean && apt-get update && apt-get install -y locales diff --git a/Gemfile.lock b/Gemfile.lock index d0d99ee..310bfd4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,16 +1,17 @@ GEM remote: https://rubygems.org/ specs: - activesupport (5.1.5) + activesupport (5.2.2) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (~> 0.7) + i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - addressable (2.5.2) + addressable (2.6.0) public_suffix (>= 2.0.2, < 4.0) ast (2.4.0) - bankscrap (2.0.6) + bankscrap (2.1.1) activesupport + json mechanize money thor @@ -18,69 +19,72 @@ GEM bankscrap-bbva (2.0.3) bankscrap (>= 2.0.0) base32 (0.3.2) - cmxl (0.2.1) + cmxl (0.2.2) rchardet19 colorize (0.8.1) - concurrent-ruby (1.0.5) + concurrent-ruby (1.1.4) + connection_pool (2.2.2) crack (0.4.3) safe_yaml (~> 1.0.0) deep_merge (1.2.1) diff-lcs (1.3) - domain_name (0.5.20170404) + domain_name (0.5.20180417) unf (>= 0.0.5, < 1.0.0) - ethon (0.11.0) + ethon (0.12.0) ffi (>= 1.3.0) - ffi (1.9.25) - hashdiff (0.3.7) + ffi (1.10.0) + hashdiff (0.3.8) http-cookie (1.0.3) domain_name (~> 0.5) - httparty (0.16.0) + httparty (0.16.3) + mime-types (~> 3.0) multi_xml (>= 0.5.2) - i18n (0.9.5) + i18n (1.5.3) concurrent-ruby (~> 1.0) json (2.1.0) - mechanize (2.7.5) + mechanize (2.7.6) domain_name (~> 0.5, >= 0.5.1) http-cookie (~> 1.0) mime-types (>= 1.17.2) net-http-digest_auth (~> 1.1, >= 1.1.1) - net-http-persistent (~> 2.5, >= 2.5.2) + net-http-persistent (>= 2.5.2) nokogiri (~> 1.6) ntlm-http (~> 0.1, >= 0.1.1) webrobots (>= 0.0.9, < 0.2) - mime-types (3.1) + mime-types (3.2.2) mime-types-data (~> 3.2015) - mime-types-data (3.2016.0521) - mini_portile2 (2.3.0) + mime-types-data (3.2018.0812) + mini_portile2 (2.4.0) minitest (5.11.3) - money (6.10.1) - i18n (>= 0.6.4, < 1.0) + money (6.13.2) + i18n (>= 0.6.4, <= 2) multi_xml (0.6.0) net-http-digest_auth (1.4.1) - net-http-persistent (2.9.4) - nokogiri (1.8.2) - mini_portile2 (~> 2.3.0) + net-http-persistent (3.0.0) + connection_pool (~> 2.2) + nokogiri (1.10.1) + mini_portile2 (~> 2.4.0) ntlm-http (0.1.1) - parallel (1.12.1) - parser (2.5.0.2) + parallel (1.13.0) + parser (2.6.0.0) ast (~> 2.4.0) - powerpack (0.1.1) - public_suffix (3.0.2) + powerpack (0.1.2) + public_suffix (3.0.3) rainbow (3.0.0) rchardet19 (1.3.7) - rspec (3.7.0) - rspec-core (~> 3.7.0) - rspec-expectations (~> 3.7.0) - rspec-mocks (~> 3.7.0) - rspec-core (3.7.1) - rspec-support (~> 3.7.0) - rspec-expectations (3.7.0) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-mocks (3.7.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-support (3.7.1) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) rubocop (0.52.1) parallel (~> 1.10) parser (>= 2.4.0.2, < 3.0) @@ -90,26 +94,26 @@ GEM unicode-display_width (~> 1.0, >= 1.0.1) rubocop-rspec (1.23.0) rubocop (>= 0.52.1) - ruby-progressbar (1.9.0) + ruby-progressbar (1.10.0) ruby_fints (0.0.3) cmxl (~> 0.2) httparty (~> 0.10) safe_yaml (1.0.4) - thor (0.20.0) + thor (0.20.3) thread_safe (0.3.6) twentysix (0.1.1) deep_merge httparty - typhoeus (1.3.0) + typhoeus (1.3.1) ethon (>= 0.9.0) tzinfo (1.2.5) thread_safe (~> 0.1) unf (0.1.4) unf_ext unf_ext (0.0.7.5) - unicode-display_width (1.3.0) + unicode-display_width (1.4.1) vcr (4.0.0) - webmock (3.3.0) + webmock (3.5.1) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff @@ -138,4 +142,4 @@ DEPENDENCIES ynab (= 1.5.0) BUNDLED WITH - 1.16.2 + 1.17.3 diff --git a/README.md b/README.md index dd707be..0595e66 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,12 @@ This is a ruby script that **pulls your transactions from your banks** and impor --- +## Disclaimer + +**Using this tool is on your own risk.** I can **not** garantee you that this will work for you. Also I can **not** garantee that it won't up your YNAB data. I highly recommend to start with a new Budget or at least test it with a new one before you use your existing Budget. + +--- + ## Supported banks * Most German and Austrian banks _(all banks that implement the FinTS standard)_ @@ -32,6 +38,7 @@ The script also includes some additional logic like detecting internal transacti # Known Problems * Currently no known problems +* Please read the notes in each Dumper _[(see Wiki)](https://github.com/schurig/ynab-bank-importer/wiki#supported-dumpers)_ to understand the limitations ____________________ @@ -39,10 +46,13 @@ ____________________ Support and contriubution of any kind is always welcome!!! -I'm not that into hardware. It would be super awesome if someone could help making this work on Raspbian. I already tried but building the docker container fails _(Dockerfile.rpi)_. The PR related to that you can find here: [18](https://github.com/schurig/ynab-bank-importer/pull/18) - # Thanks +* [@dequis](https://github.com/dequis) for preventing the script from failing in the future [PR #70](https://github.com/schurig/ynab-bank-importer/pull/70) and keeping an eye on the development experience [PR #69](https://github.com/schurig/ynab-bank-importer/pull/69) +* [@moay](https://github.com/moay), [@yuvke](https://github.com/yuvke), [@BluetriX](https://github.com/BluetriX) for the help on debugging an error with not existing dates [Issue #52](https://github.com/schurig/ynab-bank-importer/issues/52) +* [@mathijshoogland](https://github.com/mathijshoogland) for updating the dependencies [PR #49](https://github.com/schurig/ynab-bank-importer/pull/49) +* [@manuelgrabowski](https://github.com/manuelgrabowski), [@martinlabuschin](https://github.com/martinlabuschin) and [@peterjeschke](https://github.com/peterjeschke) for giving feedback on a new N26 dumper config flag that prevents transactions to be imported multiple times [PR #38](https://github.com/schurig/ynab-bank-importer/pull/38) +* [@peterjeschke](https://github.com/peterjeschke) for fixing a bug that happened when the FinTS username was an integer [PR #35](https://github.com/schurig/ynab-bank-importer/pull/35) * [@derintendant](https://github.com/derintendant) for spotting and fixing edge cases [PR #27](https://github.com/schurig/ynab-bank-importer/pull/27) (improves error messages) and [PR #28](https://github.com/schurig/ynab-bank-importer/pull/28) (truncates the payee field if it's too long) * [@manuelgrabowski](https://github.com/manuelgrabowski) for implementing a fallback in the FinTS dumper [PR #26](https://github.com/schurig/ynab-bank-importer/pull/26) * [@markuspabst](https://github.com/markuspabst) for spotting an error in the readme [PR #11](https://github.com/schurig/ynab-bank-importer/pull/11) diff --git a/config.sample.yml b/config.sample.yml index 842f1ea..8407256 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -17,4 +17,5 @@ accounts: ynab_id: # last hash in the url when you click on the account in YNAB username: # n26 username password: # n26 password + skip_pending_transactions: false # default: false, only imports transactions when they're processed set_category: false # default: false, sets the N26 category name as category diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 792b6d5..f774c23 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,5 +4,5 @@ services: build: context: . volumes: - - ./config.yml:/usr/app/config.yml + - ./:/usr/app/ command: "ruby /usr/app/run.rb" diff --git a/lib/dumper.rb b/lib/dumper.rb index 210f363..9083022 100644 --- a/lib/dumper.rb +++ b/lib/dumper.rb @@ -16,7 +16,7 @@ def self.get_dumper(name) # rubocop:disable Metrics/MethodLength def to_ynab_transaction(transaction) - return nil if date(transaction) > Date.today + return nil if date(transaction).nil? || date(transaction) > Date.today ::TransactionCreator.call( account_id: account_id, date: date(transaction), diff --git a/lib/dumper/bbva.rb b/lib/dumper/bbva.rb index 96c0d28..b9229c3 100644 --- a/lib/dumper/bbva.rb +++ b/lib/dumper/bbva.rb @@ -50,7 +50,9 @@ def amount(transaction) def withdrawal?(transaction) text = transaction.description.downcase - text.include?('cajero') || text.include?('withdrawal') + text.include?('cajero') || + text.include?('withdrawal') || + text.include?('efectivo') end def normalize_iban(iban) diff --git a/lib/dumper/fints.rb b/lib/dumper/fints.rb index 8066c4b..c5fce5d 100644 --- a/lib/dumper/fints.rb +++ b/lib/dumper/fints.rb @@ -7,8 +7,8 @@ class Fints < Dumper def initialize(params = {}) @ynab_id = params.fetch('ynab_id') - @username = params.fetch('username') - @password = params.fetch('password') + @username = params.fetch('username').to_s + @password = params.fetch('password').to_s @iban = params.fetch('iban') @endpoint = params.fetch('fints_endpoint') @blz = params.fetch('fints_blz') @@ -32,29 +32,29 @@ def account_id def date(transaction) transaction.entry_date || transaction.date + rescue NoMethodError + # https://github.com/schurig/ynab-bank-importer/issues/52 + # Some banks think Feb 29 and 30 exist in non-leap years. + entry_date(transaction) || to_date(transaction['date']) end def payee_name(transaction) - parse_transaction_at(32, transaction).try(:strip) + transaction.name.try(:strip) end def payee_iban(transaction) - parse_transaction_at(31, transaction) + transaction.iban end def memo(transaction) - parse_transaction_at(20, transaction).try(:strip) + [ + transaction.description, + transaction.information + ].compact.join(' / ').try(:strip) end def amount(transaction) - amount = - if transaction.funds_code == 'D' - "-#{transaction.amount}" - else - transaction.amount - end - - (amount.to_f * 1000).to_i + (transaction.amount * transaction.sign * 1000).to_i end def withdrawal?(transaction) @@ -65,38 +65,45 @@ def withdrawal?(transaction) end def import_id(transaction) - data = [transaction_type(transaction), - transaction.date, - transaction.amount, - transaction.funds_code, - transaction.reference.try(:downcase), - payee_iban(transaction), - payee_name(transaction).try(:downcase), - @iban].join - - Digest::MD5.hexdigest(data) + Digest::MD5.hexdigest(transaction.source) end - def transaction_type(transaction) - # Changing the result of this method will - # change the hash returned by the `import_id` which - # could will result in duplicated entries. - - str = parse_transaction_at(0, transaction).encode('iso-8859-1') - .force_encoding('utf-8') - return nil unless str - str[1..-1] + # Patches + + # taken from https://github.com/railslove/cmxl/blob/master/lib/cmxl/field.rb + # and modified so that it takes the last day of the month if the provided day + # doesn't exist in that month. + # See issue: https://github.com/schurig/ynab-bank-importer/issues/52 + DATE = /(?\d{0,2})(?\d{2})(?\d{2})/ + def to_date(date, year = nil) + if match = date.to_s.match(DATE) + year ||= "20#{match['year'] || Date.today.strftime('%y')}" + month = match['month'] + day = match['day'] + + begin + Date.new(year.to_i, month.to_i, day.to_i) + rescue ArgumentError + # Take the last day of that month + Date.civil(year.to_i, month.to_i, -1) + end + else + date + end end - def parse_transaction_at(position, transaction) - # I don't know who invented this structure but I hope - # the responsible people know how inconvenient it is. + def entry_date(transaction) + data = transaction.data + date = to_date(data['date']) - seperator = transaction.details.seperator - array = transaction.details.source.split("#{seperator}#{position}") - return nil if array.size < 2 + return unless transaction.data['entry_date'] && date - array.last.split(seperator).first + entry_date_with_date_year = to_date(data['entry_date'], date.year) + if date.month == 1 && date.month < entry_date_with_date_year.month + to_date(data['entry_date'], date.year - 1) + else + to_date(data['entry_date'], date.year) + end end end end diff --git a/lib/dumper/n26.rb b/lib/dumper/n26.rb index 66e36e3..959ddc9 100644 --- a/lib/dumper/n26.rb +++ b/lib/dumper/n26.rb @@ -17,6 +17,8 @@ def initialize(params = {}) @password = params.fetch('password') @iban = params.fetch('iban') @set_category = params.fetch('set_category', false) + @skip_pending_transactions = params.fetch('skip_pending_transactions', + false) @categories = {} end @@ -28,10 +30,15 @@ def fetch_transactions end client.transactions(count: 100) - .reject { |t| t['pending'] } # Only transactions that aren't pending + .select { |t| accept?(t) } .map { |t| to_ynab_transaction(t) } end + def accept?(transaction) + return true unless @skip_pending_transactions + already_processed?(transaction) + end + private def check_authorization!(client) @@ -81,7 +88,7 @@ def withdrawal?(transaction) end def import_id(transaction) - data = [calculated_timestamp(transaction), + data = [transaction['visibleTS'], transaction['transactionNature'], transaction['amount'], transaction['accountId']].join @@ -89,22 +96,10 @@ def import_id(transaction) Digest::MD5.hexdigest(data) end - # N26 seems to have an internal timezone mismatch in their database. - # Transactions that are not processed yet have the `visibleTS` value - # in UTC but processed transactions have timezone Europe/Berlin. - # => This method checks if the transaction was processed or not. - # If it's already processed it will just take the value, if not it will - # add the current offset to make it Europe/Berlin timezone. - def calculated_timestamp(transaction) - return transaction['visibleTS'] if alread_processed?(transaction) - offset_to_utc = Time.now.in_time_zone('Europe/Berlin').utc_offset - transaction['visibleTS'] + offset_to_utc * 1000 - end - # All very recent transactions with the credit card have # the type value set to "AA". So we assume that this is an # indicator to check if a transaction has been processed or not. - def alread_processed?(transaction) + def already_processed?(transaction) transaction['type'] != 'AA' end end diff --git a/run.rb b/run.rb index 254a3c2..0ee27a9 100644 --- a/run.rb +++ b/run.rb @@ -17,10 +17,10 @@ access_token = Settings.all['ynab'].fetch('access_token') ynab_api = YNAB::API.new(access_token) -bulk_transactions = YNAB::BulkTransactions.new(transactions: transactions) begin - ynab_api.transactions.bulk_create_transactions(budget_id, bulk_transactions) + ynab_api.transactions.create_transaction(budget_id, + transactions: transactions) rescue YNAB::ApiError => e ErrorMessage.new(e).print abort diff --git a/spec/dumper/n26_spec.rb b/spec/dumper/n26_spec.rb index 84c47e6..9362180 100644 --- a/spec/dumper/n26_spec.rb +++ b/spec/dumper/n26_spec.rb @@ -4,13 +4,15 @@ RSpec.describe Dumper::N26, vcr: vcr_options do subject(:object) { Dumper::N26.new(params) } + let(:skip_pending_transactions) { false } let(:params) do { 'ynab_id' => '123466', 'username' => 'username', 'password' => 'password', - 'iban' => 'DE89370400440532013000' + 'iban' => 'DE89370400440532013000', + 'skip_pending_transactions' => skip_pending_transactions } end @@ -159,19 +161,47 @@ it 'sets it correctly' do expect(method).to eq('46c9ccde424652bc013dca9b408dcdec') end + end + + describe '.accept?' do + subject(:accept?) { object.accept?(transaction) } - it 'is the same for a pending transaction and a processed transaction' do - expect( - object.send(:import_id, transaction_pending) - ).to eq(object.send(:import_id, transaction_processed)) + context 'when skip_pending_transactions feature is disabled' do + context 'when the transaction is pending' do + let(:transaction) { transaction_pending } + + it 'returns true' do + expect(accept?).to be_truthy + end + end + + context 'when the transaction is processed' do + let(:transaction) { transaction_processed } + + it 'returns true' do + expect(accept?).to be_truthy + end + end end - end - describe '#calculated_timestamp' do - it 'is the same for a pending transaction and a processed transaction' do - expect( - object.send(:calculated_timestamp, transaction_pending) - ).to eq(object.send(:calculated_timestamp, transaction_processed)) + context 'when skip_pending_transactions feature is disabled' do + let(:skip_pending_transactions) { true } + + context 'when the transaction is pending' do + let(:transaction) { transaction_pending } + + it 'returns false' do + expect(accept?).to be_falsy + end + end + + context 'when the transaction is processed' do + let(:transaction) { transaction_processed } + + it 'returns true' do + expect(accept?).to be_truthy + end + end end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e371173..5e1ecfc 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,6 +9,7 @@ config.cassette_library_dir = 'spec/fixtures/vcr_cassettes' config.hook_into :webmock config.configure_rspec_metadata! + config.default_cassette_options = { record: :none } end # This file was generated by the `rspec --init` command. Conventionally, all