Extend "orders" table

Add new columns "virtual" and "custom" to Orders table. "Virtual" is for
indicating virtual order, "Custom" is for adding any custom data in
string format, may be used by any strategy to add additional info about
order.

This commit also introduces database migrations mechanism via alembic
tool. Seems like it's the best solution for sqlalchemy to handle schema
updates.

dexbot/migrations/README

@@ -0,0 +1,13 @@
+Dexbot database migrations are handled by alembic. See https://alembic.sqlalchemy.org/
+
+## Create new migration
+
+```
+alembic revision -m "Short summary of changes"
+```
+
+Next, modify the migration script in dexbot/migrations/versions/
+
+Migration will be applied automatically on next run of dexbot, see `run_migrations()` in dexbot/storage.py
+
+Don't forget to change table definitions in dexbot/storage.py.

dexbot/migrations/env.py

@@ -0,0 +1,72 @@
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+# fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

dexbot/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py

@@ -0,0 +1,26 @@
+"""extend orders table
+
+Revision ID: d1e6672520b2
+Revises:
+Create Date: 2019-07-29 17:38:09.136485
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd1e6672520b2'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('orders', sa.Column('virtual', sa.Boolean(create_constraint=False)))
+    op.add_column('orders', sa.Column('custom', sa.String))
+
+
+def downgrade():
+    op.drop_column('orders', 'virtual')
+    op.drop_column('orders', 'custom')

dexbot/storage.py

@@ -1,17 +1,23 @@
 import os
+import os.path
+import inspect
 import json
 import threading
 import queue
 import uuid
+import alembic
+import alembic.config
+
 from appdirs import user_data_dir
 
 from . import helper
 from dexbot import APP_NAME, AUTHOR
 
-from sqlalchemy import create_engine, Column, String, Integer, Float
+from sqlalchemy import create_engine, Column, String, Integer, Float, Boolean
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
+
 Base = declarative_base()
 
 # For dexbot.sqlite file
@@ -39,11 +45,15 @@ class Orders(Base):
     worker = Column(String)
     order_id = Column(String)
     order = Column(String)
+    virtual = Column(Boolean)
+    custom = Column(String)
 
-    def __init__(self, worker, order_id, order):
+    def __init__(self, worker, order_id, order, virtual, custom):
         self.worker = worker
         self.order_id = order_id
         self.order = order
+        self.virtual = virtual
+        self.custom = custom
 
 
 class Balances(Base):
@@ -152,12 +162,18 @@ def __init__(self):
         super().__init__()
 
         # Obtain engine and session
-        engine = create_engine('sqlite:///%s' % sqlDataBaseFile, echo=False)
+        dsn = 'sqlite:///{}'.format(sqlDataBaseFile)
+        engine = create_engine(dsn, echo=False)
         Session = sessionmaker(bind=engine)
         self.session = Session()
         Base.metadata.create_all(engine)
         self.session.commit()
 
+        # Run migrations
+        import dexbot
+        migrations_dir = '{}/migrations'.format(os.path.dirname(inspect.getfile(dexbot)))
+        self.run_migrations(migrations_dir, dsn)
+
         self.task_queue = queue.Queue()
         self.results = {}
 
@@ -166,6 +182,18 @@ def __init__(self):
         self.daemon = True
         self.start()
 
+    @staticmethod
+    def run_migrations(script_location, dsn):
+        """ Apply database migrations using alembic
+
+            :param str script_location: path to migration scripts
+            :param str dsn: database URL
+        """
+        alembic_cfg = alembic.config.Config()
+        alembic_cfg.set_main_option('script_location', script_location)
+        alembic_cfg.set_main_option('sqlalchemy.url', dsn)
+        alembic.command.upgrade(alembic_cfg, 'head')
+
     def run(self):
         for func, args, token in iter(self.task_queue.get, None):
             if token is not None:
@@ -279,7 +307,7 @@ def _save_order(self, worker, order_id, order):
         if e:
             e.value = value
         else:
-            e = Orders(worker, order_id, value)
+            e = Orders(worker, order_id, value, None, None)
             self.session.add(e)
         self.session.commit()
 

requirements.txt

@@ -18,3 +18,4 @@ websocket-client==0.56.0
 sdnotify==0.3.2
 sqlalchemy==1.3.0
 click==7.0
+alembic==1.0.11

tests/migrations/conftest.py

@@ -0,0 +1,62 @@
+import os
+import pytest
+import tempfile
+import logging
+
+from sqlalchemy import create_engine, Column, String, Integer, Float
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+log = logging.getLogger("dexbot")
+log.setLevel(logging.DEBUG)
+
+Base = declarative_base()
+
+# Classes are represent initial table structure
+
+
+class Config(Base):
+    __tablename__ = 'config'
+
+    id = Column(Integer, primary_key=True)
+    category = Column(String)
+    key = Column(String)
+    value = Column(String)
+
+
+class Orders(Base):
+    __tablename__ = 'orders'
+
+    id = Column(Integer, primary_key=True)
+    worker = Column(String)
+    order_id = Column(String)
+    order = Column(String)
+
+
+class Balances(Base):
+    __tablename__ = 'balances'
+
+    id = Column(Integer, primary_key=True)
+    account = Column(String)
+    worker = Column(String)
+    base_total = Column(Float)
+    base_symbol = Column(String)
+    quote_total = Column(Float)
+    quote_symbol = Column(String)
+    center_price = Column(Float)
+    timestamp = Column(Integer)
+
+
+@pytest.fixture
+def initial_db():
+
+    _, db_file = tempfile.mkstemp()  # noqa: F811
+    engine = create_engine('sqlite:///{}'.format(db_file), echo=False)
+    Session = sessionmaker(bind=engine)
+    session = Session()
+    Base.metadata.create_all(engine)
+    session.commit()
+    log.debug('Prepared db on {}'.format(db_file))
+
+    yield db_file
+    os.unlink(db_file)

tests/migrations/test_migrations.py

@@ -0,0 +1,5 @@
+from dexbot.storage import DatabaseWorker
+
+
+def test_apply_migrations(initial_db):
+    DatabaseWorker.run_migrations('dexbot/migrations', 'sqlite:///{}'.format(initial_db))