diff --git a/dexbot/migrations/README b/dexbot/migrations/README new file mode 100644 index 000000000..d3f637d2c --- /dev/null +++ b/dexbot/migrations/README @@ -0,0 +1,13 @@ +Dexbot database migrations are handled by alembic. See https://alembic.sqlalchemy.org/ + +## Create new migration + +``` +alembic revision -m "Short summary of changes" +``` + +Next, modify the migration script in dexbot/migrations/versions/ + +Migration will be applied automatically on next run of dexbot, see `run_migrations()` in dexbot/storage.py + +Don't forget to change table definitions in dexbot/storage.py. diff --git a/dexbot/migrations/env.py b/dexbot/migrations/env.py new file mode 100644 index 000000000..be82b9ecc --- /dev/null +++ b/dexbot/migrations/env.py @@ -0,0 +1,72 @@ +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +# fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/dexbot/migrations/script.py.mako b/dexbot/migrations/script.py.mako new file mode 100644 index 000000000..2c0156303 --- /dev/null +++ b/dexbot/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py b/dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py new file mode 100644 index 000000000..732d5bfb1 --- /dev/null +++ b/dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py @@ -0,0 +1,26 @@ +"""extend orders table + +Revision ID: d1e6672520b2 +Revises: +Create Date: 2019-07-29 17:38:09.136485 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd1e6672520b2' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('orders', sa.Column('virtual', sa.Boolean(create_constraint=False))) + op.add_column('orders', sa.Column('custom', sa.String)) + + +def downgrade(): + op.drop_column('orders', 'virtual') + op.drop_column('orders', 'custom') diff --git a/dexbot/storage.py b/dexbot/storage.py index 115419051..558022d49 100644 --- a/dexbot/storage.py +++ b/dexbot/storage.py @@ -1,17 +1,23 @@ import os +import os.path +import inspect import json import threading import queue import uuid +import alembic +import alembic.config + from appdirs import user_data_dir from . import helper from dexbot import APP_NAME, AUTHOR -from sqlalchemy import create_engine, Column, String, Integer, Float +from sqlalchemy import create_engine, Column, String, Integer, Float, Boolean from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker + Base = declarative_base() # For dexbot.sqlite file @@ -39,11 +45,15 @@ class Orders(Base): worker = Column(String) order_id = Column(String) order = Column(String) + virtual = Column(Boolean) + custom = Column(String) - def __init__(self, worker, order_id, order): + def __init__(self, worker, order_id, order, virtual, custom): self.worker = worker self.order_id = order_id self.order = order + self.virtual = virtual + self.custom = custom class Balances(Base): @@ -152,12 +162,18 @@ def __init__(self): super().__init__() # Obtain engine and session - engine = create_engine('sqlite:///%s' % sqlDataBaseFile, echo=False) + dsn = 'sqlite:///{}'.format(sqlDataBaseFile) + engine = create_engine(dsn, echo=False) Session = sessionmaker(bind=engine) self.session = Session() Base.metadata.create_all(engine) self.session.commit() + # Run migrations + import dexbot + migrations_dir = '{}/migrations'.format(os.path.dirname(inspect.getfile(dexbot))) + self.run_migrations(migrations_dir, dsn) + self.task_queue = queue.Queue() self.results = {} @@ -166,6 +182,18 @@ def __init__(self): self.daemon = True self.start() + @staticmethod + def run_migrations(script_location, dsn): + """ Apply database migrations using alembic + + :param str script_location: path to migration scripts + :param str dsn: database URL + """ + alembic_cfg = alembic.config.Config() + alembic_cfg.set_main_option('script_location', script_location) + alembic_cfg.set_main_option('sqlalchemy.url', dsn) + alembic.command.upgrade(alembic_cfg, 'head') + def run(self): for func, args, token in iter(self.task_queue.get, None): if token is not None: @@ -279,7 +307,7 @@ def _save_order(self, worker, order_id, order): if e: e.value = value else: - e = Orders(worker, order_id, value) + e = Orders(worker, order_id, value, None, None) self.session.add(e) self.session.commit() diff --git a/requirements.txt b/requirements.txt index b38cfb0ab..9711a1be8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ websocket-client==0.56.0 sdnotify==0.3.2 sqlalchemy==1.3.0 click==7.0 +alembic==1.0.11 diff --git a/tests/migrations/conftest.py b/tests/migrations/conftest.py new file mode 100644 index 000000000..acb64d33c --- /dev/null +++ b/tests/migrations/conftest.py @@ -0,0 +1,62 @@ +import os +import pytest +import tempfile +import logging + +from sqlalchemy import create_engine, Column, String, Integer, Float +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +log = logging.getLogger("dexbot") +log.setLevel(logging.DEBUG) + +Base = declarative_base() + +# Classes are represent initial table structure + + +class Config(Base): + __tablename__ = 'config' + + id = Column(Integer, primary_key=True) + category = Column(String) + key = Column(String) + value = Column(String) + + +class Orders(Base): + __tablename__ = 'orders' + + id = Column(Integer, primary_key=True) + worker = Column(String) + order_id = Column(String) + order = Column(String) + + +class Balances(Base): + __tablename__ = 'balances' + + id = Column(Integer, primary_key=True) + account = Column(String) + worker = Column(String) + base_total = Column(Float) + base_symbol = Column(String) + quote_total = Column(Float) + quote_symbol = Column(String) + center_price = Column(Float) + timestamp = Column(Integer) + + +@pytest.fixture +def initial_db(): + + _, db_file = tempfile.mkstemp() # noqa: F811 + engine = create_engine('sqlite:///{}'.format(db_file), echo=False) + Session = sessionmaker(bind=engine) + session = Session() + Base.metadata.create_all(engine) + session.commit() + log.debug('Prepared db on {}'.format(db_file)) + + yield db_file + os.unlink(db_file) diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py new file mode 100644 index 000000000..06eab6fcf --- /dev/null +++ b/tests/migrations/test_migrations.py @@ -0,0 +1,5 @@ +from dexbot.storage import DatabaseWorker + + +def test_apply_migrations(initial_db): + DatabaseWorker.run_migrations('dexbot/migrations', 'sqlite:///{}'.format(initial_db))