version control db

.gitignore

@@ -6,3 +6,4 @@ __pycache__/
 config/config.ini
 config/*.log*
 config/sqlite.db
+.DS_STORE

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,79 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from src.models import TableNotification, TableSteamItem, TableGiveaway, Base
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py

@@ -0,0 +1,70 @@
+"""init
+
+Revision ID: 1da33402b659
+Revises: 
+Create Date: 2022-05-19 15:50:19.539401
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '1da33402b659'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('notification',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('type', sa.String(length=50), nullable=False),
+                    sa.Column('message', sa.String(length=300), nullable=False),
+                    sa.Column('medium', sa.String(length=50), nullable=False),
+                    sa.Column('success', sa.Boolean(), nullable=False),
+                    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_table('steam_item',
+                    sa.Column('steam_id', sa.String(length=15), nullable=False),
+                    sa.Column('game_name', sa.String(length=200), nullable=False),
+                    sa.Column('steam_url', sa.String(length=100), nullable=False),
+                    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.PrimaryKeyConstraint('steam_id')
+                    )
+    op.create_table('giveaway',
+                    sa.Column('giveaway_id', sa.String(length=10), nullable=False),
+                    sa.Column('steam_id', sa.Integer(), nullable=False),
+                    sa.Column('giveaway_uri', sa.String(length=200), nullable=False),
+                    sa.Column('user', sa.String(length=40), nullable=False),
+                    sa.Column('giveaway_created_at', sa.DateTime(timezone=True), nullable=False),
+                    sa.Column('giveaway_ended_at', sa.DateTime(timezone=True), nullable=False),
+                    sa.Column('cost', sa.Integer(), nullable=False),
+                    sa.Column('copies', sa.Integer(), nullable=False),
+                    sa.Column('contributor_level', sa.Integer(), nullable=False),
+                    sa.Column('entered', sa.Boolean(), nullable=False),
+                    sa.Column('won', sa.Boolean(), nullable=False),
+                    sa.Column('game_entries', sa.Integer(), nullable=False),
+                    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.ForeignKeyConstraint(['steam_id'], ['steam_item.steam_id'], ),
+                    sa.PrimaryKeyConstraint('giveaway_id', 'steam_id')
+                    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('giveaway')
+    op.drop_table('steam_item')
+    op.drop_table('notification')
+    # ### end Alembic commands ###

requirements.txt

@@ -7,3 +7,4 @@ python-dateutil==2.8.2
 Flask==2.1.2
 Flask-BasicAuth==0.2.0
 pyopenssl==22.0.0
+alembic==1.7.7

src/database.py

@@ -1,27 +1,35 @@
 from datetime import datetime, timedelta
 
+import sqlalchemy
+from alembic import command
+from alembic.config import Config
 from dateutil import tz
-from sqlalchemy import create_engine, Integer, String, Column, DateTime, Boolean, func, ForeignKey
-from sqlalchemy.orm import registry, relationship, Session
-from sqlalchemy_utils import database_exists, create_database
+from sqlalchemy import func
+from sqlalchemy.orm import Session
 
-mapper_registry = registry()
-mapper_registry.metadata
-Base = mapper_registry.generate_base()
-engine = create_engine('sqlite:///../config/sqlite.db', echo=False)
+import log
+from models import TableNotification, TableGiveaway, TableSteamItem
+
+logger = log.get_logger(__name__)
+engine = None
 
 
-class TableNotification(Base):
-    __tablename__ = 'notification'
-    id = Column(Integer, primary_key=True, nullable=False)
-    type = Column(String(50), nullable=False)
-    message = Column(String(300), nullable=False)
-    medium = Column(String(50), nullable=False)
-    success = Column(Boolean, nullable=False)
-    created_at = Column(DateTime(timezone=True), server_default=func.now())
-    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+def create_engine(db_url: str):
+    global engine
+    if not engine:
+        engine = sqlalchemy.create_engine(db_url, echo=False)
+        engine.connect()
 
-    __mapper_args__ = {"eager_defaults": True}
+
+def run_migrations(script_location: str, dsn: str) -> None:
+    logger.info('Running DB migrations in %r on %r', script_location, dsn)
+    alembic_cfg = Config()
+    alembic_cfg.set_main_option('script_location', script_location)
+    alembic_cfg.set_main_option('sqlalchemy.url', dsn)
+    command.upgrade(alembic_cfg, 'head')
+
+
+class NotificationHelper:
 
     @classmethod
     def insert(cls, type_of_error, message, medium, success):
@@ -61,37 +69,7 @@ class TableNotification(Base):
                 .all()
 
 
-class TableSteamItem(Base):
-    __tablename__ = 'steam_item'
-    steam_id = Column(String(15), primary_key=True, nullable=False)
-    game_name = Column(String(200), nullable=False)
-    steam_url = Column(String(100), nullable=False)
-    created_at = Column(DateTime(timezone=True), server_default=func.now())
-    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
-
-    giveaways = relationship("TableGiveaway", back_populates="steam_item")
-
-
-class TableGiveaway(Base):
-    __tablename__ = 'giveaway'
-    giveaway_id = Column(String(10), primary_key=True, nullable=False)
-    steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
-    giveaway_uri = Column(String(200), nullable=False)
-    user = Column(String(40), nullable=False)
-    giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
-    giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
-    cost = Column(Integer(), nullable=False)
-    copies = Column(Integer(), nullable=False)
-    contributor_level = Column(Integer(), nullable=False)
-    entered = Column(Boolean(), nullable=False)
-    won = Column(Boolean(), nullable=False)
-    game_entries = Column(Integer(), nullable=False)
-    created_at = Column(DateTime(timezone=True), server_default=func.now())
-    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
-
-    steam_item = relationship("TableSteamItem", back_populates="giveaways")
-
-    __mapper_args__ = {"eager_defaults": True}
+class GiveawayHelper:
 
     @classmethod
     def unix_timestamp_to_utc_datetime(cls, timestamp):
@@ -122,8 +100,8 @@ class TableGiveaway(Base):
                 steam_id=steam_id,
                 giveaway_uri=giveaway.giveaway_uri,
                 user=giveaway.user,
-                giveaway_created_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
-                giveaway_ended_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
+                giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
+                giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
                 cost=giveaway.cost,
                 copies=giveaway.copies,
                 contributor_level=giveaway.contributor_level,
@@ -135,9 +113,9 @@ class TableGiveaway(Base):
 
     @classmethod
     def upsert_giveaway(cls, giveaway, entered):
-        result = TableGiveaway.get_by_ids(giveaway)
+        result = GiveawayHelper.get_by_ids(giveaway)
         if not result:
-            TableGiveaway.insert(giveaway, entered)
+            GiveawayHelper.insert(giveaway, entered)
         else:
             with Session(engine) as session:
                 g = TableGiveaway(
@@ -155,13 +133,3 @@ class TableGiveaway(Base):
                     game_entries=giveaway.game_entries)
                 session.merge(g)
                 session.commit()
-
-
-if not database_exists(engine.url):
-    create_database(engine.url)
-    # emitting DDL
-    mapper_registry.metadata.create_all(engine)
-    Base.metadata.create_all(engine)
-else:
-    # Connect the database if exists.
-    engine.connect()

src/enter_giveaways.py

@@ -8,8 +8,8 @@ from requests.adapters import HTTPAdapter
 from urllib3.util import Retry
 
 import log
+from database import NotificationHelper, GiveawayHelper
 from giveaway import Giveaway
-from tables import TableNotification, TableGiveaway
 
 logger = log.get_logger(__name__)
 
@@ -90,7 +90,7 @@ class EnterGiveaways:
         won = soup.select("a[title='Giveaways Won'] div")
         if won:
             number_won = soup.select_one("a[title='Giveaways Won'] div").text
-            won_notifications = TableNotification.get_won_notifications_today()
+            won_notifications = NotificationHelper.get_won_notifications_today()
             if won_notifications and len(won_notifications) >= 1:
                 logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting "
                             "to be received. Doing nothing.")
@@ -203,15 +203,15 @@ class EnterGiveaways:
                 if if_enter_giveaway:
                     res = self.enter_giveaway(giveaway)
                     if res:
-                        TableGiveaway.upsert_giveaway(giveaway, True)
+                        GiveawayHelper.upsert_giveaway(giveaway, True)
                         self.points -= int(giveaway.cost)
                         txt = f"✅ Entered giveaway '{giveaway.game_name}'"
                         logger.info(txt)
                         sleep(randint(4, 15))
                     else:
-                        TableGiveaway.upsert_giveaway(giveaway, False)
+                        GiveawayHelper.upsert_giveaway(giveaway, False)
                 else:
-                    TableGiveaway.upsert_giveaway(giveaway, False)
+                    GiveawayHelper.upsert_giveaway(giveaway, False)
                 # if we are on any filter type except New and we get to a giveaway that exceeds our
                 # max time left amount, then we don't need to continue to look at giveaways as any
                 # after this point will also exceed the max time left

src/models.py

@@ -0,0 +1,52 @@
+from sqlalchemy import Integer, String, Column, DateTime, Boolean, func, ForeignKey
+from sqlalchemy.orm import registry, relationship
+
+mapper_registry = registry()
+metadata = mapper_registry.metadata
+Base = mapper_registry.generate_base()
+
+
+class TableNotification(Base):
+    __tablename__ = 'notification'
+    id = Column(Integer, primary_key=True, nullable=False)
+    type = Column(String(50), nullable=False)
+    message = Column(String(300), nullable=False)
+    medium = Column(String(50), nullable=False)
+    success = Column(Boolean, nullable=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    __mapper_args__ = {"eager_defaults": True}
+
+
+class TableSteamItem(Base):
+    __tablename__ = 'steam_item'
+    steam_id = Column(String(15), primary_key=True, nullable=False)
+    game_name = Column(String(200), nullable=False)
+    steam_url = Column(String(100), nullable=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    giveaways = relationship("TableGiveaway", back_populates="steam_item")
+
+
+class TableGiveaway(Base):
+    __tablename__ = 'giveaway'
+    giveaway_id = Column(String(10), primary_key=True, nullable=False)
+    steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
+    giveaway_uri = Column(String(200), nullable=False)
+    user = Column(String(40), nullable=False)
+    giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
+    giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
+    cost = Column(Integer(), nullable=False)
+    copies = Column(Integer(), nullable=False)
+    contributor_level = Column(Integer(), nullable=False)
+    entered = Column(Boolean(), nullable=False)
+    won = Column(Boolean(), nullable=False)
+    game_entries = Column(Integer(), nullable=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    steam_item = relationship("TableSteamItem", back_populates="giveaways")
+
+    __mapper_args__ = {"eager_defaults": True}

src/notification.py

@@ -2,7 +2,7 @@ import http.client
 import urllib
 
 import log
-from tables import TableNotification
+from database import NotificationHelper
 
 logger = log.get_logger(__name__)
 
@@ -48,4 +48,4 @@ class Notification:
         else:
             logger.error(f"Pushover notification failed. Code {response.getcode()}: {response.read().decode()}")
             success = False
-        TableNotification.insert(type_of_error, f"{message}", 'pushover', success)
+        NotificationHelper.insert(type_of_error, f"{message}", 'pushover', success)

src/run.py

@@ -5,26 +5,30 @@ from config_reader import ConfigReader, ConfigException
 from enter_giveaways import SteamGiftsException
 from giveaway_thread import GiveawayThread
 from notification import Notification
+from database import run_migrations, create_engine
 from webserver_thread import WebServerThread
 
 logger = log.get_logger(__name__)
+config_file_name = '../config/config.ini'
+db_url = 'sqlite:///../config/sqlite.db'
+alembic_migration_files = '../alembic'
 
 
 def run():
     logger.info("Starting Steamgifts bot.")
-    file_name = '../config/config.ini'
+
     config = None
     try:
-        config = ConfigReader(file_name)
+        config = ConfigReader(config_file_name)
     except IOError:
-        txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out."
+        txt = f"{config_file_name} doesn't exist. Rename {config_file_name}.example to {config_file_name} and fill out."
         logger.warning(txt)
         exit(-1)
     except ConfigException as e:
         logger.error(e)
         exit(-1)
 
-    config.read(file_name)
+    config.read(config_file_name)
 
     notification = Notification(config['NOTIFICATIONS'].get('notification.prefix'))
     pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled')
@@ -68,4 +72,6 @@ if __name__ == '__main__':
                                           |___/                                       
     -------------------------------------------------------------------------------------
     """)
+    run_migrations(alembic_migration_files, db_url)
+    create_engine(db_url)
     run()