Add missing example config options, fix Steam URL parsing

- will scape won games at start to update db for giveaways the bot has entered
- schedules a once a day run of scraping won giveaways

.dockerignore

@@ -1,2 +1,5 @@
 config/config.ini
-config/debug.log
+config/debug.log
+config/sqlite.db
+config/info.log
+README.ini

.gitignore

@@ -1,7 +1,8 @@
 env/
 venv/
-*.ini
 __pycache__/
 .idea
 config/config.ini
-config/debug.log
+config/*.log*
+config/sqlite*
+.DS_STORE

Dockerfile

@@ -1,20 +1,25 @@
 FROM python:3.9-alpine
 
-RUN mkdir -p /app
+RUN apk add tzdata build-base libffi-dev py3-cffi --no-cache
+
+RUN mkdir -p /app/src
 WORKDIR /app
 
-# resolves gcc issue with installing regex dependency
-RUN apk add build-base tzdata --no-cache
-
 ENV TZ=America/New_York
-ENV VIRTUAL_ENV=/app/env
+ENV VIRTUAL_ENV=/src/env
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV BOT_CONFIG_DIR="../config"
+ENV BOT_DB_URL="sqlite:///../config/sqlite.db"
+ENV BOT_ALEMBIC_CONFIG="./src/alembic"
+
 
 RUN python -m venv $VIRTUAL_ENV
-COPY requirements.txt .
+COPY requirements.txt /app/
 RUN pip3 install -r requirements.txt
 
-COPY ./src/* /app/
+
+COPY ./src/ /app/src/
+COPY main.py /app/
 VOLUME /config
 
-CMD ["python", "run.py"]
+CMD ["python3", "main.py"]

README.md

@@ -4,15 +4,32 @@ The bot is specially designed for [SteamGifts.com](https://www.steamgifts.com/)
 ### Features
 - Automatically enters giveaways.
 - Undetectable.
-- Сonfigurable.
+- Сonfigurable
+  - Can look at your steam wishlist games or the main page for games to enter
+  - When evaluating a giveaway
+    - `max_time_left` - if the time left on giveaway is > max time left, then don't enter it
+    - `max_entries` - if the entries on a giveaway are > than max entries, then don't enter it
+    - `minimum_points` - minimum number of points **in your account** needed before considering any giveaway
+    - `minimum_game_points` - if steamgifts.com point cost (ex. 1P, 5P, etc) is below this, don't enter it
+    - `blacklist_keywords` - if the giveaway name contains any of these words, don't enter it. this list can be blank.
+  - Notifications
+    - A pushover notifications can be sent to you when a win is detected.
+  - Webserver - A simple, simple, simple webserver than can be enabled (disabled by default) to show the config and logs
+    - `web.host` - the IP to listen on (ex. localhost, 0.0.0.0, 192.168.1.1, etc)
+    - `web.port` - the port to listen on
+    - `web.app_root` - the folder to serve up which can be used for reverse proxying this behind nginx/apache/etc
+    - `web.ssl` - if the traffic will be encrypted (http or https) using a self-signed cert
+    - `web.basic_auth` - simple basic auth settings can be enabled
 - Sleeps to restock the points.
 - Can run 24/7.
 
 
 ## Instructions
- 1. Rename `config/config.ini.example` to `config/config.ini`. 
- 2. Add your PHPSESSION cookie to it.
- 3. Modifying the other settings is optional.
+1. Sign in on SteamGifts.com by Steam. 
+2. Find PHPSESSID cookie in your browser.
+3. Rename `config/config.ini.example` to `config/config.ini`. 
+4. Add your PHPSESSION cookie to `cookie` in `config/config.ini`
+5. Modifying the other settings is optional as defaults are set.
 
 ### Run from sources
 
@@ -20,8 +37,7 @@ The bot is specially designed for [SteamGifts.com](https://www.steamgifts.com/)
 python -m venv env
 source env/bin/activate
 pip install -r requirements.txt
-cd src
-python run.py
+python main.py
 ```
 
 ### Docker
@@ -29,7 +45,7 @@ python run.py
 ```bash
 # Run the container
 # Set TZ based on your timezone: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-docker run --name steamgifts -e TZ=America/New_York -d -v /path/to/the/config/folder:/config mcinj/docker-steamgifts-bot:v2.0
+docker run --name steamgifts -e TZ=America/New_York -d -v /path/to/the/config/folder:/config mcinj/docker-steamgifts-bot:latest
 ```
 
 #### Or build it yourself locally
@@ -42,6 +58,3 @@ docker run --name steamgifts -e TZ=America/New_York -d -v /path/to/the/config/fo
 ```
 
 
-
-### Help
-Please leave your feedback and bugs in `Issues` page.

alembic.ini

@@ -0,0 +1,102 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = src/alembic
+
+# template used to generate migration files
+file_template = %%(year)d_%%(month).2d_%%(day).2d-%%(hour).2d_%%(minute).2d_%%(second).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions:src/alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = sqlite:///config/sqlite.db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

config/config.ini.example

@@ -1,13 +1,70 @@
 [DEFAULT]
 cookie = PHPSESSIONCOOKIEGOESHERE
-# gift_types options: All, Wishlist, Recommended, Copies, DLC, New
-gift_types = All
-pinned = true
+# should we consider giveaways on the 'ALL' page? it is the main page
+enabled = true
+
+[ALL]
 # minimum number of points in your account before entering into giveaways
-minimum_points = 50
+all.minimum_points = 50
 # max number of entries in a giveaway for it to be considered
-max_entries = 900
+all.max_entries = 2000
 # time left in minutes of a giveaway for it to be considered
-max_time_left = 180
+all.max_time_left = 300
+# points after that giveaway is considered regardless of entries and time left
+all.max_points = 350
 # the minimum point value for a giveaway to be considered
-minimum_game_points = 5
+all.minimum_game_points = 1
+# a comma separated list of keywords in game titles to ignore
+all.blacklist_keywords = hentai,adult
+
+[WISHLIST]
+# should we consider giveaways on the 'Wishlist' page?
+wishlist.enabled = true
+# minimum number of points in your account before entering into giveaways
+wishlist.minimum_points = 1
+# max number of entries in a giveaway for it to be considered
+wishlist.max_entries = 10000
+# time left in minutes of a giveaway for it to be considered
+wishlist.max_time_left = 300
+# points after that giveaway is considered regardless of entries and time left
+wishlist.max_points = 350
+
+[DLC]
+# should we consider giveaways on the 'DLC' page?
+dlc.enabled = true
+# minimum number of points in your account before entering into giveaways
+dlc.minimum_points = 1
+# max number of entries in a giveaway for it to be considered
+dlc.max_entries = 10000
+# time left in minutes of a giveaway for it to be considered
+dlc.max_time_left = 300
+# points after that giveaway is considered regardless of entries and time left
+dlc.max_points = 350
+
+[NOTIFICATIONS]
+# a prefix for messages sent via notifications
+notification.prefix = SG-Bot
+# if we should send notifications via pushover
+pushover.enabled = false
+# your specific pushover token
+pushover.token =
+# your specific pushover user key
+pushover.user_key =
+
+[WEB]
+# should we enable the webserver which is just a simple, simple, simple webui to view the logs
+web.enabled = false
+# the host to listen on. localhost or 0.0.0.0 are the two common options
+web.host = localhost
+# the port to run on
+web.port = 9547
+# the app root  / web folder / root / many other names . MUST contain a trailing '/'
+web.app_root = /
+# should this served up on http or https
+web.ssl = true
+# should we use basic auth
+web.basic_auth = true
+# basic auth username
+web.basic_auth.username = admin
+# basic auth password
+web.basic_auth.password = ChangeMe

main.py

@@ -0,0 +1,5 @@
+from src import run
+
+
+if __name__ == '__main__':
+    run.entry()

requirements.txt

@@ -1,23 +1,13 @@
-astroid==2.4.2
-beautifulsoup4==4.9.1
-certifi==2020.6.20
-chardet==3.0.4
-idna==2.9
-isort==4.3.21
-lazy-object-proxy==1.4.3
-mccabe==0.6.1
-prompt-toolkit==1.0.14
-pyconfigstore3==1.0.1
-pyfiglet==0.8.post1
-Pygments==2.6.1
-PyInquirer==1.0.3
-pylint==2.5.3
-regex==2020.6.8
-requests==2.24.0
-six==1.15.0
-soupsieve==2.0.1
-termcolor==1.1.0
-toml==0.10.1
-urllib3==1.25.9
-wcwidth==0.2.5
-wrapt==1.12.1
+requests==2.27.1
+beautifulsoup4==4.11.1
+urllib3==1.26.9
+sqlalchemy==1.4.36
+sqlalchemy_utils==0.38.2
+paginate-sqlalchemy==0.3.1
+alembic==1.7.7
+python-dateutil==2.8.2
+Flask==2.1.2
+Flask-BasicAuth==0.2.0
+pyopenssl==22.0.0
+apscheduler==3.9.1
+werkzeug==2.2.2

src/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

src/alembic/env.py

@@ -0,0 +1,81 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from src.bot.models import TableNotification, TableSteamItem, TableGiveaway, Base
+
+target_metadata = Base.metadata
+
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

src/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

src/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py

@@ -0,0 +1,70 @@
+"""init
+
+Revision ID: 1da33402b659
+Revises: 
+Create Date: 2022-05-19 15:50:19.539401
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '1da33402b659'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('notification',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('type', sa.String(length=50), nullable=False),
+                    sa.Column('message', sa.String(length=300), nullable=False),
+                    sa.Column('medium', sa.String(length=50), nullable=False),
+                    sa.Column('success', sa.Boolean(), nullable=False),
+                    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_table('steam_item',
+                    sa.Column('steam_id', sa.String(length=15), nullable=False),
+                    sa.Column('game_name', sa.String(length=200), nullable=False),
+                    sa.Column('steam_url', sa.String(length=100), nullable=False),
+                    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.PrimaryKeyConstraint('steam_id')
+                    )
+    op.create_table('giveaway',
+                    sa.Column('giveaway_id', sa.String(length=10), nullable=False),
+                    sa.Column('steam_id', sa.Integer(), nullable=False),
+                    sa.Column('giveaway_uri', sa.String(length=200), nullable=False),
+                    sa.Column('user', sa.String(length=40), nullable=False),
+                    sa.Column('giveaway_created_at', sa.DateTime(timezone=True), nullable=False),
+                    sa.Column('giveaway_ended_at', sa.DateTime(timezone=True), nullable=False),
+                    sa.Column('cost', sa.Integer(), nullable=False),
+                    sa.Column('copies', sa.Integer(), nullable=False),
+                    sa.Column('contributor_level', sa.Integer(), nullable=False),
+                    sa.Column('entered', sa.Boolean(), nullable=False),
+                    sa.Column('won', sa.Boolean(), nullable=False),
+                    sa.Column('game_entries', sa.Integer(), nullable=False),
+                    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+                              nullable=True),
+                    sa.ForeignKeyConstraint(['steam_id'], ['steam_item.steam_id'], ),
+                    sa.PrimaryKeyConstraint('giveaway_id', 'steam_id')
+                    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('giveaway')
+    op.drop_table('steam_item')
+    op.drop_table('notification')
+    # ### end Alembic commands ###

src/alembic/versions/2022_05_21-10_25_41-15c028536ef5_won_column_removed.py

@@ -0,0 +1,31 @@
+"""won column removed
+
+Revision ID: 15c028536ef5
+Revises: 1da33402b659
+Create Date: 2022-05-21 10:25:41.647723
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '15c028536ef5'
+down_revision = '1da33402b659'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column('giveaway', 'won')
+
+
+def downgrade():
+    # add columns as nullable as existing records don't have that column value set
+    op.add_column('giveaway', sa.Column('won', sa.Boolean(), nullable=True))
+    # set value on new columns for all records
+    with op.get_context().autocommit_block():
+        op.execute("UPDATE giveaway SET won=false WHERE won is null;")
+    # SQLite doesn't support ALTERs so alembic uses a batch mode
+    # Set columns to non-nullable now that all records have a value
+    with op.batch_alter_table('giveaway') as batch_op:
+        batch_op.alter_column('won', nullable=False)

src/alembic/versions/2022_05_24-08_31_25-ff0a728ba3da_add_games_won_column.py

@@ -0,0 +1,27 @@
+"""add games_won column
+
+Revision ID: ff0a728ba3da
+Revises: 15c028536ef5
+Create Date: 2022-05-24 08:31:25.684099
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ff0a728ba3da'
+down_revision = '15c028536ef5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('notification', sa.Column('games_won', sa.Integer(), nullable=True))
+    # set a default for previous won notifications
+    with op.get_context().autocommit_block():
+        op.execute("UPDATE notification SET games_won=1 WHERE type='won';")
+
+
+def downgrade():
+    op.drop_column('notification', 'games_won')

src/alembic/versions/2022_06_01-09_20_36-8c1784114d65_won_column_added_back.py

@@ -0,0 +1,31 @@
+"""won column added backed
+
+Revision ID: 8c1784114d65
+Revises: ff0a728ba3da
+Create Date: 2022-06-01 09:20:36.762279
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '8c1784114d65'
+down_revision = 'ff0a728ba3da'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # add columns as nullable as existing records don't have that column value set
+    op.add_column('giveaway', sa.Column('won', sa.Boolean(), nullable=True))
+    # set value on new columns for all records
+    with op.get_context().autocommit_block():
+        op.execute("UPDATE giveaway SET won=false WHERE won is null;")
+    # SQLite doesn't support ALTERs so alembic uses a batch mode
+    # Set columns to non-nullable now that all records have a value
+    with op.batch_alter_table('giveaway') as batch_op:
+        batch_op.alter_column('won', nullable=False)
+
+
+def downgrade():
+    op.drop_column('giveaway', 'won')

src/bot/config_reader.py

@@ -0,0 +1,159 @@
+from configparser import ConfigParser
+from random import randint, randrange
+
+from .log import get_logger
+
+logger = get_logger(__name__)
+
+
+class ConfigException(Exception):
+    pass
+
+
+def value_range(min, max):
+    return [str(x) for x in [*range(min, max + 1)]]
+
+
+def choose_user_agent():
+    user_agents = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15',
+        'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36'
+    ]
+    return user_agents[randrange(0, len(user_agents))]
+
+
+class ConfigReader(ConfigParser):
+    required_values = {
+        'DEFAULT': {
+        },
+        'ALL': {
+            'all.enabled': ('true', 'false'),
+            'all.minimum_points': '%s' % (value_range(0, 400)),
+            'all.max_entries': '%s' % (value_range(0, 100000)),
+            'all.max_time_left': '%s' % (value_range(0, 21600)),
+            'all.max_points': '%s' % (value_range(0, 400)),
+            'all.minimum_game_points': '%s' % (value_range(0, 50))
+        },
+        'WISHLIST': {
+            'wishlist.enabled': ('true', 'false'),
+            'wishlist.minimum_points': '%s' % (value_range(0, 400)),
+            'wishlist.max_entries': '%s' % (value_range(0, 100000)),
+            'wishlist.max_time_left': '%s' % (value_range(0, 21600)),
+            'wishlist.max_points': '%s' % (value_range(0, 400)),
+        },
+        'DLC': {
+            'dlc.enabled': ('true', 'false'),
+            'dlc.minimum_points': '%s' % (value_range(0, 400)),
+            'dlc.max_entries': '%s' % (value_range(0, 100000)),
+            'dlc.max_time_left': '%s' % (value_range(0, 21600)),
+            'dlc.max_points': '%s' % (value_range(0, 400)),
+        },
+        'NOTIFICATIONS': {
+            'pushover.enabled': ('true', 'false'),
+        },
+        'WEB': {
+            'web.enabled': ('true', 'false'),
+            'web.port': '%s' % (value_range(1, 65535))
+        }
+    }
+    default_values = {
+        'DEFAULT': {
+            'cookie': '',
+            'user_agent': f"{choose_user_agent()}"
+        },
+        'ALL': {
+            'all.enabled': 'true',
+            'all.minimum_points': f"{randint(20, 50)}",
+            'all.max_entries': f"{randint(1000, 2500)}",
+            'all.max_time_left': f"{randint(180, 500)}",
+            'all.max_points': f"{randint(300, 400)}",
+            'all.minimum_game_points': "0",
+            'all.blacklist_keywords': 'hentai,adult'
+        },
+        'WISHLIST': {
+            'wishlist.enabled': 'true',
+            'wishlist.minimum_points': '1',
+            'wishlist.max_entries': f"{randint(10000, 100000)}",
+            'wishlist.max_time_left': f"{randint(180, 500)}",
+            'wishlist.max_points': f"{randint(300, 400)}",
+        },
+        'DLC': {
+            'dlc.enabled': 'true',
+            'dlc.minimum_points': '1',
+            'dlc.max_entries': f"{randint(10000, 100000)}",
+            'dlc.max_time_left': f"{randint(180, 500)}",
+            'dlc.max_points': f"{randint(300, 400)}",
+        },
+        'NOTIFICATIONS': {
+            'notification.prefix': '',
+            'pushover.enabled': 'false',
+            'pushover.token': '',
+            'pushover.user_key': '',
+        },
+        'WEB': {
+            'web.enabled': 'false',
+            'web.host': '0.0.0.0',
+            'web.app_root': '/',
+            'web.port': '9647',
+            'web.ssl': 'true',
+            'web.basic_auth': 'true',
+            'web.basic_auth.username': 'admin',
+            'web.basic_auth.password': 'p@ssw0rd'
+        }
+    }
+    deprecated_values = {
+        'DEFAULT': {
+            'pinned': 'false',
+            'gift_types': 'All'
+        }
+    }
+
+    def __init__(self, config_file):
+        super(ConfigReader, self).__init__()
+        self.read(config_file)
+        modified = self.create_defaults()
+        if modified:
+            with open(config_file, 'w+') as file:
+                self.write(file)
+        self.find_deprecated()
+        self.validate_config()
+
+    def create_defaults(self):
+        modified = False
+        for section, keys in self.default_values.items():
+            if section not in self:
+                self.add_section(section)
+                modified = True
+            for key, value in keys.items():
+                if key not in self[section]:
+                    self.set(section, key, value)
+                    modified = True
+        return modified
+
+    def find_deprecated(self):
+        for section, keys in self.deprecated_values.items():
+            for key, values in keys.items():
+                if key in self[section]:
+                    logger.warn(f"config.ini : Key '{key}' in {section} is no longer used. Please remove.")
+
+    def validate_config(self):
+        for section, keys in self.required_values.items():
+            if section not in self:
+                raise ConfigException(
+                    'Missing section "%s" in the config file' % section)
+
+            for key, values in keys.items():
+                if key not in self[section] or self[section][key] == '':
+                    raise ConfigException((
+                                                  'Missing value for "%s" under section "%s" in ' +
+                                                  'the config file') % (key, section))
+
+                if values:
+                    if self[section][key] not in values:
+                        raise ConfigException((
+                                                      'Invalid value for "%s" under section "%s" in ' +
+                                                      'the config file') % (key, section))

src/bot/database.py

@@ -0,0 +1,222 @@
+import os
+from datetime import datetime, timedelta
+
+import paginate_sqlalchemy
+import sqlalchemy
+from alembic import command
+from alembic.config import Config
+from dateutil import tz
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session, joinedload
+from sqlalchemy_utils import database_exists
+
+from .log import get_logger
+from .models import TableNotification, TableGiveaway, TableSteamItem
+
+logger = get_logger(__name__)
+engine = sqlalchemy.create_engine(f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}", echo=False)
+engine.connect()
+
+
+def run_db_migrations(script_location: str, db_url: str) -> None:
+    logger.debug('Running DB migrations in %r on %r', script_location, db_url)
+    alembic_cfg = Config()
+    alembic_cfg.set_main_option('script_location', script_location)
+    alembic_cfg.set_main_option('sqlalchemy.url', db_url)
+
+    if not database_exists(db_url):
+        logger.debug(f"'{db_url}' does not exist. Running normal migration to create db and tables.")
+        command.upgrade(alembic_cfg, 'head')
+    elif database_exists(db_url):
+        logger.debug(f"'{db_url}' exists.")
+        insp = sqlalchemy.inspect(engine)
+        alembic_version_table_name = 'alembic_version'
+        has_alembic_table = insp.has_table(alembic_version_table_name)
+        if has_alembic_table:
+            logger.debug(f"Table '{alembic_version_table_name}' exists.")
+        else:
+            logger.debug(f"Table '{alembic_version_table_name}' doesn't exist so assuming it was created pre-alembic "
+                         f"setup. Setting the version to the first version created prior to alembic setup.")
+            alembic_first_ref = '1da33402b659'
+            command.stamp(alembic_cfg, alembic_first_ref)
+        logger.debug("Running migration.")
+        command.upgrade(alembic_cfg, 'head')
+
+
+class NotificationHelper:
+
+    @classmethod
+    def get(cls):
+        with Session(engine) as session:
+            return session.query(TableNotification).order_by(TableNotification.created_at.desc()).all()
+
+    @classmethod
+    def insert(cls, type_of_error, message, medium, success, number_won):
+        with Session(engine) as session:
+            n = TableNotification(type=type_of_error, message=message, medium=medium, success=success,
+                                  games_won=number_won)
+            session.add(n)
+            session.commit()
+
+    @classmethod
+    def get_won_notifications_today(cls):
+        with Session(engine) as session:
+            # with how filtering of datetimes works with a sqlite backend I couldn't figure out a better way
+            # to filter out the dates to local time when they are stored in utc in the db
+            within_3_days = session.query(TableNotification) \
+                .filter(func.DATE(TableNotification.created_at) >= (datetime.utcnow().date() - timedelta(days=1))) \
+                .filter(func.DATE(TableNotification.created_at) <= (datetime.utcnow().date() + timedelta(days=1))) \
+                .filter_by(type='won').order_by(TableNotification.created_at.asc()).all()
+            actual = []
+            for r in within_3_days:
+                if r.created_at.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal()).date() == datetime.now(
+                        tz=tz.tzlocal()).date():
+                    actual.append(r)
+            return actual
+
+    @classmethod
+    def get_won_notifications(cls):
+        with Session(engine) as session:
+            return session.query(TableNotification) \
+                .filter_by(type='won') \
+                .all()
+
+    @classmethod
+    def get_error_notifications(cls):
+        with Session(engine) as session:
+            return session.query(TableNotification) \
+                .filter_by(type='error') \
+                .all()
+
+
+class GiveawayHelper:
+
+    @classmethod
+    def get(cls):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).options(joinedload('steam_item')) \
+                .order_by(TableGiveaway.giveaway_ended_at.desc()).all()
+
+    @classmethod
+    def paginate(cls, page=1):
+        with Session(engine) as session:
+            paginated_giveaways = paginate_sqlalchemy.SqlalchemyOrmPage(session.query(TableGiveaway)
+                                                                        .options(joinedload('steam_item'))
+                                                                        .order_by(
+                TableGiveaway.giveaway_ended_at.desc()), page=page)
+            return paginated_giveaways
+
+    @classmethod
+    def total_giveaways(cls):
+        with Session(engine) as session:
+            return session.execute(select(func.count(TableGiveaway.giveaway_id))).scalar_one()
+
+    @classmethod
+    def total_entered(cls):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).filter_by(entered=True).count()
+
+    @classmethod
+    def total_won(cls):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).filter_by(won=True).count()
+
+    @classmethod
+    def get_by_giveaway_id(cls, game_id):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).filter_by(giveaway_id=game_id).one_or_none()
+
+    @classmethod
+    def unix_timestamp_to_utc_datetime(cls, timestamp):
+        return datetime.utcfromtimestamp(timestamp)
+
+    @classmethod
+    def get_by_ids(cls, giveaway):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).filter_by(giveaway_id=giveaway.giveaway_game_id,
+                                                          steam_id=giveaway.steam_app_id).all()
+
+    @classmethod
+    def mark_game_as_won(cls, game_id):
+        with Session(engine) as session:
+            result = session.query(TableGiveaway).filter_by(giveaway_id=game_id).all()
+            if result:
+                won_giveaway = result[0]
+                if not won_giveaway.won:
+                    won_giveaway.won = True
+                    session.add(won_giveaway)
+                    session.commit()
+
+    @classmethod
+    def insert(cls, giveaway, entered, won):
+        with Session(engine) as session:
+            result = session.query(TableSteamItem).filter_by(steam_id=giveaway.steam_app_id).all()
+            if result:
+                steam_id = result[0].steam_id
+            else:
+                item = TableSteamItem(
+                    steam_id=giveaway.steam_app_id,
+                    steam_url=giveaway.steam_url,
+                    game_name=giveaway.game_name)
+                session.add(item)
+                session.flush()
+                steam_id = item.steam_id
+            g = TableGiveaway(
+                giveaway_id=giveaway.giveaway_game_id,
+                steam_id=steam_id,
+                giveaway_uri=giveaway.giveaway_uri,
+                user=giveaway.user,
+                giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
+                giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
+                cost=giveaway.cost,
+                copies=giveaway.copies,
+                contributor_level=giveaway.contributor_level,
+                entered=entered,
+                won=won,
+                game_entries=giveaway.game_entries)
+            session.add(g)
+            session.commit()
+
+    @classmethod
+    def upsert_giveaway_with_details(cls, giveaway, entered, won):
+        result = GiveawayHelper.get_by_ids(giveaway)
+        if not result:
+            GiveawayHelper.insert(giveaway, entered, won)
+        else:
+            with Session(engine) as session:
+                g = TableGiveaway(
+                    giveaway_id=giveaway.giveaway_game_id,
+                    steam_id=result[0].steam_id,
+                    giveaway_uri=giveaway.giveaway_uri,
+                    user=giveaway.user,
+                    giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
+                    giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
+                    cost=giveaway.cost,
+                    copies=giveaway.copies,
+                    contributor_level=giveaway.contributor_level,
+                    entered=entered,
+                    won=won,
+                    game_entries=giveaway.game_entries)
+                session.merge(g)
+                session.commit()
+
+    @classmethod
+    def upsert_giveaway(cls, giveaway):
+        result = GiveawayHelper.get_by_ids(giveaway)
+        if not result:
+            GiveawayHelper.insert(giveaway, False, False)
+        else:
+            with Session(engine) as session:
+                g = TableGiveaway(
+                    giveaway_id=giveaway.giveaway_game_id,
+                    steam_id=result[0].steam_id,
+                    giveaway_uri=giveaway.giveaway_uri,
+                    user=giveaway.user,
+                    giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
+                    giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
+                    cost=giveaway.cost,
+                    copies=giveaway.copies,
+                    contributor_level=giveaway.contributor_level,
+                    game_entries=giveaway.game_entries)
+                session.merge(g)
+                session.commit()

src/bot/enter_giveaways.py

@@ -0,0 +1,246 @@
+import json
+from random import randint
+from time import sleep
+
+import requests
+from bs4 import BeautifulSoup
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from .log import get_logger
+from .database import NotificationHelper, GiveawayHelper
+from .giveaway_entry import GiveawayEntry
+
+logger = get_logger(__name__)
+
+
+class SteamGiftsException(Exception):
+    pass
+
+
+class EnterGiveaways:
+
+    def __init__(self, cookie, user_agent, gifts_type, pinned, min_points, max_entries,
+                 max_time_left, max_points, minimum_game_points, blacklist, notification):
+        self._contributor_level = None
+        self._xsrf_token = None
+        self._points = None
+        self._cookie = {
+            'PHPSESSID': cookie
+        }
+        self._user_agent = user_agent
+        self._gifts_type = gifts_type
+        self._pinned = pinned
+        self._min_points = int(min_points)
+        self._max_entries = int(max_entries)
+        self._max_time_left = int(max_time_left)
+        self._max_points = int(max_points)
+        self._minimum_game_points = int(minimum_game_points)
+        self._blacklist = blacklist.split(',')
+        self._notification = notification
+
+        self._base = "https://www.steamgifts.com"
+        self._session = requests.Session()
+
+        self._filter_url = {
+            'All': "search?page=%d",
+            'Wishlist': "search?page=%d&type=wishlist",
+            'Recommended': "search?page=%d&type=recommended",
+            'Copies': "search?page=%d&copy_min=2",
+            'DLC': "search?page=%d&dlc=true",
+            'New': "search?page=%d&type=new"
+        }
+
+    def start(self):
+        self._update_info()
+        if self._points >= self._min_points:
+            txt = f"〰 You have {self._points} points. Evaluating '{self._gifts_type}' giveaways..."
+            logger.info(txt)
+            self._evaluate_giveaways()
+        else:
+            txt = f"🟡 You have {self._points} points which is below your minimum point threshold of " \
+                  f"{self._min_points} points for '{self._gifts_type}' giveaways. Not evaluating right now."
+            logger.info(txt)
+
+    def _requests_retry_session(
+            self,
+            retries=5,
+            backoff_factor=0.3
+    ):
+        session = self._session or requests.Session()
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=(500, 502, 504),
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        return session
+
+    def _get_soup_from_page(self, url):
+        headers = {
+            'User-Agent': self._user_agent
+        }
+        self._requests_retry_session().get(url, headers=headers)
+        r = requests.get(url, cookies=self._cookie)
+        soup = BeautifulSoup(r.text, 'html.parser')
+        return soup
+
+    def _update_info(self):
+        soup = self._get_soup_from_page(self._base)
+
+        try:
+            self._xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value']
+            self._points = int(soup.find('span', {'class': 'nav__points'}).text)  # storage points
+            self._contributor_level = int(float(soup.select_one('nav a>span[title]')['title']))
+        except TypeError:
+            logger.error("⛔⛔⛔  Cookie is not valid. A new one must be added.⛔⛔⛔")
+            raise SteamGiftsException("Cookie is not valid. A new one must be added.")
+
+        won = soup.select("a[title='Giveaways Won'] div")
+        if won:
+            number_won = int(soup.select_one("a[title='Giveaways Won'] div").text)
+            won_notifications = NotificationHelper.get_won_notifications_today()
+            if won_notifications and len(won_notifications) >= 1:
+                if number_won == won_notifications[-1].games_won:
+                    logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting "
+                                "to be received. Doing nothing.")
+                elif number_won > won_notifications[-1].games_won:
+                    logger.info("🔥🔥 MORE win(s) detected. Notifying again.")
+                    self._notification.send_won(f"You won ANOTHER game. You now have {number_won} game(s) "
+                                               f"waiting to be claimed.", number_won)
+                else:  # we have less games waiting to be claimed than notified, meaning some have been claimed
+                    logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting "
+                                "to be received. Some have been claimed. Doing nothing.")
+            else:
+                logger.info(f"💰💰 WINNER! You have {number_won} game(s) waiting to be claimed. 💰💰")
+                self._notification.send_won(f"WINNER! You have {number_won} game(s) waiting to be claimed.", number_won)
+        else:
+            logger.debug('No wins detected. Doing nothing.')
+
+    def _should_we_enter_giveaway(self, giveaway):
+        if giveaway.time_remaining_in_minutes is None:
+            return False
+        if giveaway.time_created_in_minutes is None:
+            return False
+
+        if self._blacklist is not None and self._blacklist != ['']:
+            for keyword in self._blacklist:
+                if giveaway.game_name.lower().find(keyword.lower()) != -1:
+                    txt = f"〰️ Game {giveaway.game_name} contains the blacklisted keyword {keyword}"
+                    logger.info(txt)
+                    return False
+        if giveaway.contributor_level is None or self._contributor_level < giveaway.contributor_level:
+            txt = f"〰️ Game {giveaway.game_name} requires at least level {giveaway.contributor_level} contributor " \
+                  f"level to enter. Your level: {self._contributor_level}"
+            logger.info(txt)
+            return False
+        if giveaway.time_remaining_in_minutes > self._max_time_left and self._points < self._max_points:
+            txt = f"〰️ Game {giveaway.game_name} has {giveaway.time_remaining_in_minutes} minutes left and is " \
+                  f"above your cutoff of {self._max_time_left} minutes."
+            logger.info(txt)
+            return False
+        if giveaway.game_entries / giveaway.copies > self._max_entries and self._points < self._max_points:
+            txt = f"〰️ Game {giveaway.game_name} has {giveaway.game_entries} entries and is above your cutoff " \
+                  f"of {self._max_entries} entries."
+            logger.info(txt)
+            return False
+        if self._points - int(giveaway.cost) < 0:
+            txt = f"〰️ Not enough points to enter: {giveaway.game_name}"
+            logger.info(txt)
+            return False
+        if giveaway.cost < self._minimum_game_points:
+            txt = f"〰️ Game {giveaway.game_name} costs {giveaway.cost}P and is below your cutoff of " \
+                  f"{self._minimum_game_points}P."
+            logger.info(txt)
+            return False
+
+        return True
+
+    def _enter_giveaway(self, giveaway):
+        headers = {
+            'User-Agent': self._user_agent
+        }
+        payload = {'xsrf_token': self._xsrf_token, 'do': 'entry_insert', 'code': giveaway.giveaway_game_id}
+        logger.debug(f"Sending enter giveaway payload: {payload}")
+        entry = requests.post('https://www.steamgifts.com/ajax.php', data=payload, cookies=self._cookie,
+                              headers=headers)
+        json_data = json.loads(entry.text)
+
+        if json_data['type'] == 'success':
+            logger.debug(f"Successfully entered giveaway {giveaway.giveaway_game_id}: {json_data}")
+            return True
+        else:
+            logger.error(f"❌ Failed entering giveaway {giveaway.giveaway_game_id}: {json_data}")
+            return False
+
+    def _evaluate_giveaways(self, page=1):
+        n = page
+        run = True
+        while run and n < 3:  # hard stop safety net at page 3 as idk why we would ever get to this point
+            txt = "〰️ Retrieving games from %d page." % n
+            logger.info(txt)
+
+            filtered_url = self._filter_url[self._gifts_type] % n
+            paginated_url = f"{self._base}/giveaways/{filtered_url}"
+
+            soup = self._get_soup_from_page(paginated_url)
+
+            pinned_giveaway_count = len(soup.select('div.pinned-giveaways__outer-wrap div.giveaway__row-inner-wrap'))
+            all_games_list_count = len(soup.select('div.giveaway__row-inner-wrap'))
+            # this matches on a div with the exact class value so we discard ones
+            # that also have a class 'is-faded' containing already entered giveaways
+            unentered_game_list = soup.select('div[class=giveaway__row-inner-wrap]')
+            # game_list = soup.find_all('div', {'class': 'giveaway__row-inner-wrap'})
+
+            if not len(unentered_game_list) or (all_games_list_count == pinned_giveaway_count):
+                txt = f"🟡 We have run out of gifts to consider."
+                logger.info(txt)
+                break
+
+            for item in unentered_game_list:
+                giveaway = GiveawayEntry(item)
+                txt = f"〰 {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries " \
+                      f"(w/ {giveaway.copies} copies) - Created {giveaway.time_created_string} ago " \
+                      f"with {giveaway.time_remaining_string} remaining by {giveaway.user}."
+                logger.info(txt)
+                if giveaway.pinned and not self._pinned:
+                    logger.info(f"〰️ Giveaway {giveaway.game_name} is pinned. Ignoring.")
+                    continue
+
+                if self._points == 0 or self._points < self._min_points:
+                    txt = f"🟡 We have {self._points} points, but we need {self._min_points} to start."
+                    logger.info(txt)
+                    run = False
+                    break
+
+                if not giveaway.cost:
+                    logger.error(f"Cost could not be determined for '{giveaway.game_name}'")
+                    continue
+                if_enter_giveaway = self._should_we_enter_giveaway(giveaway)
+                if if_enter_giveaway:
+                    res = self._enter_giveaway(giveaway)
+                    if res:
+                        GiveawayHelper.upsert_giveaway_with_details(giveaway, True, False)
+                        self._points -= int(giveaway.cost)
+                        txt = f"✅ Entered giveaway '{giveaway.game_name}'"
+                        logger.info(txt)
+                        sleep(randint(4, 15))
+                    else:
+                        GiveawayHelper.upsert_giveaway_with_details(giveaway, False, False)
+                else:
+                    GiveawayHelper.upsert_giveaway(giveaway)
+                # if we are on any filter type except New and we get to a giveaway that exceeds our
+                # max time left amount, then we don't need to continue to look at giveaways as any
+                # after this point will also exceed the max time left
+                if self._gifts_type != "New" and not giveaway.pinned and \
+                        giveaway.time_remaining_in_minutes > self._max_time_left and \
+                        self._points < self._max_points:
+                    logger.info("🟡 We have run out of gifts to consider.")
+                    run = False
+                    break
+
+            n = n + 1

src/bot/evaluate_won_giveaways.py

@@ -0,0 +1,86 @@
+import requests
+from bs4 import BeautifulSoup
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from .database import GiveawayHelper
+from .log import get_logger
+from .won_entry import WonEntry
+
+logger = get_logger(__name__)
+
+
+class SteamGiftsException(Exception):
+    pass
+
+
+class EvaluateWonGiveaways:
+
+    def __init__(self, cookie, user_agent, notification):
+        self._contributor_level = None
+        self._xsrf_token = None
+        self._cookie = {
+            'PHPSESSID': cookie
+        }
+        self._user_agent = user_agent
+        self._notification = notification
+
+        self._base = "https://www.steamgifts.com/giveaways/won"
+        self._session = requests.Session()
+
+    def start(self):
+        self._evaluate_won_giveaways()
+
+    def _requests_retry_session(
+            self,
+            retries=5,
+            backoff_factor=0.3
+    ):
+        session = self._session or requests.Session()
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=(500, 502, 504),
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        return session
+
+    def _get_soup_from_page(self, url):
+        headers = {
+            'User-Agent': self._user_agent
+        }
+        self._requests_retry_session().get(url, headers=headers)
+        r = requests.get(url, cookies=self._cookie)
+        soup = BeautifulSoup(r.text, 'html.parser')
+        return soup
+
+    def _evaluate_won_giveaways(self, page=1):
+        soup = self._get_soup_from_page(self._base)
+
+        try:
+            self._xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value']
+            self._points = int(soup.find('span', {'class': 'nav__points'}).text)  # storage points
+            self._contributor_level = int(float(soup.select_one('nav a>span[title]')['title']))
+        except TypeError:
+            logger.error("⛔⛔⛔  Cookie is not valid. A new one must be added.⛔⛔⛔")
+            raise SteamGiftsException("Cookie is not valid. A new one must be added.")
+
+        won_game_list = soup.select('div[class=table__row-inner-wrap]')
+
+        if not len(won_game_list):
+            txt = f"🟡 No won games to evaluate"
+            logger.info(txt)
+
+        for item in won_game_list:
+            won_giveaway = WonEntry(item)
+            w = GiveawayHelper.get_by_giveaway_id(won_giveaway.giveaway_game_id)
+            logger.debug(f"Giveaway in db: {w}")
+            if w and not w.won and w.entered:
+                logger.info(f"Marking {won_giveaway.game_name} as won.")
+                logger.debug(f"Marking: {w}")
+                GiveawayHelper.mark_game_as_won(won_giveaway.giveaway_game_id)
+

src/bot/giveaway_entry.py

@@ -0,0 +1,110 @@
+import re
+from .log import get_logger
+import time
+
+logger = get_logger(__name__)
+
+
+class GiveawayEntry:
+
+    def __init__(self, soup_item):
+        self.steam_app_id = None
+        self.steam_url = None
+        self.game_name = None
+        self.giveaway_game_id = None
+        self.giveaway_uri = None
+        self.pinned = False
+        self.cost = None
+        self.game_entries = None
+        self.user = None
+        self.copies = None
+        self.contributor_level = None
+        self.time_created_timestamp = None
+        self.time_remaining_string = None
+        self.time_remaining_in_minutes = None
+        self.time_remaining_timestamp = None
+        self.time_created_string = None
+        self.time_created_in_minutes = None
+
+        logger.debug(f"Giveaway html: {soup_item}")
+        icons = soup_item.select('a.giveaway__icon')
+        self.steam_url = icons[0]['href']
+        self.steam_app_id = self._get_steam_app_id(self.steam_url)
+        self.game_name = soup_item.find('a', {'class': 'giveaway__heading__name'}).text
+        self.giveaway_game_id = soup_item.find('a', {'class': 'giveaway__heading__name'})['href'].split('/')[2]
+        self.giveaway_uri = soup_item.select_one('a.giveaway__heading__name')['href']
+        pin_class = soup_item.parent.parent.get("class")
+        self.pinned = pin_class is not None and len(pin_class) > 0 and pin_class[0].find('pinned') != -1
+        self.cost, self.copies = self._determine_cost_and_copies(soup_item, self.game_name, self.giveaway_game_id)
+        self.game_entries = int(soup_item.select('div.giveaway__links span')[0].text.split(' ')[0].replace(',', ''))
+        contributor_level = soup_item.select_one('div[title="Contributor Level"]')
+        self.contributor_level = self._determine_contributor_level(contributor_level)
+        self.user = soup_item.select_one('a.giveaway__username').text
+        times = soup_item.select('div span[data-timestamp]')
+        self.time_remaining_timestamp = int(times[0]['data-timestamp'])
+        self.time_remaining_string = times[0].text
+        self.time_remaining_in_minutes = self._determine_time_in_minutes(times[0]['data-timestamp'])
+        self.time_created_timestamp = int(times[1]['data-timestamp'])
+        self.time_created_string = times[1].text
+        self.time_created_in_minutes = self._determine_time_in_minutes(times[1]['data-timestamp'])
+        logger.debug(f"Scraped Giveaway: {self}")
+
+    def _determine_contributor_level(self, contributor_level):
+        if contributor_level is None:
+            return 0
+        match = re.search('^Level (?P<level>[0-9]+)\\+$', contributor_level.text, re.IGNORECASE)
+        if match:
+            return int(match.group('level'))
+        else:
+            return None
+
+    def _get_steam_app_id(self, steam_url):
+        match = re.search('^.+/[a-z0-9]+/(?P<steam_app_id>[0-9]+)(/|\?)', steam_url, re.IGNORECASE)
+        if match:
+            return match.group('steam_app_id')
+        else:
+            return None
+
+    def _determine_time_in_minutes(self, timestamp):
+        if not timestamp or not re.search('^[0-9]+$', timestamp):
+            logger.error(f"Could not determine time from string {timestamp}")
+            return None
+        now = time.localtime()
+        giveaway_endtime = time.localtime(int(timestamp))
+        return int(abs((time.mktime(giveaway_endtime) - time.mktime(now)) / 60))
+
+    def _determine_cost_and_copies(self, item, game_name, game_id):
+        item_headers = item.find_all('span', {'class': 'giveaway__heading__thin'})
+        if len(item_headers) == 1:  # then no multiple copies
+            game_cost = item_headers[0].getText().replace('(', '').replace(')', '').replace('P', '')
+            if not re.search('^[0-9]+$', game_cost):
+                txt = f"Unable to determine cost of {game_name} with id {game_id}. Cost string: {item_headers[0]}"
+                logger.error(txt)
+                return None, None
+            game_cost = int(game_cost)
+            return game_cost, 1
+        elif len(item_headers) == 2:  # then multiple copies
+            game_cost = item_headers[1].getText().replace('(', '').replace(')', '').replace('P', '')
+            if not re.search('^[0-9]+$', game_cost):
+                txt = f"Unable to determine cost of {game_name} with id {game_id}. Cost string: {item_headers[1].getText()}"
+                logger.error(txt)
+                return None, None
+            game_cost = int(game_cost)
+
+            match = re.search('(?P<copies>[0-9]+) Copies', item_headers[0].getText(), re.IGNORECASE)
+            if match:
+                num_copies_str = match.group('copies')
+                num_copies = int(num_copies_str)
+                return game_cost, num_copies
+            else:
+                txt = f"It appears there are multiple copies of {game_name} with id {game_id}, but we could not " \
+                      f"determine that. Copy string: {item_headers[0].getText()}"
+                logger.error(txt)
+                return game_cost, 1
+        else:
+            txt = f"Unable to determine cost or num copies of {game_name} with id {game_id}."
+            logger.error(txt)
+            return None, None
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)

src/bot/giveaway_thread.py

@@ -0,0 +1,155 @@
+import datetime
+import threading
+from datetime import timedelta, datetime
+from threading import Thread
+from time import sleep
+
+from .enter_giveaways import EnterGiveaways
+from .evaluate_won_giveaways import EvaluateWonGiveaways
+from .log import get_logger
+from .scheduler import Scheduler
+
+logger = get_logger(__name__)
+
+
+class GiveawayThread(threading.Thread):
+
+    def __init__(self, config, notification):
+        Thread.__init__(self)
+        self.exc = None
+        self.config = config
+        self.notification = notification
+        logger.debug("Creating scheduler")
+        self._scheduler = Scheduler()
+        self.won_giveaway_job_id = 'eval_won_giveaways'
+        self.evaluate_giveaway_job_id = 'eval_giveaways'
+
+        self._all_page = None
+        self._wishlist_page = None
+        self._dlc_page = None
+
+        cookie = config['DEFAULT'].get('cookie')
+        user_agent = config['DEFAULT'].get('user_agent')
+
+        if config['ALL'].getboolean('all.enabled'):
+            all_minimum_points = config['ALL'].getint('all.minimum_points')
+            all_max_entries = config['ALL'].getint('all.max_entries')
+            all_max_time_left = config['ALL'].getint('all.max_time_left')
+            all_max_points = config['ALL'].getint('all.max_points')
+            all_minimum_game_points = config['DEFAULT'].getint('minimum_game_points')
+            all_blacklist = config['DEFAULT'].get('blacklist_keywords')
+
+            self._all_page = EnterGiveaways(cookie, user_agent, 'All', False, all_minimum_points, all_max_entries,
+                                            all_max_time_left, all_max_points, all_minimum_game_points, all_blacklist, notification)
+
+        if config['WISHLIST'].getboolean('wishlist.enabled'):
+            wishlist_minimum_points = config['WISHLIST'].getint('wishlist.minimum_points')
+            wishlist_max_entries = config['WISHLIST'].getint('wishlist.max_entries')
+            wishlist_max_time_left = config['WISHLIST'].getint('wishlist.max_time_left')
+            wishlist_max_points = config['WISHLIST'].getint('wishlist.max_points')
+
+            self._wishlist_page = EnterGiveaways(cookie, user_agent, 'Wishlist', False, wishlist_minimum_points,
+                                                 wishlist_max_entries, wishlist_max_time_left, wishlist_max_points, 0, '', notification)
+
+        if config['DLC'].getboolean('dlc.enabled'):
+            dlc_minimum_points = config['DLC'].getint('dlc.minimum_points')
+            dlc_max_entries = config['DLC'].getint('dlc.max_entries')
+            dlc_max_time_left = config['DLC'].getint('dlc.max_time_left')
+            dlc_max_points = config['DLC'].getint('dlc.max_points')
+
+            self._dlc_page = EnterGiveaways(cookie, user_agent, 'DLC', False, dlc_minimum_points,
+                                                 dlc_max_entries, dlc_max_time_left, dlc_max_points, 0, '', notification)
+
+        if not self._all_page and not self._wishlist_page and not self._dlc_page :
+            logger.error("⁉️ 'All', 'Wishlist' and 'DLC' configurations are disabled. Nothing will run. Exiting...")
+            sleep(10)
+            exit(-1)
+
+        won_giveaway_job = self._scheduler.get_job(job_id=self.won_giveaway_job_id)
+        if won_giveaway_job:
+            logger.debug("Previous won giveaway evaluator job exists. Removing.")
+            won_giveaway_job.remove()
+        won_runner = GiveawayThread.WonRunner(EvaluateWonGiveaways(cookie, user_agent, notification),
+                                              self.won_giveaway_job_id)
+        self._scheduler.add_job(won_runner.run,
+                                id=self.won_giveaway_job_id,
+                                trigger='interval',
+                                max_instances=1,
+                                replace_existing=True,
+                                hours=12,
+                                next_run_time=datetime.now() + timedelta(minutes=5),
+                                jitter=8000)
+
+        evaluate_giveaway_job = self._scheduler.get_job(job_id=self.evaluate_giveaway_job_id)
+        if evaluate_giveaway_job:
+            logger.debug("Previous giveaway evaluator job exists. Removing.")
+            evaluate_giveaway_job.remove()
+        runner = GiveawayThread.GiveawayRunner(self._dlc_page, self._wishlist_page, self._all_page,
+                                               self.evaluate_giveaway_job_id)
+        self._scheduler.add_job(runner.run,
+                                id=self.evaluate_giveaway_job_id,
+                                trigger='interval',
+                                max_instances=1,
+                                replace_existing=True,
+                                minutes=44,
+                                next_run_time=datetime.now(),
+                                jitter=900)
+
+    def run(self):
+        # Variable that stores the exception, if raised by someFunction
+        self.exc = None
+        try:
+            self._scheduler.start()
+        except BaseException as e:
+            self.exc = e
+
+    def join(self):
+        threading.Thread.join(self)
+        # Since join() returns in caller thread
+        # we re-raise the caught exception
+        # if any was caught
+        if self.exc:
+            raise self.exc
+
+    class WonRunner:
+        def __init__(self, won_page, job_id):
+            self._won_page = won_page
+            self._job_id = job_id
+
+        def run(self):
+            logger.info(" 💍 Evaluating won giveaways. 💍")
+            if self._won_page:
+                self._won_page.start()
+            logger.info(" 💍 All won giveaways evaluated.  💍")
+            scheduler = Scheduler()
+            evaluate_giveaway_job = scheduler.get_job(job_id=self._job_id)
+            if evaluate_giveaway_job:
+                when_to_start_again = evaluate_giveaway_job.next_run_time
+                logger.info(f"💍 Going to sleep. Will start again at {when_to_start_again}")
+            else:
+                logger.info("No set time to evaluate won giveaways again.")
+
+    class GiveawayRunner:
+
+        def __init__(self, dlc_page, wishlist_page, all_page, job_id):
+            self._dlc_page = dlc_page
+            self._wishlist_page = wishlist_page
+            self._all_page = all_page
+            self._job_id = job_id
+
+        def run(self):
+            logger.info("🟢 Evaluating giveaways.")
+            if self._dlc_page:
+                self._dlc_page.start()
+            if self._wishlist_page:
+                self._wishlist_page.start()
+            if self._all_page:
+                self._all_page.start()
+            logger.info("🔴 All giveaways evaluated.")
+            scheduler = Scheduler()
+            evaluate_giveaway_job = scheduler.get_job(job_id=self._job_id)
+            if evaluate_giveaway_job:
+                when_to_start_again = evaluate_giveaway_job.next_run_time
+                logger.info(f"🛋 Going to sleep. Will start again at {when_to_start_again}")
+            else:
+                logger.info("No set time to evaluate giveaways again.")

src/bot/log.py

@@ -0,0 +1,29 @@
+import logging
+import os
+from logging.handlers import RotatingFileHandler
+
+# log info level logs to stdout and debug to debug file
+
+debug_log_format = "%(levelname)s %(asctime)s - [%(filename)s:%(lineno)d] - %(message)s"
+logging.basicConfig(
+    handlers=[RotatingFileHandler(f"{os.getenv('BOT_CONFIG_DIR', './config')}/debug.log", maxBytes=500000, backupCount=10)],
+    level=logging.DEBUG,
+    format=debug_log_format)
+
+console_output = logging.StreamHandler()
+console_output.setLevel(logging.INFO)
+console_format = logging.Formatter(debug_log_format)
+console_output.setFormatter(console_format)
+
+info_log_file = RotatingFileHandler(f"{os.getenv('BOT_CONFIG_DIR', './config')}/info.log", maxBytes=100000, backupCount=10)
+info_log_file.setLevel(logging.INFO)
+info_log_format = logging.Formatter(debug_log_format)
+info_log_file.setFormatter(info_log_format)
+
+logging.root.addHandler(console_output)
+logging.root.addHandler(info_log_file)
+
+
+def get_logger(name):
+    l = logging.getLogger(name)
+    return l

src/bot/models.py

@@ -0,0 +1,62 @@
+from sqlalchemy import Integer, String, Column, DateTime, Boolean, func, ForeignKey
+from sqlalchemy.orm import registry, relationship
+
+mapper_registry = registry()
+metadata = mapper_registry.metadata
+Base = mapper_registry.generate_base()
+
+
+class TableNotification(Base):
+    __tablename__ = 'notification'
+    id = Column(Integer, primary_key=True, nullable=False)
+    type = Column(String(50), nullable=False)
+    message = Column(String(300), nullable=False)
+    medium = Column(String(50), nullable=False)
+    success = Column(Boolean, nullable=False)
+    games_won = Column(Integer, nullable=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    __mapper_args__ = {"eager_defaults": True}
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)
+
+
+class TableSteamItem(Base):
+    __tablename__ = 'steam_item'
+    steam_id = Column(String(15), primary_key=True, nullable=False)
+    game_name = Column(String(200), nullable=False)
+    steam_url = Column(String(100), nullable=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    giveaways = relationship("TableGiveaway", back_populates="steam_item")
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)
+
+
+class TableGiveaway(Base):
+    __tablename__ = 'giveaway'
+    giveaway_id = Column(String(10), primary_key=True, nullable=False)
+    steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
+    giveaway_uri = Column(String(200), nullable=False)
+    user = Column(String(40), nullable=False)
+    giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
+    giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
+    cost = Column(Integer(), nullable=False)
+    copies = Column(Integer(), nullable=False)
+    contributor_level = Column(Integer(), nullable=False)
+    entered = Column(Boolean(), nullable=False)
+    won = Column(Boolean(), nullable=False)
+    game_entries = Column(Integer(), nullable=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    steam_item = relationship("TableSteamItem", back_populates="giveaways")
+
+    __mapper_args__ = {"eager_defaults": True}
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)

src/bot/notification.py

@@ -0,0 +1,51 @@
+import http.client
+import urllib
+
+from .log import get_logger
+from .database import NotificationHelper
+
+logger = get_logger(__name__)
+
+
+class Notification:
+
+    def __init__(self, message_prefix):
+        self.pushover = False
+        self.pushover_token = None
+        self.pushover_user_key = None
+        self.message_prefix = f"{message_prefix}: "
+
+    def send_won(self, message, number_won):
+        self.__send('won', message, number_won)
+
+    def send_error(self, message):
+        self.__send('error', message, number_won=None)
+
+    def enable_pushover(self, token, user_key):
+        logger.debug("Enabling pushover notifications.")
+        self.pushover = True
+        self.pushover_token = token
+        self.pushover_user_key = user_key
+
+    def __send(self, type_of_error, message, number_won=None):
+        logger.debug(f"Attempting to notify: '{message}'. Won: {number_won}")
+        if self.pushover:
+            logger.debug("Pushover enabled. Sending message.")
+            self.__pushover(type_of_error, message, number_won)
+
+    def __pushover(self, type_of_error, message, number_won=None):
+        conn = http.client.HTTPSConnection("api.pushover.net:443")
+        conn.request("POST", "/1/messages.json",
+                     urllib.parse.urlencode({
+                         "token": self.pushover_token,
+                         "user": self.pushover_user_key,
+                         "message": f"{self.message_prefix}{message}",
+                     }), {"Content-type": "application/x-www-form-urlencoded"})
+        response = conn.getresponse()
+        logger.debug(f"Pushover response code: {response.getcode()}")
+        if response.getcode() == 200:
+            success = True
+        else:
+            logger.error(f"Pushover notification failed. Code {response.getcode()}: {response.read().decode()}")
+            success = False
+        NotificationHelper.insert(type_of_error, f"{message}", 'pushover', success, number_won)

src/bot/scheduler.py

@@ -0,0 +1,33 @@
+import os
+
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.schedulers.blocking import BlockingScheduler
+
+from .log import get_logger
+
+logger = get_logger(__name__)
+
+
+class Scheduler:
+    class __Scheduler:
+        def __init__(self):
+            jobstores = {
+                'default': SQLAlchemyJobStore(url=f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}")
+            }
+            job_defaults = {
+                'coalesce': True,
+                'max_instances': 3
+            }
+            self.scheduler = BlockingScheduler(jobstores=jobstores, job_defaults=job_defaults)
+
+        def __getattr__(self, name):
+            return getattr(self.scheduler, name)
+
+    instance = None
+
+    def __init__(self):
+        if not Scheduler.instance:
+            Scheduler.instance = Scheduler.__Scheduler()
+
+    def __getattr__(self, name):
+        return getattr(self.instance, name)

src/bot/won_entry.py

@@ -0,0 +1,22 @@
+import re
+from .log import get_logger
+import time
+
+logger = get_logger(__name__)
+
+
+class WonEntry:
+
+    def __init__(self, soup_item):
+        self.game_name = None
+        self.giveaway_game_id = None
+        self.giveaway_uri = None
+
+        logger.debug(f"Won Giveaway html: {soup_item}")
+        self.game_name = soup_item.find('a', {'class': 'table__column__heading'}).text
+        self.giveaway_game_id = soup_item.find('a', {'class': 'table__column__heading'})['href'].split('/')[2]
+        self.giveaway_uri = soup_item.select_one('a.table__column__heading')['href']
+        logger.debug(f"Scraped Won Giveaway: {self}")
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)

src/log.py

@@ -1,20 +0,0 @@
-import logging
-from logging.handlers import RotatingFileHandler
-
-# log info level logs to stdout and debug to debug file
-
-log_format = "%(levelname)s %(asctime)s - %(message)s"
-logging.basicConfig(
-        handlers=[RotatingFileHandler('../config/debug.log', maxBytes=100000, backupCount=10)],
-        level=logging.DEBUG,
-        format=log_format)
-
-stream = logging.StreamHandler()
-stream.setLevel(logging.INFO)
-stream_format = logging.Formatter(log_format)
-stream.setFormatter(stream_format)
-
-def get_logger(name):
-    l = logging.getLogger(name)
-    l.addHandler(stream)
-    return l

src/main.py

@@ -1,249 +0,0 @@
-import json
-import re
-from random import randint
-from time import sleep
-
-import requests
-from bs4 import BeautifulSoup
-from requests.adapters import HTTPAdapter
-from urllib3.util import Retry
-
-import log
-
-logger = log.get_logger(__name__)
-
-
-class SteamGifts:
-    def __init__(self, cookie, gifts_type, pinned, min_points, max_entries, max_time_left, minimum_game_points):
-        self.cookie = {
-            'PHPSESSID': cookie
-        }
-        self.gifts_type = gifts_type
-        self.pinned = pinned
-        self.min_points = int(min_points)
-        self.max_entries = int(max_entries)
-        self.max_time_left = int(max_time_left)
-        self.minimum_game_points = int(minimum_game_points)
-
-        self.base = "https://www.steamgifts.com"
-        self.session = requests.Session()
-
-        self.filter_url = {
-            'All': "search?page=%d",
-            'Wishlist': "search?page=%d&type=wishlist",
-            'Recommended': "search?page=%d&type=recommended",
-            'Copies': "search?page=%d&copy_min=2",
-            'DLC': "search?page=%d&dlc=true",
-            'New': "search?page=%d&type=new"
-        }
-
-    def requests_retry_session(
-        self,
-        retries=5,
-        backoff_factor=0.3
-    ):
-        session = self.session or requests.Session()
-        retry = Retry(
-            total=retries,
-            read=retries,
-            connect=retries,
-            backoff_factor=backoff_factor,
-            status_forcelist=(500, 502, 504),
-        )
-        adapter = HTTPAdapter(max_retries=retry)
-        session.mount('http://', adapter)
-        session.mount('https://', adapter)
-        return session
-
-    def get_soup_from_page(self, url):
-        r = self.requests_retry_session().get(url)
-        r = requests.get(url, cookies=self.cookie)
-        soup = BeautifulSoup(r.text, 'html.parser')
-        return soup
-
-    def update_info(self):
-        soup = self.get_soup_from_page(self.base)
-
-        try:
-            self.xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value']
-            self.points = int(soup.find('span', {'class': 'nav__points'}).text)  # storage points
-        except TypeError:
-            logger.error("⛔  Cookie is not valid.")
-            sleep(10)
-            exit()
-
-    # this isn't exact because 'a week' could mean 8 days or 'a day' could mean 27 hours
-    def determine_time_in_minutes(self, string_time):
-        if not string_time:
-            logger.error(f"Could not determine time from string {string_time}")
-            return None
-        match = re.search('(?P<number>[0-9]+) (?P<time_unit>(hour|day|minute|second|week))', string_time)
-        if match:
-            number = int(match.group('number'))
-            time_unit = match.group('time_unit')
-            if time_unit == 'hour':
-                return number * 60
-            elif time_unit == 'day':
-                return number * 24 * 60
-            elif time_unit == 'minute':
-                return number
-            elif time_unit == 'second':
-                return 1
-            elif time_unit == 'week':
-                return number * 7 * 24 * 60
-            else:
-                logger.error(f"Unknown time unit displayed in giveaway: {string_time}")
-                return None
-        else:
-            return None
-
-    def determine_cost_and_copies(self, item, game_name, game_id):
-        item_headers = item.find_all('span', {'class': 'giveaway__heading__thin'})
-        if len(item_headers) == 1:  # then no multiple copies
-            game_cost = item_headers[0].getText().replace('(', '').replace(')', '').replace('P', '')
-            if not re.search('^[0-9]+$', game_cost):
-                txt = f"Unable to determine cost of {game_name} with id {game_id}. Cost string: {item_headers[0]}"
-                logger.error(txt)
-                return None, None
-            game_cost = int(game_cost)
-            return game_cost, 1
-        elif len(item_headers) == 2:  # then multiple copies
-            game_cost = item_headers[1].getText().replace('(', '').replace(')', '').replace('P', '')
-            if not re.search('^[0-9]+$', game_cost):
-                txt = f"Unable to determine cost of {game_name} with id {game_id}. Cost string: {item_headers[1].getText()}"
-                logger.error(txt)
-                return None, None
-            game_cost = int(game_cost)
-
-            match = re.search('(?P<copies>[0-9]+) Copies', item_headers[0].getText(), re.IGNORECASE)
-            if match:
-                num_copies_str = match.group('copies')
-                num_copies = int(num_copies_str)
-                return game_cost, num_copies
-            else:
-                txt = f"It appears there are multiple copies of {game_name} with id {game_id}, but we could not " \
-                      f"determine that. Copy string: {item_headers[0].getText()}"
-                logger.error(txt)
-                return game_cost, None
-        else:
-            txt = f"Unable to determine cost or num copies of {game_name} with id {game_id}."
-            logger.error(txt)
-            return None, None
-
-    def should_we_enter_giveaway(self, item, game_name, game_cost, copies):
-        times = item.select('div span[data-timestamp]')
-        game_remaining = times[0].text
-        game_remaining_in_minutes = self.determine_time_in_minutes(game_remaining)
-        if game_remaining_in_minutes is None:
-            return False
-        game_created = times[1].text
-        game_created_in_minutes = self.determine_time_in_minutes(game_created)
-        if game_created_in_minutes is None:
-            return False
-        game_entries = int(item.select('div.giveaway__links span')[0].text.split(' ')[0].replace(',', ''))
-
-        txt = f"{game_name} - {game_cost}P - {game_entries} entries (w/ {copies} copies) - " \
-              f"Created {game_created} ago with {game_remaining} remaining."
-        logger.debug(txt)
-
-        if self.points - int(game_cost) < 0:
-            txt = f"⛔ Not enough points to enter: {game_name}"
-            logger.debug(txt)
-            return False
-        if game_cost < self.minimum_game_points:
-            txt = f"Game {game_name} costs {game_cost}P and is below your cutoff of {self.minimum_game_points}P."
-            logger.debug(txt)
-            return False
-        if game_remaining_in_minutes > self.max_time_left:
-            txt = f"Game {game_name} has {game_remaining_in_minutes} minutes left and is above your cutoff of {self.max_time_left} minutes."
-            logger.debug(txt)
-            return False
-        if game_entries / copies > self.max_entries:
-            txt = f"Game {game_name} has {game_entries} entries and is above your cutoff of {self.max_entries} entries."
-            logger.debug(txt)
-            return False
-
-        return True
-
-    def enter_giveaway(self, game_id):
-        payload = {'xsrf_token': self.xsrf_token, 'do': 'entry_insert', 'code': game_id}
-        entry = requests.post('https://www.steamgifts.com/ajax.php', data=payload, cookies=self.cookie)
-        json_data = json.loads(entry.text)
-
-        if json_data['type'] == 'success':
-            return True
-
-    def get_game_content(self, page=1):
-        n = page
-        while True:
-            txt = "⚙️  Retrieving games from %d page." % n
-            logger.info(txt)
-
-            filtered_url = self.filter_url[self.gifts_type] % n
-            paginated_url = f"{self.base}/giveaways/{filtered_url}"
-
-            soup = self.get_soup_from_page(paginated_url)
-
-            # this matches on a div with the exact class value so we discard ones that also have a class 'is-faded' containing already entered giveaways
-            game_list = soup.select('div[class=giveaway__row-inner-wrap]')
-            # game_list = soup.find_all('div', {'class': 'giveaway__row-inner-wrap'})
-
-            if not len(game_list):
-                random_seconds = randint(900, 1400)
-                txt = f"We have run out of gifts to consider. Trying again in {random_seconds} seconds."
-                logger.info(txt)
-                sleep(random_seconds)
-                self.start()
-                continue
-
-            for item in game_list:
-                if len(item.get('lass', [])) == 2 and not self.pinned:
-                    continue
-
-                if self.points == 0 or self.points < self.min_points:
-                    random_seconds = randint(900, 1400)
-                    txt = f"🛋️  Sleeping {random_seconds} seconds to get more points. We have {self.points} points, but we need {self.min_points} to start."
-                    logger.info(txt)
-                    sleep(random_seconds)
-                    self.start()
-                    break
-
-                game_name = item.find('a', {'class': 'giveaway__heading__name'}).text
-                game_id = item.find('a', {'class': 'giveaway__heading__name'})['href'].split('/')[2]
-
-                game_cost, copies = self.determine_cost_and_copies(item, game_name, game_id)
-
-                if not game_cost:
-                    continue
-                if_enter_giveaway = self.should_we_enter_giveaway(item, game_name, game_cost, copies)
-
-                if if_enter_giveaway:
-                    res = self.enter_giveaway(game_id)
-                    if res:
-                        self.points -= int(game_cost)
-                        txt = f"🎉 One more game! Has just entered {game_name}"
-                        logger.info(txt)
-                        sleep(randint(4, 15))
-                else:
-                    continue
-
-            n = n+1
-
-        logger.info("🛋️  List of games is ended. Waiting 2 mins to update...")
-        sleep(120)
-        self.start()
-
-    def start(self):
-        self.update_info()
-
-        if self.points >= self.min_points:
-            txt = "🤖 You have %d points. Evaluating giveaways..." % self.points
-            logger.info(txt)
-            self.get_game_content()
-        else:
-            random_seconds = randint(900, 1400)
-            txt = f"You have {self.points} points which is below your minimum point threshold of {self.min_points} points. Sleeping for {random_seconds} seconds."
-            logger.info(txt)
-            sleep(random_seconds)
-            self.get_game_content()
-

src/run.py

@@ -1,85 +1,81 @@
-import configparser
-from configparser import ConfigParser
+import os
+from time import sleep
 
-import log
+from src.bot.log import get_logger
+from src.bot.config_reader import ConfigReader, ConfigException
+from src.bot.enter_giveaways import SteamGiftsException
+from src.bot.giveaway_thread import GiveawayThread
+from src.bot.notification import Notification
+from src.bot.database import run_db_migrations
+from src.web.webserver_thread import WebServerThread
 
-logger = log.get_logger(__name__)
-config = configparser.ConfigParser()
-
-
-class MyException(Exception):
-    pass
-
-
-def value_range(min, max):
-    return [str(x) for x in [*range(min, max + 1)]]
-
-
-class MyConfig(ConfigParser):
-    def __init__(self, config_file):
-        super(MyConfig, self).__init__()
-
-        self.read(config_file)
-        self.validate_config()
-
-    def validate_config(self):
-        required_values = {
-            'DEFAULT': {
-                'gift_types': ('All', 'Wishlist', 'Recommended', 'Copies', 'DLC', 'New'),
-                'pinned': ('true', 'false'),
-                'minimum_points': '%s' % (value_range(0,400)),
-                'max_entries': '%s' % (value_range(0,10000)),
-                'max_time_left': '%s' % (value_range(0,21600)),
-                'minimum_game_points': '%s' % (value_range(0,50))
-            }
-        }
-
-        for section, keys in required_values.items():
-            if section not in self:
-                raise MyException(
-                    'Missing section %s in the config file' % section)
-
-            for key, values in keys.items():
-                if key not in self[section] or self[section][key] == '':
-                    raise MyException((
-                                              'Missing value for %s under section %s in ' +
-                                              'the config file') % (key, section))
-
-                if values:
-                    if self[section][key] not in values:
-                        raise MyException((
-                                                  'Invalid value for %s under section %s in ' +
-                                                  'the config file') % (key, section))
+logger = get_logger(__name__)
+config_file_name = f"{os.getenv('BOT_CONFIG_DIR', './config')}/config.ini"
+db_url = f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}"
+alembic_migration_files = os.getenv('BOT_ALEMBIC_CONFIG_DIR', './src/alembic')
 
 
 def run():
-    from main import SteamGifts as SG
+    logger.info("Starting Steamgifts bot.")
 
-    file_name = '../config/config.ini'
+    config = None
     try:
-        with open(file_name) as f:
-            config.read_file(f)
-            MyConfig(file_name)
+        config = ConfigReader(config_file_name)
     except IOError:
-        txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out."
+        txt = f"{config_file_name} doesn't exist. Rename {config_file_name}.example to {config_file_name} and fill out."
         logger.warning(txt)
         exit(-1)
-    except MyException as e:
+    except ConfigException as e:
         logger.error(e)
         exit(-1)
 
-    config.read(file_name)
-    cookie = config['DEFAULT'].get('cookie')
-    pinned_games = config['DEFAULT'].getboolean('pinned')
-    gift_types = config['DEFAULT'].get('gift_types')
-    minimum_points = config['DEFAULT'].getint('minimum_points')
-    max_entries = config['DEFAULT'].getint('max_entries')
-    max_time_left = config['DEFAULT'].getint('max_time_left')
-    minimum_game_points = config['DEFAULT'].getint('minimum_game_points')
+    config.read(config_file_name)
 
-    s = SG(cookie, gift_types, pinned_games, minimum_points, max_entries, max_time_left, minimum_game_points)
-    s.start()
+    notification = Notification(config['NOTIFICATIONS'].get('notification.prefix'))
+    pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled')
+    pushover_token = config['NOTIFICATIONS'].get('pushover.token')
+    pushover_user_key = config['NOTIFICATIONS'].get('pushover.user_key')
+    if pushover_enabled:
+        notification.enable_pushover(pushover_token, pushover_user_key)
+    try:
+        g = GiveawayThread(config, notification)
+        g.setName("Giveaway Enterer")
+        g.start()
+
+        w = WebServerThread(config)
+        w.setName("WebServer")
+        # if the giveaway thread dies then this daemon thread will die by design
+        w.setDaemon(True)
+        w.start()
+
+        g.join()
+    except SteamGiftsException as e:
+        notification.send_error(e)
+        sleep(5)
+        exit(-1)
+    except Exception as e:
+        logger.error(e)
+        notification.send_error("Something happened and the bot had to quit!")
+        sleep(10)
+        exit(-1)
+
+
+def entry():
+    logger.info("""
+        -------------------------------------------------------------------------------------
+           _____  _                                  _   __  _          ____          _   
+          / ____|| |                                (_) / _|| |        |  _ \        | |  
+         | (___  | |_  ___   __ _  _ __ ___    __ _  _ | |_ | |_  ___  | |_) |  ___  | |_ 
+          \___ \ | __|/ _ \ / _` || '_ ` _ \  / _` || ||  _|| __|/ __| |  _ <  / _ \ | __|
+          ____) || |_|  __/| (_| || | | | | || (_| || || |  | |_ \__ \ | |_) || (_) || |_ 
+         |_____/  \__|\___| \__,_||_| |_| |_| \__, ||_||_|   \__||___/ |____/  \___/  \__|
+                                               __/ |                                      
+                                              |___/                                       
+        -------------------------------------------------------------------------------------
+        """)
+    run_db_migrations(alembic_migration_files, db_url)
+    run()
 
 
 if __name__ == '__main__':
-    run()
+    entry()

src/web/static/css/main.css

@@ -0,0 +1,106 @@
+body {
+  margin: 0;
+  padding: 0;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  color: #444;
+}
+/*
+ * Formatting the header area
+ */
+header {
+  background-color: #DFB887;
+  height: 35px;
+  width: 100%;
+  opacity: .9;
+  margin-bottom: 10px;
+}
+header h1.logo {
+  margin: 0;
+  font-size: 1.7em;
+  color: #fff;
+  text-transform: uppercase;
+  float: left;
+}
+header h1.logo:hover {
+  color: #fff;
+  text-decoration: none;
+}
+/*
+ * Centering the body content
+ */
+.container {
+  width: 1200px;
+  margin: 0 auto;
+}
+div.home {
+  padding: 10px 0 30px 0;
+  background-color: #E6E6FA;
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+}
+div.about {
+  padding: 10px 0 30px 0;
+  background-color: #E6E6FA;
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+}
+h2 {
+  font-size: 3em;
+  margin-top: 40px;
+  text-align: center;
+  letter-spacing: -2px;
+}
+h3 {
+  font-size: 1.7em;
+  font-weight: 100;
+  margin-top: 30px;
+  text-align: center;
+  letter-spacing: -1px;
+  color: #999;
+}
+.menu {
+  float: right;
+  margin-top: 8px;
+}
+.menu li {
+  display: inline;
+}
+.menu li + li {
+  margin-left: 35px;
+}
+.menu li a {
+  color: #444;
+  text-decoration: none;
+}
+.styled-table {
+    border-collapse: collapse;
+    margin: 25px 0;
+    font-size: 0.9em;
+    font-family: sans-serif;
+    min-width: 400px;
+    box-shadow: 0 0 20px rgba(0, 0, 0, 0.15);
+}
+.styled-table thead tr {
+    background-color: #009879;
+    color: #ffffff;
+    text-align: left;
+}
+.styled-table th,
+.styled-table td {
+    padding: 12px 15px;
+}
+.styled-table tbody tr {
+    border-bottom: 1px solid #dddddd;
+}
+.styled-table tbody tr:nth-of-type(even) {
+    background-color: #f3f3f3;
+}
+.styled-table tbody tr:last-of-type {
+    border-bottom: 2px solid #009879;
+}
+.styled-table tbody tr.active-row {
+    font-weight: bold;
+    color: #009879;
+}

src/web/templates/base.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>{% block title %} {% endblock %}</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}">
+  </head>
+  <body>
+    <header>
+      <div class="container">
+        <h1 class="logo">Steamgifts Bot</h1>
+        <strong><nav>
+          <ul class="menu">
+            <li><a href="{{ url_for('config') }}">Config</a></li>
+            <li><a href="{{ url_for('stats') }}">Stats</a></li>
+            <li><a href="{{ url_for('log_info') }}">Info Logs</a></li>
+            <li><a href="{{ url_for('log_debug') }}">Debug Logs</a></li>
+            <li><a href="{{ url_for('db_giveaways') }}">Giveaways</a></li>
+            <li><a href="{{ url_for('db_notifications') }}">Notifications</a></li>
+          </ul>
+        </nav></strong>
+      </div>
+    </header>
+    <div class="container">
+      {% block content %} {% endblock %}
+    </div>
+  </body>
+</html>

src/web/templates/configuration.html

@@ -0,0 +1,6 @@
+{% extends 'base.html' %}
+
+{% block title %} {{name}} Steamgifts Bot Configuration {% endblock %}
+{% block content %}
+    <pre>{{data}}</pre>
+{% endblock %}

src/web/templates/giveaways.html

@@ -0,0 +1,53 @@
+{% extends 'base.html' %}
+
+{% block title %} {{name}} Steamgifts Bot Giveaways {% endblock %}
+{% block content %}
+    <ul class="pagination">
+                    {% if data.previous_page %}
+                      <li class="page-item"> <a class="page-link" href="{{ url_for('db_giveaways', page=data.previous_page) }}">Previous</a></li>
+                    {% else %}
+                      <li class="page-item">Previous</li>
+                    {% endif %}
+                    {% if data.next_page %}
+                      <li class="page-item"> <a class="page-link" href="{{ url_for('db_giveaways', page=data.next_page) }}">Next</a></li>
+                    {% else %}
+                      <li class="page-item">Next</li>
+                    {% endif %}
+                </ul>
+    <table class="styled-table">
+        <thead>
+        <tr>
+            <th>Giveaway ID</th>
+            <th>Steam ID</th>
+            <th>Giveaway URI</th>
+            <th>Name</th>
+            <th>Steam URL</th>
+            <th>User</th>
+            <th>Giveaway Created At</th>
+            <th>Giveaway Ended At</th>
+            <th>Cost</th>
+            <th>Copies</th>
+            <th>Contributor Level</th>
+            <th>Entered</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for row in data.items %}
+        <tr>
+            <td>{{row.giveaway_id}}</td>
+            <td>{{row.steam_id}}</td>
+            <td>{{row.giveaway_uri}}</td>
+            <td>{{row.steam_item.game_name}}</td>
+            <td>{{row.steam_item.steam_url}}</td>
+            <td>{{row.user}}</td>
+            <td>{{row.giveaway_created_at}}</td>
+            <td>{{row.giveaway_ended_at}}</td>
+            <td>{{row.cost}}</td>
+            <td>{{row.copies}}</td>
+            <td>{{row.contributor_level}}</td>
+            <td>{{row.entered}}</td>
+        </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+{% endblock %}

src/web/templates/log.html

@@ -0,0 +1,6 @@
+{% extends 'base.html' %}
+
+{% block title %} {{name}} Steamgifts Bot {{log_type}} Logs{% endblock %}
+{% block content %}
+    <pre id="output">{{data}}</pre>
+{% endblock %}

src/web/templates/notifications.html

@@ -0,0 +1,29 @@
+{% extends 'base.html' %}
+
+{% block title %} {{name}} Steamgifts Bot Notifications {% endblock %}
+{% block content %}
+<table>
+        <thead>
+        <tr>
+            <th>ID</th>
+            <th>Medium</th>
+            <th>Type</th>
+            <th>Success</th>
+            <th>Created At</th>
+            <th>Message</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for row in data %}
+        <tr>
+            <td>{{row.id}}</td>
+            <td>{{row.medium}}</td>
+            <td>{{row.type}}</td>
+            <td>{{row.success}}</td>
+            <td>{{row.created_at}}</td>
+            <td>{{row.message}}</td>
+        </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+{% endblock %}

src/web/templates/stats.html

@@ -0,0 +1,10 @@
+{% extends 'base.html' %}
+
+{% block title %} {{name}} Steamgifts Bot Configuration {% endblock %}
+{% block content %}
+    <ul>
+        <li>Total Giveaways Considered: {{totals}}</li>
+        <li>Giveaways Entered: {{entered}}</li>
+        <li>Giveaways Won (with the bot): {{won}}</li>
+    </ul>
+{% endblock %}

src/web/webserver_thread.py

@@ -0,0 +1,103 @@
+import os
+import threading
+from threading import Thread
+
+from flask_basicauth import BasicAuth
+
+from src.bot.database import NotificationHelper, GiveawayHelper
+from src.bot.log import get_logger
+
+logger = get_logger(__name__)
+
+
+class WebServerThread(threading.Thread):
+
+    def __init__(self, config):
+        Thread.__init__(self)
+        self.exc = None
+        self.config = config
+        self.prefix = config['NOTIFICATIONS'].get('notification.prefix')
+        self.host = config['WEB'].get('web.host')
+        self.port = config['WEB'].getint('web.port')
+        self.ssl = config['WEB'].getboolean('web.ssl')
+        self.enabled = config['WEB'].getboolean('web.enabled')
+        self.app_root = config['WEB'].get('web.app_root')
+        self.basic_auth = config['WEB'].getboolean('web.basic_auth')
+        self.basic_auth_username = config['WEB'].get('web.basic_auth.username')
+        self.basic_auth_password = config['WEB'].get('web.basic_auth.password')
+
+    def run_webserver(self):
+        from flask import Flask
+        from flask import render_template
+
+        app = Flask(__name__)
+
+        if self.basic_auth:
+            app.config['BASIC_AUTH_USERNAME'] = self.basic_auth_username
+            app.config['BASIC_AUTH_PASSWORD'] = self.basic_auth_password
+
+        app.config['BASIC_AUTH_FORCE'] = self.basic_auth
+        basic_auth = BasicAuth(app)
+
+        @app.route(f"{self.app_root}")
+        def config():
+            with open(f"{os.getenv('BOT_CONFIG_DIR', './config')}/config.ini", 'r') as f:
+                data = f.read()
+            return render_template('configuration.html', name=self.prefix, data=data)
+
+        @app.route(f"{self.app_root}log_info")
+        def log_info():
+            with open(f"{os.getenv('BOT_CONFIG_DIR', './config')}/info.log", 'r') as f:
+                data = f.read()
+                return render_template('log.html', name=self.prefix, log_type='info', data=data)
+
+        @app.route(f"{self.app_root}log_debug")
+        def log_debug():
+            with open(f"{os.getenv('BOT_CONFIG_DIR', './config')}/debug.log", 'r') as f:
+                data = f.read()
+                return render_template('log.html', name=self.prefix, log_type='debug', data=data)
+
+        @app.route(f"{self.app_root}alive")
+        def alive():
+            return 'OK'
+
+        @app.route(f"{self.app_root}notifications")
+        def db_notifications():
+            return render_template('notifications.html', name=self.prefix, data=NotificationHelper.get())
+
+        @app.route(f"{self.app_root}giveaways", methods=['GET'], defaults={"page": 1})
+        @app.route(f"{self.app_root}giveaways/<int:page>", methods=['GET'])
+        def db_giveaways(page):
+            return render_template('giveaways.html', name=self.prefix, data=GiveawayHelper.paginate(page=page))
+
+        @app.route(f"{self.app_root}stats")
+        def stats():
+            totals = GiveawayHelper.total_giveaways()
+            entered = GiveawayHelper.total_entered()
+            won = GiveawayHelper.total_won()
+            return render_template('stats.html', name=self.prefix, totals=totals, entered=entered, won=won)
+
+        if self.enabled:
+            logger.info("Webserver Enabled. Running")
+            if self.ssl:
+                app.run(port=self.port, host=self.host, ssl_context='adhoc')
+            else:
+                app.run(port=self.port, host=self.host)
+        else:
+            logger.info("Webserver NOT Enabled.")
+
+    def run(self):
+        # Variable that stores the exception, if raised by someFunction
+        self.exc = None
+        try:
+            self.run_webserver()
+        except BaseException as e:
+            self.exc = e
+
+    def join(self):
+        threading.Thread.join(self)
+        # Since join() returns in caller thread
+        # we re-raise the caught exception
+        # if any was caught
+        if self.exc:
+            raise self.exc