From ceb55bec68e4d5a24159f386f19e22679105dd84 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 11:43:59 -0400
Subject: [PATCH 01/37] assigne user agent from list
---
src/ConfigReader.py | 23 +++++++++++++++++++----
src/SteamGifts.py | 7 ++++---
src/run.py | 6 +++---
3 files changed, 26 insertions(+), 10 deletions(-)
diff --git a/src/ConfigReader.py b/src/ConfigReader.py
index 1f64ed4..2d4714c 100644
--- a/src/ConfigReader.py
+++ b/src/ConfigReader.py
@@ -1,5 +1,5 @@
from configparser import ConfigParser
-from random import randint
+from random import randint, randrange
import log
@@ -10,9 +10,23 @@ class ConfigException(Exception):
pass
+def value_range(min, max):
+ return [str(x) for x in [*range(min, max + 1)]]
+
+
+def choose_user_agent():
+ user_agents = [
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36',
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36',
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15',
+ 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36'
+ ]
+ return user_agents[randrange(0, len(user_agents))]
+
class ConfigReader(ConfigParser):
- def value_range(min, max):
- return [str(x) for x in [*range(min, max + 1)]]
+
required_values = {
'DEFAULT': {
@@ -35,6 +49,7 @@ class ConfigReader(ConfigParser):
default_values = {
'DEFAULT': {
'cookie': '',
+ 'user_agent': f"{choose_user_agent()}",
'enabled': 'true',
'minimum_points': f"{randint(20, 50)}",
'max_entries': f"{randint(1000, 2500)}",
@@ -107,4 +122,4 @@ class ConfigReader(ConfigParser):
if self[section][key] not in values:
raise ConfigException((
'Invalid value for "%s" under section "%s" in ' +
- 'the config file') % (key, section))
\ No newline at end of file
+ 'the config file') % (key, section))
diff --git a/src/SteamGifts.py b/src/SteamGifts.py
index 659af0f..08c01e8 100644
--- a/src/SteamGifts.py
+++ b/src/SteamGifts.py
@@ -19,7 +19,7 @@ class SteamGiftsException(Exception):
class SteamGifts:
- def __init__(self, cookie, gifts_type, pinned, min_points, max_entries,
+ def __init__(self, cookie, user_agent, gifts_type, pinned, min_points, max_entries,
max_time_left, minimum_game_points, blacklist, notification):
self.contributor_level = None
self.xsrf_token = None
@@ -27,6 +27,7 @@ class SteamGifts:
self.cookie = {
'PHPSESSID': cookie
}
+ self.user_agent = user_agent
self.gifts_type = gifts_type
self.pinned = pinned
self.min_points = int(min_points)
@@ -68,7 +69,7 @@ class SteamGifts:
def get_soup_from_page(self, url):
headers = {
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36',
+ 'User-Agent': self.user_agent
}
self.requests_retry_session().get(url, headers=headers)
r = requests.get(url, cookies=self.cookie)
@@ -144,7 +145,7 @@ class SteamGifts:
def enter_giveaway(self, giveaway):
headers = {
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36',
+ 'User-Agent': self.user_agent
}
payload = {'xsrf_token': self.xsrf_token, 'do': 'entry_insert', 'code': giveaway.giveaway_game_id}
entry = requests.post('https://www.steamgifts.com/ajax.php', data=payload, cookies=self.cookie,
diff --git a/src/run.py b/src/run.py
index 3276ed4..c861b3e 100644
--- a/src/run.py
+++ b/src/run.py
@@ -34,7 +34,7 @@ def run():
try:
cookie = config['DEFAULT'].get('cookie')
-
+ user_agent = config['DEFAULT'].get('user_agent')
main_page_enabled = config['DEFAULT'].getboolean('enabled')
minimum_points = config['DEFAULT'].getint('minimum_points')
max_entries = config['DEFAULT'].getint('max_entries')
@@ -42,7 +42,7 @@ def run():
minimum_game_points = config['DEFAULT'].getint('minimum_game_points')
blacklist = config['DEFAULT'].get('blacklist_keywords')
- all_page = SteamGifts(cookie, 'All', False, minimum_points, max_entries,
+ all_page = SteamGifts(cookie, user_agent, 'All', False, minimum_points, max_entries,
max_time_left, minimum_game_points, blacklist, notification)
wishlist_page_enabled = config['WISHLIST'].getboolean('wishlist.enabled')
@@ -50,7 +50,7 @@ def run():
wishlist_max_entries = config['WISHLIST'].getint('wishlist.max_entries')
wishlist_max_time_left = config['WISHLIST'].getint('wishlist.max_time_left')
- wishlist_page = SteamGifts(cookie, 'Wishlist', False, wishlist_minimum_points,
+ wishlist_page = SteamGifts(cookie, user_agent, 'Wishlist', False, wishlist_minimum_points,
wishlist_max_entries, wishlist_max_time_left, 0, '', notification)
if not main_page_enabled and not wishlist_page_enabled:
From d250ed48b5aeb6a0ddbc0a9cdcad801938eb3652 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 12:34:56 -0400
Subject: [PATCH 02/37] log just info to info.log
---
.dockerignore | 3 ++-
.gitignore | 2 +-
src/log.py | 36 ++++++++++++++++++++++++++++--------
src/run.py | 1 +
4 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index d74cc4b..d17bf38 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +1,4 @@
config/config.ini
config/debug.log
-config/sqlite.db
\ No newline at end of file
+config/sqlite.db
+config/info.log
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 1599e04..42e2346 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,5 +4,5 @@ venv/
__pycache__/
.idea
config/config.ini
-config/debug.log
+config/*.log*
config/sqlite.db
\ No newline at end of file
diff --git a/src/log.py b/src/log.py
index ff345ae..dedfe66 100644
--- a/src/log.py
+++ b/src/log.py
@@ -5,17 +5,37 @@ from logging.handlers import RotatingFileHandler
log_format = "%(levelname)s %(asctime)s - %(message)s"
logging.basicConfig(
- handlers=[RotatingFileHandler('../config/debug.log', maxBytes=500000, backupCount=10)],
- level=logging.DEBUG,
- format=log_format)
+ handlers=[RotatingFileHandler('../config/debug.log', maxBytes=500000, backupCount=10)],
+ level=logging.DEBUG,
+ format=log_format)
-stream = logging.StreamHandler()
-stream.setLevel(logging.INFO)
-stream_format = logging.Formatter(log_format)
-stream.setFormatter(stream_format)
+console_output = logging.StreamHandler()
+console_output.setLevel(logging.INFO)
+console_format = logging.Formatter(log_format)
+console_output.setFormatter(console_format)
+
+info_log_file = RotatingFileHandler('../config/info.log', maxBytes=500000, backupCount=10)
+info_log_file.setLevel(logging.INFO)
+info_log_format = logging.Formatter(log_format)
+info_log_file.setFormatter(info_log_format)
+
+logging.root.addHandler(console_output)
+logging.root.addHandler(info_log_file)
+
+logging.info("""
+-------------------------------------------------------------------------------------
+ _____ _ _ __ _ ____ _
+ / ____|| | (_) / _|| | | _ \ | |
+ | (___ | |_ ___ __ _ _ __ ___ __ _ _ | |_ | |_ ___ | |_) | ___ | |_
+ \___ \ | __|/ _ \ / _` || '_ ` _ \ / _` || || _|| __|/ __| | _ < / _ \ | __|
+ ____) || |_| __/| (_| || | | | | || (_| || || | | |_ \__ \ | |_) || (_) || |_
+ |_____/ \__|\___| \__,_||_| |_| |_| \__, ||_||_| \__||___/ |____/ \___/ \__|
+ __/ |
+ |___/
+-------------------------------------------------------------------------------------
+""")
def get_logger(name):
l = logging.getLogger(name)
- l.addHandler(stream)
return l
diff --git a/src/run.py b/src/run.py
index c861b3e..29579f1 100644
--- a/src/run.py
+++ b/src/run.py
@@ -11,6 +11,7 @@ logger = log.get_logger(__name__)
def run():
+ logger.info("Starting Steamgifts bot.")
file_name = '../config/config.ini'
config = None
try:
From 4f4709776b1dfc07b95235671080aeac8f1147f2 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 13:29:40 -0400
Subject: [PATCH 03/37] threading
- added a simple webserver thread that runs alongside the steamgift enterer
---
src/index.html | 14 ++++++
src/run.py | 116 ++++++++++++++++++++++++++++++++++++++++---------
2 files changed, 109 insertions(+), 21 deletions(-)
create mode 100644 src/index.html
diff --git a/src/index.html b/src/index.html
new file mode 100644
index 0000000..c6b9cdb
--- /dev/null
+++ b/src/index.html
@@ -0,0 +1,14 @@
+
+
+
+
+
+ Hello!
+
+
+
+
Hello World!
+
This is a simple paragraph.
+
+
+
\ No newline at end of file
diff --git a/src/run.py b/src/run.py
index 29579f1..5acf7de 100644
--- a/src/run.py
+++ b/src/run.py
@@ -1,3 +1,4 @@
+import threading
from random import randint
from time import sleep
@@ -5,35 +6,57 @@ import log
from ConfigReader import ConfigReader, ConfigException
from SteamGifts import SteamGifts, SteamGiftsException
from notification import Notification
+from threading import Thread
logger = log.get_logger(__name__)
-def run():
- logger.info("Starting Steamgifts bot.")
- file_name = '../config/config.ini'
- config = None
- try:
- config = ConfigReader(file_name)
- except IOError:
- txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out."
- logger.warning(txt)
- exit(-1)
- except ConfigException as e:
- logger.error(e)
- exit(-1)
+class WebServerThread(threading.Thread):
- config.read(file_name)
+ def run_webserver(self):
+ import http.server
+ import socketserver
- notification = Notification(config['NOTIFICATIONS'].get('notification.prefix'))
- pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled')
- pushover_token = config['NOTIFICATIONS'].get('pushover.token')
- pushover_user_key = config['NOTIFICATIONS'].get('pushover.user_key')
- if pushover_enabled:
- notification.enable_pushover(pushover_token, pushover_user_key)
+ PORT = 8000
- try:
+ class MyHttpRequestHandler(http.server.SimpleHTTPRequestHandler):
+ def do_GET(self):
+ self.path = 'index.html'
+ return http.server.SimpleHTTPRequestHandler.do_GET(self)
+
+ Handler = MyHttpRequestHandler
+
+ with socketserver.TCPServer(("", PORT), Handler) as httpd:
+ print("Http Server Serving at port", PORT)
+ httpd.serve_forever()
+
+ def run(self):
+ # Variable that stores the exception, if raised by someFunction
+ self.exc = None
+ try:
+ self.run_webserver()
+ except BaseException as e:
+ self.exc = e
+
+ def join(self):
+ threading.Thread.join(self)
+ # Since join() returns in caller thread
+ # we re-raise the caught exception
+ # if any was caught
+ if self.exc:
+ raise self.exc
+
+
+class GiveawayEntererThread(threading.Thread):
+
+ def __init__(self, config, notification):
+ Thread.__init__(self)
+ self.exc = None
+ self.config = config
+ self.notification = notification
+
+ def run_steam_gifts(self, config, notification):
cookie = config['DEFAULT'].get('cookie')
user_agent = config['DEFAULT'].get('user_agent')
main_page_enabled = config['DEFAULT'].getboolean('enabled')
@@ -68,6 +91,57 @@ def run():
random_seconds = randint(1740, 3540) # sometime between 29-59 minutes
logger.info(f"Going to sleep for {random_seconds / 60} minutes.")
sleep(random_seconds)
+
+ def run(self):
+ # Variable that stores the exception, if raised by someFunction
+ self.exc = None
+ try:
+ self.run_steam_gifts(self.config, self.notification)
+ except BaseException as e:
+ self.exc = e
+
+ def join(self):
+ threading.Thread.join(self)
+ # Since join() returns in caller thread
+ # we re-raise the caught exception
+ # if any was caught
+ if self.exc:
+ raise self.exc
+
+
+def run():
+ logger.info("Starting Steamgifts bot.")
+ file_name = '../config/config.ini'
+ config = None
+ try:
+ config = ConfigReader(file_name)
+ except IOError:
+ txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out."
+ logger.warning(txt)
+ exit(-1)
+ except ConfigException as e:
+ logger.error(e)
+ exit(-1)
+
+ config.read(file_name)
+
+ notification = Notification(config['NOTIFICATIONS'].get('notification.prefix'))
+ pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled')
+ pushover_token = config['NOTIFICATIONS'].get('pushover.token')
+ pushover_user_key = config['NOTIFICATIONS'].get('pushover.user_key')
+ if pushover_enabled:
+ notification.enable_pushover(pushover_token, pushover_user_key)
+ try:
+ g = GiveawayEntererThread(config, notification)
+ g.setName("Giveaway Enterer")
+ g.start()
+
+ w = WebServerThread()
+ w.setName("WebServer")
+ w.setDaemon(True)
+ w.start()
+
+ g.join()
except SteamGiftsException as e:
notification.send_error(e)
sleep(5)
From f60220f3029c98abe52270d13a8e070a7976b5a6 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 16:55:52 -0400
Subject: [PATCH 04/37] web
- show config and logs in webserver
---
requirements.txt | 3 +-
src/ConfigReader.py | 11 ++++-
src/run.py | 36 ++++++++++-----
src/static/css/main.css | 76 ++++++++++++++++++++++++++++++++
src/templates/configuration.html | 25 +++++++++++
src/templates/log.html | 35 +++++++++++++++
6 files changed, 172 insertions(+), 14 deletions(-)
create mode 100644 src/static/css/main.css
create mode 100644 src/templates/configuration.html
create mode 100644 src/templates/log.html
diff --git a/requirements.txt b/requirements.txt
index 2bb5028..30e751c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,5 @@ beautifulsoup4==4.11.1
urllib3==1.26.9
sqlalchemy==1.4.36
sqlalchemy_utils==0.38.2
-python-dateutil==2.8.2
\ No newline at end of file
+python-dateutil==2.8.2
+Flask==2.1.2
\ No newline at end of file
diff --git a/src/ConfigReader.py b/src/ConfigReader.py
index 2d4714c..66e49ad 100644
--- a/src/ConfigReader.py
+++ b/src/ConfigReader.py
@@ -25,8 +25,8 @@ def choose_user_agent():
]
return user_agents[randrange(0, len(user_agents))]
-class ConfigReader(ConfigParser):
+class ConfigReader(ConfigParser):
required_values = {
'DEFAULT': {
@@ -44,6 +44,10 @@ class ConfigReader(ConfigParser):
},
'NOTIFICATIONS': {
'pushover.enabled': ('true', 'false'),
+ },
+ 'WEB': {
+ 'web.enabled': ('true', 'false'),
+ 'web.port': '%s' % (value_range(1, 65535))
}
}
default_values = {
@@ -68,7 +72,10 @@ class ConfigReader(ConfigParser):
'pushover.enabled': 'false',
'pushover.token': '',
'pushover.user_key': '',
-
+ },
+ 'WEB': {
+ 'web.enabled': 'false',
+ 'web.port': '9647'
}
}
deprecated_values = {
diff --git a/src/run.py b/src/run.py
index 5acf7de..8cfe941 100644
--- a/src/run.py
+++ b/src/run.py
@@ -2,6 +2,8 @@ import threading
from random import randint
from time import sleep
+from pygtail import Pygtail
+
import log
from ConfigReader import ConfigReader, ConfigException
from SteamGifts import SteamGifts, SteamGiftsException
@@ -15,21 +17,32 @@ logger = log.get_logger(__name__)
class WebServerThread(threading.Thread):
def run_webserver(self):
- import http.server
- import socketserver
+ from flask import Flask
+ from flask import render_template
- PORT = 8000
+ app = Flask(__name__)
- class MyHttpRequestHandler(http.server.SimpleHTTPRequestHandler):
- def do_GET(self):
- self.path = 'index.html'
- return http.server.SimpleHTTPRequestHandler.do_GET(self)
+ @app.route("/")
+ def config():
+ with open('../config/config.ini', 'r') as f:
+ content = f.read()
+ return render_template('configuration.html', config=content)
- Handler = MyHttpRequestHandler
+ @app.route("/log")
+ def logs():
+ return render_template('log.html')
- with socketserver.TCPServer(("", PORT), Handler) as httpd:
- print("Http Server Serving at port", PORT)
- httpd.serve_forever()
+ @app.route("/stream")
+ def stream():
+ def generate():
+ with open('../config/debug.log') as f:
+ while True:
+ yield f.read()
+ sleep(10)
+
+ return app.response_class(generate(), mimetype='text/plain')
+
+ app.run(port=9647)
def run(self):
# Variable that stores the exception, if raised by someFunction
@@ -138,6 +151,7 @@ def run():
w = WebServerThread()
w.setName("WebServer")
+ # if the giveaway thread dies then this daemon thread will die by definition
w.setDaemon(True)
w.start()
diff --git a/src/static/css/main.css b/src/static/css/main.css
new file mode 100644
index 0000000..9613db1
--- /dev/null
+++ b/src/static/css/main.css
@@ -0,0 +1,76 @@
+body {
+ margin: 0;
+ padding: 0;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ color: #444;
+}
+/*
+ * Formatting the header area
+ */
+header {
+ background-color: #DFB887;
+ height: 35px;
+ width: 100%;
+ opacity: .9;
+ margin-bottom: 10px;
+}
+header h1.logo {
+ margin: 0;
+ font-size: 1.7em;
+ color: #fff;
+ text-transform: uppercase;
+ float: left;
+}
+header h1.logo:hover {
+ color: #fff;
+ text-decoration: none;
+}
+/*
+ * Centering the body content
+ */
+.container {
+ width: 1200px;
+ margin: 0 auto;
+}
+div.home {
+ padding: 10px 0 30px 0;
+ background-color: #E6E6FA;
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+}
+div.about {
+ padding: 10px 0 30px 0;
+ background-color: #E6E6FA;
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+}
+h2 {
+ font-size: 3em;
+ margin-top: 40px;
+ text-align: center;
+ letter-spacing: -2px;
+}
+h3 {
+ font-size: 1.7em;
+ font-weight: 100;
+ margin-top: 30px;
+ text-align: center;
+ letter-spacing: -1px;
+ color: #999;
+}
+.menu {
+ float: right;
+ margin-top: 8px;
+}
+.menu li {
+ display: inline;
+}
+.menu li + li {
+ margin-left: 35px;
+}
+.menu li a {
+ color: #444;
+ text-decoration: none;
+}
\ No newline at end of file
diff --git a/src/templates/configuration.html b/src/templates/configuration.html
new file mode 100644
index 0000000..05b6f59
--- /dev/null
+++ b/src/templates/configuration.html
@@ -0,0 +1,25 @@
+
+
+
+ Flask app
+
+
+
+
+
+
Steamgifts Bot
+
+
+
+
+
{{config}}
+ {% block content %}
+ {% endblock %}
+
+
+
\ No newline at end of file
diff --git a/src/templates/log.html b/src/templates/log.html
new file mode 100644
index 0000000..e365d92
--- /dev/null
+++ b/src/templates/log.html
@@ -0,0 +1,35 @@
+
+
+
+ Flask app
+
+
+
+
+
+
Steamgifts Bot
+
+
+
+
+
+
+
+
+
\ No newline at end of file
From 4478b14e20b21904a8ef749e44da06a88b1b516c Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 16:58:47 -0400
Subject: [PATCH 05/37] removed
---
src/run.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/run.py b/src/run.py
index 8cfe941..516351a 100644
--- a/src/run.py
+++ b/src/run.py
@@ -2,8 +2,6 @@ import threading
from random import randint
from time import sleep
-from pygtail import Pygtail
-
import log
from ConfigReader import ConfigReader, ConfigException
from SteamGifts import SteamGifts, SteamGiftsException
From f3ddfc8297eac4d049cb1f5a0f0521f60ac2a627 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 17:06:11 -0400
Subject: [PATCH 06/37] running in docker
---
Dockerfile | 2 +-
src/run.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 0ad015e..42b358f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@ RUN python -m venv $VIRTUAL_ENV
COPY requirements.txt .
RUN pip3 install -r requirements.txt
-COPY ./src/* /app/
+COPY ./src/ /app/
VOLUME /config
CMD ["python3", "run.py"]
diff --git a/src/run.py b/src/run.py
index 516351a..fa1ec81 100644
--- a/src/run.py
+++ b/src/run.py
@@ -40,7 +40,7 @@ class WebServerThread(threading.Thread):
return app.response_class(generate(), mimetype='text/plain')
- app.run(port=9647)
+ app.run(port=9647, host="0.0.0.0")
def run(self):
# Variable that stores the exception, if raised by someFunction
From 6a1c7a36d824799ec610bda1a0f4ae099c59f5c7 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 13 May 2022 22:33:10 -0400
Subject: [PATCH 07/37] changing log level
---
Dockerfile | 1 -
src/SteamGifts.py | 29 +++++++++++++++--------------
src/giveaway.py | 4 ++++
src/log.py | 15 +--------------
src/run.py | 18 +++++++++++++++++-
5 files changed, 37 insertions(+), 30 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 42b358f..ebc1613 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,6 @@ FROM python:3.9-alpine
RUN mkdir -p /app
WORKDIR /app
-# resolves gcc issue with installing regex dependency
RUN apk add tzdata --no-cache
ENV TZ=America/New_York
diff --git a/src/SteamGifts.py b/src/SteamGifts.py
index 08c01e8..213d8b7 100644
--- a/src/SteamGifts.py
+++ b/src/SteamGifts.py
@@ -92,10 +92,9 @@ class SteamGifts:
number_won = soup.select_one("a[title='Giveaways Won'] div").text
won_notifications = TableNotification.get_won_notifications_today()
if won_notifications and len(won_notifications) >= 1:
- logger.debug("Win(s) detected, but we have already notified that there are won games waiting "
- "to be received. Doing nothing.")
+ logger.info("Win(s) detected, but we have already notified that there are won games waiting "
+ "to be received. Doing nothing.")
else:
- logger.debug("Win(s) detected. Going to send a notification.")
logger.info(f"WINNER! You have {number_won} game(s) waiting to be claimed.")
self.notification.send_won(f"WINNER! You have {number_won} game(s) waiting to be claimed.")
else:
@@ -106,39 +105,36 @@ class SteamGifts:
return False
if giveaway.time_created_in_minutes is None:
return False
- txt = f"{giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries (w/ {giveaway.copies} " \
- f"copies) - Created {giveaway.time_created_string} ago with {giveaway.time_remaining_string} remaining."
- logger.debug(txt)
if self.blacklist is not None and self.blacklist != ['']:
for keyword in self.blacklist:
if giveaway.game_name.lower().find(keyword.lower()) != -1:
txt = f"Game {giveaway.game_name} contains the blacklisted keyword {keyword}"
- logger.debug(txt)
+ logger.info(txt)
return False
if giveaway.contributor_level is None or self.contributor_level < giveaway.contributor_level:
txt = f"Game {giveaway.game_name} requires at least level {giveaway.contributor_level} contributor level " \
f"to enter. Your level: {self.contributor_level}"
- logger.debug(txt)
+ logger.info(txt)
return False
if self.points - int(giveaway.cost) < 0:
txt = f"⛔ Not enough points to enter: {giveaway.game_name}"
- logger.debug(txt)
+ logger.info(txt)
return False
if giveaway.cost < self.minimum_game_points:
txt = f"Game {giveaway.game_name} costs {giveaway.cost}P and is below your cutoff of " \
f"{self.minimum_game_points}P."
- logger.debug(txt)
+ logger.info(txt)
return False
if giveaway.time_remaining_in_minutes > self.max_time_left:
txt = f"Game {giveaway.game_name} has {giveaway.time_remaining_in_minutes} minutes left and is " \
f"above your cutoff of {self.max_time_left} minutes."
- logger.debug(txt)
+ logger.info(txt)
return False
if giveaway.game_entries / giveaway.copies > self.max_entries:
txt = f"Game {giveaway.game_name} has {giveaway.game_entries} entries and is above your cutoff " \
f"of {self.max_entries} entries."
- logger.debug(txt)
+ logger.info(txt)
return False
return True
@@ -148,15 +144,16 @@ class SteamGifts:
'User-Agent': self.user_agent
}
payload = {'xsrf_token': self.xsrf_token, 'do': 'entry_insert', 'code': giveaway.giveaway_game_id}
+ logger.debug(f"Sending enter giveaway payload: {payload}")
entry = requests.post('https://www.steamgifts.com/ajax.php', data=payload, cookies=self.cookie,
headers=headers)
json_data = json.loads(entry.text)
if json_data['type'] == 'success':
- logger.debug(f"Successfully entered giveaway {giveaway.giveaway_game_id}")
+ logger.debug(f"Successfully entered giveaway {giveaway.giveaway_game_id}: {json_data}")
return True
else:
- logger.error(f"Failed entering giveaway {giveaway.giveaway_game_id}")
+ logger.error(f"Failed entering giveaway {giveaway.giveaway_game_id}: {json_data}")
return False
def evaluate_giveaways(self, page=1):
@@ -185,7 +182,11 @@ class SteamGifts:
for item in unentered_game_list:
giveaway = Giveaway(item)
+ txt = f"{giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries (w/ {giveaway.copies} " \
+ f"copies) - Created {giveaway.time_created_string} ago with {giveaway.time_remaining_string} remaining."
+ logger.info(txt)
if giveaway.pinned and not self.pinned:
+ logger.info(f"Giveaway {giveaway.game_name} is pinned. Ignoring.")
continue
if self.points == 0 or self.points < self.min_points:
diff --git a/src/giveaway.py b/src/giveaway.py
index 9ffc1fc..e648152 100644
--- a/src/giveaway.py
+++ b/src/giveaway.py
@@ -47,6 +47,7 @@ class Giveaway:
self.time_created_timestamp = int(times[1]['data-timestamp'])
self.time_created_string = times[1].text
self.time_created_in_minutes = self.determine_time_in_minutes(times[1]['data-timestamp'])
+ logger.debug(f"Scraped Giveaway: {self}")
def determine_contributor_level(self, contributor_level):
if contributor_level is None:
@@ -104,3 +105,6 @@ class Giveaway:
txt = f"Unable to determine cost or num copies of {game_name} with id {game_id}."
logger.error(txt)
return None, None
+
+ def __str__(self):
+ return str(self.__class__) + ": " + str(self.__dict__)
diff --git a/src/log.py b/src/log.py
index dedfe66..562e677 100644
--- a/src/log.py
+++ b/src/log.py
@@ -14,7 +14,7 @@ console_output.setLevel(logging.INFO)
console_format = logging.Formatter(log_format)
console_output.setFormatter(console_format)
-info_log_file = RotatingFileHandler('../config/info.log', maxBytes=500000, backupCount=10)
+info_log_file = RotatingFileHandler('../config/info.log', maxBytes=10000, backupCount=10)
info_log_file.setLevel(logging.INFO)
info_log_format = logging.Formatter(log_format)
info_log_file.setFormatter(info_log_format)
@@ -22,19 +22,6 @@ info_log_file.setFormatter(info_log_format)
logging.root.addHandler(console_output)
logging.root.addHandler(info_log_file)
-logging.info("""
--------------------------------------------------------------------------------------
- _____ _ _ __ _ ____ _
- / ____|| | (_) / _|| | | _ \ | |
- | (___ | |_ ___ __ _ _ __ ___ __ _ _ | |_ | |_ ___ | |_) | ___ | |_
- \___ \ | __|/ _ \ / _` || '_ ` _ \ / _` || || _|| __|/ __| | _ < / _ \ | __|
- ____) || |_| __/| (_| || | | | | || (_| || || | | |_ \__ \ | |_) || (_) || |_
- |_____/ \__|\___| \__,_||_| |_| |_| \__, ||_||_| \__||___/ |____/ \___/ \__|
- __/ |
- |___/
--------------------------------------------------------------------------------------
-""")
-
def get_logger(name):
l = logging.getLogger(name)
diff --git a/src/run.py b/src/run.py
index fa1ec81..327b155 100644
--- a/src/run.py
+++ b/src/run.py
@@ -14,6 +14,10 @@ logger = log.get_logger(__name__)
class WebServerThread(threading.Thread):
+ def __init__(self):
+ Thread.__init__(self)
+ self.exc = None
+
def run_webserver(self):
from flask import Flask
from flask import render_template
@@ -33,7 +37,7 @@ class WebServerThread(threading.Thread):
@app.route("/stream")
def stream():
def generate():
- with open('../config/debug.log') as f:
+ with open('../config/info.log') as f:
while True:
yield f.read()
sleep(10)
@@ -166,4 +170,16 @@ def run():
if __name__ == '__main__':
+ logger.info("""
+ -------------------------------------------------------------------------------------
+ _____ _ _ __ _ ____ _
+ / ____|| | (_) / _|| | | _ \ | |
+ | (___ | |_ ___ __ _ _ __ ___ __ _ _ | |_ | |_ ___ | |_) | ___ | |_
+ \___ \ | __|/ _ \ / _` || '_ ` _ \ / _` || || _|| __|/ __| | _ < / _ \ | __|
+ ____) || |_| __/| (_| || | | | | || (_| || || | | |_ \__ \ | |_) || (_) || |_
+ |_____/ \__|\___| \__,_||_| |_| |_| \__, ||_||_| \__||___/ |____/ \___/ \__|
+ __/ |
+ |___/
+ -------------------------------------------------------------------------------------
+ """)
run()
From 54054b4f6de1626b26692adb3c3b7870a32dcd2e Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sat, 14 May 2022 00:47:15 -0400
Subject: [PATCH 08/37] docker. basic auth.
---
Dockerfile | 2 +-
config/config.ini.example | 18 +++++++++++++++++-
requirements.txt | 4 +++-
src/ConfigReader.py | 7 ++++++-
src/log.py | 2 +-
src/run.py | 36 ++++++++++++++++++++++++++++++------
6 files changed, 58 insertions(+), 11 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index ebc1613..7835b0f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,7 @@ FROM python:3.9-alpine
RUN mkdir -p /app
WORKDIR /app
-RUN apk add tzdata --no-cache
+RUN apk add tzdata build-base libffi-dev py3-cffi --no-cache
ENV TZ=America/New_York
ENV VIRTUAL_ENV=/app/env
diff --git a/config/config.ini.example b/config/config.ini.example
index 2fb7b1d..6d0fee8 100644
--- a/config/config.ini.example
+++ b/config/config.ini.example
@@ -31,4 +31,20 @@ pushover.enabled = false
# your specific pushover token
pushover.token =
# your specific pushover user key
-pushover.user_key =
\ No newline at end of file
+pushover.user_key =
+
+[WEB]
+# should we enable the webserver which is just a simple, simple, simple webui to view the logs
+web.enabled = false
+# the port to run on
+web.port = 9547
+# the app root / web folder / root / many other names . MUST contain a trailing '/'
+web.app_root = /
+# should this served up on http or https
+web.ssl = true
+# should we use basic auth
+web.basic_auth = true
+# basic auth username
+web.basic_auth.username = admin
+# basic auth password
+web.basic_auth.password = admin
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 30e751c..ec416c3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,4 +4,6 @@ urllib3==1.26.9
sqlalchemy==1.4.36
sqlalchemy_utils==0.38.2
python-dateutil==2.8.2
-Flask==2.1.2
\ No newline at end of file
+Flask==2.1.2
+Flask-BasicAuth==0.2.0
+pyopenssl==22.0.0
\ No newline at end of file
diff --git a/src/ConfigReader.py b/src/ConfigReader.py
index 66e49ad..e69c3d7 100644
--- a/src/ConfigReader.py
+++ b/src/ConfigReader.py
@@ -75,7 +75,12 @@ class ConfigReader(ConfigParser):
},
'WEB': {
'web.enabled': 'false',
- 'web.port': '9647'
+ 'web.app_root': '/',
+ 'web.port': '9647',
+ 'web.ssl': 'true',
+ 'web.basic_auth': 'true',
+ 'web.basic_auth.username': 'admin',
+ 'web.basic_auth.password': 'p@ssw0rd'
}
}
deprecated_values = {
diff --git a/src/log.py b/src/log.py
index 562e677..36c702c 100644
--- a/src/log.py
+++ b/src/log.py
@@ -14,7 +14,7 @@ console_output.setLevel(logging.INFO)
console_format = logging.Formatter(log_format)
console_output.setFormatter(console_format)
-info_log_file = RotatingFileHandler('../config/info.log', maxBytes=10000, backupCount=10)
+info_log_file = RotatingFileHandler('../config/info.log', maxBytes=100000, backupCount=10)
info_log_file.setLevel(logging.INFO)
info_log_format = logging.Formatter(log_format)
info_log_file.setFormatter(info_log_format)
diff --git a/src/run.py b/src/run.py
index 327b155..48c98c0 100644
--- a/src/run.py
+++ b/src/run.py
@@ -2,6 +2,8 @@ import threading
from random import randint
from time import sleep
+from flask_basicauth import BasicAuth
+
import log
from ConfigReader import ConfigReader, ConfigException
from SteamGifts import SteamGifts, SteamGiftsException
@@ -14,9 +16,17 @@ logger = log.get_logger(__name__)
class WebServerThread(threading.Thread):
- def __init__(self):
+ def __init__(self, config):
Thread.__init__(self)
self.exc = None
+ self.config = config
+ self.port = config['WEB'].getint('web.port')
+ self.ssl = config['WEB'].getboolean('web.ssl')
+ self.enabled = config['WEB'].getboolean('web.enabled')
+ self.app_root = config['WEB'].get('web.app_root')
+ self.basic_auth = config['WEB'].getboolean('web.basic_auth')
+ self.basic_auth_username = config['WEB'].get('web.basic_auth.username')
+ self.basic_auth_password = config['WEB'].get('web.basic_auth.password')
def run_webserver(self):
from flask import Flask
@@ -24,17 +34,24 @@ class WebServerThread(threading.Thread):
app = Flask(__name__)
- @app.route("/")
+ if self.basic_auth:
+ app.config['BASIC_AUTH_USERNAME'] = self.basic_auth_username
+ app.config['BASIC_AUTH_PASSWORD'] = self.basic_auth_password
+
+ app.config['BASIC_AUTH_FORCE'] = self.basic_auth
+ basic_auth = BasicAuth(app)
+
+ @app.route(f"{self.app_root}")
def config():
with open('../config/config.ini', 'r') as f:
content = f.read()
return render_template('configuration.html', config=content)
- @app.route("/log")
+ @app.route(f"{self.app_root}log")
def logs():
return render_template('log.html')
- @app.route("/stream")
+ @app.route(f"{self.app_root}stream")
def stream():
def generate():
with open('../config/info.log') as f:
@@ -44,7 +61,14 @@ class WebServerThread(threading.Thread):
return app.response_class(generate(), mimetype='text/plain')
- app.run(port=9647, host="0.0.0.0")
+ if self.enabled:
+ logger.info("Webserver Enabled. Running")
+ if self.ssl:
+ app.run(port=self.port, host="0.0.0.0", ssl_context='adhoc')
+ else:
+ app.run(port=self.port, host="0.0.0.0")
+ else:
+ logger.info("Webserver NOT Enabled.")
def run(self):
# Variable that stores the exception, if raised by someFunction
@@ -151,7 +175,7 @@ def run():
g.setName("Giveaway Enterer")
g.start()
- w = WebServerThread()
+ w = WebServerThread(config)
w.setName("WebServer")
# if the giveaway thread dies then this daemon thread will die by definition
w.setDaemon(True)
From d76452bdc3893f8fd29197b179684f3d472c56eb Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sat, 14 May 2022 11:13:29 -0400
Subject: [PATCH 09/37] pretty logs
---
src/SteamGifts.py | 42 +++++++++++++++++++++---------------------
src/index.html | 14 --------------
src/run.py | 6 ++++--
3 files changed, 25 insertions(+), 37 deletions(-)
delete mode 100644 src/index.html
diff --git a/src/SteamGifts.py b/src/SteamGifts.py
index 213d8b7..6bf3294 100644
--- a/src/SteamGifts.py
+++ b/src/SteamGifts.py
@@ -84,18 +84,18 @@ class SteamGifts:
self.points = int(soup.find('span', {'class': 'nav__points'}).text) # storage points
self.contributor_level = int(float(soup.select_one('nav a>span[title]')['title']))
except TypeError:
- logger.error("⛔ Cookie is not valid.")
- raise SteamGiftsException("Cookie is not valid.")
+ logger.error("⛔⛔⛔ Cookie is not valid. A new one must be added.⛔⛔⛔")
+ raise SteamGiftsException("Cookie is not valid. A new one must be added.")
won = soup.select("a[title='Giveaways Won'] div")
if won:
number_won = soup.select_one("a[title='Giveaways Won'] div").text
won_notifications = TableNotification.get_won_notifications_today()
if won_notifications and len(won_notifications) >= 1:
- logger.info("Win(s) detected, but we have already notified that there are won games waiting "
+ logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting "
"to be received. Doing nothing.")
else:
- logger.info(f"WINNER! You have {number_won} game(s) waiting to be claimed.")
+ logger.info(f"💰💰 WINNER! You have {number_won} game(s) waiting to be claimed. 💰💰")
self.notification.send_won(f"WINNER! You have {number_won} game(s) waiting to be claimed.")
else:
logger.debug('No wins detected. Doing nothing.')
@@ -109,30 +109,30 @@ class SteamGifts:
if self.blacklist is not None and self.blacklist != ['']:
for keyword in self.blacklist:
if giveaway.game_name.lower().find(keyword.lower()) != -1:
- txt = f"Game {giveaway.game_name} contains the blacklisted keyword {keyword}"
+ txt = f"〰️ Game {giveaway.game_name} contains the blacklisted keyword {keyword}"
logger.info(txt)
return False
if giveaway.contributor_level is None or self.contributor_level < giveaway.contributor_level:
- txt = f"Game {giveaway.game_name} requires at least level {giveaway.contributor_level} contributor level " \
- f"to enter. Your level: {self.contributor_level}"
+ txt = f"〰️ Game {giveaway.game_name} requires at least level {giveaway.contributor_level} contributor " \
+ f"level to enter. Your level: {self.contributor_level}"
logger.info(txt)
return False
if self.points - int(giveaway.cost) < 0:
- txt = f"⛔ Not enough points to enter: {giveaway.game_name}"
+ txt = f"〰️ Not enough points to enter: {giveaway.game_name}"
logger.info(txt)
return False
if giveaway.cost < self.minimum_game_points:
- txt = f"Game {giveaway.game_name} costs {giveaway.cost}P and is below your cutoff of " \
+ txt = f"〰️ Game {giveaway.game_name} costs {giveaway.cost}P and is below your cutoff of " \
f"{self.minimum_game_points}P."
logger.info(txt)
return False
if giveaway.time_remaining_in_minutes > self.max_time_left:
- txt = f"Game {giveaway.game_name} has {giveaway.time_remaining_in_minutes} minutes left and is " \
+ txt = f"〰️ Game {giveaway.game_name} has {giveaway.time_remaining_in_minutes} minutes left and is " \
f"above your cutoff of {self.max_time_left} minutes."
logger.info(txt)
return False
if giveaway.game_entries / giveaway.copies > self.max_entries:
- txt = f"Game {giveaway.game_name} has {giveaway.game_entries} entries and is above your cutoff " \
+ txt = f"〰️ Game {giveaway.game_name} has {giveaway.game_entries} entries and is above your cutoff " \
f"of {self.max_entries} entries."
logger.info(txt)
return False
@@ -153,14 +153,14 @@ class SteamGifts:
logger.debug(f"Successfully entered giveaway {giveaway.giveaway_game_id}: {json_data}")
return True
else:
- logger.error(f"Failed entering giveaway {giveaway.giveaway_game_id}: {json_data}")
+ logger.error(f"❌ Failed entering giveaway {giveaway.giveaway_game_id}: {json_data}")
return False
def evaluate_giveaways(self, page=1):
n = page
run = True
while run and n < 3: # hard stop safety net at page 3 as idk why we would ever get to this point
- txt = "⚙️ Retrieving games from %d page." % n
+ txt = "〰️ Retrieving games from %d page." % n
logger.info(txt)
filtered_url = self.filter_url[self.gifts_type] % n
@@ -176,21 +176,21 @@ class SteamGifts:
# game_list = soup.find_all('div', {'class': 'giveaway__row-inner-wrap'})
if not len(unentered_game_list) or (all_games_list_count == pinned_giveaway_count):
- txt = f"We have run out of gifts to consider."
+ txt = f"〰️ We have run out of gifts to consider."
logger.info(txt)
break
for item in unentered_game_list:
giveaway = Giveaway(item)
- txt = f"{giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries (w/ {giveaway.copies} " \
+ txt = f"〰️ {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries (w/ {giveaway.copies} " \
f"copies) - Created {giveaway.time_created_string} ago with {giveaway.time_remaining_string} remaining."
logger.info(txt)
if giveaway.pinned and not self.pinned:
- logger.info(f"Giveaway {giveaway.game_name} is pinned. Ignoring.")
+ logger.info(f"〰️ Giveaway {giveaway.game_name} is pinned. Ignoring.")
continue
if self.points == 0 or self.points < self.min_points:
- txt = f"🛋️ We have {self.points} points, but we need {self.min_points} to start."
+ txt = f"〰 We have {self.points} points, but we need {self.min_points} to start."
logger.info(txt)
run = False
break
@@ -203,7 +203,7 @@ class SteamGifts:
if res:
TableGiveaway.upsert_giveaway(giveaway, True)
self.points -= int(giveaway.cost)
- txt = f"🎉 One more game! Has just entered {giveaway.game_name}"
+ txt = f"✅ Entered giveaway '{giveaway.game_name}'"
logger.info(txt)
sleep(randint(4, 15))
else:
@@ -215,7 +215,7 @@ class SteamGifts:
# after this point will also exceed the max time left
if self.gifts_type != "New" and not giveaway.pinned and \
giveaway.time_remaining_in_minutes > self.max_time_left:
- logger.info("We have run out of gifts to consider.")
+ logger.info("🟡 We have run out of gifts to consider.")
run = False
break
@@ -224,10 +224,10 @@ class SteamGifts:
def start(self):
self.update_info()
if self.points >= self.min_points:
- txt = f"🤖 You have {self.points} points. Evaluating '{self.gifts_type}' giveaways..."
+ txt = f"〰 You have {self.points} points. Evaluating '{self.gifts_type}' giveaways..."
logger.info(txt)
self.evaluate_giveaways()
else:
- txt = f"You have {self.points} points which is below your minimum point threshold of " \
+ txt = f"🟡 You have {self.points} points which is below your minimum point threshold of " \
f"{self.min_points} points for '{self.gifts_type}' giveaways. Not evaluating right now."
logger.info(txt)
diff --git a/src/index.html b/src/index.html
deleted file mode 100644
index c6b9cdb..0000000
--- a/src/index.html
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-
-
- Hello!
-
-
-
-
Hello World!
-
This is a simple paragraph.
-
-
-
\ No newline at end of file
diff --git a/src/run.py b/src/run.py
index 48c98c0..e01c8a6 100644
--- a/src/run.py
+++ b/src/run.py
@@ -117,18 +117,20 @@ class GiveawayEntererThread(threading.Thread):
wishlist_max_entries, wishlist_max_time_left, 0, '', notification)
if not main_page_enabled and not wishlist_page_enabled:
- logger.error("Both 'Default' and 'Wishlist' configurations are disabled. Nothing will run. Exiting...")
+ logger.error("⁉️ Both 'Default' and 'Wishlist' configurations are disabled. Nothing will run. Exiting...")
sleep(10)
exit(-1)
while True:
+ logger.info("🟢 Evaluating giveaways.")
if wishlist_page_enabled:
wishlist_page.start()
if main_page_enabled:
all_page.start()
+ logger.info("🔴 All giveaways evaluated.")
random_seconds = randint(1740, 3540) # sometime between 29-59 minutes
- logger.info(f"Going to sleep for {random_seconds / 60} minutes.")
+ logger.info(f"🛋 Going to sleep for {random_seconds / 60} minutes.")
sleep(random_seconds)
def run(self):
From e8596fb544245f6d5587ffc7a7b02890f297a2b4 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sat, 14 May 2022 18:50:19 -0400
Subject: [PATCH 10/37] stuff
---
src/SteamGifts.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/SteamGifts.py b/src/SteamGifts.py
index 6bf3294..5c015b5 100644
--- a/src/SteamGifts.py
+++ b/src/SteamGifts.py
@@ -182,20 +182,22 @@ class SteamGifts:
for item in unentered_game_list:
giveaway = Giveaway(item)
- txt = f"〰️ {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries (w/ {giveaway.copies} " \
- f"copies) - Created {giveaway.time_created_string} ago with {giveaway.time_remaining_string} remaining."
+ txt = f"〰️ {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries " \
+ f"(w/ {giveaway.copies} copies) - Created {giveaway.time_created_string} ago " \
+ f"with {giveaway.time_remaining_string} remaining by {giveaway.user}."
logger.info(txt)
if giveaway.pinned and not self.pinned:
logger.info(f"〰️ Giveaway {giveaway.game_name} is pinned. Ignoring.")
continue
if self.points == 0 or self.points < self.min_points:
- txt = f"〰 We have {self.points} points, but we need {self.min_points} to start."
+ txt = f"🟡 We have {self.points} points, but we need {self.min_points} to start."
logger.info(txt)
run = False
break
if not giveaway.cost:
+ logger.error(f"Cost could not be determined for '{giveaway.game_name}'")
continue
if_enter_giveaway = self.should_we_enter_giveaway(giveaway)
if if_enter_giveaway:
From fd9e5ba23e2f9a76b2b3d5168d7bca922da3d83a Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sun, 15 May 2022 11:39:44 -0400
Subject: [PATCH 11/37] stuff
---
config/config.ini.example | 2 +-
src/SteamGifts.py | 2 +-
src/giveaway.py | 3 +--
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/config/config.ini.example b/config/config.ini.example
index 6d0fee8..fd6f43f 100644
--- a/config/config.ini.example
+++ b/config/config.ini.example
@@ -47,4 +47,4 @@ web.basic_auth = true
# basic auth username
web.basic_auth.username = admin
# basic auth password
-web.basic_auth.password = admin
\ No newline at end of file
+web.basic_auth.password = ChangeMe
\ No newline at end of file
diff --git a/src/SteamGifts.py b/src/SteamGifts.py
index 5c015b5..875b62f 100644
--- a/src/SteamGifts.py
+++ b/src/SteamGifts.py
@@ -182,7 +182,7 @@ class SteamGifts:
for item in unentered_game_list:
giveaway = Giveaway(item)
- txt = f"〰️ {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries " \
+ txt = f"〰 {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries " \
f"(w/ {giveaway.copies} copies) - Created {giveaway.time_created_string} ago " \
f"with {giveaway.time_remaining_string} remaining by {giveaway.user}."
logger.info(txt)
diff --git a/src/giveaway.py b/src/giveaway.py
index e648152..936b846 100644
--- a/src/giveaway.py
+++ b/src/giveaway.py
@@ -8,7 +8,6 @@ logger = log.get_logger(__name__)
class Giveaway:
def __init__(self, soup_item):
- self.soup_item = soup_item
self.steam_app_id = None
self.steam_url = None
self.game_name = None
@@ -35,7 +34,7 @@ class Giveaway:
self.giveaway_uri = soup_item.select_one('a.giveaway__heading__name')['href']
pin_class = soup_item.parent.parent.get("class")
self.pinned = pin_class is not None and len(pin_class) > 0 and pin_class[0].find('pinned') != -1
- self.cost, self.copies = self.determine_cost_and_copies(self.soup_item, self.game_name, self.giveaway_game_id)
+ self.cost, self.copies = self.determine_cost_and_copies(soup_item, self.game_name, self.giveaway_game_id)
self.game_entries = int(soup_item.select('div.giveaway__links span')[0].text.split(' ')[0].replace(',', ''))
contributor_level = soup_item.select_one('div[title="Contributor Level"]')
self.contributor_level = self.determine_contributor_level(contributor_level)
From b51712cb8750fcd42debafc19584460f4a521cf8 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sun, 15 May 2022 13:56:28 -0400
Subject: [PATCH 12/37] rename all the things
---
src/{ConfigReader.py => config_reader.py} | 0
src/{SteamGifts.py => enter_giveaways.py} | 2 +-
src/giveaway_thread.py | 72 ++++++++++
src/run.py | 154 +---------------------
src/templates/configuration.html | 2 +-
src/templates/log.html | 2 +-
src/webserver_thread.py | 86 ++++++++++++
7 files changed, 168 insertions(+), 150 deletions(-)
rename src/{ConfigReader.py => config_reader.py} (100%)
rename src/{SteamGifts.py => enter_giveaways.py} (97%)
create mode 100644 src/giveaway_thread.py
create mode 100644 src/webserver_thread.py
diff --git a/src/ConfigReader.py b/src/config_reader.py
similarity index 100%
rename from src/ConfigReader.py
rename to src/config_reader.py
diff --git a/src/SteamGifts.py b/src/enter_giveaways.py
similarity index 97%
rename from src/SteamGifts.py
rename to src/enter_giveaways.py
index 875b62f..878af86 100644
--- a/src/SteamGifts.py
+++ b/src/enter_giveaways.py
@@ -18,7 +18,7 @@ class SteamGiftsException(Exception):
pass
-class SteamGifts:
+class EnterGiveaways:
def __init__(self, cookie, user_agent, gifts_type, pinned, min_points, max_entries,
max_time_left, minimum_game_points, blacklist, notification):
self.contributor_level = None
diff --git a/src/giveaway_thread.py b/src/giveaway_thread.py
new file mode 100644
index 0000000..bf12294
--- /dev/null
+++ b/src/giveaway_thread.py
@@ -0,0 +1,72 @@
+import threading
+from random import randint
+from threading import Thread
+from time import sleep
+
+import log
+from enter_giveaways import EnterGiveaways
+
+logger = log.get_logger(__name__)
+
+
+class GiveawayThread(threading.Thread):
+
+ def __init__(self, config, notification):
+ Thread.__init__(self)
+ self.exc = None
+ self.config = config
+ self.notification = notification
+
+ def run_steam_gifts(self, config, notification):
+ cookie = config['DEFAULT'].get('cookie')
+ user_agent = config['DEFAULT'].get('user_agent')
+ main_page_enabled = config['DEFAULT'].getboolean('enabled')
+ minimum_points = config['DEFAULT'].getint('minimum_points')
+ max_entries = config['DEFAULT'].getint('max_entries')
+ max_time_left = config['DEFAULT'].getint('max_time_left')
+ minimum_game_points = config['DEFAULT'].getint('minimum_game_points')
+ blacklist = config['DEFAULT'].get('blacklist_keywords')
+
+ all_page = EnterGiveaways(cookie, user_agent, 'All', False, minimum_points, max_entries,
+ max_time_left, minimum_game_points, blacklist, notification)
+
+ wishlist_page_enabled = config['WISHLIST'].getboolean('wishlist.enabled')
+ wishlist_minimum_points = config['WISHLIST'].getint('wishlist.minimum_points')
+ wishlist_max_entries = config['WISHLIST'].getint('wishlist.max_entries')
+ wishlist_max_time_left = config['WISHLIST'].getint('wishlist.max_time_left')
+
+ wishlist_page = EnterGiveaways(cookie, user_agent, 'Wishlist', False, wishlist_minimum_points,
+ wishlist_max_entries, wishlist_max_time_left, 0, '', notification)
+
+ if not main_page_enabled and not wishlist_page_enabled:
+ logger.error("⁉️ Both 'Default' and 'Wishlist' configurations are disabled. Nothing will run. Exiting...")
+ sleep(10)
+ exit(-1)
+
+ while True:
+ logger.info("🟢 Evaluating giveaways.")
+ if wishlist_page_enabled:
+ wishlist_page.start()
+ if main_page_enabled:
+ all_page.start()
+
+ logger.info("🔴 All giveaways evaluated.")
+ random_seconds = randint(1740, 3540) # sometime between 29-59 minutes
+ logger.info(f"🛋 Going to sleep for {random_seconds / 60} minutes.")
+ sleep(random_seconds)
+
+ def run(self):
+ # Variable that stores the exception, if raised by someFunction
+ self.exc = None
+ try:
+ self.run_steam_gifts(self.config, self.notification)
+ except BaseException as e:
+ self.exc = e
+
+ def join(self):
+ threading.Thread.join(self)
+ # Since join() returns in caller thread
+ # we re-raise the caught exception
+ # if any was caught
+ if self.exc:
+ raise self.exc
\ No newline at end of file
diff --git a/src/run.py b/src/run.py
index e01c8a6..320ca17 100644
--- a/src/run.py
+++ b/src/run.py
@@ -1,155 +1,15 @@
-import threading
-from random import randint
from time import sleep
-from flask_basicauth import BasicAuth
-
import log
-from ConfigReader import ConfigReader, ConfigException
-from SteamGifts import SteamGifts, SteamGiftsException
+from config_reader import ConfigReader, ConfigException
+from enter_giveaways import SteamGiftsException
+from giveaway_thread import GiveawayThread
from notification import Notification
-from threading import Thread
-
+from webserver_thread import WebServerThread
logger = log.get_logger(__name__)
-class WebServerThread(threading.Thread):
-
- def __init__(self, config):
- Thread.__init__(self)
- self.exc = None
- self.config = config
- self.port = config['WEB'].getint('web.port')
- self.ssl = config['WEB'].getboolean('web.ssl')
- self.enabled = config['WEB'].getboolean('web.enabled')
- self.app_root = config['WEB'].get('web.app_root')
- self.basic_auth = config['WEB'].getboolean('web.basic_auth')
- self.basic_auth_username = config['WEB'].get('web.basic_auth.username')
- self.basic_auth_password = config['WEB'].get('web.basic_auth.password')
-
- def run_webserver(self):
- from flask import Flask
- from flask import render_template
-
- app = Flask(__name__)
-
- if self.basic_auth:
- app.config['BASIC_AUTH_USERNAME'] = self.basic_auth_username
- app.config['BASIC_AUTH_PASSWORD'] = self.basic_auth_password
-
- app.config['BASIC_AUTH_FORCE'] = self.basic_auth
- basic_auth = BasicAuth(app)
-
- @app.route(f"{self.app_root}")
- def config():
- with open('../config/config.ini', 'r') as f:
- content = f.read()
- return render_template('configuration.html', config=content)
-
- @app.route(f"{self.app_root}log")
- def logs():
- return render_template('log.html')
-
- @app.route(f"{self.app_root}stream")
- def stream():
- def generate():
- with open('../config/info.log') as f:
- while True:
- yield f.read()
- sleep(10)
-
- return app.response_class(generate(), mimetype='text/plain')
-
- if self.enabled:
- logger.info("Webserver Enabled. Running")
- if self.ssl:
- app.run(port=self.port, host="0.0.0.0", ssl_context='adhoc')
- else:
- app.run(port=self.port, host="0.0.0.0")
- else:
- logger.info("Webserver NOT Enabled.")
-
- def run(self):
- # Variable that stores the exception, if raised by someFunction
- self.exc = None
- try:
- self.run_webserver()
- except BaseException as e:
- self.exc = e
-
- def join(self):
- threading.Thread.join(self)
- # Since join() returns in caller thread
- # we re-raise the caught exception
- # if any was caught
- if self.exc:
- raise self.exc
-
-
-class GiveawayEntererThread(threading.Thread):
-
- def __init__(self, config, notification):
- Thread.__init__(self)
- self.exc = None
- self.config = config
- self.notification = notification
-
- def run_steam_gifts(self, config, notification):
- cookie = config['DEFAULT'].get('cookie')
- user_agent = config['DEFAULT'].get('user_agent')
- main_page_enabled = config['DEFAULT'].getboolean('enabled')
- minimum_points = config['DEFAULT'].getint('minimum_points')
- max_entries = config['DEFAULT'].getint('max_entries')
- max_time_left = config['DEFAULT'].getint('max_time_left')
- minimum_game_points = config['DEFAULT'].getint('minimum_game_points')
- blacklist = config['DEFAULT'].get('blacklist_keywords')
-
- all_page = SteamGifts(cookie, user_agent, 'All', False, minimum_points, max_entries,
- max_time_left, minimum_game_points, blacklist, notification)
-
- wishlist_page_enabled = config['WISHLIST'].getboolean('wishlist.enabled')
- wishlist_minimum_points = config['WISHLIST'].getint('wishlist.minimum_points')
- wishlist_max_entries = config['WISHLIST'].getint('wishlist.max_entries')
- wishlist_max_time_left = config['WISHLIST'].getint('wishlist.max_time_left')
-
- wishlist_page = SteamGifts(cookie, user_agent, 'Wishlist', False, wishlist_minimum_points,
- wishlist_max_entries, wishlist_max_time_left, 0, '', notification)
-
- if not main_page_enabled and not wishlist_page_enabled:
- logger.error("⁉️ Both 'Default' and 'Wishlist' configurations are disabled. Nothing will run. Exiting...")
- sleep(10)
- exit(-1)
-
- while True:
- logger.info("🟢 Evaluating giveaways.")
- if wishlist_page_enabled:
- wishlist_page.start()
- if main_page_enabled:
- all_page.start()
-
- logger.info("🔴 All giveaways evaluated.")
- random_seconds = randint(1740, 3540) # sometime between 29-59 minutes
- logger.info(f"🛋 Going to sleep for {random_seconds / 60} minutes.")
- sleep(random_seconds)
-
- def run(self):
- # Variable that stores the exception, if raised by someFunction
- self.exc = None
- try:
- self.run_steam_gifts(self.config, self.notification)
- except BaseException as e:
- self.exc = e
-
- def join(self):
- threading.Thread.join(self)
- # Since join() returns in caller thread
- # we re-raise the caught exception
- # if any was caught
- if self.exc:
- raise self.exc
-
-
def run():
logger.info("Starting Steamgifts bot.")
file_name = '../config/config.ini'
@@ -173,13 +33,13 @@ def run():
if pushover_enabled:
notification.enable_pushover(pushover_token, pushover_user_key)
try:
- g = GiveawayEntererThread(config, notification)
+ g = GiveawayThread(config, notification)
g.setName("Giveaway Enterer")
g.start()
w = WebServerThread(config)
w.setName("WebServer")
- # if the giveaway thread dies then this daemon thread will die by definition
+ # if the giveaway thread dies then this daemon thread will die by design
w.setDaemon(True)
w.start()
@@ -191,7 +51,7 @@ def run():
except Exception as e:
logger.error(e)
notification.send_error("Something happened and the bot had to quit!")
- sleep(5)
+ sleep(10)
exit(-1)
diff --git a/src/templates/configuration.html b/src/templates/configuration.html
index 05b6f59..320839d 100644
--- a/src/templates/configuration.html
+++ b/src/templates/configuration.html
@@ -1,7 +1,7 @@
- Flask app
+ Steamgifts Bot Configuration
diff --git a/src/templates/log.html b/src/templates/log.html
index e365d92..155c1f4 100644
--- a/src/templates/log.html
+++ b/src/templates/log.html
@@ -1,7 +1,7 @@
- Flask app
+ Steamgifts Bot Logs
diff --git a/src/webserver_thread.py b/src/webserver_thread.py
new file mode 100644
index 0000000..3e66503
--- /dev/null
+++ b/src/webserver_thread.py
@@ -0,0 +1,86 @@
+import threading
+from threading import Thread
+from time import sleep
+
+from flask_basicauth import BasicAuth
+
+import log
+
+logger = log.get_logger(__name__)
+
+
+class WebServerThread(threading.Thread):
+
+ def __init__(self, config):
+ Thread.__init__(self)
+ self.exc = None
+ self.config = config
+ self.port = config['WEB'].getint('web.port')
+ self.ssl = config['WEB'].getboolean('web.ssl')
+ self.enabled = config['WEB'].getboolean('web.enabled')
+ self.app_root = config['WEB'].get('web.app_root')
+ self.basic_auth = config['WEB'].getboolean('web.basic_auth')
+ self.basic_auth_username = config['WEB'].get('web.basic_auth.username')
+ self.basic_auth_password = config['WEB'].get('web.basic_auth.password')
+
+ def run_webserver(self):
+ from flask import Flask
+ from flask import render_template
+
+ app = Flask(__name__)
+
+ if self.basic_auth:
+ app.config['BASIC_AUTH_USERNAME'] = self.basic_auth_username
+ app.config['BASIC_AUTH_PASSWORD'] = self.basic_auth_password
+
+ app.config['BASIC_AUTH_FORCE'] = self.basic_auth
+ basic_auth = BasicAuth(app)
+
+ @app.route(f"{self.app_root}")
+ def config():
+ with open('../config/config.ini', 'r') as f:
+ content = f.read()
+ return render_template('configuration.html', config=content)
+
+ @app.route(f"{self.app_root}log")
+ def logs():
+ return render_template('log.html')
+
+ @app.route(f"{self.app_root}stream")
+ def stream():
+ def generate():
+ with open('../config/info.log') as f:
+ while True:
+ yield f.read()
+ sleep(10)
+
+ return app.response_class(generate(), mimetype='text/plain')
+
+ @app.route(f"{self.app_root}alive")
+ def alive():
+ return 'OK'
+
+ if self.enabled:
+ logger.info("Webserver Enabled. Running")
+ if self.ssl:
+ app.run(port=self.port, host="0.0.0.0", ssl_context='adhoc')
+ else:
+ app.run(port=self.port, host="0.0.0.0")
+ else:
+ logger.info("Webserver NOT Enabled.")
+
+ def run(self):
+ # Variable that stores the exception, if raised by someFunction
+ self.exc = None
+ try:
+ self.run_webserver()
+ except BaseException as e:
+ self.exc = e
+
+ def join(self):
+ threading.Thread.join(self)
+ # Since join() returns in caller thread
+ # we re-raise the caught exception
+ # if any was caught
+ if self.exc:
+ raise self.exc
\ No newline at end of file
From 6cb493eba7aebf670c62c16200110799b9a1c789 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Mon, 16 May 2022 16:03:12 -0400
Subject: [PATCH 13/37] add host
- small clean up
---
config/config.ini.example | 2 ++
src/config_reader.py | 16 ++++++++--------
src/enter_giveaways.py | 2 +-
src/giveaway_thread.py | 10 ++++++++--
src/notification.py | 4 +---
src/tables.py | 9 +++++----
src/webserver_thread.py | 7 ++++---
7 files changed, 29 insertions(+), 21 deletions(-)
diff --git a/config/config.ini.example b/config/config.ini.example
index fd6f43f..33efbae 100644
--- a/config/config.ini.example
+++ b/config/config.ini.example
@@ -36,6 +36,8 @@ pushover.user_key =
[WEB]
# should we enable the webserver which is just a simple, simple, simple webui to view the logs
web.enabled = false
+# the host to listen on. localhost or 0.0.0.0 are the two common options
+web.host = localhost
# the port to run on
web.port = 9547
# the app root / web folder / root / many other names . MUST contain a trailing '/'
diff --git a/src/config_reader.py b/src/config_reader.py
index e69c3d7..d0e6520 100644
--- a/src/config_reader.py
+++ b/src/config_reader.py
@@ -27,7 +27,6 @@ def choose_user_agent():
class ConfigReader(ConfigParser):
-
required_values = {
'DEFAULT': {
'enabled': ('true', 'false'),
@@ -51,13 +50,13 @@ class ConfigReader(ConfigParser):
}
}
default_values = {
- 'DEFAULT': {
+ 'DEFAULT': {
'cookie': '',
'user_agent': f"{choose_user_agent()}",
'enabled': 'true',
'minimum_points': f"{randint(20, 50)}",
'max_entries': f"{randint(1000, 2500)}",
- 'max_time_left': f"{randint(180,500)}",
+ 'max_time_left': f"{randint(180, 500)}",
'minimum_game_points': "0",
'blacklist_keywords': 'hentai,adult'
},
@@ -65,7 +64,7 @@ class ConfigReader(ConfigParser):
'wishlist.enabled': 'true',
'wishlist.minimum_points': '1',
'wishlist.max_entries': f"{randint(10000, 100000)}",
- 'wishlist.max_time_left': f"{randint(180,500)}"
+ 'wishlist.max_time_left': f"{randint(180, 500)}"
},
'NOTIFICATIONS': {
'notification.prefix': '',
@@ -75,6 +74,7 @@ class ConfigReader(ConfigParser):
},
'WEB': {
'web.enabled': 'false',
+ 'web.host': '0.0.0.0',
'web.app_root': '/',
'web.port': '9647',
'web.ssl': 'true',
@@ -127,11 +127,11 @@ class ConfigReader(ConfigParser):
for key, values in keys.items():
if key not in self[section] or self[section][key] == '':
raise ConfigException((
- 'Missing value for "%s" under section "%s" in ' +
- 'the config file') % (key, section))
+ 'Missing value for "%s" under section "%s" in ' +
+ 'the config file') % (key, section))
if values:
if self[section][key] not in values:
raise ConfigException((
- 'Invalid value for "%s" under section "%s" in ' +
- 'the config file') % (key, section))
+ 'Invalid value for "%s" under section "%s" in ' +
+ 'the config file') % (key, section))
diff --git a/src/enter_giveaways.py b/src/enter_giveaways.py
index 878af86..fabb27d 100644
--- a/src/enter_giveaways.py
+++ b/src/enter_giveaways.py
@@ -69,7 +69,7 @@ class EnterGiveaways:
def get_soup_from_page(self, url):
headers = {
- 'User-Agent': self.user_agent
+ 'User-Agent': self.user_agent
}
self.requests_retry_session().get(url, headers=headers)
r = requests.get(url, cookies=self.cookie)
diff --git a/src/giveaway_thread.py b/src/giveaway_thread.py
index bf12294..d1076e2 100644
--- a/src/giveaway_thread.py
+++ b/src/giveaway_thread.py
@@ -1,8 +1,12 @@
+import datetime
import threading
+from datetime import timedelta, datetime
from random import randint
from threading import Thread
from time import sleep
+from dateutil import tz
+
import log
from enter_giveaways import EnterGiveaways
@@ -52,7 +56,9 @@ class GiveawayThread(threading.Thread):
logger.info("🔴 All giveaways evaluated.")
random_seconds = randint(1740, 3540) # sometime between 29-59 minutes
- logger.info(f"🛋 Going to sleep for {random_seconds / 60} minutes.")
+ when_to_start_again = datetime.now(tz=tz.tzlocal()) + timedelta(seconds=random_seconds)
+ logger.info(f"🛋 Going to sleep for {random_seconds / 60} minutes. "
+ f"Will start again at {when_to_start_again}")
sleep(random_seconds)
def run(self):
@@ -69,4 +75,4 @@ class GiveawayThread(threading.Thread):
# we re-raise the caught exception
# if any was caught
if self.exc:
- raise self.exc
\ No newline at end of file
+ raise self.exc
diff --git a/src/notification.py b/src/notification.py
index 1e51f45..16b1f86 100644
--- a/src/notification.py
+++ b/src/notification.py
@@ -1,10 +1,8 @@
import http.client
import urllib
-from sqlalchemy.orm import Session
-
-from tables import TableNotification
import log
+from tables import TableNotification
logger = log.get_logger(__name__)
diff --git a/src/tables.py b/src/tables.py
index 348e010..366339f 100644
--- a/src/tables.py
+++ b/src/tables.py
@@ -35,13 +35,14 @@ class TableNotification(Base):
with Session(engine) as session:
# with how filtering of datetimes works with a sqlite backend I couldn't figure out a better way
# to filter out the dates to local time when they are stored in utc in the db
- within_3_days = session.query(TableNotification)\
- .filter(func.DATE(TableNotification.created_at) >= (datetime.utcnow().date() - timedelta(days=1)))\
- .filter(func.DATE(TableNotification.created_at) <= (datetime.utcnow().date() + timedelta(days=1)))\
+ within_3_days = session.query(TableNotification) \
+ .filter(func.DATE(TableNotification.created_at) >= (datetime.utcnow().date() - timedelta(days=1))) \
+ .filter(func.DATE(TableNotification.created_at) <= (datetime.utcnow().date() + timedelta(days=1))) \
.filter_by(type='won').all()
actual = []
for r in within_3_days:
- if r.created_at.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal()).date() == datetime.now(tz=tz.tzlocal()).date():
+ if r.created_at.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal()).date() == datetime.now(
+ tz=tz.tzlocal()).date():
actual.append(r)
return actual
diff --git a/src/webserver_thread.py b/src/webserver_thread.py
index 3e66503..06e25cd 100644
--- a/src/webserver_thread.py
+++ b/src/webserver_thread.py
@@ -15,6 +15,7 @@ class WebServerThread(threading.Thread):
Thread.__init__(self)
self.exc = None
self.config = config
+ self.host = config['WEB'].get('web.host')
self.port = config['WEB'].getint('web.port')
self.ssl = config['WEB'].getboolean('web.ssl')
self.enabled = config['WEB'].getboolean('web.enabled')
@@ -63,9 +64,9 @@ class WebServerThread(threading.Thread):
if self.enabled:
logger.info("Webserver Enabled. Running")
if self.ssl:
- app.run(port=self.port, host="0.0.0.0", ssl_context='adhoc')
+ app.run(port=self.port, host=self.host, ssl_context='adhoc')
else:
- app.run(port=self.port, host="0.0.0.0")
+ app.run(port=self.port, host=self.host)
else:
logger.info("Webserver NOT Enabled.")
@@ -83,4 +84,4 @@ class WebServerThread(threading.Thread):
# we re-raise the caught exception
# if any was caught
if self.exc:
- raise self.exc
\ No newline at end of file
+ raise self.exc
From 9b67cdacf1de8bed521d460d58e713164ae21328 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Mon, 16 May 2022 23:56:47 -0400
Subject: [PATCH 14/37] changes
---
requirements.txt | 3 ++-
src/enter_giveaways.py | 2 +-
src/templates/configuration.html | 4 +---
src/templates/log.html | 14 +-------------
src/webserver_thread.py | 16 ++++------------
5 files changed, 9 insertions(+), 30 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index ec416c3..47e10c5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,5 @@ sqlalchemy_utils==0.38.2
python-dateutil==2.8.2
Flask==2.1.2
Flask-BasicAuth==0.2.0
-pyopenssl==22.0.0
\ No newline at end of file
+pyopenssl==22.0.0
+alembic==1.7.7
\ No newline at end of file
diff --git a/src/enter_giveaways.py b/src/enter_giveaways.py
index fabb27d..42e1004 100644
--- a/src/enter_giveaways.py
+++ b/src/enter_giveaways.py
@@ -176,7 +176,7 @@ class EnterGiveaways:
# game_list = soup.find_all('div', {'class': 'giveaway__row-inner-wrap'})
if not len(unentered_game_list) or (all_games_list_count == pinned_giveaway_count):
- txt = f"〰️ We have run out of gifts to consider."
+ txt = f"🟡 We have run out of gifts to consider."
logger.info(txt)
break
diff --git a/src/templates/configuration.html b/src/templates/configuration.html
index 320839d..db2c119 100644
--- a/src/templates/configuration.html
+++ b/src/templates/configuration.html
@@ -17,9 +17,7 @@
-
{{config}}
- {% block content %}
- {% endblock %}
+
{{content}}
\ No newline at end of file
diff --git a/src/templates/log.html b/src/templates/log.html
index 155c1f4..80b7b71 100644
--- a/src/templates/log.html
+++ b/src/templates/log.html
@@ -17,19 +17,7 @@
-
-
+
{{content}}
\ No newline at end of file
diff --git a/src/webserver_thread.py b/src/webserver_thread.py
index 06e25cd..348f0bf 100644
--- a/src/webserver_thread.py
+++ b/src/webserver_thread.py
@@ -41,21 +41,13 @@ class WebServerThread(threading.Thread):
def config():
with open('../config/config.ini', 'r') as f:
content = f.read()
- return render_template('configuration.html', config=content)
+ return render_template('configuration.html', content=content)
@app.route(f"{self.app_root}log")
def logs():
- return render_template('log.html')
-
- @app.route(f"{self.app_root}stream")
- def stream():
- def generate():
- with open('../config/info.log') as f:
- while True:
- yield f.read()
- sleep(10)
-
- return app.response_class(generate(), mimetype='text/plain')
+ with open('../config/info.log', 'r') as f:
+ content = f.read()
+ return render_template('log.html', content=content)
@app.route(f"{self.app_root}alive")
def alive():
From 948281dbfb8cc3daf1642aeb1c4431af9f281a04 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Wed, 18 May 2022 22:08:03 -0400
Subject: [PATCH 15/37] not used
---
requirements.txt | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index 47e10c5..ec416c3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,5 +6,4 @@ sqlalchemy_utils==0.38.2
python-dateutil==2.8.2
Flask==2.1.2
Flask-BasicAuth==0.2.0
-pyopenssl==22.0.0
-alembic==1.7.7
\ No newline at end of file
+pyopenssl==22.0.0
\ No newline at end of file
From abc2e66c293f608f0fdb9e58ad2e673806e27c39 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Thu, 19 May 2022 18:03:35 -0400
Subject: [PATCH 16/37] version control db
---
.gitignore | 3 +-
alembic/README | 1 +
alembic/env.py | 79 ++++++++++++++++
alembic/script.py.mako | 24 +++++
.../2022_05_19-15_50_19-1da33402b659_init.py | 70 ++++++++++++++
requirements.txt | 3 +-
src/{tables.py => database.py} | 94 ++++++-------------
src/enter_giveaways.py | 10 +-
src/models.py | 52 ++++++++++
src/notification.py | 4 +-
src/run.py | 14 ++-
11 files changed, 278 insertions(+), 76 deletions(-)
create mode 100644 alembic/README
create mode 100644 alembic/env.py
create mode 100644 alembic/script.py.mako
create mode 100644 alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py
rename src/{tables.py => database.py} (57%)
create mode 100644 src/models.py
diff --git a/.gitignore b/.gitignore
index 42e2346..e292dc2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@ __pycache__/
.idea
config/config.ini
config/*.log*
-config/sqlite.db
\ No newline at end of file
+config/sqlite.db
+.DS_STORE
\ No newline at end of file
diff --git a/alembic/README b/alembic/README
new file mode 100644
index 0000000..98e4f9c
--- /dev/null
+++ b/alembic/README
@@ -0,0 +1 @@
+Generic single-database configuration.
\ No newline at end of file
diff --git a/alembic/env.py b/alembic/env.py
new file mode 100644
index 0000000..d622584
--- /dev/null
+++ b/alembic/env.py
@@ -0,0 +1,79 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from src.models import TableNotification, TableSteamItem, TableGiveaway, Base
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+ """Run migrations in 'offline' mode.
+
+ This configures the context with just a URL
+ and not an Engine, though an Engine is acceptable
+ here as well. By skipping the Engine creation
+ we don't even need a DBAPI to be available.
+
+ Calls to context.execute() here emit the given string to the
+ script output.
+
+ """
+ url = config.get_main_option("sqlalchemy.url")
+ context.configure(
+ url=url,
+ target_metadata=target_metadata,
+ literal_binds=True,
+ dialect_opts={"paramstyle": "named"},
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine
+ and associate a connection with the context.
+
+ """
+ connectable = engine_from_config(
+ config.get_section(config.config_ini_section),
+ prefix="sqlalchemy.",
+ poolclass=pool.NullPool,
+ )
+
+ with connectable.connect() as connection:
+ context.configure(
+ connection=connection, target_metadata=target_metadata
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/alembic/script.py.mako b/alembic/script.py.mako
new file mode 100644
index 0000000..2c01563
--- /dev/null
+++ b/alembic/script.py.mako
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+ ${downgrades if downgrades else "pass"}
diff --git a/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py b/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py
new file mode 100644
index 0000000..cbb80be
--- /dev/null
+++ b/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py
@@ -0,0 +1,70 @@
+"""init
+
+Revision ID: 1da33402b659
+Revises:
+Create Date: 2022-05-19 15:50:19.539401
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '1da33402b659'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('notification',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('type', sa.String(length=50), nullable=False),
+ sa.Column('message', sa.String(length=300), nullable=False),
+ sa.Column('medium', sa.String(length=50), nullable=False),
+ sa.Column('success', sa.Boolean(), nullable=False),
+ sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+ nullable=True),
+ sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+ nullable=True),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('steam_item',
+ sa.Column('steam_id', sa.String(length=15), nullable=False),
+ sa.Column('game_name', sa.String(length=200), nullable=False),
+ sa.Column('steam_url', sa.String(length=100), nullable=False),
+ sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+ nullable=True),
+ sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+ nullable=True),
+ sa.PrimaryKeyConstraint('steam_id')
+ )
+ op.create_table('giveaway',
+ sa.Column('giveaway_id', sa.String(length=10), nullable=False),
+ sa.Column('steam_id', sa.Integer(), nullable=False),
+ sa.Column('giveaway_uri', sa.String(length=200), nullable=False),
+ sa.Column('user', sa.String(length=40), nullable=False),
+ sa.Column('giveaway_created_at', sa.DateTime(timezone=True), nullable=False),
+ sa.Column('giveaway_ended_at', sa.DateTime(timezone=True), nullable=False),
+ sa.Column('cost', sa.Integer(), nullable=False),
+ sa.Column('copies', sa.Integer(), nullable=False),
+ sa.Column('contributor_level', sa.Integer(), nullable=False),
+ sa.Column('entered', sa.Boolean(), nullable=False),
+ sa.Column('won', sa.Boolean(), nullable=False),
+ sa.Column('game_entries', sa.Integer(), nullable=False),
+ sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+ nullable=True),
+ sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
+ nullable=True),
+ sa.ForeignKeyConstraint(['steam_id'], ['steam_item.steam_id'], ),
+ sa.PrimaryKeyConstraint('giveaway_id', 'steam_id')
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('giveaway')
+ op.drop_table('steam_item')
+ op.drop_table('notification')
+ # ### end Alembic commands ###
diff --git a/requirements.txt b/requirements.txt
index ec416c3..47e10c5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,5 @@ sqlalchemy_utils==0.38.2
python-dateutil==2.8.2
Flask==2.1.2
Flask-BasicAuth==0.2.0
-pyopenssl==22.0.0
\ No newline at end of file
+pyopenssl==22.0.0
+alembic==1.7.7
\ No newline at end of file
diff --git a/src/tables.py b/src/database.py
similarity index 57%
rename from src/tables.py
rename to src/database.py
index 366339f..d699dc2 100644
--- a/src/tables.py
+++ b/src/database.py
@@ -1,27 +1,35 @@
from datetime import datetime, timedelta
+import sqlalchemy
+from alembic import command
+from alembic.config import Config
from dateutil import tz
-from sqlalchemy import create_engine, Integer, String, Column, DateTime, Boolean, func, ForeignKey
-from sqlalchemy.orm import registry, relationship, Session
-from sqlalchemy_utils import database_exists, create_database
+from sqlalchemy import func
+from sqlalchemy.orm import Session
-mapper_registry = registry()
-mapper_registry.metadata
-Base = mapper_registry.generate_base()
-engine = create_engine('sqlite:///../config/sqlite.db', echo=False)
+import log
+from models import TableNotification, TableGiveaway, TableSteamItem
+
+logger = log.get_logger(__name__)
+engine = None
-class TableNotification(Base):
- __tablename__ = 'notification'
- id = Column(Integer, primary_key=True, nullable=False)
- type = Column(String(50), nullable=False)
- message = Column(String(300), nullable=False)
- medium = Column(String(50), nullable=False)
- success = Column(Boolean, nullable=False)
- created_at = Column(DateTime(timezone=True), server_default=func.now())
- updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+def create_engine(db_url: str):
+ global engine
+ if not engine:
+ engine = sqlalchemy.create_engine(db_url, echo=False)
+ engine.connect()
- __mapper_args__ = {"eager_defaults": True}
+
+def run_migrations(script_location: str, dsn: str) -> None:
+ logger.info('Running DB migrations in %r on %r', script_location, dsn)
+ alembic_cfg = Config()
+ alembic_cfg.set_main_option('script_location', script_location)
+ alembic_cfg.set_main_option('sqlalchemy.url', dsn)
+ command.upgrade(alembic_cfg, 'head')
+
+
+class NotificationHelper:
@classmethod
def insert(cls, type_of_error, message, medium, success):
@@ -61,37 +69,7 @@ class TableNotification(Base):
.all()
-class TableSteamItem(Base):
- __tablename__ = 'steam_item'
- steam_id = Column(String(15), primary_key=True, nullable=False)
- game_name = Column(String(200), nullable=False)
- steam_url = Column(String(100), nullable=False)
- created_at = Column(DateTime(timezone=True), server_default=func.now())
- updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
-
- giveaways = relationship("TableGiveaway", back_populates="steam_item")
-
-
-class TableGiveaway(Base):
- __tablename__ = 'giveaway'
- giveaway_id = Column(String(10), primary_key=True, nullable=False)
- steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
- giveaway_uri = Column(String(200), nullable=False)
- user = Column(String(40), nullable=False)
- giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
- giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
- cost = Column(Integer(), nullable=False)
- copies = Column(Integer(), nullable=False)
- contributor_level = Column(Integer(), nullable=False)
- entered = Column(Boolean(), nullable=False)
- won = Column(Boolean(), nullable=False)
- game_entries = Column(Integer(), nullable=False)
- created_at = Column(DateTime(timezone=True), server_default=func.now())
- updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
-
- steam_item = relationship("TableSteamItem", back_populates="giveaways")
-
- __mapper_args__ = {"eager_defaults": True}
+class GiveawayHelper:
@classmethod
def unix_timestamp_to_utc_datetime(cls, timestamp):
@@ -122,8 +100,8 @@ class TableGiveaway(Base):
steam_id=steam_id,
giveaway_uri=giveaway.giveaway_uri,
user=giveaway.user,
- giveaway_created_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
- giveaway_ended_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
+ giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
+ giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
cost=giveaway.cost,
copies=giveaway.copies,
contributor_level=giveaway.contributor_level,
@@ -135,9 +113,9 @@ class TableGiveaway(Base):
@classmethod
def upsert_giveaway(cls, giveaway, entered):
- result = TableGiveaway.get_by_ids(giveaway)
+ result = GiveawayHelper.get_by_ids(giveaway)
if not result:
- TableGiveaway.insert(giveaway, entered)
+ GiveawayHelper.insert(giveaway, entered)
else:
with Session(engine) as session:
g = TableGiveaway(
@@ -154,14 +132,4 @@ class TableGiveaway(Base):
won=False,
game_entries=giveaway.game_entries)
session.merge(g)
- session.commit()
-
-
-if not database_exists(engine.url):
- create_database(engine.url)
- # emitting DDL
- mapper_registry.metadata.create_all(engine)
- Base.metadata.create_all(engine)
-else:
- # Connect the database if exists.
- engine.connect()
+ session.commit()
\ No newline at end of file
diff --git a/src/enter_giveaways.py b/src/enter_giveaways.py
index 42e1004..81a3f99 100644
--- a/src/enter_giveaways.py
+++ b/src/enter_giveaways.py
@@ -8,8 +8,8 @@ from requests.adapters import HTTPAdapter
from urllib3.util import Retry
import log
+from database import NotificationHelper, GiveawayHelper
from giveaway import Giveaway
-from tables import TableNotification, TableGiveaway
logger = log.get_logger(__name__)
@@ -90,7 +90,7 @@ class EnterGiveaways:
won = soup.select("a[title='Giveaways Won'] div")
if won:
number_won = soup.select_one("a[title='Giveaways Won'] div").text
- won_notifications = TableNotification.get_won_notifications_today()
+ won_notifications = NotificationHelper.get_won_notifications_today()
if won_notifications and len(won_notifications) >= 1:
logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting "
"to be received. Doing nothing.")
@@ -203,15 +203,15 @@ class EnterGiveaways:
if if_enter_giveaway:
res = self.enter_giveaway(giveaway)
if res:
- TableGiveaway.upsert_giveaway(giveaway, True)
+ GiveawayHelper.upsert_giveaway(giveaway, True)
self.points -= int(giveaway.cost)
txt = f"✅ Entered giveaway '{giveaway.game_name}'"
logger.info(txt)
sleep(randint(4, 15))
else:
- TableGiveaway.upsert_giveaway(giveaway, False)
+ GiveawayHelper.upsert_giveaway(giveaway, False)
else:
- TableGiveaway.upsert_giveaway(giveaway, False)
+ GiveawayHelper.upsert_giveaway(giveaway, False)
# if we are on any filter type except New and we get to a giveaway that exceeds our
# max time left amount, then we don't need to continue to look at giveaways as any
# after this point will also exceed the max time left
diff --git a/src/models.py b/src/models.py
new file mode 100644
index 0000000..f9174f8
--- /dev/null
+++ b/src/models.py
@@ -0,0 +1,52 @@
+from sqlalchemy import Integer, String, Column, DateTime, Boolean, func, ForeignKey
+from sqlalchemy.orm import registry, relationship
+
+mapper_registry = registry()
+metadata = mapper_registry.metadata
+Base = mapper_registry.generate_base()
+
+
+class TableNotification(Base):
+ __tablename__ = 'notification'
+ id = Column(Integer, primary_key=True, nullable=False)
+ type = Column(String(50), nullable=False)
+ message = Column(String(300), nullable=False)
+ medium = Column(String(50), nullable=False)
+ success = Column(Boolean, nullable=False)
+ created_at = Column(DateTime(timezone=True), server_default=func.now())
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+ __mapper_args__ = {"eager_defaults": True}
+
+
+class TableSteamItem(Base):
+ __tablename__ = 'steam_item'
+ steam_id = Column(String(15), primary_key=True, nullable=False)
+ game_name = Column(String(200), nullable=False)
+ steam_url = Column(String(100), nullable=False)
+ created_at = Column(DateTime(timezone=True), server_default=func.now())
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+ giveaways = relationship("TableGiveaway", back_populates="steam_item")
+
+
+class TableGiveaway(Base):
+ __tablename__ = 'giveaway'
+ giveaway_id = Column(String(10), primary_key=True, nullable=False)
+ steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
+ giveaway_uri = Column(String(200), nullable=False)
+ user = Column(String(40), nullable=False)
+ giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
+ giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
+ cost = Column(Integer(), nullable=False)
+ copies = Column(Integer(), nullable=False)
+ contributor_level = Column(Integer(), nullable=False)
+ entered = Column(Boolean(), nullable=False)
+ won = Column(Boolean(), nullable=False)
+ game_entries = Column(Integer(), nullable=False)
+ created_at = Column(DateTime(timezone=True), server_default=func.now())
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+ steam_item = relationship("TableSteamItem", back_populates="giveaways")
+
+ __mapper_args__ = {"eager_defaults": True}
diff --git a/src/notification.py b/src/notification.py
index 16b1f86..57fb3b0 100644
--- a/src/notification.py
+++ b/src/notification.py
@@ -2,7 +2,7 @@ import http.client
import urllib
import log
-from tables import TableNotification
+from database import NotificationHelper
logger = log.get_logger(__name__)
@@ -48,4 +48,4 @@ class Notification:
else:
logger.error(f"Pushover notification failed. Code {response.getcode()}: {response.read().decode()}")
success = False
- TableNotification.insert(type_of_error, f"{message}", 'pushover', success)
+ NotificationHelper.insert(type_of_error, f"{message}", 'pushover', success)
diff --git a/src/run.py b/src/run.py
index 320ca17..51a68ef 100644
--- a/src/run.py
+++ b/src/run.py
@@ -5,26 +5,30 @@ from config_reader import ConfigReader, ConfigException
from enter_giveaways import SteamGiftsException
from giveaway_thread import GiveawayThread
from notification import Notification
+from database import run_migrations, create_engine
from webserver_thread import WebServerThread
logger = log.get_logger(__name__)
+config_file_name = '../config/config.ini'
+db_url = 'sqlite:///../config/sqlite.db'
+alembic_migration_files = '../alembic'
def run():
logger.info("Starting Steamgifts bot.")
- file_name = '../config/config.ini'
+
config = None
try:
- config = ConfigReader(file_name)
+ config = ConfigReader(config_file_name)
except IOError:
- txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out."
+ txt = f"{config_file_name} doesn't exist. Rename {config_file_name}.example to {config_file_name} and fill out."
logger.warning(txt)
exit(-1)
except ConfigException as e:
logger.error(e)
exit(-1)
- config.read(file_name)
+ config.read(config_file_name)
notification = Notification(config['NOTIFICATIONS'].get('notification.prefix'))
pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled')
@@ -68,4 +72,6 @@ if __name__ == '__main__':
|___/
-------------------------------------------------------------------------------------
""")
+ run_migrations(alembic_migration_files, db_url)
+ create_engine(db_url)
run()
From 5c30089b72024909ec3fa7443ad0c3c2d500f2ec Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Thu, 19 May 2022 18:23:35 -0400
Subject: [PATCH 17/37] bad reference
---
src/database.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/database.py b/src/database.py
index d699dc2..7703781 100644
--- a/src/database.py
+++ b/src/database.py
@@ -123,8 +123,8 @@ class GiveawayHelper:
steam_id=result[0].steam_id,
giveaway_uri=giveaway.giveaway_uri,
user=giveaway.user,
- giveaway_created_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
- giveaway_ended_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
+ giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
+ giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
cost=giveaway.cost,
copies=giveaway.copies,
contributor_level=giveaway.contributor_level,
From 3670d1522f77184e7eea3183c4b89653badc8caf Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Thu, 19 May 2022 22:01:12 -0400
Subject: [PATCH 18/37] migrations
- handles the case where the first db that was not versioned with alembic by applying the version info
- will migrate on start to ensure db is always caught up with the latest
---
src/database.py | 31 +++++++++++++++++++++++++++----
1 file changed, 27 insertions(+), 4 deletions(-)
diff --git a/src/database.py b/src/database.py
index 7703781..58f5542 100644
--- a/src/database.py
+++ b/src/database.py
@@ -6,6 +6,7 @@ from alembic.config import Config
from dateutil import tz
from sqlalchemy import func
from sqlalchemy.orm import Session
+from sqlalchemy_utils import database_exists
import log
from models import TableNotification, TableGiveaway, TableSteamItem
@@ -19,14 +20,36 @@ def create_engine(db_url: str):
if not engine:
engine = sqlalchemy.create_engine(db_url, echo=False)
engine.connect()
+ return engine
-def run_migrations(script_location: str, dsn: str) -> None:
- logger.info('Running DB migrations in %r on %r', script_location, dsn)
+def run_migrations(script_location: str, db_url: str) -> None:
+ logger.debug('Running DB migrations in %r on %r', script_location, db_url)
alembic_cfg = Config()
alembic_cfg.set_main_option('script_location', script_location)
- alembic_cfg.set_main_option('sqlalchemy.url', dsn)
- command.upgrade(alembic_cfg, 'head')
+ alembic_cfg.set_main_option('sqlalchemy.url', db_url)
+
+ if not database_exists(db_url):
+ logger.debug(f"'{db_url}' does not exist. Running normal migration to create db and tables." )
+ command.upgrade(alembic_cfg, 'head')
+ if database_exists(db_url):
+ logger.debug(f"'{db_url} exists.")
+ e = create_engine(db_url)
+ insp = sqlalchemy.inspect(e)
+ alembic_version_table_name = 'alembic_version'
+ has_alembic_table = insp.has_table(alembic_version_table_name)
+ if has_alembic_table:
+ logger.debug(f"Table '{alembic_version_table_name}' exists.")
+ else:
+ logger.debug(f"Table '{alembic_version_table_name}' doesn't exist so assuming it was created pre-alembic "
+ f"setup. Setting the version to the first version created prior to alembic setup.")
+ alembic_first_ref = '1da33402b659'
+ command.stamp(alembic_cfg, alembic_first_ref)
+ logger.debug("Running migration.")
+ command.upgrade(alembic_cfg, 'head')
+
+
+
class NotificationHelper:
From a101447eeb027b08b097df1666ca8b1ff66c1da4 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 20 May 2022 12:20:08 -0400
Subject: [PATCH 19/37] reorganization
- Had to reorganize so that Alembic's env.py could reference the tables in both local environment and a docker environment without manually modifying the python sys path. Maybe there is a better way idk
---
.dockerignore | 4 +-
Dockerfile | 20 +++++---
README.md | 3 +-
main.py | 5 ++
{alembic => src/alembic}/README | 0
{alembic => src/alembic}/env.py | 4 +-
{alembic => src/alembic}/script.py.mako | 0
.../2022_05_19-15_50_19-1da33402b659_init.py | 0
src/{ => bot}/__init__.py | 0
src/{ => bot}/config_reader.py | 4 +-
src/{ => bot}/database.py | 17 +++----
src/{ => bot}/enter_giveaways.py | 8 +--
src/{ => bot}/giveaway.py | 4 +-
src/{ => bot}/giveaway_thread.py | 6 +--
src/{ => bot}/log.py | 5 +-
src/{ => bot}/models.py | 0
src/{ => bot}/notification.py | 6 +--
src/{ => bot}/run.py | 50 ++++++++++---------
src/{ => bot}/static/css/main.css | 0
src/{ => bot}/templates/configuration.html | 0
src/{ => bot}/templates/log.html | 0
src/{ => bot}/webserver_thread.py | 5 +-
22 files changed, 77 insertions(+), 64 deletions(-)
create mode 100644 main.py
rename {alembic => src/alembic}/README (100%)
rename {alembic => src/alembic}/env.py (96%)
rename {alembic => src/alembic}/script.py.mako (100%)
rename {alembic => src/alembic}/versions/2022_05_19-15_50_19-1da33402b659_init.py (100%)
rename src/{ => bot}/__init__.py (100%)
rename src/{ => bot}/config_reader.py (98%)
rename src/{ => bot}/database.py (95%)
rename src/{ => bot}/enter_giveaways.py (96%)
rename src/{ => bot}/giveaway.py (98%)
rename src/{ => bot}/giveaway_thread.py (96%)
rename src/{ => bot}/log.py (72%)
rename src/{ => bot}/models.py (100%)
rename src/{ => bot}/notification.py (94%)
rename src/{ => bot}/run.py (50%)
rename src/{ => bot}/static/css/main.css (100%)
rename src/{ => bot}/templates/configuration.html (100%)
rename src/{ => bot}/templates/log.html (100%)
rename src/{ => bot}/webserver_thread.py (97%)
diff --git a/.dockerignore b/.dockerignore
index d17bf38..fb2bb68 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,6 @@
config/config.ini
config/debug.log
config/sqlite.db
-config/info.log
\ No newline at end of file
+config/info.log
+alembic.ini
+README.ini
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 7835b0f..a3c179c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,19 +1,25 @@
FROM python:3.9-alpine
-RUN mkdir -p /app
-WORKDIR /app
-
RUN apk add tzdata build-base libffi-dev py3-cffi --no-cache
+RUN mkdir -p /app/src
+WORKDIR /app
+
ENV TZ=America/New_York
-ENV VIRTUAL_ENV=/app/env
+ENV VIRTUAL_ENV=/src/env
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV BOT_CONFIG_DIR="../config"
+ENV BOT_DB_URL="sqlite:///../config/sqlite.db"
+ENV BOT_ALEMBIC_CONFIG="./src/alembic"
+
RUN python -m venv $VIRTUAL_ENV
-COPY requirements.txt .
+COPY requirements.txt /app/
RUN pip3 install -r requirements.txt
-COPY ./src/ /app/
+
+COPY ./src/ /app/src/
+COPY main.py /app/
VOLUME /config
-CMD ["python3", "run.py"]
+CMD ["python3", "main.py"]
diff --git a/README.md b/README.md
index 7b9b8c3..66151da 100644
--- a/README.md
+++ b/README.md
@@ -20,8 +20,7 @@ The bot is specially designed for [SteamGifts.com](https://www.steamgifts.com/)
python -m venv env
source env/bin/activate
pip install -r requirements.txt
-cd src
-python run.py
+python main.py
```
### Docker
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..3131670
--- /dev/null
+++ b/main.py
@@ -0,0 +1,5 @@
+from src.bot import run
+
+
+if __name__ == '__main__':
+ run.entry()
\ No newline at end of file
diff --git a/alembic/README b/src/alembic/README
similarity index 100%
rename from alembic/README
rename to src/alembic/README
diff --git a/alembic/env.py b/src/alembic/env.py
similarity index 96%
rename from alembic/env.py
rename to src/alembic/env.py
index d622584..f5b2ad8 100644
--- a/alembic/env.py
+++ b/src/alembic/env.py
@@ -18,9 +18,11 @@ if config.config_file_name is not None:
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
-from src.models import TableNotification, TableSteamItem, TableGiveaway, Base
+from src.bot.models import TableNotification, TableSteamItem, TableGiveaway, Base
+
target_metadata = Base.metadata
+
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
diff --git a/alembic/script.py.mako b/src/alembic/script.py.mako
similarity index 100%
rename from alembic/script.py.mako
rename to src/alembic/script.py.mako
diff --git a/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py b/src/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py
similarity index 100%
rename from alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py
rename to src/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py
diff --git a/src/__init__.py b/src/bot/__init__.py
similarity index 100%
rename from src/__init__.py
rename to src/bot/__init__.py
diff --git a/src/config_reader.py b/src/bot/config_reader.py
similarity index 98%
rename from src/config_reader.py
rename to src/bot/config_reader.py
index d0e6520..5450023 100644
--- a/src/config_reader.py
+++ b/src/bot/config_reader.py
@@ -1,9 +1,9 @@
from configparser import ConfigParser
from random import randint, randrange
-import log
+from .log import get_logger
-logger = log.get_logger(__name__)
+logger = get_logger(__name__)
class ConfigException(Exception):
diff --git a/src/database.py b/src/bot/database.py
similarity index 95%
rename from src/database.py
rename to src/bot/database.py
index 58f5542..849fcad 100644
--- a/src/database.py
+++ b/src/bot/database.py
@@ -1,3 +1,4 @@
+import os
from datetime import datetime, timedelta
import sqlalchemy
@@ -8,18 +9,15 @@ from sqlalchemy import func
from sqlalchemy.orm import Session
from sqlalchemy_utils import database_exists
-import log
-from models import TableNotification, TableGiveaway, TableSteamItem
+from .log import get_logger
+from .models import TableNotification, TableGiveaway, TableSteamItem
-logger = log.get_logger(__name__)
-engine = None
+logger = get_logger(__name__)
+engine = sqlalchemy.create_engine(f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}", echo=False)
+engine.connect()
def create_engine(db_url: str):
- global engine
- if not engine:
- engine = sqlalchemy.create_engine(db_url, echo=False)
- engine.connect()
return engine
@@ -49,9 +47,6 @@ def run_migrations(script_location: str, db_url: str) -> None:
command.upgrade(alembic_cfg, 'head')
-
-
-
class NotificationHelper:
@classmethod
diff --git a/src/enter_giveaways.py b/src/bot/enter_giveaways.py
similarity index 96%
rename from src/enter_giveaways.py
rename to src/bot/enter_giveaways.py
index 81a3f99..bcd25c8 100644
--- a/src/enter_giveaways.py
+++ b/src/bot/enter_giveaways.py
@@ -7,11 +7,11 @@ from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from urllib3.util import Retry
-import log
-from database import NotificationHelper, GiveawayHelper
-from giveaway import Giveaway
+from .log import get_logger
+from .database import NotificationHelper, GiveawayHelper
+from .giveaway import Giveaway
-logger = log.get_logger(__name__)
+logger = get_logger(__name__)
class SteamGiftsException(Exception):
diff --git a/src/giveaway.py b/src/bot/giveaway.py
similarity index 98%
rename from src/giveaway.py
rename to src/bot/giveaway.py
index 936b846..ae23574 100644
--- a/src/giveaway.py
+++ b/src/bot/giveaway.py
@@ -1,8 +1,8 @@
import re
-import log
+from .log import get_logger
import time
-logger = log.get_logger(__name__)
+logger = get_logger(__name__)
class Giveaway:
diff --git a/src/giveaway_thread.py b/src/bot/giveaway_thread.py
similarity index 96%
rename from src/giveaway_thread.py
rename to src/bot/giveaway_thread.py
index d1076e2..be15ae9 100644
--- a/src/giveaway_thread.py
+++ b/src/bot/giveaway_thread.py
@@ -7,10 +7,10 @@ from time import sleep
from dateutil import tz
-import log
-from enter_giveaways import EnterGiveaways
+from .log import get_logger
+from .enter_giveaways import EnterGiveaways
-logger = log.get_logger(__name__)
+logger = get_logger(__name__)
class GiveawayThread(threading.Thread):
diff --git a/src/log.py b/src/bot/log.py
similarity index 72%
rename from src/log.py
rename to src/bot/log.py
index 36c702c..3228982 100644
--- a/src/log.py
+++ b/src/bot/log.py
@@ -1,11 +1,12 @@
import logging
+import os
from logging.handlers import RotatingFileHandler
# log info level logs to stdout and debug to debug file
log_format = "%(levelname)s %(asctime)s - %(message)s"
logging.basicConfig(
- handlers=[RotatingFileHandler('../config/debug.log', maxBytes=500000, backupCount=10)],
+ handlers=[RotatingFileHandler(f"{os.getenv('BOT_CONFIG_DIR', './config')}/debug.log", maxBytes=500000, backupCount=10)],
level=logging.DEBUG,
format=log_format)
@@ -14,7 +15,7 @@ console_output.setLevel(logging.INFO)
console_format = logging.Formatter(log_format)
console_output.setFormatter(console_format)
-info_log_file = RotatingFileHandler('../config/info.log', maxBytes=100000, backupCount=10)
+info_log_file = RotatingFileHandler(f"{os.getenv('BOT_CONFIG_DIR', './config')}/info.log", maxBytes=100000, backupCount=10)
info_log_file.setLevel(logging.INFO)
info_log_format = logging.Formatter(log_format)
info_log_file.setFormatter(info_log_format)
diff --git a/src/models.py b/src/bot/models.py
similarity index 100%
rename from src/models.py
rename to src/bot/models.py
diff --git a/src/notification.py b/src/bot/notification.py
similarity index 94%
rename from src/notification.py
rename to src/bot/notification.py
index 57fb3b0..7a3089b 100644
--- a/src/notification.py
+++ b/src/bot/notification.py
@@ -1,10 +1,10 @@
import http.client
import urllib
-import log
-from database import NotificationHelper
+from .log import get_logger
+from .database import NotificationHelper
-logger = log.get_logger(__name__)
+logger = get_logger(__name__)
class Notification:
diff --git a/src/run.py b/src/bot/run.py
similarity index 50%
rename from src/run.py
rename to src/bot/run.py
index 51a68ef..c5993fb 100644
--- a/src/run.py
+++ b/src/bot/run.py
@@ -1,17 +1,18 @@
+import os
from time import sleep
-import log
-from config_reader import ConfigReader, ConfigException
-from enter_giveaways import SteamGiftsException
-from giveaway_thread import GiveawayThread
-from notification import Notification
-from database import run_migrations, create_engine
-from webserver_thread import WebServerThread
+from .log import get_logger
+from .config_reader import ConfigReader, ConfigException
+from .enter_giveaways import SteamGiftsException
+from .giveaway_thread import GiveawayThread
+from .notification import Notification
+from .database import run_migrations, create_engine
+from .webserver_thread import WebServerThread
-logger = log.get_logger(__name__)
-config_file_name = '../config/config.ini'
-db_url = 'sqlite:///../config/sqlite.db'
-alembic_migration_files = '../alembic'
+logger = get_logger(__name__)
+config_file_name = f"{os.getenv('BOT_CONFIG_DIR', './config')}/config.ini"
+db_url = f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}"
+alembic_migration_files = os.getenv('BOT_ALEMBIC_CONFIG_DIR', './src/alembic')
def run():
@@ -59,19 +60,22 @@ def run():
exit(-1)
-if __name__ == '__main__':
+def entry():
logger.info("""
- -------------------------------------------------------------------------------------
- _____ _ _ __ _ ____ _
- / ____|| | (_) / _|| | | _ \ | |
- | (___ | |_ ___ __ _ _ __ ___ __ _ _ | |_ | |_ ___ | |_) | ___ | |_
- \___ \ | __|/ _ \ / _` || '_ ` _ \ / _` || || _|| __|/ __| | _ < / _ \ | __|
- ____) || |_| __/| (_| || | | | | || (_| || || | | |_ \__ \ | |_) || (_) || |_
- |_____/ \__|\___| \__,_||_| |_| |_| \__, ||_||_| \__||___/ |____/ \___/ \__|
- __/ |
- |___/
- -------------------------------------------------------------------------------------
- """)
+ -------------------------------------------------------------------------------------
+ _____ _ _ __ _ ____ _
+ / ____|| | (_) / _|| | | _ \ | |
+ | (___ | |_ ___ __ _ _ __ ___ __ _ _ | |_ | |_ ___ | |_) | ___ | |_
+ \___ \ | __|/ _ \ / _` || '_ ` _ \ / _` || || _|| __|/ __| | _ < / _ \ | __|
+ ____) || |_| __/| (_| || | | | | || (_| || || | | |_ \__ \ | |_) || (_) || |_
+ |_____/ \__|\___| \__,_||_| |_| |_| \__, ||_||_| \__||___/ |____/ \___/ \__|
+ __/ |
+ |___/
+ -------------------------------------------------------------------------------------
+ """)
run_migrations(alembic_migration_files, db_url)
create_engine(db_url)
run()
+
+if __name__ == '__main__':
+ entry()
diff --git a/src/static/css/main.css b/src/bot/static/css/main.css
similarity index 100%
rename from src/static/css/main.css
rename to src/bot/static/css/main.css
diff --git a/src/templates/configuration.html b/src/bot/templates/configuration.html
similarity index 100%
rename from src/templates/configuration.html
rename to src/bot/templates/configuration.html
diff --git a/src/templates/log.html b/src/bot/templates/log.html
similarity index 100%
rename from src/templates/log.html
rename to src/bot/templates/log.html
diff --git a/src/webserver_thread.py b/src/bot/webserver_thread.py
similarity index 97%
rename from src/webserver_thread.py
rename to src/bot/webserver_thread.py
index 348f0bf..c62ebf0 100644
--- a/src/webserver_thread.py
+++ b/src/bot/webserver_thread.py
@@ -1,12 +1,11 @@
import threading
from threading import Thread
-from time import sleep
from flask_basicauth import BasicAuth
-import log
+from .log import get_logger
-logger = log.get_logger(__name__)
+logger = get_logger(__name__)
class WebServerThread(threading.Thread):
From 5e198e2e9d6d0d640b4647d4ea9cd86e32ad8ad6 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 20 May 2022 16:56:36 -0400
Subject: [PATCH 20/37] readme update
---
README.md | 28 +++++++++++++++++++++-------
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md
index 66151da..5db342e 100644
--- a/README.md
+++ b/README.md
@@ -4,15 +4,32 @@ The bot is specially designed for [SteamGifts.com](https://www.steamgifts.com/)
### Features
- Automatically enters giveaways.
- Undetectable.
-- Сonfigurable.
+- Сonfigurable
+ - Can look at your steam wishlist games or the main page for games to enter
+ - When evaluating a giveaway
+ - `max_time_left` - if the time left on giveaway is > max time left, then don't enter it
+ - `max_entries` - if the entries on a giveaway are > than max entries, then don't enter it
+ - `minimum_points` - minimum number of points **in your account** needed before considering any giveaway
+ - `minimum_game_points` - if steamgifts.com point cost (ex. 1P, 5P, etc) is below this, don't enter it
+ - `blacklist_keywords` - if the giveaway name contains any of these words, don't enter it. this list can be blank.
+ - Notifications
+ - A pushover notifications can be sent to you when a win is detected.
+ - Webserver - A simple, simple, simple webserver than can be enabled (disabled by default) to show the config and logs
+ - `web.host` - the IP to listen on (ex. localhost, 0.0.0.0, 192.168.1.1, etc)
+ - `web.port` - the port to listen on
+ - `web.app_root` - the folder to serve up which can be used for reverse proxying this behind nginx/apache/etc
+ - `web.ssl` - if the traffic will be encrypted (http or https) using a self-signed cert
+ - `web.basic_auth` - simple basic auth settings can be enabled
- Sleeps to restock the points.
- Can run 24/7.
## Instructions
- 1. Rename `config/config.ini.example` to `config/config.ini`.
- 2. Add your PHPSESSION cookie to it.
- 3. Modifying the other settings is optional.
+1. Sign in on SteamGifts.com by Steam.
+2. Find PHPSESSID cookie in your browser.
+3. Rename `config/config.ini.example` to `config/config.ini`.
+4. Add your PHPSESSION cookie to `cookie` in `config/config.ini`
+5. Modifying the other settings is optional as defaults are set.
### Run from sources
@@ -41,6 +58,3 @@ docker run --name steamgifts -e TZ=America/New_York -d -v /path/to/the/config/fo
```
-
-### Help
-Please leave your feedback and bugs in `Issues` page.
From 3005228416cb769603e5e6cae557f76870120234 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 20 May 2022 17:37:43 -0400
Subject: [PATCH 21/37] more logging
---
src/bot/giveaway.py | 1 +
src/bot/log.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/bot/giveaway.py b/src/bot/giveaway.py
index ae23574..64a9dbf 100644
--- a/src/bot/giveaway.py
+++ b/src/bot/giveaway.py
@@ -26,6 +26,7 @@ class Giveaway:
self.time_created_string = None
self.time_created_in_minutes = None
+ logger.debug(f"Giveaway html: {soup_item}")
icons = soup_item.select('a.giveaway__icon')
self.steam_url = icons[0]['href']
self.steam_app_id = self.get_steam_app_id(self.steam_url)
diff --git a/src/bot/log.py b/src/bot/log.py
index 3228982..cbdce24 100644
--- a/src/bot/log.py
+++ b/src/bot/log.py
@@ -4,20 +4,20 @@ from logging.handlers import RotatingFileHandler
# log info level logs to stdout and debug to debug file
-log_format = "%(levelname)s %(asctime)s - %(message)s"
+debug_log_format = "%(levelname)s %(asctime)s - [%(filename)s:%(lineno)d] - %(message)s"
logging.basicConfig(
handlers=[RotatingFileHandler(f"{os.getenv('BOT_CONFIG_DIR', './config')}/debug.log", maxBytes=500000, backupCount=10)],
level=logging.DEBUG,
- format=log_format)
+ format=debug_log_format)
console_output = logging.StreamHandler()
console_output.setLevel(logging.INFO)
-console_format = logging.Formatter(log_format)
+console_format = logging.Formatter(debug_log_format)
console_output.setFormatter(console_format)
info_log_file = RotatingFileHandler(f"{os.getenv('BOT_CONFIG_DIR', './config')}/info.log", maxBytes=100000, backupCount=10)
info_log_file.setLevel(logging.INFO)
-info_log_format = logging.Formatter(log_format)
+info_log_format = logging.Formatter(debug_log_format)
info_log_file.setFormatter(info_log_format)
logging.root.addHandler(console_output)
From 6a5e679aecfc37ff17794ebbf816480da25b1e73 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Fri, 20 May 2022 23:42:42 -0400
Subject: [PATCH 22/37] fix ref
---
src/bot/templates/configuration.html | 3 ++-
src/bot/templates/log.html | 3 ++-
src/bot/webserver_thread.py | 15 +++++++++++----
3 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/src/bot/templates/configuration.html b/src/bot/templates/configuration.html
index db2c119..7268e62 100644
--- a/src/bot/templates/configuration.html
+++ b/src/bot/templates/configuration.html
@@ -11,7 +11,8 @@
diff --git a/src/bot/templates/log.html b/src/bot/templates/log.html
index 80b7b71..9b486de 100644
--- a/src/bot/templates/log.html
+++ b/src/bot/templates/log.html
@@ -11,7 +11,8 @@
diff --git a/src/bot/webserver_thread.py b/src/bot/webserver_thread.py
index c62ebf0..6f6aeba 100644
--- a/src/bot/webserver_thread.py
+++ b/src/bot/webserver_thread.py
@@ -1,3 +1,4 @@
+import os
import threading
from threading import Thread
@@ -38,13 +39,19 @@ class WebServerThread(threading.Thread):
@app.route(f"{self.app_root}")
def config():
- with open('../config/config.ini', 'r') as f:
+ with open(f"{os.getenv('BOT_CONFIG_DIR', './config')}/config.ini", 'r') as f:
content = f.read()
return render_template('configuration.html', content=content)
- @app.route(f"{self.app_root}log")
- def logs():
- with open('../config/info.log', 'r') as f:
+ @app.route(f"{self.app_root}log_info")
+ def log_info():
+ with open(f"{os.getenv('BOT_CONFIG_DIR', './config')}/info.log", 'r') as f:
+ content = f.read()
+ return render_template('log.html', content=content)
+
+ @app.route(f"{self.app_root}log_debug")
+ def log_debug():
+ with open(f"{os.getenv('BOT_CONFIG_DIR', './config')}/debug.log", 'r') as f:
content = f.read()
return render_template('log.html', content=content)
From 9f2b2e7be2f066b4d6c0e1d899b51e524bfd57a6 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sat, 21 May 2022 13:12:04 -0400
Subject: [PATCH 23/37] removed unused column
---
.dockerignore | 1 -
.gitignore | 1 -
alembic.ini | 102 ++++++++++++++++++
...0_25_41-15c028536ef5_won_column_removed.py | 31 ++++++
src/bot/database.py | 2 -
src/bot/models.py | 1 -
6 files changed, 133 insertions(+), 5 deletions(-)
create mode 100644 alembic.ini
create mode 100644 src/alembic/versions/2022_05_21-10_25_41-15c028536ef5_won_column_removed.py
diff --git a/.dockerignore b/.dockerignore
index fb2bb68..c56db2a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,4 @@ config/config.ini
config/debug.log
config/sqlite.db
config/info.log
-alembic.ini
README.ini
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index e292dc2..6dc7536 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,5 @@
env/
venv/
-*.ini
__pycache__/
.idea
config/config.ini
diff --git a/alembic.ini b/alembic.ini
new file mode 100644
index 0000000..06ec989
--- /dev/null
+++ b/alembic.ini
@@ -0,0 +1,102 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = src/alembic
+
+# template used to generate migration files
+file_template = %%(year)d_%%(month).2d_%%(day).2d-%%(hour).2d_%%(minute).2d_%%(second).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions. When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions:src/alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = sqlite:///config/sqlite.db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts. See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/src/alembic/versions/2022_05_21-10_25_41-15c028536ef5_won_column_removed.py b/src/alembic/versions/2022_05_21-10_25_41-15c028536ef5_won_column_removed.py
new file mode 100644
index 0000000..7abd9af
--- /dev/null
+++ b/src/alembic/versions/2022_05_21-10_25_41-15c028536ef5_won_column_removed.py
@@ -0,0 +1,31 @@
+"""won column removed
+
+Revision ID: 15c028536ef5
+Revises: 1da33402b659
+Create Date: 2022-05-21 10:25:41.647723
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '15c028536ef5'
+down_revision = '1da33402b659'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.drop_column('giveaway', 'won')
+
+
+def downgrade():
+ # add columns as nullable as existing records don't have that column value set
+ op.add_column('giveaway', sa.Column('won', sa.Boolean(), nullable=True))
+ # set value on new columns for all records
+ with op.get_context().autocommit_block():
+ op.execute("UPDATE giveaway SET won=false WHERE won is null;")
+ # SQLite doesn't support ALTERs so alembic uses a batch mode
+ # Set columns to non-nullable now that all records have a value
+ with op.batch_alter_table('giveaway') as batch_op:
+ batch_op.alter_column('won', nullable=False)
diff --git a/src/bot/database.py b/src/bot/database.py
index 849fcad..ec6beea 100644
--- a/src/bot/database.py
+++ b/src/bot/database.py
@@ -124,7 +124,6 @@ class GiveawayHelper:
copies=giveaway.copies,
contributor_level=giveaway.contributor_level,
entered=entered,
- won=False,
game_entries=giveaway.game_entries)
session.add(g)
session.commit()
@@ -147,7 +146,6 @@ class GiveawayHelper:
copies=giveaway.copies,
contributor_level=giveaway.contributor_level,
entered=entered,
- won=False,
game_entries=giveaway.game_entries)
session.merge(g)
session.commit()
\ No newline at end of file
diff --git a/src/bot/models.py b/src/bot/models.py
index f9174f8..c16b9f2 100644
--- a/src/bot/models.py
+++ b/src/bot/models.py
@@ -42,7 +42,6 @@ class TableGiveaway(Base):
copies = Column(Integer(), nullable=False)
contributor_level = Column(Integer(), nullable=False)
entered = Column(Boolean(), nullable=False)
- won = Column(Boolean(), nullable=False)
game_entries = Column(Integer(), nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
From 7c6493cea87586f3ac92c8479765f3dd89b248b2 Mon Sep 17 00:00:00 2001
From: mcinj <98779161+mcinj@users.noreply.github.com>
Date: Sun, 22 May 2022 14:20:44 -0400
Subject: [PATCH 24/37] re-org and db views
---
.gitignore | 2 +-
main.py | 2 +-
src/bot/database.py | 25 +++++---
src/{bot => }/run.py | 18 +++---
src/{bot => web}/static/css/main.css | 0
src/{bot => web}/templates/configuration.html | 4 +-
src/web/templates/giveaways.html | 63 +++++++++++++++++++
src/{bot => web}/templates/log.html | 4 +-
src/web/templates/notifications.html | 51 +++++++++++++++
src/{bot => web}/webserver_thread.py | 18 ++++--
10 files changed, 160 insertions(+), 27 deletions(-)
rename src/{bot => }/run.py (86%)
rename src/{bot => web}/static/css/main.css (100%)
rename src/{bot => web}/templates/configuration.html (74%)
create mode 100644 src/web/templates/giveaways.html
rename src/{bot => web}/templates/log.html (74%)
create mode 100644 src/web/templates/notifications.html
rename src/{bot => web}/webserver_thread.py (76%)
diff --git a/.gitignore b/.gitignore
index 6dc7536..8304bd8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,5 +4,5 @@ __pycache__/
.idea
config/config.ini
config/*.log*
-config/sqlite.db
+config/sqlite*
.DS_STORE
\ No newline at end of file
diff --git a/main.py b/main.py
index 3131670..d19fa56 100644
--- a/main.py
+++ b/main.py
@@ -1,4 +1,4 @@
-from src.bot import run
+from src import run
if __name__ == '__main__':
diff --git a/src/bot/database.py b/src/bot/database.py
index ec6beea..e3fc60e 100644
--- a/src/bot/database.py
+++ b/src/bot/database.py
@@ -6,7 +6,7 @@ from alembic import command
from alembic.config import Config
from dateutil import tz
from sqlalchemy import func
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, joinedload
from sqlalchemy_utils import database_exists
from .log import get_logger
@@ -17,11 +17,7 @@ engine = sqlalchemy.create_engine(f"{os.getenv('BOT_DB_URL', 'sqlite:///./config
engine.connect()
-def create_engine(db_url: str):
- return engine
-
-
-def run_migrations(script_location: str, db_url: str) -> None:
+def run_db_migrations(script_location: str, db_url: str) -> None:
logger.debug('Running DB migrations in %r on %r', script_location, db_url)
alembic_cfg = Config()
alembic_cfg.set_main_option('script_location', script_location)
@@ -30,10 +26,9 @@ def run_migrations(script_location: str, db_url: str) -> None:
if not database_exists(db_url):
logger.debug(f"'{db_url}' does not exist. Running normal migration to create db and tables." )
command.upgrade(alembic_cfg, 'head')
- if database_exists(db_url):
- logger.debug(f"'{db_url} exists.")
- e = create_engine(db_url)
- insp = sqlalchemy.inspect(e)
+ elif database_exists(db_url):
+ logger.debug(f"'{db_url}' exists.")
+ insp = sqlalchemy.inspect(engine)
alembic_version_table_name = 'alembic_version'
has_alembic_table = insp.has_table(alembic_version_table_name)
if has_alembic_table:
@@ -49,6 +44,11 @@ def run_migrations(script_location: str, db_url: str) -> None:
class NotificationHelper:
+ @classmethod
+ def get(cls):
+ with Session(engine) as session:
+ return session.query(TableNotification).order_by(TableNotification.created_at.desc()).all()
+
@classmethod
def insert(cls, type_of_error, message, medium, success):
with Session(engine) as session:
@@ -89,6 +89,11 @@ class NotificationHelper:
class GiveawayHelper:
+ @classmethod
+ def get(cls):
+ with Session(engine) as session:
+ return session.query(TableGiveaway).options(joinedload('steam_item')).order_by(TableGiveaway.giveaway_ended_at.desc()).all()
+
@classmethod
def unix_timestamp_to_utc_datetime(cls, timestamp):
return datetime.utcfromtimestamp(timestamp)
diff --git a/src/bot/run.py b/src/run.py
similarity index 86%
rename from src/bot/run.py
rename to src/run.py
index c5993fb..2375878 100644
--- a/src/bot/run.py
+++ b/src/run.py
@@ -1,13 +1,13 @@
import os
from time import sleep
-from .log import get_logger
-from .config_reader import ConfigReader, ConfigException
-from .enter_giveaways import SteamGiftsException
-from .giveaway_thread import GiveawayThread
-from .notification import Notification
-from .database import run_migrations, create_engine
-from .webserver_thread import WebServerThread
+from src.bot.log import get_logger
+from src.bot.config_reader import ConfigReader, ConfigException
+from src.bot.enter_giveaways import SteamGiftsException
+from src.bot.giveaway_thread import GiveawayThread
+from src.bot.notification import Notification
+from src.bot.database import run_db_migrations
+from src.web.webserver_thread import WebServerThread
logger = get_logger(__name__)
config_file_name = f"{os.getenv('BOT_CONFIG_DIR', './config')}/config.ini"
@@ -73,9 +73,9 @@ def entry():
|___/
-------------------------------------------------------------------------------------
""")
- run_migrations(alembic_migration_files, db_url)
- create_engine(db_url)
+ run_db_migrations(alembic_migration_files, db_url)
run()
+
if __name__ == '__main__':
entry()
diff --git a/src/bot/static/css/main.css b/src/web/static/css/main.css
similarity index 100%
rename from src/bot/static/css/main.css
rename to src/web/static/css/main.css
diff --git a/src/bot/templates/configuration.html b/src/web/templates/configuration.html
similarity index 74%
rename from src/bot/templates/configuration.html
rename to src/web/templates/configuration.html
index 7268e62..670abb9 100644
--- a/src/bot/templates/configuration.html
+++ b/src/web/templates/configuration.html
@@ -1,7 +1,7 @@
- Steamgifts Bot Configuration
+ {{name}} Steamgifts Bot Configuration
@@ -13,6 +13,8 @@