diff --git a/src/alembic/versions/2022_06_01-09_20_36-8c1784114d65_won_column_added_back.py b/src/alembic/versions/2022_06_01-09_20_36-8c1784114d65_won_column_added_back.py new file mode 100644 index 0000000..7694dfb --- /dev/null +++ b/src/alembic/versions/2022_06_01-09_20_36-8c1784114d65_won_column_added_back.py @@ -0,0 +1,31 @@ +"""won column added backed + +Revision ID: 8c1784114d65 +Revises: ff0a728ba3da +Create Date: 2022-06-01 09:20:36.762279 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '8c1784114d65' +down_revision = 'ff0a728ba3da' +branch_labels = None +depends_on = None + + +def upgrade(): + # add columns as nullable as existing records don't have that column value set + op.add_column('giveaway', sa.Column('won', sa.Boolean(), nullable=True)) + # set value on new columns for all records + with op.get_context().autocommit_block(): + op.execute("UPDATE giveaway SET won=false WHERE won is null;") + # SQLite doesn't support ALTERs so alembic uses a batch mode + # Set columns to non-nullable now that all records have a value + with op.batch_alter_table('giveaway') as batch_op: + batch_op.alter_column('won', nullable=False) + + +def downgrade(): + op.drop_column('giveaway', 'won') diff --git a/src/bot/database.py b/src/bot/database.py index 65ebbe0..c261518 100644 --- a/src/bot/database.py +++ b/src/bot/database.py @@ -11,7 +11,7 @@ from sqlalchemy.orm import Session, joinedload from sqlalchemy_utils import database_exists from .log import get_logger -from .models import TableNotification, TableGiveaway, TableSteamItem +from .models import TableNotification, TableGiveaway, TableSteamItem, TableTask logger = get_logger(__name__) engine = sqlalchemy.create_engine(f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}", echo=False) @@ -127,7 +127,7 @@ class GiveawayHelper: steam_id=giveaway.steam_app_id).all() @classmethod - def insert(cls, giveaway, entered): + def insert(cls, giveaway, entered, won): with Session(engine) as session: result = session.query(TableSteamItem).filter_by(steam_id=giveaway.steam_app_id).all() if result: @@ -149,17 +149,18 @@ class GiveawayHelper: giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), cost=giveaway.cost, copies=giveaway.copies, - contributor_level=giveaway.contributor_level, + contributor_level=giveaway._contributor_level, entered=entered, + won=won, game_entries=giveaway.game_entries) session.add(g) session.commit() @classmethod - def upsert_giveaway(cls, giveaway, entered): + def upsert_giveaway(cls, giveaway, entered, won): result = GiveawayHelper.get_by_ids(giveaway) if not result: - GiveawayHelper.insert(giveaway, entered) + GiveawayHelper.insert(giveaway, entered, won) else: with Session(engine) as session: g = TableGiveaway( @@ -171,8 +172,30 @@ class GiveawayHelper: giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), cost=giveaway.cost, copies=giveaway.copies, - contributor_level=giveaway.contributor_level, + contributor_level=giveaway._contributor_level, entered=entered, + won=won, + game_entries=giveaway.game_entries) + session.merge(g) + session.commit() + + @classmethod + def upsert_giveaway(cls, giveaway): + result = GiveawayHelper.get_by_ids(giveaway) + if not result: + GiveawayHelper.insert(giveaway, False, False) + else: + with Session(engine) as session: + g = TableGiveaway( + giveaway_id=giveaway.giveaway_game_id, + steam_id=result[0].steam_id, + giveaway_uri=giveaway.giveaway_uri, + user=giveaway.user, + giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp), + giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), + cost=giveaway.cost, + copies=giveaway.copies, + contributor_level=giveaway._contributor_level, game_entries=giveaway.game_entries) session.merge(g) session.commit() diff --git a/src/bot/enter_giveaways.py b/src/bot/enter_giveaways.py index 9455c9b..7eeb922 100644 --- a/src/bot/enter_giveaways.py +++ b/src/bot/enter_giveaways.py @@ -9,7 +9,7 @@ from urllib3.util import Retry from .log import get_logger from .database import NotificationHelper, GiveawayHelper -from .giveaway import Giveaway +from .giveaway_entry import GiveawayEntry logger = get_logger(__name__) @@ -19,28 +19,29 @@ class SteamGiftsException(Exception): class EnterGiveaways: + def __init__(self, cookie, user_agent, gifts_type, pinned, min_points, max_entries, max_time_left, minimum_game_points, blacklist, notification): - self.contributor_level = None - self.xsrf_token = None - self.points = None - self.cookie = { + self._contributor_level = None + self._xsrf_token = None + self._points = None + self._cookie = { 'PHPSESSID': cookie } - self.user_agent = user_agent - self.gifts_type = gifts_type - self.pinned = pinned - self.min_points = int(min_points) - self.max_entries = int(max_entries) - self.max_time_left = int(max_time_left) - self.minimum_game_points = int(minimum_game_points) - self.blacklist = blacklist.split(',') - self.notification = notification + self._user_agent = user_agent + self._gifts_type = gifts_type + self._pinned = pinned + self._min_points = int(min_points) + self._max_entries = int(max_entries) + self._max_time_left = int(max_time_left) + self._minimum_game_points = int(minimum_game_points) + self._blacklist = blacklist.split(',') + self._notification = notification - self.base = "https://www.steamgifts.com" - self.session = requests.Session() + self._base = "https://www.steamgifts.com" + self._session = requests.Session() - self.filter_url = { + self._filter_url = { 'All': "search?page=%d", 'Wishlist': "search?page=%d&type=wishlist", 'Recommended': "search?page=%d&type=recommended", @@ -49,12 +50,23 @@ class EnterGiveaways: 'New': "search?page=%d&type=new" } - def requests_retry_session( + def start(self): + self._update_info() + if self._points >= self._min_points: + txt = f"〰 You have {self._points} points. Evaluating '{self._gifts_type}' giveaways..." + logger.info(txt) + self._evaluate_giveaways() + else: + txt = f"🟡 You have {self._points} points which is below your minimum point threshold of " \ + f"{self._min_points} points for '{self._gifts_type}' giveaways. Not evaluating right now." + logger.info(txt) + + def _requests_retry_session( self, retries=5, backoff_factor=0.3 ): - session = self.session or requests.Session() + session = self._session or requests.Session() retry = Retry( total=retries, read=retries, @@ -67,22 +79,22 @@ class EnterGiveaways: session.mount('https://', adapter) return session - def get_soup_from_page(self, url): + def _get_soup_from_page(self, url): headers = { - 'User-Agent': self.user_agent + 'User-Agent': self._user_agent } - self.requests_retry_session().get(url, headers=headers) - r = requests.get(url, cookies=self.cookie) + self._requests_retry_session().get(url, headers=headers) + r = requests.get(url, cookies=self._cookie) soup = BeautifulSoup(r.text, 'html.parser') return soup - def update_info(self): - soup = self.get_soup_from_page(self.base) + def _update_info(self): + soup = self._get_soup_from_page(self._base) try: - self.xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value'] - self.points = int(soup.find('span', {'class': 'nav__points'}).text) # storage points - self.contributor_level = int(float(soup.select_one('nav a>span[title]')['title'])) + self._xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value'] + self._points = int(soup.find('span', {'class': 'nav__points'}).text) # storage points + self._contributor_level = int(float(soup.select_one('nav a>span[title]')['title'])) except TypeError: logger.error("⛔⛔⛔ Cookie is not valid. A new one must be added.⛔⛔⛔") raise SteamGiftsException("Cookie is not valid. A new one must be added.") @@ -97,63 +109,63 @@ class EnterGiveaways: "to be received. Doing nothing.") elif number_won > won_notifications[-1].games_won: logger.info("🔥🔥 MORE win(s) detected. Notifying again.") - self.notification.send_won(f"You won ANOTHER game. You now have {number_won} game(s) " + self._notification.send_won(f"You won ANOTHER game. You now have {number_won} game(s) " f"waiting to be claimed.", number_won) else: # we have less games waiting to be claimed than notified, meaning some have been claimed logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting " "to be received. Some have been claimed. Doing nothing.") else: logger.info(f"💰💰 WINNER! You have {number_won} game(s) waiting to be claimed. 💰💰") - self.notification.send_won(f"WINNER! You have {number_won} game(s) waiting to be claimed.", number_won) + self._notification.send_won(f"WINNER! You have {number_won} game(s) waiting to be claimed.", number_won) else: logger.debug('No wins detected. Doing nothing.') - def should_we_enter_giveaway(self, giveaway): + def _should_we_enter_giveaway(self, giveaway): if giveaway.time_remaining_in_minutes is None: return False if giveaway.time_created_in_minutes is None: return False - if self.blacklist is not None and self.blacklist != ['']: - for keyword in self.blacklist: + if self._blacklist is not None and self._blacklist != ['']: + for keyword in self._blacklist: if giveaway.game_name.lower().find(keyword.lower()) != -1: txt = f"〰️ Game {giveaway.game_name} contains the blacklisted keyword {keyword}" logger.info(txt) return False - if giveaway.contributor_level is None or self.contributor_level < giveaway.contributor_level: + if giveaway.contributor_level is None or self._contributor_level < giveaway.contributor_level: txt = f"〰️ Game {giveaway.game_name} requires at least level {giveaway.contributor_level} contributor " \ - f"level to enter. Your level: {self.contributor_level}" + f"level to enter. Your level: {self._contributor_level}" logger.info(txt) return False - if self.points - int(giveaway.cost) < 0: + if giveaway.time_remaining_in_minutes > self._max_time_left: + txt = f"〰️ Game {giveaway.game_name} has {giveaway.time_remaining_in_minutes} minutes left and is " \ + f"above your cutoff of {self._max_time_left} minutes." + logger.info(txt) + return False + if giveaway.game_entries / giveaway.copies > self._max_entries: + txt = f"〰️ Game {giveaway.game_name} has {giveaway.game_entries} entries and is above your cutoff " \ + f"of {self._max_entries} entries." + logger.info(txt) + return False + if self._points - int(giveaway.cost) < 0: txt = f"〰️ Not enough points to enter: {giveaway.game_name}" logger.info(txt) return False - if giveaway.cost < self.minimum_game_points: + if giveaway.cost < self._minimum_game_points: txt = f"〰️ Game {giveaway.game_name} costs {giveaway.cost}P and is below your cutoff of " \ - f"{self.minimum_game_points}P." - logger.info(txt) - return False - if giveaway.time_remaining_in_minutes > self.max_time_left: - txt = f"〰️ Game {giveaway.game_name} has {giveaway.time_remaining_in_minutes} minutes left and is " \ - f"above your cutoff of {self.max_time_left} minutes." - logger.info(txt) - return False - if giveaway.game_entries / giveaway.copies > self.max_entries: - txt = f"〰️ Game {giveaway.game_name} has {giveaway.game_entries} entries and is above your cutoff " \ - f"of {self.max_entries} entries." + f"{self._minimum_game_points}P." logger.info(txt) return False return True - def enter_giveaway(self, giveaway): + def _enter_giveaway(self, giveaway): headers = { - 'User-Agent': self.user_agent + 'User-Agent': self._user_agent } - payload = {'xsrf_token': self.xsrf_token, 'do': 'entry_insert', 'code': giveaway.giveaway_game_id} + payload = {'xsrf_token': self._xsrf_token, 'do': 'entry_insert', 'code': giveaway.giveaway_game_id} logger.debug(f"Sending enter giveaway payload: {payload}") - entry = requests.post('https://www.steamgifts.com/ajax.php', data=payload, cookies=self.cookie, + entry = requests.post('https://www.steamgifts.com/ajax.php', data=payload, cookies=self._cookie, headers=headers) json_data = json.loads(entry.text) @@ -164,17 +176,17 @@ class EnterGiveaways: logger.error(f"❌ Failed entering giveaway {giveaway.giveaway_game_id}: {json_data}") return False - def evaluate_giveaways(self, page=1): + def _evaluate_giveaways(self, page=1): n = page run = True while run and n < 3: # hard stop safety net at page 3 as idk why we would ever get to this point txt = "〰️ Retrieving games from %d page." % n logger.info(txt) - filtered_url = self.filter_url[self.gifts_type] % n - paginated_url = f"{self.base}/giveaways/{filtered_url}" + filtered_url = self._filter_url[self._gifts_type] % n + paginated_url = f"{self._base}/giveaways/{filtered_url}" - soup = self.get_soup_from_page(paginated_url) + soup = self._get_soup_from_page(paginated_url) pinned_giveaway_count = len(soup.select('div.pinned-giveaways__outer-wrap div.giveaway__row-inner-wrap')) all_games_list_count = len(soup.select('div.giveaway__row-inner-wrap')) @@ -189,17 +201,17 @@ class EnterGiveaways: break for item in unentered_game_list: - giveaway = Giveaway(item) + giveaway = GiveawayEntry(item) txt = f"〰 {giveaway.game_name} - {giveaway.cost}P - {giveaway.game_entries} entries " \ f"(w/ {giveaway.copies} copies) - Created {giveaway.time_created_string} ago " \ f"with {giveaway.time_remaining_string} remaining by {giveaway.user}." logger.info(txt) - if giveaway.pinned and not self.pinned: + if giveaway.pinned and not self._pinned: logger.info(f"〰️ Giveaway {giveaway.game_name} is pinned. Ignoring.") continue - if self.points == 0 or self.points < self.min_points: - txt = f"🟡 We have {self.points} points, but we need {self.min_points} to start." + if self._points == 0 or self._points < self._min_points: + txt = f"🟡 We have {self._points} points, but we need {self._min_points} to start." logger.info(txt) run = False break @@ -207,37 +219,26 @@ class EnterGiveaways: if not giveaway.cost: logger.error(f"Cost could not be determined for '{giveaway.game_name}'") continue - if_enter_giveaway = self.should_we_enter_giveaway(giveaway) + if_enter_giveaway = self._should_we_enter_giveaway(giveaway) if if_enter_giveaway: - res = self.enter_giveaway(giveaway) + res = self._enter_giveaway(giveaway) if res: - GiveawayHelper.upsert_giveaway(giveaway, True) - self.points -= int(giveaway.cost) + GiveawayHelper.upsert_giveaway(giveaway, True, False) + self._points -= int(giveaway.cost) txt = f"✅ Entered giveaway '{giveaway.game_name}'" logger.info(txt) sleep(randint(4, 15)) else: - GiveawayHelper.upsert_giveaway(giveaway, False) + GiveawayHelper.upsert_giveaway(giveaway, False, False) else: - GiveawayHelper.upsert_giveaway(giveaway, False) + GiveawayHelper.upsert_giveaway(giveaway) # if we are on any filter type except New and we get to a giveaway that exceeds our # max time left amount, then we don't need to continue to look at giveaways as any # after this point will also exceed the max time left - if self.gifts_type != "New" and not giveaway.pinned and \ - giveaway.time_remaining_in_minutes > self.max_time_left: + if self._gifts_type != "New" and not giveaway.pinned and \ + giveaway.time_remaining_in_minutes > self._max_time_left: logger.info("🟡 We have run out of gifts to consider.") run = False break n = n + 1 - - def start(self): - self.update_info() - if self.points >= self.min_points: - txt = f"〰 You have {self.points} points. Evaluating '{self.gifts_type}' giveaways..." - logger.info(txt) - self.evaluate_giveaways() - else: - txt = f"🟡 You have {self.points} points which is below your minimum point threshold of " \ - f"{self.min_points} points for '{self.gifts_type}' giveaways. Not evaluating right now." - logger.info(txt) diff --git a/src/bot/giveaway.py b/src/bot/giveaway_entry.py similarity index 86% rename from src/bot/giveaway.py rename to src/bot/giveaway_entry.py index 64a9dbf..34e2b2f 100644 --- a/src/bot/giveaway.py +++ b/src/bot/giveaway_entry.py @@ -5,7 +5,7 @@ import time logger = get_logger(__name__) -class Giveaway: +class GiveawayEntry: def __init__(self, soup_item): self.steam_app_id = None @@ -29,27 +29,27 @@ class Giveaway: logger.debug(f"Giveaway html: {soup_item}") icons = soup_item.select('a.giveaway__icon') self.steam_url = icons[0]['href'] - self.steam_app_id = self.get_steam_app_id(self.steam_url) + self.steam_app_id = self._get_steam_app_id(self.steam_url) self.game_name = soup_item.find('a', {'class': 'giveaway__heading__name'}).text self.giveaway_game_id = soup_item.find('a', {'class': 'giveaway__heading__name'})['href'].split('/')[2] self.giveaway_uri = soup_item.select_one('a.giveaway__heading__name')['href'] pin_class = soup_item.parent.parent.get("class") self.pinned = pin_class is not None and len(pin_class) > 0 and pin_class[0].find('pinned') != -1 - self.cost, self.copies = self.determine_cost_and_copies(soup_item, self.game_name, self.giveaway_game_id) + self.cost, self.copies = self._determine_cost_and_copies(soup_item, self.game_name, self.giveaway_game_id) self.game_entries = int(soup_item.select('div.giveaway__links span')[0].text.split(' ')[0].replace(',', '')) contributor_level = soup_item.select_one('div[title="Contributor Level"]') - self.contributor_level = self.determine_contributor_level(contributor_level) + self.contributor_level = self._determine_contributor_level(contributor_level) self.user = soup_item.select_one('a.giveaway__username').text times = soup_item.select('div span[data-timestamp]') self.time_remaining_timestamp = int(times[0]['data-timestamp']) self.time_remaining_string = times[0].text - self.time_remaining_in_minutes = self.determine_time_in_minutes(times[0]['data-timestamp']) + self.time_remaining_in_minutes = self._determine_time_in_minutes(times[0]['data-timestamp']) self.time_created_timestamp = int(times[1]['data-timestamp']) self.time_created_string = times[1].text - self.time_created_in_minutes = self.determine_time_in_minutes(times[1]['data-timestamp']) + self.time_created_in_minutes = self._determine_time_in_minutes(times[1]['data-timestamp']) logger.debug(f"Scraped Giveaway: {self}") - def determine_contributor_level(self, contributor_level): + def _determine_contributor_level(self, contributor_level): if contributor_level is None: return 0 match = re.search('^Level (?P[0-9]+)\\+$', contributor_level.text, re.IGNORECASE) @@ -58,14 +58,14 @@ class Giveaway: else: return None - def get_steam_app_id(self, steam_url): + def _get_steam_app_id(self, steam_url): match = re.search('^.+/[a-z0-9]+/(?P[0-9]+)/$', steam_url, re.IGNORECASE) if match: return match.group('steam_app_id') else: return None - def determine_time_in_minutes(self, timestamp): + def _determine_time_in_minutes(self, timestamp): if not timestamp or not re.search('^[0-9]+$', timestamp): logger.error(f"Could not determine time from string {timestamp}") return None @@ -73,7 +73,7 @@ class Giveaway: giveaway_endtime = time.localtime(int(timestamp)) return int(abs((time.mktime(giveaway_endtime) - time.mktime(now)) / 60)) - def determine_cost_and_copies(self, item, game_name, game_id): + def _determine_cost_and_copies(self, item, game_name, game_id): item_headers = item.find_all('span', {'class': 'giveaway__heading__thin'}) if len(item_headers) == 1: # then no multiple copies game_cost = item_headers[0].getText().replace('(', '').replace(')', '').replace('P', '')