132 lines
5.5 KiB
Python
132 lines
5.5 KiB
Python
from datetime import datetime
|
|
from time import sleep
|
|
|
|
from sqlalchemy import func
|
|
from sqlalchemy.exc import IntegrityError, PendingRollbackError
|
|
|
|
from data.database import dal, MegaMillionsDraw, PowerballDraw
|
|
import lottery_predictor.collect as predictor
|
|
from util.drawing import get_mega_millions_object, get_powerball_object
|
|
from util.environment import load_environment_variables
|
|
from util.scrape import scrape_game_history
|
|
|
|
project_variables = load_environment_variables()
|
|
|
|
def main():
|
|
print("Lottery Analyzer initializing...")
|
|
# convert specific variables for use
|
|
mm_start_year = int(project_variables["LP_MEGA_MILLIONS_INCEPTION"])
|
|
pb_start_year = int(project_variables["LP_POWERBALL_INCEPTION"])
|
|
end_year = datetime.today().year
|
|
date_format = project_variables["LP_DATE_INSERT_FORMAT"]
|
|
max_mm_main = int(project_variables["LP_MEGA_MILLIONS_MAX_MAIN_BALL"])
|
|
max_megaball = int(project_variables["LP_MEGA_MILLIONS_MAX_MEGA_BALL"])
|
|
max_pb_main = int(project_variables["LP_POWERBALL_MAX_MAIN_BALL"])
|
|
max_powerball = int(project_variables["LP_POWERBALL_MAX_POWERBALL"])
|
|
|
|
|
|
# connect to the database
|
|
print("Connecting to the database...")
|
|
dal.connect()
|
|
session = dal.Session()
|
|
|
|
print("Collecting and processing missing Mega Millions draws...")
|
|
# if there are no records in the MegaMillionsDraw table...
|
|
if predictor.get_record_count('MegaMillionsDraw') == 0:
|
|
# loop over all years and scrape the data for each
|
|
for year in range(mm_start_year, end_year + 1):
|
|
print(f"\tCollecting {year}...")
|
|
scrape_data = scrape_game_history(game='mega-millions', year=year)
|
|
# convert and insert the records
|
|
for key, value in scrape_data.items():
|
|
record = get_mega_millions_object(draw_date=key, draw_result=value)
|
|
try:
|
|
session.add(record)
|
|
session.commit()
|
|
except IntegrityError:
|
|
pass
|
|
except PendingRollbackError:
|
|
session.rollback()
|
|
finally:
|
|
session.close()
|
|
# sleep between records and years to avoid querying the site too often
|
|
sleep(60)
|
|
|
|
# commit all the records to the database
|
|
else:
|
|
most_recent = session.query(func.max(MegaMillionsDraw.draw_date)).first()
|
|
start_year = datetime.strptime(str(most_recent[0]), date_format).year
|
|
# loop over the years needed and gather records
|
|
for year in range(start_year, end_year + 1):
|
|
print(f"\tCollecting {year}...")
|
|
scrape_data = scrape_game_history(game='powerball', year=year)
|
|
# convert and insert the records
|
|
for key, value in scrape_data.items():
|
|
record = get_powerball_object(draw_date=key, draw_result=value)
|
|
try:
|
|
session.add(record)
|
|
session.commit()
|
|
except IntegrityError:
|
|
pass
|
|
except PendingRollbackError:
|
|
session.rollback()
|
|
finally:
|
|
session.close()
|
|
# sleep between records and years to avoid querying the site too often
|
|
sleep(60)
|
|
|
|
print("Collecting and processing missing Powerball draws...")
|
|
# if there are no records in the PowerBallDraw table...
|
|
if len(session.query(PowerballDraw).all()) == 0:
|
|
# loop over all years and scrape the data for each
|
|
for year in range(pb_start_year, end_year + 1):
|
|
print(f"\tCollecting {year}...")
|
|
scrape_data = scrape_game_history(game='powerball', year=year)
|
|
# convert and insert the records
|
|
for key, value in scrape_data.items():
|
|
record = get_powerball_object(draw_date=key, draw_result=value)
|
|
try:
|
|
session.add(record)
|
|
session.commit()
|
|
except IntegrityError:
|
|
pass
|
|
except PendingRollbackError:
|
|
session.rollback()
|
|
finally:
|
|
session.close()
|
|
# sleep between records and years to avoid querying the site too often
|
|
sleep(60)
|
|
|
|
# commit all the records to the database
|
|
session.commit()
|
|
else:
|
|
most_recent = session.query(func.max(PowerballDraw.draw_date)).first()
|
|
start_year = datetime.strptime(str(most_recent[0]), str(date_format)).year
|
|
# loop over the years needed and gather records
|
|
for year in range(start_year, end_year + 1):
|
|
print(f"\tCollecting {year}...")
|
|
scrape_data = scrape_game_history(game='powerball', year=year)
|
|
# convert and insert the records
|
|
for key, value in scrape_data.items():
|
|
record = get_powerball_object(draw_date=key, draw_result=value)
|
|
try:
|
|
session.add(record)
|
|
session.commit()
|
|
except IntegrityError:
|
|
pass
|
|
except PendingRollbackError:
|
|
session.rollback()
|
|
finally:
|
|
session.close()
|
|
# sleep between records and years to avoid querying the site too often
|
|
sleep(60)
|
|
|
|
mm_count = session.query(MegaMillionsDraw).count()
|
|
pb_count = session.query(PowerballDraw).count()
|
|
|
|
print(f"There are {mm_count} Mega Millions draw records and {pb_count} Powerball records")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|