92 lines
2.8 KiB
Python
Executable File
92 lines
2.8 KiB
Python
Executable File
"""
|
|
util/scrape.py
|
|
|
|
Utility functions for scraping web data relative to selected games
|
|
"""
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
from dateutil import parser, utils
|
|
|
|
from util.environment import load_environment_variables
|
|
|
|
|
|
project_variables: dict = load_environment_variables()
|
|
|
|
|
|
def scrape_game_history(game: str, year: int = utils.today().year) -> dict:
|
|
"""
|
|
Scrape game data from the url provided. This function requires
|
|
specific environment variables be present
|
|
in order to function properly.
|
|
:param:game The target game to scrape.
|
|
:param:year The year to scrape. If no year is provided, scrapes
|
|
games from the current year.
|
|
|
|
:return: dict()
|
|
|
|
Examples:
|
|
>>>scrape_game_history(game='mega-millions')
|
|
draw_date {'main_balls': ...}
|
|
|
|
>>>scrape_game_history(game='mega-millions', year=2020)
|
|
draw_date {'main_balls': ...}
|
|
"""
|
|
drawings = {}
|
|
|
|
url = f"{project_variables['LP_BASE_SCRAPE_URL']}{game}/numbers/{year}"
|
|
page = httpx.get(url)
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
results = soup.find_all(
|
|
project_variables['LP_RESULT_CONTAINER'],
|
|
class_=project_variables['LP_RESULT_CLASS'],
|
|
)
|
|
|
|
for result in results:
|
|
pick5 = []
|
|
draw_date = parser.parse(
|
|
result.find(
|
|
project_variables['LP_DRAW_DATE_CONTAINER'],
|
|
class_=project_variables['LP_DRAW_DATE_CLASS'],
|
|
).text,
|
|
).date().strftime('%Y-%m-%d')
|
|
main_balls = result.find_all(
|
|
project_variables['LP_BALL_CONTAINER'],
|
|
class_=project_variables['LP_BALL_CLASS'],
|
|
)
|
|
game_ball = result.find(
|
|
project_variables['LP_BALL_CONTAINER'],
|
|
class_=[
|
|
project_variables['LP_POWERBALL_CLASS'],
|
|
project_variables['LP_MEGABALL_CLASS'],
|
|
],
|
|
)
|
|
multiplier = result.find(
|
|
project_variables['LP_BALL_CONTAINER'],
|
|
class_=[
|
|
project_variables['LP_POWERBALL_MULTIPLIER_CLASS'],
|
|
project_variables['LP_MEGA_MILLIONS_MULTIPLIER_CLASS'],
|
|
],
|
|
)
|
|
|
|
for ball in main_balls:
|
|
pick5.append(int(ball.text.strip()))
|
|
|
|
game_ball = int(game_ball.find('span').text)
|
|
if multiplier:
|
|
multiplier = int(multiplier.find('span').text)
|
|
else:
|
|
multiplier = 1
|
|
|
|
if game == 'mega-millions':
|
|
drawings[draw_date] = {
|
|
'main_balls': pick5, 'mega-ball': game_ball,
|
|
'multiplier': multiplier,
|
|
}
|
|
elif game == 'powerball':
|
|
drawings[draw_date] = {
|
|
'main_balls': pick5, 'powerball': game_ball,
|
|
'multiplier': multiplier,
|
|
}
|
|
|
|
return drawings
|