Reformat code to pythonic standards
This commit is contained in:
1 parent
9630657710
commit
31b335b7e8
4 files changed
+128
-61
No files matched your search
@@ -1,7 +1,8 @@
|
||||
"""
|
||||
collect.py
|
||||
|
||||
Functions for collecting data for insertion into the databse or pulling data from the database for analysis
|
||||
Functions for collecting data for insertion into the databse or pulling
|
||||
data from the database for analysis
|
||||
"""
|
||||
from datetime import datetime, timedelta
|
||||
from time import sleep
|
||||
@@ -9,10 +10,10 @@ from time import sleep
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from data.database import Base, dal
|
||||
from lottery_predictor.config import GAME_INFO, PROJECT_VARIABLES
|
||||
import util.drawing as drawutil
|
||||
import util.scrape as scrapeutil
|
||||
from data.database import Base, dal
|
||||
from lottery_predictor.config import GAME_INFO, PROJECT_VARIABLES
|
||||
|
||||
|
||||
def get_record_count(game: str) -> int | None:
|
||||
@@ -26,7 +27,8 @@ def get_record_count(game: str) -> int | None:
|
||||
dal.connect()
|
||||
session = dal.Session()
|
||||
target_table = Base.metadata.tables.get(game_dict['db_table_name'])
|
||||
return session.query(func.count(target_table.columns.draw_date)).first()[0]
|
||||
return \
|
||||
session.query(func.count(target_table.columns.draw_date)).first()[0]
|
||||
else:
|
||||
raise ValueError('An invalid table name was specified.')
|
||||
|
||||
@@ -42,7 +44,8 @@ def get_recent_draw_date(game: str) -> datetime | None:
|
||||
dal.connect()
|
||||
session = dal.Session()
|
||||
target_table = Base.metadata.tables.get(game_dict['db_table_name'])
|
||||
return session.query(func.max(target_table.columns.draw_date)).first()[0]
|
||||
return session.query(func.max(target_table.columns.draw_date)).first()[
|
||||
0]
|
||||
else:
|
||||
raise ValueError('An invalid table name was specified.')
|
||||
|
||||
@@ -67,20 +70,39 @@ def get_missing_records(game: str) -> int:
|
||||
record_count = get_record_count(game=game)
|
||||
target_game = game_dict['game_name']
|
||||
end_year = datetime.today().year + 1
|
||||
recent_draw_date = (get_recent_draw_date(game=game) + timedelta(days=1)).strftime(date_format)
|
||||
following_draw_date = datetime.strptime(drawutil.get_next_draw_date(game=target_game,
|
||||
from_date=recent_draw_date), date_format)
|
||||
start_year = game_dict['inception_year'] if record_count == 0 else following_draw_date.year
|
||||
check_date = datetime.strptime(drawutil.get_previous_draw_date(game=target_game, from_date=
|
||||
drawutil.get_next_draw_date(game=target_game)), date_format).date()
|
||||
recent_draw_date = (get_recent_draw_date(game=game) + timedelta(
|
||||
days=1,
|
||||
)).strftime(
|
||||
date_format,
|
||||
)
|
||||
following_draw_date = datetime.strptime(
|
||||
drawutil.get_next_draw_date(
|
||||
game=target_game,
|
||||
from_date=recent_draw_date,
|
||||
), date_format,
|
||||
)
|
||||
start_year = game_dict[
|
||||
'inception_year'] if record_count == 0 else following_draw_date.year
|
||||
check_date = datetime.strptime(
|
||||
drawutil.get_previous_draw_date(
|
||||
game=target_game, from_date=
|
||||
drawutil.get_next_draw_date(game=target_game),
|
||||
), date_format,
|
||||
).date()
|
||||
|
||||
if session.query(target_table).filter_by(draw_date=check_date).first() is None:
|
||||
if session.query(target_table).filter_by(
|
||||
draw_date=check_date,
|
||||
).first() is None:
|
||||
# loop over the range of years and insert missing records
|
||||
for year in range(start_year, end_year):
|
||||
scrape_data = scrapeutil.scrape_game_history(game=target_game, year=year)
|
||||
scrape_data = scrapeutil.scrape_game_history(
|
||||
game=target_game, year=year,
|
||||
)
|
||||
# convert and insert records, discarding duplicate dates
|
||||
for draw_date, draw_info in scrape_data.items():
|
||||
record = game_dict['record_type'](draw_date=draw_date, draw_result=draw_info)
|
||||
record = game_dict['record_type'](
|
||||
draw_date=draw_date, draw_result=draw_info,
|
||||
)
|
||||
try:
|
||||
session.add(record)
|
||||
session.commit()
|
||||
@@ -88,7 +110,10 @@ def get_missing_records(game: str) -> int:
|
||||
except IntegrityError:
|
||||
session.rollback()
|
||||
if year != end_year - 1:
|
||||
sleep(30) # sleep between calls to the site for data to avoid being blocked
|
||||
sleep(
|
||||
30,
|
||||
) # sleep between calls to the site for data to
|
||||
# avoid being blocked
|
||||
|
||||
return insert_count
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user