From 9a3ebe2d5c27552c423fad68caa1df0f9580bf8c Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 6 Mar 2026 14:50:52 -0500 Subject: [PATCH] Clean up imports, complete get_missing_records --- lottery_predictor/collect.py | 39 ++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/lottery_predictor/collect.py b/lottery_predictor/collect.py index da3c200..bfded28 100644 --- a/lottery_predictor/collect.py +++ b/lottery_predictor/collect.py @@ -4,13 +4,13 @@ Functions for collecting data for insertion into the databse or pulling data from the database for analysis """ from datetime import datetime, timedelta -from enum import Enum +from doctest import script_from_examples from time import sleep -from sqlalchemy import func, text +from sqlalchemy import func from sqlalchemy.exc import IntegrityError -from data.database import Base, dal, MegaMillionsDraw, PowerballDraw +from data.database import Base, dal import util.drawing as drawutil import util.environment as envutil import util.scrape as scrapeutil @@ -47,22 +47,39 @@ def get_missing_records(table_name: str) -> int: # if the table_name is valid game_dict = drawutil.check_table_name(table_name=table_name) if game_dict: + insert_count = 0 # connect to the database and establish a session dal.connect() session = dal.Session() - # get the ORM table value + + date_format = project_variables['LP_DATE_PASS_FORMAT'] + target_table = Base.metadata.tables.get(game_dict['db_table_name']) record_count = get_record_count(table_name=table_name) target_game = game_dict['game_name'] end_year = datetime.today().year + 1 - start_year = game_dict['inception_year'] if record_count > 0 else get_recent_draw_date(table_name=table_name).year - - # check to see if there are records in the table - if get_record_count(table_name) > 0: - pass - else: - pass + recent_draw_date = (get_recent_draw_date(table_name=table_name) + timedelta(days=1)).strftime(date_format) + following_draw_date = datetime.strptime(drawutil.get_next_draw_date(game=target_game, + from_date=recent_draw_date), date_format) + start_year = game_dict['inception_year'] if record_count == 0 else following_draw_date.year + check_date = datetime.strptime(drawutil.get_previous_draw_date(game=target_game, from_date= + drawutil.get_next_draw_date(game=target_game)), date_format).date() + if session.query(target_table).filter_by(draw_date=check_date).first() is None: + # loop over the range of years and insert missing records + for year in range(start_year, end_year): + scrape_data = scrapeutil.scrape_game_history(game=target_game, year=year) + # convert and insert records, discarding duplicate dates + for draw_date, draw_info in scrape_data.items(): + record = game_dict['record_type'](draw_date=draw_date, draw_result=draw_info) + try: + session.add(record) + session.commit() + insert_count += 1 + except IntegrityError: + session.rollback() + sleep(30) # sleep between calls to the site for data to avoid being blocked + return insert_count else: raise ValueError('An invalid table name was specified.')