diff options
Diffstat (limited to 'server/services/get_data.py')
| -rw-r--r-- | server/services/get_data.py | 67 |
1 files changed, 48 insertions, 19 deletions
diff --git a/server/services/get_data.py b/server/services/get_data.py index 1b446c6..3014ea1 100644 --- a/server/services/get_data.py +++ b/server/services/get_data.py @@ -11,12 +11,16 @@ MARKET_CLASSES: Dict[str, Type] = { "NYISO": NYISO, } + def get_iso_instance(market: str): market = market.upper() if market not in MARKET_CLASSES: - raise ValueError(f"Unsupported market '{market}'. Supported: {list(MARKET_CLASSES.keys())}") + raise ValueError( + f"Unsupported market '{market}'. Supported: {list(MARKET_CLASSES.keys())}" + ) return MARKET_CLASSES[market]() + def update_market_data(): db: Session = SessionLocal() @@ -24,20 +28,30 @@ def update_market_data(): print(f"Processing {market_name}") iso = get_iso_instance(market_name) - # --- Real-Time Data (5 min) --- - last_realtime = db.query(MarketDataDB)\ - .filter(MarketDataDB.market == market_name, MarketDataDB.type == "REALTIME")\ - .order_by(MarketDataDB.timestamp.desc())\ + last_realtime = ( + db.query(MarketDataDB) + .filter(MarketDataDB.market == market_name, MarketDataDB.type == "REALTIME") + .order_by(MarketDataDB.timestamp.desc()) .first() + ) - if not last_realtime or (datetime.now(UTC) - last_realtime.timestamp.replace(tzinfo=UTC) > timedelta(minutes=5)): + if not last_realtime or ( + datetime.now(UTC) - last_realtime.timestamp.replace(tzinfo=UTC) + > timedelta(minutes=5) + ): print(f"Getting realtime data for {market_name}") df = iso.get_lmp(date="latest", market="REAL_TIME_5_MIN", locations="ALL") - df["Interval Start"] = df["Interval Start"].dt.tz_convert('UTC') - grouped = df.groupby("Interval Start")[["LMP", "Energy", "Congestion", "Loss"]].mean().reset_index() + df["Interval Start"] = df["Interval Start"].dt.tz_convert("UTC") + grouped = ( + df.groupby("Interval Start")[["LMP", "Energy", "Congestion", "Loss"]] + .mean() + .reset_index() + ) for _, row in grouped.iterrows(): - if last_realtime and row["Interval Start"] <= last_realtime.timestamp.replace(tzinfo=UTC): + if last_realtime and row[ + "Interval Start" + ] <= last_realtime.timestamp.replace(tzinfo=UTC): continue # Skip old data entry = MarketDataDB( timestamp=row["Interval Start"], @@ -50,24 +64,38 @@ def update_market_data(): ) db.add(entry) - # --- Day-Ahead Hourly Data (1 hour) --- - last_dayahead = db.query(MarketDataDB)\ - .filter(MarketDataDB.market == market_name, MarketDataDB.type == "DAYAHEAD")\ - .order_by(MarketDataDB.timestamp.desc())\ + last_dayahead = ( + db.query(MarketDataDB) + .filter(MarketDataDB.market == market_name, MarketDataDB.type == "DAYAHEAD") + .order_by(MarketDataDB.timestamp.desc()) .first() + ) - if not last_dayahead or (datetime.now(UTC) - last_dayahead.timestamp.replace(tzinfo=UTC) > timedelta(hours=1)): + if not last_dayahead or ( + datetime.now(UTC) - last_dayahead.timestamp.replace(tzinfo=UTC) + > timedelta(hours=1) + ): print(f"Getting day-ahead data for {market_name}") now_utc = datetime.now(UTC) day_ahead_date = now_utc.date() - if now_utc.hour >= 18: # After 6PM UTC, markets usually publish next day's data + if ( + now_utc.hour >= 18 + ): # After 6PM UTC, markets usually publish next day's data day_ahead_date += timedelta(days=1) - df = iso.get_lmp(date=day_ahead_date, market="DAY_AHEAD_HOURLY", locations="ALL") - df["Interval Start"] = df["Interval Start"].dt.tz_convert('UTC') - grouped = df.groupby("Interval Start")[["LMP", "Energy", "Congestion", "Loss"]].mean().reset_index() + df = iso.get_lmp( + date=day_ahead_date, market="DAY_AHEAD_HOURLY", locations="ALL" + ) + df["Interval Start"] = df["Interval Start"].dt.tz_convert("UTC") + grouped = ( + df.groupby("Interval Start")[["LMP", "Energy", "Congestion", "Loss"]] + .mean() + .reset_index() + ) for _, row in grouped.iterrows(): - if last_dayahead and row["Interval Start"] <= last_dayahead.timestamp.replace(tzinfo=UTC): + if last_dayahead and row[ + "Interval Start" + ] <= last_dayahead.timestamp.replace(tzinfo=UTC): continue entry = MarketDataDB( timestamp=row["Interval Start"], @@ -83,5 +111,6 @@ def update_market_data(): db.commit() db.close() + if __name__ == "__main__": update_market_data() |