diff --git a/quickstart_etl/assets/nyboe.py b/quickstart_etl/assets/nyboe.py index 190a748..7f5ad95 100644 --- a/quickstart_etl/assets/nyboe.py +++ b/quickstart_etl/assets/nyboe.py @@ -73,23 +73,22 @@ NY_DAILY_PARTITION = dg.DailyPartitionsDefinition( ) def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult: """Fetch the day before the partition date""" - end_date = pendulum.parse(context.partition_key).subtract(days=1) + target_date = pendulum.parse(context.partition_key) - start_date = end_date.subtract(days=1) with curl_cffi.Session() as s: - res = get_cookies(s, start_date, end_date) + res = get_cookies(s, target_date, target_date) if not res.json()["aaData"]: return None req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure", - params=gen_ie_query(start_date, end_date), + params=gen_ie_query(target_date, target_date), impersonate="chrome", ) df = pd.read_csv(StringIO(req.text), index_col=False) engine = create_engine("postgresql://superset:PASSWORD@IP_ADDR/superset") df.to_sql( - "independent_expenditures_raw_redo", + "independent_expenditures_raw", con=engine, if_exists="append", )