adjust date targeting

This commit is contained in:
Alex Fox
2025-05-25 22:04:09 +00:00
parent 4e4f37c9fc
commit cee5b89127

View File

@@ -73,23 +73,22 @@ NY_DAILY_PARTITION = dg.DailyPartitionsDefinition(
)
def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult:
"""Fetch the day before the partition date"""
end_date = pendulum.parse(context.partition_key).subtract(days=1)
target_date = pendulum.parse(context.partition_key)
start_date = end_date.subtract(days=1)
with curl_cffi.Session() as s:
res = get_cookies(s, start_date, end_date)
res = get_cookies(s, target_date, target_date)
if not res.json()["aaData"]:
return None
req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure",
params=gen_ie_query(start_date, end_date),
params=gen_ie_query(target_date, target_date),
impersonate="chrome",
)
df = pd.read_csv(StringIO(req.text), index_col=False)
engine = create_engine("postgresql://superset:PASSWORD@IP_ADDR/superset")
df.to_sql(
"independent_expenditures_raw_redo",
"independent_expenditures_raw",
con=engine,
if_exists="append",
)