arfox/nyboe-dag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adjust date targeting

Browse Source
This commit is contained in:
Alex Fox
2025-05-25 22:04:09 +00:00
parent 4e4f37c9fc
commit cee5b89127
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
quickstart_etl/assets/nyboe.py
View File

@@ -73,23 +73,22 @@ NY_DAILY_PARTITION = dg.DailyPartitionsDefinition(
) )
def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult: def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult:
"""Fetch the day before the partition date""" """Fetch the day before the partition date"""
end_date = pendulum.parse(context.partition_key).subtract(days=1) target_date = pendulum.parse(context.partition_key)
start_date = end_date.subtract(days=1)
with curl_cffi.Session() as s: with curl_cffi.Session() as s:
res = get_cookies(s, start_date, end_date) res = get_cookies(s, target_date, target_date)
if not res.json()["aaData"]: if not res.json()["aaData"]:
return None return None
req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure", req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure",
params=gen_ie_query(start_date, end_date), params=gen_ie_query(target_date, target_date),
impersonate="chrome", impersonate="chrome",
) )
df = pd.read_csv(StringIO(req.text), index_col=False) df = pd.read_csv(StringIO(req.text), index_col=False)
engine = create_engine("postgresql://superset:PASSWORD@IP_ADDR/superset") engine = create_engine("postgresql://superset:PASSWORD@IP_ADDR/superset")
df.to_sql( df.to_sql(
"independent_expenditures_raw_redo", "independent_expenditures_raw",
con=engine, con=engine,
if_exists="append", if_exists="append",
) )