arfox/nyboe-dag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: arfox:main
Branches Tags
arfox:main
..
compare: arfox:46c8fbe63ff27ac9d83c42a7d5a5c9038c31c139
Branches Tags
arfox:main

1 Commits
main ... 46c8fbe63f

Author SHA1 Message Date
Alex Fox
46c8fbe63f evade tls fingerprinting 2025-05-24 23:59:53 +00:00
1 changed files with 6 additions and 5 deletions
Expand all files Collapse all files

11
quickstart_etl/assets/nyboe.py
View File

@@ -73,22 +73,23 @@ NY_DAILY_PARTITION = dg.DailyPartitionsDefinition(
) )
def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult: def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult:
"""Fetch the day before the partition date""" """Fetch the day before the partition date"""
target_date = pendulum.parse(context.partition_key) end_date = pendulum.parse(context.partition_key).subtract(days=1)
start_date = end_date.subtract(days=1)
with curl_cffi.Session() as s: with curl_cffi.Session() as s:
res = get_cookies(s, target_date, target_date) res = get_cookies(s, start_date, end_date)
if not res.json()["aaData"]: if not res.json()["aaData"]:
return None return None
req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure", req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure",
params=gen_ie_query(target_date, target_date), params=gen_ie_query(start_date, end_date),
impersonate="chrome", impersonate="chrome",
) )
df = pd.read_csv(StringIO(req.text), index_col=False) df = pd.read_csv(StringIO(req.text), index_col=False)
engine = create_engine("postgresql://superset:PASSWORD@IP_ADDR/superset") engine = create_engine("postgresql://superset:aeMuiRied2ke0doigaew@192.168.68.69/superset")
df.to_sql( df.to_sql(
"independent_expenditures_raw", "independent_expenditures_raw_redo",
con=engine, con=engine,
if_exists="append", if_exists="append",
) )