adjust date targeting
This commit is contained in:
@@ -73,23 +73,22 @@ NY_DAILY_PARTITION = dg.DailyPartitionsDefinition(
|
||||
)
|
||||
def fetch_expenditures(context: dg.AssetExecutionContext) -> dg.MaterializeResult:
|
||||
"""Fetch the day before the partition date"""
|
||||
end_date = pendulum.parse(context.partition_key).subtract(days=1)
|
||||
target_date = pendulum.parse(context.partition_key)
|
||||
|
||||
start_date = end_date.subtract(days=1)
|
||||
with curl_cffi.Session() as s:
|
||||
res = get_cookies(s, start_date, end_date)
|
||||
res = get_cookies(s, target_date, target_date)
|
||||
if not res.json()["aaData"]:
|
||||
return None
|
||||
|
||||
req = s.get(f"{IE_ENDPOINT}/IndependentExpenditure",
|
||||
params=gen_ie_query(start_date, end_date),
|
||||
params=gen_ie_query(target_date, target_date),
|
||||
impersonate="chrome",
|
||||
)
|
||||
df = pd.read_csv(StringIO(req.text), index_col=False)
|
||||
|
||||
engine = create_engine("postgresql://superset:PASSWORD@IP_ADDR/superset")
|
||||
df.to_sql(
|
||||
"independent_expenditures_raw_redo",
|
||||
"independent_expenditures_raw",
|
||||
con=engine,
|
||||
if_exists="append",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user