From ad8185ec5b3f4d7a382be7c7aad67ebf32e78036 Mon Sep 17 00:00:00 2001 From: Alex Fox Date: Tue, 20 May 2025 00:18:10 +0000 Subject: [PATCH] rename to remove quickstart --- README.md | 14 +++++++------- dagster_cloud.yaml | 4 ++-- {quickstart_etl => etl}/__init__.py | 0 {quickstart_etl => etl}/assets/__init__.py | 0 {quickstart_etl => etl}/assets/hackernews.py | 0 {quickstart_etl => etl}/definitions.py | 2 +- {quickstart_etl => etl}/py.typed | 0 {quickstart_etl_tests => etl_tests}/__init__.py | 0 {quickstart_etl_tests => etl_tests}/test_defs.py | 2 +- pyproject.toml | 4 ++-- setup.cfg | 2 +- setup.py | 4 ++-- 12 files changed, 16 insertions(+), 16 deletions(-) rename {quickstart_etl => etl}/__init__.py (100%) rename {quickstart_etl => etl}/assets/__init__.py (100%) rename {quickstart_etl => etl}/assets/hackernews.py (100%) rename {quickstart_etl => etl}/definitions.py (92%) rename {quickstart_etl => etl}/py.typed (100%) rename {quickstart_etl_tests => etl_tests}/__init__.py (100%) rename {quickstart_etl_tests => etl_tests}/test_defs.py (62%) diff --git a/README.md b/README.md index c392559..ae4cb5d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ An asset is a software object that models a data asset, which can be a file in y Dagster visualizes upstream and downstream dependencies vertically. Assets below other assets connected by arrows implies a dependency relationship. So we can tell from the UI that the asset `hackernews_topstories` depends on `hackernews_topstory_ids` (i.e. `hackernews_topstories` takes `hackernews_topstory_ids`'s output as an input) and `hackernews_stories_word_cloud` depends on `hackernews_topstories`. -All three assets are defined [in `quickstart_etl/assets/hackernews.py`](./quickstart_etl/assets/hackernews.py). Typically, you'll define assets by annotating ordinary Python functions with the [`@asset`](https://docs.dagster.io/concepts/assets/software-defined-assets#a-basic-software-defined-asset) decorator. +All three assets are defined [in `etl/assets/hackernews.py`](./etl/assets/hackernews.py). Typically, you'll define assets by annotating ordinary Python functions with the [`@asset`](https://docs.dagster.io/concepts/assets/software-defined-assets#a-basic-software-defined-asset) decorator. This project also comes with ways to better organize the assets: @@ -133,7 +133,7 @@ Click **Show Markdown**. You'll see a word cloud of the top 500 HackerNews story

-The metadata is recorded in the `hackernews_topstories_word_cloud` asset [in `quickstart_etl/assets/hackernews.py`](./quickstart_etl/assets/hackernews.py). Dagster supports attaching arbitrary [metadata](https://docs.dagster.io/_apidocs/ops#dagster.MetadataValue) to asset materializations. This metadata is also be displayed on the **Activity** tab of the **Asset Details** page in the UI or in the **Asset Lineage** view after selecting an asset. From the compute logs of a run, you can click the **View Asset** to go to the **Asset Details** page. +The metadata is recorded in the `hackernews_topstories_word_cloud` asset [in `etl/assets/hackernews.py`](./etl/assets/hackernews.py). Dagster supports attaching arbitrary [metadata](https://docs.dagster.io/_apidocs/ops#dagster.MetadataValue) to asset materializations. This metadata is also be displayed on the **Activity** tab of the **Asset Details** page in the UI or in the **Asset Lineage** view after selecting an asset. From the compute logs of a run, you can click the **View Asset** to go to the **Asset Details** page.

@@ -145,7 +145,7 @@ This metadata would be useful for monitoring and maintaining the asset as you it

-In the results, you'll see that the `hackernews_topstories` asset has two metadata entries: `num_records` and `preview`. Both are defined [in `quickstart_etl/assets/hackernews.py`](./quickstart_etl/assets/hackernews.py), in which we record the first five rows of the output Pandas DataFrame in the `preview` metadata entry using the Markdown type. This could help debug and keep your assets easily monitored. Click **Show Markdown** to view a preview of the output data frame: +In the results, you'll see that the `hackernews_topstories` asset has two metadata entries: `num_records` and `preview`. Both are defined [in `etl/assets/hackernews.py`](./etl/assets/hackernews.py), in which we record the first five rows of the output Pandas DataFrame in the `preview` metadata entry using the Markdown type. This could help debug and keep your assets easily monitored. Click **Show Markdown** to view a preview of the output data frame:

@@ -157,7 +157,7 @@ Note: You'll find a `path` metadata attached to every asset. This is because ass Finally, let's refresh our plots every day so we can monitor popular topics over time. To do so, we can use [schedules](https://docs.dagster.io/concepts/partitions-schedules-sensors/schedules#schedules). -We've defined a daily schedule and job in [`quickstart_etl/definitions.py`](./quickstart_etl/definitions.py) for all assets that are defined in the [`quickstart_etl/assets/`](./quickstart_etl/assets) module. +We've defined a daily schedule and job in [`etl/definitions.py`](./etl/definitions.py) for all assets that are defined in the [`etl/assets/`](./etl/assets) module. Now, let's turn on the daily schedule within Dagster. @@ -168,7 +168,7 @@ Now, let's turn on the daily schedule within Dagster.

-You can now turn on the schedule switch to set up the daily job we defined in [quickstart_etl/definitions.py](./quickstart_etl/definitions.py). +You can now turn on the schedule switch to set up the daily job we defined in [etl/definitions.py](./etl/definitions.py).

@@ -221,8 +221,8 @@ You can specify new Python dependencies in `setup.py`. ### Testing -Tests are in the `quickstart_etl_tests` directory and you can run tests using `pytest`: +Tests are in the `etl_tests` directory and you can run tests using `pytest`: ```bash -pytest quickstart_etl_tests +pytest etl_tests ``` diff --git a/dagster_cloud.yaml b/dagster_cloud.yaml index 7f35298..17cf38e 100644 --- a/dagster_cloud.yaml +++ b/dagster_cloud.yaml @@ -1,4 +1,4 @@ locations: - - location_name: quickstart_etl + - location_name: etl code_source: - module_name: quickstart_etl.definitions + module_name: etl.definitions diff --git a/quickstart_etl/__init__.py b/etl/__init__.py similarity index 100% rename from quickstart_etl/__init__.py rename to etl/__init__.py diff --git a/quickstart_etl/assets/__init__.py b/etl/assets/__init__.py similarity index 100% rename from quickstart_etl/assets/__init__.py rename to etl/assets/__init__.py diff --git a/quickstart_etl/assets/hackernews.py b/etl/assets/hackernews.py similarity index 100% rename from quickstart_etl/assets/hackernews.py rename to etl/assets/hackernews.py diff --git a/quickstart_etl/definitions.py b/etl/definitions.py similarity index 92% rename from quickstart_etl/definitions.py rename to etl/definitions.py index af2842d..e9c5a1b 100644 --- a/quickstart_etl/definitions.py +++ b/etl/definitions.py @@ -42,7 +42,7 @@ my_assets = link_code_references_to_git( git_branch="master", file_path_mapping=AnchorBasedFilePathMapping( local_file_anchor=Path(__file__).parent, - file_anchor_path_in_repository="examples/quickstart_etl/quickstart_etl/", + file_anchor_path_in_repository="examples/etl/quickstart_etl/", ), ) diff --git a/quickstart_etl/py.typed b/etl/py.typed similarity index 100% rename from quickstart_etl/py.typed rename to etl/py.typed diff --git a/quickstart_etl_tests/__init__.py b/etl_tests/__init__.py similarity index 100% rename from quickstart_etl_tests/__init__.py rename to etl_tests/__init__.py diff --git a/quickstart_etl_tests/test_defs.py b/etl_tests/test_defs.py similarity index 62% rename from quickstart_etl_tests/test_defs.py rename to etl_tests/test_defs.py index 20bb90e..9573ecb 100644 --- a/quickstart_etl_tests/test_defs.py +++ b/etl_tests/test_defs.py @@ -1,4 +1,4 @@ -from quickstart_etl.definitions import defs +from etl.definitions import defs def test_def_can_load(): diff --git a/pyproject.toml b/pyproject.toml index ef6a022..fa8cce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,5 +3,5 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] -module_name = "quickstart_etl.definitions" -code_location_name = "quickstart_etl" \ No newline at end of file +module_name = "etl.definitions" +code_location_name = "etl" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c39bb92..522eb3b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] -name = quickstart_etl +name = etl diff --git a/setup.py b/setup.py index 30294c8..d014123 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from setuptools import find_packages, setup setup( - name="quickstart_etl", - packages=find_packages(exclude=["quickstart_etl_tests"]), + name="etl", + packages=find_packages(exclude=["etl_tests"]), install_requires=[ "dagster", "dagster-cloud",