Skip to content
Open
Changes from 1 commit
Commits
Show all changes
67 commits
Select commit Hold shift + click to select a range
c0f61f3
added dagster_defs and dg.Dockerfile
jkislin Oct 7, 2025
c5c46ea
simplified names
jkislin Oct 7, 2025
08b1939
moved dagster to its own folder
jkislin Oct 7, 2025
842d7a4
dagster_defs.py a la sandbox; containerfile updates to include dagster
jkislin Oct 8, 2025
8996067
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 9, 2025
066894b
Merge branch 'main' into jk-dagster-sandbox
jkislin Oct 9, 2025
5307f58
quick fix to dagster_defs.py
Oct 14, 2025
6746820
some simplification for dagster. pending debugging tomorrow.
jkislin Oct 14, 2025
f2df9ef
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Oct 14, 2025
3472034
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 14, 2025
631f982
tons of incremental updates toward a working dagster pyrenew-h. check…
jkislin Oct 16, 2025
38cd584
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 16, 2025
500df4d
blobfuse mounts in local dagster docker executor!
jkislin Oct 17, 2025
01c2497
pyrenew-h-output for now
jkislin Oct 21, 2025
d0e6947
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Oct 21, 2025
884bb44
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 21, 2025
dbcf189
Merge branch 'main' of https://github.com/CDCgov/pyrenew-hew into jk-…
jkislin Oct 21, 2025
c5d9de6
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Oct 21, 2025
947d34d
output subdir back to ./
jkislin Oct 21, 2025
60e9f6c
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 21, 2025
07aa576
fix asset execution context
jkislin Oct 21, 2025
abd1ed4
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Oct 21, 2025
aa03b33
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 21, 2025
5264571
pyrenew asset builder
jkislin Oct 21, 2025
125c4ff
An initial working example for pyrenew-hew!
jkislin Oct 29, 2025
f10171a
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Oct 29, 2025
282c869
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 29, 2025
40ec05a
readme update and testing for the timeseries and batch executor
jkislin Nov 3, 2025
9235f55
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 3, 2025
2609614
blobfuse refinements
jkislin Nov 3, 2025
1082032
switch to docker executor for testing
jkislin Nov 3, 2025
499a753
caj executor docker image tag
jkislin Nov 4, 2025
bf1116e
fixed batch executor source volumes
jkislin Nov 4, 2025
d23d306
updated docker executor to also user username versioned tags
jkislin Nov 4, 2025
8f57a82
reverting to batch executor for testing
jkislin Nov 4, 2025
5ba84d3
Success! Batch Executor works
jkislin Nov 4, 2025
82a762e
some mount cleanup
jkislin Nov 21, 2025
e981428
update definitions
jkislin Nov 25, 2025
e6c9b09
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 25, 2025
c3369d9
remove unnecessary commented and redundant code
jkislin Dec 2, 2025
bf6a20d
Merge branch 'main' of https://github.com/CDCgov/pyrenew-hew into jk-…
jkislin Dec 2, 2025
d615505
adding env variables for Azure Command Center
jkislin Dec 2, 2025
3da4c81
allow different models to have different partitions
jkislin Dec 2, 2025
40a72cb
an attempt at clean venv separation between pyrenew and dagster
jkislin Dec 2, 2025
b8d1f1a
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 2, 2025
74cd8a7
Merge branch 'main' into jk-dagster-sandbox
jkislin Dec 4, 2025
a3d891d
simplified blobfuse
jkislin Dec 4, 2025
e3f1a2b
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Dec 4, 2025
eaaf510
merged the dagster and project dependencies together
jkislin Dec 5, 2025
b2415d3
Merge branch 'main' of https://github.com/CDCgov/pyrenew-hew into jk-…
jkislin Dec 5, 2025
1a77853
new dagster_defs with precommit run
jkislin Dec 5, 2025
93f7e7b
Merge branch 'main' into jk-dagster-sandbox
jkislin Dec 9, 2025
df0d8ed
partition definitions are all the same; we exclude in runs themselves…
jkislin Dec 10, 2025
c592804
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Dec 10, 2025
612cff9
Merge branch 'main' of https://github.com/CDCgov/pyrenew-hew into jk-…
jkislin Dec 10, 2025
21f1d9f
preparing for central code location usage; updated cfa dagster
jkislin Dec 11, 2025
59781de
readme, github actions, makefile
jkislin Dec 11, 2025
aa67bb3
Merge branch 'main' into jk-dagster-sandbox
jkislin Dec 11, 2025
7b49243
Update blobfuse/README.md
jkislin Dec 11, 2025
b7190ae
Update Makefile
jkislin Dec 11, 2025
dc1d8cb
replaced ignoring dagster_defs.py with ruff format off for a single c…
jkislin Dec 11, 2025
89e8a59
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Dec 11, 2025
c28c60f
Apply suggestion from @Copilot
jkislin Dec 11, 2025
30ead07
images
jkislin Dec 11, 2025
1ad893f
Merge branch 'main' into jk-dagster-sandbox
jkislin Dec 12, 2025
09e0670
add context to run_pyrenew_model calls
jkislin Dec 12, 2025
5e8f858
Merge branch 'jk-dagster-sandbox' of https://github.com/CDCgov/pyrene…
jkislin Dec 12, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
pyrenew-h-output for now
  • Loading branch information
jkislin committed Oct 21, 2025
commit 01c24979ea29b6814cea150c76b5554799d8ec23
27 changes: 18 additions & 9 deletions dagster_defs.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,16 +128,28 @@ class PyrenewAssetConfig(dg.Config):
# # These should generate the outputs by submitting to azure batch.
# return "pyrenew-hew-output"

disease_list = ["COVID-19", "INFLUENZA", "RSV"]
disease_list = ["COVID-19", "Influenza", "RSV"]
disease_partitions = dg.StaticPartitionsDefinition(disease_list)

state_list = ["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA"]
state_list = [
"AL", "AK", "AZ", "AR",
"CA", "CO", "CT", "DE",
"FL", "GA", "HI", "ID",
"IL", "IN", "IA", "KS",
"KY", "LA", "ME", "MD",
"MA", "MI", "MN", "MS",
"MO", "MT", "NE", "NV",
"NH", "NJ", "NM", "NY",
"NC", "ND", "OH", "OK",
"OR", "PA", "RI", "SC",
"SD", "TN", "TX", "UT",
"VT", "VA", "WA", "WV",
"WI", "WY", "DC", "US"
]
state_partitions = dg.StaticPartitionsDefinition(state_list)
two_dimensional_partitions = dg.MultiPartitionsDefinition(
{"disease": disease_partitions, "loc": state_partitions}
)


class PyrenewHOutputConfig(dg.Config):
# when using the docker_executor, specify the image you'd like to use
image: str = "pyrenew-dagster"
Expand All @@ -161,7 +173,7 @@ def pyrenew_h_output(
model_letters = "h"
n_warmup = 1000
additional_forecast_letters = "h"
output_subdir = "./"
output_subdir = "."
additional_args = (
f"--n-warmup {n_warmup} "
"--nwss-data-dir nwss-vintages "
Expand Down Expand Up @@ -196,10 +208,7 @@ def pyrenew_h_output(
output_dir=str(Path("output", output_subdir)),
)
run = subprocess.run(base_call, shell=True, check=True)
run.check_returncode()
print(run.stderr)
print(run.stdout)
return run
return "pyrenew-h-output"


# @dg.asset(
Expand Down