|
24 | 24 | POSSIBILITY OF SUCH DAMAGE. |
25 | 25 | """ |
26 | 26 | import os |
27 | | -from azureml.core import Model, Run |
| 27 | +from azureml.core import Model, Run, Workspace, Experiment |
28 | 28 | import argparse |
| 29 | +from azureml.core.authentication import ServicePrincipalAuthentication |
| 30 | +import traceback |
29 | 31 |
|
30 | | - |
31 | | -# Get workspace |
32 | 32 | run = Run.get_context() |
33 | | -exp = run.experiment |
34 | | -ws = run.experiment.workspace |
| 33 | +if (run.id.startswith('OfflineRun')): |
| 34 | + from dotenv import load_dotenv |
| 35 | + # For local development, set values in this section |
| 36 | + load_dotenv() |
| 37 | + workspace_name = os.environ.get("WORKSPACE_NAME") |
| 38 | + experiment_name = os.environ.get("EXPERIMENT_NAME") |
| 39 | + resource_group = os.environ.get("RESOURCE_GROUP") |
| 40 | + subscription_id = os.environ.get("SUBSCRIPTION_ID") |
| 41 | + tenant_id = os.environ.get("TENANT_ID") |
| 42 | + model_name = os.environ.get("MODEL_NAME") |
| 43 | + app_id = os.environ.get('SP_APP_ID') |
| 44 | + app_secret = os.environ.get('SP_APP_SECRET') |
| 45 | + build_id = os.environ.get('BUILD_BUILDID') |
| 46 | + service_principal = ServicePrincipalAuthentication( |
| 47 | + tenant_id=tenant_id, |
| 48 | + service_principal_id=app_id, |
| 49 | + service_principal_password=app_secret) |
35 | 50 |
|
| 51 | + aml_workspace = Workspace.get( |
| 52 | + name=workspace_name, |
| 53 | + subscription_id=subscription_id, |
| 54 | + resource_group=resource_group, |
| 55 | + auth=service_principal |
| 56 | + ) |
| 57 | + ws = aml_workspace |
| 58 | + exp = Experiment(ws, experiment_name) |
| 59 | + run_id = "e78b2c27-5ceb-49d9-8e84-abe7aecf37d5" |
| 60 | +else: |
| 61 | + exp = run.experiment |
| 62 | + ws = run.experiment.workspace |
| 63 | + run_id = 'amlcompute' |
36 | 64 |
|
37 | 65 | parser = argparse.ArgumentParser("evaluate") |
38 | 66 | parser.add_argument( |
39 | | - "--release_id", |
| 67 | + "--build_id", |
| 68 | + type=str, |
| 69 | + help="The Build ID of the build triggering this pipeline run", |
| 70 | +) |
| 71 | +parser.add_argument( |
| 72 | + "--run_id", |
40 | 73 | type=str, |
41 | | - help="The ID of the release triggering this pipeline run", |
| 74 | + help="Training run ID", |
42 | 75 | ) |
43 | 76 | parser.add_argument( |
44 | 77 | "--model_name", |
45 | 78 | type=str, |
46 | 79 | help="Name of the Model", |
47 | 80 | default="sklearn_regression_model.pkl", |
48 | 81 | ) |
49 | | -args = parser.parse_args() |
50 | 82 |
|
51 | | -print("Argument 1: %s" % args.release_id) |
52 | | -print("Argument 2: %s" % args.model_name) |
| 83 | +args = parser.parse_args() |
| 84 | +if (args.build_id is not None): |
| 85 | + build_id = args.build_id |
| 86 | +if (args.run_id is not None): |
| 87 | + run_id = args.run_id |
| 88 | +if (run_id == 'amlcompute'): |
| 89 | + run_id = run.parent.id |
53 | 90 | model_name = args.model_name |
54 | | -release_id = args.release_id |
| 91 | +metric_eval = "mse" |
| 92 | +run.tag("BuildId", value=build_id) |
55 | 93 |
|
56 | | -# Paramaterize the matrics on which the models should be compared |
| 94 | +# Paramaterize the matrices on which the models should be compared |
57 | 95 | # Add golden data set on which all the model performance can be evaluated |
58 | | - |
59 | | -all_runs = exp.get_runs( |
60 | | - properties={"release_id": release_id, "run_type": "train"}, |
61 | | - include_children=True |
62 | | - ) |
63 | | -new_model_run = next(all_runs) |
64 | | -new_model_run_id = new_model_run.id |
65 | | -print(f'New Run found with Run ID of: {new_model_run_id}') |
66 | | - |
67 | 96 | try: |
68 | | - # Get most recently registered model, we assume that |
69 | | - # is the model in production. |
70 | | - # Download this model and compare it with the recently |
71 | | - # trained model by running test with same data set. |
72 | 97 | model_list = Model.list(ws) |
73 | | - production_model = next( |
74 | | - filter( |
75 | | - lambda x: x.created_time == max( |
76 | | - model.created_time for model in model_list), |
77 | | - model_list, |
| 98 | + if (len(model_list) > 0): |
| 99 | + production_model = next( |
| 100 | + filter( |
| 101 | + lambda x: x.created_time == max( |
| 102 | + model.created_time for model in model_list), |
| 103 | + model_list, |
| 104 | + ) |
78 | 105 | ) |
79 | | - ) |
80 | | - production_model_run_id = production_model.tags.get("run_id") |
81 | | - run_list = exp.get_runs() |
| 106 | + production_model_run_id = production_model.run_id |
82 | 107 |
|
83 | | - # Get the run history for both production model and |
84 | | - # newly trained model and compare mse |
85 | | - production_model_run = Run(exp, run_id=production_model_run_id) |
86 | | - new_model_run = Run(exp, run_id=new_model_run_id) |
| 108 | + # Get the run history for both production model and |
| 109 | + # newly trained model and compare mse |
| 110 | + production_model_run = Run(exp, run_id=production_model_run_id) |
| 111 | + new_model_run = run.parent |
| 112 | + print("Production model run is", production_model_run) |
87 | 113 |
|
88 | | - production_model_mse = production_model_run.get_metrics().get("mse") |
89 | | - new_model_mse = new_model_run.get_metrics().get("mse") |
90 | | - print( |
91 | | - "Current Production model mse: {}, New trained model mse: {}".format( |
92 | | - production_model_mse, new_model_mse |
93 | | - ) |
94 | | - ) |
| 114 | + production_model_mse = \ |
| 115 | + production_model_run.get_metrics().get(metric_eval) |
| 116 | + new_model_mse = new_model_run.get_metrics().get(metric_eval) |
| 117 | + if (production_model_mse is None or new_model_mse is None): |
| 118 | + print("Unable to find", metric_eval, "metrics, " |
| 119 | + "exiting evaluation") |
| 120 | + run.parent.cancel() |
| 121 | + else: |
| 122 | + print( |
| 123 | + "Current Production model mse: {}, " |
| 124 | + "New trained model mse: {}".format( |
| 125 | + production_model_mse, new_model_mse |
| 126 | + ) |
| 127 | + ) |
95 | 128 |
|
96 | | - promote_new_model = False |
97 | | - if new_model_mse < production_model_mse: |
98 | | - promote_new_model = True |
99 | | - print("New trained model performs better, thus it will be registered") |
| 129 | + if (new_model_mse < production_model_mse): |
| 130 | + print("New trained model performs better, " |
| 131 | + "thus it should be registered") |
| 132 | + else: |
| 133 | + print("New trained model metric is less than or equal to " |
| 134 | + "production model so skipping model registration.") |
| 135 | + run.parent.cancel() |
| 136 | + else: |
| 137 | + print("This is the first model, " |
| 138 | + "thus it should be registered") |
100 | 139 | except Exception: |
101 | | - promote_new_model = True |
102 | | - print("This is the first model to be trained, \ |
103 | | - thus nothing to evaluate for now") |
104 | | - |
105 | | - |
106 | | -# Writing the run id to /aml_config/run_id.json |
107 | | -if promote_new_model: |
108 | | - model_path = os.path.join('outputs', model_name) |
109 | | - new_model_run.register_model( |
110 | | - model_name=model_name, |
111 | | - model_path=model_path, |
112 | | - properties={"release_id": release_id}) |
113 | | - print("Registered new model!") |
| 140 | + traceback.print_exc(limit=None, file=None, chain=True) |
| 141 | + print("Something went wrong trying to evaluate. Exiting.") |
| 142 | + raise |
0 commit comments