Merge pull request Azure#1184 from Azure/release_update/Release-70

update samples from Release-70 as a part of  SDK 1.16.0 release

Author: harneetvirk

configuration.ipynb

@@ -103,7 +103,7 @@
       "source": [
         "import azureml.core\n",
         "\n",
-        "print(\"This notebook was created using version 1.15.0 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.16.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },

contrib/fairness/fairlearn-azureml-mitigation.ipynb

@@ -82,8 +82,7 @@
         "from sklearn import svm\n",
         "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
         "from sklearn.linear_model import LogisticRegression\n",
-        "import pandas as pd\n",
-        "import shap"
+        "import pandas as pd"
       ]
     },
     {
@@ -99,8 +98,12 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "X_raw, Y = shap.datasets.adult()\n",
-        "X_raw[\"Race\"].value_counts().to_dict()"
+        "from sklearn.datasets import fetch_openml\n",
+        "data = fetch_openml(data_id=1590, as_frame=True)\n",
+        "X_raw = data.data\n",
+        "Y = (data.target == '>50K') * 1\n",
+        "\n",
+        "X_raw[\"race\"].value_counts().to_dict()"
       ]
     },
     {
@@ -116,9 +119,13 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "A = X_raw[['Sex','Race']]\n",
-        "X = X_raw.drop(labels=['Sex', 'Race'],axis = 1)\n",
-        "X = pd.get_dummies(X)\n",
+        "A = X_raw[['sex','race']]\n",
+        "X = X_raw.drop(labels=['sex', 'race'],axis = 1)\n",
+        "X_dummies = pd.get_dummies(X)\n",
+        "\n",
+        "sc = StandardScaler()\n",
+        "X_scaled = sc.fit_transform(X_dummies)\n",
+        "X_scaled = pd.DataFrame(X_scaled, columns=X_dummies.columns)\n",
         "\n",
         "\n",
         "le = LabelEncoder()\n",
@@ -139,7 +146,7 @@
       "outputs": [],
       "source": [
         "from sklearn.model_selection import train_test_split\n",
-        "X_train, X_test, Y_train, Y_test, A_train, A_test = train_test_split(X_raw, \n",
+        "X_train, X_test, Y_train, Y_test, A_train, A_test = train_test_split(X_scaled, \n",
         "                                                    Y, \n",
         "                                                    A,\n",
         "                                                    test_size = 0.2,\n",
@@ -150,18 +157,7 @@
         "X_train = X_train.reset_index(drop=True)\n",
         "A_train = A_train.reset_index(drop=True)\n",
         "X_test = X_test.reset_index(drop=True)\n",
-        "A_test = A_test.reset_index(drop=True)\n",
-        "\n",
-        "# Improve labels\n",
-        "A_test.Sex.loc[(A_test['Sex'] == 0)] = 'female'\n",
-        "A_test.Sex.loc[(A_test['Sex'] == 1)] = 'male'\n",
-        "\n",
-        "\n",
-        "A_test.Race.loc[(A_test['Race'] == 0)] = 'Amer-Indian-Eskimo'\n",
-        "A_test.Race.loc[(A_test['Race'] == 1)] = 'Asian-Pac-Islander'\n",
-        "A_test.Race.loc[(A_test['Race'] == 2)] = 'Black'\n",
-        "A_test.Race.loc[(A_test['Race'] == 3)] = 'Other'\n",
-        "A_test.Race.loc[(A_test['Race'] == 4)] = 'White'"
+        "A_test = A_test.reset_index(drop=True)"
       ]
     },
     {
@@ -251,7 +247,7 @@
       "outputs": [],
       "source": [
         "sweep.fit(X_train, Y_train,\n",
-        "          sensitive_features=A_train.Sex)\n",
+        "          sensitive_features=A_train.sex)\n",
         "\n",
         "predictors = sweep._predictors"
       ]
@@ -274,9 +270,9 @@
         "    classifier = lambda X: m.predict(X)\n",
         "    \n",
         "    error = ErrorRate()\n",
-        "    error.load_data(X_train, pd.Series(Y_train), sensitive_features=A_train.Sex)\n",
+        "    error.load_data(X_train, pd.Series(Y_train), sensitive_features=A_train.sex)\n",
         "    disparity = DemographicParity()\n",
-        "    disparity.load_data(X_train, pd.Series(Y_train), sensitive_features=A_train.Sex)\n",
+        "    disparity.load_data(X_train, pd.Series(Y_train), sensitive_features=A_train.sex)\n",
         "    \n",
         "    errors.append(error.gamma(classifier)[0])\n",
         "    disparities.append(disparity.gamma(classifier).max())\n",
@@ -440,7 +436,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "sf = { 'sex': A_test.Sex, 'race': A_test.Race }\n",
+        "sf = { 'sex': A_test.sex, 'race': A_test.race }\n",
         "\n",
         "from fairlearn.metrics._group_metric_set import _create_group_metric_set\n",
         "\n",

contrib/fairness/fairlearn-azureml-mitigation.yml

@@ -5,4 +5,3 @@ dependencies:
   - azureml-contrib-fairness
   - fairlearn==0.4.6
   - joblib
-  - shap

contrib/fairness/upload-fairness-dashboard.ipynb

@@ -82,8 +82,7 @@
         "from sklearn import svm\n",
         "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
         "from sklearn.linear_model import LogisticRegression\n",
-        "import pandas as pd\n",
-        "import shap"
+        "import pandas as pd"
       ]
     },
     {
@@ -99,7 +98,10 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "X_raw, Y = shap.datasets.adult()"
+        "from sklearn.datasets import fetch_openml\n",
+        "data = fetch_openml(data_id=1590, as_frame=True)\n",
+        "X_raw = data.data\n",
+        "Y = (data.target == '>50K') * 1"
       ]
     },
     {
@@ -115,7 +117,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(X_raw[\"Race\"].value_counts().to_dict())"
+        "print(X_raw[\"race\"].value_counts().to_dict())"
       ]
     },
     {
@@ -134,9 +136,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "A = X_raw[['Sex','Race']]\n",
-        "X = X_raw.drop(labels=['Sex', 'Race'],axis = 1)\n",
-        "X = pd.get_dummies(X)"
+        "A = X_raw[['sex','race']]\n",
+        "X = X_raw.drop(labels=['sex', 'race'],axis = 1)\n",
+        "X_dummies = pd.get_dummies(X)"
       ]
     },
     {
@@ -153,8 +155,8 @@
       "outputs": [],
       "source": [
         "sc = StandardScaler()\n",
-        "X_scaled = sc.fit_transform(X)\n",
-        "X_scaled = pd.DataFrame(X_scaled, columns=X.columns)\n",
+        "X_scaled = sc.fit_transform(X_dummies)\n",
+        "X_scaled = pd.DataFrame(X_scaled, columns=X_dummies.columns)\n",
         "\n",
         "le = LabelEncoder()\n",
         "Y = le.fit_transform(Y)"
@@ -185,18 +187,7 @@
         "X_train = X_train.reset_index(drop=True)\n",
         "A_train = A_train.reset_index(drop=True)\n",
         "X_test = X_test.reset_index(drop=True)\n",
-        "A_test = A_test.reset_index(drop=True)\n",
-        "\n",
-        "# Improve labels\n",
-        "A_test.Sex.loc[(A_test['Sex'] == 0)] = 'female'\n",
-        "A_test.Sex.loc[(A_test['Sex'] == 1)] = 'male'\n",
-        "\n",
-        "\n",
-        "A_test.Race.loc[(A_test['Race'] == 0)] = 'Amer-Indian-Eskimo'\n",
-        "A_test.Race.loc[(A_test['Race'] == 1)] = 'Asian-Pac-Islander'\n",
-        "A_test.Race.loc[(A_test['Race'] == 2)] = 'Black'\n",
-        "A_test.Race.loc[(A_test['Race'] == 3)] = 'Other'\n",
-        "A_test.Race.loc[(A_test['Race'] == 4)] = 'White'"
+        "A_test = A_test.reset_index(drop=True)"
       ]
     },
     {
@@ -380,7 +371,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "sf = { 'Race': A_test.Race, 'Sex': A_test.Sex }\n",
+        "sf = { 'Race': A_test.race, 'Sex': A_test.sex }\n",
         "\n",
         "from fairlearn.metrics._group_metric_set import _create_group_metric_set\n",
         "\n",
@@ -499,7 +490,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.8"
+      "version": "3.6.10"
     }
   },
   "nbformat": 4,

contrib/fairness/upload-fairness-dashboard.yml

@@ -5,4 +5,3 @@ dependencies:
   - azureml-contrib-fairness
   - fairlearn==0.4.6
   - joblib
-  - shap

how-to-use-azureml/automated-machine-learning/README.md

@@ -173,7 +173,7 @@ The main code of the file must be indented so that it is under this condition.
 ## automl_setup fails
 1. On Windows, make sure that you are running automl_setup from an Anconda Prompt window rather than a regular cmd window.  You can launch the "Anaconda Prompt" window by hitting the Start button and typing "Anaconda Prompt".  If you don't see the application "Anaconda Prompt", you might not have conda or mini conda installed.  In that case, you can install it [here](https://conda.io/miniconda.html)
 2. Check that you have conda 64-bit installed rather than 32-bit.  You can check this with the command `conda info`.  The `platform` should be `win-64` for Windows or `osx-64` for Mac.
-3. Check that you have conda 4.4.10 or later.  You can check the version with the command `conda -V`.  If you have a previous version installed, you can update it using the command: `conda update conda`.
+3. Check that you have conda 4.7.8 or later.  You can check the version with the command `conda -V`.  If you have a previous version installed, you can update it using the command: `conda update conda`.
 4. On Linux, if the error is `gcc: error trying to exec 'cc1plus': execvp: No such file or directory`, install build essentials using the command `sudo apt-get install build-essential`.
 5. Pass a new name as the first parameter to automl_setup so that it creates a new conda environment. You can view existing conda environments using `conda env list` and remove them with `conda env remove -n <environmentname>`.
 

how-to-use-azureml/automated-machine-learning/automl_env.yml

@@ -6,7 +6,7 @@ dependencies:
 - python>=3.5.2,<3.6.8
 - nb_conda
 - matplotlib==2.1.0
-- numpy~=1.18.0
+- numpy==1.18.5
 - cython
 - urllib3<1.24
 - scipy==1.4.1
@@ -24,5 +24,5 @@ dependencies:
   - pytorch-transformers==1.0.0
   - spacy==2.1.8
   - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
-  - -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.15.0/validated_win32_requirements.txt [--no-deps]
+  - -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.16.0/validated_win32_requirements.txt [--no-deps]
 

how-to-use-azureml/automated-machine-learning/automl_env_linux.yml

@@ -6,7 +6,7 @@ dependencies:
 - python>=3.5.2,<3.6.8
 - nb_conda
 - matplotlib==2.1.0
-- numpy~=1.18.0
+- numpy==1.18.5
 - cython
 - urllib3<1.24
 - scipy==1.4.1
@@ -24,5 +24,5 @@ dependencies:
   - pytorch-transformers==1.0.0
   - spacy==2.1.8
   - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
-  - -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.15.0/validated_linux_requirements.txt [--no-deps]
+  - -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.16.0/validated_linux_requirements.txt [--no-deps]
 

how-to-use-azureml/automated-machine-learning/automl_env_mac.yml

@@ -7,7 +7,7 @@ dependencies:
 - python>=3.5.2,<3.6.8
 - nb_conda
 - matplotlib==2.1.0
-- numpy~=1.18.0
+- numpy==1.18.5
 - cython
 - urllib3<1.24
 - scipy==1.4.1
@@ -25,4 +25,4 @@ dependencies:
   - pytorch-transformers==1.0.0
   - spacy==2.1.8
   - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz  
-  - -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.15.0/validated_darwin_requirements.txt [--no-deps]
+  - -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.16.0/validated_darwin_requirements.txt [--no-deps]

how-to-use-azureml/automated-machine-learning/automl_setup.cmd

@@ -6,11 +6,22 @@ set PIP_NO_WARN_SCRIPT_LOCATION=0
 
 IF "%conda_env_name%"=="" SET conda_env_name="azure_automl"
 IF "%automl_env_file%"=="" SET automl_env_file="automl_env.yml"
+SET check_conda_version_script="check_conda_version.py"
 
 IF NOT EXIST %automl_env_file% GOTO YmlMissing
 
 IF "%CONDA_EXE%"=="" GOTO CondaMissing
 
+IF NOT EXIST %check_conda_version_script% GOTO VersionCheckMissing
+
+python "%check_conda_version_script%"
+IF errorlevel 1 GOTO ErrorExit:
+
+SET replace_version_script="replace_latest_version.ps1"
+IF EXIST %replace_version_script% (
+  powershell -file %replace_version_script% %automl_env_file%
+)
+
 call conda activate %conda_env_name% 2>nul:
 
 if not errorlevel 1 (
@@ -54,6 +65,10 @@ echo If you are running an older version of Miniconda or Anaconda,
 echo you can upgrade using the command: conda update conda
 goto End
 
+:VersionCheckMissing
+echo File %check_conda_version_script% not found.
+goto End
+
 :YmlMissing
 echo File %automl_env_file% not found.