From 292e28bbb5a9488de1853a985a994882781068ac Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 18 Oct 2021 16:59:40 +0000 Subject: [PATCH 001/262] Acquire SSH cert from Cloud Shell IMDS Cloud Shell Detection PoC: Silent flow utilizes Cloud Shell IMDS Introduce get_accounts(username=msal.CURRENT_USER) A reasonable-effort to convert scope to resource Replace get_accounts(username=msal.CURRENT_USER) by acquire_token_interactive(..., prompt="none") Detect unsupported Portal so that AzCLI could fallback --- msal/application.py | 33 ++++++++++-- msal/cloudshell.py | 122 ++++++++++++++++++++++++++++++++++++++++++++ msal/token_cache.py | 9 ++-- tests/test_e2e.py | 17 ++++++ 4 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 msal/cloudshell.py diff --git a/msal/application.py b/msal/application.py index 7ca62d7c..812abbfb 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,13 +21,14 @@ import msal.telemetry from .region import _detect_region from .throttled_http_client import ThrottledHttpClient +from .cloudshell import _is_running_in_cloud_shell # The __init__.py will import this. Not the other way around. __version__ = "1.17.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) - +_AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" def extract_certs(public_cert_content): # Parses raw public certificate file contents and returns a list of strings @@ -986,6 +987,10 @@ def get_accounts(self, username=None): return accounts def _find_msal_accounts(self, environment): + interested_authority_types = [ + TokenCache.AuthorityType.ADFS, TokenCache.AuthorityType.MSSTS] + if _is_running_in_cloud_shell(): + interested_authority_types.append(_AUTHORITY_TYPE_CLOUDSHELL) grouped_accounts = { a.get("home_account_id"): # Grouped by home tenant's id { # These are minimal amount of non-tenant-specific account info @@ -1001,8 +1006,7 @@ def _find_msal_accounts(self, environment): for a in self.token_cache.find( TokenCache.CredentialType.ACCOUNT, query={"environment": environment}) - if a["authority_type"] in ( - TokenCache.AuthorityType.ADFS, TokenCache.AuthorityType.MSSTS) + if a["authority_type"] in interested_authority_types } return list(grouped_accounts.values()) @@ -1062,6 +1066,21 @@ def _forget_me(self, home_account): TokenCache.CredentialType.ACCOUNT, query=owned_by_home_account): self.token_cache.remove_account(a) + def _acquire_token_by_cloud_shell(self, scopes, data=None): + from .cloudshell import _obtain_token + response = _obtain_token( + self.http_client, scopes, client_id=self.client_id, data=data) + if "error" not in response: + self.token_cache.add(dict( + client_id=self.client_id, + scope=response["scope"].split() if "scope" in response else scopes, + token_endpoint=self.authority.token_endpoint, + response=response.copy(), + data=data or {}, + authority_type=_AUTHORITY_TYPE_CLOUDSHELL, + )) + return response + def acquire_token_silent( self, scopes, # type: List[str] @@ -1195,6 +1214,7 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( authority, # This can be different than self.authority force_refresh=False, # type: Optional[boolean] claims_challenge=None, + correlation_id=None, **kwargs): access_token_from_cache = None if not (force_refresh or claims_challenge): # Bypass AT when desired or using claims @@ -1233,9 +1253,13 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( refresh_reason = msal.telemetry.FORCE_REFRESH # TODO: It could also mean claims_challenge assert refresh_reason, "It should have been established at this point" try: + if account and account.get("authority_type") == _AUTHORITY_TYPE_CLOUDSHELL: + return self._acquire_token_by_cloud_shell( + scopes, data=kwargs.get("data")) result = _clean_up(self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( authority, self._decorate_scope(scopes), account, refresh_reason=refresh_reason, claims_challenge=claims_challenge, + correlation_id=correlation_id, **kwargs)) if (result and "error" not in result) or (not access_token_from_cache): return result @@ -1574,6 +1598,9 @@ def acquire_token_interactive( - A dict containing an "error" key, when token refresh failed. """ self._validate_ssh_cert_input_data(kwargs.get("data", {})) + if _is_running_in_cloud_shell() and prompt == "none": + return self._acquire_token_by_cloud_shell( + scopes, data=kwargs.pop("data", {})) claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) telemetry_context = self._build_telemetry_context( diff --git a/msal/cloudshell.py b/msal/cloudshell.py new file mode 100644 index 00000000..f4feaf44 --- /dev/null +++ b/msal/cloudshell.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft Corporation. +# All rights reserved. +# +# This code is licensed under the MIT License. + +"""This module wraps Cloud Shell's IMDS-like interface inside an OAuth2-like helper""" +import base64 +import json +import logging +import os +import time +try: # Python 2 + from urlparse import urlparse +except: # Python 3 + from urllib.parse import urlparse +from .oauth2cli.oidc import decode_part + + +logger = logging.getLogger(__name__) + + +def _is_running_in_cloud_shell(): + return os.environ.get("AZUREPS_HOST_ENVIRONMENT", "").startswith("cloud-shell") + + +def _scope_to_resource(scope): # This is an experimental reasonable-effort approach + cloud_shell_supported_audiences = [ + "https://analysis.windows.net/powerbi/api", # Came from https://msazure.visualstudio.com/One/_git/compute-CloudShell?path=/src/images/agent/env/envconfig.PROD.json + "https://pas.windows.net/CheckMyAccess/Linux/.default", # Cloud Shell accepts it as-is + ] + for a in cloud_shell_supported_audiences: + if scope.startswith(a): + return a + u = urlparse(scope) + if u.scheme: + return "{}://{}".format(u.scheme, u.netloc) + return scope # There is no much else we can do here + + +def _obtain_token(http_client, scopes, client_id=None, data=None): + resp = http_client.post( + "http://localhost:50342/oauth2/token", + data=dict( + data or {}, + resource=" ".join(map(_scope_to_resource, scopes))), + headers={"Metadata": "true"}, + ) + if resp.status_code >= 300: + logger.debug("Cloud Shell IMDS error: %s", resp.text) + cs_error = json.loads(resp.text).get("error", {}) + return {k: v for k, v in { + "error": cs_error.get("code"), + "error_description": cs_error.get("message"), + }.items() if v} + imds_payload = json.loads(resp.text) + BEARER = "Bearer" + oauth2_response = { + "access_token": imds_payload["access_token"], + "expires_in": int(imds_payload["expires_in"]), + "token_type": imds_payload.get("token_type", BEARER), + } + expected_token_type = (data or {}).get("token_type", BEARER) + if oauth2_response["token_type"] != expected_token_type: + return { # Generate a normal error (rather than an intrusive exception) + "error": "broker_error", + "error_description": "token_type {} is not supported by this version of Azure Portal".format( + expected_token_type), + } + parts = imds_payload["access_token"].split(".") + + # The following default values are useful in SSH Cert scenario + client_info = { # Default value, in case the real value will be unavailable + "uid": "user", + "utid": "cloudshell", + } + now = time.time() + preferred_username = "currentuser@cloudshell" + oauth2_response["id_token_claims"] = { # First 5 claims are required per OIDC + "iss": "cloudshell", + "sub": "user", + "aud": client_id, + "exp": now + 3600, + "iat": now, + "preferred_username": preferred_username, # Useful as MSAL account's username + } + + if len(parts) == 3: # Probably a JWT. Use it to derive client_info and id token. + try: + # Data defined in https://docs.microsoft.com/en-us/azure/active-directory/develop/access-tokens#payload-claims + jwt_payload = json.loads(decode_part(parts[1])) + client_info = { + # Mimic a real home_account_id, + # so that this pseudo account and a real account would interop. + "uid": jwt_payload.get("oid", "user"), + "utid": jwt_payload.get("tid", "cloudshell"), + } + oauth2_response["id_token_claims"] = { + "iss": jwt_payload["iss"], + "sub": jwt_payload["sub"], # Could use oid instead + "aud": client_id, + "exp": jwt_payload["exp"], + "iat": jwt_payload["iat"], + "preferred_username": jwt_payload.get("preferred_username") # V2 + or jwt_payload.get("unique_name") # V1 + or preferred_username, + } + except ValueError: + logger.debug("Unable to decode jwt payload: %s", parts[1]) + oauth2_response["client_info"] = base64.b64encode( + # Mimic a client_info, so that MSAL would create an account + json.dumps(client_info).encode("utf-8")).decode("utf-8") + oauth2_response["id_token_claims"]["tid"] = client_info["utid"] # TBD + + ## Note: Decided to not surface resource back as scope, + ## because they would cause the downstream OAuth2 code path to + ## cache the token with a different scope and won't hit them later. + #if imds_payload.get("resource"): + # oauth2_response["scope"] = imds_payload["resource"] + if imds_payload.get("refresh_token"): + oauth2_response["refresh_token"] = imds_payload["refresh_token"] + return oauth2_response + diff --git a/msal/token_cache.py b/msal/token_cache.py index 2ed819d7..f7d9f955 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -113,6 +113,7 @@ def wipe(dictionary, sensitive_fields): # Masks sensitive info return self.__add(event, now=now) finally: wipe(event.get("response", {}), ( # These claims were useful during __add() + "id_token_claims", # Provided by broker "access_token", "refresh_token", "id_token", "username")) wipe(event, ["username"]) # Needed for federated ROPC logger.debug("event=%s", json.dumps( @@ -150,7 +151,8 @@ def __add(self, event, now=None): id_token = response.get("id_token") id_token_claims = ( decode_id_token(id_token, client_id=event["client_id"]) - if id_token else {}) + if id_token + else response.get("id_token_claims", {})) # Broker would provide id_token_claims client_info, home_account_id = self.__parse_account(response, id_token_claims) target = ' '.join(event.get("scope") or []) # Per schema, we don't sort it @@ -195,9 +197,10 @@ def __add(self, event, now=None): or data.get("username") # Falls back to ROPC username or event.get("username") # Falls back to Federated ROPC username or "", # The schema does not like null - "authority_type": + "authority_type": event.get( + "authority_type", # Honor caller's choice of authority_type self.AuthorityType.ADFS if realm == "adfs" - else self.AuthorityType.MSSTS, + else self.AuthorityType.MSSTS), # "client_info": response.get("client_info"), # Optional } self.modify(self.CredentialType.ACCOUNT, account, account) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index f74c0767..9a971f46 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -185,12 +185,14 @@ def _test_acquire_token_interactive( self, client_id=None, authority=None, scope=None, port=None, username_uri="", # But you would want to provide one data=None, # Needed by ssh-cert feature + prompt=None, **ignored): assert client_id and authority and scope self.app = msal.PublicClientApplication( client_id, authority=authority, http_client=MinimalHttpClient()) result = self.app.acquire_token_interactive( scope, + prompt=prompt, timeout=120, port=port, welcome_template= # This is an undocumented feature for testing @@ -237,6 +239,7 @@ def test_ssh_cert_for_user(self): scope=self.SCOPE, data=self.DATA1, username_uri="https://msidlab.com/api/user?usertype=cloud", + prompt="none" if msal.application._is_running_in_cloud_shell() else None, ) # It already tests reading AT from cache, and using RT to refresh # acquire_token_silent() would work because we pass in the same key self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( @@ -254,6 +257,20 @@ def test_ssh_cert_for_user(self): self.assertNotEqual(result["access_token"], refreshed_ssh_cert['access_token']) +@unittest.skipUnless( + msal.application._is_running_in_cloud_shell(), + "Manually run this test case from inside Cloud Shell") +class CloudShellTestCase(E2eTestCase): + app = msal.PublicClientApplication("client_id") + scope_that_requires_no_managed_device = "https://management.core.windows.net/" # Scopes came from https://msazure.visualstudio.com/One/_git/compute-CloudShell?path=/src/images/agent/env/envconfig.PROD.json&version=GBmaster&_a=contents + def test_access_token_should_be_obtained_for_a_supported_scope(self): + result = self.app.acquire_token_interactive( + [self.scope_that_requires_no_managed_device], prompt="none") + self.assertEqual( + "Bearer", result.get("token_type"), "Unexpected result: %s" % result) + self.assertIsNotNone(result.get("access_token")) + + THIS_FOLDER = os.path.dirname(__file__) CONFIG = os.path.join(THIS_FOLDER, "config.json") @unittest.skipUnless(os.path.exists(CONFIG), "Optional %s not found" % CONFIG) From ea188290cfa68d26195d6b7d5079602511f240df Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 18 May 2022 22:57:28 -0700 Subject: [PATCH 002/262] MSAL Python 1.18.0b1 Bump cryptography --- msal/application.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index 812abbfb..1f6d50f2 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.17.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.18.0b1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" diff --git a/setup.py b/setup.py index 8523c2e3..f8bdd7d7 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ 'requests>=2.0.0,<3', 'PyJWT[crypto]>=1.0.0,<3', - 'cryptography>=0.6,<39', + 'cryptography>=0.6,<40', # load_pem_private_key() is available since 0.6 # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 # From 7964e1cc09695b1c0bde3e85218ff06fb8e8076e Mon Sep 17 00:00:00 2001 From: Alexander Overvoorde <60606100+OvervCW@users.noreply.github.com> Date: Mon, 30 May 2022 11:17:59 +0200 Subject: [PATCH 003/262] Fix typo in code I stumbled upon this typo while investigating a different issue in this file. --- msal/oauth2cli/assertion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/oauth2cli/assertion.py b/msal/oauth2cli/assertion.py index 855bd16b..419bb14e 100644 --- a/msal/oauth2cli/assertion.py +++ b/msal/oauth2cli/assertion.py @@ -115,7 +115,7 @@ def create_normal_assertion( payload, self.key, algorithm=self.algorithm, headers=self.headers) return _str2bytes(str_or_bytes) # We normalize them into bytes except: - if self.algorithm.startswith("RS") or self.algorithm.starswith("ES"): + if self.algorithm.startswith("RS") or self.algorithm.startswith("ES"): logger.exception( 'Some algorithms requires "pip install cryptography". ' 'See https://pyjwt.readthedocs.io/en/latest/installation.html#cryptographic-dependencies-optional') From 5d1d6c6df675a96289e14c88cad422e1d5a8763d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 31 May 2022 12:51:18 -0700 Subject: [PATCH 004/262] MSAL Python 1.18.0 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 1f6d50f2..9ac8a3bd 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.18.0b1" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.18.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 349bd530b79959ad2e98426f67486f733edb1edb Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 31 May 2022 14:26:16 -0700 Subject: [PATCH 005/262] Document our findings on addressing CVE-2022-29217 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8bdd7d7..814627f9 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ # See https://stackoverflow.com/a/14211600/728675 for more detail install_requires=[ 'requests>=2.0.0,<3', - 'PyJWT[crypto]>=1.0.0,<3', + 'PyJWT[crypto]>=1.0.0,<3', # MSAL does not use jwt.decode(), therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ 'cryptography>=0.6,<40', # load_pem_private_key() is available since 0.6 From 4af02b0080aa9aeaa898242727611677a6b1f46c Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 31 May 2022 14:56:21 -0700 Subject: [PATCH 006/262] Disable test for China cloud --- tests/test_authority.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_authority.py b/tests/test_authority.py index 9fdc83c5..ee81c15e 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -37,8 +37,9 @@ def test_wellknown_host_and_tenant(self): def test_wellknown_host_and_tenant_using_new_authority_builder(self): self._test_authority_builder(AZURE_PUBLIC, "consumers") - self._test_authority_builder(AZURE_CHINA, "organizations") self._test_authority_builder(AZURE_US_GOVERNMENT, "common") + ## AZURE_CHINA is prone to some ConnectionError. We skip it to speed up our tests. + # self._test_authority_builder(AZURE_CHINA, "organizations") @unittest.skip("As of Jan 2017, the server no longer returns V1 endpoint") def test_lessknown_host_will_return_a_set_of_v1_endpoints(self): From 5be7faf4b3f9ddfb2ad3fffac44e4ed5e9c65fca Mon Sep 17 00:00:00 2001 From: Alexander Overvoorde <60606100+OvervCW@users.noreply.github.com> Date: Mon, 30 May 2022 11:17:59 +0200 Subject: [PATCH 007/262] Fix typo in code I stumbled upon this typo while investigating a different issue in this file. --- oauth2cli/assertion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oauth2cli/assertion.py b/oauth2cli/assertion.py index 0cf58799..f01bb2d0 100644 --- a/oauth2cli/assertion.py +++ b/oauth2cli/assertion.py @@ -116,7 +116,7 @@ def create_normal_assertion( payload, self.key, algorithm=self.algorithm, headers=self.headers) return _str2bytes(str_or_bytes) # We normalize them into bytes except: - if self.algorithm.startswith("RS") or self.algorithm.starswith("ES"): + if self.algorithm.startswith("RS") or self.algorithm.startswith("ES"): logger.exception( 'Some algorithms requires "pip install cryptography". ' 'See https://pyjwt.readthedocs.io/en/latest/installation.html#cryptographic-dependencies-optional') From 1ebc175a4d895b38bda1dd7ad3f8fd25513789d7 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 2 Jun 2022 19:02:41 -0700 Subject: [PATCH 008/262] Tolerate absence of client_id to allow edge cases --- oauth2cli/oauth2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oauth2cli/oauth2.py b/oauth2cli/oauth2.py index e092b3dd..6cb31bbb 100644 --- a/oauth2cli/oauth2.py +++ b/oauth2cli/oauth2.py @@ -141,8 +141,7 @@ def __init__( """ if not server_configuration: raise ValueError("Missing input parameter server_configuration") - if not client_id: - raise ValueError("Missing input parameter client_id") + # Generally we should have client_id, but we tolerate its absence self.configuration = server_configuration self.client_id = client_id self.client_secret = client_secret From 053af22a7204b036c4ca3d9d0add7e3b2204e50b Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 8 Jun 2022 11:35:22 -0700 Subject: [PATCH 009/262] Tolerate home_account_id to be None --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 9ac8a3bd..829c35b0 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1340,7 +1340,7 @@ def _acquire_token_silent_by_finding_specific_refresh_token( reverse=True): logger.debug("Cache attempts an RT") headers = telemetry_context.generate_headers() - if "home_account_id" in query: # Then use it as CCS Routing info + if query.get("home_account_id"): # Then use it as CCS Routing info headers["X-AnchorMailbox"] = "Oid:{}".format( # case-insensitive value query["home_account_id"].replace(".", "@")) response = client.obtain_token_by_refresh_token( From 5935f0b6d34270f0dc001bb925371eeababc1ad0 Mon Sep 17 00:00:00 2001 From: sarathys <2991011+sarathys@users.noreply.github.com> Date: Tue, 28 Jun 2022 18:01:07 -0700 Subject: [PATCH 010/262] Use provided authority port when building the tenant discovery endpoint (#484) * Use provided authority port when building the tenant discovery endpoint * address PR comment * Polish the implementation Co-authored-by: Ray Luo --- msal/authority.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msal/authority.py b/msal/authority.py index 4fb6e829..81788200 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -91,8 +91,9 @@ def __init__(self, authority_url, http_client, validate_authority=True): tenant_discovery_endpoint = payload['tenant_discovery_endpoint'] else: tenant_discovery_endpoint = ( - 'https://{}{}{}/.well-known/openid-configuration'.format( + 'https://{}:{}{}{}/.well-known/openid-configuration'.format( self.instance, + 443 if authority.port is None else authority.port, authority.path, # In B2C scenario, it is "/tenant/policy" "" if tenant == "adfs" else "/v2.0" # the AAD v2 endpoint )) From d2f9b9c2fa75c2aa6d759a49af90ce22ea3d538d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 17 Jun 2022 12:37:22 -0700 Subject: [PATCH 011/262] Right regional endpoint for login.microsoft.com We got it right in PR 358 based on the specs at that time, but we were using a fragile approach, which caused the "login.microsoft.com" to be left out in subsequent PR 394. Lesson learned. Explicit is better than implicit. https://peps.python.org/pep-0020/ --- msal/application.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 829c35b0..838a28d8 100644 --- a/msal/application.py +++ b/msal/application.py @@ -526,8 +526,10 @@ def _get_regional_authority(self, central_authority): if region_to_use: regional_host = ("{}.r.login.microsoftonline.com".format(region_to_use) if central_authority.instance in ( - # The list came from https://github.com/AzureAD/microsoft-authentication-library-for-python/pull/358/files#r629400328 + # The list came from point 3 of the algorithm section in this internal doc + # https://identitydivision.visualstudio.com/DevEx/_git/AuthLibrariesApiReview?path=/PinAuthToRegion/AAD%20SDK%20Proposal%20to%20Pin%20Auth%20to%20region.md&anchor=algorithm&_a=preview "login.microsoftonline.com", + "login.microsoft.com", "login.windows.net", "sts.windows.net", ) From d43ebe9df77c15a88d99657737b05bf3f5fbad7d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 29 Jun 2022 12:46:03 -0700 Subject: [PATCH 012/262] Disable more tests for China cloud --- tests/test_authority.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_authority.py b/tests/test_authority.py index ee81c15e..fc6e12fc 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -33,7 +33,8 @@ def _test_authority_builder(self, host, tenant): def test_wellknown_host_and_tenant(self): # Assert all well known authority hosts are using their own "common" tenant for host in WELL_KNOWN_AUTHORITY_HOSTS: - self._test_given_host_and_tenant(host, "common") + if host != AZURE_CHINA: # It is prone to ConnectionError + self._test_given_host_and_tenant(host, "common") def test_wellknown_host_and_tenant_using_new_authority_builder(self): self._test_authority_builder(AZURE_PUBLIC, "consumers") From 20fff27684305af129f3477a431a45181693482d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 25 Jul 2022 12:14:57 +1000 Subject: [PATCH 013/262] Test latest 3.11 beta --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 10afc207..8950c15a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11.0-alpha.5"] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev"] steps: - uses: actions/checkout@v2 From 4736584738573b6a3f7fc37505e8c016053744ce Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 25 Aug 2022 16:50:14 -0700 Subject: [PATCH 014/262] Enable sphinx-paramlinks Troubleshooting --- docs/conf.py | 5 +++-- docs/requirements.txt | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 810dfc02..024451d5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,8 +40,9 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', + 'sphinx.ext.autodoc', # This seems need to be the first extension to load 'sphinx.ext.githubpages', + 'sphinx_paramlinks', ] # Add any paths that contain templates here, relative to this directory. @@ -182,4 +183,4 @@ epub_exclude_files = ['search.html'] -# -- Extension configuration ------------------------------------------------- \ No newline at end of file +# -- Extension configuration ------------------------------------------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index d5de57fe..0fd0c33a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ furo --r ../requirements.txt \ No newline at end of file +sphinx-paramlinks +-r ../requirements.txt From 76785c49f93ea823b41ccfaa7eacdfef99a4ca34 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 25 Aug 2022 18:16:10 -0700 Subject: [PATCH 015/262] Fix doc typo --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 838a28d8..62fc6cb8 100644 --- a/msal/application.py +++ b/msal/application.py @@ -300,7 +300,7 @@ def __init__( Client capability is implemented using "claims" parameter on the wire, for now. MSAL will combine them into - `claims parameter `_ which you will later provide via one of the acquire-token request. :param str azure_region: From e6114791b42c3ffb149a255c41d435fbba4fb88b Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 28 Aug 2022 23:56:51 -0700 Subject: [PATCH 016/262] Broker (WAM) integration Disabled SSH Cert when using broker --- msal/application.py | 283 +++++++++++++++++++++++++++++++++-- msal/authority.py | 5 +- msal/broker.py | 239 +++++++++++++++++++++++++++++ msal/token_cache.py | 20 ++- sample/interactive_sample.py | 6 + setup.py | 10 +- tests/msaltest.py | 15 +- tests/test_broker.py | 63 ++++++++ tests/test_e2e.py | 36 ++++- 9 files changed, 642 insertions(+), 35 deletions(-) create mode 100644 msal/broker.py create mode 100644 tests/test_broker.py diff --git a/msal/application.py b/msal/application.py index 62fc6cb8..a16e1844 100644 --- a/msal/application.py +++ b/msal/application.py @@ -17,7 +17,7 @@ from .mex import send_request as mex_send_request from .wstrust_request import send_request as wst_send_request from .wstrust_response import * -from .token_cache import TokenCache +from .token_cache import TokenCache, _get_username import msal.telemetry from .region import _detect_region from .throttled_http_client import ThrottledHttpClient @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.18.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.20.0b1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" @@ -67,8 +67,12 @@ def _str2bytes(raw): def _clean_up(result): if isinstance(result, dict): - result.pop("refresh_in", None) # MSAL handled refresh_in, customers need not - return result + return { + k: result[k] for k in result + if k != "refresh_in" # MSAL handled refresh_in, customers need not + and not k.startswith('_') # Skim internal properties + } + return result # It could be None def _preferred_browser(): @@ -174,6 +178,7 @@ def __init__( # when we would eventually want to add this feature to PCA in future. exclude_scopes=None, http_cache=None, + allow_broker=None, ): """Create an instance of application. @@ -409,6 +414,34 @@ def __init__( Personally Identifiable Information (PII). Encryption is unnecessary. New in version 1.16.0. + + :param boolean allow_broker: + Brokers provide Single-Sign-On, device identification, + and application identification verification. + If this parameter is set to True, + MSAL will use the broker and return either a token or an error, + when your scenario is supported by a broker, + otherwise it will automatically fall back to non-broker behavior. + This also means you could set this flag as True universally, + as long as your app meets the following prerequisite: + + * Installed optional dependency, e.g. ``pip install msal[broker]>=1.20,<2``. + (Note that broker is currently only available on Windows 10+) + + * Register a new redirect_uri for your desktop app as: + ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` + + * Tested your app in following scenarios: + + * Windows 10+ + + * PublicClientApplication's following methods:: + acquire_token_interactive(), acquire_token_by_username_password(), + acquire_token_silent() (or acquire_token_silent_with_error()). + + * AAD and MSA accounts (i.e. Non-ADFS, non-B2C) + + New in version 1.20.0. """ self.client_id = client_id self.client_credential = client_credential @@ -467,6 +500,15 @@ def __init__( self.http_client, validate_authority=False) else: raise + is_confidential_app = bool( + isinstance(self, ConfidentialClientApplication) or self.client_credential) + if is_confidential_app and allow_broker: + raise ValueError("allow_broker=True is only supported in PublicClientApplication") + self._enable_broker = bool( + allow_broker and not is_confidential_app + and sys.platform == "win32" + and not self.authority.is_adfs and not self.authority._is_b2c) + logger.debug("Broker enabled? %s", self._enable_broker) self.token_cache = token_cache or TokenCache() self._region_configured = azure_region @@ -1028,6 +1070,15 @@ def _get_authority_aliases(self, instance): def remove_account(self, account): """Sign me out and forget me from token cache""" self._forget_me(account) + if self._enable_broker: + try: + from .broker import _signout_silently + except RuntimeError: # TODO: TBD + logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") + else: + error = _signout_silently(self.client_id, account["local_account_id"]) + if error: + logger.debug("_signout_silently() returns error: %s", error) def _sign_out(self, home_account): # Remove all relevant RTs and ATs from token cache @@ -1255,9 +1306,28 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( refresh_reason = msal.telemetry.FORCE_REFRESH # TODO: It could also mean claims_challenge assert refresh_reason, "It should have been established at this point" try: + data = kwargs.get("data", {}) if account and account.get("authority_type") == _AUTHORITY_TYPE_CLOUDSHELL: - return self._acquire_token_by_cloud_shell( - scopes, data=kwargs.get("data")) + return self._acquire_token_by_cloud_shell(scopes, data=data) + + if self._enable_broker and account is not None and data.get("token_type") != "ssh-cert": + try: + from .broker import _acquire_token_silently + except RuntimeError: # TODO: TBD + logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") + else: + response = _acquire_token_silently( + "https://{}/{}".format(self.authority.instance, self.authority.tenant), + self.client_id, + account["local_account_id"], + scopes, + claims=_merge_claims_challenge_and_capabilities( + self._client_capabilities, claims_challenge), + correlation_id=correlation_id, + **data) + if response: # The broker provided a decisive outcome, so we use it + return self._process_broker_response(response, scopes, data) + result = _clean_up(self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( authority, self._decorate_scope(scopes), account, refresh_reason=refresh_reason, claims_challenge=claims_challenge, @@ -1271,6 +1341,18 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( raise # We choose to bubble up the exception return access_token_from_cache + def _process_broker_response(self, response, scopes, data): + if "error" not in response: + self.token_cache.add(dict( + client_id=self.client_id, + scope=response["scope"].split() if "scope" in response else scopes, + token_endpoint=self.authority.token_endpoint, + response=response.copy(), + data=data, + _account_id=response["_account_id"], + )) + return _clean_up(response) + def _acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( self, authority, scopes, account, **kwargs): query = { @@ -1447,14 +1529,33 @@ def acquire_token_by_username_password( - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". """ + claims = _merge_claims_challenge_and_capabilities( + self._client_capabilities, claims_challenge) + if self._enable_broker: + try: + from .broker import _signin_silently + except RuntimeError: # TODO: TBD + logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") + else: + response = _signin_silently( + "https://{}/{}".format(self.authority.instance, self.authority.tenant), + self.client_id, + scopes, # Decorated scopes won't work due to offline_access + MSALRuntime_Username=username, + MSALRuntime_Password=password, + validateAuthority="no" + if self.authority._validate_authority is False + or self.authority.is_adfs or self.authority._is_b2c + else None, + claims=claims, + ) + return self._process_broker_response(response, scopes, kwargs.get("data", {})) + scopes = self._decorate_scope(scopes) telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_BY_USERNAME_PASSWORD_ID) headers = telemetry_context.generate_headers() - data = dict( - kwargs.pop("data", {}), - claims=_merge_claims_challenge_and_capabilities( - self._client_capabilities, claims_challenge)) + data = dict(kwargs.pop("data", {}), claims=claims) if not self.authority.is_adfs: user_realm_result = self.authority.user_realm_discovery( username, correlation_id=headers[msal.telemetry.CLIENT_REQUEST_ID]) @@ -1520,6 +1621,7 @@ def _acquire_token_by_username_password_federated( class PublicClientApplication(ClientApplication): # browser app or mobile app DEVICE_FLOW_CORRELATION_ID = "_correlation_id" + CONSOLE_WINDOW_HANDLE = object() def __init__(self, client_id, client_credential=None, **kwargs): if client_credential is not None: @@ -1538,11 +1640,16 @@ def acquire_token_interactive( port=None, extra_scopes_to_consent=None, max_age=None, + parent_window_handle=None, + on_before_launching_ui=None, **kwargs): """Acquire token interactively i.e. via a local browser. Prerequisite: In Azure Portal, configure the Redirect URI of your "Mobile and Desktop application" as ``http://localhost``. + If you opts in to use broker during ``PublicClientApplication`` creation, + your app also need this Redirect URI: + ``ms-appx-web://Microsoft.AAD.BrokerPlugin/YOUR_CLIENT_ID`` :param list scopes: It is a list of case-sensitive strings. @@ -1594,17 +1701,79 @@ def acquire_token_interactive( New in version 1.15. + :param int parent_window_handle: + OPTIONAL. If your app is a GUI app running on modern Windows system, + and your app opts in to use broker, + you are recommended to also provide its window handle, + so that the sign in UI window will properly pop up on top of your window. + + New in version 1.20.0. + + :param function on_before_launching_ui: + A callback with the form of + ``lambda ui="xyz", **kwargs: print("A {} will be launched".format(ui))``, + where ``ui`` will be either "browser" or "broker". + You can use it to inform your end user to expect a pop-up window. + + New in version 1.20.0. + :return: - A dict containing no "error" key, and typically contains an "access_token" key. - A dict containing an "error" key, when token refresh failed. """ - self._validate_ssh_cert_input_data(kwargs.get("data", {})) + data = kwargs.pop("data", {}) + self._validate_ssh_cert_input_data(data) + if not on_before_launching_ui: + on_before_launching_ui = lambda **kwargs: None if _is_running_in_cloud_shell() and prompt == "none": - return self._acquire_token_by_cloud_shell( - scopes, data=kwargs.pop("data", {})) + # Note: _acquire_token_by_cloud_shell() is always silent, + # so we would not fire on_before_launching_ui() + return self._acquire_token_by_cloud_shell(scopes, data=data) claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) + if self._enable_broker and data.get("token_type") != "ssh-cert": + if parent_window_handle is None: + raise ValueError( + "parent_window_handle is required when you opted into using broker. " + "You need to provide the window handle of your GUI application, " + "or use msal.PublicClientApplication.CONSOLE_WINDOW_HANDLE " + "when and only when your application is a console app.") + if extra_scopes_to_consent: + logger.warning( + "Ignoring parameter extra_scopes_to_consent, " + "which is not supported by broker") + enable_msa_passthrough = kwargs.pop( + "enable_msa_passthrough", # Keep it as a hidden param, for now. + # OPTIONAL. MSA-Passthrough is a legacy configuration, + # needed by a small amount of Microsoft first-party apps, + # which would login MSA accounts via ".../organizations" authority. + # If you app belongs to this category, AND you are enabling broker, + # you would want to enable this flag. Default value is equivalent to False. + self.client_id in [ + # Experimental: Automatically enable MSA-PT mode for known MSA-PT apps + # More background of MSA-PT is available from this internal docs: + # https://microsoft.sharepoint.com/:w:/t/Identity-DevEx/EatIUauX3c9Ctw1l7AQ6iM8B5CeBZxc58eoQCE0IuZ0VFw?e=tgc3jP&CID=39c853be-76ea-79d7-ee73-f1b2706ede05 + "04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI + "04f0c124-f2bc-4f59-8241-bf6df9866bbd", # Visual Studio + ] and data.get("token_type") != "ssh-cert" # Work around a known issue as of PyMsalRuntime 0.8 + ) + try: + return self._acquire_token_interactive_via_broker( + scopes, + parent_window_handle, + enable_msa_passthrough, + claims, + data, + on_before_launching_ui, + prompt=prompt, + login_hint=login_hint, + max_age=max_age, + ) + except RuntimeError: # TODO: TBD + logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") + + on_before_launching_ui(ui="browser") telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_INTERACTIVE) response = _clean_up(self.client.obtain_token_by_browser( @@ -1621,13 +1790,99 @@ def acquire_token_interactive( "claims": claims, "domain_hint": domain_hint, }, - data=dict(kwargs.pop("data", {}), claims=claims), + data=dict(data, claims=claims), headers=telemetry_context.generate_headers(), browser_name=_preferred_browser(), **kwargs)) telemetry_context.update_telemetry(response) return response + def _acquire_token_interactive_via_broker( + self, + scopes, # type: list[str] + parent_window_handle, # type: int + enable_msa_passthrough, # type: boolean + claims, # type: str + data, # type: dict + on_before_launching_ui, # type: callable + prompt=None, + login_hint=None, # type: Optional[str] + max_age=None, + **kwargs): + from .broker import _signin_interactively, _signin_silently, _acquire_token_silently + if "welcome_template" in kwargs: + logger.debug(kwargs["welcome_template"]) # Experimental + authority = "https://{}/{}".format( + self.authority.instance, self.authority.tenant) + validate_authority = ( + "no" if self.authority._validate_authority is False + or self.authority.is_adfs or self.authority._is_b2c + else None) + # Calls different broker methods to mimic the OIDC behaviors + if login_hint and prompt != "select_account": # OIDC prompts when the user did not sign in + accounts = self.get_accounts(username=login_hint) + if len(accounts) == 1: # Unambiguously proceed with this account + response = _acquire_token_silently( # When it works, it bypasses prompt + authority, + self.client_id, + accounts[0]["local_account_id"], + scopes, + claims=claims, + **data) + if response and "error" not in response: + return self._process_broker_response(response, scopes, data) + # login_hint undecisive or not exists + if prompt == "none" or not prompt: # Must/Can attempt _signin_silently() + response = _signin_silently( # Unlike OIDC, it doesn't honor login_hint + authority, self.client_id, scopes, + validateAuthority=validate_authority, + claims=claims, + max_age=max_age, + enable_msa_pt=enable_msa_passthrough, + **data) + is_wrong_account = bool( + # _signin_silently() only gets tokens for default account, + # but this seems to have been fixed in PyMsalRuntime 0.11.2 + "access_token" in response and login_hint + and response.get("id_token_claims", {}) != login_hint) + wrong_account_error_message = ( + 'prompt="none" will not work for login_hint="non-default-user"') + if is_wrong_account: + logger.debug(wrong_account_error_message) + if prompt == "none": + return self._process_broker_response( # It is either token or error + response, scopes, data + ) if not is_wrong_account else { + "error": "broker_error", + "error_description": wrong_account_error_message, + } + else: + assert bool(prompt) is False + from pymsalruntime import Response_Status + recoverable_errors = frozenset([ + Response_Status.Status_AccountUnusable, + Response_Status.Status_InteractionRequired, + ]) + if is_wrong_account or "error" in response and response.get( + "_broker_status") in recoverable_errors: + pass # It will fall back to the _signin_interactively() + else: + return self._process_broker_response(response, scopes, data) + # Falls back to _signin_interactively() + on_before_launching_ui(ui="broker") + response = _signin_interactively( + authority, self.client_id, scopes, + None if parent_window_handle is self.CONSOLE_WINDOW_HANDLE + else parent_window_handle, + validateAuthority=validate_authority, + login_hint=login_hint, + prompt=prompt, + claims=claims, + max_age=max_age, + enable_msa_pt=enable_msa_passthrough, + **data) + return self._process_broker_response(response, scopes, data) + def initiate_device_flow(self, scopes=None, **kwargs): """Initiate a Device Flow instance, which will be used in :func:`~acquire_token_by_device_flow`. diff --git a/msal/authority.py b/msal/authority.py index 81788200..93aafeea 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -72,9 +72,10 @@ def __init__(self, authority_url, http_client, validate_authority=True): authority_url = str(authority_url) authority, self.instance, tenant = canonicalize(authority_url) parts = authority.path.split('/') - is_b2c = any(self.instance.endswith("." + d) for d in WELL_KNOWN_B2C_HOSTS) or ( + self._is_b2c = any(self.instance.endswith("." + d) for d in WELL_KNOWN_B2C_HOSTS) or ( len(parts) == 3 and parts[2].lower().startswith("b2c_")) - if (tenant != "adfs" and (not is_b2c) and validate_authority + self._validate_authority = True if validate_authority is None else bool(validate_authority) + if (tenant != "adfs" and (not self._is_b2c) and self._validate_authority and self.instance not in WELL_KNOWN_AUTHORITY_HOSTS): payload = instance_discovery( "https://{}{}/oauth2/v2.0/authorize".format( diff --git a/msal/broker.py b/msal/broker.py new file mode 100644 index 00000000..e9bbe28c --- /dev/null +++ b/msal/broker.py @@ -0,0 +1,239 @@ +"""This module is an adaptor to the underlying broker. +It relies on PyMsalRuntime which is the package providing broker's functionality. +""" +from threading import Event +import json +import logging +import time +import uuid + + +logger = logging.getLogger(__name__) +try: + import pymsalruntime # ImportError would be raised on unsupported platforms such as Windows 8 + # Its API description is available in site-packages/pymsalruntime/PyMsalRuntime.pyi + pymsalruntime.register_logging_callback(lambda message, level: { # New in pymsalruntime 0.7 + pymsalruntime.LogLevel.TRACE: logger.debug, # Python has no TRACE level + pymsalruntime.LogLevel.DEBUG: logger.debug, + # Let broker's excess info, warning and error logs map into default DEBUG, for now + #pymsalruntime.LogLevel.INFO: logger.info, + #pymsalruntime.LogLevel.WARNING: logger.warning, + #pymsalruntime.LogLevel.ERROR: logger.error, + pymsalruntime.LogLevel.FATAL: logger.critical, + }.get(level, logger.debug)(message)) +except (ImportError, AttributeError): # AttributeError happens when a prior pymsalruntime uninstallation somehow leaved an empty folder behind + # PyMsalRuntime currently supports these Windows versions, listed in this MSFT internal link + # https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/2406/files + raise ImportError( # TODO: Remove or adjust this line right before merging this PR + 'You need to install dependency by: pip install "msal[broker]>=1.20.0b1,<2"') +# Other exceptions (possibly RuntimeError) would be raised if its initialization fails + + +class RedirectUriError(ValueError): + pass + + +class TokenTypeError(ValueError): + pass + + +class _CallbackData: + def __init__(self): + self.signal = Event() + self.result = None + + def complete(self, result): + self.signal.set() + self.result = result + + +def _convert_error(error, client_id): + context = error.get_context() # Available since pymsalruntime 0.0.4 + if ( + "AADSTS50011" in context # In WAM, this could happen on both interactive and silent flows + or "AADSTS7000218" in context # This "request body must contain ... client_secret" is just a symptom of current app has no WAM redirect_uri + ): + raise RedirectUriError( # This would be seen by either the app developer or end user + "MsalRuntime won't work unless this one more redirect_uri is registered to current app: " + "ms-appx-web://Microsoft.AAD.BrokerPlugin/{}".format(client_id)) + # OTOH, AAD would emit other errors when other error handling branch was hit first, + # so, the AADSTS50011/RedirectUriError is not guaranteed to happen. + return { + "error": "broker_error", # Note: Broker implies your device needs to be compliant. + # You may use "dsregcmd /status" to check your device state + # https://docs.microsoft.com/en-us/azure/active-directory/devices/troubleshoot-device-dsregcmd + "error_description": "{}. Status: {}, Error code: {}, Tag: {}".format( + context, + error.get_status(), error.get_error_code(), error.get_tag()), + "_broker_status": error.get_status(), + "_broker_error_code": error.get_error_code(), + "_broker_tag": error.get_tag(), + } + + +def _read_account_by_id(account_id, correlation_id): + """Return an instance of MSALRuntimeError or MSALRuntimeAccount, or None""" + callback_data = _CallbackData() + pymsalruntime.read_account_by_id( + account_id, + correlation_id, + lambda result, callback_data=callback_data: callback_data.complete(result) + ) + callback_data.signal.wait() + return (callback_data.result.get_error() or callback_data.result.get_account() + or None) # None happens when the account was not created by broker + + +def _convert_result(result, client_id, expected_token_type=None): # Mimic an on-the-wire response from AAD + error = result.get_error() + if error: + return _convert_error(error, client_id) + id_token_claims = json.loads(result.get_id_token()) if result.get_id_token() else {} + account = result.get_account() + assert account, "Account is expected to be always available" + ## Note: As of pymsalruntime 0.1.0, only wam_account_ids property is available + #account.get_account_property("wam_account_ids") + return_value = {k: v for k, v in { + "access_token": result.get_access_token(), + "expires_in": result.get_access_token_expiry_time() - int(time.time()), # Convert epoch to count-down + "id_token": result.get_raw_id_token(), # New in pymsalruntime 0.8.1 + "id_token_claims": id_token_claims, + "client_info": account.get_client_info(), + "_account_id": account.get_account_id(), + "token_type": expected_token_type or "Bearer", # Workaround its absence from broker + }.items() if v} + likely_a_cert = return_value["access_token"].startswith("AAAA") # Empirical observation + if return_value["token_type"].lower() == "ssh-cert" and not likely_a_cert: + raise TokenTypeError("Broker could not get an SSH Cert: {}...".format( + return_value["access_token"][:8])) + granted_scopes = result.get_granted_scopes() # New in pymsalruntime 0.3.x + if granted_scopes: + return_value["scope"] = " ".join(granted_scopes) # Mimic the on-the-wire data format + return return_value + + +def _get_new_correlation_id(): + return str(uuid.uuid4()) + + +def _enable_msa_pt(params): + params.set_additional_parameter("msal_request_type", "consumer_passthrough") # PyMsalRuntime 0.8+ + + +def _signin_silently( + authority, client_id, scopes, correlation_id=None, claims=None, + enable_msa_pt=False, + **kwargs): + params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params.set_requested_scopes(scopes) + if claims: + params.set_decoded_claims(claims) + callback_data = _CallbackData() + for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. + if v is not None: + params.set_additional_parameter(k, str(v)) + if enable_msa_pt: + _enable_msa_pt(params) + pymsalruntime.signin_silently( + params, + correlation_id or _get_new_correlation_id(), + lambda result, callback_data=callback_data: callback_data.complete(result)) + callback_data.signal.wait() + return _convert_result( + callback_data.result, client_id, expected_token_type=kwargs.get("token_type")) + + +def _signin_interactively( + authority, client_id, scopes, + parent_window_handle, # None means auto-detect for console apps + prompt=None, + login_hint=None, + claims=None, + correlation_id=None, + enable_msa_pt=False, + **kwargs): + params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params.set_requested_scopes(scopes) + params.set_redirect_uri("placeholder") # pymsalruntime 0.1 requires non-empty str, + # the actual redirect_uri will be overridden by a value hardcoded by the broker + if prompt: + if prompt == "select_account": + if login_hint: + # FWIW, AAD's browser interactive flow would honor select_account + # and ignore login_hint in such a case. + # But pymsalruntime 0.3.x would pop up a meaningless account picker + # and then force the account_hint user to re-input password. Not what we want. + # https://identitydivision.visualstudio.com/Engineering/_workitems/edit/1744492 + login_hint = None # Mimicing the AAD behavior + logger.warning("Using both select_account and login_hint is ambiguous. Ignoring login_hint.") + else: + logger.warning("prompt=%s is not supported by this module", prompt) + if parent_window_handle is None: + # This fixes account picker hanging in IDE debug mode on some machines + params.set_additional_parameter("msal_gui_thread", "true") # Since pymsalruntime 0.8.1 + if enable_msa_pt: + _enable_msa_pt(params) + for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. + if v is not None: + params.set_additional_parameter(k, str(v)) + if claims: + params.set_decoded_claims(claims) + callback_data = _CallbackData() + pymsalruntime.signin_interactively( + parent_window_handle or pymsalruntime.get_console_window() or pymsalruntime.get_desktop_window(), # Since pymsalruntime 0.2+ + params, + correlation_id or _get_new_correlation_id(), + login_hint, # None value will be accepted since pymsalruntime 0.3+ + lambda result, callback_data=callback_data: callback_data.complete(result)) + callback_data.signal.wait() + return _convert_result( + callback_data.result, client_id, expected_token_type=kwargs.get("token_type")) + + +def _acquire_token_silently( + authority, client_id, account_id, scopes, claims=None, correlation_id=None, + **kwargs): + # For MSA PT scenario where you use the /organizations, yes, + # acquireTokenSilently is expected to fail. - Sam Wilson + correlation_id = correlation_id or _get_new_correlation_id() + account = _read_account_by_id(account_id, correlation_id) + if isinstance(account, pymsalruntime.MSALRuntimeError): + return _convert_error(account, client_id) + if account is None: + return + params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params.set_requested_scopes(scopes) + if claims: + params.set_decoded_claims(claims) + for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. + if v is not None: + params.set_additional_parameter(k, str(v)) + callback_data = _CallbackData() + pymsalruntime.acquire_token_silently( + params, + correlation_id, + account, + lambda result, callback_data=callback_data: callback_data.complete(result)) + callback_data.signal.wait() + return _convert_result( + callback_data.result, client_id, expected_token_type=kwargs.get("token_type")) + + +def _signout_silently(client_id, account_id, correlation_id=None): + correlation_id = correlation_id or _get_new_correlation_id() + account = _read_account_by_id(account_id, correlation_id) + if isinstance(account, pymsalruntime.MSALRuntimeError): + return _convert_error(account, client_id) + if account is None: + return + callback_data = _CallbackData() + pymsalruntime.signout_silently( # New in PyMsalRuntime 0.7 + client_id, + correlation_id, + account, + lambda result, callback_data=callback_data: callback_data.complete(result)) + callback_data.signal.wait() + error = callback_data.result.get_error() + if error: + return _convert_error(error, client_id) + diff --git a/msal/token_cache.py b/msal/token_cache.py index f7d9f955..dc26e843 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -12,6 +12,10 @@ def is_subdict_of(small, big): return dict(big, **small) == big +def _get_username(id_token_claims): + return id_token_claims.get( + "preferred_username", # AAD + id_token_claims.get("upn")) # ADFS 2019 class TokenCache(object): """This is considered as a base class containing minimal cache behavior. @@ -149,10 +153,9 @@ def __add(self, event, now=None): access_token = response.get("access_token") refresh_token = response.get("refresh_token") id_token = response.get("id_token") - id_token_claims = ( - decode_id_token(id_token, client_id=event["client_id"]) - if id_token - else response.get("id_token_claims", {})) # Broker would provide id_token_claims + id_token_claims = response.get("id_token_claims") or ( # Prefer the claims from broker + # Only use decode_id_token() when necessary, it contains time-sensitive validation + decode_id_token(id_token, client_id=event["client_id"]) if id_token else {}) client_info, home_account_id = self.__parse_account(response, id_token_claims) target = ' '.join(event.get("scope") or []) # Per schema, we don't sort it @@ -190,10 +193,11 @@ def __add(self, event, now=None): "home_account_id": home_account_id, "environment": environment, "realm": realm, - "local_account_id": id_token_claims.get( - "oid", id_token_claims.get("sub")), - "username": id_token_claims.get("preferred_username") # AAD - or id_token_claims.get("upn") # ADFS 2019 + "local_account_id": event.get( + "_account_id", # Came from mid-tier code path. + # Emperically, it is the oid in AAD or cid in MSA. + id_token_claims.get("oid", id_token_claims.get("sub"))), + "username": _get_username(id_token_claims) or data.get("username") # Falls back to ROPC username or event.get("username") # Falls back to Federated ROPC username or "", # The schema does not like null diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index 530892e5..98acd29f 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -1,4 +1,7 @@ """ +Prerequisite is documented here: +https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_interactive + The configuration file would look like this: { @@ -30,6 +33,8 @@ # Create a preferably long-lived app instance which maintains a token cache. app = msal.PublicClientApplication( config["client_id"], authority=config["authority"], + #allow_broker=True, # If opted in, you will be guided to meet the prerequisites, when applicable + # See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition # token_cache=... # Default cache is in memory only. # You can learn how to use SerializableTokenCache from # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache @@ -55,6 +60,7 @@ print("A local browser window will be open for you to sign in. CTRL+C to cancel.") result = app.acquire_token_interactive( # Only works if your app is registered with redirect_uri as http://localhost config["scope"], + #parent_window_handle=..., # If broker is enabled, you will be guided to provide a window handle login_hint=config.get("username"), # Optional. # If you know the username ahead of time, this parameter can pre-fill # the username (or email address) field of the sign-in page for the user, diff --git a/setup.py b/setup.py index 814627f9..1384634d 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,14 @@ # https://cryptography.io/en/latest/api-stability/#deprecation "mock;python_version<'3.3'", - ] + ], + extras_require={ # It does not seem to work if being defined inside setup.cfg + "broker": [ + # The broker is defined as optional dependency, + # so that downstream apps can opt in. The opt-in is needed, partially because + # most existing MSAL Python apps do not have the redirect_uri needed by broker. + "pymsalruntime>=0.11.2,<0.12;python_version>='3.6' and platform_system=='Windows'", + ], + }, ) diff --git a/tests/msaltest.py b/tests/msaltest.py index c1ef1e7c..5d33f2c2 100644 --- a/tests/msaltest.py +++ b/tests/msaltest.py @@ -74,12 +74,14 @@ def _acquire_token_interactive(app, scopes, data=None): # login_hint is unnecessary when prompt=select_account, # but we still let tester input login_hint, just for testing purpose. [None] + [a["username"] for a in app.get_accounts()], - header="login_hint? (If you have multiple signed-in sessions in browser, and you specify a login_hint to match one of them, you will bypass the account picker.)", + header="login_hint? (If you have multiple signed-in sessions in browser/broker, and you specify a login_hint to match one of them, you will bypass the account picker.)", accept_nonempty_string=True, ) login_hint = raw_login_hint["username"] if isinstance(raw_login_hint, dict) else raw_login_hint result = app.acquire_token_interactive( - scopes, prompt=prompt, login_hint=login_hint, data=data or {}) + scopes, + parent_window_handle=app.CONSOLE_WINDOW_HANDLE, # This test app is a console app + prompt=prompt, login_hint=login_hint, data=data or {}) if login_hint and "id_token_claims" in result: signed_in_user = result.get("id_token_claims", {}).get("preferred_username") if signed_in_user != login_hint: @@ -127,9 +129,13 @@ def remove_account(app): app.remove_account(account) print('Account "{}" and/or its token(s) are signed out from MSAL Python'.format(account["username"])) -def exit(_): +def exit(app): """Exit""" - bug_link = "https://github.com/AzureAD/microsoft-authentication-library-for-python/issues/new/choose" + bug_link = ( + "https://identitydivision.visualstudio.com/Engineering/_queries/query/79b3a352-a775-406f-87cd-a487c382a8ed/" + if app._enable_broker else + "https://github.com/AzureAD/microsoft-authentication-library-for-python/issues/new/choose" + ) print("Bye. If you found a bug, please report it here: {}".format(bug_link)) sys.exit() @@ -155,6 +161,7 @@ def main(): header="Input authority (Note that MSA-PT apps would NOT use the /common authority)", accept_nonempty_string=True, ), + allow_broker=_input_boolean("Allow broker? (Azure CLI currently only supports @microsoft.com accounts when enabling broker)"), ) if _input_boolean("Enable MSAL Python's DEBUG log?"): logging.basicConfig(level=logging.DEBUG) diff --git a/tests/test_broker.py b/tests/test_broker.py new file mode 100644 index 00000000..bb7d928e --- /dev/null +++ b/tests/test_broker.py @@ -0,0 +1,63 @@ +from tests import unittest +import logging +import sys + +if not sys.platform.startswith("win"): + raise unittest.SkipTest("Currently, our broker supports Windows") +from msal.broker import ( # Import them after the platform check + _signin_silently, _signin_interactively, _acquire_token_silently, RedirectUriError, + _signout_silently, _read_account_by_id, + ) + + +logging.basicConfig(level=logging.DEBUG) + +class BrokerTestCase(unittest.TestCase): + """These are the unit tests for the thin broker.py layer. + + It currently hardcodes some test apps which might be changed/gone in the future. + The existing test_e2e.py is sophisticated to pull test configuration securely from lab. + """ + _client_id = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" # Visual Studio + _authority = "https://login.microsoftonline.com/common" + _scopes = ["https://graph.microsoft.com/.default"] + + def test_signin_interactive_then_acquire_token_silent_then_signout(self): + result = _signin_interactively(self._authority, self._client_id, self._scopes, None) + self.assertIsNotNone(result.get("access_token"), result) + + account_id = result["_account_id"] + self.assertIsNotNone(_read_account_by_id(account_id, "correlation_id")) + result = _acquire_token_silently( + self._authority, self._client_id, account_id, self._scopes) + self.assertIsNotNone(result.get("access_token"), result) + + signout_error = _signout_silently(self._client_id, account_id) + self.assertIsNone(signout_error, msg=signout_error) + account = _read_account_by_id(account_id, "correlation_id") + self.assertIsNotNone(account, msg="pymsalruntime still has this account") + result = _acquire_token_silently( + self._authority, self._client_id, account_id, self._scopes) + self.assertIn("Status_AccountUnusable", result.get("error_description", "")) + + def test_unconfigured_app_should_raise_exception(self): + app_without_needed_redirect_uri = "289a413d-284b-4303-9c79-94380abe5d22" + with self.assertRaises(RedirectUriError): + _signin_interactively( + self._authority, app_without_needed_redirect_uri, self._scopes, None) + # Note: _acquire_token_silently() would raise same exception, + # we skip its test here due to the lack of a valid account_id + + def test_signin_interactively_and_select_account(self): + print("An account picker UI will pop up. See whether the auth result matches your account") + result = _signin_interactively( + self._authority, self._client_id, self._scopes, None, prompt="select_account") + self.assertIsNotNone(result.get("access_token"), result) + if "access_token" in result: + result["access_token"] = "********" + import pprint; pprint.pprint(result) + + def test_signin_silently(self): + result = _signin_silently(self._authority, self._client_id, self._scopes) + self.assertIsNotNone(result.get("access_token"), result) + diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9a971f46..57ef8208 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -134,6 +134,31 @@ def assertCacheWorksForApp(self, result_from_wire, scope): result_from_wire['access_token'], result_from_cache['access_token'], "We should get a cached AT") + @classmethod + def _build_app(cls, + client_id, + client_credential=None, + authority="https://login.microsoftonline.com/common", + scopes=["https://graph.microsoft.com/.default"], # Microsoft Graph + http_client=None, + azure_region=None, + **kwargs): + try: + import pymsalruntime + broker_available = True + except ImportError: + broker_available = False + return (msal.ConfidentialClientApplication + if client_credential else msal.PublicClientApplication)( + client_id, + client_credential=client_credential, + authority=authority, + azure_region=azure_region, + http_client=http_client or MinimalHttpClient(), + allow_broker=broker_available # This way, we reuse same test cases, by run them with and without broker + and not client_credential, + ) + def _test_username_password(self, authority=None, client_id=None, username=None, password=None, scope=None, client_secret=None, # Since MSAL 1.11, confidential client has ROPC too @@ -141,9 +166,9 @@ def _test_username_password(self, http_client=None, **ignored): assert authority and client_id and username and password and scope - self.app = msal.ClientApplication( + self.app = self._build_app( client_id, authority=authority, - http_client=http_client or MinimalHttpClient(), + http_client=http_client, azure_region=azure_region, # Regional endpoint does not support ROPC. # Here we just use it to test a regional app won't break ROPC. client_credential=client_secret) @@ -158,8 +183,7 @@ def _test_username_password(self, def _test_device_flow( self, client_id=None, authority=None, scope=None, **ignored): assert client_id and authority and scope - self.app = msal.PublicClientApplication( - client_id, authority=authority, http_client=MinimalHttpClient()) + self.app = self._build_app(client_id, authority=authority) flow = self.app.initiate_device_flow(scopes=scope) assert "user_code" in flow, "DF does not seem to be provisioned: %s".format( json.dumps(flow, indent=4)) @@ -188,13 +212,13 @@ def _test_acquire_token_interactive( prompt=None, **ignored): assert client_id and authority and scope - self.app = msal.PublicClientApplication( - client_id, authority=authority, http_client=MinimalHttpClient()) + self.app = self._build_app(client_id, authority=authority) result = self.app.acquire_token_interactive( scope, prompt=prompt, timeout=120, port=port, + parent_window_handle=self.app.CONSOLE_WINDOW_HANDLE, # This test app is a console app welcome_template= # This is an undocumented feature for testing """

{id}

  1. Get a username from the upn shown at here
    2. From 6e9110907f0ba214b08f4c02e735168102cb4539 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 29 Aug 2022 12:31:06 -0700 Subject: [PATCH 017/262] Refine document --- msal/application.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/msal/application.py b/msal/application.py index a16e1844..ef2cfd90 100644 --- a/msal/application.py +++ b/msal/application.py @@ -418,11 +418,12 @@ def __init__( :param boolean allow_broker: Brokers provide Single-Sign-On, device identification, and application identification verification. + This flag defaults to None, which means MSAL will not utilize broker. If this parameter is set to True, - MSAL will use the broker and return either a token or an error, - when your scenario is supported by a broker, - otherwise it will automatically fall back to non-broker behavior. - This also means you could set this flag as True universally, + MSAL will use the broker whenever possible, + and automatically fall back to non-broker behavior. + That also means your app does not need to enable broker conditionally, + you can always set allow_broker to True, as long as your app meets the following prerequisite: * Installed optional dependency, e.g. ``pip install msal[broker]>=1.20,<2``. From 9c34873901f46ddefcb842045eb91bf2d84d84f4 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 12 Sep 2022 14:36:57 -0700 Subject: [PATCH 018/262] Gracefully handle RuntimeError upfront --- msal/application.py | 112 ++++++++++++++++++++------------------------ msal/broker.py | 5 +- 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/msal/application.py b/msal/application.py index ef2cfd90..66370f57 100644 --- a/msal/application.py +++ b/msal/application.py @@ -505,10 +505,17 @@ def __init__( isinstance(self, ConfidentialClientApplication) or self.client_credential) if is_confidential_app and allow_broker: raise ValueError("allow_broker=True is only supported in PublicClientApplication") - self._enable_broker = bool( - allow_broker and not is_confidential_app - and sys.platform == "win32" - and not self.authority.is_adfs and not self.authority._is_b2c) + self._enable_broker = False + if (allow_broker and not is_confidential_app + and sys.platform == "win32" + and not self.authority.is_adfs and not self.authority._is_b2c): + try: + from . import broker # Trigger Broker's initialization + self._enable_broker = True + except RuntimeError: + logger.exception( + "Broker is unavailable on this platform. " + "We will fallback to non-broker.") logger.debug("Broker enabled? %s", self._enable_broker) self.token_cache = token_cache or TokenCache() @@ -1072,14 +1079,10 @@ def remove_account(self, account): """Sign me out and forget me from token cache""" self._forget_me(account) if self._enable_broker: - try: - from .broker import _signout_silently - except RuntimeError: # TODO: TBD - logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") - else: - error = _signout_silently(self.client_id, account["local_account_id"]) - if error: - logger.debug("_signout_silently() returns error: %s", error) + from .broker import _signout_silently + error = _signout_silently(self.client_id, account["local_account_id"]) + if error: + logger.debug("_signout_silently() returns error: %s", error) def _sign_out(self, home_account): # Remove all relevant RTs and ATs from token cache @@ -1312,22 +1315,18 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( return self._acquire_token_by_cloud_shell(scopes, data=data) if self._enable_broker and account is not None and data.get("token_type") != "ssh-cert": - try: - from .broker import _acquire_token_silently - except RuntimeError: # TODO: TBD - logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") - else: - response = _acquire_token_silently( - "https://{}/{}".format(self.authority.instance, self.authority.tenant), - self.client_id, - account["local_account_id"], - scopes, - claims=_merge_claims_challenge_and_capabilities( - self._client_capabilities, claims_challenge), - correlation_id=correlation_id, - **data) - if response: # The broker provided a decisive outcome, so we use it - return self._process_broker_response(response, scopes, data) + from .broker import _acquire_token_silently + response = _acquire_token_silently( + "https://{}/{}".format(self.authority.instance, self.authority.tenant), + self.client_id, + account["local_account_id"], + scopes, + claims=_merge_claims_challenge_and_capabilities( + self._client_capabilities, claims_challenge), + correlation_id=correlation_id, + **data) + if response: # The broker provided a decisive outcome, so we use it + return self._process_broker_response(response, scopes, data) result = _clean_up(self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( authority, self._decorate_scope(scopes), account, @@ -1533,24 +1532,20 @@ def acquire_token_by_username_password( claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) if self._enable_broker: - try: - from .broker import _signin_silently - except RuntimeError: # TODO: TBD - logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") - else: - response = _signin_silently( - "https://{}/{}".format(self.authority.instance, self.authority.tenant), - self.client_id, - scopes, # Decorated scopes won't work due to offline_access - MSALRuntime_Username=username, - MSALRuntime_Password=password, - validateAuthority="no" - if self.authority._validate_authority is False - or self.authority.is_adfs or self.authority._is_b2c - else None, - claims=claims, - ) - return self._process_broker_response(response, scopes, kwargs.get("data", {})) + from .broker import _signin_silently + response = _signin_silently( + "https://{}/{}".format(self.authority.instance, self.authority.tenant), + self.client_id, + scopes, # Decorated scopes won't work due to offline_access + MSALRuntime_Username=username, + MSALRuntime_Password=password, + validateAuthority="no" + if self.authority._validate_authority is False + or self.authority.is_adfs or self.authority._is_b2c + else None, + claims=claims, + ) + return self._process_broker_response(response, scopes, kwargs.get("data", {})) scopes = self._decorate_scope(scopes) telemetry_context = self._build_telemetry_context( @@ -1759,20 +1754,17 @@ def acquire_token_interactive( "04f0c124-f2bc-4f59-8241-bf6df9866bbd", # Visual Studio ] and data.get("token_type") != "ssh-cert" # Work around a known issue as of PyMsalRuntime 0.8 ) - try: - return self._acquire_token_interactive_via_broker( - scopes, - parent_window_handle, - enable_msa_passthrough, - claims, - data, - on_before_launching_ui, - prompt=prompt, - login_hint=login_hint, - max_age=max_age, - ) - except RuntimeError: # TODO: TBD - logger.debug("Broker is unavailable on this platform. Fallback to non-broker.") + return self._acquire_token_interactive_via_broker( + scopes, + parent_window_handle, + enable_msa_passthrough, + claims, + data, + on_before_launching_ui, + prompt=prompt, + login_hint=login_hint, + max_age=max_age, + ) on_before_launching_ui(ui="browser") telemetry_context = self._build_telemetry_context( diff --git a/msal/broker.py b/msal/broker.py index e9bbe28c..1443505b 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -10,8 +10,7 @@ logger = logging.getLogger(__name__) try: - import pymsalruntime # ImportError would be raised on unsupported platforms such as Windows 8 - # Its API description is available in site-packages/pymsalruntime/PyMsalRuntime.pyi + import pymsalruntime # Its API description is available in site-packages/pymsalruntime/PyMsalRuntime.pyi pymsalruntime.register_logging_callback(lambda message, level: { # New in pymsalruntime 0.7 pymsalruntime.LogLevel.TRACE: logger.debug, # Python has no TRACE level pymsalruntime.LogLevel.DEBUG: logger.debug, @@ -26,7 +25,7 @@ # https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/2406/files raise ImportError( # TODO: Remove or adjust this line right before merging this PR 'You need to install dependency by: pip install "msal[broker]>=1.20.0b1,<2"') -# Other exceptions (possibly RuntimeError) would be raised if its initialization fails +# It could throw RuntimeError when running on ancient versions of Windows class RedirectUriError(ValueError): From de8d4b27cb61d31bc9a96b473f4c049225c31dd8 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 14 Sep 2022 16:13:59 -0700 Subject: [PATCH 019/262] Implement instance_discovery only --- msal/application.py | 78 +++++++++++++++++++++++++++++++++-------- msal/authority.py | 44 +++++++++++++++-------- tests/test_authority.py | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 30 deletions(-) diff --git a/msal/application.py b/msal/application.py index 62fc6cb8..61e61580 100644 --- a/msal/application.py +++ b/msal/application.py @@ -13,7 +13,7 @@ from .oauth2cli import Client, JwtAssertionCreator from .oauth2cli.oidc import decode_part -from .authority import Authority +from .authority import Authority, WORLD_WIDE from .mex import send_request as mex_send_request from .wstrust_request import send_request as wst_send_request from .wstrust_response import * @@ -146,7 +146,6 @@ def obtain_token_by_username_password(self, username, password, **kwargs): class ClientApplication(object): - ACQUIRE_TOKEN_SILENT_ID = "84" ACQUIRE_TOKEN_BY_REFRESH_TOKEN = "85" ACQUIRE_TOKEN_BY_USERNAME_PASSWORD_ID = "301" @@ -174,6 +173,7 @@ def __init__( # when we would eventually want to add this feature to PCA in future. exclude_scopes=None, http_cache=None, + instance_discovery=None, ): """Create an instance of application. @@ -409,11 +409,40 @@ def __init__( Personally Identifiable Information (PII). Encryption is unnecessary. New in version 1.16.0. + + :param boolean instance_discovery: + Historically, MSAL would connect to a central endpoint located at + ``https://login.microsoftonline.com`` to acquire some metadata, + especially when using an unfamiliar authority. + This behavior is known as Instance Discovery. + + This parameter defaults to None, which enables the Instance Discovery. + + If you know some authorities which you allow MSAL to operate with as-is, + without involving any Instance Discovery, the recommended pattern is:: + + known_authorities = frozenset([ # Treat your known authorities as const + "https://contoso.com/adfs", "https://login.azs/foo"]) + ... + authority = "https://contoso.com/adfs" # Assuming your app will use this + app1 = PublicClientApplication( + "client_id", + authority=authority, + # Conditionally disable Instance Discovery for known authorities + instance_discovery=authority not in known_authorities, + ) + + If you do not know some authorities beforehand, + yet still want MSAL to accept any authority that you will provide, + you can use a ``False`` to unconditionally disable Instance Discovery. + + New in version 1.19.0. """ self.client_id = client_id self.client_credential = client_credential self.client_claims = client_claims self._client_capabilities = client_capabilities + self._instance_discovery = instance_discovery if exclude_scopes and not isinstance(exclude_scopes, list): raise ValueError( @@ -453,9 +482,13 @@ def __init__( # Here the self.authority will not be the same type as authority in input try: + authority_to_use = authority or "https://{}/common/".format(WORLD_WIDE) self.authority = Authority( - authority or "https://login.microsoftonline.com/common/", - self.http_client, validate_authority=validate_authority) + authority_to_use, + self.http_client, + validate_authority=validate_authority, + instance_discovery=self._instance_discovery, + ) except ValueError: # Those are explicit authority validation errors raise except Exception: # The rest are typically connection errors @@ -463,8 +496,10 @@ def __init__( # Since caller opts in to use region, here we tolerate connection # errors happened during authority validation at non-region endpoint self.authority = Authority( - authority or "https://login.microsoftonline.com/common/", - self.http_client, validate_authority=False) + authority_to_use, + self.http_client, + instance_discovery=False, + ) else: raise @@ -534,10 +569,11 @@ def _get_regional_authority(self, central_authority): "sts.windows.net", ) else "{}.{}".format(region_to_use, central_authority.instance)) - return Authority( + return Authority( # The central_authority has already been validated "https://{}/{}".format(regional_host, central_authority.tenant), self.http_client, - validate_authority=False) # The central_authority has already been validated + instance_discovery=False, + ) return None def _build_client(self, client_credential, authority, skip_regional_client=False): @@ -789,7 +825,8 @@ def get_authorization_request_url( # Multi-tenant app can use new authority on demand the_authority = Authority( authority, - self.http_client + self.http_client, + instance_discovery=self._instance_discovery, ) if authority else self.authority client = _ClientWithCcsRoutingInfo( @@ -1012,14 +1049,23 @@ def _find_msal_accounts(self, environment): } return list(grouped_accounts.values()) + def _get_instance_metadata(self): # This exists so it can be mocked in unit test + resp = self.http_client.get( + "https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=https://login.microsoftonline.com/common/oauth2/authorize", # TBD: We may extend this to use self._instance_discovery endpoint + headers={'Accept': 'application/json'}) + resp.raise_for_status() + return json.loads(resp.text)['metadata'] + def _get_authority_aliases(self, instance): + if self._instance_discovery is False: + return [] + if self.authority._is_known_to_developer: + # Then it is an ADFS/B2C/known_authority_hosts situation + # which may not reach the central endpoint, so we skip it. + return [] if not self.authority_groups: - resp = self.http_client.get( - "https://login.microsoftonline.com/common/discovery/instance?api-version=1.1&authorization_endpoint=https://login.microsoftonline.com/common/oauth2/authorize", - headers={'Accept': 'application/json'}) - resp.raise_for_status() self.authority_groups = [ - set(group['aliases']) for group in json.loads(resp.text)['metadata']] + set(group['aliases']) for group in self._get_instance_metadata()] for group in self.authority_groups: if instance in group: return [alias for alias in group if alias != instance] @@ -1168,6 +1214,7 @@ def acquire_token_silent_with_error( # the_authority = Authority( # authority, # self.http_client, + # instance_discovery=self._instance_discovery, # ) if authority else self.authority result = self._acquire_token_silent_from_cache_and_possibly_refresh_it( scopes, account, self.authority, force_refresh=force_refresh, @@ -1189,7 +1236,8 @@ def acquire_token_silent_with_error( the_authority = Authority( "https://" + alias + "/" + self.authority.tenant, self.http_client, - validate_authority=False) + instance_discovery=False, + ) result = self._acquire_token_silent_from_cache_and_possibly_refresh_it( scopes, account, the_authority, force_refresh=force_refresh, claims_challenge=claims_challenge, diff --git a/msal/authority.py b/msal/authority.py index 81788200..216fed5d 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -58,7 +58,11 @@ def http_client(self): # Obsolete. We will remove this eventually "authority.http_client might be removed in MSAL Python 1.21+", DeprecationWarning) return self._http_client - def __init__(self, authority_url, http_client, validate_authority=True): + def __init__( + self, authority_url, http_client, + validate_authority=True, + instance_discovery=None, + ): """Creates an authority instance, and also validates it. :param validate_authority: @@ -67,19 +71,34 @@ def __init__(self, authority_url, http_client, validate_authority=True): This parameter only controls whether an instance discovery will be performed. """ + # :param instance_discovery: + # By default, the known-to-Microsoft validation will use an + # instance discovery endpoint located at ``login.microsoftonline.com``. + # You can customize the endpoint by providing a url as a string. + # Or you can turn this behavior off by passing in a False here. self._http_client = http_client if isinstance(authority_url, AuthorityBuilder): authority_url = str(authority_url) authority, self.instance, tenant = canonicalize(authority_url) + self.is_adfs = tenant.lower() == 'adfs' parts = authority.path.split('/') - is_b2c = any(self.instance.endswith("." + d) for d in WELL_KNOWN_B2C_HOSTS) or ( - len(parts) == 3 and parts[2].lower().startswith("b2c_")) - if (tenant != "adfs" and (not is_b2c) and validate_authority - and self.instance not in WELL_KNOWN_AUTHORITY_HOSTS): - payload = instance_discovery( + is_b2c = any( + self.instance.endswith("." + d) for d in WELL_KNOWN_B2C_HOSTS + ) or (len(parts) == 3 and parts[2].lower().startswith("b2c_")) + self._is_known_to_developer = self.is_adfs or is_b2c or not validate_authority + is_known_to_microsoft = self.instance in WELL_KNOWN_AUTHORITY_HOSTS + instance_discovery_endpoint = 'https://{}/common/discovery/instance'.format( # Note: This URL seemingly returns V1 endpoint only + WORLD_WIDE # Historically using WORLD_WIDE. Could use self.instance too + # See https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.0.0/src/Microsoft.Identity.Client/Instance/AadInstanceDiscovery.cs#L101-L103 + # and https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.0.0/src/Microsoft.Identity.Client/Instance/AadAuthority.cs#L19-L33 + ) if instance_discovery in (None, True) else instance_discovery + if instance_discovery_endpoint and not ( + is_known_to_microsoft or self._is_known_to_developer): + payload = _instance_discovery( "https://{}{}/oauth2/v2.0/authorize".format( self.instance, authority.path), - self._http_client) + self._http_client, + instance_discovery_endpoint) if payload.get("error") == "invalid_instance": raise ValueError( "invalid_instance: " @@ -113,7 +132,6 @@ def __init__(self, authority_url, http_client, validate_authority=True): self.token_endpoint = openid_config['token_endpoint'] self.device_authorization_endpoint = openid_config.get('device_authorization_endpoint') _, _, self.tenant = canonicalize(self.token_endpoint) # Usually a GUID - self.is_adfs = self.tenant.lower() == 'adfs' def user_realm_discovery(self, username, correlation_id=None, response=None): # It will typically return a dict containing "ver", "account_type", @@ -145,13 +163,9 @@ def canonicalize(authority_url): % authority_url) return authority, authority.hostname, parts[1] -def instance_discovery(url, http_client, **kwargs): - resp = http_client.get( # Note: This URL seemingly returns V1 endpoint only - 'https://{}/common/discovery/instance'.format( - WORLD_WIDE # Historically using WORLD_WIDE. Could use self.instance too - # See https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.0.0/src/Microsoft.Identity.Client/Instance/AadInstanceDiscovery.cs#L101-L103 - # and https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.0.0/src/Microsoft.Identity.Client/Instance/AadAuthority.cs#L19-L33 - ), +def _instance_discovery(url, http_client, instance_discovery_endpoint, **kwargs): + resp = http_client.get( + instance_discovery_endpoint, params={'authorization_endpoint': url, 'api-version': '1.0'}, **kwargs) return json.loads(resp.text) diff --git a/tests/test_authority.py b/tests/test_authority.py index fc6e12fc..dd91afbb 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -1,5 +1,10 @@ import os +try: + from unittest.mock import patch +except: + from mock import patch +import msal from msal.authority import * from tests import unittest from tests.http_client import MinimalHttpClient @@ -123,3 +128,64 @@ class MockResponse(object): finally: # MUST NOT let the previous test changes affect other test cases Authority._domains_without_user_realm_discovery = set([]) + +@patch("msal.authority.tenant_discovery", return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", + }) +@patch("msal.authority._instance_discovery") +@patch.object(msal.ClientApplication, "_get_instance_metadata", return_value=[]) +class TestMsalBehaviorsWithoutAndWithInstanceDiscoveryBoolean(unittest.TestCase): + """Test cases use ClientApplication, which is a base class of both PCA and CCA""" + + def test_by_default_a_known_to_microsoft_authority_should_skip_validation_but_still_use_instance_metadata( + self, instance_metadata, known_to_microsoft_validation, _): + app = msal.ClientApplication("id", authority="https://login.microsoftonline.com/common") + known_to_microsoft_validation.assert_not_called() + app.get_accounts() # This could make an instance metadata call for authority aliases + instance_metadata.assert_called_once_with() + + def test_validate_authority_boolean_should_skip_validation_and_instance_metadata( + self, instance_metadata, known_to_microsoft_validation, _): + """Pending deprecation, but kept for backward compatibility, for now""" + app = msal.ClientApplication( + "id", authority="https://contoso.com/common", validate_authority=False) + known_to_microsoft_validation.assert_not_called() + app.get_accounts() # This could make an instance metadata call for authority aliases + instance_metadata.assert_not_called() + + def test_by_default_adfs_should_skip_validation_and_instance_metadata( + self, instance_metadata, known_to_microsoft_validation, _): + """Not strictly required, but when/if we already supported it, we better keep it""" + app = msal.ClientApplication("id", authority="https://contoso.com/adfs") + known_to_microsoft_validation.assert_not_called() + app.get_accounts() # This could make an instance metadata call for authority aliases + instance_metadata.assert_not_called() + + def test_by_default_b2c_should_skip_validation_and_instance_metadata( + self, instance_metadata, known_to_microsoft_validation, _): + """Not strictly required, but when/if we already supported it, we better keep it""" + app = msal.ClientApplication( + "id", authority="https://login.b2clogin.com/contoso/b2c_policy") + known_to_microsoft_validation.assert_not_called() + app.get_accounts() # This could make an instance metadata call for authority aliases + instance_metadata.assert_not_called() + + def test_turning_off_instance_discovery_should_work_for_all_kinds_of_clouds( + self, instance_metadata, known_to_microsoft_validation, _): + for authority in [ + "https://login.microsoftonline.com/common", # Known to Microsoft + "https://contoso.com/adfs", # ADFS + "https://login.b2clogin.com/contoso/b2c_policy", # B2C + "https://private.cloud/foo", # Private Cloud + ]: + self._test_turning_off_instance_discovery_should_skip_authority_validation_and_instance_metadata( + authority, instance_metadata, known_to_microsoft_validation) + + def _test_turning_off_instance_discovery_should_skip_authority_validation_and_instance_metadata( + self, authority, instance_metadata, known_to_microsoft_validation): + app = msal.ClientApplication("id", authority=authority, instance_discovery=False) + known_to_microsoft_validation.assert_not_called() + app.get_accounts() # This could make an instance metadata call for authority aliases + instance_metadata.assert_not_called() + From 60506eb0fe38c186fda6f5fc5b4e8c117008e867 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 19 Sep 2022 20:39:29 -0700 Subject: [PATCH 020/262] MSAL Python 1.19.0 Bump version number --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 61e61580..caa4b830 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.18.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.19.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From f0f00afd2e8424cc3e166a3e9c49115746409203 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 19 Sep 2022 20:48:53 -0700 Subject: [PATCH 021/262] Bump up cryptography upperbound We have reviewed https://cryptography.io/en/latest/changelog/ --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 814627f9..64fd07df 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ 'requests>=2.0.0,<3', 'PyJWT[crypto]>=1.0.0,<3', # MSAL does not use jwt.decode(), therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ - 'cryptography>=0.6,<40', + 'cryptography>=0.6,<41', # load_pem_private_key() is available since 0.6 # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 # From 06590ab3f78cd1eec575ee44fed69fe24774a350 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 20 Sep 2022 22:34:00 -0700 Subject: [PATCH 022/262] Test acquire_token_silent() for confidential client --- tests/test_e2e.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9a971f46..51da742b 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -133,6 +133,11 @@ def assertCacheWorksForApp(self, result_from_wire, scope): self.assertEqual( result_from_wire['access_token'], result_from_cache['access_token'], "We should get a cached AT") + self.app.acquire_token_silent( + # Result will typically be None, because client credential grant returns no RT. + # But we care more on this call should succeed without exception. + scope, account=None, + force_refresh=True) # Mimic the AT already expires def _test_username_password(self, authority=None, client_id=None, username=None, password=None, scope=None, @@ -618,11 +623,12 @@ def _test_acquire_token_by_client_secret( self, client_id=None, client_secret=None, authority=None, scope=None, **ignored): assert client_id and client_secret and authority and scope - app = msal.ConfidentialClientApplication( + self.app = msal.ConfidentialClientApplication( client_id, client_credential=client_secret, authority=authority, http_client=MinimalHttpClient()) - result = app.acquire_token_for_client(scope) + result = self.app.acquire_token_for_client(scope) self.assertIsNotNone(result.get("access_token"), "Got %s instead" % result) + self.assertCacheWorksForApp(result, scope) class WorldWideTestCase(LabBasedTestCase): From f41d5469cbec95e1781396f883e5abeb94ec7acd Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 27 Sep 2022 22:25:15 -0700 Subject: [PATCH 023/262] Adopt pymsalruntime 0.13 --- msal/broker.py | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/msal/broker.py b/msal/broker.py index 1443505b..c9d0e38e 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -90,8 +90,7 @@ def _convert_result(result, client_id, expected_token_type=None): # Mimic an on id_token_claims = json.loads(result.get_id_token()) if result.get_id_token() else {} account = result.get_account() assert account, "Account is expected to be always available" - ## Note: As of pymsalruntime 0.1.0, only wam_account_ids property is available - #account.get_account_property("wam_account_ids") + # Note: There are more account attribute getters available in pymsalruntime 0.13+ return_value = {k: v for k, v in { "access_token": result.get_access_token(), "expires_in": result.get_access_token_expiry_time() - int(time.time()), # Convert epoch to count-down diff --git a/setup.py b/setup.py index 31f585b3..effc825c 100644 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ # The broker is defined as optional dependency, # so that downstream apps can opt in. The opt-in is needed, partially because # most existing MSAL Python apps do not have the redirect_uri needed by broker. - "pymsalruntime>=0.11.2,<0.12;python_version>='3.6' and platform_system=='Windows'", + "pymsalruntime>=0.11.2,<0.14;python_version>='3.6' and platform_system=='Windows'", ], }, ) From 28b45a394cec06ebea4dcd4ff165f2938af8cd96 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 27 Sep 2022 22:33:58 -0700 Subject: [PATCH 024/262] Remove automatic msa-pt for Azure CLI and Visual Studio --- msal/application.py | 26 +++++++++++--------------- tests/msaltest.py | 10 ++++++++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/msal/application.py b/msal/application.py index 5c8b7ba9..73992351 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1766,6 +1766,17 @@ def acquire_token_interactive( - A dict containing an "error" key, when token refresh failed. """ data = kwargs.pop("data", {}) + enable_msa_passthrough = kwargs.pop( # MUST remove it from kwargs + "enable_msa_passthrough", # Keep it as a hidden param, for now. + # OPTIONAL. MSA-Passthrough is a legacy configuration, + # needed by a small amount of Microsoft first-party apps, + # which would login MSA accounts via ".../organizations" authority. + # If you app belongs to this category, AND you are enabling broker, + # you would want to enable this flag. Default value is False. + # More background of MSA-PT is available from this internal docs: + # https://microsoft.sharepoint.com/:w:/t/Identity-DevEx/EatIUauX3c9Ctw1l7AQ6iM8B5CeBZxc58eoQCE0IuZ0VFw?e=tgc3jP&CID=39c853be-76ea-79d7-ee73-f1b2706ede05 + False + ) and data.get("token_type") != "ssh-cert" # Work around a known issue as of PyMsalRuntime 0.8 self._validate_ssh_cert_input_data(data) if not on_before_launching_ui: on_before_launching_ui = lambda **kwargs: None @@ -1786,21 +1797,6 @@ def acquire_token_interactive( logger.warning( "Ignoring parameter extra_scopes_to_consent, " "which is not supported by broker") - enable_msa_passthrough = kwargs.pop( - "enable_msa_passthrough", # Keep it as a hidden param, for now. - # OPTIONAL. MSA-Passthrough is a legacy configuration, - # needed by a small amount of Microsoft first-party apps, - # which would login MSA accounts via ".../organizations" authority. - # If you app belongs to this category, AND you are enabling broker, - # you would want to enable this flag. Default value is equivalent to False. - self.client_id in [ - # Experimental: Automatically enable MSA-PT mode for known MSA-PT apps - # More background of MSA-PT is available from this internal docs: - # https://microsoft.sharepoint.com/:w:/t/Identity-DevEx/EatIUauX3c9Ctw1l7AQ6iM8B5CeBZxc58eoQCE0IuZ0VFw?e=tgc3jP&CID=39c853be-76ea-79d7-ee73-f1b2706ede05 - "04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI - "04f0c124-f2bc-4f59-8241-bf6df9866bbd", # Visual Studio - ] and data.get("token_type") != "ssh-cert" # Work around a known issue as of PyMsalRuntime 0.8 - ) return self._acquire_token_interactive_via_broker( scopes, parent_window_handle, diff --git a/tests/msaltest.py b/tests/msaltest.py index 5d33f2c2..cc4e5606 100644 --- a/tests/msaltest.py +++ b/tests/msaltest.py @@ -1,6 +1,9 @@ import getpass, logging, pprint, sys, msal +AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" +VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" + def _input_boolean(message): return input( "{} (N/n/F/f or empty means False, otherwise it is True): ".format(message) @@ -81,6 +84,9 @@ def _acquire_token_interactive(app, scopes, data=None): result = app.acquire_token_interactive( scopes, parent_window_handle=app.CONSOLE_WINDOW_HANDLE, # This test app is a console app + enable_msa_passthrough=app.client_id in [ # Apps are expected to set this right + AZURE_CLI, VISUAL_STUDIO, + ], # Here this test app mimics the setting for some known MSA-PT apps prompt=prompt, login_hint=login_hint, data=data or {}) if login_hint and "id_token_claims" in result: signed_in_user = result.get("id_token_claims", {}).get("preferred_username") @@ -142,8 +148,8 @@ def exit(app): def main(): print("Welcome to the Msal Python Console Test App, committed at 2022-5-2\n") chosen_app = _select_options([ - {"client_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", "name": "Azure CLI (Correctly configured for MSA-PT)"}, - {"client_id": "04f0c124-f2bc-4f59-8241-bf6df9866bbd", "name": "Visual Studio (Correctly configured for MSA-PT)"}, + {"client_id": AZURE_CLI, "name": "Azure CLI (Correctly configured for MSA-PT)"}, + {"client_id": VISUAL_STUDIO, "name": "Visual Studio (Correctly configured for MSA-PT)"}, {"client_id": "95de633a-083e-42f5-b444-a4295d8e9314", "name": "Whiteboard Services (Non MSA-PT app. Accepts AAD & MSA accounts.)"}, ], option_renderer=lambda a: a["name"], From 3d6e977f7c2e7df2a3d1ad92fe1d960c2094730d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 29 Sep 2022 19:11:05 -0700 Subject: [PATCH 025/262] Refactor test infrastructure to expose a known bug Apply the refactor to similar code path --- msal/application.py | 5 +- msal/broker.py | 2 +- tests/test_e2e.py | 113 +++++++++++++++++++++++++++++++------------- 3 files changed, 85 insertions(+), 35 deletions(-) diff --git a/msal/application.py b/msal/application.py index 73992351..b1324284 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1857,6 +1857,7 @@ def _acquire_token_interactive_via_broker( if login_hint and prompt != "select_account": # OIDC prompts when the user did not sign in accounts = self.get_accounts(username=login_hint) if len(accounts) == 1: # Unambiguously proceed with this account + logger.debug("Calling broker._acquire_token_silently()") response = _acquire_token_silently( # When it works, it bypasses prompt authority, self.client_id, @@ -1868,6 +1869,7 @@ def _acquire_token_interactive_via_broker( return self._process_broker_response(response, scopes, data) # login_hint undecisive or not exists if prompt == "none" or not prompt: # Must/Can attempt _signin_silently() + logger.debug("Calling broker._signin_silently()") response = _signin_silently( # Unlike OIDC, it doesn't honor login_hint authority, self.client_id, scopes, validateAuthority=validate_authority, @@ -1903,7 +1905,8 @@ def _acquire_token_interactive_via_broker( pass # It will fall back to the _signin_interactively() else: return self._process_broker_response(response, scopes, data) - # Falls back to _signin_interactively() + + logger.debug("Falls back to broker._signin_interactively()") on_before_launching_ui(ui="broker") response = _signin_interactively( authority, self.client_id, scopes, diff --git a/msal/broker.py b/msal/broker.py index c9d0e38e..f1de262c 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -144,7 +144,7 @@ def _signin_silently( def _signin_interactively( authority, client_id, scopes, parent_window_handle, # None means auto-detect for console apps - prompt=None, + prompt=None, # Note: This function does not really use this parameter login_hint=None, claims=None, correlation_id=None, diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 607af6e4..cd3ee467 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -54,6 +54,25 @@ def _get_app_and_auth_code( assert ac is not None return (app, ac, redirect_uri) +def _render(url, description=None): + # Render a url in html if description is available, otherwise return url as-is + return "{description}".format( + url=url, description=description) if description else url + + +def _get_hint(html_mode=None, username=None, lab_name=None, username_uri=None): + return "Sign in with {user} whose password is available from {lab}".format( + user=("{}".format(username) if html_mode else username) + if username + else "the upn from {}".format(_render( + username_uri, description="here" if html_mode else None)), + lab=_render( + "https://aka.ms/GetLabUserSecret?Secret=" + (lab_name or "msidlabXYZ"), + description="this password api" if html_mode else None, + ), + ) + + @unittest.skipIf(os.getenv("TRAVIS_TAG"), "Skip e2e tests during tagged release") class E2eTestCase(unittest.TestCase): @@ -212,25 +231,31 @@ def _test_device_flow( def _test_acquire_token_interactive( self, client_id=None, authority=None, scope=None, port=None, - username_uri="", # But you would want to provide one + username=None, lab_name=None, + username_uri="", # Unnecessary if you provided username and lab_name data=None, # Needed by ssh-cert feature prompt=None, + enable_msa_passthrough=None, **ignored): assert client_id and authority and scope self.app = self._build_app(client_id, authority=authority) + logger.info(_get_hint( # Useful when testing broker which shows no welcome_template + username=username, lab_name=lab_name, username_uri=username_uri)) result = self.app.acquire_token_interactive( scope, + login_hint=username, prompt=prompt, timeout=120, port=port, parent_window_handle=self.app.CONSOLE_WINDOW_HANDLE, # This test app is a console app + enable_msa_passthrough=enable_msa_passthrough, # Needed when testing MSA-PT app welcome_template= # This is an undocumented feature for testing """

    {id}

      -
    1. Get a username from the upn shown at here
      2. -
    2. Get its password from https://aka.ms/GetLabUserSecret?Secret=msidlabXYZ - (replace the lab name with the labName from the link above).
      3. +
    3. {hint}
    4. Sign In or Abort
      5. -
    """.format(id=self.id(), username_uri=username_uri), +
""".format(id=self.id(), hint=_get_hint( + html_mode=True, + username=username, lab_name=lab_name, username_uri=username_uri)), data=data or {}, ) self.assertIn( @@ -239,6 +264,11 @@ def _test_acquire_token_interactive( # Note: No interpolation here, cause error won't always present error=result.get("error"), error_description=result.get("error_description"))) + if username and result.get("id_token_claims", {}).get("preferred_username"): + self.assertEqual( + username, result["id_token_claims"]["preferred_username"], + "You are expected to sign in as account {}, but tokens returned is for {}".format( + username, result["id_token_claims"]["preferred_username"])) self.assertCacheWorksForUser(result, scope, username=None, data=data or {}) return result # For further testing @@ -260,7 +290,7 @@ def test_ssh_cert_for_service_principal(self): self.assertEqual("ssh-cert", result["token_type"]) @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") - def test_ssh_cert_for_user(self): + def test_ssh_cert_for_user_should_work_with_any_account(self): result = self._test_acquire_token_interactive( client_id="04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI is one # of the only 2 clients that are PreAuthz to use ssh cert feature @@ -555,7 +585,8 @@ def _test_acquire_token_by_auth_code( def _test_acquire_token_by_auth_code_flow( self, client_id=None, authority=None, port=None, scope=None, - username_uri="", # But you would want to provide one + username=None, lab_name=None, + username_uri="", # Optional if you provided username and lab_name **ignored): assert client_id and authority and scope self.app = msal.ClientApplication( @@ -568,11 +599,11 @@ def _test_acquire_token_by_auth_code_flow( auth_response = receiver.get_auth_response( auth_uri=flow["auth_uri"], state=flow["state"], timeout=60, welcome_template="""

{id}

    -
  1. Get a username from the upn shown at here
    2. -
  2. Get its password from https://aka.ms/GetLabUserSecret?Secret=msidlabXYZ - (replace the lab name with the labName from the link above).
    3. +
  3. {hint}
  4. Sign In or Abort
    5. -
""".format(id=self.id(), username_uri=username_uri), + """.format(id=self.id(), hint=_get_hint( + html_mode=True, + username=username, lab_name=lab_name, username_uri=username_uri)), ) if auth_response is None: self.skipTest("Timed out. Did not have test settings in hand? Prepare and retry.") @@ -592,6 +623,11 @@ def _test_acquire_token_by_auth_code_flow( # Note: No interpolation here, cause error won't always present error=result.get("error"), error_description=result.get("error_description"))) + if username and result.get("id_token_claims", {}).get("preferred_username"): + self.assertEqual( + username, result["id_token_claims"]["preferred_username"], + "You are expected to sign in as account {}, but tokens returned is for {}".format( + username, result["id_token_claims"]["preferred_username"])) self.assertCacheWorksForUser(result, scope, username=None) def _test_acquire_token_obo(self, config_pca, config_cca, @@ -689,10 +725,23 @@ def test_adfs2019_fed_user(self): @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_cloud_acquire_token_interactive(self): - config = self.get_lab_user(usertype="cloud") - self._test_acquire_token_interactive( - username_uri="https://msidlab.com/api/user?usertype=cloud", - **config) + self._test_acquire_token_interactive(**self.get_lab_user(usertype="cloud")) + + @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") + def test_msa_pt_app_signin_via_organizations_authority_without_login_hint(self): + """There is/was an upstream bug. See test case full docstring for the details. + + When a MSAL-PT flow that account control is launched, user has 2+ AAD accounts in WAM, + selects an AAD account that is NOT the default AAD account from the OS, + it will incorrectly get tokens for default AAD account. + """ + self._test_acquire_token_interactive(**dict( + self.get_lab_user(usertype="cloud"), # This is generally not the current laptop's default AAD account + authority="https://login.microsoftonline.com/organizations", + client_id="04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI is an MSA-PT app + enable_msa_passthrough=True, + prompt="select_account", # In MSAL Python, this resets login_hint + )) def test_ropc_adfs2019_onprem(self): # Configuration is derived from https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.7.0/tests/Microsoft.Identity.Test.Common/TestConstants.cs#L250-L259 @@ -719,22 +768,22 @@ def test_adfs2019_onprem_acquire_token_by_auth_code(self): @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_adfs2019_onprem_acquire_token_by_auth_code_flow(self): config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") - config["authority"] = "https://fs.%s.com/adfs" % config["lab_name"] - config["scope"] = self.adfs2019_scopes - config["port"] = 8080 - self._test_acquire_token_by_auth_code_flow( - username_uri="https://msidlab.com/api/user?usertype=onprem&federationprovider=ADFSv2019", - **config) + self._test_acquire_token_by_auth_code_flow(**dict( + config, + authority="https://fs.%s.com/adfs" % config["lab_name"], + scope=self.adfs2019_scopes, + port=8080, + )) @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_adfs2019_onprem_acquire_token_interactive(self): config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") - config["authority"] = "https://fs.%s.com/adfs" % config["lab_name"] - config["scope"] = self.adfs2019_scopes - config["port"] = 8080 - self._test_acquire_token_interactive( - username_uri="https://msidlab.com/api/user?usertype=onprem&federationprovider=ADFSv2019", - **config) + self._test_acquire_token_interactive(**dict( + config, + authority="https://fs.%s.com/adfs" % config["lab_name"], + scope=self.adfs2019_scopes, + port=8080, + )) @unittest.skipUnless( os.getenv("LAB_OBO_CLIENT_SECRET"), @@ -816,14 +865,12 @@ def test_b2c_acquire_token_by_auth_code(self): @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_b2c_acquire_token_by_auth_code_flow(self): - config = self.get_lab_app_object(azureenvironment="azureb2ccloud") - self._test_acquire_token_by_auth_code_flow( + self._test_acquire_token_by_auth_code_flow(**dict( + self.get_lab_user(usertype="b2c", b2cprovider="local"), authority=self._build_b2c_authority("B2C_1_SignInPolicy"), - client_id=config["appId"], port=3843, # Lab defines 4 of them: [3843, 4584, 4843, 60000] - scope=config["scopes"], - username_uri="https://msidlab.com/api/user?usertype=b2c&b2cprovider=local", - ) + scope=self.get_lab_app_object(azureenvironment="azureb2ccloud")["scopes"], + )) def test_b2c_acquire_token_by_ropc(self): config = self.get_lab_app_object(azureenvironment="azureb2ccloud") From 66a90824267efe5ddd2a2413dd6050aaee3607cc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 6 Oct 2022 21:20:32 -0700 Subject: [PATCH 026/262] Add more docs --- msal/application.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/msal/application.py b/msal/application.py index b1324284..8c803f70 100644 --- a/msal/application.py +++ b/msal/application.py @@ -444,9 +444,22 @@ def __init__( New in version 1.19.0. :param boolean allow_broker: - Brokers provide Single-Sign-On, device identification, - and application identification verification. - This flag defaults to None, which means MSAL will not utilize broker. + A broker is a component installed on your device. + Broker implicitly gives your device an identity. By using a broker, + your device becomes a factor that can satisfy MFA (Multi-factor authentication). + This factor would become mandatory + if a tenant's admin enables a corresponding Conditional Access (CA) policy. + The broker's presence allows Microsoft identity platform + to have higher confidence that the tokens are being issued to your device, + and that is more secure. + + An additional benefit of broker is, + it runs as a long-lived process with your device's OS, + and maintains its own cache, + so that your broker-enabled apps (even a CLI) + could automatically SSO from a previously established signed-in session. + + This parameter defaults to None, which means MSAL will not utilize a broker. If this parameter is set to True, MSAL will use the broker whenever possible, and automatically fall back to non-broker behavior. From 789342c4bc57f80314a83662af614f6f02a207d0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 6 Oct 2022 21:32:36 -0700 Subject: [PATCH 027/262] MSAL Python 1.20.0 Bumping version number --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 8c803f70..916f7170 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.20.0b1" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.20.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From a044c324cbb59945547f8c7745e5f985b5320a55 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 6 Oct 2022 21:39:47 -0700 Subject: [PATCH 028/262] Update installation instruction --- msal/broker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/broker.py b/msal/broker.py index f1de262c..8b997c61 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -24,7 +24,7 @@ # PyMsalRuntime currently supports these Windows versions, listed in this MSFT internal link # https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/2406/files raise ImportError( # TODO: Remove or adjust this line right before merging this PR - 'You need to install dependency by: pip install "msal[broker]>=1.20.0b1,<2"') + 'You need to install dependency by: pip install "msal[broker]>=1.20,<2"') # It could throw RuntimeError when running on ancient versions of Windows From 8aecacb23f2f622039e5e0fc3596eb5ad8e92fd8 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 2 Nov 2022 16:27:53 -0700 Subject: [PATCH 029/262] Test matrix covers Python 3.11 --- .github/workflows/python-package.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8950c15a..9c11210f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev"] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index effc825c..dd86f2ee 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', ], From 2c79aaba9facfb5c36b30570a17bcdecd145f334 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 14 Nov 2022 14:51:50 -0800 Subject: [PATCH 030/262] Switch to new region endpoints --- msal/application.py | 2 +- tests/test_e2e.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index 916f7170..b469c350 100644 --- a/msal/application.py +++ b/msal/application.py @@ -622,7 +622,7 @@ def _get_regional_authority(self, central_authority): else self._region_configured) # It will retain the None i.e. opted out logger.debug('Region to be used: {}'.format(repr(region_to_use))) if region_to_use: - regional_host = ("{}.r.login.microsoftonline.com".format(region_to_use) + regional_host = ("{}.login.microsoft.com".format(region_to_use) if central_authority.instance in ( # The list came from point 3 of the algorithm section in this internal doc # https://identitydivision.visualstudio.com/DevEx/_git/AuthLibrariesApiReview?path=/PinAuthToRegion/AAD%20SDK%20Proposal%20to%20Pin%20Auth%20to%20region.md&anchor=algorithm&_a=preview diff --git a/tests/test_e2e.py b/tests/test_e2e.py index cd3ee467..ae3683e5 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -904,7 +904,7 @@ def _test_acquire_token_for_client(self, configured_region, expected_region): self.app.http_client, "post", return_value=MinimalResponse( status_code=400, text='{"error": "mock"}')) as mocked_method: self.app.acquire_token_for_client(scopes) - expected_host = '{}.r.login.microsoftonline.com'.format( + expected_host = '{}.login.microsoft.com'.format( expected_region) if expected_region else 'login.microsoftonline.com' mocked_method.assert_called_with( 'https://{}/{}/oauth2/v2.0/token'.format( From 8805f00cf45804d14a70ddb245b5230aba1a0762 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 8 Dec 2022 10:00:19 -0800 Subject: [PATCH 031/262] Test only Python versions available on github's ubuntu 22.04 --- .github/workflows/python-package.yml | 4 ++-- tests/test_authority.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9c11210f..cf56cb2a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -23,10 +23,10 @@ jobs: LAB_OBO_PUBLIC_CLIENT_ID: ${{ secrets.LAB_OBO_PUBLIC_CLIENT_ID }} # Derived from https://docs.github.com/en/actions/guides/building-and-testing-python#starting-with-the-python-workflow-template - runs-on: ubuntu-latest + runs-on: ubuntu-latest # It switched to 22.04 shortly after 2022-Nov-8 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [2.7, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12-dev"] steps: - uses: actions/checkout@v2 diff --git a/tests/test_authority.py b/tests/test_authority.py index dd91afbb..ca0bc68f 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -59,7 +59,10 @@ def test_lessknown_host_will_return_a_set_of_v1_endpoints(self): self.assertNotIn('v2.0', a.token_endpoint) def test_unknown_host_wont_pass_instance_discovery(self): - _assert = getattr(self, "assertRaisesRegex", self.assertRaisesRegexp) # Hack + _assert = ( + # Was Regexp, added alias Regex in Py 3.2, and Regexp will be gone in Py 3.12 + getattr(self, "assertRaisesRegex", None) or + getattr(self, "assertRaisesRegexp", None)) with _assert(ValueError, "invalid_instance"): Authority('https://example.com/tenant_doesnt_matter_in_this_case', MinimalHttpClient()) From 600c11020be6a5990a7197eb382a9529dabd35b3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 9 Dec 2022 10:50:12 -0800 Subject: [PATCH 032/262] Cleaner skip declaration --- tests/test_e2e.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ae3683e5..6cf9e629 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -196,6 +196,8 @@ def _test_username_password(self, azure_region=azure_region, # Regional endpoint does not support ROPC. # Here we just use it to test a regional app won't break ROPC. client_credential=client_secret) + self.assertEqual( + self.app.get_accounts(username=username), [], "Cache starts empty") result = self.app.acquire_token_by_username_password( username, password, scopes=scope) self.assertLoosely(result) @@ -204,6 +206,9 @@ def _test_username_password(self, username=username, # Our implementation works even when "profile" scope was not requested, or when profile claims is unavailable in B2C ) + @unittest.skipIf( + os.getenv("TRAVIS"), # It is set when running on TravisCI or Github Actions + "Although it is doable, we still choose to skip device flow to save time") def _test_device_flow( self, client_id=None, authority=None, scope=None, **ignored): assert client_id and authority and scope @@ -229,6 +234,7 @@ def _test_device_flow( logger.info( "%s obtained tokens: %s", self.id(), json.dumps(result, indent=4)) + @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def _test_acquire_token_interactive( self, client_id=None, authority=None, scope=None, port=None, username=None, lab_name=None, @@ -289,7 +295,6 @@ def test_ssh_cert_for_service_principal(self): result.get("error"), result.get("error_description"))) self.assertEqual("ssh-cert", result["token_type"]) - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_ssh_cert_for_user_should_work_with_any_account(self): result = self._test_acquire_token_interactive( client_id="04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI is one @@ -524,8 +529,8 @@ def tearDownClass(cls): cls.session.close() @classmethod - def get_lab_app_object(cls, **query): # https://msidlab.com/swagger/index.html - url = "https://msidlab.com/api/app" + def get_lab_app_object(cls, client_id=None, **query): # https://msidlab.com/swagger/index.html + url = "https://msidlab.com/api/app/{}".format(client_id or "") resp = cls.session.get(url, params=query) result = resp.json()[0] result["scopes"] = [ # Raw data has extra space, such as "s1, s2" @@ -561,6 +566,7 @@ def get_lab_user(cls, **query): # https://docs.msidlab.com/labapi/userapi.html "scope": scope, } + @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def _test_acquire_token_by_auth_code( self, client_id=None, authority=None, port=None, scope=None, **ignored): @@ -583,6 +589,7 @@ def _test_acquire_token_by_auth_code( error_description=result.get("error_description"))) self.assertCacheWorksForUser(result, scope, username=None) + @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def _test_acquire_token_by_auth_code_flow( self, client_id=None, authority=None, port=None, scope=None, username=None, lab_name=None, @@ -723,11 +730,9 @@ def test_adfs2019_fed_user(self): self.skipTest("MEX endpoint in our test environment tends to fail") raise - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_cloud_acquire_token_interactive(self): self._test_acquire_token_interactive(**self.get_lab_user(usertype="cloud")) - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_msa_pt_app_signin_via_organizations_authority_without_login_hint(self): """There is/was an upstream bug. See test case full docstring for the details. @@ -751,7 +756,6 @@ def test_ropc_adfs2019_onprem(self): config["password"] = self.get_lab_user_secret(config["lab_name"]) self._test_username_password(**config) - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_adfs2019_onprem_acquire_token_by_auth_code(self): """When prompted, you can manually login using this account: @@ -765,7 +769,6 @@ def test_adfs2019_onprem_acquire_token_by_auth_code(self): config["port"] = 8080 self._test_acquire_token_by_auth_code(**config) - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_adfs2019_onprem_acquire_token_by_auth_code_flow(self): config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") self._test_acquire_token_by_auth_code_flow(**dict( @@ -775,7 +778,6 @@ def test_adfs2019_onprem_acquire_token_by_auth_code_flow(self): port=8080, )) - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_adfs2019_onprem_acquire_token_interactive(self): config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") self._test_acquire_token_interactive(**dict( @@ -846,7 +848,6 @@ def _build_b2c_authority(self, policy): base = "https://msidlabb2c.b2clogin.com/msidlabb2c.onmicrosoft.com" return base + "/" + policy # We do not support base + "?p=" + policy - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_b2c_acquire_token_by_auth_code(self): """ When prompted, you can manually login using this account: @@ -863,7 +864,6 @@ def test_b2c_acquire_token_by_auth_code(self): scope=config["scopes"], ) - @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def test_b2c_acquire_token_by_auth_code_flow(self): self._test_acquire_token_by_auth_code_flow(**dict( self.get_lab_user(usertype="b2c", b2cprovider="local"), From cff23eecc76b845eebc38518a3c7797e19904f67 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 8 Dec 2022 09:43:51 -0800 Subject: [PATCH 033/262] Fallback to expires_on when expires_in is absent --- msal/token_cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index dc26e843..0259522f 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -164,8 +164,11 @@ def __add(self, event, now=None): now = int(time.time() if now is None else now) if access_token: + default_expires_in = ( # https://www.rfc-editor.org/rfc/rfc6749#section-5.1 + int(response.get("expires_on")) - now # Some Managed Identity emits this + ) if response.get("expires_on") else 600 expires_in = int( # AADv1-like endpoint returns a string - response.get("expires_in", 3599)) + response.get("expires_in", default_expires_in)) ext_expires_in = int( # AADv1-like endpoint returns a string response.get("ext_expires_in", expires_in)) at = { From e2c079b16a2715ce31bc2cb034384c39b468dd69 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 20 Jan 2023 18:35:26 -0800 Subject: [PATCH 034/262] Show lab api error, useful when trying api params --- tests/test_e2e.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 6cf9e629..5c43f4f0 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -551,6 +551,8 @@ def get_lab_user_secret(cls, lab_name="msidlab4"): def get_lab_user(cls, **query): # https://docs.msidlab.com/labapi/userapi.html resp = cls.session.get("https://msidlab.com/api/user", params=query) result = resp.json()[0] + assert result.get("upn"), "Found no test user but {}".format( + json.dumps(result, indent=2)) _env = query.get("azureenvironment", "").lower() authority_base = { "azureusgovernment": "https://login.microsoftonline.us/" From a08dd9e3e814893165703c1432357b4989b3b7c3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 21 Oct 2022 16:36:24 -0700 Subject: [PATCH 035/262] Clarify when (not) to read API section --- docs/index.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 95b89b98..b376f52d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -62,6 +62,16 @@ API === The following section is the API Reference of MSAL Python. +The API Reference is like a dictionary. You **read this API section when and only when**: + +* You already followed our sample(s) above and have your app up and running, + but want to know more on how you could tweak the authentication experience + by using other optional parameters (there are plenty of them!) +* You read the MSAL Python source code and found a helper function that is useful to you, + then you would want to double check whether that helper is documented below. + Only documented APIs are considered part of the MSAL Python public API, + which are guaranteed to be backward-compatible in MSAL Python 1.x series. + Undocumented internal helpers are subject to change anytime, without prior notice. .. note:: From 20667dc718f69a321d568cd403acbb573ef039c0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 10 Nov 2022 12:07:49 -0800 Subject: [PATCH 036/262] Use broker for SSH Cert feature It is OK to use PyMsalRuntime now, since ESTS has deployed the fix. https://identitydivision.visualstudio.com/Engineering/_workitems/edit/2060332 --- msal/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index b469c350..7e3ec16e 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1375,7 +1375,7 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( if account and account.get("authority_type") == _AUTHORITY_TYPE_CLOUDSHELL: return self._acquire_token_by_cloud_shell(scopes, data=data) - if self._enable_broker and account is not None and data.get("token_type") != "ssh-cert": + if self._enable_broker and account is not None: from .broker import _acquire_token_silently response = _acquire_token_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), @@ -1799,7 +1799,7 @@ def acquire_token_interactive( return self._acquire_token_by_cloud_shell(scopes, data=data) claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) - if self._enable_broker and data.get("token_type") != "ssh-cert": + if self._enable_broker: if parent_window_handle is None: raise ValueError( "parent_window_handle is required when you opted into using broker. " From 94561cd79b78a70c40ddf2f7f792919f527f7ffe Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 27 Jan 2023 12:07:31 -0800 Subject: [PATCH 037/262] Allow using client_id as scope. Needed by B2C. --- msal/application.py | 15 +++------------ tests/test_application.py | 15 +++++++++++++++ tests/test_e2e.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/msal/application.py b/msal/application.py index 7e3ec16e..62ba4b5c 100644 --- a/msal/application.py +++ b/msal/application.py @@ -588,18 +588,9 @@ def _decorate_scope( raise ValueError( "API does not accept {} value as user-provided scopes".format( reserved_scope)) - if self.client_id in scope_set: - if len(scope_set) > 1: - # We make developers pass their client id, so that they can express - # the intent that they want the token for themselves (their own - # app). - # If we do not restrict them to passing only client id then they - # could write code where they expect an id token but end up getting - # access_token. - raise ValueError("Client Id can only be provided as a single scope") - decorated = set(reserved_scope) # Make a writable copy - else: - decorated = scope_set | reserved_scope + + # client_id can also be used as a scope in B2C + decorated = scope_set | reserved_scope decorated -= self._exclude_scopes return list(decorated) diff --git a/tests/test_application.py b/tests/test_application.py index 804ccb82..b62f41d5 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -625,3 +625,18 @@ def test_organizations_authority_should_emit_warnning(self): self._test_certain_authority_should_emit_warnning( authority="https://login.microsoftonline.com/organizations") + +class TestScopeDecoration(unittest.TestCase): + def _test_client_id_should_be_a_valid_scope(self, client_id, other_scopes): + # B2C needs this https://learn.microsoft.com/en-us/azure/active-directory-b2c/access-tokens#openid-connect-scopes + reserved_scope = ['openid', 'profile', 'offline_access'] + scopes_to_use = [client_id] + other_scopes + self.assertEqual( + set(ClientApplication(client_id)._decorate_scope(scopes_to_use)), + set(scopes_to_use + reserved_scope), + "Scope decoration should return input scopes plus reserved scopes") + + def test_client_id_should_be_a_valid_scope(self): + self._test_client_id_should_be_a_valid_scope("client_id", []) + self._test_client_id_should_be_a_valid_scope("client_id", ["foo"]) + diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 5c43f4f0..48ffe47a 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -884,6 +884,18 @@ def test_b2c_acquire_token_by_ropc(self): scope=config["scopes"], ) + def test_b2c_allows_using_client_id_as_scope(self): + # See also https://learn.microsoft.com/en-us/azure/active-directory-b2c/access-tokens#openid-connect-scopes + config = self.get_lab_app_object(azureenvironment="azureb2ccloud") + config["scopes"] = [config["appId"]] + self._test_username_password( + authority=self._build_b2c_authority("B2C_1_ROPC_Auth"), + client_id=config["appId"], + username="b2clocal@msidlabb2c.onmicrosoft.com", + password=self.get_lab_user_secret("msidlabb2c"), + scope=config["scopes"], + ) + class WorldWideRegionalEndpointTestCase(LabBasedTestCase): region = "westus" From 0081f3085782d69472e2ad030e45f9fb0903f4b9 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 30 Jan 2023 11:25:05 -0800 Subject: [PATCH 038/262] Bump PyMsalRuntime to 0.13.2+ --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dd86f2ee..73be693f 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,9 @@ # The broker is defined as optional dependency, # so that downstream apps can opt in. The opt-in is needed, partially because # most existing MSAL Python apps do not have the redirect_uri needed by broker. - "pymsalruntime>=0.11.2,<0.14;python_version>='3.6' and platform_system=='Windows'", + # MSAL Python uses a subset of API from PyMsalRuntime 0.11.2+, + # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) + "pymsalruntime>=0.13.2,<0.14;python_version>='3.6' and platform_system=='Windows'", ], }, ) From b8ff2e4da2d78636e5daaf8b59977a3bd9ebad18 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 31 Jan 2023 09:30:04 -0800 Subject: [PATCH 039/262] MSAL Python 1.21.0 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 62ba4b5c..e024252c 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.20.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.21.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From f88f26b7b37fa48295619460dd34c53bbe92a218 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 28 Jan 2023 18:50:30 -0800 Subject: [PATCH 040/262] Remind user to use proper helper for ssh cert test --- tests/msaltest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/msaltest.py b/tests/msaltest.py index cc4e5606..b1556106 100644 --- a/tests/msaltest.py +++ b/tests/msaltest.py @@ -34,7 +34,7 @@ def _select_options( return raw_data def _input_scopes(): - return _select_options([ + scopes = _select_options([ "https://graph.microsoft.com/.default", "https://management.azure.com/.default", "User.Read", @@ -42,7 +42,10 @@ def _input_scopes(): ], header="Select a scope (multiple scopes can only be input by manually typing them, delimited by space):", accept_nonempty_string=True, - ).split() + ).split() # It also converts the input string(s) into a list + if "https://pas.windows.net/CheckMyAccess/Linux/.default" in scopes: + raise ValueError("SSH Cert scope shall be tested by its dedicated functions") + return scopes def _select_account(app): accounts = app.get_accounts() @@ -183,6 +186,8 @@ def main(): ], option_renderer=lambda f: f.__doc__, header="MSAL Python APIs:") try: func(app) + except ValueError as e: + logging.error("Invalid input: %s", e) except KeyboardInterrupt: # Useful for bailing out a stuck interactive flow print("Aborted") From 7f72e23dbc97030c08273015dbf31eff1c014d1d Mon Sep 17 00:00:00 2001 From: Dickson Mwendia <64727760+Dickson-Mwendia@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:04:42 +0300 Subject: [PATCH 041/262] point to correct quickstart --- README.md | 2 +- docs/index.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9088b60a..3f1b01e8 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Not sure whether this is the SDK you are looking for your app? There are other M Quick links: -| [Getting Started](https://docs.microsoft.com/azure/active-directory/develop/quickstart-v2-python-webapp) | [Docs](https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki) | [Samples](https://aka.ms/aaddevsamplesv2) | [Support](README.md#community-help-and-support) | [Feedback](https://forms.office.com/r/TMjZkDbzjY) | +| [Getting Started](https://learn.microsoft.com/azure/active-directory/develop/web-app-quickstart?pivots=devlang-python| [Docs](https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki) | [Samples](https://aka.ms/aaddevsamplesv2) | [Support](README.md#community-help-and-support) | [Feedback](https://forms.office.com/r/TMjZkDbzjY) | | --- | --- | --- | --- | --- | ## Scenarios supported diff --git a/docs/index.rst b/docs/index.rst index b376f52d..fecc8ee0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ You can find high level conceptual documentations in the project Scenarios ========= -There are many `different application scenarios `_. +There are many `different application scenarios `_. MSAL Python supports some of them. **The following diagram serves as a map. Locate your application scenario on the map.** **If the corresponding icon is clickable, it will bring you to an MSAL Python sample for that scenario.** @@ -24,15 +24,15 @@ MSAL Python supports some of them. .. raw:: html - + Web app + alt="Web app" title="Web app" href="https://learn.microsoft.com/azure/active-directory/develop/web-app-quickstart?pivots=devlang-python> Web app + alt="Web app" title="Web app" href="https://learn.microsoft.com/azure/active-directory/develop/web-app-quickstart?pivots=devlang-python> Desktop App From 9dbe305cab6df45b176f990c849516102c560c0e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 2 Mar 2023 10:48:22 -0800 Subject: [PATCH 042/262] Fix Markdown link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f1b01e8..3fab9682 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Not sure whether this is the SDK you are looking for your app? There are other M Quick links: -| [Getting Started](https://learn.microsoft.com/azure/active-directory/develop/web-app-quickstart?pivots=devlang-python| [Docs](https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki) | [Samples](https://aka.ms/aaddevsamplesv2) | [Support](README.md#community-help-and-support) | [Feedback](https://forms.office.com/r/TMjZkDbzjY) | +| [Getting Started](https://learn.microsoft.com/azure/active-directory/develop/web-app-quickstart?pivots=devlang-python)| [Docs](https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki) | [Samples](https://aka.ms/aaddevsamplesv2) | [Support](README.md#community-help-and-support) | [Feedback](https://forms.office.com/r/TMjZkDbzjY) | | --- | --- | --- | --- | --- | ## Scenarios supported From 56e39dc6a36cb5e61a4d2c313be865b940384659 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 13 Feb 2023 13:34:04 -0800 Subject: [PATCH 043/262] Remove effectiveless in-place clean --- msal/application.py | 4 ++-- msal/token_cache.py | 45 +++++++++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/msal/application.py b/msal/application.py index e024252c..b6b584a7 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1182,7 +1182,7 @@ def _acquire_token_by_cloud_shell(self, scopes, data=None): client_id=self.client_id, scope=response["scope"].split() if "scope" in response else scopes, token_endpoint=self.authority.token_endpoint, - response=response.copy(), + response=response, data=data or {}, authority_type=_AUTHORITY_TYPE_CLOUDSHELL, )) @@ -1399,7 +1399,7 @@ def _process_broker_response(self, response, scopes, data): client_id=self.client_id, scope=response["scope"].split() if "scope" in response else scopes, token_endpoint=self.authority.token_endpoint, - response=response.copy(), + response=response, data=data, _account_id=response["_account_id"], )) diff --git a/msal/token_cache.py b/msal/token_cache.py index 0259522f..4f6d225c 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -103,29 +103,30 @@ def find(self, credential_type, target=None, query=None): def add(self, event, now=None): # type: (dict) -> None - """Handle a token obtaining event, and add tokens into cache. - - Known side effects: This function modifies the input event in place. - """ - def wipe(dictionary, sensitive_fields): # Masks sensitive info - for sensitive in sensitive_fields: - if sensitive in dictionary: - dictionary[sensitive] = "********" - wipe(event.get("data", {}), - ("password", "client_secret", "refresh_token", "assertion")) - try: - return self.__add(event, now=now) - finally: - wipe(event.get("response", {}), ( # These claims were useful during __add() + """Handle a token obtaining event, and add tokens into cache.""" + def make_clean_copy(dictionary, sensitive_fields): # Masks sensitive info + return { + k: "********" if k in sensitive_fields else v + for k, v in dictionary.items() + } + clean_event = dict( + event, + data=make_clean_copy(event.get("data", {}), ( + "password", "client_secret", "refresh_token", "assertion", + )), + response=make_clean_copy(event.get("response", {}), ( "id_token_claims", # Provided by broker - "access_token", "refresh_token", "id_token", "username")) - wipe(event, ["username"]) # Needed for federated ROPC - logger.debug("event=%s", json.dumps( - # We examined and concluded that this log won't have Log Injection risk, - # because the event payload is already in JSON so CR/LF will be escaped. - event, indent=4, sort_keys=True, - default=str, # A workaround when assertion is in bytes in Python 3 - )) + "access_token", "refresh_token", "id_token", "username", + )), + ) + logger.debug("event=%s", json.dumps( + # We examined and concluded that this log won't have Log Injection risk, + # because the event payload is already in JSON so CR/LF will be escaped. + clean_event, + indent=4, sort_keys=True, + default=str, # assertion is in bytes in Python 3 + )) + return self.__add(event, now=now) def __parse_account(self, response, id_token_claims): """Return client_info and home_account_id""" From 4ab91faaa0eade5a3b2aafac283d415eb6e028cb Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 16 Feb 2023 15:05:41 -0800 Subject: [PATCH 044/262] Simplify and easier debugging --- msal/authority.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/msal/authority.py b/msal/authority.py index 407ff7cc..7c82b161 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -174,16 +174,14 @@ def tenant_discovery(tenant_discovery_endpoint, http_client, **kwargs): # Returns Openid Configuration resp = http_client.get(tenant_discovery_endpoint, **kwargs) if resp.status_code == 200: - payload = json.loads(resp.text) # It could raise ValueError - if 'authorization_endpoint' in payload and 'token_endpoint' in payload: - return payload # Happy path - raise ValueError("OIDC Discovery does not provide enough information") + return json.loads(resp.text) # It could raise ValueError if 400 <= resp.status_code < 500: # Nonexist tenant would hit this path # e.g. https://login.microsoftonline.com/nonexist_tenant/v2.0/.well-known/openid-configuration - raise ValueError( - "OIDC Discovery endpoint rejects our request. Error: {}".format( - resp.text # Expose it as-is b/c OIDC defines no error response format + raise ValueError("OIDC Discovery failed on {}. HTTP status: {}, Error: {}".format( + tenant_discovery_endpoint, + resp.status_code, + resp.text, # Expose it as-is b/c OIDC defines no error response format )) # Transient network error would hit this path resp.raise_for_status() From d3212a1097d070cf3f965da4b192d6cab44c5f46 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 5 Apr 2023 21:19:58 -0700 Subject: [PATCH 045/262] Update setup.cfg with documentation URL (#539) * Update setup.cfg * Update setup.cfg Co-authored-by: Ray Luo --------- Co-authored-by: Ray Luo --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 7e543541..013719f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,6 @@ universal=1 [metadata] project_urls = Changelog = https://github.com/AzureAD/microsoft-authentication-library-for-python/releases + Documentation = https://msal-python.readthedocs.io/ + Questions = https://stackoverflow.com/questions/tagged/msal+python + Feature/Bug Tracker = https://github.com/AzureAD/microsoft-authentication-library-for-python/issues From f75b48a33318f7cb75fcc05c1782629423ca9575 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 20 Feb 2023 16:45:50 -0800 Subject: [PATCH 046/262] Fix type introduced in #537 --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index fecc8ee0..8f24a58d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,9 +30,9 @@ MSAL Python supports some of them. usemap="#public-map"> Web app Web app Desktop App From 09d5ff9a71a51c998b83b356047230677825a622 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 20 Feb 2023 08:14:26 -0800 Subject: [PATCH 047/262] Remove a deprecated attribute, scheduled for 1.21+ Actually, it was broken since MSAL 1.17 because we forgot to import warnings. So, that makes the removal today even easier. Nobody needs it anymore. --- msal/authority.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/msal/authority.py b/msal/authority.py index 7c82b161..13aafa7f 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -52,12 +52,6 @@ class Authority(object): """ _domains_without_user_realm_discovery = set([]) - @property - def http_client(self): # Obsolete. We will remove this eventually - warnings.warn( - "authority.http_client might be removed in MSAL Python 1.21+", DeprecationWarning) - return self._http_client - def __init__( self, authority_url, http_client, validate_authority=True, From c46ff0cca7483cd1e4b9e52de27d045beb5da070 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 8 Dec 2022 09:05:12 -0800 Subject: [PATCH 048/262] CIAM end-to-end test cases based on new lab API --- tests/test_e2e.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 48ffe47a..44c1d5f2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -897,6 +897,57 @@ def test_b2c_allows_using_client_id_as_scope(self): ) +class CiamTestCase(LabBasedTestCase): + # Test cases below show you what scenarios need to be covered for CIAM. + # Detail test behaviors have already been implemented in preexisting helpers. + + @classmethod + def setUpClass(cls): + super(CiamTestCase, cls).setUpClass() + cls.user = cls.get_lab_user( + federationProvider="ciam", signinAudience="azureadmyorg", publicClient="No") + # FYI: Only single- or multi-tenant CIAM app can have other-than-OIDC + # delegated permissions on Microsoft Graph. + cls.app_config = cls.get_lab_app_object(cls.user["client_id"]) + + def test_ciam_acquire_token_interactive(self): + self._test_acquire_token_interactive( + authority=self.app_config["authority"], + client_id=self.app_config["appId"], + scope=self.app_config["scopes"], + username=self.user["username"], + lab_name=self.user["lab_name"], + ) + + def test_ciam_acquire_token_for_client(self): + self._test_acquire_token_by_client_secret( + client_id=self.app_config["appId"], + client_secret=self.get_lab_user_secret( + self.app_config["clientSecret"].split("=")[-1]), + authority=self.app_config["authority"], + scope=["{}/.default".format(self.app_config["appId"])], # App permission + ) + + def test_ciam_acquire_token_by_ropc(self): + # Somehow, this would only work after creating a secret for the test app + # and enabling "Allow public client flows". + # Otherwise it would hit AADSTS7000218. + self._test_username_password( + authority=self.app_config["authority"], + client_id=self.app_config["appId"], + username=self.user["username"], + password=self.get_lab_user_secret(self.user["lab_name"]), + scope=self.app_config["scopes"], + ) + + def test_ciam_device_flow(self): + self._test_device_flow( + authority=self.app_config["authority"], + client_id=self.app_config["appId"], + scope=self.app_config["scopes"], + ) + + class WorldWideRegionalEndpointTestCase(LabBasedTestCase): region = "westus" timeout = 2 # Short timeout makes this test case responsive on non-VM From 9850cf4582179eee82183ae01c9ca9d8de9a2b59 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 1 Mar 2023 10:20:19 -0800 Subject: [PATCH 049/262] Support https://contoso.ciamlogin.com as authority --- msal/authority.py | 53 +++++++++++++++++++++++++---------------- tests/test_authority.py | 20 ++++++++++++++++ 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/msal/authority.py b/msal/authority.py index 13aafa7f..6eb294f1 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -5,8 +5,6 @@ from urlparse import urlparse import logging -from .exceptions import MsalServiceError - logger = logging.getLogger(__name__) @@ -28,7 +26,9 @@ "b2clogin.cn", "b2clogin.us", "b2clogin.de", + "ciamlogin.com", ] +_CIAM_DOMAIN_SUFFIX = ".ciamlogin.com" class AuthorityBuilder(object): @@ -74,7 +74,8 @@ def __init__( if isinstance(authority_url, AuthorityBuilder): authority_url = str(authority_url) authority, self.instance, tenant = canonicalize(authority_url) - self.is_adfs = tenant.lower() == 'adfs' + is_ciam = self.instance.endswith(_CIAM_DOMAIN_SUFFIX) + self.is_adfs = tenant.lower() == 'adfs' and not is_ciam parts = authority.path.split('/') self._is_b2c = any( self.instance.endswith("." + d) for d in WELL_KNOWN_B2C_HOSTS @@ -103,13 +104,13 @@ def __init__( % authority_url) tenant_discovery_endpoint = payload['tenant_discovery_endpoint'] else: - tenant_discovery_endpoint = ( - 'https://{}:{}{}{}/.well-known/openid-configuration'.format( - self.instance, - 443 if authority.port is None else authority.port, - authority.path, # In B2C scenario, it is "/tenant/policy" - "" if tenant == "adfs" else "/v2.0" # the AAD v2 endpoint - )) + tenant_discovery_endpoint = authority._replace( + path="{prefix}{version}/.well-known/openid-configuration".format( + prefix=tenant if is_ciam and len(authority.path) <= 1 # Path-less CIAM + else authority.path, # In B2C, it is "/tenant/policy" + version="" if self.is_adfs else "/v2.0", + ) + ).geturl() # Keeping original port and query. Query is useful for test. try: openid_config = tenant_discovery( tenant_discovery_endpoint, @@ -144,18 +145,28 @@ def user_realm_discovery(self, username, correlation_id=None, response=None): return {} # This can guide the caller to fall back normal ROPC flow -def canonicalize(authority_url): +def canonicalize(authority_or_auth_endpoint): # Returns (url_parsed_result, hostname_in_lowercase, tenant) - authority = urlparse(authority_url) - parts = authority.path.split("/") - if authority.scheme != "https" or len(parts) < 2 or not parts[1]: - raise ValueError( - "Your given address (%s) should consist of " - "an https url with a minimum of one segment in a path: e.g. " - "https://login.microsoftonline.com/ " - "or https://.b2clogin.com/.onmicrosoft.com/policy" - % authority_url) - return authority, authority.hostname, parts[1] + authority = urlparse(authority_or_auth_endpoint) + if authority.scheme == "https": + parts = authority.path.split("/") + first_part = parts[1] if len(parts) >= 2 and parts[1] else None + if authority.hostname.endswith(_CIAM_DOMAIN_SUFFIX): # CIAM + # Use path in CIAM authority. It will be validated by OIDC Discovery soon + tenant = first_part if first_part else "{}.onmicrosoft.com".format( + # Fallback to sub domain name. This variation may not be advertised + authority.hostname.rsplit(_CIAM_DOMAIN_SUFFIX, 1)[0]) + return authority, authority.hostname, tenant + # AAD + if len(parts) >= 2 and parts[1]: + return authority, authority.hostname, parts[1] + raise ValueError( + "Your given address (%s) should consist of " + "an https url with a minimum of one segment in a path: e.g. " + "https://login.microsoftonline.com/ " + "or https://.ciamlogin.com/ " + "or https://.b2clogin.com/.onmicrosoft.com/policy" + % authority_or_auth_endpoint) def _instance_discovery(url, http_client, instance_discovery_endpoint, **kwargs): resp = http_client.get( diff --git a/tests/test_authority.py b/tests/test_authority.py index ca0bc68f..2ced23f8 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -79,6 +79,26 @@ def test_invalid_host_skipping_validation_can_be_turned_off(self): pass # Those are expected for this unittest case +@patch("msal.authority.tenant_discovery", return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", + }) +class TestCiamAuthority(unittest.TestCase): + http_client = MinimalHttpClient() + + def test_path_less_authority_should_work(self, oidc_discovery): + Authority('https://contoso.ciamlogin.com', self.http_client) + oidc_discovery.assert_called_once_with( + "https://contoso.ciamlogin.com/contoso.onmicrosoft.com/v2.0/.well-known/openid-configuration", + self.http_client) + + def test_authority_with_path_should_be_used_as_is(self, oidc_discovery): + Authority('https://contoso.ciamlogin.com/anything', self.http_client) + oidc_discovery.assert_called_once_with( + "https://contoso.ciamlogin.com/anything/v2.0/.well-known/openid-configuration", + self.http_client) + + class TestAuthorityInternalHelperCanonicalize(unittest.TestCase): def test_canonicalize_tenant_followed_by_extra_paths(self): From 1bb5476fa15d80c44c33915ebe5c8c2b7f7a88d5 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 3 Apr 2023 23:16:44 -0700 Subject: [PATCH 050/262] Bumping version numbers --- msal/application.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index b6b584a7..b3e2c209 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.21.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.22.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" diff --git a/setup.py b/setup.py index 73be693f..721baa6d 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ 'requests>=2.0.0,<3', 'PyJWT[crypto]>=1.0.0,<3', # MSAL does not use jwt.decode(), therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ - 'cryptography>=0.6,<41', + 'cryptography>=0.6,<43', # load_pem_private_key() is available since 0.6 # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 # From f788a3bf83187d77b7179ab44e2330dbcff97593 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 30 Mar 2023 10:12:13 -0700 Subject: [PATCH 051/262] Turns out they changed to a new tag for MSAL. Fix #539 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 013719f3..3ec1c6ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,5 +5,5 @@ universal=1 project_urls = Changelog = https://github.com/AzureAD/microsoft-authentication-library-for-python/releases Documentation = https://msal-python.readthedocs.io/ - Questions = https://stackoverflow.com/questions/tagged/msal+python + Questions = https://stackoverflow.com/questions/tagged/azure-ad-msal+python Feature/Bug Tracker = https://github.com/AzureAD/microsoft-authentication-library-for-python/issues From 2168027c4694b243ddeff08592396557d9848de0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 17 Mar 2023 21:02:33 -0700 Subject: [PATCH 052/262] Clarify that allow_broker is not applicable to ConfidentialClientApplication It is applicable to PublicClientApplication and base class ClientApplication --- msal/application.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msal/application.py b/msal/application.py index b3e2c209..a3295cc2 100644 --- a/msal/application.py +++ b/msal/application.py @@ -444,6 +444,8 @@ def __init__( New in version 1.19.0. :param boolean allow_broker: + This parameter is NOT applicable to :class:`ConfidentialClientApplication`. + A broker is a component installed on your device. Broker implicitly gives your device an identity. By using a broker, your device becomes a factor that can satisfy MFA (Multi-factor authentication). From 79efb89b195fbd5f317be3614f1cfd58c05a261f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 26 Feb 2023 15:29:15 -0800 Subject: [PATCH 053/262] Backport test improvements --- tests/http_client.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/http_client.py b/tests/http_client.py index 4bff9b45..c532b94b 100644 --- a/tests/http_client.py +++ b/tests/http_client.py @@ -10,21 +10,27 @@ def __init__(self, verify=True, proxies=None, timeout=None): self.timeout = timeout def post(self, url, params=None, data=None, headers=None, **kwargs): + assert not kwargs, "Our stack shouldn't leak extra kwargs: %s" % kwargs return MinimalResponse(requests_resp=self.session.post( url, params=params, data=data, headers=headers, timeout=self.timeout)) def get(self, url, params=None, headers=None, **kwargs): + assert not kwargs, "Our stack shouldn't leak extra kwargs: %s" % kwargs return MinimalResponse(requests_resp=self.session.get( url, params=params, headers=headers, timeout=self.timeout)) + def close(self): # Not required, but we use it to avoid a warning in unit test + self.session.close() + class MinimalResponse(object): # Not for production use def __init__(self, requests_resp=None, status_code=None, text=None): self.status_code = status_code or requests_resp.status_code - self.text = text or requests_resp.text + self.text = text if text is not None else requests_resp.text self._raw_resp = requests_resp def raise_for_status(self): - if self._raw_resp: + if self._raw_resp is not None: # Turns out `if requests.response` won't work + # cause it would be True when 200<=status<400 self._raw_resp.raise_for_status() From a07c3ef99d0fec923560eb4994573e2e05bb0394 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 4 Mar 2023 15:35:16 -0800 Subject: [PATCH 054/262] Expand http interface to include response.headers --- oauth2cli/http.py | 5 +++++ tests/http_client.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/oauth2cli/http.py b/oauth2cli/http.py index 12e2dac6..668b98ff 100644 --- a/oauth2cli/http.py +++ b/oauth2cli/http.py @@ -58,6 +58,11 @@ class Response(object): # but a `text` would be more generic, # when downstream packages would potentially access some XML endpoints. + headers = {} # Duplicated headers are expected to be combined into one header + # with its value as a comma-separated string. + # https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.2 + # Popular HTTP libraries model it as a case-insensitive dict. + def raise_for_status(self): """Raise an exception when http response status contains error""" raise NotImplementedError("Your implementation should provide this") diff --git a/tests/http_client.py b/tests/http_client.py index c532b94b..f6f24739 100644 --- a/tests/http_client.py +++ b/tests/http_client.py @@ -25,9 +25,10 @@ def close(self): # Not required, but we use it to avoid a warning in unit test class MinimalResponse(object): # Not for production use - def __init__(self, requests_resp=None, status_code=None, text=None): + def __init__(self, requests_resp=None, status_code=None, text=None, headers=None): self.status_code = status_code or requests_resp.status_code self.text = text if text is not None else requests_resp.text + self.headers = {} if headers is None else headers self._raw_resp = requests_resp def raise_for_status(self): From 21755029cbe5e9f520c84ab5897906603d864416 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 15 Mar 2023 17:31:16 -0700 Subject: [PATCH 055/262] No need for DummyHttpResponse --- tests/test_throttled_http_client.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_throttled_http_client.py b/tests/test_throttled_http_client.py index 93820505..aa20060d 100644 --- a/tests/test_throttled_http_client.py +++ b/tests/test_throttled_http_client.py @@ -11,19 +11,13 @@ logging.basicConfig(level=logging.DEBUG) -class DummyHttpResponse(MinimalResponse): - def __init__(self, headers=None, **kwargs): - self.headers = {} if headers is None else headers - super(DummyHttpResponse, self).__init__(**kwargs) - - class DummyHttpClient(object): def __init__(self, status_code=None, response_headers=None): self._status_code = status_code self._response_headers = response_headers def _build_dummy_response(self): - return DummyHttpResponse( + return MinimalResponse( status_code=self._status_code, headers=self._response_headers, text=random(), # So that we'd know whether a new response is received From 49090cb692707eadf2f2248b3ea5b8d082d1e84c Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 28 Apr 2023 23:56:28 -0700 Subject: [PATCH 056/262] Adjustment for new CIAM partition --- tests/test_e2e.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 44c1d5f2..657e777e 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -925,10 +925,16 @@ def test_ciam_acquire_token_for_client(self): client_secret=self.get_lab_user_secret( self.app_config["clientSecret"].split("=")[-1]), authority=self.app_config["authority"], - scope=["{}/.default".format(self.app_config["appId"])], # App permission + #scope=["{}/.default".format(self.app_config["appId"])], # AADSTS500207: The account type can't be used for the resource you're trying to access. + #scope=["api://{}/.default".format(self.app_config["appId"])], # AADSTS500011: The resource principal named api://ced781e7-bdb0-4c99-855c-d3bacddea88a was not found in the tenant named MSIDLABCIAM2. This can happen if the application has not been installed by the administrator of the tenant or consented to by any user in the tenant. You might have sent your authentication request to the wrong tenant. + scope=self.app_config["scopes"], # It shall ends with "/.default" ) def test_ciam_acquire_token_by_ropc(self): + """CIAM does not officially support ROPC, especially not for external emails. + + We keep this test case for now, because the test data will use a local email. + """ # Somehow, this would only work after creating a secret for the test app # and enabling "Allow public client flows". # Otherwise it would hit AADSTS7000218. From 6b2f3375064ab6d302ee788cdeeb08c30f3c219e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 26 May 2023 11:34:27 -0700 Subject: [PATCH 057/262] Improve logs --- msal/application.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index a3295cc2..659a7409 100644 --- a/msal/application.py +++ b/msal/application.py @@ -65,6 +65,12 @@ def _str2bytes(raw): return raw +def _pii_less_home_account_id(home_account_id): + parts = home_account_id.split(".") # It could contain one or two parts + parts[0] = "********" + return ".".join(parts) + + def _clean_up(result): if isinstance(result, dict): return { @@ -1460,7 +1466,10 @@ def _acquire_token_silent_by_finding_specific_refresh_token( self.token_cache.CredentialType.REFRESH_TOKEN, # target=scopes, # AAD RTs are scope-independent query=query) - logger.debug("Found %d RTs matching %s", len(matches), query) + logger.debug("Found %d RTs matching %s", len(matches), { + k: _pii_less_home_account_id(v) if k == "home_account_id" and v else v + for k, v in query.items() + }) response = None # A distinguishable value to mean cache is empty if not matches: # Then exit early to avoid expensive operations From 1b7db8db56e27e9b57542ae5eb48d348a94e9d60 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 31 May 2023 13:58:32 -0700 Subject: [PATCH 058/262] Add more sections into TOC for the now long doc --- docs/index.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8f24a58d..5c49a7ba 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -MSAL Python documentation +MSAL Python Documentation ========================= .. toctree:: @@ -6,8 +6,11 @@ MSAL Python documentation :caption: Contents: :hidden: - MSAL Documentation - GitHub Repository + index + +.. + Comment: Perhaps because of the theme, only the first level sections will show in TOC, + regardless of maxdepth setting. You can find high level conceptual documentations in the project `README `_. @@ -58,8 +61,8 @@ MSAL Python supports some of them. `_. -API -=== +API Reference +============= The following section is the API Reference of MSAL Python. The API Reference is like a dictionary. You **read this API section when and only when**: @@ -88,8 +91,10 @@ MSAL proposes a clean separation between They are implemented as two separated classes, with different methods for different authentication scenarios. + + PublicClientApplication ------------------------ +======================= .. autoclass:: msal.PublicClientApplication :members: @@ -98,7 +103,7 @@ PublicClientApplication .. automethod:: __init__ ConfidentialClientApplication ------------------------------ +============================= .. autoclass:: msal.ConfidentialClientApplication :members: @@ -107,7 +112,7 @@ ConfidentialClientApplication .. automethod:: __init__ TokenCache ----------- +========== One of the parameters accepted by both `PublicClientApplication` and `ConfidentialClientApplication` From 10c8dd50bbe25bd51371c85f4121b5b396494815 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 2 Jun 2023 14:43:40 -0700 Subject: [PATCH 059/262] Remove many Sphinx warnings --- docs/index.rst | 10 +++++++--- msal/application.py | 31 ++++++++++++++++++++----------- msal/token_cache.py | 1 - 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5c49a7ba..e608fe6b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -91,14 +91,20 @@ MSAL proposes a clean separation between They are implemented as two separated classes, with different methods for different authentication scenarios. +ClientApplication +================= +.. autoclass:: msal.ClientApplication + :members: + :inherited-members: + + .. automethod:: __init__ PublicClientApplication ======================= .. autoclass:: msal.PublicClientApplication :members: - :inherited-members: .. automethod:: __init__ @@ -107,9 +113,7 @@ ConfidentialClientApplication .. autoclass:: msal.ConfidentialClientApplication :members: - :inherited-members: - .. automethod:: __init__ TokenCache ========== diff --git a/msal/application.py b/msal/application.py index 659a7409..48b6575b 100644 --- a/msal/application.py +++ b/msal/application.py @@ -156,6 +156,9 @@ def obtain_token_by_username_password(self, username, password, **kwargs): class ClientApplication(object): + """You do not usually directly use this class. Use its subclasses instead: + :class:`PublicClientApplication` and :class:`ConfidentialClientApplication`. + """ ACQUIRE_TOKEN_SILENT_ID = "84" ACQUIRE_TOKEN_BY_REFRESH_TOKEN = "85" ACQUIRE_TOKEN_BY_USERNAME_PASSWORD_ID = "301" @@ -319,7 +322,7 @@ def __init__( to keep their traffic remain inside that region. As of 2021 May, regional service is only available for - ``acquire_token_for_client()`` sent by any of the following scenarios:: + ``acquire_token_for_client()`` sent by any of the following scenarios: 1. An app powered by a capable MSAL (MSAL Python 1.12+ will be provisioned) @@ -764,9 +767,9 @@ def initiate_auth_code_flow( Can be one of "consumers" or "organizations" or your tenant domain "contoso.com". If included, it will skip the email-based discovery process that user goes through on the sign-in page, leading to a slightly more streamlined user experience. - More information on possible values - `here `_ and - `here `_. + More information on possible values available in + `Auth Code Flow doc `_ and + `domain_hint doc `_. :param int max_age: OPTIONAL. Maximum Authentication Age. @@ -804,7 +807,7 @@ def initiate_auth_code_flow( "...": "...", // Everything else are reserved and internal } - The caller is expected to:: + The caller is expected to: 1. somehow store this content, typically inside the current session, 2. guide the end user (i.e. resource owner) to visit that auth_uri, @@ -868,9 +871,9 @@ def get_authorization_request_url( Can be one of "consumers" or "organizations" or your tenant domain "contoso.com". If included, it will skip the email-based discovery process that user goes through on the sign-in page, leading to a slightly more streamlined user experience. - More information on possible values - `here `_ and - `here `_. + More information on possible values available in + `Auth Code Flow doc `_ and + `domain_hint doc `_. :param claims_challenge: The claims_challenge parameter requests specific claims requested by the resource provider in the form of a claims_challenge directive in the www-authenticate header to be @@ -1682,6 +1685,9 @@ class PublicClientApplication(ClientApplication): # browser app or mobile app CONSOLE_WINDOW_HANDLE = object() def __init__(self, client_id, client_credential=None, **kwargs): + """Same as :func:`ClientApplication.__init__`, + except that ``client_credential`` parameter shall remain ``None``. + """ if client_credential is not None: raise ValueError("Public Client should not possess credentials") super(PublicClientApplication, self).__init__( @@ -1722,9 +1728,9 @@ def acquire_token_interactive( Can be one of "consumers" or "organizations" or your tenant domain "contoso.com". If included, it will skip the email-based discovery process that user goes through on the sign-in page, leading to a slightly more streamlined user experience. - More information on possible values - `here `_ and - `here `_. + More information on possible values available in + `Auth Code Flow doc `_ and + `domain_hint doc `_. :param claims_challenge: The claims_challenge parameter requests specific claims requested by the resource provider @@ -1994,6 +2000,9 @@ def acquire_token_by_device_flow(self, flow, claims_challenge=None, **kwargs): class ConfidentialClientApplication(ClientApplication): # server-side web app + """Same as :func:`ClientApplication.__init__`, + except that ``allow_broker`` parameter shall remain ``None``. + """ def acquire_token_for_client(self, scopes, claims_challenge=None, **kwargs): """Acquires token for the current confidential client, not for an end user. diff --git a/msal/token_cache.py b/msal/token_cache.py index 4f6d225c..49262069 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -102,7 +102,6 @@ def find(self, credential_type, target=None, query=None): ] def add(self, event, now=None): - # type: (dict) -> None """Handle a token obtaining event, and add tokens into cache.""" def make_clean_copy(dictionary, sensitive_fields): # Masks sensitive info return { From 3e3b97a6d8dfc218dd0a7b290f7d08bad0fbe817 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 7 Jun 2023 17:44:59 -0700 Subject: [PATCH 060/262] Github removes Python 2.7 support on 2023-6-19 See also https://github.com/actions/setup-python/issues/672 MSAL downloads from Python 2.7 is less than 0.2% according to https://pypistats.org/packages/msal --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cf56cb2a..9d24904a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest # It switched to 22.04 shortly after 2022-Nov-8 strategy: matrix: - python-version: [2.7, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12-dev"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12-dev"] steps: - uses: actions/checkout@v2 From 2288b7792148892769b8753cb9df22c946fa5c40 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 28 Jun 2023 23:52:47 -0700 Subject: [PATCH 061/262] Remove acquire_token_silent(..., account=None) usage in a backward-compatible way Now acquire_token_for_client()'s cache behavior will have corresponding api id Continue to disallow acquire_token_for_client(..., force_refresh=True) --- msal/application.py | 90 +++++++++++++------ .../confidential_client_certificate_sample.py | 14 +-- sample/confidential_client_secret_sample.py | 14 +-- tests/test_application.py | 31 +++++-- tests/test_e2e.py | 12 ++- 5 files changed, 99 insertions(+), 62 deletions(-) diff --git a/msal/application.py b/msal/application.py index 48b6575b..29e3cb28 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1209,32 +1209,24 @@ def acquire_token_silent( **kwargs): """Acquire an access token for given account, without user interaction. - It is done either by finding a valid access token from cache, - or by finding a valid refresh token from cache and then automatically - use it to redeem a new access token. - + It has same parameters as the :func:`~acquire_token_silent_with_error`. + The difference is the behavior of the return value. This method will combine the cache empty and refresh error into one return value, `None`. If your app does not care about the exact token refresh error during token cache look-up, then this method is easier and recommended. - Internally, this method calls :func:`~acquire_token_silent_with_error`. - - :param claims_challenge: - The claims_challenge parameter requests specific claims requested by the resource provider - in the form of a claims_challenge directive in the www-authenticate header to be - returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. - It is a string of a JSON object which contains lists of claims being requested from these locations. - :return: - A dict containing no "error" key, and typically contains an "access_token" key, if cache lookup succeeded. - None when cache lookup does not yield a token. """ - result = self.acquire_token_silent_with_error( + if not account: + return None # A backward-compatible NO-OP to drop the account=None usage + result = _clean_up(self._acquire_token_silent_with_error( scopes, account, authority=authority, force_refresh=force_refresh, - claims_challenge=claims_challenge, **kwargs) + claims_challenge=claims_challenge, **kwargs)) return result if result and "error" not in result else None def acquire_token_silent_with_error( @@ -1258,9 +1250,10 @@ def acquire_token_silent_with_error( :param list[str] scopes: (Required) Scopes requested to access a protected API (a resource). - :param account: - one of the account object returned by :func:`~get_accounts`, - or use None when you want to find an access token for this client. + :param account: (Required) + One of the account object returned by :func:`~get_accounts`. + Starting from MSAL Python 1.23, + a ``None`` input will become a NO-OP and always return ``None``. :param force_refresh: If True, it will skip Access Token look-up, and try to find a Refresh Token to obtain a new Access Token. @@ -1276,6 +1269,20 @@ def acquire_token_silent_with_error( - None when there is simply no token in the cache. - A dict containing an "error" key, when token refresh failed. """ + if not account: + return None # A backward-compatible NO-OP to drop the account=None usage + return _clean_up(self._acquire_token_silent_with_error( + scopes, account, authority=authority, force_refresh=force_refresh, + claims_challenge=claims_challenge, **kwargs)) + + def _acquire_token_silent_with_error( + self, + scopes, # type: List[str] + account, # type: Optional[Account] + authority=None, # See get_authorization_request_url() + force_refresh=False, # type: Optional[boolean] + claims_challenge=None, + **kwargs): assert isinstance(scopes, list), "Invalid parameter type" self._validate_ssh_cert_input_data(kwargs.get("data", {})) correlation_id = msal.telemetry._get_new_correlation_id() @@ -1335,7 +1342,11 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( force_refresh=False, # type: Optional[boolean] claims_challenge=None, correlation_id=None, + http_exceptions=None, **kwargs): + # This internal method has two calling patterns: + # it accepts a non-empty account to find token for a user, + # and accepts account=None to find a token for the current app. access_token_from_cache = None if not (force_refresh or claims_challenge): # Bypass AT when desired or using claims query={ @@ -1372,6 +1383,10 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( else: refresh_reason = msal.telemetry.FORCE_REFRESH # TODO: It could also mean claims_challenge assert refresh_reason, "It should have been established at this point" + if not http_exceptions: # It can be a tuple of exceptions + # The exact HTTP exceptions are transportation-layer dependent + from requests.exceptions import RequestException # Lazy load + http_exceptions = (RequestException,) try: data = kwargs.get("data", {}) if account and account.get("authority_type") == _AUTHORITY_TYPE_CLOUDSHELL: @@ -1391,14 +1406,19 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( if response: # The broker provided a decisive outcome, so we use it return self._process_broker_response(response, scopes, data) - result = _clean_up(self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( - authority, self._decorate_scope(scopes), account, - refresh_reason=refresh_reason, claims_challenge=claims_challenge, - correlation_id=correlation_id, - **kwargs)) + if account: + result = self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( + authority, self._decorate_scope(scopes), account, + refresh_reason=refresh_reason, claims_challenge=claims_challenge, + correlation_id=correlation_id, + **kwargs) + else: # The caller is acquire_token_for_client() + result = self._acquire_token_for_client( + scopes, refresh_reason, claims_challenge=claims_challenge, + **kwargs) if (result and "error" not in result) or (not access_token_from_cache): return result - except: # The exact HTTP exception is transportation-layer dependent + except http_exceptions: # Typically network error. Potential AAD outage? if not access_token_from_cache: # It means there is no fall back option raise # We choose to bubble up the exception @@ -2007,6 +2027,9 @@ class ConfidentialClientApplication(ClientApplication): # server-side web app def acquire_token_for_client(self, scopes, claims_challenge=None, **kwargs): """Acquires token for the current confidential client, not for an end user. + Since MSAL Python 1.23, it will automatically look for token from cache, + and only send request to Identity Provider when cache misses. + :param list[str] scopes: (Required) Scopes requested to access a protected API (a resource). :param claims_challenge: @@ -2020,7 +2043,20 @@ def acquire_token_for_client(self, scopes, claims_challenge=None, **kwargs): - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". """ - # TBD: force_refresh behavior + if kwargs.get("force_refresh"): + raise ValueError( # We choose to disallow force_refresh + "Historically, this method does not support force_refresh behavior. " + ) + return _clean_up(self._acquire_token_silent_with_error( + scopes, None, claims_challenge=claims_challenge, **kwargs)) + + def _acquire_token_for_client( + self, + scopes, + refresh_reason, + claims_challenge=None, + **kwargs + ): if self.authority.tenant.lower() in ["common", "organizations"]: warnings.warn( "Using /common or /organizations authority " @@ -2028,16 +2064,16 @@ def acquire_token_for_client(self, scopes, claims_challenge=None, **kwargs): "Please use a specific tenant instead.", DeprecationWarning) self._validate_ssh_cert_input_data(kwargs.get("data", {})) telemetry_context = self._build_telemetry_context( - self.ACQUIRE_TOKEN_FOR_CLIENT_ID) + self.ACQUIRE_TOKEN_FOR_CLIENT_ID, refresh_reason=refresh_reason) client = self._regional_client or self.client - response = _clean_up(client.obtain_token_for_client( + response = client.obtain_token_for_client( scope=scopes, # This grant flow requires no scope decoration headers=telemetry_context.generate_headers(), data=dict( kwargs.pop("data", {}), claims=_merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge)), - **kwargs)) + **kwargs) telemetry_context.update_telemetry(response) return response diff --git a/sample/confidential_client_certificate_sample.py b/sample/confidential_client_certificate_sample.py index 7e5d8069..6cd22a86 100644 --- a/sample/confidential_client_certificate_sample.py +++ b/sample/confidential_client_certificate_sample.py @@ -51,17 +51,9 @@ # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache ) -# The pattern to acquire a token looks like this. -result = None - -# Firstly, looks up a token from cache -# Since we are looking for token for the current app, NOT for an end user, -# notice we give account parameter as None. -result = app.acquire_token_silent(config["scope"], account=None) - -if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - result = app.acquire_token_for_client(scopes=config["scope"]) +# Since MSAL 1.23, acquire_token_for_client(...) will automatically look up +# a token from cache, and fall back to acquire a fresh token when needed. +result = app.acquire_token_for_client(scopes=config["scope"]) if "access_token" in result: # Calling graph using the access token diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_secret_sample.py index d4c06e20..61fd1db7 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_secret_sample.py @@ -50,17 +50,9 @@ # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache ) -# The pattern to acquire a token looks like this. -result = None - -# Firstly, looks up a token from cache -# Since we are looking for token for the current app, NOT for an end user, -# notice we give account parameter as None. -result = app.acquire_token_silent(config["scope"], account=None) - -if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - result = app.acquire_token_for_client(scopes=config["scope"]) +# Since MSAL 1.23, acquire_token_for_client(...) will automatically look up +# a token from cache, and fall back to acquire a fresh token when needed. +result = app.acquire_token_for_client(scopes=config["scope"]) if "access_token" in result: # Calling graph using the access token diff --git a/tests/test_application.py b/tests/test_application.py index b62f41d5..0d93737e 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -382,8 +382,8 @@ def test_aging_token_and_unavailable_aad_should_return_old_token(self): old_at = "old AT" self.populate_cache(access_token=old_at, expires_in=3599, refresh_in=-1) def mock_post(url, headers=None, *args, **kwargs): - self.assertEqual("4|84,2|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) - return MinimalResponse(status_code=400, text=json.dumps({"error": error})) + self.assertEqual("4|84,4|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) + return MinimalResponse(status_code=400, text=json.dumps({"error": "foo"})) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) self.assertEqual(old_at, result.get("access_token")) @@ -549,12 +549,31 @@ def setUpClass(cls): # Initialization at runtime, not interpret-time authority="https://login.microsoftonline.com/common") def test_acquire_token_for_client(self): - at = "this is an access token" def mock_post(url, headers=None, *args, **kwargs): - self.assertEqual("4|730,0|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) - return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) + self.assertEqual("4|730,2|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) + return MinimalResponse(status_code=200, text=json.dumps({ + "access_token": "AT 1", + "expires_in": 0, + })) result = self.app.acquire_token_for_client(["scope"], post=mock_post) - self.assertEqual(at, result.get("access_token")) + self.assertEqual("AT 1", result.get("access_token"), "Shall get a new token") + + def mock_post(url, headers=None, *args, **kwargs): + self.assertEqual("4|730,3|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) + return MinimalResponse(status_code=200, text=json.dumps({ + "access_token": "AT 2", + "expires_in": 3600, + "refresh_in": -100, # A hack to make sure it will attempt refresh + })) + result = self.app.acquire_token_for_client(["scope"], post=mock_post) + self.assertEqual("AT 2", result.get("access_token"), "Shall get a new token") + + def mock_post(url, headers=None, *args, **kwargs): + # 1/0 # TODO: Make sure this was called + self.assertEqual("4|730,4|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) + return MinimalResponse(status_code=400, text=json.dumps({"error": "foo"})) + result = self.app.acquire_token_for_client(["scope"], post=mock_post) + self.assertEqual("AT 2", result.get("access_token"), "Shall get aging token") def test_acquire_token_on_behalf_of(self): at = "this is an access token" diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 657e777e..d1fc50dd 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -146,17 +146,15 @@ def assertCacheWorksForApp(self, result_from_wire, scope): json.dumps(self.app.token_cache._cache, indent=4), json.dumps(result_from_wire.get("id_token_claims"), indent=4), ) - # Going to test acquire_token_silent(...) to locate an AT from cache - result_from_cache = self.app.acquire_token_silent(scope, account=None) + self.assertIsNone( + self.app.acquire_token_silent(scope, account=None), + "acquire_token_silent(..., account=None) shall always return None") + # Going to test acquire_token_for_client(...) to locate an AT from cache + result_from_cache = self.app.acquire_token_for_client(scope) self.assertIsNotNone(result_from_cache) self.assertEqual( result_from_wire['access_token'], result_from_cache['access_token'], "We should get a cached AT") - self.app.acquire_token_silent( - # Result will typically be None, because client credential grant returns no RT. - # But we care more on this call should succeed without exception. - scope, account=None, - force_refresh=True) # Mimic the AT already expires @classmethod def _build_app(cls, From 44c3bfbc6edaf7927c4372f6a85b6dd3ef3ea50c Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 11 Jul 2023 17:58:19 -0700 Subject: [PATCH 062/262] Bumping up version numbers --- msal/application.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index 29e3cb28..16fbac28 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.22.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.23.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" diff --git a/setup.py b/setup.py index 721baa6d..f7a2a4a1 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ 'requests>=2.0.0,<3', 'PyJWT[crypto]>=1.0.0,<3', # MSAL does not use jwt.decode(), therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ - 'cryptography>=0.6,<43', + 'cryptography>=0.6,<44', # load_pem_private_key() is available since 0.6 # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 # From df7b6c52988c65672c350ad5a6017fbd05b5cbbc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 14 Jun 2023 02:07:34 -0700 Subject: [PATCH 063/262] msaltest.py switches from confusing pprint to json --- tests/msaltest.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/msaltest.py b/tests/msaltest.py index b1556106..fec57419 100644 --- a/tests/msaltest.py +++ b/tests/msaltest.py @@ -1,9 +1,12 @@ -import getpass, logging, pprint, sys, msal +import getpass, json, logging, sys, msal AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" +def print_json(blob): + print(json.dumps(blob, indent=2)) + def _input_boolean(message): return input( "{} (N/n/F/f or empty means False, otherwise it is True): ".format(message) @@ -62,7 +65,7 @@ def acquire_token_silent(app): """acquire_token_silent() - with an account already signed into MSAL Python.""" account = _select_account(app) if account: - pprint.pprint(app.acquire_token_silent( + print_json(app.acquire_token_silent( _input_scopes(), account=account, force_refresh=_input_boolean("Bypass MSAL Python's token cache?"), @@ -99,11 +102,11 @@ def _acquire_token_interactive(app, scopes, data=None): def acquire_token_interactive(app): """acquire_token_interactive() - User will be prompted if app opts to do select_account.""" - pprint.pprint(_acquire_token_interactive(app, _input_scopes())) + print_json(_acquire_token_interactive(app, _input_scopes())) def acquire_token_by_username_password(app): """acquire_token_by_username_password() - See constraints here: https://docs.microsoft.com/en-us/azure/active-directory/develop/msal-authentication-flows#constraints-for-ropc""" - pprint.pprint(app.acquire_token_by_username_password( + print_json(app.acquire_token_by_username_password( _input("username: "), getpass.getpass("password: "), scopes=_input_scopes())) _JWK1 = """{"kty":"RSA", "n":"2tNr73xwcj6lH7bqRZrFzgSLj7OeLfbn8216uOMDHuaZ6TEUBDN8Uz0ve8jAlKsP9CQFCSVoSNovdE-fs7c15MxEGHjDcNKLWonznximj8pDGZQjVdfK-7mG6P6z-lgVcLuYu5JcWU_PeEqIKg5llOaz-qeQ4LEDS4T1D2qWRGpAra4rJX1-kmrWmX_XIamq30C9EIO0gGuT4rc2hJBWQ-4-FnE1NXmy125wfT3NdotAJGq5lMIfhjfglDbJCwhc8Oe17ORjO3FsB5CLuBRpYmP7Nzn66lRY3Fe11Xz8AEBl3anKFSJcTvlMnFtu3EpD-eiaHfTgRBU7CztGQqVbiQ", "e":"AQAB"}""" @@ -120,14 +123,14 @@ def acquire_ssh_cert_silently(app): data=SSH_CERT_DATA, force_refresh=_input_boolean("Bypass MSAL Python's token cache?"), ) - pprint.pprint(result) + print_json(result) if result and result.get("token_type") != "ssh-cert": logging.error("Unable to acquire an ssh-cert.") def acquire_ssh_cert_interactive(app): """Acquire an SSH Cert interactively - This typically only works with Azure CLI""" result = _acquire_token_interactive(app, SSH_CERT_SCOPE, data=SSH_CERT_DATA) - pprint.pprint(result) + print_json(result) if result.get("token_type") != "ssh-cert": logging.error("Unable to acquire an ssh-cert") @@ -149,7 +152,7 @@ def exit(app): sys.exit() def main(): - print("Welcome to the Msal Python Console Test App, committed at 2022-5-2\n") + print("Welcome to the Msal Python {} Tester\n".format(msal.__version__)) chosen_app = _select_options([ {"client_id": AZURE_CLI, "name": "Azure CLI (Correctly configured for MSA-PT)"}, {"client_id": VISUAL_STUDIO, "name": "Visual Studio (Correctly configured for MSA-PT)"}, From cb4aa61f06787bf4b9f5bdf8464c64476e506efb Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 19 Jun 2023 21:59:01 -0700 Subject: [PATCH 064/262] Switch from setup.py to setup.cfg --- setup.cfg | 67 +++++++++++++++++++++++++++++++++++ setup.py | 102 ++---------------------------------------------------- 2 files changed, 69 insertions(+), 100 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3ec1c6ab..a69dbdd7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,76 @@ +# Format https://setuptools.pypa.io/en/latest/userguide/declarative_config.html + [bdist_wheel] universal=1 [metadata] +name = msal +version = attr: msal.__version__ +description = + The Microsoft Authentication Library (MSAL) for Python library + enables your app to access the Microsoft Cloud + by supporting authentication of users with + Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) + using industry standard OAuth2 and OpenID Connect. +long_description = file: README.md +long_description_content_type = text/markdown +license = MIT +author = Microsoft Corporation +author_email = nugetaad@microsoft.com +url = https://github.com/AzureAD/microsoft-authentication-library-for-python +classifiers = + Development Status :: 5 - Production/Stable + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + project_urls = Changelog = https://github.com/AzureAD/microsoft-authentication-library-for-python/releases Documentation = https://msal-python.readthedocs.io/ Questions = https://stackoverflow.com/questions/tagged/azure-ad-msal+python Feature/Bug Tracker = https://github.com/AzureAD/microsoft-authentication-library-for-python/issues + + +[options] +include_package_data = False # We used to ship LICENSE, but our __init__.py already mentions MIT +packages = find: +python_requires = >=2.7 +install_requires = + requests>=2.0.0,<3 + + # MSAL does not use jwt.decode(), + # therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ + PyJWT[crypto]>=1.0.0,<3 + + # load_pem_private_key() is available since 0.6 + # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 + # + # And we will use the cryptography (X+3).0.0 as the upper bound, + # based on their latest deprecation policy + # https://cryptography.io/en/latest/api-stability/#deprecation + cryptography>=0.6,<44 + + mock; python_version<'3.3' + +[options.extras_require] +broker = + # The broker is defined as optional dependency, + # so that downstream apps can opt in. The opt-in is needed, partially because + # most existing MSAL Python apps do not have the redirect_uri needed by broker. + # MSAL Python uses a subset of API from PyMsalRuntime 0.11.2+, + # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) + pymsalruntime>=0.13.2,<0.14; python_version>='3.6' and platform_system=='Windows' + +[options.packages.find] +exclude = + tests diff --git a/setup.py b/setup.py index f7a2a4a1..1f21e1d5 100644 --- a/setup.py +++ b/setup.py @@ -1,101 +1,3 @@ -#!/usr/bin/env python -#------------------------------------------------------------------------------ -# -# Copyright (c) Microsoft Corporation. -# All rights reserved. -# -# This code is licensed under the MIT License. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files(the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions : -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#------------------------------------------------------------------------------ - -from setuptools import setup, find_packages -import re, io - -# setup.py shall not import main package -__version__ = re.search( - r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', # It excludes inline comment too - io.open('msal/application.py', encoding='utf_8_sig').read() - ).group(1) - -long_description = open('README.md').read() - -setup( - name='msal', - version=__version__, - description=' '.join( - """The Microsoft Authentication Library (MSAL) for Python library - enables your app to access the Microsoft Cloud - by supporting authentication of users with - Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) - using industry standard OAuth2 and OpenID Connect.""".split()), - long_description=long_description, - long_description_content_type="text/markdown", - license='MIT', - author='Microsoft Corporation', - author_email='nugetaad@microsoft.com', - url='https://github.com/AzureAD/microsoft-authentication-library-for-python', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - ], - packages=find_packages(exclude=["tests"]), - package_data={'': ['LICENSE']}, # Do not use data_files=[...], - # which would cause the LICENSE being copied to /usr/local, - # and tend to fail because of insufficient permission. - # See https://stackoverflow.com/a/14211600/728675 for more detail - install_requires=[ - 'requests>=2.0.0,<3', - 'PyJWT[crypto]>=1.0.0,<3', # MSAL does not use jwt.decode(), therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ - - 'cryptography>=0.6,<44', - # load_pem_private_key() is available since 0.6 - # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 - # - # And we will use the cryptography (X+3).0.0 as the upper bound, - # based on their latest deprecation policy - # https://cryptography.io/en/latest/api-stability/#deprecation - - "mock;python_version<'3.3'", - ], - extras_require={ # It does not seem to work if being defined inside setup.cfg - "broker": [ - # The broker is defined as optional dependency, - # so that downstream apps can opt in. The opt-in is needed, partially because - # most existing MSAL Python apps do not have the redirect_uri needed by broker. - # MSAL Python uses a subset of API from PyMsalRuntime 0.11.2+, - # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) - "pymsalruntime>=0.13.2,<0.14;python_version>='3.6' and platform_system=='Windows'", - ], - }, -) +from setuptools import setup +setup() From 45a0aee6da46063454cf8c64edd42b4507d868f4 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 20 Jun 2023 21:59:25 -0700 Subject: [PATCH 065/262] Ship release- branch of a non-draft PR to TestPyPI --- .github/workflows/python-package.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9d24904a..95f3d4cb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -65,7 +65,12 @@ jobs: cd: needs: ci - if: github.event_name == 'push' && (startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/main') + if: | + github.event_name == 'push' && + ( + startsWith(github.ref, 'refs/tags') || + startsWith(github.ref, 'refs/heads/release-') + ) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -77,14 +82,16 @@ jobs: run: | python -m pip install build --user python -m build --sdist --wheel --outdir dist/ . - - name: Publish to TestPyPI + - name: | + Publish to TestPyPI when pushing to release-* branch. + You better test with a1, a2, b1, b2 releases first. uses: pypa/gh-action-pypi-publish@v1.4.2 - if: github.ref == 'refs/heads/main' + if: startsWith(github.ref, 'refs/heads/release-') with: user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ - - name: Publish to PyPI + - name: Publish to PyPI when tagged if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@v1.4.2 with: From fea7ea94de7539649bc6603013653deada789de3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 23 Jun 2023 02:15:40 -0700 Subject: [PATCH 066/262] Surface msal telemetry as a long opaque string Remove wam_telemetry, for now --- msal/application.py | 7 ++++++- msal/broker.py | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/msal/application.py b/msal/application.py index 16fbac28..5a750598 100644 --- a/msal/application.py +++ b/msal/application.py @@ -73,6 +73,11 @@ def _pii_less_home_account_id(home_account_id): def _clean_up(result): if isinstance(result, dict): + if "_msalruntime_telemetry" in result or "_msal_python_telemetry" in result: + result["msal_telemetry"] = json.dumps({ # Telemetry as an opaque string + "msalruntime_telemetry": result.get("_msalruntime_telemetry"), + "msal_python_telemetry": result.get("_msal_python_telemetry"), + }, separators=(",", ":")) return { k: result[k] for k in result if k != "refresh_in" # MSAL handled refresh_in, customers need not @@ -966,7 +971,7 @@ def authorize(): # A controller in a web app self._validate_ssh_cert_input_data(kwargs.get("data", {})) telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_BY_AUTHORIZATION_CODE_ID) - response =_clean_up(self.client.obtain_token_by_auth_code_flow( + response = _clean_up(self.client.obtain_token_by_auth_code_flow( auth_code_flow, auth_response, scope=self._decorate_scope(scopes) if scopes else None, diff --git a/msal/broker.py b/msal/broker.py index 8b997c61..ce7a9bde 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -23,8 +23,7 @@ except (ImportError, AttributeError): # AttributeError happens when a prior pymsalruntime uninstallation somehow leaved an empty folder behind # PyMsalRuntime currently supports these Windows versions, listed in this MSFT internal link # https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/2406/files - raise ImportError( # TODO: Remove or adjust this line right before merging this PR - 'You need to install dependency by: pip install "msal[broker]>=1.20,<2"') + raise ImportError('You need to install dependency by: pip install "msal[broker]>=1.20,<2"') # It could throw RuntimeError when running on ancient versions of Windows @@ -84,9 +83,11 @@ def _read_account_by_id(account_id, correlation_id): def _convert_result(result, client_id, expected_token_type=None): # Mimic an on-the-wire response from AAD + telemetry = result.get_telemetry_data() + telemetry.pop("wam_telemetry", None) # In pymsalruntime 0.13, it contains PII "account_id" error = result.get_error() if error: - return _convert_error(error, client_id) + return dict(_convert_error(error, client_id), _msalruntime_telemetry=telemetry) id_token_claims = json.loads(result.get_id_token()) if result.get_id_token() else {} account = result.get_account() assert account, "Account is expected to be always available" @@ -107,7 +108,7 @@ def _convert_result(result, client_id, expected_token_type=None): # Mimic an on granted_scopes = result.get_granted_scopes() # New in pymsalruntime 0.3.x if granted_scopes: return_value["scope"] = " ".join(granted_scopes) # Mimic the on-the-wire data format - return return_value + return dict(return_value, _msalruntime_telemetry=telemetry) def _get_new_correlation_id(): From 03cc43bd2e0baebe5de2b6bad2d9581c6153bee2 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 24 Jul 2023 11:31:08 -0700 Subject: [PATCH 067/262] Use a neutral name to hopefully avoid false alarm --- tests/test_e2e.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d1fc50dd..f2c3d97a 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -465,7 +465,7 @@ def test_device_flow(self): def get_lab_app( env_client_id="LAB_APP_CLIENT_ID", - env_client_secret="LAB_APP_CLIENT_SECRET", + env_name2="LAB_APP_CLIENT_SECRET", # A var name that hopefully avoids false alarm authority="https://login.microsoftonline.com/" "72f988bf-86f1-41af-91ab-2d7cd011db47", # Microsoft tenant ID timeout=None, @@ -477,18 +477,17 @@ def get_lab_app( logger.info( "Reading ENV variables %s and %s for lab app defined at " "https://docs.msidlab.com/accounts/confidentialclient.html", - env_client_id, env_client_secret) - if os.getenv(env_client_id) and os.getenv(env_client_secret): + env_client_id, env_name2) + if os.getenv(env_client_id) and os.getenv(env_name2): # A shortcut mainly for running tests on developer's local development machine # or it could be setup on Travis CI # https://docs.travis-ci.com/user/environment-variables/#defining-variables-in-repository-settings # Data came from here # https://docs.msidlab.com/accounts/confidentialclient.html client_id = os.getenv(env_client_id) - client_secret = os.getenv(env_client_secret) + client_secret = os.getenv(env_name2) else: - logger.info("ENV variables %s and/or %s are not defined. Fall back to MSI.", - env_client_id, env_client_secret) + logger.info("ENV variables are not defined. Fall back to MSI.") # See also https://microsoft.sharepoint-df.com/teams/MSIDLABSExtended/SitePages/Programmatically-accessing-LAB-API's.aspx raise unittest.SkipTest("MSI-based mechanism has not been implemented yet") return msal.ConfidentialClientApplication( From fa2e7006e38c6bbaa4e1c5a86c979a069797fcc9 Mon Sep 17 00:00:00 2001 From: "Benjamin A. Beasley" Date: Wed, 9 Aug 2023 09:17:28 -0400 Subject: [PATCH 068/262] =?UTF-8?q?Fix=20typo=20in=20test=20names=20(warnn?= =?UTF-8?q?ing=20=E2=86=92=20warning)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_application.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_application.py b/tests/test_application.py index 0d93737e..df87a05b 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -627,7 +627,7 @@ def test_get_accounts(self): sys.version_info[0] >= 3 and sys.version_info[1] >= 2, "assertWarns() is only available in Python 3.2+") class TestClientCredentialGrant(unittest.TestCase): - def _test_certain_authority_should_emit_warnning(self, authority): + def _test_certain_authority_should_emit_warning(self, authority): app = ConfidentialClientApplication( "client_id", client_credential="secret", authority=authority) def mock_post(url, headers=None, *args, **kwargs): @@ -636,12 +636,12 @@ def mock_post(url, headers=None, *args, **kwargs): with self.assertWarns(DeprecationWarning): app.acquire_token_for_client(["scope"], post=mock_post) - def test_common_authority_should_emit_warnning(self): - self._test_certain_authority_should_emit_warnning( + def test_common_authority_should_emit_warning(self): + self._test_certain_authority_should_emit_warning( authority="https://login.microsoftonline.com/common") - def test_organizations_authority_should_emit_warnning(self): - self._test_certain_authority_should_emit_warnning( + def test_organizations_authority_should_emit_warning(self): + self._test_certain_authority_should_emit_warning( authority="https://login.microsoftonline.com/organizations") From 46ffcb169885bfb88d29a90731d45922428b36af Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 26 Jun 2023 19:49:34 -0700 Subject: [PATCH 069/262] Explicitly pip cache seems unnecessary --- .github/workflows/python-package.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 95f3d4cb..e7ae1643 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,22 +32,11 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 + # It automatically takes care of pip cache, according to + # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#about-caching-workflow-dependencies with: python-version: ${{ matrix.python-version }} - # Derived from https://github.com/actions/cache/blob/main/examples.md#using-pip-to-get-cache-location - # However, a before-and-after test shows no improvement in this repo, - # possibly because the bottlenect was not in downloading those small python deps. - - name: Get pip cache dir from pip 20.1+ - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-${{ hashFiles('**/setup.py', '**/requirements.txt') }} - - name: Install dependencies run: | python -m pip install --upgrade pip From 3826c6b4acc09f6c5a53e89d3805b9ce894f5771 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 11 Aug 2023 14:56:55 -0700 Subject: [PATCH 070/262] Add enable_pii_log and wire it up with MsalRuntime --- msal/application.py | 10 ++++++++++ msal/broker.py | 3 +++ setup.cfg | 2 +- tests/msaltest.py | 8 ++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/msal/application.py b/msal/application.py index 5a750598..7803702f 100644 --- a/msal/application.py +++ b/msal/application.py @@ -193,6 +193,7 @@ def __init__( http_cache=None, instance_discovery=None, allow_broker=None, + enable_pii_log=None, ): """Create an instance of application. @@ -500,6 +501,13 @@ def __init__( * AAD and MSA accounts (i.e. Non-ADFS, non-B2C) New in version 1.20.0. + + :param boolean enable_pii_log: + When enabled, logs may include PII (Personal Identifiable Information). + This can be useful in troubleshooting broker behaviors. + The default behavior is False. + + New in version 1.24.0. """ self.client_id = client_id self.client_credential = client_credential @@ -576,6 +584,8 @@ def __init__( try: from . import broker # Trigger Broker's initialization self._enable_broker = True + if enable_pii_log: + broker._enable_pii_log() except RuntimeError: logger.exception( "Broker is unavailable on this platform. " diff --git a/msal/broker.py b/msal/broker.py index ce7a9bde..81b14a2a 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -236,3 +236,6 @@ def _signout_silently(client_id, account_id, correlation_id=None): if error: return _convert_error(error, client_id) +def _enable_pii_log(): + pymsalruntime.set_is_pii_enabled(1) # New in PyMsalRuntime 0.13.0 + diff --git a/setup.cfg b/setup.cfg index a69dbdd7..39b8524e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ broker = # The broker is defined as optional dependency, # so that downstream apps can opt in. The opt-in is needed, partially because # most existing MSAL Python apps do not have the redirect_uri needed by broker. - # MSAL Python uses a subset of API from PyMsalRuntime 0.11.2+, + # MSAL Python uses a subset of API from PyMsalRuntime 0.13.0+, # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) pymsalruntime>=0.13.2,<0.14; python_version>='3.6' and platform_system=='Windows' diff --git a/tests/msaltest.py b/tests/msaltest.py index fec57419..21f78bd4 100644 --- a/tests/msaltest.py +++ b/tests/msaltest.py @@ -161,6 +161,9 @@ def main(): option_renderer=lambda a: a["name"], header="Impersonate this app (or you can type in the client_id of your own app)", accept_nonempty_string=True) + allow_broker = _input_boolean("Allow broker?") + enable_debug_log = _input_boolean("Enable MSAL Python's DEBUG log?") + enable_pii_log = _input_boolean("Enable PII in broker's log?") if allow_broker and enable_debug_log else False app = msal.PublicClientApplication( chosen_app["client_id"] if isinstance(chosen_app, dict) else chosen_app, authority=_select_options([ @@ -173,9 +176,10 @@ def main(): header="Input authority (Note that MSA-PT apps would NOT use the /common authority)", accept_nonempty_string=True, ), - allow_broker=_input_boolean("Allow broker? (Azure CLI currently only supports @microsoft.com accounts when enabling broker)"), + allow_broker=allow_broker, + enable_pii_log=enable_pii_log, ) - if _input_boolean("Enable MSAL Python's DEBUG log?"): + if enable_debug_log: logging.basicConfig(level=logging.DEBUG) while True: func = _select_options([ From b2c157daae583a89987143cfbc0095e4b379eb25 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 3 Jul 2023 16:07:41 -0700 Subject: [PATCH 071/262] Guarding against perf regression for acquire_token_for_client() Turns out we do not really need a full-blown Timeable class Refactor to use pytest and pytest-benchmark Calibrate ratios Adjust detection calculation Experimenting different reference workload Add more iterations to quick test cases Tune reference and each test case to be in tenth of second Go with fewer loop in hoping for more stable time Relax threshold to 20% One more run Use 40% threshold Use larger threshold 0.4 * 3 Refactor to potentially use PyPerf Automatically choose the right number and repeat Remove local regression detection, for now --- requirements.txt | 1 + tests/simulator.py | 69 +++++++++++++++++++++++++++++++++++++++++ tests/test_benchmark.py | 20 ++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 tests/simulator.py create mode 100644 tests/test_benchmark.py diff --git a/requirements.txt b/requirements.txt index d078afb9..d1b00aea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ . python-dotenv +pytest-benchmark diff --git a/tests/simulator.py b/tests/simulator.py new file mode 100644 index 00000000..ea82cf38 --- /dev/null +++ b/tests/simulator.py @@ -0,0 +1,69 @@ +"""Simulator(s) that can be used to create MSAL instance +whose token cache is in a certain state, and remains unchanged. +This generic simulator(s) becomes the test subject for different benchmark tools. + +For example, you can install pyperf and then run: + + pyperf timeit -s "from tests.simulator import ClientCredentialGrantSimulator as T; t=T(tokens_per_tenant=1, cache_hit=True)" "t.run()" +""" +import json +import logging +import random +from unittest.mock import patch + +import msal +from tests.http_client import MinimalResponse + + +logger = logging.getLogger(__name__) + + +def _count_access_tokens(app): + return len(app.token_cache._cache[app.token_cache.CredentialType.ACCESS_TOKEN]) + + +class ClientCredentialGrantSimulator(object): + + def __init__(self, number_of_tenants=1, tokens_per_tenant=1, cache_hit=False): + logger.info( + "number_of_tenants=%d, tokens_per_tenant=%d, cache_hit=%s", + number_of_tenants, tokens_per_tenant, cache_hit) + with patch.object(msal.authority, "tenant_discovery", return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", + }) as _: # Otherwise it would fail on OIDC discovery + self.apps = [ # In MSAL Python, each CCA binds to one tenant only + msal.ConfidentialClientApplication( + "client_id", client_credential="foo", + authority="https://login.microsoftonline.com/tenant_%s" % t, + ) for t in range(number_of_tenants) + ] + for app in self.apps: + for i in range(tokens_per_tenant): # Populate token cache + self.run(app=app, scope="scope_{}".format(i)) + assert tokens_per_tenant == _count_access_tokens(app), ( + "Token cache did not populate correctly: {}".format(json.dumps( + app.token_cache._cache, indent=4))) + + if cache_hit: + self.run(app=app)["access_token"] # Populate 1 token to be hit + expected_tokens = tokens_per_tenant + 1 + else: + expected_tokens = tokens_per_tenant + app.token_cache.modify = lambda *args, **kwargs: None # Cache becomes read-only + self.run(app=app)["access_token"] + assert expected_tokens == _count_access_tokens(app), "Cache shall not grow" + + def run(self, app=None, scope=None): + # This implementation shall be as concise as possible + app = app or random.choice(self.apps) + #return app.acquire_token_for_client([scope or "scope"], post=_fake) + return app.acquire_token_for_client( + [scope or "scope"], + post=lambda url, **kwargs: MinimalResponse( # Using an inline lambda is as fast as a standalone function + status_code=200, text='''{ + "access_token": "AT for %s", + "token_type": "bearer" + }''' % kwargs["data"]["scope"], + )) + diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py new file mode 100644 index 00000000..dbc54da9 --- /dev/null +++ b/tests/test_benchmark.py @@ -0,0 +1,20 @@ +from tests.simulator import ClientCredentialGrantSimulator as CcaTester + +# Here come benchmark test cases, powered by pytest-benchmark +# Func names will become diag names. +def test_cca_1_tenant_with_10_tokens_per_tenant_and_cache_hit(benchmark): + tester = CcaTester(tokens_per_tenant=10, cache_hit=True) + benchmark(tester.run) + +def test_cca_many_tenants_with_10_tokens_per_tenant_and_cache_hit(benchmark): + tester = CcaTester(number_of_tenants=1000, tokens_per_tenant=10, cache_hit=True) + benchmark(tester.run) + +def test_cca_1_tenant_with_10_tokens_per_tenant_and_cache_miss(benchmark): + tester = CcaTester(tokens_per_tenant=10, cache_hit=False) + benchmark(tester.run) + +def test_cca_many_tenants_with_10_tokens_per_tenant_and_cache_miss(benchmark): + tester = CcaTester(number_of_tenants=1000, tokens_per_tenant=10, cache_hit=False) + benchmark(tester.run) + From 4a4be7b0a405ca801e2b65e4aef1b44408d643ef Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 4 Jul 2023 16:06:45 -0700 Subject: [PATCH 072/262] Add benchmark action and publish it to gh-pages Experimenting not using GPO Use vanilla git command to publish Do not run benchmark in matrix Skip chatty test case discovery during benchmark --- .github/workflows/python-package.yml | 29 ++++++++++++++++++++++++++-- README.md | 6 +++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e7ae1643..1bac8ebb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,15 +42,40 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with pytest + run: pytest --benchmark-skip - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names #flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide #flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest + + cb: + # Benchmark only after the correctness has been tested by CI, + # and then run benchmark only once (sampling with only one Python version). + needs: ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies run: | - pytest + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Run benchmark + run: pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py + - name: Render benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'pytest' + output-file-path: benchmark.json + fail-on-alert: true + - name: Publish Gibhub Pages + run: git push origin gh-pages cd: needs: ci diff --git a/README.md b/README.md index 3fab9682..9d72fdfe 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Microsoft Authentication Library (MSAL) for Python -| `dev` branch | Reference Docs | # of Downloads per different platforms | # of Downloads per recent MSAL versions | -|---------------|---------------|----------------------------------------|-----------------------------------------| - [![Build status](https://github.com/AzureAD/microsoft-authentication-library-for-python/actions/workflows/python-package.yml/badge.svg?branch=dev)](https://github.com/AzureAD/microsoft-authentication-library-for-python/actions) | [![Documentation Status](https://readthedocs.org/projects/msal-python/badge/?version=latest)](https://msal-python.readthedocs.io/en/latest/?badge=latest) | [![Downloads](https://pepy.tech/badge/msal)](https://pypistats.org/packages/msal) | [![Download monthly](https://pepy.tech/badge/msal/month)](https://pepy.tech/project/msal) +| `dev` branch | Reference Docs | # of Downloads per different platforms | # of Downloads per recent MSAL versions | Benchmark Diagram | +|:------------:|:--------------:|:--------------------------------------:|:---------------------------------------:|:-----------------:| + [![Build status](https://github.com/AzureAD/microsoft-authentication-library-for-python/actions/workflows/python-package.yml/badge.svg?branch=dev)](https://github.com/AzureAD/microsoft-authentication-library-for-python/actions) | [![Documentation Status](https://readthedocs.org/projects/msal-python/badge/?version=latest)](https://msal-python.readthedocs.io/en/latest/?badge=latest) | [![Downloads](https://static.pepy.tech/badge/msal)](https://pypistats.org/packages/msal) | [![Download monthly](https://static.pepy.tech/badge/msal/month)](https://pepy.tech/project/msal) | [📉](https://azuread.github.io/microsoft-authentication-library-for-python/dev/bench/) The Microsoft Authentication Library for Python enables applications to integrate with the [Microsoft identity platform](https://aka.ms/aaddevv2). It allows you to sign in users or apps with Microsoft identities ([Azure AD](https://azure.microsoft.com/services/active-directory/), [Microsoft Accounts](https://account.microsoft.com) and [Azure AD B2C](https://azure.microsoft.com/services/active-directory-b2c/) accounts) and obtain tokens to call Microsoft APIs such as [Microsoft Graph](https://graph.microsoft.io/) or your own APIs registered with the Microsoft identity platform. It is built using industry standard OAuth2 and OpenID Connect protocols From 1dd4a22f280bbc32b3c156228aac4e009abb6c88 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 31 Jul 2023 19:03:23 -0700 Subject: [PATCH 073/262] Automatically check cryptography version --- tests/certificate-with-password.pem | 51 +++++++++++++++++++++++ tests/test_cryptography.py | 63 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 tests/certificate-with-password.pem create mode 100644 tests/test_cryptography.py diff --git a/tests/certificate-with-password.pem b/tests/certificate-with-password.pem new file mode 100644 index 00000000..bca1bb58 --- /dev/null +++ b/tests/certificate-with-password.pem @@ -0,0 +1,51 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIpDOLr9sNuTwCAggA +MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECIWEFMkTyS60BIIEyPvMPGyf1shr +ql0UnZzMWDz/9bqdRIZe5N7F1qYjgZNms/QXVzZOQ2J9YwaLHbwpEv2QigfHXq/3 +nLyTm4HFyg3qpCqWa33A0r/v5B6WtjgYbuPePqpM2UV34CMGylkmhMVUUbs1X6j7 +ezwmipq/paOslokC0RYl16cQD/uLTD0usTDtWoEs3S4gbcGUj/b9Ll2urG9zBYwI +faSWQcwcRgfYk/OZSbv6zNT74dYMAOW3mjsS/ackc/+h2XWFQSVjN7BBXamfCQ3C +qE27Py85PP8Qt7MbHuoj8rMuoaQ3NIi++1RkW6cyFo5n+HBi7YYCP74loG1OCTN8 +H5StIi2aLbls9ZbQJHLM2+J1tBwJqIR/UogITESV++17ZVHfDLk8uaad7i6Kj+u5 +6vbruehnFqo5P80lZRuqHfGf/5v8Hbsve/zL24wdQ+tFDHaC6v+kiz9unnO/+k86 +9gph/WTly4N4wJhdxhoYxJLMdPcWk6AxA7ZsJ/mI9+t8iHdSOZY91FyN3sDlDB4C +yLi8t1WP+VB9KfMjSN0AuULWrwwQ1YGRUsKaS9pxTy8MbXQ1OgXGGHzHKDm6vqyp +Jow9wD8Ql+V7zPsNgBpeRWXzA5VS6nEyIuOolkJnNoC5d69/LtDaBn58TZQ4z1Ja +wGXG6n9BeFrgwgH5X5kGslLDXZ71V/aT0HHoBbiAPWf90teccGJ5nVXv3kMaC/zc +klzNCrQ2koFphQgW1bU/FZ46yd2rvlFo8wbxAwRieldZpkwRcFVKz+cRh4/QsTpl +uPKtPpI0c2jgiSReNXi2kRlkOPg1UVHnapvv2yRUoq4NvzaOegSVJG1oe/XdLS02 +5JoDNajEcIvHQpLZL1ecQSwpge843mW4F5twm8/1MKY8G2CTXjKif77n3WVR2Tvp +RdOm86TbIB4FpbLAqiN11A/8cLNfVmioQkNdULLELfKiOeQZABSMPJMVGVxgODN0 +nP31BMibPayqApLLEFIQbSLhZvWIJ5ircZ5XKPHeuqpnxFoeIwrGQuqHo6gvSp5J +CPv4Pul43y0s3vxRpJAqmXO4aAzsPsrYGJiNckbD43OxRV9ZDDeD1Wrc4u82zQxo +frSy5XhVPsKH1lFZ7l6te4Tkro4vMxRVu+W2acToJI6QZct0xrlp+ScmD/9CEOGU +Wj5SN4Y8YW+UfeYhDAhntzruRx4HjdocbvwYsY9gAkim4P9AdiG6eeRUyU9GCUwd +MrjMt5HUzsTQiIyN9jnv9yWNdYmzgJ2V0ZOwVHaEZhZnkgYoK0O/NXSg0FsPo5LU +qdYncK+BMAGnEl/riaJRmnsIH29jKPjZDOvfo1+0UJM3+zPjYy4985+CH7xWGWnb +NQFBtwiPyWzlDyV3119T4rY+Ad6z90vG4hgvpvue1Qcuaure8NwkUEEh1/d8PN20 +kP4vWhpDeHbO5R5byXlJMNzmgVm3mc2t6mA/ouUcMmUOTvjdYALXqgw9RgOsqkob +DNjEK4VZUu3Vd6AsK+s796KTLgQvZhrcahoer/88j7Nu0PyQGrVN202IfmbjIwer +NNcieLmok6r2k8GvyUYP51hpdkXO5j3BsrtBeq4xn3qxzOtEUL8ITZ/BU20+xJq6 +GoGDjvCSBpzesnQFlvUtEw== +-----END ENCRYPTED PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIUHBH8mppwjLI2dFOQ7haLnd6iRjQwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMDAxMDMxODE2MzlaGA8yMTk5 +MDYwODE4MTYzOVowRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBANMXamdgXR+0B2b6zt2nURcYcwC0YrqjvTH/ofF3 +MjUzZ1uKziPNxxAYrUY0O0zIcZWo9Aqfi10vS5oNya/aDrKoWxVRCsLltAV9dbLJ +65zF7wbVE7ZnZ7Nknop+ytd1t1VNTlpbxgWdT6z/WTn4ydqH7Hlh0Ucu2Q3QGQL3 +G9He0kOMog4Y0myxP2xNGjLoig2kh60KEwtxbudOxVN4rLpqhT/1n/L5s+7rznKc +cB4MRqPJMdycIYhTD2mfp/E9hDWRcVJY+9GlqzyxXFTsDsO1SzGgpMEjdO5mtc6N +A0dd8fZQLt1BHLFJlpsuk5Fk40y7HtT3kYKUcD55Xd0pd6ECAwEAAaNTMFEwHQYD +VR0OBBYEFKG65qd+cChhFLB8y4po+vL3HwxuMB8GA1UdIwQYMBaAFKG65qd+cChh +FLB8y4po+vL3HwxuMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB +AAlEqWO8tGKprQrqfdZDsMyKyne+WYYA7RqRxYzKJ5j1zsPa4f7RKhLYzx6TqWK3 +keBc6aw9aeNYNqTBeDLGHRHRRwdHwwU5HdqowhE0sEQzOIJvs2JK5L+LcoSkTViy +PzidZ0qoAHRluuFw8Ag9sahcQfao6rqJOFY/16KEjDthATGo/4mHRsuAM+xza+2m +GbqJH/iO/q0lsPb3culm8aoJNxULTHrU5YWhuGvRypSYrfdL7RBkzW4VEt5LcRK6 +KcfmfHMrjPl/XxSSvrBmly7nYNH80DGSMRP/lnrQ8OS+hSiDy1KBaCcNhja5Dyzn +K0dXlMGmWrnDMs8m+4cUoIM= +-----END CERTIFICATE----- diff --git a/tests/test_cryptography.py b/tests/test_cryptography.py new file mode 100644 index 00000000..5af5722a --- /dev/null +++ b/tests/test_cryptography.py @@ -0,0 +1,63 @@ +import configparser +import os +import re +from unittest import TestCase +import warnings +import xml.etree.ElementTree as ET + +import requests + +from msal.application import _str2bytes + + +latest_cryptography_version = ET.fromstring( + requests.get("https://pypi.org/rss/project/cryptography/releases.xml").text + ).findall("./channel/item/title")[0].text + + +def get_current_ceiling(): + parser = configparser.ConfigParser() + parser.read("setup.cfg") + for line in parser["options"]["install_requires"].splitlines(): + if line.startswith("cryptography"): + match = re.search(r"<(\d+)", line) + if match: + return int(match.group(1)) + raise RuntimeError("Unable to find cryptography info from setup.cfg") + + +class CryptographyTestCase(TestCase): + + def test_should_be_run_with_latest_version_of_cryptography(self): + import cryptography + self.assertEqual( + cryptography.__version__, latest_cryptography_version, + "We are using cryptography {} but we should test with latest {} instead. " + "Run 'pip install -U cryptography'.".format( + cryptography.__version__, latest_cryptography_version)) + + def test_latest_cryptography_should_support_our_usage_without_warnings(self): + with open(os.path.join( + os.path.dirname(__file__), "certificate-with-password.pem")) as f: + cert = f.read() + with warnings.catch_warnings(record=True) as encountered_warnings: + # The usage was copied from application.py + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.backends import default_backend + unencrypted_private_key = serialization.load_pem_private_key( + _str2bytes(cert), + _str2bytes("password"), + backend=default_backend(), # It was a required param until 2020 + ) + self.assertEqual(0, len(encountered_warnings), + "Did cryptography deprecate the functions that we used?") + + def test_ceiling_should_be_latest_cryptography_version_plus_three(self): + expected_ceiling = int(latest_cryptography_version.split(".")[0]) + 3 + self.assertEqual( + expected_ceiling, get_current_ceiling(), + "Test passed with latest cryptography, so we shall bump ceiling to N+3={}, " + "based on their latest deprecation policy " + "https://cryptography.io/en/latest/api-stability/#deprecation".format( + expected_ceiling)) + From bbde57695be9dcd9350679402e934d39fa147e49 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 21 Aug 2023 21:15:53 -0700 Subject: [PATCH 074/262] Placeholders in some error will use curly brackets This way, it will remain visible even if it is rendered on web. The choice of curly brackets is influenced by URL Template RFC 6570. --- msal/authority.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msal/authority.py b/msal/authority.py index 6eb294f1..ae3ebf74 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -163,9 +163,9 @@ def canonicalize(authority_or_auth_endpoint): raise ValueError( "Your given address (%s) should consist of " "an https url with a minimum of one segment in a path: e.g. " - "https://login.microsoftonline.com/ " - "or https://.ciamlogin.com/ " - "or https://.b2clogin.com/.onmicrosoft.com/policy" + "https://login.microsoftonline.com/{tenant} " + "or https://{tenant_name}.ciamlogin.com/{tenant} " + "or https://{tenant_name}.b2clogin.com/{tenant_name}.onmicrosoft.com/policy" % authority_or_auth_endpoint) def _instance_discovery(url, http_client, instance_discovery_endpoint, **kwargs): From 237bd8aa1ababfde1c79aad2cdd40c98ae776a02 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 24 Aug 2023 09:36:32 -0700 Subject: [PATCH 075/262] Provide guidance on how to DIY the pkcs12-to-pem --- msal/application.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 7803702f..b3afbccf 100644 --- a/msal/application.py +++ b/msal/application.py @@ -206,12 +206,19 @@ def __init__( or an X509 certificate container in this form:: { - "private_key": "...-----BEGIN PRIVATE KEY-----...", + "private_key": "...-----BEGIN PRIVATE KEY-----... in PEM format", "thumbprint": "A1B2C3D4E5F6...", "public_certificate": "...-----BEGIN CERTIFICATE-----... (Optional. See below.)", "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", } + MSAL Python requires a "private_key" in PEM format. + If your cert is in a PKCS12 (.pfx) format, you can also + `convert it to PEM and get the thumbprint `_. + + The thumbprint is available in your app's registration in Azure Portal. + Alternatively, you can `calculate the thumbprint `_. + *Added in version 0.5.0*: public_certificate (optional) is public key certificate which will be sent through 'x5c' JWT header only for From 7819dada86a16d55234add4d0e06d9ee471892bc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 16 Aug 2023 18:45:24 -0700 Subject: [PATCH 076/262] Experimental: More precise regression detection --- .github/workflows/python-package.yml | 6 ++++++ .gitignore | 3 ++- requirements.txt | 11 ++++++++--- tests/test_benchmark.py | 8 ++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1bac8ebb..87125759 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -66,6 +66,12 @@ jobs: run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Setup an updatable cache for Performance Baselines + uses: actions/cache@v3 + with: + path: .perf.baseline + key: ${{ runner.os }}-performance-${{ hashFiles('tests/test_benchmark.py') }} + restore-keys: ${{ runner.os }}-performance- - name: Run benchmark run: pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py - name: Render benchmark result diff --git a/.gitignore b/.gitignore index 18dae08c..36b43713 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,5 @@ docs/_build/ tests/config.json -.env \ No newline at end of file +.env +.perf.baseline diff --git a/requirements.txt b/requirements.txt index d1b00aea..7252b96e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,8 @@ -. -python-dotenv -pytest-benchmark +-e . + +# python-dotenv 1.0+ no longer supports Python 3.7 +python-dotenv>=0.21,<2 + +pytest-benchmark>=4,<5 +perf_baseline>=0.1,<0.2 + diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index dbc54da9..9aaeac05 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,20 +1,28 @@ from tests.simulator import ClientCredentialGrantSimulator as CcaTester +from perf_baseline import Baseline + + +baseline = Baseline(".perf.baseline", threshold=1.5) # Up to 1.5x slower than baseline # Here come benchmark test cases, powered by pytest-benchmark # Func names will become diag names. def test_cca_1_tenant_with_10_tokens_per_tenant_and_cache_hit(benchmark): tester = CcaTester(tokens_per_tenant=10, cache_hit=True) + baseline.set_or_compare(tester.run) benchmark(tester.run) def test_cca_many_tenants_with_10_tokens_per_tenant_and_cache_hit(benchmark): tester = CcaTester(number_of_tenants=1000, tokens_per_tenant=10, cache_hit=True) + baseline.set_or_compare(tester.run) benchmark(tester.run) def test_cca_1_tenant_with_10_tokens_per_tenant_and_cache_miss(benchmark): tester = CcaTester(tokens_per_tenant=10, cache_hit=False) + baseline.set_or_compare(tester.run) benchmark(tester.run) def test_cca_many_tenants_with_10_tokens_per_tenant_and_cache_miss(benchmark): tester = CcaTester(number_of_tenants=1000, tokens_per_tenant=10, cache_hit=False) + baseline.set_or_compare(tester.run) benchmark(tester.run) From 24882498432d209864a7ad09e10d33e6d81a7748 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 29 Aug 2023 13:54:15 -0700 Subject: [PATCH 077/262] Refactor SshCert e2e test to use lab user --- tests/test_e2e.py | 95 ++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index f2c3d97a..12231a2a 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -277,48 +277,6 @@ def _test_acquire_token_interactive( return result # For further testing -class SshCertTestCase(E2eTestCase): - _JWK1 = """{"kty":"RSA", "n":"2tNr73xwcj6lH7bqRZrFzgSLj7OeLfbn8216uOMDHuaZ6TEUBDN8Uz0ve8jAlKsP9CQFCSVoSNovdE-fs7c15MxEGHjDcNKLWonznximj8pDGZQjVdfK-7mG6P6z-lgVcLuYu5JcWU_PeEqIKg5llOaz-qeQ4LEDS4T1D2qWRGpAra4rJX1-kmrWmX_XIamq30C9EIO0gGuT4rc2hJBWQ-4-FnE1NXmy125wfT3NdotAJGq5lMIfhjfglDbJCwhc8Oe17ORjO3FsB5CLuBRpYmP7Nzn66lRY3Fe11Xz8AEBl3anKFSJcTvlMnFtu3EpD-eiaHfTgRBU7CztGQqVbiQ", "e":"AQAB"}""" - _JWK2 = """{"kty":"RSA", "n":"72u07mew8rw-ssw3tUs9clKstGO2lvD7ZNxJU7OPNKz5PGYx3gjkhUmtNah4I4FP0DuF1ogb_qSS5eD86w10Wb1ftjWcoY8zjNO9V3ph-Q2tMQWdDW5kLdeU3-EDzc0HQeou9E0udqmfQoPbuXFQcOkdcbh3eeYejs8sWn3TQprXRwGh_TRYi-CAurXXLxQ8rp-pltUVRIr1B63fXmXhMeCAGwCPEFX9FRRs-YHUszUJl9F9-E0nmdOitiAkKfCC9LhwB9_xKtjmHUM9VaEC9jWOcdvXZutwEoW2XPMOg0Ky-s197F9rfpgHle2gBrXsbvVMvS0D-wXg6vsq6BAHzQ", "e":"AQAB"}""" - DATA1 = {"token_type": "ssh-cert", "key_id": "key1", "req_cnf": _JWK1} - DATA2 = {"token_type": "ssh-cert", "key_id": "key2", "req_cnf": _JWK2} - _SCOPE_USER = ["https://pas.windows.net/CheckMyAccess/Linux/user_impersonation"] - _SCOPE_SP = ["https://pas.windows.net/CheckMyAccess/Linux/.default"] - SCOPE = _SCOPE_SP # Historically there was a separation, at 2021 it is unified - - def test_ssh_cert_for_service_principal(self): - # Any SP can obtain an ssh-cert. Here we use the lab app. - result = get_lab_app().acquire_token_for_client(self.SCOPE, data=self.DATA1) - self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( - result.get("error"), result.get("error_description"))) - self.assertEqual("ssh-cert", result["token_type"]) - - def test_ssh_cert_for_user_should_work_with_any_account(self): - result = self._test_acquire_token_interactive( - client_id="04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI is one - # of the only 2 clients that are PreAuthz to use ssh cert feature - authority="https://login.microsoftonline.com/common", - scope=self.SCOPE, - data=self.DATA1, - username_uri="https://msidlab.com/api/user?usertype=cloud", - prompt="none" if msal.application._is_running_in_cloud_shell() else None, - ) # It already tests reading AT from cache, and using RT to refresh - # acquire_token_silent() would work because we pass in the same key - self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( - result.get("error"), result.get("error_description"))) - self.assertEqual("ssh-cert", result["token_type"]) - logger.debug("%s.cache = %s", - self.id(), json.dumps(self.app.token_cache._cache, indent=4)) - - # refresh_token grant can fetch an ssh-cert bound to a different key - account = self.app.get_accounts()[0] - refreshed_ssh_cert = self.app.acquire_token_silent( - self.SCOPE, account=account, data=self.DATA2) - self.assertIsNotNone(refreshed_ssh_cert) - self.assertEqual(refreshed_ssh_cert["token_type"], "ssh-cert") - self.assertNotEqual(result["access_token"], refreshed_ssh_cert['access_token']) - - @unittest.skipUnless( msal.application._is_running_in_cloud_shell(), "Manually run this test case from inside Cloud Shell") @@ -697,6 +655,59 @@ def _test_acquire_token_by_client_secret( self.assertCacheWorksForApp(result, scope) +class PopWithExternalKeyTestCase(LabBasedTestCase): + def _test_service_principal(self): + # Any SP can obtain an ssh-cert. Here we use the lab app. + result = get_lab_app().acquire_token_for_client(self.SCOPE, data=self.DATA1) + self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( + result.get("error"), result.get("error_description"))) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, result["token_type"]) + + def _test_user_account(self): + lab_user = self.get_lab_user(usertype="cloud") + result = self._test_acquire_token_interactive( + client_id="04b07795-8ddb-461a-bbee-02f9e1bf7b46", # Azure CLI is one + # of the only 2 clients that are PreAuthz to use ssh cert feature + authority="https://login.microsoftonline.com/common", + scope=self.SCOPE, + data=self.DATA1, + username=lab_user["username"], + lab_name=lab_user["lab_name"], + prompt="none" if msal.application._is_running_in_cloud_shell() else None, + ) # It already tests reading AT from cache, and using RT to refresh + # acquire_token_silent() would work because we pass in the same key + self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( + result.get("error"), result.get("error_description"))) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, result["token_type"]) + logger.debug("%s.cache = %s", + self.id(), json.dumps(self.app.token_cache._cache, indent=4)) + + # refresh_token grant can fetch an ssh-cert bound to a different key + account = self.app.get_accounts()[0] + refreshed_ssh_cert = self.app.acquire_token_silent( + self.SCOPE, account=account, data=self.DATA2) + self.assertIsNotNone(refreshed_ssh_cert) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, refreshed_ssh_cert["token_type"]) + self.assertNotEqual(result["access_token"], refreshed_ssh_cert['access_token']) + + +class SshCertTestCase(PopWithExternalKeyTestCase): + EXPECTED_TOKEN_TYPE = "ssh-cert" + _JWK1 = """{"kty":"RSA", "n":"2tNr73xwcj6lH7bqRZrFzgSLj7OeLfbn8216uOMDHuaZ6TEUBDN8Uz0ve8jAlKsP9CQFCSVoSNovdE-fs7c15MxEGHjDcNKLWonznximj8pDGZQjVdfK-7mG6P6z-lgVcLuYu5JcWU_PeEqIKg5llOaz-qeQ4LEDS4T1D2qWRGpAra4rJX1-kmrWmX_XIamq30C9EIO0gGuT4rc2hJBWQ-4-FnE1NXmy125wfT3NdotAJGq5lMIfhjfglDbJCwhc8Oe17ORjO3FsB5CLuBRpYmP7Nzn66lRY3Fe11Xz8AEBl3anKFSJcTvlMnFtu3EpD-eiaHfTgRBU7CztGQqVbiQ", "e":"AQAB"}""" + _JWK2 = """{"kty":"RSA", "n":"72u07mew8rw-ssw3tUs9clKstGO2lvD7ZNxJU7OPNKz5PGYx3gjkhUmtNah4I4FP0DuF1ogb_qSS5eD86w10Wb1ftjWcoY8zjNO9V3ph-Q2tMQWdDW5kLdeU3-EDzc0HQeou9E0udqmfQoPbuXFQcOkdcbh3eeYejs8sWn3TQprXRwGh_TRYi-CAurXXLxQ8rp-pltUVRIr1B63fXmXhMeCAGwCPEFX9FRRs-YHUszUJl9F9-E0nmdOitiAkKfCC9LhwB9_xKtjmHUM9VaEC9jWOcdvXZutwEoW2XPMOg0Ky-s197F9rfpgHle2gBrXsbvVMvS0D-wXg6vsq6BAHzQ", "e":"AQAB"}""" + DATA1 = {"token_type": "ssh-cert", "key_id": "key1", "req_cnf": _JWK1} + DATA2 = {"token_type": "ssh-cert", "key_id": "key2", "req_cnf": _JWK2} + _SCOPE_USER = ["https://pas.windows.net/CheckMyAccess/Linux/user_impersonation"] + _SCOPE_SP = ["https://pas.windows.net/CheckMyAccess/Linux/.default"] + SCOPE = _SCOPE_SP # Historically there was a separation, at 2021 it is unified + + def test_service_principal(self): + self._test_service_principal() + + def test_user_account(self): + self._test_user_account() + + class WorldWideTestCase(LabBasedTestCase): def test_aad_managed_user(self): # Pure cloud From e6d7398920f32f96156bbb274b29da4888b5a7fa Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 30 Aug 2023 13:55:02 -0700 Subject: [PATCH 078/262] E2E test for Azure CLI's connectedk8s AT POP --- tests/test_e2e.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 12231a2a..9deec8f7 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -10,7 +10,7 @@ load_dotenv() # take environment variables from .env. except: pass - +import base64 import logging import os import json @@ -708,6 +708,32 @@ def test_user_account(self): self._test_user_account() +def _data_for_pop(key): + raw_req_cnf = json.dumps({"kid": key, "xms_ksl": "sw"}) + return { # Sampled from Azure CLI's plugin connectedk8s + 'token_type': 'pop', + 'key_id': key, + "req_cnf": base64.urlsafe_b64encode(raw_req_cnf.encode('utf-8')).decode('utf-8').rstrip('='), + # Note: Sending raw_req_cnf without base64 encoding would result in an http 500 error + } # See also https://github.com/Azure/azure-cli-extensions/blob/main/src/connectedk8s/azext_connectedk8s/_clientproxyutils.py#L86-L92 + + +class AtPopWithExternalKeyTestCase(PopWithExternalKeyTestCase): + EXPECTED_TOKEN_TYPE = "pop" + DATA1 = _data_for_pop('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-AAAAAAAA') # Fake key with a certain format and length + DATA2 = _data_for_pop('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB-BBBBBBBB') # Fake key with a certain format and length + SCOPE = [ + '6256c85f-0aad-4d50-b960-e6e9b21efe35/.default', # Azure CLI's connectedk8s plugin uses this + # https://github.com/Azure/azure-cli-extensions/pull/4468/files#diff-a47efa3186c7eb4f1176e07d0b858ead0bf4a58bfd51e448ee3607a5b4ef47f6R116 + ] + + def test_service_principal(self): + self._test_service_principal() + + def test_user_account(self): + self._test_user_account() + + class WorldWideTestCase(LabBasedTestCase): def test_aad_managed_user(self): # Pure cloud From ede22de3505dcad792269abd7fc7b61d57ff9390 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 31 Aug 2023 13:55:20 -0700 Subject: [PATCH 079/262] Add POP test function --- tests/msaltest.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/msaltest.py b/tests/msaltest.py index 21f78bd4..0721e38f 100644 --- a/tests/msaltest.py +++ b/tests/msaltest.py @@ -1,11 +1,11 @@ -import getpass, json, logging, sys, msal +import base64, getpass, json, logging, sys, msal AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" def print_json(blob): - print(json.dumps(blob, indent=2)) + print(json.dumps(blob, indent=2, sort_keys=True)) def _input_boolean(message): return input( @@ -134,6 +134,24 @@ def acquire_ssh_cert_interactive(app): if result.get("token_type") != "ssh-cert": logging.error("Unable to acquire an ssh-cert") +POP_KEY_ID = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-AAAAAAAA' # Fake key with a certain format and length +RAW_REQ_CNF = json.dumps({"kid": POP_KEY_ID, "xms_ksl": "sw"}) +POP_DATA = { # Sampled from Azure CLI's plugin connectedk8s + 'token_type': 'pop', + 'key_id': POP_KEY_ID, + "req_cnf": base64.urlsafe_b64encode(RAW_REQ_CNF.encode('utf-8')).decode('utf-8').rstrip('='), + # Note: Sending RAW_REQ_CNF without base64 encoding would result in an http 500 error +} # See also https://github.com/Azure/azure-cli-extensions/blob/main/src/connectedk8s/azext_connectedk8s/_clientproxyutils.py#L86-L92 + +def acquire_pop_token_interactive(app): + """Acquire a POP token interactively - This typically only works with Azure CLI""" + POP_SCOPE = ['6256c85f-0aad-4d50-b960-e6e9b21efe35/.default'] # KAP 1P Server App Scope, obtained from https://github.com/Azure/azure-cli-extensions/pull/4468/files#diff-a47efa3186c7eb4f1176e07d0b858ead0bf4a58bfd51e448ee3607a5b4ef47f6R116 + result = _acquire_token_interactive(app, POP_SCOPE, data=POP_DATA) + print_json(result) + if result.get("token_type") != "pop": + logging.error("Unable to acquire a pop token") + + def remove_account(app): """remove_account() - Invalidate account and/or token(s) from cache, so that acquire_token_silent() would be reset""" account = _select_account(app) @@ -188,6 +206,7 @@ def main(): acquire_token_by_username_password, acquire_ssh_cert_silently, acquire_ssh_cert_interactive, + acquire_pop_token_interactive, remove_account, exit, ], option_renderer=lambda f: f.__doc__, header="MSAL Python APIs:") From d2c2b42ca3af9a61b577de00307d7e94ff96b727 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 4 Sep 2023 08:15:16 -0700 Subject: [PATCH 080/262] Calls out that each commit triggers a TestPyPI release --- .github/workflows/python-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 87125759..ab544fb1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,6 +85,10 @@ jobs: cd: needs: ci + # Note: github.event.pull_request.draft == false WON'T WORK in "if" statement, + # because the triggered event is a push, not a pull_request. + # This means each commit will trigger a release on TestPyPI. + # Those releases will only succeed when each push has a new version number: a1, a2, a3, etc. if: | github.event_name == 'push' && ( From 66fc6ebdb4c25609bfb918eb90528402f5963e41 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 6 Sep 2023 23:46:25 -0700 Subject: [PATCH 081/262] Bumping version number --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index b3afbccf..2c63d202 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.23.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.24.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From e3963c9b6dc06b7ad3b4f9e60395a1f6d86c7614 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 8 Sep 2023 19:16:02 -0700 Subject: [PATCH 082/262] CLI tester will be shipped with msal library --- tests/msaltest.py => msal/__main__.py | 95 ++++++++++++++------------- 1 file changed, 51 insertions(+), 44 deletions(-) rename tests/msaltest.py => msal/__main__.py (78%) diff --git a/tests/msaltest.py b/msal/__main__.py similarity index 78% rename from tests/msaltest.py rename to msal/__main__.py index 0721e38f..cd40b8b4 100644 --- a/tests/msaltest.py +++ b/msal/__main__.py @@ -1,8 +1,19 @@ +# It is currently shipped inside msal library. +# Pros: It is always available wherever msal is installed. +# Cons: Its 3rd-party dependencies (if any) may become msal's dependency. +"""MSAL Python Tester + +Usage 1: Run it on the fly. + python -m msal + +Usage 2: Build an all-in-one executable file for bug bash. + shiv -e msal.__main__._main -o msaltest-on-os-name.pyz . + Note: We choose to not define a console script to avoid name conflict. +""" import base64, getpass, json, logging, sys, msal - -AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" -VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" +_AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" +_VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" def print_json(blob): print(json.dumps(blob, indent=2, sort_keys=True)) @@ -61,7 +72,7 @@ def _select_account(app): else: print("No account available inside MSAL Python. Use other methods to acquire token first.") -def acquire_token_silent(app): +def _acquire_token_silent(app): """acquire_token_silent() - with an account already signed into MSAL Python.""" account = _select_account(app) if account: @@ -71,7 +82,8 @@ def acquire_token_silent(app): force_refresh=_input_boolean("Bypass MSAL Python's token cache?"), )) -def _acquire_token_interactive(app, scopes, data=None): +def _acquire_token_interactive(app, scopes=None, data=None): + """acquire_token_interactive() - User will be prompted if app opts to do select_account.""" prompt = _select_options([ {"value": None, "description": "Unspecified. Proceed silently with a default account (if any), fallback to prompt."}, {"value": "none", "description": "none. Proceed silently with a default account (if any), or error out."}, @@ -88,78 +100,73 @@ def _acquire_token_interactive(app, scopes, data=None): ) login_hint = raw_login_hint["username"] if isinstance(raw_login_hint, dict) else raw_login_hint result = app.acquire_token_interactive( - scopes, + scopes or _input_scopes(), parent_window_handle=app.CONSOLE_WINDOW_HANDLE, # This test app is a console app enable_msa_passthrough=app.client_id in [ # Apps are expected to set this right - AZURE_CLI, VISUAL_STUDIO, + _AZURE_CLI, _VISUAL_STUDIO, ], # Here this test app mimics the setting for some known MSA-PT apps prompt=prompt, login_hint=login_hint, data=data or {}) if login_hint and "id_token_claims" in result: signed_in_user = result.get("id_token_claims", {}).get("preferred_username") if signed_in_user != login_hint: logging.warning('Signed-in user "%s" does not match login_hint', signed_in_user) + print_json(result) return result -def acquire_token_interactive(app): - """acquire_token_interactive() - User will be prompted if app opts to do select_account.""" - print_json(_acquire_token_interactive(app, _input_scopes())) - -def acquire_token_by_username_password(app): +def _acquire_token_by_username_password(app): """acquire_token_by_username_password() - See constraints here: https://docs.microsoft.com/en-us/azure/active-directory/develop/msal-authentication-flows#constraints-for-ropc""" print_json(app.acquire_token_by_username_password( _input("username: "), getpass.getpass("password: "), scopes=_input_scopes())) _JWK1 = """{"kty":"RSA", "n":"2tNr73xwcj6lH7bqRZrFzgSLj7OeLfbn8216uOMDHuaZ6TEUBDN8Uz0ve8jAlKsP9CQFCSVoSNovdE-fs7c15MxEGHjDcNKLWonznximj8pDGZQjVdfK-7mG6P6z-lgVcLuYu5JcWU_PeEqIKg5llOaz-qeQ4LEDS4T1D2qWRGpAra4rJX1-kmrWmX_XIamq30C9EIO0gGuT4rc2hJBWQ-4-FnE1NXmy125wfT3NdotAJGq5lMIfhjfglDbJCwhc8Oe17ORjO3FsB5CLuBRpYmP7Nzn66lRY3Fe11Xz8AEBl3anKFSJcTvlMnFtu3EpD-eiaHfTgRBU7CztGQqVbiQ", "e":"AQAB"}""" -SSH_CERT_DATA = {"token_type": "ssh-cert", "key_id": "key1", "req_cnf": _JWK1} -SSH_CERT_SCOPE = ["https://pas.windows.net/CheckMyAccess/Linux/.default"] +_SSH_CERT_DATA = {"token_type": "ssh-cert", "key_id": "key1", "req_cnf": _JWK1} +_SSH_CERT_SCOPE = ["https://pas.windows.net/CheckMyAccess/Linux/.default"] -def acquire_ssh_cert_silently(app): +def _acquire_ssh_cert_silently(app): """Acquire an SSH Cert silently- This typically only works with Azure CLI""" account = _select_account(app) if account: result = app.acquire_token_silent( - SSH_CERT_SCOPE, + _SSH_CERT_SCOPE, account, - data=SSH_CERT_DATA, + data=_SSH_CERT_DATA, force_refresh=_input_boolean("Bypass MSAL Python's token cache?"), ) print_json(result) if result and result.get("token_type") != "ssh-cert": logging.error("Unable to acquire an ssh-cert.") -def acquire_ssh_cert_interactive(app): +def _acquire_ssh_cert_interactive(app): """Acquire an SSH Cert interactively - This typically only works with Azure CLI""" - result = _acquire_token_interactive(app, SSH_CERT_SCOPE, data=SSH_CERT_DATA) - print_json(result) + result = _acquire_token_interactive(app, scopes=_SSH_CERT_SCOPE, data=_SSH_CERT_DATA) if result.get("token_type") != "ssh-cert": logging.error("Unable to acquire an ssh-cert") -POP_KEY_ID = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-AAAAAAAA' # Fake key with a certain format and length -RAW_REQ_CNF = json.dumps({"kid": POP_KEY_ID, "xms_ksl": "sw"}) -POP_DATA = { # Sampled from Azure CLI's plugin connectedk8s +_POP_KEY_ID = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-AAAAAAAA' # Fake key with a certain format and length +_RAW_REQ_CNF = json.dumps({"kid": _POP_KEY_ID, "xms_ksl": "sw"}) +_POP_DATA = { # Sampled from Azure CLI's plugin connectedk8s 'token_type': 'pop', - 'key_id': POP_KEY_ID, - "req_cnf": base64.urlsafe_b64encode(RAW_REQ_CNF.encode('utf-8')).decode('utf-8').rstrip('='), - # Note: Sending RAW_REQ_CNF without base64 encoding would result in an http 500 error + 'key_id': _POP_KEY_ID, + "req_cnf": base64.urlsafe_b64encode(_RAW_REQ_CNF.encode('utf-8')).decode('utf-8').rstrip('='), + # Note: Sending _RAW_REQ_CNF without base64 encoding would result in an http 500 error } # See also https://github.com/Azure/azure-cli-extensions/blob/main/src/connectedk8s/azext_connectedk8s/_clientproxyutils.py#L86-L92 -def acquire_pop_token_interactive(app): +def _acquire_pop_token_interactive(app): """Acquire a POP token interactively - This typically only works with Azure CLI""" POP_SCOPE = ['6256c85f-0aad-4d50-b960-e6e9b21efe35/.default'] # KAP 1P Server App Scope, obtained from https://github.com/Azure/azure-cli-extensions/pull/4468/files#diff-a47efa3186c7eb4f1176e07d0b858ead0bf4a58bfd51e448ee3607a5b4ef47f6R116 - result = _acquire_token_interactive(app, POP_SCOPE, data=POP_DATA) + result = _acquire_token_interactive(app, scopes=POP_SCOPE, data=_POP_DATA) print_json(result) if result.get("token_type") != "pop": logging.error("Unable to acquire a pop token") - -def remove_account(app): +def _remove_account(app): """remove_account() - Invalidate account and/or token(s) from cache, so that acquire_token_silent() would be reset""" account = _select_account(app) if account: app.remove_account(account) print('Account "{}" and/or its token(s) are signed out from MSAL Python'.format(account["username"])) -def exit(app): +def _exit(app): """Exit""" bug_link = ( "https://identitydivision.visualstudio.com/Engineering/_queries/query/79b3a352-a775-406f-87cd-a487c382a8ed/" @@ -169,11 +176,11 @@ def exit(app): print("Bye. If you found a bug, please report it here: {}".format(bug_link)) sys.exit() -def main(): - print("Welcome to the Msal Python {} Tester\n".format(msal.__version__)) +def _main(): + print("Welcome to the Msal Python {} Tester (Experimental)\n".format(msal.__version__)) chosen_app = _select_options([ - {"client_id": AZURE_CLI, "name": "Azure CLI (Correctly configured for MSA-PT)"}, - {"client_id": VISUAL_STUDIO, "name": "Visual Studio (Correctly configured for MSA-PT)"}, + {"client_id": _AZURE_CLI, "name": "Azure CLI (Correctly configured for MSA-PT)"}, + {"client_id": _VISUAL_STUDIO, "name": "Visual Studio (Correctly configured for MSA-PT)"}, {"client_id": "95de633a-083e-42f5-b444-a4295d8e9314", "name": "Whiteboard Services (Non MSA-PT app. Accepts AAD & MSA accounts.)"}, ], option_renderer=lambda a: a["name"], @@ -201,14 +208,14 @@ def main(): logging.basicConfig(level=logging.DEBUG) while True: func = _select_options([ - acquire_token_silent, - acquire_token_interactive, - acquire_token_by_username_password, - acquire_ssh_cert_silently, - acquire_ssh_cert_interactive, - acquire_pop_token_interactive, - remove_account, - exit, + _acquire_token_silent, + _acquire_token_interactive, + _acquire_token_by_username_password, + _acquire_ssh_cert_silently, + _acquire_ssh_cert_interactive, + _acquire_pop_token_interactive, + _remove_account, + _exit, ], option_renderer=lambda f: f.__doc__, header="MSAL Python APIs:") try: func(app) @@ -218,5 +225,5 @@ def main(): print("Aborted") if __name__ == "__main__": - main() + _main() From f9ddc9d17fe0f062e28fbe0d295cc4ae755e3025 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 10 Sep 2023 09:55:47 -0700 Subject: [PATCH 083/262] Fix regression on input order for interactive test --- msal/__main__.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/msal/__main__.py b/msal/__main__.py index cd40b8b4..ca8b9a87 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -84,6 +84,7 @@ def _acquire_token_silent(app): def _acquire_token_interactive(app, scopes=None, data=None): """acquire_token_interactive() - User will be prompted if app opts to do select_account.""" + scopes = scopes or _input_scopes() # Let user input scope param before less important prompt and login_hint prompt = _select_options([ {"value": None, "description": "Unspecified. Proceed silently with a default account (if any), fallback to prompt."}, {"value": "none", "description": "none. Proceed silently with a default account (if any), or error out."}, @@ -91,21 +92,23 @@ def _acquire_token_interactive(app, scopes=None, data=None): ], option_renderer=lambda o: o["description"], header="Prompt behavior?")["value"] - raw_login_hint = _select_options( - # login_hint is unnecessary when prompt=select_account, - # but we still let tester input login_hint, just for testing purpose. - [None] + [a["username"] for a in app.get_accounts()], - header="login_hint? (If you have multiple signed-in sessions in browser/broker, and you specify a login_hint to match one of them, you will bypass the account picker.)", - accept_nonempty_string=True, - ) - login_hint = raw_login_hint["username"] if isinstance(raw_login_hint, dict) else raw_login_hint + if prompt == "select_account": + login_hint = None # login_hint is unnecessary when prompt=select_account + else: + raw_login_hint = _select_options( + [None] + [a["username"] for a in app.get_accounts()], + header="login_hint? (If you have multiple signed-in sessions in browser/broker, and you specify a login_hint to match one of them, you will bypass the account picker.)", + accept_nonempty_string=True, + ) + login_hint = raw_login_hint["username"] if isinstance(raw_login_hint, dict) else raw_login_hint result = app.acquire_token_interactive( - scopes or _input_scopes(), + scopes, parent_window_handle=app.CONSOLE_WINDOW_HANDLE, # This test app is a console app enable_msa_passthrough=app.client_id in [ # Apps are expected to set this right _AZURE_CLI, _VISUAL_STUDIO, ], # Here this test app mimics the setting for some known MSA-PT apps - prompt=prompt, login_hint=login_hint, data=data or {}) + prompt=prompt, login_hint=login_hint, data=data or {}, + ) if login_hint and "id_token_claims" in result: signed_in_user = result.get("id_token_claims", {}).get("preferred_username") if signed_in_user != login_hint: From fcd9779720d225a3d831b9c48690a3d2c66b7a91 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 31 Dec 2022 11:35:05 -0800 Subject: [PATCH 084/262] Lazy loading requests for quick start --- oauth2cli/oauth2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oauth2cli/oauth2.py b/oauth2cli/oauth2.py index 6cb31bbb..90f576af 100644 --- a/oauth2cli/oauth2.py +++ b/oauth2cli/oauth2.py @@ -17,8 +17,6 @@ import string import hashlib -import requests - from .authcode import AuthCodeReceiver as _AuthCodeReceiver try: @@ -158,6 +156,8 @@ def __init__( "when http_client is in use") self._http_client = http_client else: + import requests # Lazy loading + self._http_client = requests.Session() self._http_client.verify = True if verify is None else verify self._http_client.proxies = proxies From 9ae3b9563996d5d37492d71ed2e074c97a533076 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 30 Dec 2022 11:43:24 -0800 Subject: [PATCH 085/262] Improve a test case by using dynamic port --- tests/test_authcode.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_authcode.py b/tests/test_authcode.py index 385100fd..57bc33bc 100644 --- a/tests/test_authcode.py +++ b/tests/test_authcode.py @@ -17,10 +17,9 @@ def test_setup_at_a_ephemeral_port_and_teardown(self): self.assertNotEqual(port, receiver.get_port()) def test_no_two_concurrent_receivers_can_listen_on_same_port(self): - port = 12345 # Assuming this port is available - with AuthCodeReceiver(port=port) as receiver: + with AuthCodeReceiver() as receiver: expected_error = OSError if sys.version_info[0] > 2 else socket.error with self.assertRaises(expected_error): - with AuthCodeReceiver(port=port) as receiver2: + with AuthCodeReceiver(port=receiver.get_port()): pass From 3427c2577bb674196a59e58cbf5aa91ad92cba7d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 12 Aug 2023 12:44:10 -0700 Subject: [PATCH 086/262] Escape unsafe query string when rendering to html --- oauth2cli/authcode.py | 29 ++++++++++++++++++++++------- tests/test_authcode.py | 17 +++++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/oauth2cli/authcode.py b/oauth2cli/authcode.py index bcef60b8..d2b14613 100644 --- a/oauth2cli/authcode.py +++ b/oauth2cli/authcode.py @@ -15,10 +15,12 @@ try: # Python 3 from http.server import HTTPServer, BaseHTTPRequestHandler from urllib.parse import urlparse, parse_qs, urlencode + from html import escape except ImportError: # Fall back to Python 2 from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler from urlparse import urlparse, parse_qs from urllib import urlencode + from cgi import escape logger = logging.getLogger(__name__) @@ -77,25 +79,37 @@ def _qs2kv(qs): for k, v in qs.items()} +def _is_html(text): + return text.startswith("<") # Good enough for our purpose + + +def _escape(key_value_pairs): + return {k: escape(v) for k, v in key_value_pairs.items()} + + class _AuthCodeHandler(BaseHTTPRequestHandler): def do_GET(self): # For flexibility, we choose to not check self.path matching redirect_uri #assert self.path.startswith('/THE_PATH_REGISTERED_BY_THE_APP') qs = parse_qs(urlparse(self.path).query) if qs.get('code') or qs.get("error"): # So, it is an auth response - self.server.auth_response = _qs2kv(qs) - logger.debug("Got auth response: %s", self.server.auth_response) + auth_response = _qs2kv(qs) + logger.debug("Got auth response: %s", auth_response) template = (self.server.success_template if "code" in qs else self.server.error_template) - self._send_full_response( - template.safe_substitute(**self.server.auth_response)) + if _is_html(template.template): + safe_data = _escape(auth_response) + else: + safe_data = auth_response + self._send_full_response(template.safe_substitute(**safe_data)) + self.server.auth_response = auth_response # Set it now, after the response is likely sent # NOTE: Don't do self.server.shutdown() here. It'll halt the server. else: self._send_full_response(self.server.welcome_page) def _send_full_response(self, body, is_ok=True): self.send_response(200 if is_ok else 400) - content_type = 'text/html' if body.startswith('<') else 'text/plain' + content_type = 'text/html' if _is_html(body) else 'text/plain' self.send_header('Content-type', content_type) self.end_headers() self.wfile.write(body.encode("utf-8")) @@ -318,6 +332,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): default="https://login.microsoftonline.com/common/oauth2/v2.0/authorize") p.add_argument('client_id', help="The client_id of your application") p.add_argument('--port', type=int, default=0, help="The port in redirect_uri") + p.add_argument('--timeout', type=int, default=60, help="Timeout value, in second") p.add_argument('--host', default="127.0.0.1", help="The host of redirect_uri") p.add_argument('--scope', default=None, help="The scope list") args = parser.parse_args() @@ -331,8 +346,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): auth_uri=flow["auth_uri"], welcome_template= "Sign In, or Abort<tag>foo</tag>", + requests.get("http://localhost:{}?error=foo".format( + receiver.get_port())).text, + "Unsafe data in HTML should be escaped", + ))] + receiver.get_auth_response( # Starts server and hang until timeout + timeout=3, + error_template="$error", + ) + From b2b00eb4c4ca9d78130456dca4c0638a2f27826f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 13 Aug 2023 12:44:31 -0700 Subject: [PATCH 087/262] AuthCodeReceiver checks state early now --- oauth2cli/authcode.py | 27 +++++++++++++++------------ oauth2cli/oauth2.py | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/oauth2cli/authcode.py b/oauth2cli/authcode.py index d2b14613..2c626716 100644 --- a/oauth2cli/authcode.py +++ b/oauth2cli/authcode.py @@ -95,17 +95,22 @@ def do_GET(self): if qs.get('code') or qs.get("error"): # So, it is an auth response auth_response = _qs2kv(qs) logger.debug("Got auth response: %s", auth_response) - template = (self.server.success_template - if "code" in qs else self.server.error_template) - if _is_html(template.template): - safe_data = _escape(auth_response) + if self.server.auth_state and self.server.auth_state != auth_response.get("state"): + # OAuth2 successful and error responses contain state when it was used + # https://www.rfc-editor.org/rfc/rfc6749#section-4.2.2.1 + self._send_full_response("State mismatch") # Possibly an attack else: - safe_data = auth_response - self._send_full_response(template.safe_substitute(**safe_data)) - self.server.auth_response = auth_response # Set it now, after the response is likely sent - # NOTE: Don't do self.server.shutdown() here. It'll halt the server. + template = (self.server.success_template + if "code" in qs else self.server.error_template) + if _is_html(template.template): + safe_data = _escape(auth_response) # Foiling an XSS attack + else: + safe_data = auth_response + self._send_full_response(template.safe_substitute(**safe_data)) + self.server.auth_response = auth_response # Set it now, after the response is likely sent else: self._send_full_response(self.server.welcome_page) + # NOTE: Don't do self.server.shutdown() here. It'll halt the server. def _send_full_response(self, body, is_ok=True): self.send_response(200 if is_ok else 400) @@ -295,16 +300,14 @@ def _get_auth_response(self, result, auth_uri=None, timeout=None, state=None, self._server.timeout = timeout # Otherwise its handle_timeout() won't work self._server.auth_response = {} # Shared with _AuthCodeHandler + self._server.auth_state = state # So handler will check it before sending response while not self._closing: # Otherwise, the handle_request() attempt # would yield noisy ValueError trace # Derived from # https://docs.python.org/2/library/basehttpserver.html#more-examples self._server.handle_request() if self._server.auth_response: - if state and state != self._server.auth_response.get("state"): - logger.debug("State mismatch. Ignoring this noise.") - else: - break + break result.update(self._server.auth_response) # Return via writable result param def close(self): diff --git a/oauth2cli/oauth2.py b/oauth2cli/oauth2.py index 90f576af..01ac78a9 100644 --- a/oauth2cli/oauth2.py +++ b/oauth2cli/oauth2.py @@ -666,7 +666,7 @@ def _obtain_token_by_browser( **(auth_params or {})) auth_response = auth_code_receiver.get_auth_response( auth_uri=flow["auth_uri"], - state=flow["state"], # Optional but we choose to do it upfront + state=flow["state"], # So receiver can check it early timeout=timeout, welcome_template=welcome_template, success_template=success_template, From 4a88b63309d5c3c825e54c7580e267d1b7931d50 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 25 Sep 2023 20:44:57 -0700 Subject: [PATCH 088/262] Bumping version number --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 2c63d202..73a69a35 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.24.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.24.1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 0b014e2941eb76eb193cc8db4acf5f46aedb2209 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 6 Oct 2023 12:54:34 +1100 Subject: [PATCH 089/262] Mark package as supporting Python 3.12 --- .github/workflows/python-package.yml | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ab544fb1..198767d7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest # It switched to 22.04 shortly after 2022-Nov-8 strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12-dev"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 diff --git a/setup.cfg b/setup.cfg index 39b8524e..adf13aba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: MIT License Operating System :: OS Independent From 2e119d98a206342a32cb0ec6fa249b349b45d3fa Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 5 Oct 2023 20:55:30 -0700 Subject: [PATCH 090/262] Remove x-client-cpu --- msal/application.py | 1 - 1 file changed, 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 73a69a35..71b0afcd 100644 --- a/msal/application.py +++ b/msal/application.py @@ -669,7 +669,6 @@ def _build_client(self, client_credential, authority, skip_regional_client=False default_headers = { "x-client-sku": "MSAL.Python", "x-client-ver": __version__, "x-client-os": sys.platform, - "x-client-cpu": "x64" if sys.maxsize > 2 ** 32 else "x86", "x-ms-lib-capability": "retry-after, h429", } if self.app_name: From acd2e357c55728b34d7fd30d4c2ea85aebf15503 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 14 Sep 2023 14:39:57 -0700 Subject: [PATCH 091/262] Resolve warnings node12 deprecation warnings https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/ --- .github/workflows/python-package.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 198767d7..461eb959 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,9 +29,9 @@ jobs: python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 # It automatically takes care of pip cache, according to # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#about-caching-workflow-dependencies with: @@ -57,9 +57,9 @@ jobs: needs: ci runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install dependencies @@ -97,9 +97,9 @@ jobs: ) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Build a package for release From 32d987bd2d5c05a7d06be98a3fb3c2e0ee2f71d5 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 27 Sep 2023 13:15:08 -0700 Subject: [PATCH 092/262] Switch to ReadTheDocs configuration file v2 See also https://blog.readthedocs.com/migrate-configuration-v2/ --- .readthedocs.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..d74e663d --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt From 21359e87789b1770dacbaf327c828d772a8d4c39 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 8 Oct 2023 18:48:03 -0700 Subject: [PATCH 093/262] AuthCodeReceiver supports running inside docker --- oauth2cli/authcode.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/oauth2cli/authcode.py b/oauth2cli/authcode.py index 2c626716..5d465288 100644 --- a/oauth2cli/authcode.py +++ b/oauth2cli/authcode.py @@ -6,6 +6,7 @@ After obtaining an auth code, the web server will automatically shut down. """ import logging +import os import socket import sys from string import Template @@ -38,6 +39,20 @@ def obtain_auth_code(listen_port, auth_uri=None): # Historically only used in t ).get("code") +def _is_inside_docker(): + try: + with open("/proc/1/cgroup") as f: # https://stackoverflow.com/a/20012536/728675 + # Search keyword "/proc/pid/cgroup" in this link for the file format + # https://man7.org/linux/man-pages/man7/cgroups.7.html + for line in f.readlines(): + cgroup_path = line.split(":", 2)[2].strip() + if cgroup_path.strip() != "/": + return True + except IOError: + pass # We are probably not running on Linux + return os.path.exists("/.dockerenv") # Docker on Mac will run this line + + def is_wsl(): # "Official" way of detecting WSL: https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 # Run `uname -a` to get 'release' without python @@ -165,7 +180,7 @@ def __init__(self, port=None, scheduled_actions=None): then the receiver would call that lambda function after waiting the response for 10 seconds. """ - address = "127.0.0.1" # Hardcode, for now, Not sure what to expose, yet. + address = "0.0.0.0" if _is_inside_docker() else "127.0.0.1" # Hardcode # Per RFC 8252 (https://tools.ietf.org/html/rfc8252#section-8.3): # * Clients should listen on the loopback network interface only. # (It is not recommended to use "" shortcut to bind all addr.) @@ -283,13 +298,15 @@ def _get_auth_response(self, result, auth_uri=None, timeout=None, state=None, logger.warning( "Found no browser in current environment. " "If this program is being run inside a container " - "which has access to host network " + "which either (1) has access to host network " "(i.e. started by `docker run --net=host -it ...`), " + "or (2) published port {port} to host network " + "(i.e. started by `docker run -p 127.0.0.1:{port}:{port} -it ...`), " "you can use browser on host to visit the following link. " "Otherwise, this auth attempt would either timeout " "(current timeout setting is {timeout}) " "or be aborted by CTRL+C. Auth URI: {auth_uri}".format( - auth_uri=_uri, timeout=timeout)) + auth_uri=_uri, timeout=timeout, port=self.get_port())) else: # Then it is the auth_uri_callback()'s job to inform the user auth_uri_callback(_uri) From 216e78f1370691d0785e40c629f72772c07d110b Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 12 Oct 2023 00:43:28 -0700 Subject: [PATCH 094/262] Hard code port for testing purpose --- msal/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/msal/__main__.py b/msal/__main__.py index ca8b9a87..48a50f6c 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -107,6 +107,7 @@ def _acquire_token_interactive(app, scopes=None, data=None): enable_msa_passthrough=app.client_id in [ # Apps are expected to set this right _AZURE_CLI, _VISUAL_STUDIO, ], # Here this test app mimics the setting for some known MSA-PT apps + port=1234, # Hard coded for testing. Real app typically uses default value. prompt=prompt, login_hint=login_hint, data=data or {}, ) if login_hint and "id_token_claims" in result: From ffb198859c2f70f23eae49d0a8ec7541fe8025ba Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 2 Oct 2023 08:58:50 -0700 Subject: [PATCH 095/262] Fix a typo in api reference doc --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 71b0afcd..eea14697 100644 --- a/msal/application.py +++ b/msal/application.py @@ -280,7 +280,7 @@ def __init__( :param bool validate_authority: (optional) Turns authority validation on or off. This parameter default to true. - :param TokenCache cache: + :param TokenCache token_cache: Sets the token cache used by this ClientApplication instance. By default, an in-memory cache will be created and used. :param http_client: (optional) From 8bab1dd6fea2aa6beb922eeae6c5c183c685cb4e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 4 Oct 2023 22:19:41 -0700 Subject: [PATCH 096/262] Explain how to use global token cache and app --- .../confidential_client_certificate_sample.py | 49 +++++--- sample/confidential_client_secret_sample.py | 50 +++++--- sample/device_flow_sample.py | 117 ++++++++++-------- sample/interactive_sample.py | 111 +++++++++-------- sample/username_password_sample.py | 82 +++++++----- sample/vault_jwt_sample.py | 63 +++++----- 6 files changed, 269 insertions(+), 203 deletions(-) diff --git a/sample/confidential_client_certificate_sample.py b/sample/confidential_client_certificate_sample.py index 6cd22a86..b388cbd4 100644 --- a/sample/confidential_client_certificate_sample.py +++ b/sample/confidential_client_certificate_sample.py @@ -31,6 +31,7 @@ import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import json import logging +import time import requests import msal @@ -42,27 +43,37 @@ config = json.load(open(sys.argv[1])) -# Create a preferably long-lived app instance which maintains a token cache. -app = msal.ConfidentialClientApplication( +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.ConfidentialClientApplication( config["client_id"], authority=config["authority"], client_credential={"thumbprint": config["thumbprint"], "private_key": open(config['private_key_file']).read()}, - # token_cache=... # Default cache is in memory only. - # You can learn how to use SerializableTokenCache from - # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache ) -# Since MSAL 1.23, acquire_token_for_client(...) will automatically look up -# a token from cache, and fall back to acquire a fresh token when needed. -result = app.acquire_token_for_client(scopes=config["scope"]) - -if "access_token" in result: - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) -else: - print(result.get("error")) - print(result.get("error_description")) - print(result.get("correlation_id")) # You may need this when reporting a bug + +def acquire_and_use_token(): + # Since MSAL 1.23, acquire_token_for_client(...) will automatically look up + # a token from cache, and fall back to acquire a fresh token when needed. + result = global_app.acquire_token_for_client(scopes=config["scope"]) + + if "access_token" in result: + # Calling graph using the access token + graph_data = requests.get( # Use token to call downstream service + config["endpoint"], + headers={'Authorization': 'Bearer ' + result['access_token']},).json() + print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + else: + print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_secret_sample.py index 61fd1db7..3a06cded 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_secret_sample.py @@ -30,6 +30,7 @@ import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import json import logging +import time import requests import msal @@ -41,28 +42,37 @@ config = json.load(open(sys.argv[1])) -# Create a preferably long-lived app instance which maintains a token cache. -app = msal.ConfidentialClientApplication( +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.ConfidentialClientApplication( config["client_id"], authority=config["authority"], client_credential=config["secret"], - # token_cache=... # Default cache is in memory only. - # You can learn how to use SerializableTokenCache from - # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache ) -# Since MSAL 1.23, acquire_token_for_client(...) will automatically look up -# a token from cache, and fall back to acquire a fresh token when needed. -result = app.acquire_token_for_client(scopes=config["scope"]) - -if "access_token" in result: - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) - -else: - print(result.get("error")) - print(result.get("error_description")) - print(result.get("correlation_id")) # You may need this when reporting a bug + +def acquire_and_use_token(): + # Since MSAL 1.23, acquire_token_for_client(...) will automatically look up + # a token from cache, and fall back to acquire a fresh token when needed. + result = global_app.acquire_token_for_client(scopes=config["scope"]) + + if "access_token" in result: + # Calling graph using the access token + graph_data = requests.get( # Use token to call downstream service + config["endpoint"], + headers={'Authorization': 'Bearer ' + result['access_token']},).json() + print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + else: + print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes diff --git a/sample/device_flow_sample.py b/sample/device_flow_sample.py index 48f8e7f4..e894a7a3 100644 --- a/sample/device_flow_sample.py +++ b/sample/device_flow_sample.py @@ -20,6 +20,7 @@ import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import json import logging +import time import requests import msal @@ -31,58 +32,70 @@ config = json.load(open(sys.argv[1])) -# Create a preferably long-lived app instance which maintains a token cache. -app = msal.PublicClientApplication( +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.PublicClientApplication( config["client_id"], authority=config["authority"], - # token_cache=... # Default cache is in memory only. - # You can learn how to use SerializableTokenCache from - # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache ) -# The pattern to acquire a token looks like this. -result = None - -# Note: If your device-flow app does not have any interactive ability, you can -# completely skip the following cache part. But here we demonstrate it anyway. -# We now check the cache to see if we have some end users signed in before. -accounts = app.get_accounts() -if accounts: - logging.info("Account(s) exists in cache, probably with token too. Let's try.") - print("Pick the account you want to use to proceed:") - for a in accounts: - print(a["username"]) - # Assuming the end user chose this one - chosen = accounts[0] - # Now let's try to find a token in cache for this account - result = app.acquire_token_silent(config["scope"], account=chosen) - -if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - - flow = app.initiate_device_flow(scopes=config["scope"]) - if "user_code" not in flow: - raise ValueError( - "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) - - print(flow["message"]) - sys.stdout.flush() # Some terminal needs this to ensure the message is shown - - # Ideally you should wait here, in order to save some unnecessary polling - # input("Press Enter after signing in from another device to proceed, CTRL+C to abort.") - - result = app.acquire_token_by_device_flow(flow) # By default it will block - # You can follow this instruction to shorten the block time - # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow - # or you may even turn off the blocking behavior, - # and then keep calling acquire_token_by_device_flow(flow) in your own customized loop. - -if "access_token" in result: - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) -else: - print(result.get("error")) - print(result.get("error_description")) - print(result.get("correlation_id")) # You may need this when reporting a bug + +def acquire_and_use_token(): + # The pattern to acquire a token looks like this. + result = None + + # Note: If your device-flow app does not have any interactive ability, you can + # completely skip the following cache part. But here we demonstrate it anyway. + # We now check the cache to see if we have some end users signed in before. + accounts = global_app.get_accounts() + if accounts: + logging.info("Account(s) exists in cache, probably with token too. Let's try.") + print("Pick the account you want to use to proceed:") + for a in accounts: + print(a["username"]) + # Assuming the end user chose this one + chosen = accounts[0] + # Now let's try to find a token in cache for this account + result = global_app.acquire_token_silent(config["scope"], account=chosen) + + if not result: + logging.info("No suitable token exists in cache. Let's get a new one from AAD.") + + flow = global_app.initiate_device_flow(scopes=config["scope"]) + if "user_code" not in flow: + raise ValueError( + "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) + + print(flow["message"]) + sys.stdout.flush() # Some terminal needs this to ensure the message is shown + + # Ideally you should wait here, in order to save some unnecessary polling + # input("Press Enter after signing in from another device to proceed, CTRL+C to abort.") + + result = global_app.acquire_token_by_device_flow(flow) # By default it will block + # You can follow this instruction to shorten the block time + # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow + # or you may even turn off the blocking behavior, + # and then keep calling acquire_token_by_device_flow(flow) in your own customized loop. + + if "access_token" in result: + # Calling graph using the access token + graph_data = requests.get( # Use token to call downstream service + config["endpoint"], + headers={'Authorization': 'Bearer ' + result['access_token']},).json() + print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + else: + print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes. + # The first acquire_and_use_token() call will prompt. Others hit the cache. + diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index 98acd29f..6d48ee75 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -20,9 +20,8 @@ python sample.py parameters.json """ - import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] -import json, logging, msal, requests +import json, logging, time, msal, requests # Optional logging # logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script @@ -30,56 +29,68 @@ config = json.load(open(sys.argv[1])) -# Create a preferably long-lived app instance which maintains a token cache. -app = msal.PublicClientApplication( +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.PublicClientApplication( config["client_id"], authority=config["authority"], #allow_broker=True, # If opted in, you will be guided to meet the prerequisites, when applicable # See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition - # token_cache=... # Default cache is in memory only. - # You can learn how to use SerializableTokenCache from - # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache ) -# The pattern to acquire a token looks like this. -result = None - -# Firstly, check the cache to see if this end user has signed in before -accounts = app.get_accounts(username=config.get("username")) -if accounts: - logging.info("Account(s) exists in cache, probably with token too. Let's try.") - print("Account(s) already signed in:") - for a in accounts: - print(a["username"]) - chosen = accounts[0] # Assuming the end user chose this one to proceed - print("Proceed with account: %s" % chosen["username"]) - # Now let's try to find a token in cache for this account - result = app.acquire_token_silent(config["scope"], account=chosen) - -if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - print("A local browser window will be open for you to sign in. CTRL+C to cancel.") - result = app.acquire_token_interactive( # Only works if your app is registered with redirect_uri as http://localhost - config["scope"], - #parent_window_handle=..., # If broker is enabled, you will be guided to provide a window handle - login_hint=config.get("username"), # Optional. - # If you know the username ahead of time, this parameter can pre-fill - # the username (or email address) field of the sign-in page for the user, - # Often, apps use this parameter during reauthentication, - # after already extracting the username from an earlier sign-in - # by using the preferred_username claim from returned id_token_claims. - - #prompt=msal.Prompt.SELECT_ACCOUNT, # Or simply "select_account". Optional. It forces to show account selector page - #prompt=msal.Prompt.CREATE, # Or simply "create". Optional. It brings user to a self-service sign-up flow. - # Prerequisite: https://docs.microsoft.com/en-us/azure/active-directory/external-identities/self-service-sign-up-user-flow - ) - -if "access_token" in result: - # Calling graph using the access token - graph_response = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},) - print("Graph API call result: %s ..." % graph_response.text[:100]) -else: - print(result.get("error")) - print(result.get("error_description")) - print(result.get("correlation_id")) # You may need this when reporting a bug + +def acquire_and_use_token(): + # The pattern to acquire a token looks like this. + result = None + + # Firstly, check the cache to see if this end user has signed in before + accounts = global_app.get_accounts(username=config.get("username")) + if accounts: + logging.info("Account(s) exists in cache, probably with token too. Let's try.") + print("Account(s) already signed in:") + for a in accounts: + print(a["username"]) + chosen = accounts[0] # Assuming the end user chose this one to proceed + print("Proceed with account: %s" % chosen["username"]) + # Now let's try to find a token in cache for this account + result = global_app.acquire_token_silent(config["scope"], account=chosen) + + if not result: + logging.info("No suitable token exists in cache. Let's get a new one from AAD.") + print("A local browser window will be open for you to sign in. CTRL+C to cancel.") + result = global_app.acquire_token_interactive( # Only works if your app is registered with redirect_uri as http://localhost + config["scope"], + #parent_window_handle=..., # If broker is enabled, you will be guided to provide a window handle + login_hint=config.get("username"), # Optional. + # If you know the username ahead of time, this parameter can pre-fill + # the username (or email address) field of the sign-in page for the user, + # Often, apps use this parameter during reauthentication, + # after already extracting the username from an earlier sign-in + # by using the preferred_username claim from returned id_token_claims. + + #prompt=msal.Prompt.SELECT_ACCOUNT, # Or simply "select_account". Optional. It forces to show account selector page + #prompt=msal.Prompt.CREATE, # Or simply "create". Optional. It brings user to a self-service sign-up flow. + # Prerequisite: https://docs.microsoft.com/en-us/azure/active-directory/external-identities/self-service-sign-up-user-flow + ) + + if "access_token" in result: + # Calling graph using the access token + graph_response = requests.get( # Use token to call downstream service + config["endpoint"], + headers={'Authorization': 'Bearer ' + result['access_token']},) + print("Graph API call result: %s ..." % graph_response.text[:100]) + else: + print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes + # The first acquire_and_use_token() call will prompt. Others hit the cache. + diff --git a/sample/username_password_sample.py b/sample/username_password_sample.py index c5b98632..13bf7bed 100644 --- a/sample/username_password_sample.py +++ b/sample/username_password_sample.py @@ -22,6 +22,7 @@ import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import json import logging +import time import requests import msal @@ -33,41 +34,54 @@ config = json.load(open(sys.argv[1])) -# Create a preferably long-lived app instance which maintains a token cache. -app = msal.ClientApplication( +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.PublicClientApplication( config["client_id"], authority=config["authority"], client_credential=config.get("client_secret"), - # token_cache=... # Default cache is in memory only. - # You can learn how to use SerializableTokenCache from - # https://msal-python.readthedocs.io/en/latest/#msal.SerializableTokenCache + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache ) -# The pattern to acquire a token looks like this. -result = None - -# Firstly, check the cache to see if this end user has signed in before -accounts = app.get_accounts(username=config["username"]) -if accounts: - logging.info("Account(s) exists in cache, probably with token too. Let's try.") - result = app.acquire_token_silent(config["scope"], account=accounts[0]) - -if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - # See this page for constraints of Username Password Flow. - # https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki/Username-Password-Authentication - result = app.acquire_token_by_username_password( - config["username"], config["password"], scopes=config["scope"]) - -if "access_token" in result: - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) -else: - print(result.get("error")) - print(result.get("error_description")) - print(result.get("correlation_id")) # You may need this when reporting a bug - if 65001 in result.get("error_codes", []): # Not mean to be coded programatically, but... - # AAD requires user consent for U/P flow - print("Visit this to consent:", app.get_authorization_request_url(config["scope"])) + +def acquire_and_use_token(): + # The pattern to acquire a token looks like this. + result = None + + # Firstly, check the cache to see if this end user has signed in before + accounts = global_app.get_accounts(username=config["username"]) + if accounts: + logging.info("Account(s) exists in cache, probably with token too. Let's try.") + result = global_app.acquire_token_silent(config["scope"], account=accounts[0]) + + if not result: + logging.info("No suitable token exists in cache. Let's get a new one from AAD.") + # See this page for constraints of Username Password Flow. + # https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki/Username-Password-Authentication + result = global_app.acquire_token_by_username_password( + config["username"], config["password"], scopes=config["scope"]) + + if "access_token" in result: + # Calling graph using the access token + graph_data = requests.get( # Use token to call downstream service + config["endpoint"], + headers={'Authorization': 'Bearer ' + result['access_token']},).json() + print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + else: + print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + print(result) + if 65001 in result.get("error_codes", []): # Not mean to be coded programatically, but... + raise RuntimeError( + "AAD requires user consent for U/P flow to succeed. " + "Run acquire_token_interactive() instead.") + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes. + diff --git a/sample/vault_jwt_sample.py b/sample/vault_jwt_sample.py index 131732e1..0a46a9ed 100644 --- a/sample/vault_jwt_sample.py +++ b/sample/vault_jwt_sample.py @@ -103,32 +103,39 @@ def make_vault_jwt(): return full_token -authority = "https://login.microsoftonline.com/%s" % config['tenant'] - -app = msal.ConfidentialClientApplication( - config['client_id'], authority=authority, client_credential={"client_assertion": make_vault_jwt()} - ) - -# The pattern to acquire a token looks like this. -result = None - -# Firstly, looks up a token from cache -# Since we are looking for token for the current app, NOT for an end user, -# notice we give account parameter as None. -result = app.acquire_token_silent(config["scope"], account=None) - -if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - result = app.acquire_token_for_client(scopes=config["scope"]) - -if "access_token" in result: - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) -else: - print(result.get("error")) - print(result.get("error_description")) - print(result.get("correlation_id")) # You may need this when reporting a bug + +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.ConfidentialClientApplication( + config['client_id'], + authority="https://login.microsoftonline.com/%s" % config['tenant'], + client_credential={"client_assertion": make_vault_jwt()}, + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache + ) + + +def acquire_and_use_token(): + # Since MSAL 1.23, acquire_token_for_client(...) will automatically look up + # a token from cache, and fall back to acquire a fresh token when needed. + result = global_app.acquire_token_for_client(scopes=config["scope"]) + + if "access_token" in result: + # Calling graph using the access token + graph_data = requests.get( # Use token to call downstream service + config["endpoint"], + headers={'Authorization': 'Bearer ' + result['access_token']},).json() + print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + else: + print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes From 0ca81d82bc0d96004fa3a673bfb400bbfee93388 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 18 Oct 2023 23:40:55 -0700 Subject: [PATCH 097/262] Expose token_source for observability --- msal/application.py | 44 ++++++++++++++----- .../confidential_client_certificate_sample.py | 1 + sample/confidential_client_secret_sample.py | 1 + sample/device_flow_sample.py | 1 + sample/interactive_sample.py | 1 + sample/username_password_sample.py | 1 + sample/vault_jwt_sample.py | 1 + tests/test_application.py | 17 +++++++ 8 files changed, 56 insertions(+), 11 deletions(-) diff --git a/msal/application.py b/msal/application.py index eea14697..95fa86c1 100644 --- a/msal/application.py +++ b/msal/application.py @@ -176,6 +176,10 @@ class ClientApplication(object): REMOVE_ACCOUNT_ID = "903" ATTEMPT_REGION_DISCOVERY = True # "TryAutoDetect" + _TOKEN_SOURCE = "token_source" + _TOKEN_SOURCE_IDP = "identity_provider" + _TOKEN_SOURCE_CACHE = "cache" + _TOKEN_SOURCE_BROKER = "broker" def __init__( self, client_id, @@ -998,6 +1002,8 @@ def authorize(): # A controller in a web app self._client_capabilities, auth_code_flow.pop("claims_challenge", None))), **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response @@ -1070,6 +1076,8 @@ def acquire_token_by_authorization_code( self._client_capabilities, claims_challenge)), nonce=nonce, **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response @@ -1218,6 +1226,8 @@ def _acquire_token_by_cloud_shell(self, scopes, data=None): data=data or {}, authority_type=_AUTHORITY_TYPE_CLOUDSHELL, )) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_BROKER return response def acquire_token_silent( @@ -1395,6 +1405,7 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( "access_token": entry["secret"], "token_type": entry.get("token_type", "Bearer"), "expires_in": int(expires_in), # OAuth2 specs defines it as int + self._TOKEN_SOURCE: self._TOKEN_SOURCE_CACHE, } if "refresh_on" in entry and int(entry["refresh_on"]) < now: # aging refresh_reason = msal.telemetry.AT_AGING @@ -1437,6 +1448,8 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( result = self._acquire_token_for_client( scopes, refresh_reason, claims_challenge=claims_challenge, **kwargs) + if result and "access_token" in result: + result[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP if (result and "error" not in result) or (not access_token_from_cache): return result except http_exceptions: @@ -1455,6 +1468,7 @@ def _process_broker_response(self, response, scopes, data): data=data, _account_id=response["_account_id"], )) + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_BROKER return _clean_up(response) def _acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( @@ -1611,6 +1625,8 @@ def acquire_token_by_refresh_token(self, refresh_token, scopes, **kwargs): on_updating_rt=False, on_removing_rt=lambda rt_item: None, # No OP **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response @@ -1658,6 +1674,7 @@ def acquire_token_by_username_password( self.ACQUIRE_TOKEN_BY_USERNAME_PASSWORD_ID) headers = telemetry_context.generate_headers() data = dict(kwargs.pop("data", {}), claims=claims) + response = None if not self.authority.is_adfs: user_realm_result = self.authority.user_realm_discovery( username, correlation_id=headers[msal.telemetry.CLIENT_REQUEST_ID]) @@ -1666,13 +1683,14 @@ def acquire_token_by_username_password( user_realm_result, username, password, scopes=scopes, data=data, headers=headers, **kwargs)) - telemetry_context.update_telemetry(response) - return response - response = _clean_up(self.client.obtain_token_by_username_password( + if response is None: # Either ADFS or not federated + response = _clean_up(self.client.obtain_token_by_username_password( username, password, scope=scopes, headers=headers, data=data, **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response @@ -1859,7 +1877,7 @@ def acquire_token_interactive( logger.warning( "Ignoring parameter extra_scopes_to_consent, " "which is not supported by broker") - return self._acquire_token_interactive_via_broker( + response = self._acquire_token_interactive_via_broker( scopes, parent_window_handle, enable_msa_passthrough, @@ -1870,6 +1888,7 @@ def acquire_token_interactive( login_hint=login_hint, max_age=max_age, ) + return self._process_broker_response(response, scopes, data) on_before_launching_ui(ui="browser") telemetry_context = self._build_telemetry_context( @@ -1892,6 +1911,8 @@ def acquire_token_interactive( headers=telemetry_context.generate_headers(), browser_name=_preferred_browser(), **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response @@ -1928,7 +1949,7 @@ def _acquire_token_interactive_via_broker( claims=claims, **data) if response and "error" not in response: - return self._process_broker_response(response, scopes, data) + return response # login_hint undecisive or not exists if prompt == "none" or not prompt: # Must/Can attempt _signin_silently() logger.debug("Calling broker._signin_silently()") @@ -1949,9 +1970,7 @@ def _acquire_token_interactive_via_broker( if is_wrong_account: logger.debug(wrong_account_error_message) if prompt == "none": - return self._process_broker_response( # It is either token or error - response, scopes, data - ) if not is_wrong_account else { + return response if not is_wrong_account else { "error": "broker_error", "error_description": wrong_account_error_message, } @@ -1966,11 +1985,11 @@ def _acquire_token_interactive_via_broker( "_broker_status") in recoverable_errors: pass # It will fall back to the _signin_interactively() else: - return self._process_broker_response(response, scopes, data) + return response logger.debug("Falls back to broker._signin_interactively()") on_before_launching_ui(ui="broker") - response = _signin_interactively( + return _signin_interactively( authority, self.client_id, scopes, None if parent_window_handle is self.CONSOLE_WINDOW_HANDLE else parent_window_handle, @@ -1981,7 +2000,6 @@ def _acquire_token_interactive_via_broker( max_age=max_age, enable_msa_pt=enable_msa_passthrough, **data) - return self._process_broker_response(response, scopes, data) def initiate_device_flow(self, scopes=None, **kwargs): """Initiate a Device Flow instance, @@ -2036,6 +2054,8 @@ def acquire_token_by_device_flow(self, flow, claims_challenge=None, **kwargs): ), headers=telemetry_context.generate_headers(), **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response @@ -2145,5 +2165,7 @@ def acquire_token_on_behalf_of(self, user_assertion, scopes, claims_challenge=No headers=telemetry_context.generate_headers(), # TBD: Expose a login_hint (or ccs_routing_hint) param for web app **kwargs)) + if "access_token" in response: + response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP telemetry_context.update_telemetry(response) return response diff --git a/sample/confidential_client_certificate_sample.py b/sample/confidential_client_certificate_sample.py index b388cbd4..93c72ee9 100644 --- a/sample/confidential_client_certificate_sample.py +++ b/sample/confidential_client_certificate_sample.py @@ -63,6 +63,7 @@ def acquire_and_use_token(): result = global_app.acquire_token_for_client(scopes=config["scope"]) if "access_token" in result: + print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 # Calling graph using the access token graph_data = requests.get( # Use token to call downstream service config["endpoint"], diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_secret_sample.py index 3a06cded..9c616d53 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_secret_sample.py @@ -62,6 +62,7 @@ def acquire_and_use_token(): result = global_app.acquire_token_for_client(scopes=config["scope"]) if "access_token" in result: + print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 # Calling graph using the access token graph_data = requests.get( # Use token to call downstream service config["endpoint"], diff --git a/sample/device_flow_sample.py b/sample/device_flow_sample.py index e894a7a3..89dccd1c 100644 --- a/sample/device_flow_sample.py +++ b/sample/device_flow_sample.py @@ -84,6 +84,7 @@ def acquire_and_use_token(): # and then keep calling acquire_token_by_device_flow(flow) in your own customized loop. if "access_token" in result: + print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 # Calling graph using the access token graph_data = requests.get( # Use token to call downstream service config["endpoint"], diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index 6d48ee75..b063661c 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -79,6 +79,7 @@ def acquire_and_use_token(): ) if "access_token" in result: + print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 # Calling graph using the access token graph_response = requests.get( # Use token to call downstream service config["endpoint"], diff --git a/sample/username_password_sample.py b/sample/username_password_sample.py index 13bf7bed..a25407d0 100644 --- a/sample/username_password_sample.py +++ b/sample/username_password_sample.py @@ -66,6 +66,7 @@ def acquire_and_use_token(): config["username"], config["password"], scopes=config["scope"]) if "access_token" in result: + print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 # Calling graph using the access token graph_data = requests.get( # Use token to call downstream service config["endpoint"], diff --git a/sample/vault_jwt_sample.py b/sample/vault_jwt_sample.py index 0a46a9ed..9410039c 100644 --- a/sample/vault_jwt_sample.py +++ b/sample/vault_jwt_sample.py @@ -125,6 +125,7 @@ def acquire_and_use_token(): result = global_app.acquire_token_for_client(scopes=config["scope"]) if "access_token" in result: + print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 # Calling graph using the access token graph_data = requests.get( # Use token to call downstream service config["endpoint"], diff --git a/tests/test_application.py b/tests/test_application.py index df87a05b..fc529f01 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -109,6 +109,7 @@ def tester(url, **kwargs): self.scopes, self.account, post=tester) self.assertEqual("", result.get("classification")) + class TestClientApplicationAcquireTokenSilentFociBehaviors(unittest.TestCase): def setUp(self): @@ -263,6 +264,7 @@ def test_get_accounts_should_find_accounts_under_different_alias(self): def test_acquire_token_silent_should_find_at_under_different_alias(self): result = self.app.acquire_token_silent(self.scopes, self.account) self.assertNotEqual(None, result) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_CACHE) self.assertEqual(self.access_token, result.get('access_token')) def test_acquire_token_silent_should_find_rt_under_different_alias(self): @@ -360,6 +362,7 @@ def test_fresh_token_should_be_returned_from_cache(self): post=lambda url, *args, **kwargs: # Utilize the undocumented test feature self.fail("I/O shouldn't happen in cache hit AT scenario") ) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_CACHE) self.assertEqual(access_token, result.get("access_token")) self.assertNotIn("refresh_in", result, "Customers need not know refresh_in") @@ -374,6 +377,7 @@ def mock_post(url, headers=None, *args, **kwargs): "refresh_in": 123, })) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(new_access_token, result.get("access_token")) self.assertNotIn("refresh_in", result, "Customers need not know refresh_in") @@ -385,6 +389,7 @@ def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|84,4|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=400, text=json.dumps({"error": "foo"})) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_CACHE) self.assertEqual(old_at, result.get("access_token")) def test_expired_token_and_unavailable_aad_should_return_error(self): @@ -409,6 +414,7 @@ def mock_post(url, headers=None, *args, **kwargs): "refresh_in": 123, })) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(new_access_token, result.get("access_token")) self.assertNotIn("refresh_in", result, "Customers need not know refresh_in") @@ -444,6 +450,7 @@ def test_maintaining_offline_state_and_sending_them(self): post=lambda url, *args, **kwargs: # Utilize the undocumented test feature self.fail("I/O shouldn't happen in cache hit AT scenario") ) + self.assertEqual(result[app._TOKEN_SOURCE], app._TOKEN_SOURCE_CACHE) self.assertEqual(cached_access_token, result.get("access_token")) error1 = "error_1" @@ -477,6 +484,7 @@ def mock_post(url, headers=None, *args, **kwargs): "The previous error should result in same success counter plus latest error info") return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) result = app.acquire_token_by_device_flow({"device_code": "123"}, post=mock_post) + self.assertEqual(result[app._TOKEN_SOURCE], app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) def mock_post(url, headers=None, *args, **kwargs): @@ -485,6 +493,7 @@ def mock_post(url, headers=None, *args, **kwargs): "The previous success should reset all offline telemetry counters") return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) result = app.acquire_token_by_device_flow({"device_code": "123"}, post=mock_post) + self.assertEqual(result[app._TOKEN_SOURCE], app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) @@ -503,6 +512,7 @@ def mock_post(url, headers=None, *args, **kwargs): result = self.app.acquire_token_by_auth_code_flow( {"state": state, "code_verifier": "bar"}, {"state": state, "code": "012"}, post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) def test_acquire_token_by_refresh_token(self): @@ -511,6 +521,7 @@ def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|85,1|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) result = self.app.acquire_token_by_refresh_token("rt", ["s"], post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) @@ -529,6 +540,7 @@ def mock_post(url, headers=None, *args, **kwargs): return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) result = self.app.acquire_token_by_device_flow( {"device_code": "123"}, post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) def test_acquire_token_by_username_password(self): @@ -538,6 +550,7 @@ def mock_post(url, headers=None, *args, **kwargs): return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) result = self.app.acquire_token_by_username_password( "username", "password", ["scope"], post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) @@ -556,6 +569,7 @@ def mock_post(url, headers=None, *args, **kwargs): "expires_in": 0, })) result = self.app.acquire_token_for_client(["scope"], post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual("AT 1", result.get("access_token"), "Shall get a new token") def mock_post(url, headers=None, *args, **kwargs): @@ -566,6 +580,7 @@ def mock_post(url, headers=None, *args, **kwargs): "refresh_in": -100, # A hack to make sure it will attempt refresh })) result = self.app.acquire_token_for_client(["scope"], post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual("AT 2", result.get("access_token"), "Shall get a new token") def mock_post(url, headers=None, *args, **kwargs): @@ -573,6 +588,7 @@ def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|730,4|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=400, text=json.dumps({"error": "foo"})) result = self.app.acquire_token_for_client(["scope"], post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_CACHE) self.assertEqual("AT 2", result.get("access_token"), "Shall get aging token") def test_acquire_token_on_behalf_of(self): @@ -581,6 +597,7 @@ def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|523,0|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=200, text=json.dumps({"access_token": at})) result = self.app.acquire_token_on_behalf_of("assertion", ["s"], post=mock_post) + self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(at, result.get("access_token")) From 3d3d02f5a86f668a4662a9cbd7125d70e759a8da Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 25 Oct 2023 14:46:43 -0700 Subject: [PATCH 098/262] Deprecate allow_broker, use enable_broker_on_windows --- msal/__main__.py | 6 +- msal/application.py | 132 ++++++++++++++++++++--------------- sample/interactive_sample.py | 4 +- tests/test_e2e.py | 34 +++++---- 4 files changed, 100 insertions(+), 76 deletions(-) diff --git a/msal/__main__.py b/msal/__main__.py index 48a50f6c..b0fb6b7a 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -190,9 +190,9 @@ def _main(): option_renderer=lambda a: a["name"], header="Impersonate this app (or you can type in the client_id of your own app)", accept_nonempty_string=True) - allow_broker = _input_boolean("Allow broker?") + enable_broker = _input_boolean("Enable broker? It will error out later if your app has not registered some redirect URI") enable_debug_log = _input_boolean("Enable MSAL Python's DEBUG log?") - enable_pii_log = _input_boolean("Enable PII in broker's log?") if allow_broker and enable_debug_log else False + enable_pii_log = _input_boolean("Enable PII in broker's log?") if enable_broker and enable_debug_log else False app = msal.PublicClientApplication( chosen_app["client_id"] if isinstance(chosen_app, dict) else chosen_app, authority=_select_options([ @@ -205,7 +205,7 @@ def _main(): header="Input authority (Note that MSA-PT apps would NOT use the /common authority)", accept_nonempty_string=True, ), - allow_broker=allow_broker, + enable_broker_on_windows=enable_broker, enable_pii_log=enable_pii_log, ) if enable_debug_log: diff --git a/msal/application.py b/msal/application.py index 95fa86c1..a5d97b47 100644 --- a/msal/application.py +++ b/msal/application.py @@ -181,6 +181,8 @@ class ClientApplication(object): _TOKEN_SOURCE_CACHE = "cache" _TOKEN_SOURCE_BROKER = "broker" + _enable_broker = False + def __init__( self, client_id, client_credential=None, authority=None, validate_authority=True, @@ -470,48 +472,7 @@ def __init__( New in version 1.19.0. :param boolean allow_broker: - This parameter is NOT applicable to :class:`ConfidentialClientApplication`. - - A broker is a component installed on your device. - Broker implicitly gives your device an identity. By using a broker, - your device becomes a factor that can satisfy MFA (Multi-factor authentication). - This factor would become mandatory - if a tenant's admin enables a corresponding Conditional Access (CA) policy. - The broker's presence allows Microsoft identity platform - to have higher confidence that the tokens are being issued to your device, - and that is more secure. - - An additional benefit of broker is, - it runs as a long-lived process with your device's OS, - and maintains its own cache, - so that your broker-enabled apps (even a CLI) - could automatically SSO from a previously established signed-in session. - - This parameter defaults to None, which means MSAL will not utilize a broker. - If this parameter is set to True, - MSAL will use the broker whenever possible, - and automatically fall back to non-broker behavior. - That also means your app does not need to enable broker conditionally, - you can always set allow_broker to True, - as long as your app meets the following prerequisite: - - * Installed optional dependency, e.g. ``pip install msal[broker]>=1.20,<2``. - (Note that broker is currently only available on Windows 10+) - - * Register a new redirect_uri for your desktop app as: - ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` - - * Tested your app in following scenarios: - - * Windows 10+ - - * PublicClientApplication's following methods:: - acquire_token_interactive(), acquire_token_by_username_password(), - acquire_token_silent() (or acquire_token_silent_with_error()). - - * AAD and MSA accounts (i.e. Non-ADFS, non-B2C) - - New in version 1.20.0. + Deprecated. Please use ``enable_broker_on_windows`` instead. :param boolean enable_pii_log: When enabled, logs may include PII (Personal Identifiable Information). @@ -584,34 +545,47 @@ def __init__( ) else: raise - is_confidential_app = bool( - isinstance(self, ConfidentialClientApplication) or self.client_credential) + + self._decide_broker(allow_broker, enable_pii_log) + self.token_cache = token_cache or TokenCache() + self._region_configured = azure_region + self._region_detected = None + self.client, self._regional_client = self._build_client( + client_credential, self.authority) + self.authority_groups = None + self._telemetry_buffer = {} + self._telemetry_lock = Lock() + + def _decide_broker(self, allow_broker, enable_pii_log): + is_confidential_app = self.client_credential or isinstance( + self, ConfidentialClientApplication) if is_confidential_app and allow_broker: raise ValueError("allow_broker=True is only supported in PublicClientApplication") - self._enable_broker = False - if (allow_broker and not is_confidential_app - and sys.platform == "win32" + # Historically, we chose to support ClientApplication("client_id", allow_broker=True) + if allow_broker: + warnings.warn( + "allow_broker is deprecated. " + "Please use PublicClientApplication(..., enable_broker_on_windows=True)", + DeprecationWarning) + self._enable_broker = self._enable_broker or ( + # When we started the broker project on Windows platform, + # the allow_broker was meant to be cross-platform. Now we realize + # that other platforms have different redirect_uri requirements, + # so the old allow_broker is deprecated and will only for Windows. + allow_broker and sys.platform == "win32") + if (self._enable_broker and not is_confidential_app and not self.authority.is_adfs and not self.authority._is_b2c): try: from . import broker # Trigger Broker's initialization - self._enable_broker = True if enable_pii_log: broker._enable_pii_log() except RuntimeError: + self._enable_broker = False logger.exception( "Broker is unavailable on this platform. " "We will fallback to non-broker.") logger.debug("Broker enabled? %s", self._enable_broker) - self.token_cache = token_cache or TokenCache() - self._region_configured = azure_region - self._region_detected = None - self.client, self._regional_client = self._build_client( - client_credential, self.authority) - self.authority_groups = None - self._telemetry_buffer = {} - self._telemetry_lock = Lock() - def _decorate_scope( self, scopes, reserved_scope=frozenset(['openid', 'profile', 'offline_access'])): @@ -1746,9 +1720,53 @@ class PublicClientApplication(ClientApplication): # browser app or mobile app def __init__(self, client_id, client_credential=None, **kwargs): """Same as :func:`ClientApplication.__init__`, except that ``client_credential`` parameter shall remain ``None``. + + .. note:: + + You may set enable_broker_on_windows to True. + + What is a broker, and why use it? + + A broker is a component installed on your device. + Broker implicitly gives your device an identity. By using a broker, + your device becomes a factor that can satisfy MFA (Multi-factor authentication). + This factor would become mandatory + if a tenant's admin enables a corresponding Conditional Access (CA) policy. + The broker's presence allows Microsoft identity platform + to have higher confidence that the tokens are being issued to your device, + and that is more secure. + + An additional benefit of broker is, + it runs as a long-lived process with your device's OS, + and maintains its own cache, + so that your broker-enabled apps (even a CLI) + could automatically SSO from a previously established signed-in session. + + ADFS and B2C do not support broker. + MSAL will automatically fallback to use browser. + + You shall only enable broker when your app: + + 1. is running on supported platforms, + and already registered their corresponding redirect_uri + + * ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` + if your app is expected to run on Windows 10+ + + 2. installed broker dependency, + e.g. ``pip install msal[broker]>=1.25,<2``. + + 3. tested with ``acquire_token_interactive()`` and ``acquire_token_silent()``. + + :param boolean enable_broker_on_windows: + This setting is only effective if your app is running on Windows 10+. + This parameter defaults to None, which means MSAL will not utilize a broker. """ if client_credential is not None: raise ValueError("Public Client should not possess credentials") + # Using kwargs notation for now. We will switch to keyword-only arguments. + enable_broker_on_windows = kwargs.pop("enable_broker_on_windows", False) + self._enable_broker = enable_broker_on_windows and sys.platform == "win32" super(PublicClientApplication, self).__init__( client_id, client_credential=None, **kwargs) diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index b063661c..f283ed29 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -37,8 +37,8 @@ # Create a preferably long-lived app instance, to avoid the overhead of app creation global_app = msal.PublicClientApplication( config["client_id"], authority=config["authority"], - #allow_broker=True, # If opted in, you will be guided to meet the prerequisites, when applicable - # See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition + #enable_broker_on_windows=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already + # See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9deec8f7..36e7a445 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -165,21 +165,27 @@ def _build_app(cls, http_client=None, azure_region=None, **kwargs): - try: - import pymsalruntime - broker_available = True - except ImportError: - broker_available = False - return (msal.ConfidentialClientApplication - if client_credential else msal.PublicClientApplication)( - client_id, - client_credential=client_credential, - authority=authority, - azure_region=azure_region, - http_client=http_client or MinimalHttpClient(), - allow_broker=broker_available # This way, we reuse same test cases, by run them with and without broker - and not client_credential, + if client_credential: + return msal.ConfidentialClientApplication( + client_id, + client_credential=client_credential, + authority=authority, + azure_region=azure_region, + http_client=http_client or MinimalHttpClient(), ) + else: + # Reuse same test cases, by run them with and without broker + try: + import pymsalruntime + broker_available = True + except ImportError: + broker_available = False + return msal.PublicClientApplication( + client_id, + authority=authority, + http_client=http_client or MinimalHttpClient(), + enable_broker_on_windows=broker_available, + ) def _test_username_password(self, authority=None, client_id=None, username=None, password=None, scope=None, From e90f6cf2f514179f4f4553dc23bc4e40a3d540ce Mon Sep 17 00:00:00 2001 From: jennyf19 Date: Mon, 30 Oct 2023 08:29:03 -0700 Subject: [PATCH 099/262] add triage labels to bug report (#612) --- .github/ISSUE_TEMPLATE/bug_report.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 58bfecda..8d823e82 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,8 +1,8 @@ --- name: Bug report about: Create a report to help us improve -title: '' -labels: '' +title: '[Bug] ' +labels: ["untriaged", "needs attention"] assignees: '' --- From 6973a99e6bac932845c57eaee84bdbed366e5b40 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 1 Nov 2023 22:53:22 -0700 Subject: [PATCH 100/262] Only invoke broker for selected flows (grants) ROPC also bypass broker, for now Unit tests --- msal/__main__.py | 48 ++++++++++++++++++++-- msal/application.py | 20 ++++++--- msal/broker.py | 16 ++++---- msal/token_cache.py | 7 ++++ tests/test_account_source.py | 79 ++++++++++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 16 deletions(-) create mode 100644 tests/test_account_source.py diff --git a/msal/__main__.py b/msal/__main__.py index b0fb6b7a..8bd19d33 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -5,12 +5,20 @@ Usage 1: Run it on the fly. python -m msal + Note: We choose to not define a console script to avoid name conflict. Usage 2: Build an all-in-one executable file for bug bash. shiv -e msal.__main__._main -o msaltest-on-os-name.pyz . - Note: We choose to not define a console script to avoid name conflict. """ -import base64, getpass, json, logging, sys, msal +import base64, getpass, json, logging, sys, os, atexit, msal + +_token_cache_filename = "msal_cache.bin" +global_cache = msal.SerializableTokenCache() +atexit.register(lambda: + open(_token_cache_filename, "w").write(global_cache.serialize()) + # Hint: The following optional line persists only when state changed + if global_cache.has_state_changed else None + ) _AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" _VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" @@ -66,7 +74,7 @@ def _select_account(app): if accounts: return _select_options( accounts, - option_renderer=lambda a: a["username"], + option_renderer=lambda a: "{}, came from {}".format(a["username"], a["account_source"]), header="Account(s) already signed in inside MSAL Python:", ) else: @@ -76,7 +84,7 @@ def _acquire_token_silent(app): """acquire_token_silent() - with an account already signed into MSAL Python.""" account = _select_account(app) if account: - print_json(app.acquire_token_silent( + print_json(app.acquire_token_silent_with_error( _input_scopes(), account=account, force_refresh=_input_boolean("Bypass MSAL Python's token cache?"), @@ -122,6 +130,15 @@ def _acquire_token_by_username_password(app): print_json(app.acquire_token_by_username_password( _input("username: "), getpass.getpass("password: "), scopes=_input_scopes())) +def _acquire_token_by_device_flow(app): + """acquire_token_by_device_flow() - Note that this one does not go through broker""" + flow = app.initiate_device_flow(scopes=_input_scopes()) + print(flow["message"]) + sys.stdout.flush() # Some terminal needs this to ensure the message is shown + input("After you completed the step above, press ENTER in this console to continue...") + result = app.acquire_token_by_device_flow(flow) # By default it will block + print_json(result) + _JWK1 = """{"kty":"RSA", "n":"2tNr73xwcj6lH7bqRZrFzgSLj7OeLfbn8216uOMDHuaZ6TEUBDN8Uz0ve8jAlKsP9CQFCSVoSNovdE-fs7c15MxEGHjDcNKLWonznximj8pDGZQjVdfK-7mG6P6z-lgVcLuYu5JcWU_PeEqIKg5llOaz-qeQ4LEDS4T1D2qWRGpAra4rJX1-kmrWmX_XIamq30C9EIO0gGuT4rc2hJBWQ-4-FnE1NXmy125wfT3NdotAJGq5lMIfhjfglDbJCwhc8Oe17ORjO3FsB5CLuBRpYmP7Nzn66lRY3Fe11Xz8AEBl3anKFSJcTvlMnFtu3EpD-eiaHfTgRBU7CztGQqVbiQ", "e":"AQAB"}""" _SSH_CERT_DATA = {"token_type": "ssh-cert", "key_id": "key1", "req_cnf": _JWK1} _SSH_CERT_SCOPE = ["https://pas.windows.net/CheckMyAccess/Linux/.default"] @@ -182,6 +199,27 @@ def _exit(app): def _main(): print("Welcome to the Msal Python {} Tester (Experimental)\n".format(msal.__version__)) + cache_choice = _select_options([ + { + "choice": "empty", + "desc": "Start with an empty token cache. Suitable for one-off tests.", + }, + { + "choice": "reuse", + "desc": "Reuse the previous token cache {} (if any) " + "which was created during last test app exit. " + "Useful for testing acquire_token_silent() repeatedly".format( + _token_cache_filename), + }, + ], + option_renderer=lambda o: o["desc"], + header="What token cache state do you want to begin with?", + accept_nonempty_string=False) + if cache_choice["choice"] == "reuse" and os.path.exists(_token_cache_filename): + try: + global_cache.deserialize(open(_token_cache_filename, "r").read()) + except IOError: + pass # Use empty token cache chosen_app = _select_options([ {"client_id": _AZURE_CLI, "name": "Azure CLI (Correctly configured for MSA-PT)"}, {"client_id": _VISUAL_STUDIO, "name": "Visual Studio (Correctly configured for MSA-PT)"}, @@ -207,6 +245,7 @@ def _main(): ), enable_broker_on_windows=enable_broker, enable_pii_log=enable_pii_log, + token_cache=global_cache, ) if enable_debug_log: logging.basicConfig(level=logging.DEBUG) @@ -215,6 +254,7 @@ def _main(): _acquire_token_silent, _acquire_token_interactive, _acquire_token_by_username_password, + _acquire_token_by_device_flow, _acquire_ssh_cert_silently, _acquire_ssh_cert_interactive, _acquire_pop_token_interactive, diff --git a/msal/application.py b/msal/application.py index a5d97b47..d868ff36 100644 --- a/msal/application.py +++ b/msal/application.py @@ -17,7 +17,7 @@ from .mex import send_request as mex_send_request from .wstrust_request import send_request as wst_send_request from .wstrust_response import * -from .token_cache import TokenCache, _get_username +from .token_cache import TokenCache, _get_username, _GRANT_TYPE_BROKER import msal.telemetry from .region import _detect_region from .throttled_http_client import ThrottledHttpClient @@ -1104,6 +1104,7 @@ def _find_msal_accounts(self, environment): "home_account_id": a.get("home_account_id"), "environment": a.get("environment"), "username": a.get("username"), + "account_source": a.get("account_source"), # The following fields for backward compatibility, for now "authority_type": a.get("authority_type"), @@ -1398,7 +1399,10 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( if account and account.get("authority_type") == _AUTHORITY_TYPE_CLOUDSHELL: return self._acquire_token_by_cloud_shell(scopes, data=data) - if self._enable_broker and account is not None: + if self._enable_broker and account and account.get("account_source") in ( + _GRANT_TYPE_BROKER, # Broker successfully established this account previously. + None, # Unknown data from older MSAL. Broker might still work. + ): from .broker import _acquire_token_silently response = _acquire_token_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), @@ -1409,8 +1413,12 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( self._client_capabilities, claims_challenge), correlation_id=correlation_id, **data) - if response: # The broker provided a decisive outcome, so we use it - return self._process_broker_response(response, scopes, data) + if response: # Broker provides a decisive outcome + account_was_established_by_broker = account.get( + "account_source") == _GRANT_TYPE_BROKER + broker_attempt_succeeded_just_now = "error" not in response + if account_was_established_by_broker or broker_attempt_succeeded_just_now: + return self._process_broker_response(response, scopes, data) if account: result = self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( @@ -1441,6 +1449,8 @@ def _process_broker_response(self, response, scopes, data): response=response, data=data, _account_id=response["_account_id"], + environment=self.authority.instance, # Be consistent with non-broker flows + grant_type=_GRANT_TYPE_BROKER, # A pseudo grant type for TokenCache to mark account_source as broker )) response[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_BROKER return _clean_up(response) @@ -1628,7 +1638,7 @@ def acquire_token_by_username_password( """ claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) - if self._enable_broker: + if False: # Disabled, for now. It was if self._enable_broker: from .broker import _signin_silently response = _signin_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), diff --git a/msal/broker.py b/msal/broker.py index 81b14a2a..ea0366b3 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -70,7 +70,7 @@ def _convert_error(error, client_id): def _read_account_by_id(account_id, correlation_id): - """Return an instance of MSALRuntimeError or MSALRuntimeAccount, or None""" + """Return an instance of MSALRuntimeAccount, or log error and return None""" callback_data = _CallbackData() pymsalruntime.read_account_by_id( account_id, @@ -78,8 +78,14 @@ def _read_account_by_id(account_id, correlation_id): lambda result, callback_data=callback_data: callback_data.complete(result) ) callback_data.signal.wait() - return (callback_data.result.get_error() or callback_data.result.get_account() - or None) # None happens when the account was not created by broker + error = callback_data.result.get_error() + if error: + logger.debug("read_account_by_id() error: %s", _convert_error(error, None)) + return None + account = callback_data.result.get_account() + if account: + return account + return None # None happens when the account was not created by broker def _convert_result(result, client_id, expected_token_type=None): # Mimic an on-the-wire response from AAD @@ -196,8 +202,6 @@ def _acquire_token_silently( # acquireTokenSilently is expected to fail. - Sam Wilson correlation_id = correlation_id or _get_new_correlation_id() account = _read_account_by_id(account_id, correlation_id) - if isinstance(account, pymsalruntime.MSALRuntimeError): - return _convert_error(account, client_id) if account is None: return params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) @@ -221,8 +225,6 @@ def _acquire_token_silently( def _signout_silently(client_id, account_id, correlation_id=None): correlation_id = correlation_id or _get_new_correlation_id() account = _read_account_by_id(account_id, correlation_id) - if isinstance(account, pymsalruntime.MSALRuntimeError): - return _convert_error(account, client_id) if account is None: return callback_data = _CallbackData() diff --git a/msal/token_cache.py b/msal/token_cache.py index 49262069..bd6d8a6f 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -5,9 +5,11 @@ from .authority import canonicalize from .oauth2cli.oidc import decode_part, decode_id_token +from .oauth2cli.oauth2 import Client logger = logging.getLogger(__name__) +_GRANT_TYPE_BROKER = "broker" def is_subdict_of(small, big): return dict(big, **small) == big @@ -210,6 +212,11 @@ def __add(self, event, now=None): else self.AuthorityType.MSSTS), # "client_info": response.get("client_info"), # Optional } + grant_types_that_establish_an_account = ( + _GRANT_TYPE_BROKER, "authorization_code", "password", + Client.DEVICE_FLOW["GRANT_TYPE"]) + if event.get("grant_type") in grant_types_that_establish_an_account: + account["account_source"] = event["grant_type"] self.modify(self.CredentialType.ACCOUNT, account, account) if id_token: diff --git a/tests/test_account_source.py b/tests/test_account_source.py new file mode 100644 index 00000000..b8713992 --- /dev/null +++ b/tests/test_account_source.py @@ -0,0 +1,79 @@ +import json +try: + from unittest.mock import patch +except: + from mock import patch +try: + import pymsalruntime + broker_available = True +except ImportError: + broker_available = False +import msal +from tests import unittest +from tests.test_token_cache import build_response +from tests.http_client import MinimalResponse + + +SCOPE = "scope_foo" +TOKEN_RESPONSE = build_response( + access_token="at", + uid="uid", utid="utid", # So that it will create an account + scope=SCOPE, refresh_token="rt", # So that non-broker's acquire_token_silent() would work +) + +def _mock_post(url, headers=None, *args, **kwargs): + return MinimalResponse(status_code=200, text=json.dumps(TOKEN_RESPONSE)) + +@unittest.skipUnless(broker_available, "These test cases need pip install msal[broker]") +@patch("msal.broker._acquire_token_silently", return_value=dict( + TOKEN_RESPONSE, _account_id="placeholder")) +@patch.object(msal.authority, "tenant_discovery", return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", +}) # Otherwise it would fail on OIDC discovery +class TestAccountSourceBehavior(unittest.TestCase): + + def test_device_flow_and_its_silent_call_should_bypass_broker(self, _, mocked_broker_ats): + app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) + result = app.acquire_token_by_device_flow({"device_code": "123"}, post=_mock_post) + self.assertEqual(result["token_source"], "identity_provider") + + account = app.get_accounts()[0] + self.assertEqual(account["account_source"], "urn:ietf:params:oauth:grant-type:device_code") + + result = app.acquire_token_silent_with_error( + [SCOPE], account, force_refresh=True, post=_mock_post) + mocked_broker_ats.assert_not_called() + self.assertEqual(result["token_source"], "identity_provider") + + def test_ropc_flow_and_its_silent_call_should_bypass_broker(self, _, mocked_broker_ats): + app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) + with patch.object(app.authority, "user_realm_discovery", return_value={}): + result = app.acquire_token_by_username_password( + "username", "placeholder", [SCOPE], post=_mock_post) + self.assertEqual(result["token_source"], "identity_provider") + + account = app.get_accounts()[0] + self.assertEqual(account["account_source"], "password") + + result = app.acquire_token_silent_with_error( + [SCOPE], account, force_refresh=True, post=_mock_post) + mocked_broker_ats.assert_not_called() + self.assertEqual(result["token_source"], "identity_provider") + + def test_interactive_flow_and_its_silent_call_should_invoke_broker(self, _, mocked_broker_ats): + app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) + with patch.object(app, "_acquire_token_interactive_via_broker", return_value=dict( + TOKEN_RESPONSE, _account_id="placeholder")): + result = app.acquire_token_interactive( + [SCOPE], parent_window_handle=app.CONSOLE_WINDOW_HANDLE) + self.assertEqual(result["token_source"], "broker") + + account = app.get_accounts()[0] + self.assertEqual(account["account_source"], "broker") + + result = app.acquire_token_silent_with_error( + [SCOPE], account, force_refresh=True, post=_mock_post) + mocked_broker_ats.assert_called_once() + self.assertEqual(result["token_source"], "broker") + From bcff60f44dcfa29adb87c0696be43919c43a3b8d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 6 Nov 2023 15:47:00 -0800 Subject: [PATCH 101/262] MSAL Python 1.25 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index d868ff36..bfcbb7d1 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.24.1" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.25.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 1666d851f4c4536e47607ee1d27ba5db9e7d2b3c Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 8 Nov 2023 21:21:20 -0800 Subject: [PATCH 102/262] Add more docs --- msal/application.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msal/application.py b/msal/application.py index bfcbb7d1..2f33e965 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1771,6 +1771,8 @@ def __init__(self, client_id, client_credential=None, **kwargs): :param boolean enable_broker_on_windows: This setting is only effective if your app is running on Windows 10+. This parameter defaults to None, which means MSAL will not utilize a broker. + + New in MSAL Python 1.25.0. """ if client_credential is not None: raise ValueError("Public Client should not possess credentials") From 56bdab93938e54199a10d3507d587c01f3a44c51 Mon Sep 17 00:00:00 2001 From: micwoj92 <45581170+micwoj92@users.noreply.github.com> Date: Sat, 18 Nov 2023 01:43:47 +0100 Subject: [PATCH 103/262] Remove newlines from description. --- setup.cfg | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index adf13aba..75df4f9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,12 +6,7 @@ universal=1 [metadata] name = msal version = attr: msal.__version__ -description = - The Microsoft Authentication Library (MSAL) for Python library - enables your app to access the Microsoft Cloud - by supporting authentication of users with - Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) - using industry standard OAuth2 and OpenID Connect. +description = The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect. long_description = file: README.md long_description_content_type = text/markdown license = MIT From 460dc66acd6074ff805a2134f046ff784c7e4b74 Mon Sep 17 00:00:00 2001 From: Bogdan Gavril Date: Fri, 1 Dec 2023 08:26:41 +0000 Subject: [PATCH 104/262] #629 - skip region discory when region=None (#630) * #629 - skip region discory when region=None * Tidy up --------- Co-authored-by: Ray Luo --- msal/application.py | 53 +++++++++++---------------------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/msal/application.py b/msal/application.py index 2f33e965..52bd6835 100644 --- a/msal/application.py +++ b/msal/application.py @@ -336,51 +336,22 @@ def __init__( `claims parameter `_ which you will later provide via one of the acquire-token request. - :param str azure_region: - AAD provides regional endpoints for apps to opt in - to keep their traffic remain inside that region. + :param str azure_region: (optional) + Instructs MSAL to use the Entra regional token service. This legacy feature is only available to + first-party applications. Only ``acquire_token_for_client()`` is supported. - As of 2021 May, regional service is only available for - ``acquire_token_for_client()`` sent by any of the following scenarios: + Supports 3 values: - 1. An app powered by a capable MSAL - (MSAL Python 1.12+ will be provisioned) - - 2. An app with managed identity, which is formerly known as MSI. - (However MSAL Python does not support managed identity, - so this one does not apply.) - - 3. An app authenticated by - `Subject Name/Issuer (SNI) `_. - - 4. An app which already onboard to the region's allow-list. - - This parameter defaults to None, which means region behavior remains off. - - App developer can opt in to a regional endpoint, - by provide its region name, such as "westus", "eastus2". - You can find a full list of regions by running - ``az account list-locations -o table``, or referencing to - `this doc `_. - - An app running inside Azure Functions and Azure VM can use a special keyword - ``ClientApplication.ATTEMPT_REGION_DISCOVERY`` to auto-detect region. + ``azure_region=None`` - meaning no region is used. This is the default value. + ``azure_region="some_region"`` - meaning the specified region is used. + ``azure_region=True`` - meaning MSAL will try to auto-detect the region. This is not recommended. .. note:: + Region auto-discovery has been tested on VMs and on Azure Functions. It is unreliable. + Applications using this option should configure a short timeout. - Setting ``azure_region`` to non-``None`` for an app running - outside of Azure Function/VM could hang indefinitely. - - You should consider opting in/out region behavior on-demand, - by loading ``azure_region=None`` or ``azure_region="westus"`` - or ``azure_region=True`` (which means opt-in and auto-detect) - from your per-deployment configuration, and then do - ``app = ConfidentialClientApplication(..., azure_region=azure_region)``. - - Alternatively, you can configure a short timeout, - or provide a custom http_client which has a short timeout. - That way, the latency would be under your control, - but still less performant than opting out of region feature. + For more details and for the values of the region string + see https://learn.microsoft.com/entra/msal/dotnet/resources/region-discovery-troubleshooting New in version 1.12.0. @@ -612,6 +583,8 @@ def _build_telemetry_context( correlation_id=correlation_id, refresh_reason=refresh_reason) def _get_regional_authority(self, central_authority): + if not self._region_configured: # User did not opt-in to ESTS-R + return None # Short circuit to completely bypass region detection self._region_detected = self._region_detected or _detect_region( self.http_client if self._region_configured is not None else None) if (self._region_configured != self.ATTEMPT_REGION_DISCOVERY From 607e702632ae94a7659e4c466868262d527ba995 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 5 Dec 2023 00:31:04 -0800 Subject: [PATCH 105/262] AT POP for Public Client based on broker (#511) * AT POP for Public Client based on broker Pop test case * Use token source during e2e tests * WIP: unsuccessful e2e test for POP SHR --- msal/__init__.py | 1 + msal/__main__.py | 11 +++ msal/application.py | 60 +++++++++++++- msal/auth_scheme.py | 34 ++++++++ msal/broker.py | 23 +++++- tests/test_e2e.py | 192 ++++++++++++++++++++++++++++++++++++-------- 6 files changed, 283 insertions(+), 38 deletions(-) create mode 100644 msal/auth_scheme.py diff --git a/msal/__init__.py b/msal/__init__.py index 4e2faaed..09b7a504 100644 --- a/msal/__init__.py +++ b/msal/__init__.py @@ -33,4 +33,5 @@ ) from .oauth2cli.oidc import Prompt from .token_cache import TokenCache, SerializableTokenCache +from .auth_scheme import PopAuthScheme diff --git a/msal/__main__.py b/msal/__main__.py index 8bd19d33..aeb123b0 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -22,6 +22,11 @@ _AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" _VISUAL_STUDIO = "04f0c124-f2bc-4f59-8241-bf6df9866bbd" +placeholder_auth_scheme = msal.PopAuthScheme( + http_method=msal.PopAuthScheme.HTTP_GET, + url="https://example.com/endpoint", + nonce="placeholder", + ) def print_json(blob): print(json.dumps(blob, indent=2, sort_keys=True)) @@ -88,6 +93,9 @@ def _acquire_token_silent(app): _input_scopes(), account=account, force_refresh=_input_boolean("Bypass MSAL Python's token cache?"), + auth_scheme=placeholder_auth_scheme + if app.is_pop_supported() and _input_boolean("Acquire AT POP via Broker?") + else None, )) def _acquire_token_interactive(app, scopes=None, data=None): @@ -117,6 +125,9 @@ def _acquire_token_interactive(app, scopes=None, data=None): ], # Here this test app mimics the setting for some known MSA-PT apps port=1234, # Hard coded for testing. Real app typically uses default value. prompt=prompt, login_hint=login_hint, data=data or {}, + auth_scheme=placeholder_auth_scheme + if app.is_pop_supported() and _input_boolean("Acquire AT POP via Broker?") + else None, ) if login_hint and "id_token_claims" in result: signed_in_user = result.get("id_token_claims", {}).get("preferred_username") diff --git a/msal/application.py b/msal/application.py index 52bd6835..a7ae7bc2 100644 --- a/msal/application.py +++ b/msal/application.py @@ -182,6 +182,10 @@ class ClientApplication(object): _TOKEN_SOURCE_BROKER = "broker" _enable_broker = False + _AUTH_SCHEME_UNSUPPORTED = ( + "auth_scheme is currently only available from broker. " + "You can enable broker by following these instructions. " + "https://msal-python.readthedocs.io/en/latest/#publicclientapplication") def __init__( self, client_id, @@ -557,6 +561,10 @@ def _decide_broker(self, allow_broker, enable_pii_log): "We will fallback to non-broker.") logger.debug("Broker enabled? %s", self._enable_broker) + def is_pop_supported(self): + """Returns True if this client supports Proof-of-Possession Access Token.""" + return self._enable_broker + def _decorate_scope( self, scopes, reserved_scope=frozenset(['openid', 'profile', 'offline_access'])): @@ -1185,6 +1193,7 @@ def acquire_token_silent( authority=None, # See get_authorization_request_url() force_refresh=False, # type: Optional[boolean] claims_challenge=None, + auth_scheme=None, **kwargs): """Acquire an access token for given account, without user interaction. @@ -1205,7 +1214,7 @@ def acquire_token_silent( return None # A backward-compatible NO-OP to drop the account=None usage result = _clean_up(self._acquire_token_silent_with_error( scopes, account, authority=authority, force_refresh=force_refresh, - claims_challenge=claims_challenge, **kwargs)) + claims_challenge=claims_challenge, auth_scheme=auth_scheme, **kwargs)) return result if result and "error" not in result else None def acquire_token_silent_with_error( @@ -1215,6 +1224,7 @@ def acquire_token_silent_with_error( authority=None, # See get_authorization_request_url() force_refresh=False, # type: Optional[boolean] claims_challenge=None, + auth_scheme=None, **kwargs): """Acquire an access token for given account, without user interaction. @@ -1241,6 +1251,12 @@ def acquire_token_silent_with_error( in the form of a claims_challenge directive in the www-authenticate header to be returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. It is a string of a JSON object which contains lists of claims being requested from these locations. + :param object auth_scheme: + You can provide an ``msal.auth_scheme.PopAuthScheme`` object + so that MSAL will get a Proof-of-Possession (POP) token for you. + + New in version 1.26.0. + :return: - A dict containing no "error" key, and typically contains an "access_token" key, @@ -1252,7 +1268,7 @@ def acquire_token_silent_with_error( return None # A backward-compatible NO-OP to drop the account=None usage return _clean_up(self._acquire_token_silent_with_error( scopes, account, authority=authority, force_refresh=force_refresh, - claims_challenge=claims_challenge, **kwargs)) + claims_challenge=claims_challenge, auth_scheme=auth_scheme, **kwargs)) def _acquire_token_silent_with_error( self, @@ -1261,6 +1277,7 @@ def _acquire_token_silent_with_error( authority=None, # See get_authorization_request_url() force_refresh=False, # type: Optional[boolean] claims_challenge=None, + auth_scheme=None, **kwargs): assert isinstance(scopes, list), "Invalid parameter type" self._validate_ssh_cert_input_data(kwargs.get("data", {})) @@ -1276,6 +1293,7 @@ def _acquire_token_silent_with_error( scopes, account, self.authority, force_refresh=force_refresh, claims_challenge=claims_challenge, correlation_id=correlation_id, + auth_scheme=auth_scheme, **kwargs) if result and "error" not in result: return result @@ -1298,6 +1316,7 @@ def _acquire_token_silent_with_error( scopes, account, the_authority, force_refresh=force_refresh, claims_challenge=claims_challenge, correlation_id=correlation_id, + auth_scheme=auth_scheme, **kwargs) if result: if "error" not in result: @@ -1322,12 +1341,13 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( claims_challenge=None, correlation_id=None, http_exceptions=None, + auth_scheme=None, **kwargs): # This internal method has two calling patterns: # it accepts a non-empty account to find token for a user, # and accepts account=None to find a token for the current app. access_token_from_cache = None - if not (force_refresh or claims_challenge): # Bypass AT when desired or using claims + if not (force_refresh or claims_challenge or auth_scheme): # Then attempt AT cache query={ "client_id": self.client_id, "environment": authority.instance, @@ -1370,6 +1390,8 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( try: data = kwargs.get("data", {}) if account and account.get("authority_type") == _AUTHORITY_TYPE_CLOUDSHELL: + if auth_scheme: + raise ValueError("auth_scheme is not supported in Cloud Shell") return self._acquire_token_by_cloud_shell(scopes, data=data) if self._enable_broker and account and account.get("account_source") in ( @@ -1385,6 +1407,7 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( claims=_merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge), correlation_id=correlation_id, + auth_scheme=auth_scheme, **data) if response: # Broker provides a decisive outcome account_was_established_by_broker = account.get( @@ -1393,6 +1416,8 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( if account_was_established_by_broker or broker_attempt_succeeded_just_now: return self._process_broker_response(response, scopes, data) + if auth_scheme: + raise ValueError(self._AUTH_SCHEME_UNSUPPORTED) if account: result = self._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( authority, self._decorate_scope(scopes), account, @@ -1588,7 +1613,11 @@ def acquire_token_by_refresh_token(self, refresh_token, scopes, **kwargs): return response def acquire_token_by_username_password( - self, username, password, scopes, claims_challenge=None, **kwargs): + self, username, password, scopes, claims_challenge=None, + # Note: We shouldn't need to surface enable_msa_passthrough, + # because this ROPC won't work with MSA account anyway. + auth_scheme=None, + **kwargs): """Gets a token for a given resource via user credentials. See this page for constraints of Username Password Flow. @@ -1604,6 +1633,12 @@ def acquire_token_by_username_password( returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. It is a string of a JSON object which contains lists of claims being requested from these locations. + :param object auth_scheme: + You can provide an ``msal.auth_scheme.PopAuthScheme`` object + so that MSAL will get a Proof-of-Possession (POP) token for you. + + New in version 1.26.0. + :return: A dict representing the json response from AAD: - A successful response would contain "access_token" key, @@ -1623,9 +1658,12 @@ def acquire_token_by_username_password( self.authority._is_known_to_developer or self._instance_discovery is False) else None, claims=claims, + auth_scheme=auth_scheme, ) return self._process_broker_response(response, scopes, kwargs.get("data", {})) + if auth_scheme: + raise ValueError(self._AUTH_SCHEME_UNSUPPORTED) scopes = self._decorate_scope(scopes) telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_BY_USERNAME_PASSWORD_ID) @@ -1768,6 +1806,7 @@ def acquire_token_interactive( max_age=None, parent_window_handle=None, on_before_launching_ui=None, + auth_scheme=None, **kwargs): """Acquire token interactively i.e. via a local browser. @@ -1843,6 +1882,12 @@ def acquire_token_interactive( New in version 1.20.0. + :param object auth_scheme: + You can provide an ``msal.auth_scheme.PopAuthScheme`` object + so that MSAL will get a Proof-of-Possession (POP) token for you. + + New in version 1.26.0. + :return: - A dict containing no "error" key, and typically contains an "access_token" key. @@ -1887,12 +1932,15 @@ def acquire_token_interactive( claims, data, on_before_launching_ui, + auth_scheme, prompt=prompt, login_hint=login_hint, max_age=max_age, ) return self._process_broker_response(response, scopes, data) + if auth_scheme: + raise ValueError("auth_scheme is currently only available from broker") on_before_launching_ui(ui="browser") telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_INTERACTIVE) @@ -1927,6 +1975,7 @@ def _acquire_token_interactive_via_broker( claims, # type: str data, # type: dict on_before_launching_ui, # type: callable + auth_scheme, # type: object prompt=None, login_hint=None, # type: Optional[str] max_age=None, @@ -1950,6 +1999,7 @@ def _acquire_token_interactive_via_broker( accounts[0]["local_account_id"], scopes, claims=claims, + auth_scheme=auth_scheme, **data) if response and "error" not in response: return response @@ -1962,6 +2012,7 @@ def _acquire_token_interactive_via_broker( claims=claims, max_age=max_age, enable_msa_pt=enable_msa_passthrough, + auth_scheme=auth_scheme, **data) is_wrong_account = bool( # _signin_silently() only gets tokens for default account, @@ -2002,6 +2053,7 @@ def _acquire_token_interactive_via_broker( claims=claims, max_age=max_age, enable_msa_pt=enable_msa_passthrough, + auth_scheme=auth_scheme, **data) def initiate_device_flow(self, scopes=None, **kwargs): diff --git a/msal/auth_scheme.py b/msal/auth_scheme.py new file mode 100644 index 00000000..841adab3 --- /dev/null +++ b/msal/auth_scheme.py @@ -0,0 +1,34 @@ +try: + from urllib.parse import urlparse +except ImportError: # Fall back to Python 2 + from urlparse import urlparse + +# We may support more auth schemes in the future +class PopAuthScheme(object): + HTTP_GET = "GET" + HTTP_POST = "POST" + HTTP_PUT = "PUT" + HTTP_DELETE = "DELETE" + HTTP_PATCH = "PATCH" + _HTTP_METHODS = (HTTP_GET, HTTP_POST, HTTP_PUT, HTTP_DELETE, HTTP_PATCH) + # Internal design: https://identitydivision.visualstudio.com/DevEx/_git/AuthLibrariesApiReview?path=/PoPTokensProtocol/PopTokensProtocol.md + def __init__(self, http_method=None, url=None, nonce=None): + """Create an auth scheme which is needed to obtain a Proof-of-Possession token. + + :param str http_method: + Its value is an uppercase http verb, such as "GET" and "POST". + :param str url: + The url to be signed. + :param str nonce: + The nonce came from resource's challenge. + """ + if not (http_method and url and nonce): + # In the future, we may also support accepting an http_response as input + raise ValueError("All http_method, url and nonce are required parameters") + if http_method not in self._HTTP_METHODS: + raise ValueError("http_method must be uppercase, according to " + "https://datatracker.ietf.org/doc/html/draft-ietf-oauth-signed-http-request-03#section-3") + self._http_method = http_method + self._url = urlparse(url) + self._nonce = nonce + diff --git a/msal/broker.py b/msal/broker.py index ea0366b3..a0904199 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -99,13 +99,17 @@ def _convert_result(result, client_id, expected_token_type=None): # Mimic an on assert account, "Account is expected to be always available" # Note: There are more account attribute getters available in pymsalruntime 0.13+ return_value = {k: v for k, v in { - "access_token": result.get_access_token(), + "access_token": + result.get_authorization_header() # It returns "pop SignedHttpRequest" + .split()[1] + if result.is_pop_authorization() else result.get_access_token(), "expires_in": result.get_access_token_expiry_time() - int(time.time()), # Convert epoch to count-down "id_token": result.get_raw_id_token(), # New in pymsalruntime 0.8.1 "id_token_claims": id_token_claims, "client_info": account.get_client_info(), "_account_id": account.get_account_id(), - "token_type": expected_token_type or "Bearer", # Workaround its absence from broker + "token_type": "pop" if result.is_pop_authorization() else ( + expected_token_type or "bearer"), # Workaround "ssh-cert"'s absence from broker }.items() if v} likely_a_cert = return_value["access_token"].startswith("AAAA") # Empirical observation if return_value["token_type"].lower() == "ssh-cert" and not likely_a_cert: @@ -128,11 +132,16 @@ def _enable_msa_pt(params): def _signin_silently( authority, client_id, scopes, correlation_id=None, claims=None, enable_msa_pt=False, + auth_scheme=None, **kwargs): params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) params.set_requested_scopes(scopes) if claims: params.set_decoded_claims(claims) + if auth_scheme: + params.set_pop_params( + auth_scheme._http_method, auth_scheme._url.netloc, auth_scheme._url.path, + auth_scheme._nonce) callback_data = _CallbackData() for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. if v is not None: @@ -156,6 +165,7 @@ def _signin_interactively( claims=None, correlation_id=None, enable_msa_pt=False, + auth_scheme=None, **kwargs): params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) params.set_requested_scopes(scopes) @@ -178,6 +188,10 @@ def _signin_interactively( params.set_additional_parameter("msal_gui_thread", "true") # Since pymsalruntime 0.8.1 if enable_msa_pt: _enable_msa_pt(params) + if auth_scheme: + params.set_pop_params( + auth_scheme._http_method, auth_scheme._url.netloc, auth_scheme._url.path, + auth_scheme._nonce) for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. if v is not None: params.set_additional_parameter(k, str(v)) @@ -197,6 +211,7 @@ def _signin_interactively( def _acquire_token_silently( authority, client_id, account_id, scopes, claims=None, correlation_id=None, + auth_scheme=None, **kwargs): # For MSA PT scenario where you use the /organizations, yes, # acquireTokenSilently is expected to fail. - Sam Wilson @@ -208,6 +223,10 @@ def _acquire_token_silently( params.set_requested_scopes(scopes) if claims: params.set_decoded_claims(claims) + if auth_scheme: + params.set_pop_params( + auth_scheme._http_method, auth_scheme._url.netloc, auth_scheme._url.path, + auth_scheme._nonce) for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. if v is not None: params.set_additional_parameter(k, str(v)) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 36e7a445..ace95a53 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1,4 +1,4 @@ -"""If the following ENV VAR are available, many end-to-end test cases would run. +"""If the following ENV VAR were available, many end-to-end test cases would run. LAB_APP_CLIENT_SECRET=... LAB_OBO_CLIENT_SECRET=... LAB_APP_CLIENT_ID=... @@ -27,10 +27,23 @@ import msal from tests.http_client import MinimalHttpClient, MinimalResponse from msal.oauth2cli import AuthCodeReceiver +from msal.oauth2cli.oidc import decode_part +try: + import pymsalruntime + broker_available = True +except ImportError: + broker_available = False logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG if "-v" in sys.argv else logging.INFO) +try: + from dotenv import load_dotenv # Use this only in local dev machine + load_dotenv() # take environment variables from .env. +except ImportError: + logger.warn("Run pip install -r requirements.txt for optional dependency") + +_AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" def _get_app_and_auth_code( client_id, @@ -93,7 +106,7 @@ def assertLoosely(self, response, assertion=None, assertion() def assertCacheWorksForUser( - self, result_from_wire, scope, username=None, data=None): + self, result_from_wire, scope, username=None, data=None, auth_scheme=None): logger.debug( "%s: cache = %s, id_token_claims = %s", self.id(), @@ -109,35 +122,34 @@ def assertCacheWorksForUser( set(scope) <= set(result_from_wire["scope"].split(" ")) ): # Going to test acquire_token_silent(...) to locate an AT from cache - result_from_cache = self.app.acquire_token_silent( - scope, account=account, data=data or {}) - self.assertIsNotNone(result_from_cache) + silent_result = self.app.acquire_token_silent( + scope, account=account, data=data or {}, auth_scheme=auth_scheme) + self.assertIsNotNone(silent_result) self.assertIsNone( - result_from_cache.get("refresh_token"), "A cache hit returns no RT") - self.assertEqual( - result_from_wire['access_token'], result_from_cache['access_token'], - "We should get a cached AT") + silent_result.get("refresh_token"), "acquire_token_silent() should return no RT") + if auth_scheme: + self.assertNotEqual( + self.app._TOKEN_SOURCE_CACHE, silent_result[self.app._TOKEN_SOURCE]) + else: + self.assertEqual( + self.app._TOKEN_SOURCE_CACHE, silent_result[self.app._TOKEN_SOURCE]) if "refresh_token" in result_from_wire: + assert auth_scheme is None # Going to test acquire_token_silent(...) to obtain an AT by a RT from cache self.app.token_cache._cache["AccessToken"] = {} # A hacky way to clear ATs - result_from_cache = self.app.acquire_token_silent( - scope, account=account, data=data or {}) - if "refresh_token" not in result_from_wire: - self.assertEqual( - result_from_cache["access_token"], result_from_wire["access_token"], - "The previously cached AT should be returned") - self.assertIsNotNone(result_from_cache, + silent_result = self.app.acquire_token_silent( + scope, account=account, data=data or {}) + self.assertIsNotNone(silent_result, "We should get a result from acquire_token_silent(...) call") - self.assertIsNotNone( - # We used to assert it this way: - # result_from_wire['access_token'] != result_from_cache['access_token'] - # but ROPC in B2C tends to return the same AT we obtained seconds ago. - # Now looking back, "refresh_token grant would return a brand new AT" - # was just an empirical observation but never a commitment in specs, - # so we adjust our way to assert here. - (result_from_cache or {}).get("access_token"), - "We should get an AT from acquire_token_silent(...) call") + self.assertEqual( + # We used to assert it this way: + # result_from_wire['access_token'] != silent_result['access_token'] + # but ROPC in B2C tends to return the same AT we obtained seconds ago. + # Now looking back, "refresh_token grant would return a brand new AT" + # was just an empirical observation but never a commitment in specs, + # so we adjust our way to assert here. + self.app._TOKEN_SOURCE_IDP, silent_result[self.app._TOKEN_SOURCE]) def assertCacheWorksForApp(self, result_from_wire, scope): logger.debug( @@ -150,11 +162,9 @@ def assertCacheWorksForApp(self, result_from_wire, scope): self.app.acquire_token_silent(scope, account=None), "acquire_token_silent(..., account=None) shall always return None") # Going to test acquire_token_for_client(...) to locate an AT from cache - result_from_cache = self.app.acquire_token_for_client(scope) - self.assertIsNotNone(result_from_cache) - self.assertEqual( - result_from_wire['access_token'], result_from_cache['access_token'], - "We should get a cached AT") + silent_result = self.app.acquire_token_for_client(scope) + self.assertIsNotNone(silent_result) + self.assertEqual(self.app._TOKEN_SOURCE_CACHE, silent_result[self.app._TOKEN_SOURCE]) @classmethod def _build_app(cls, @@ -192,6 +202,7 @@ def _test_username_password(self, client_secret=None, # Since MSAL 1.11, confidential client has ROPC too azure_region=None, http_client=None, + auth_scheme=None, **ignored): assert authority and client_id and username and password and scope self.app = self._build_app( @@ -203,12 +214,14 @@ def _test_username_password(self, self.assertEqual( self.app.get_accounts(username=username), [], "Cache starts empty") result = self.app.acquire_token_by_username_password( - username, password, scopes=scope) + username, password, scopes=scope, auth_scheme=auth_scheme) self.assertLoosely(result) self.assertCacheWorksForUser( result, scope, username=username, # Our implementation works even when "profile" scope was not requested, or when profile claims is unavailable in B2C + auth_scheme=auth_scheme, ) + return result @unittest.skipIf( os.getenv("TRAVIS"), # It is set when running on TravisCI or Github Actions @@ -246,6 +259,7 @@ def _test_acquire_token_interactive( data=None, # Needed by ssh-cert feature prompt=None, enable_msa_passthrough=None, + auth_scheme=None, **ignored): assert client_id and authority and scope self.app = self._build_app(client_id, authority=authority) @@ -266,6 +280,7 @@ def _test_acquire_token_interactive( """.format(id=self.id(), hint=_get_hint( html_mode=True, username=username, lab_name=lab_name, username_uri=username_uri)), + auth_scheme=auth_scheme, data=data or {}, ) self.assertIn( @@ -279,7 +294,8 @@ def _test_acquire_token_interactive( username, result["id_token_claims"]["preferred_username"], "You are expected to sign in as account {}, but tokens returned is for {}".format( username, result["id_token_claims"]["preferred_username"])) - self.assertCacheWorksForUser(result, scope, username=None, data=data or {}) + self.assertCacheWorksForUser( + result, scope, username=None, data=data or {}, auth_scheme=auth_scheme) return result # For further testing @@ -1147,5 +1163,117 @@ def test_acquire_token_silent_with_an_empty_cache_should_return_none(self): # If this test case passes without exception, # it means MSAL Python is not affected by that. + +@unittest.skipUnless(broker_available, "AT POP feature is only supported by using broker") +class PopTestCase(LabBasedTestCase): + def test_at_pop_should_contain_pop_scheme_content(self): + auth_scheme = msal.PopAuthScheme( + http_method=msal.PopAuthScheme.HTTP_GET, + url="https://www.Contoso.com/Path1/Path2?queryParam1=a&queryParam2=b", + nonce="placeholder", + ) + result = self._test_acquire_token_interactive( + # Lab test users tend to get kicked out from WAM, we use local user to test + client_id=_AZURE_CLI, + authority="https://login.microsoftonline.com/organizations", + scope=["https://management.azure.com/.default"], + auth_scheme=auth_scheme, + ) # It also tests assertCacheWorksForUser() + self.assertEqual(result["token_source"], "broker", "POP is only supported by broker") + self.assertEqual(result["token_type"], "pop") + payload = json.loads(decode_part(result["access_token"].split(".")[1])) + logger.debug("AT POP payload = %s", json.dumps(payload, indent=2)) + self.assertEqual(payload["m"], auth_scheme._http_method) + self.assertEqual(payload["u"], auth_scheme._url.netloc) + self.assertEqual(payload["p"], auth_scheme._url.path) + self.assertEqual(payload["nonce"], auth_scheme._nonce) + + # TODO: Remove this, as ROPC support is removed by Broker-on-Win + def test_at_pop_via_testingsts_service(self): + """Based on https://testingsts.azurewebsites.net/ServerNonce""" + self.skipTest("ROPC support is removed by Broker-on-Win") + auth_scheme = msal.PopAuthScheme( + http_method="POST", + url="https://www.Contoso.com/Path1/Path2?queryParam1=a&queryParam2=b", + nonce=requests.get( + # TODO: Could use ".../missing" and then parse its WWW-Authenticate header + "https://testingsts.azurewebsites.net/servernonce/get").text, + ) + config = self.get_lab_user(usertype="cloud") + config["password"] = self.get_lab_user_secret(config["lab_name"]) + result = self._test_username_password(auth_scheme=auth_scheme, **config) + self.assertEqual(result["token_type"], "pop") + shr = result["access_token"] + payload = json.loads(decode_part(result["access_token"].split(".")[1])) + logger.debug("AT POP payload = %s", json.dumps(payload, indent=2)) + self.assertEqual(payload["m"], auth_scheme._http_method) + self.assertEqual(payload["u"], auth_scheme._url.netloc) + self.assertEqual(payload["p"], auth_scheme._url.path) + self.assertEqual(payload["nonce"], auth_scheme._nonce) + + validation = requests.post( + # TODO: This endpoint does not seem to validate the url + "https://testingsts.azurewebsites.net/servernonce/validateshr", + data={"SHR": shr}, + ) + self.assertEqual(validation.status_code, 200) + + def test_at_pop_calling_pattern(self): + # The calling pattern was described here: + # https://identitydivision.visualstudio.com/DevEx/_git/AuthLibrariesApiReview?path=/PoPTokensProtocol/PoP_API_In_MSAL.md&_a=preview&anchor=proposal-2---optional-isproofofposessionsupportedbyclient-helper-(accepted) + + # It is supposed to call app.is_pop_supported() first, + # and then fallback to bearer token code path. + # We skip it here because this test case has not yet initialize self.app + # assert self.app.is_pop_supported() + api_endpoint = "https://20.190.132.47/beta/me" + resp = requests.get(api_endpoint, verify=False) + self.assertEqual(resp.status_code, 401, "Initial call should end with an http 401 error") + result = self._get_shr_pop(**dict( + self.get_lab_user(usertype="cloud"), # This is generally not the current laptop's default AAD account + scope=["https://graph.microsoft.com/.default"], + auth_scheme=msal.PopAuthScheme( + http_method=msal.PopAuthScheme.HTTP_GET, + url=api_endpoint, + nonce=self._extract_pop_nonce(resp.headers.get("WWW-Authenticate")), + ), + )) + resp = requests.get(api_endpoint, verify=False, headers={ + "Authorization": "pop {}".format(result["access_token"]), + }) + if resp.status_code != 200: + # TODO https://teams.microsoft.com/l/message/19:b1697a70b1de43ddaea281d98ff2e985@thread.v2/1700184847801?context=%7B%22contextType%22%3A%22chat%22%7D + self.skipTest("We haven't got this end-to-end test case working") + self.assertEqual(resp.status_code, 200, "POP resource should be accessible") + + def _extract_pop_nonce(self, www_authenticate): + # This is a hack for testing purpose only. Do not use this in prod. + # FYI: There is a www-authenticate package but it falters when encountering realm="" + import re + found = re.search(r'nonce="(.+?)"', www_authenticate) + if found: + return found.group(1) + + def _get_shr_pop( + self, client_id=None, authority=None, scope=None, auth_scheme=None, + **kwargs): + result = self._test_acquire_token_interactive( + # Lab test users tend to get kicked out from WAM, we use local user to test + client_id=client_id, + authority=authority, + scope=scope, + auth_scheme=auth_scheme, + **kwargs) # It also tests assertCacheWorksForUser() + self.assertEqual(result["token_source"], "broker", "POP is only supported by broker") + self.assertEqual(result["token_type"], "pop") + payload = json.loads(decode_part(result["access_token"].split(".")[1])) + logger.debug("AT POP payload = %s", json.dumps(payload, indent=2)) + self.assertEqual(payload["m"], auth_scheme._http_method) + self.assertEqual(payload["u"], auth_scheme._url.netloc) + self.assertEqual(payload["p"], auth_scheme._url.path) + self.assertEqual(payload["nonce"], auth_scheme._nonce) + return result + + if __name__ == "__main__": unittest.main() From 35310b5953b4524461ddb38b82bec93ee535cad0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 29 Nov 2023 00:46:38 -0800 Subject: [PATCH 106/262] Prepare 1.26 release --- msal/application.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/msal/application.py b/msal/application.py index a7ae7bc2..49de7cda 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.25.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.26.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" @@ -2201,8 +2201,7 @@ def acquire_token_on_behalf_of(self, user_assertion, scopes, claims_challenge=No """ telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_ON_BEHALF_OF_ID) - # The implementation is NOT based on Token Exchange - # https://tools.ietf.org/html/draft-ietf-oauth-token-exchange-16 + # The implementation is NOT based on Token Exchange (RFC 8693) response = _clean_up(self.client.obtain_token_by_assertion( # bases on assertion RFC 7521 user_assertion, self.client.GRANT_TYPE_JWT, # IDTs and AAD ATs are all JWTs From f7b3f4e07fa139528f9af8c7a2d0ae040719a0af Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 15 May 2023 15:18:13 -0700 Subject: [PATCH 107/262] Integrate with PyMsalRuntime on mac Use new enable_broker Use 2 flags, one per supported platform Documents the requirement on parent_window_handle --- msal/__main__.py | 1 + msal/application.py | 47 +++++++++++++++++++++++++++++------- msal/broker.py | 37 ++++++++++++++-------------- sample/interactive_sample.py | 5 +++- setup.cfg | 6 ++--- tests/test_e2e.py | 1 + 6 files changed, 66 insertions(+), 31 deletions(-) diff --git a/msal/__main__.py b/msal/__main__.py index aeb123b0..dc083fb0 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -255,6 +255,7 @@ def _main(): accept_nonempty_string=True, ), enable_broker_on_windows=enable_broker, + enable_broker_on_mac=enable_broker, enable_pii_log=enable_pii_log, token_cache=global_cache, ) diff --git a/msal/application.py b/msal/application.py index 49de7cda..c69a4be8 100644 --- a/msal/application.py +++ b/msal/application.py @@ -187,6 +187,8 @@ class ClientApplication(object): "You can enable broker by following these instructions. " "https://msal-python.readthedocs.io/en/latest/#publicclientapplication") + _enable_broker = False + def __init__( self, client_id, client_credential=None, authority=None, validate_authority=True, @@ -540,7 +542,9 @@ def _decide_broker(self, allow_broker, enable_pii_log): if allow_broker: warnings.warn( "allow_broker is deprecated. " - "Please use PublicClientApplication(..., enable_broker_on_windows=True)", + "Please use PublicClientApplication(..., " + "enable_broker_on_windows=True, " + "enable_broker_on_mac=...)", DeprecationWarning) self._enable_broker = self._enable_broker or ( # When we started the broker project on Windows platform, @@ -1646,7 +1650,8 @@ def acquire_token_by_username_password( """ claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) - if False: # Disabled, for now. It was if self._enable_broker: + if False: # Disabled, for now. It was if self._enable_broker and sys.platform != "darwin": + # _signin_silently() won't work on Mac. We may revisit on whether it shall work on Windows. from .broker import _signin_silently response = _signin_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), @@ -1744,7 +1749,7 @@ def __init__(self, client_id, client_credential=None, **kwargs): .. note:: - You may set enable_broker_on_windows to True. + You may set enable_broker_on_windows and/or enable_broker_on_mac to True. What is a broker, and why use it? @@ -1773,9 +1778,11 @@ def __init__(self, client_id, client_credential=None, **kwargs): * ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` if your app is expected to run on Windows 10+ + * ``msauth.com.msauth.unsignedapp://auth`` + if your app is expected to run on Mac 2. installed broker dependency, - e.g. ``pip install msal[broker]>=1.25,<2``. + e.g. ``pip install msal[broker]>=1.27.0b1,<2``. 3. tested with ``acquire_token_interactive()`` and ``acquire_token_silent()``. @@ -1784,12 +1791,21 @@ def __init__(self, client_id, client_credential=None, **kwargs): This parameter defaults to None, which means MSAL will not utilize a broker. New in MSAL Python 1.25.0. + + :param boolean enable_broker_on_mac: + This setting is only effective if your app is running on Mac. + This parameter defaults to None, which means MSAL will not utilize a broker. + + New in MSAL Python 1.27.0. """ if client_credential is not None: raise ValueError("Public Client should not possess credentials") # Using kwargs notation for now. We will switch to keyword-only arguments. enable_broker_on_windows = kwargs.pop("enable_broker_on_windows", False) - self._enable_broker = enable_broker_on_windows and sys.platform == "win32" + enable_broker_on_mac = kwargs.pop("enable_broker_on_mac", False) + self._enable_broker = bool( + enable_broker_on_windows and sys.platform == "win32" + or enable_broker_on_mac and sys.platform == "darwin") super(PublicClientApplication, self).__init__( client_id, client_credential=None, **kwargs) @@ -1867,10 +1883,23 @@ def acquire_token_interactive( New in version 1.15. :param int parent_window_handle: - OPTIONAL. If your app is a GUI app running on modern Windows system, - and your app opts in to use broker, - you are recommended to also provide its window handle, - so that the sign in UI window will properly pop up on top of your window. + OPTIONAL. + + * If your app does not opt in to use broker, + you do not need to provide a ``parent_window_handle`` here. + + * If your app opts in to use broker, + ``parent_window_handle`` is required. + + - If your app is a GUI app running on modern Windows system, + you are required to also provide its window handle, + so that the sign-in window will pop up on top of your window. + - If your app is a console app runnong on Windows system, + you can use a placeholder + ``PublicClientApplication.CONSOLE_WINDOW_HANDLE``. + - If your app is running on Mac, + you can use a placeholder + ``PublicClientApplication.CONSOLE_WINDOW_HANDLE``. New in version 1.20.0. diff --git a/msal/broker.py b/msal/broker.py index a0904199..74b96543 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -1,7 +1,6 @@ """This module is an adaptor to the underlying broker. It relies on PyMsalRuntime which is the package providing broker's functionality. """ -from threading import Event import json import logging import time @@ -35,14 +34,12 @@ class TokenTypeError(ValueError): pass -class _CallbackData: - def __init__(self): - self.signal = Event() - self.result = None - - def complete(self, result): - self.signal.set() - self.result = result +_redirect_uri_on_mac = "msauth.com.msauth.unsignedapp://auth" # Note: + # On Mac, the native Python has a team_id which links to bundle id + # com.apple.python3 however it won't give Python scripts better security. + # Besides, the homebrew-installed Pythons have no team_id + # so they have to use a generic placeholder anyway. + # The v-team chose to combine two situations into using same placeholder. def _convert_error(error, client_id): @@ -52,8 +49,9 @@ def _convert_error(error, client_id): or "AADSTS7000218" in context # This "request body must contain ... client_secret" is just a symptom of current app has no WAM redirect_uri ): raise RedirectUriError( # This would be seen by either the app developer or end user - "MsalRuntime won't work unless this one more redirect_uri is registered to current app: " - "ms-appx-web://Microsoft.AAD.BrokerPlugin/{}".format(client_id)) + "MsalRuntime needs the current app to register these redirect_uri " + "(1) ms-appx-web://Microsoft.AAD.BrokerPlugin/{} (2) {}".format( + client_id, _redirect_uri_on_mac)) # OTOH, AAD would emit other errors when other error handling branch was hit first, # so, the AADSTS50011/RedirectUriError is not guaranteed to happen. return { @@ -70,8 +68,8 @@ def _convert_error(error, client_id): def _read_account_by_id(account_id, correlation_id): - """Return an instance of MSALRuntimeAccount, or log error and return None""" - callback_data = _CallbackData() + """Return an instance of MSALRuntimeError or MSALRuntimeAccount, or None""" + callback_data = pymsalruntime.CallbackData() pymsalruntime.read_account_by_id( account_id, correlation_id, @@ -142,7 +140,7 @@ def _signin_silently( params.set_pop_params( auth_scheme._http_method, auth_scheme._url.netloc, auth_scheme._url.path, auth_scheme._nonce) - callback_data = _CallbackData() + callback_data = pymsalruntime.CallbackData() for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. if v is not None: params.set_additional_parameter(k, str(v)) @@ -169,8 +167,11 @@ def _signin_interactively( **kwargs): params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) params.set_requested_scopes(scopes) - params.set_redirect_uri("placeholder") # pymsalruntime 0.1 requires non-empty str, + params.set_redirect_uri( + # pymsalruntime on Windows requires non-empty str, # the actual redirect_uri will be overridden by a value hardcoded by the broker + _redirect_uri_on_mac, + ) if prompt: if prompt == "select_account": if login_hint: @@ -197,7 +198,7 @@ def _signin_interactively( params.set_additional_parameter(k, str(v)) if claims: params.set_decoded_claims(claims) - callback_data = _CallbackData() + callback_data = pymsalruntime.CallbackData(is_interactive=True) pymsalruntime.signin_interactively( parent_window_handle or pymsalruntime.get_console_window() or pymsalruntime.get_desktop_window(), # Since pymsalruntime 0.2+ params, @@ -230,7 +231,7 @@ def _acquire_token_silently( for k, v in kwargs.items(): # This can be used to support domain_hint, max_age, etc. if v is not None: params.set_additional_parameter(k, str(v)) - callback_data = _CallbackData() + callback_data = pymsalruntime.CallbackData() pymsalruntime.acquire_token_silently( params, correlation_id, @@ -246,7 +247,7 @@ def _signout_silently(client_id, account_id, correlation_id=None): account = _read_account_by_id(account_id, correlation_id) if account is None: return - callback_data = _CallbackData() + callback_data = pymsalruntime.CallbackData() pymsalruntime.signout_silently( # New in PyMsalRuntime 0.7 client_id, correlation_id, diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index f283ed29..b4e77309 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -37,8 +37,11 @@ # Create a preferably long-lived app instance, to avoid the overhead of app creation global_app = msal.PublicClientApplication( config["client_id"], authority=config["authority"], + + # You may opt in to use broker. See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition #enable_broker_on_windows=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already - # See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition + #enable_broker_on_mac=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already + token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) diff --git a/setup.cfg b/setup.cfg index 75df4f9d..800311b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,9 +63,9 @@ broker = # The broker is defined as optional dependency, # so that downstream apps can opt in. The opt-in is needed, partially because # most existing MSAL Python apps do not have the redirect_uri needed by broker. - # MSAL Python uses a subset of API from PyMsalRuntime 0.13.0+, - # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) - pymsalruntime>=0.13.2,<0.14; python_version>='3.6' and platform_system=='Windows' + # We need pymsalruntime.CallbackData introduced in PyMsalRuntime 0.14 + pymsalruntime>=0.14,<0.15; python_version>='3.6' and platform_system=='Windows' + pymsalruntime>=0.14,<0.15; python_version>='3.8' and platform_system=='Darwin' [options.packages.find] exclude = diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ace95a53..28833ce5 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -195,6 +195,7 @@ def _build_app(cls, authority=authority, http_client=http_client or MinimalHttpClient(), enable_broker_on_windows=broker_available, + enable_broker_on_mac=broker_available, ) def _test_username_password(self, From 77f64f4f610045b55c973e99bfce3b197fcff1f5 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 9 Aug 2023 11:26:01 -0700 Subject: [PATCH 108/262] How to smoke test MSAL Python --- tests/smoke-test.md | 68 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/smoke-test.md diff --git a/tests/smoke-test.md b/tests/smoke-test.md new file mode 100644 index 00000000..a0d35daf --- /dev/null +++ b/tests/smoke-test.md @@ -0,0 +1,68 @@ +# How to Smoke Test MSAL Python + +The experimental `python -m msal` usage is designed to be an interactive tool, +which can impersonate arbitrary apps and test most of the MSAL Python APIs. +Note that MSAL Python API's behavior is modeled after OIDC behavior in browser, +which are not exactly the same as the broker API's behavior, +despite that the two sets of API happen to have similar names. + +Tokens acquired during the tests will be cached by MSAL Python. +MSAL Python uses an in-memory token cache by default. +This test tool, however, saves a token cache snapshot on disk upon each exit, +and you may choose to reuse it or start afresh during start up. + +Typical test cases are listed below. + +1. The tool starts with an empty token cache. + In this state, acquire_token_silent() shall always return empty result. + +2. When testing with broker, apps would need to register a certain redirect_uri + for the test cases below to work. + We will also test an app without the required redirect_uri registration, + MSAL Python shall return a meaningful error message on what URIs to register. + +3. Interactive acquire_token_interactive() shall get a token. In particular, + + * The prompt=none option shall succeed when there is a default account, + or error out otherwise. + * The prompt=select_account option shall always prompt with an account picker. + * The prompt=absent option shall prompt an account picker UI + if there are multiple accounts available in browser + and none of them is considered a default account. + In such a case, an optional login_hint=`one_of_the_account@contoso.com` + shall bypass the account picker. + + With a broker, the behavior shall largely match the browser behavior, + unless stated otherwise below. + + * Broker (PyMsalRuntime) on Mac does not support silent signin, + so the prompt=absent will also always prompt. + +4. ROPC (Resource Owner Password Credential, a.k.a. the username password flow). + The acquire_token_by_username_password() is supported by broker on Windows. + As of Oct 2023, it is not yet supported by broker on Mac, + so it will fall back to non-broker behavior. + +5. After step 3 or 4, the acquire_token_silently() shall return a token fast, + because that is the same token returned by step 3 or 4, cached in MSAL Python. + We shall also retest this with the force_refresh=True, + a new token shall be obtained, + typically slower than a token served from MSAL Python's token cache. + +6. POP token. + POP token is supported via broker. + This tool test the POP token by using a hardcoded Signed Http Request (SHR). + A test is successful if the POP test function return a token with type as POP. + +7. SSH Cert. + The interactive test and silent test shall behave similarly to + their non ssh-cert counterparts, only the `token_type` would be different. + +8. Test the remove_account() API. It shall always be successful. + This effectively signs out an account from MSAL Python, + we can confirm that by running acquire_token_silent() + and see that account was gone. + + The remove_account() shall also sign out from broker (if broker was enabled), + it does not sign out account from browser (even when browser was used). + From 21a625c195aa1d48bce04703fe67ff8e1c682077 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 8 Dec 2023 17:15:02 -0800 Subject: [PATCH 109/262] Preparing MSAL Python 1.27.0 beta release(s) --- msal/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index c69a4be8..2901fabc 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.26.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.27.0b2" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" @@ -1782,7 +1782,7 @@ def __init__(self, client_id, client_credential=None, **kwargs): if your app is expected to run on Mac 2. installed broker dependency, - e.g. ``pip install msal[broker]>=1.27.0b1,<2``. + e.g. ``pip install msal[broker]>=1.27.0b2,<2``. 3. tested with ``acquire_token_interactive()`` and ``acquire_token_silent()``. From 1ae2d19c6e5f8544531566ac74d0422baa0a4a8a Mon Sep 17 00:00:00 2001 From: Bogdan Gavril Date: Wed, 20 Dec 2023 15:40:14 +0000 Subject: [PATCH 110/262] Update issue templates (#642) * Update issue templates * Update feature_request.md * Update feature_request.md * Remove excess spaces, and rename .md to .yaml --------- Co-authored-by: Ray Luo --- .github/ISSUE_TEMPLATE/bug_report.md | 4 +-- .github/ISSUE_TEMPLATE/feature_request.yaml | 40 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8d823e82..cbd8381f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,8 +1,8 @@ --- name: Bug report about: Create a report to help us improve -title: '[Bug] ' -labels: ["untriaged", "needs attention"] +title: "[Bug] " +labels: needs attention, untriaged assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 00000000..ddc73b5c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,40 @@ +name: Feature request +description: Suggest a new feature for MSAL Python. +labels: ["feature request", "untriaged", "needs attention"] +title : '[Feature Request] ' +body: +- type: markdown + attributes: + value: | + ## Before submitting your feature request + Please make sure that your question or issue is not already covered in [MSAL documentation](https://learn.microsoft.com/entra/msal/python/) or [samples](https://learn.microsoft.com/azure/active-directory/develop/sample-v2-code?tabs=apptype). + +- type: markdown + attributes: + value: | + ## Feature request for MSAL Python + +- type: dropdown + attributes: + label: MSAL client type + description: Are you using Public Client (desktop apps, CLI apps) or Confidential Client (web apps, web APIs, service-to-service, managed identity)? + multiple: true + options: + - "Public" + - "Confidential" + validations: + required: true + +- type: textarea + attributes: + label: Problem Statement + description: "Describe the problem or context for this feature request." + validations: + required: true + +- type: textarea + attributes: + label: Proposed solution + description: "Describe the solution you'd like." + validations: + required: false From 72b853d91e1b70844fde98eb1135b0dac93ff52f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 3 Dec 2023 15:40:27 -0800 Subject: [PATCH 111/262] No more gibberish log from https request to mess up the current terminal --- oauth2cli/authcode.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/oauth2cli/authcode.py b/oauth2cli/authcode.py index 5d465288..ba266223 100644 --- a/oauth2cli/authcode.py +++ b/oauth2cli/authcode.py @@ -102,6 +102,11 @@ def _escape(key_value_pairs): return {k: escape(v) for k, v in key_value_pairs.items()} +def _printify(text): + # If an https request is sent to an http server, the text needs to be repr-ed + return repr(text) if isinstance(text, str) and not text.isprintable() else text + + class _AuthCodeHandler(BaseHTTPRequestHandler): def do_GET(self): # For flexibility, we choose to not check self.path matching redirect_uri @@ -135,7 +140,8 @@ def _send_full_response(self, body, is_ok=True): self.wfile.write(body.encode("utf-8")) def log_message(self, format, *args): - logger.debug(format, *args) # To override the default log-to-stderr behavior + # To override the default log-to-stderr behavior + logger.debug(format, *map(_printify, args)) class _AuthCodeHttpServer(HTTPServer, object): From 866ba2b725bcbd49add91d777d0469d7019dceb0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 28 Dec 2023 11:45:39 -0800 Subject: [PATCH 112/262] AT POP with SHR is tested with Graph end-to-end --- tests/test_e2e.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ace95a53..33c8cf54 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1241,9 +1241,6 @@ def test_at_pop_calling_pattern(self): resp = requests.get(api_endpoint, verify=False, headers={ "Authorization": "pop {}".format(result["access_token"]), }) - if resp.status_code != 200: - # TODO https://teams.microsoft.com/l/message/19:b1697a70b1de43ddaea281d98ff2e985@thread.v2/1700184847801?context=%7B%22contextType%22%3A%22chat%22%7D - self.skipTest("We haven't got this end-to-end test case working") self.assertEqual(resp.status_code, 200, "POP resource should be accessible") def _extract_pop_nonce(self, www_authenticate): From c1a0ce19f6c1d3252db2e0b38e7e6381a17568cd Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 1 Jan 2024 21:50:36 -0800 Subject: [PATCH 113/262] Sort scopes before writing to token cache --- msal/token_cache.py | 2 +- tests/test_token_cache.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index bd6d8a6f..da67e078 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -160,7 +160,7 @@ def __add(self, event, now=None): decode_id_token(id_token, client_id=event["client_id"]) if id_token else {}) client_info, home_account_id = self.__parse_account(response, id_token_claims) - target = ' '.join(event.get("scope") or []) # Per schema, we don't sort it + target = ' '.join(sorted(event.get("scope") or [])) # Schema should have required sorting with self._lock: now = int(time.time() if now is None else now) diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py index 2fe486c2..94bf4969 100644 --- a/tests/test_token_cache.py +++ b/tests/test_token_cache.py @@ -76,11 +76,11 @@ def testAddByAad(self): 'home_account_id': "uid.utid", 'realm': 'contoso', 'secret': 'an access token', - 'target': 's2 s1 s3', + 'target': 's1 s2 s3', # Sorted 'token_type': 'some type', }, self.cache._cache["AccessToken"].get( - 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s2 s1 s3') + 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3') ) self.assertEqual( { @@ -90,10 +90,10 @@ def testAddByAad(self): 'home_account_id': "uid.utid", 'last_modification_time': '1000', 'secret': 'a refresh token', - 'target': 's2 s1 s3', + 'target': 's1 s2 s3', # Sorted }, self.cache._cache["RefreshToken"].get( - 'uid.utid-login.example.com-refreshtoken-my_client_id--s2 s1 s3') + 'uid.utid-login.example.com-refreshtoken-my_client_id--s1 s2 s3') ) self.assertEqual( { @@ -150,11 +150,11 @@ def testAddByAdfs(self): 'home_account_id': "subject", 'realm': 'adfs', 'secret': 'an access token', - 'target': 's2 s1 s3', + 'target': 's1 s2 s3', # Sorted 'token_type': 'some type', }, self.cache._cache["AccessToken"].get( - 'subject-fs.msidlab8.com-accesstoken-my_client_id-adfs-s2 s1 s3') + 'subject-fs.msidlab8.com-accesstoken-my_client_id-adfs-s1 s2 s3') ) self.assertEqual( { @@ -164,10 +164,10 @@ def testAddByAdfs(self): 'home_account_id': "subject", 'last_modification_time': "1000", 'secret': 'a refresh token', - 'target': 's2 s1 s3', + 'target': 's1 s2 s3', # Sorted }, self.cache._cache["RefreshToken"].get( - 'subject-fs.msidlab8.com-refreshtoken-my_client_id--s2 s1 s3') + 'subject-fs.msidlab8.com-refreshtoken-my_client_id--s1 s2 s3') ) self.assertEqual( { @@ -214,7 +214,7 @@ def test_key_id_is_also_recorded(self): refresh_token="a refresh token"), }, now=1000) cached_key_id = self.cache._cache["AccessToken"].get( - 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s2 s1 s3', + 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3', {}).get("key_id") self.assertEqual(my_key_id, cached_key_id, "AT should be bound to the key") @@ -229,7 +229,7 @@ def test_refresh_in_should_be_recorded_as_refresh_on(self): # Sounds weird. Yep ), #refresh_token="a refresh token"), }, now=1000) refresh_on = self.cache._cache["AccessToken"].get( - 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s2 s1 s3', + 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3', {}).get("refresh_on") self.assertEqual("2800", refresh_on, "Should save refresh_on") From 313d7219c9b098d727ce3b66cdf0019c3fc74c5f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 4 Jan 2024 23:26:33 -0800 Subject: [PATCH 114/262] O(1) happy path for access token hits --- msal/application.py | 11 +++++---- msal/token_cache.py | 56 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/msal/application.py b/msal/application.py index 49de7cda..89d0aecb 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1357,13 +1357,14 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( key_id = kwargs.get("data", {}).get("key_id") if key_id: # Some token types (SSH-certs, POP) are bound to a key query["key_id"] = key_id - matches = self.token_cache.find( - self.token_cache.CredentialType.ACCESS_TOKEN, - target=scopes, - query=query) now = time.time() refresh_reason = msal.telemetry.AT_ABSENT - for entry in matches: + for entry in self.token_cache._find( # It returns a generator + self.token_cache.CredentialType.ACCESS_TOKEN, + target=scopes, + query=query, + ): # Note that _find() holds a lock during this for loop; + # that is fine because this loop is fast expires_in = int(entry["expires_on"]) - now if expires_in < 5*60: # Then consider it expired refresh_reason = msal.telemetry.AT_EXPIRED diff --git a/msal/token_cache.py b/msal/token_cache.py index da67e078..ae408a9c 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -88,20 +88,60 @@ def __init__(self): "appmetadata-{}-{}".format(environment or "", client_id or ""), } - def find(self, credential_type, target=None, query=None): - target = target or [] + def _get_access_token( + self, + home_account_id, environment, client_id, realm, target, # Together they form a compound key + default=None, + ): # O(1) + return self._get( + self.CredentialType.ACCESS_TOKEN, + self.key_makers[TokenCache.CredentialType.ACCESS_TOKEN]( + home_account_id=home_account_id, + environment=environment, + client_id=client_id, + realm=realm, + target=" ".join(target), + ), + default=default) + + def _get(self, credential_type, key, default=None): # O(1) + with self._lock: + return self._cache.get(credential_type, {}).get(key, default) + + def _find(self, credential_type, target=None, query=None): # O(n) generator + """Returns a generator of matching entries. + + It is O(1) for AT hits, and O(n) for other types. + Note that it holds a lock during the entire search. + """ + target = sorted(target or []) # Match the order sorted by add() assert isinstance(target, list), "Invalid parameter type" + + preferred_result = None + if (credential_type == self.CredentialType.ACCESS_TOKEN + and "home_account_id" in query and "environment" in query + and "client_id" in query and "realm" in query and target + ): # Special case for O(1) AT lookup + preferred_result = self._get_access_token( + query["home_account_id"], query["environment"], + query["client_id"], query["realm"], target) + if preferred_result: + yield preferred_result + target_set = set(target) with self._lock: # Since the target inside token cache key is (per schema) unsorted, # there is no point to attempt an O(1) key-value search here. # So we always do an O(n) in-memory search. - return [entry - for entry in self._cache.get(credential_type, {}).values() - if is_subdict_of(query or {}, entry) - and (target_set <= set(entry.get("target", "").split()) - if target else True) - ] + for entry in self._cache.get(credential_type, {}).values(): + if is_subdict_of(query or {}, entry) and ( + target_set <= set(entry.get("target", "").split()) + if target else True): + if entry != preferred_result: # Avoid yielding the same entry twice + yield entry + + def find(self, credential_type, target=None, query=None): # Obsolete. Use _find() instead. + return list(self._find(credential_type, target=target, query=query)) def add(self, event, now=None): """Handle a token obtaining event, and add tokens into cache.""" From 5272fbd8a86ca635f8af2662c8a3a0ce67b39f31 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 5 Jan 2024 23:26:01 -0800 Subject: [PATCH 115/262] Might as well refactor a _get_app_metadata() --- msal/application.py | 6 ++---- msal/token_cache.py | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/msal/application.py b/msal/application.py index 89d0aecb..afa08f5a 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1493,10 +1493,8 @@ def _acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( **kwargs) or last_resp def _get_app_metadata(self, environment): - apps = self.token_cache.find( # Use find(), rather than token_cache.get(...) - TokenCache.CredentialType.APP_METADATA, query={ - "environment": environment, "client_id": self.client_id}) - return apps[0] if apps else {} + return self.token_cache._get_app_metadata( + environment=environment, client_id=self.client_id, default={}) def _acquire_token_silent_by_finding_specific_refresh_token( self, authority, scopes, query, diff --git a/msal/token_cache.py b/msal/token_cache.py index ae408a9c..d19a1dbf 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -104,6 +104,15 @@ def _get_access_token( ), default=default) + def _get_app_metadata(self, environment, client_id, default=None): # O(1) + return self._get( + self.CredentialType.APP_METADATA, + self.key_makers[TokenCache.CredentialType.APP_METADATA]( + environment=environment, + client_id=client_id, + ), + default=default) + def _get(self, credential_type, key, default=None): # O(1) with self._lock: return self._cache.get(credential_type, {}).get(key, default) From 84bdfabfc3cbd73bca485b2420fcb7ccc01191d0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 9 Jan 2024 13:52:18 -0800 Subject: [PATCH 116/262] Prevent crash on token_cache.find(..., query=None) --- msal/token_cache.py | 1 + tests/test_token_cache.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index d19a1dbf..f9a55800 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -128,6 +128,7 @@ def _find(self, credential_type, target=None, query=None): # O(n) generator preferred_result = None if (credential_type == self.CredentialType.ACCESS_TOKEN + and isinstance(query, dict) and "home_account_id" in query and "environment" in query and "client_id" in query and "realm" in query and target ): # Special case for O(1) AT lookup diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py index 94bf4969..4e301fa3 100644 --- a/tests/test_token_cache.py +++ b/tests/test_token_cache.py @@ -65,8 +65,7 @@ def testAddByAad(self): expires_in=3600, access_token="an access token", id_token=id_token, refresh_token="a refresh token"), }, now=1000) - self.assertEqual( - { + access_token_entry = { 'cached_at': "1000", 'client_id': 'my_client_id', 'credential_type': 'AccessToken', @@ -78,10 +77,16 @@ def testAddByAad(self): 'secret': 'an access token', 'target': 's1 s2 s3', # Sorted 'token_type': 'some type', - }, + } + self.assertEqual( + access_token_entry, self.cache._cache["AccessToken"].get( 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3') ) + self.assertIn( + access_token_entry, + self.cache.find(self.cache.CredentialType.ACCESS_TOKEN), + "find(..., query=None) should not crash, even though MSAL does not use it") self.assertEqual( { 'client_id': 'my_client_id', From 49a919827ca8c799e6019039d1af39eb42e69d14 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 17 Jan 2024 22:08:53 -0800 Subject: [PATCH 117/262] Attempts account removal from broker first --- msal/application.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index afa08f5a..96277e4a 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1123,12 +1123,13 @@ def _get_authority_aliases(self, instance): def remove_account(self, account): """Sign me out and forget me from token cache""" - self._forget_me(account) if self._enable_broker: from .broker import _signout_silently error = _signout_silently(self.client_id, account["local_account_id"]) if error: logger.debug("_signout_silently() returns error: %s", error) + # Broker sign-out has been attempted, even if the _forget_me() below throws. + self._forget_me(account) def _sign_out(self, home_account): # Remove all relevant RTs and ATs from token cache From c131b9b57770de03cb82a32c871cca84a9f1162f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 19 Jan 2024 13:14:10 -0800 Subject: [PATCH 118/262] Adding docs for PopAuthScheme --- docs/index.rst | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index e608fe6b..2129e106 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,3 +1,4 @@ +========================= MSAL Python Documentation ========================= @@ -11,6 +12,8 @@ MSAL Python Documentation .. Comment: Perhaps because of the theme, only the first level sections will show in TOC, regardless of maxdepth setting. + UPDATE: And now (early 2024) suddenly a function-level, long TOC is generated, + even though maxdepth is set to 2. You can find high level conceptual documentations in the project `README `_. @@ -92,7 +95,7 @@ They are implemented as two separated classes, with different methods for different authentication scenarios. ClientApplication -================= +----------------- .. autoclass:: msal.ClientApplication :members: @@ -101,7 +104,7 @@ ClientApplication .. automethod:: __init__ PublicClientApplication -======================= +----------------------- .. autoclass:: msal.PublicClientApplication :members: @@ -109,14 +112,14 @@ PublicClientApplication .. automethod:: __init__ ConfidentialClientApplication -============================= +----------------------------- .. autoclass:: msal.ConfidentialClientApplication :members: TokenCache -========== +---------- One of the parameters accepted by both `PublicClientApplication` and `ConfidentialClientApplication` @@ -130,3 +133,18 @@ See `SerializableTokenCache` for example. .. autoclass:: msal.SerializableTokenCache :members: + + +PopAuthScheme +------------- + +This is used as the `auth_scheme` parameter in many of the acquire token methods +to support for Proof of Possession (PoP) tokens. + +New in MSAL Python 1.26 + +.. autoclass:: msal.PopAuthScheme + :members: + + .. automethod:: __init__ + From d7331f26c634240c3b1d1eeabeebc2b963c2597c Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 22 Jan 2024 01:14:34 -0800 Subject: [PATCH 119/262] Tested with latest cryptography 42.x --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 75df4f9d..d3b7e40d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,7 +54,7 @@ install_requires = # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=0.6,<44 + cryptography>=0.6,<45 mock; python_version<'3.3' From 3e68838f3aaffb658bf61f1f403f09d79eb4dbce Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 21 Feb 2023 17:54:00 -0800 Subject: [PATCH 120/262] Mention instance_discovery instead of validate_authority in an error message --- msal/__main__.py | 26 ++++++++++++++++---------- msal/authority.py | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/msal/__main__.py b/msal/__main__.py index aeb123b0..79776b8f 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -242,18 +242,24 @@ def _main(): enable_broker = _input_boolean("Enable broker? It will error out later if your app has not registered some redirect URI") enable_debug_log = _input_boolean("Enable MSAL Python's DEBUG log?") enable_pii_log = _input_boolean("Enable PII in broker's log?") if enable_broker and enable_debug_log else False + authority = _select_options([ + "https://login.microsoftonline.com/common", + "https://login.microsoftonline.com/organizations", + "https://login.microsoftonline.com/microsoft.onmicrosoft.com", + "https://login.microsoftonline.com/msidlab4.onmicrosoft.com", + "https://login.microsoftonline.com/consumers", + ], + header="Input authority (Note that MSA-PT apps would NOT use the /common authority)", + accept_nonempty_string=True, + ) + instance_discovery = _input_boolean( + "You input an unusual authority which might fail the Instance Discovery. " + "Now, do you want to perform Instance Discovery on your input authority?" + ) if not authority.startswith("https://login.microsoftonline.com") else None app = msal.PublicClientApplication( chosen_app["client_id"] if isinstance(chosen_app, dict) else chosen_app, - authority=_select_options([ - "https://login.microsoftonline.com/common", - "https://login.microsoftonline.com/organizations", - "https://login.microsoftonline.com/microsoft.onmicrosoft.com", - "https://login.microsoftonline.com/msidlab4.onmicrosoft.com", - "https://login.microsoftonline.com/consumers", - ], - header="Input authority (Note that MSA-PT apps would NOT use the /common authority)", - accept_nonempty_string=True, - ), + authority=authority, + instance_discovery=instance_discovery, enable_broker_on_windows=enable_broker, enable_pii_log=enable_pii_log, token_cache=global_cache, diff --git a/msal/authority.py b/msal/authority.py index ae3ebf74..923f3ffb 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -100,7 +100,7 @@ def __init__( "The authority you provided, %s, is not whitelisted. " "If it is indeed your legit customized domain name, " "you can turn off this check by passing in " - "validate_authority=False" + "instance_discovery=False" % authority_url) tenant_discovery_endpoint = payload['tenant_discovery_endpoint'] else: From d52459563defadffdb5fa25060d7c98593abaa87 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sun, 28 Jan 2024 15:30:22 -0800 Subject: [PATCH 121/262] Tolerate ID token time errors --- oauth2cli/__init__.py | 2 +- oauth2cli/oidc.py | 80 ++++++++++++++++++++++++++++++++++++------- tests/test_oidc.py | 21 ++++++++++++ 3 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 tests/test_oidc.py diff --git a/oauth2cli/__init__.py b/oauth2cli/__init__.py index 60bf2595..d9978726 100644 --- a/oauth2cli/__init__.py +++ b/oauth2cli/__init__.py @@ -1,6 +1,6 @@ __version__ = "0.4.0" -from .oidc import Client +from .oidc import Client, IdTokenError from .assertion import JwtAssertionCreator from .assertion import JwtSigner # Obsolete. For backward compatibility. from .authcode import AuthCodeReceiver diff --git a/oauth2cli/oidc.py b/oauth2cli/oidc.py index d4d3a927..01ee7894 100644 --- a/oauth2cli/oidc.py +++ b/oauth2cli/oidc.py @@ -5,9 +5,13 @@ import string import warnings import hashlib +import logging from . import oauth2 + +logger = logging.getLogger(__name__) + def decode_part(raw, encoding="utf-8"): """Decode a part of the JWT. @@ -32,6 +36,45 @@ def decode_part(raw, encoding="utf-8"): base64decode = decode_part # Obsolete. For backward compatibility only. +def _epoch_to_local(epoch): + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(epoch)) + +class IdTokenError(RuntimeError): # We waised RuntimeError before, so keep it + """In unlikely event of an ID token is malformed, this exception will be raised.""" + def __init__(self, reason, now, claims): + super(IdTokenError, self).__init__( + "%s Current epoch = %s. The id_token was approximately: %s" % ( + reason, _epoch_to_local(now), json.dumps(dict( + claims, + iat=_epoch_to_local(claims["iat"]) if claims.get("iat") else None, + exp=_epoch_to_local(claims["exp"]) if claims.get("exp") else None, + ), indent=2))) + +class _IdTokenTimeError(IdTokenError): # This is not intended to be raised and caught + _SUGGESTION = "Make sure your computer's time and time zone are both correct." + def __init__(self, reason, now, claims): + super(_IdTokenTimeError, self).__init__(reason+ " " + self._SUGGESTION, now, claims) + def log(self): + # Influenced by JWT specs https://tools.ietf.org/html/rfc7519#section-4.1.5 + # and OIDC specs https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation + # We used to raise this error, but now we just log it as warning, because: + # 1. If it is caused by incorrect local machine time, + # then the token(s) are still correct and probably functioning, + # so, there is no point to error out. + # 2. If it is caused by incorrect IdP time, then it is IdP's fault, + # There is not much a client can do, so, we might as well return the token(s) + # and let downstream components to decide what to do. + logger.warning(str(self)) + +class IdTokenIssuerError(IdTokenError): + pass + +class IdTokenAudienceError(IdTokenError): + pass + +class IdTokenNonceError(IdTokenError): + pass + def decode_id_token(id_token, client_id=None, issuer=None, nonce=None, now=None): """Decodes and validates an id_token and returns its claims as a dictionary. @@ -41,41 +84,52 @@ def decode_id_token(id_token, client_id=None, issuer=None, nonce=None, now=None) `maybe more `_ """ decoded = json.loads(decode_part(id_token.split('.')[1])) - err = None # https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation + # Based on https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation _now = int(now or time.time()) skew = 120 # 2 minutes - TIME_SUGGESTION = "Make sure your computer's time and time zone are both correct." + if _now + skew < decoded.get("nbf", _now - 1): # nbf is optional per JWT specs # This is not an ID token validation, but a JWT validation # https://tools.ietf.org/html/rfc7519#section-4.1.5 - err = "0. The ID token is not yet valid. " + TIME_SUGGESTION + _IdTokenTimeError("0. The ID token is not yet valid.", _now, decoded).log() + if issuer and issuer != decoded["iss"]: # https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderConfigurationResponse - err = ('2. The Issuer Identifier for the OpenID Provider, "%s", ' + raise IdTokenIssuerError( + '2. The Issuer Identifier for the OpenID Provider, "%s", ' "(which is typically obtained during Discovery), " - "MUST exactly match the value of the iss (issuer) Claim.") % issuer + "MUST exactly match the value of the iss (issuer) Claim." % issuer, + _now, + decoded) + if client_id: valid_aud = client_id in decoded["aud"] if isinstance( decoded["aud"], list) else client_id == decoded["aud"] if not valid_aud: - err = ( + raise IdTokenAudienceError( "3. The aud (audience) claim must contain this client's client_id " '"%s", case-sensitively. Was your client_id in wrong casing?' # Some IdP accepts wrong casing request but issues right casing IDT - ) % client_id + % client_id, + _now, + decoded) + # Per specs: # 6. If the ID Token is received via direct communication between # the Client and the Token Endpoint (which it is during _obtain_token()), # the TLS server validation MAY be used to validate the issuer # in place of checking the token signature. + if _now - skew > decoded["exp"]: - err = "9. The ID token already expires. " + TIME_SUGGESTION + _IdTokenTimeError("9. The ID token already expires.", _now, decoded).log() + if nonce and nonce != decoded.get("nonce"): - err = ("11. Nonce must be the same value " - "as the one that was sent in the Authentication Request.") - if err: - raise RuntimeError("%s Current epoch = %s. The id_token was: %s" % ( - err, _now, json.dumps(decoded, indent=2))) + raise IdTokenNonceError( + "11. Nonce must be the same value " + "as the one that was sent in the Authentication Request.", + _now, + decoded) + return decoded diff --git a/tests/test_oidc.py b/tests/test_oidc.py new file mode 100644 index 00000000..d6a929bc --- /dev/null +++ b/tests/test_oidc.py @@ -0,0 +1,21 @@ +from tests import unittest + +import oauth2cli + + +class TestIdToken(unittest.TestCase): + EXPIRED_ID_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJpYXQiOjE3MDY1NzA3MzIsImV4cCI6MTY3NDk0ODMzMiwiYXVkIjoiZm9vIiwic3ViIjoic3ViamVjdCJ9.wyWNFxnE35SMP6FpxnWZmWQAy4KD0No_Q1rUy5bNnLs" + + def test_id_token_should_tolerate_time_error(self): + self.assertEqual(oauth2cli.oidc.decode_id_token(self.EXPIRED_ID_TOKEN), { + "iss": "issuer", + "iat": 1706570732, + "exp": 1674948332, # 2023-1-28 + "aud": "foo", + "sub": "subject", + }, "id_token is decoded correctly, without raising exception") + + def test_id_token_should_error_out_on_client_id_error(self): + with self.assertRaises(oauth2cli.IdTokenError): + oauth2cli.oidc.decode_id_token(self.EXPIRED_ID_TOKEN, client_id="not foo") + From 386ea2e02a533373ab2d557da6d5aa55a748d7d3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 25 Jan 2024 23:48:09 -0800 Subject: [PATCH 122/262] Tolerate ID token time errors --- docs/index.rst | 9 +++++++++ msal/__init__.py | 3 +-- tests/test_oidc.py | 5 +++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2129e106..11dd9b05 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -148,3 +148,12 @@ New in MSAL Python 1.26 .. automethod:: __init__ + +Exceptions +---------- +These are exceptions that MSAL Python may raise. +You should not need to create them directly. +You may want to catch them to provide a better error message to your end users. + +.. autoclass:: msal.IdTokenError + diff --git a/msal/__init__.py b/msal/__init__.py index 09b7a504..5c5292fa 100644 --- a/msal/__init__.py +++ b/msal/__init__.py @@ -31,7 +31,6 @@ ConfidentialClientApplication, PublicClientApplication, ) -from .oauth2cli.oidc import Prompt +from .oauth2cli.oidc import Prompt, IdTokenError from .token_cache import TokenCache, SerializableTokenCache from .auth_scheme import PopAuthScheme - diff --git a/tests/test_oidc.py b/tests/test_oidc.py index d6a929bc..297dfeb5 100644 --- a/tests/test_oidc.py +++ b/tests/test_oidc.py @@ -1,6 +1,7 @@ from tests import unittest -import oauth2cli +import msal +from msal import oauth2cli class TestIdToken(unittest.TestCase): @@ -16,6 +17,6 @@ def test_id_token_should_tolerate_time_error(self): }, "id_token is decoded correctly, without raising exception") def test_id_token_should_error_out_on_client_id_error(self): - with self.assertRaises(oauth2cli.IdTokenError): + with self.assertRaises(msal.IdTokenError): oauth2cli.oidc.decode_id_token(self.EXPIRED_ID_TOKEN, client_id="not foo") From 1a19c4b4ffa44f951fa63e84ded49f6558a35512 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 15 May 2023 12:43:56 -0700 Subject: [PATCH 123/262] Provide examples for B2C and CIAM --- msal/authority.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msal/authority.py b/msal/authority.py index 923f3ffb..5e0131f3 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -120,6 +120,8 @@ def __init__( "Unable to get authority configuration for {}. " "Authority would typically be in a format of " "https://login.microsoftonline.com/your_tenant " + "or https://tenant_name.ciamlogin.com " + "or https://tenant_name.b2clogin.com/tenant.onmicrosoft.com/policy. " "Also please double check your tenant name or GUID is correct.".format( authority_url)) logger.debug("openid_config = %s", openid_config) From 5d9b2211a5bd9e18e05b6ce77f139bac9b7c17da Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 2 Feb 2024 16:39:36 -0800 Subject: [PATCH 124/262] Give a hint on where the client_id came from --- sample/confidential_client_certificate_sample.py | 2 +- sample/confidential_client_secret_sample.py | 2 +- sample/device_flow_sample.py | 2 +- sample/interactive_sample.py | 2 +- sample/migrate_rt.py | 2 +- sample/username_password_sample.py | 2 +- sample/vault_jwt_sample.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sample/confidential_client_certificate_sample.py b/sample/confidential_client_certificate_sample.py index 93c72ee9..c8c5f3c6 100644 --- a/sample/confidential_client_certificate_sample.py +++ b/sample/confidential_client_certificate_sample.py @@ -3,7 +3,7 @@ { "authority": "https://login.microsoftonline.com/Enter_the_Tenant_Name_Here", - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "scope": ["https://graph.microsoft.com/.default"], // Specific to Client Credentials Grant i.e. acquire_token_for_client(), // you don't specify, in the code, the individual scopes you want to access. diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_secret_sample.py index 9c616d53..48948ff5 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_secret_sample.py @@ -3,7 +3,7 @@ { "authority": "https://login.microsoftonline.com/Enter_the_Tenant_Name_Here", - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "scope": ["https://graph.microsoft.com/.default"], // Specific to Client Credentials Grant i.e. acquire_token_for_client(), // you don't specify, in the code, the individual scopes you want to access. diff --git a/sample/device_flow_sample.py b/sample/device_flow_sample.py index 89dccd1c..816bbb18 100644 --- a/sample/device_flow_sample.py +++ b/sample/device_flow_sample.py @@ -3,7 +3,7 @@ { "authority": "https://login.microsoftonline.com/common", - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "scope": ["User.ReadBasic.All"], // You can find the other permission names from this document // https://docs.microsoft.com/en-us/graph/permissions-reference diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index f283ed29..f4feb6ec 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -6,7 +6,7 @@ { "authority": "https://login.microsoftonline.com/organizations", - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "scope": ["User.ReadBasic.All"], // You can find the other permission names from this document // https://docs.microsoft.com/en-us/graph/permissions-reference diff --git a/sample/migrate_rt.py b/sample/migrate_rt.py index ed0011ed..e854866d 100644 --- a/sample/migrate_rt.py +++ b/sample/migrate_rt.py @@ -3,7 +3,7 @@ { "authority": "https://login.microsoftonline.com/organizations", - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "scope": ["User.ReadBasic.All"], // You can find the other permission names from this document // https://docs.microsoft.com/en-us/graph/permissions-reference diff --git a/sample/username_password_sample.py b/sample/username_password_sample.py index a25407d0..25e49ffd 100644 --- a/sample/username_password_sample.py +++ b/sample/username_password_sample.py @@ -3,7 +3,7 @@ { "authority": "https://login.microsoftonline.com/organizations", - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "username": "your_username@your_tenant.com", "password": "This is a sample only. You better NOT persist your password.", "scope": ["User.ReadBasic.All"], diff --git a/sample/vault_jwt_sample.py b/sample/vault_jwt_sample.py index 9410039c..e2448fc7 100644 --- a/sample/vault_jwt_sample.py +++ b/sample/vault_jwt_sample.py @@ -3,7 +3,7 @@ { "tenant": "your_tenant_name", // Your target tenant, DNS name - "client_id": "your_client_id", + "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", // Target app ID in Azure AD "scope": ["https://graph.microsoft.com/.default"], // Specific to Client Credentials Grant i.e. acquire_token_for_client(), From 4b34dd6ea0b5a511a5fbc76dfefa9cd9fff7e004 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 1 Feb 2024 12:12:46 -0800 Subject: [PATCH 125/262] Allow github action to write perf result into repo This is needed because our org has transitioned to a read-only GITHUB_TOKEN for GitHub Action workflows. This change fixes #653 --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 461eb959..0ef8fe32 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -56,6 +56,8 @@ jobs: # and then run benchmark only once (sampling with only one Python version). needs: ci runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 - name: Set up Python 3.9 From b28654038fbaf684bb633b456db618a32372e1f7 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 27 Feb 2023 11:25:39 -0800 Subject: [PATCH 126/262] Adding attributes that were not auto documented --- docs/conf.py | 4 ++-- docs/index.rst | 47 ++++++++++++++++++++++++++++----------------- msal/application.py | 27 +++++++++++++++----------- 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 024451d5..f9762ca2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,7 +62,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -95,7 +95,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. diff --git a/docs/index.rst b/docs/index.rst index 11dd9b05..15ee4a0a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,8 +7,6 @@ MSAL Python Documentation :caption: Contents: :hidden: - index - .. Comment: Perhaps because of the theme, only the first level sections will show in TOC, regardless of maxdepth setting. @@ -26,7 +24,7 @@ MSAL Python supports some of them. **The following diagram serves as a map. Locate your application scenario on the map.** **If the corresponding icon is clickable, it will bring you to an MSAL Python sample for that scenario.** -* Most authentication scenarios acquire tokens on behalf of signed-in users. +* Most authentication scenarios acquire tokens representing the signed-in user. .. raw:: html @@ -46,7 +44,7 @@ MSAL Python supports some of them. alt="Browserless app" title="Browserless app" href="https://github.com/Azure-Samples/ms-identity-python-devicecodeflow"> -* There are also daemon apps. In these scenarios, applications acquire tokens on behalf of themselves with no user. +* There are also daemon apps, who acquire tokens representing themselves, not a user. .. raw:: html @@ -66,26 +64,24 @@ MSAL Python supports some of them. API Reference ============= +.. note:: + + Only the contents inside + `this source file `_ + and their documented methods (unless otherwise marked as deprecated) + are MSAL Python public API, + which are guaranteed to be backward-compatible until the next major version. + + Everything else, regardless of their naming, are all internal helpers, + which could change at anytime in the future, without prior notice. The following section is the API Reference of MSAL Python. -The API Reference is like a dictionary. You **read this API section when and only when**: +The API Reference is like a dictionary, which is useful when: * You already followed our sample(s) above and have your app up and running, but want to know more on how you could tweak the authentication experience by using other optional parameters (there are plenty of them!) -* You read the MSAL Python source code and found a helper function that is useful to you, - then you would want to double check whether that helper is documented below. - Only documented APIs are considered part of the MSAL Python public API, - which are guaranteed to be backward-compatible in MSAL Python 1.x series. - Undocumented internal helpers are subject to change anytime, without prior notice. - -.. note:: - - Only APIs and their parameters documented in this section are part of public API, - with guaranteed backward compatibility for the entire 1.x series. - - Other modules in the source code are all considered as internal helpers, - which could change at anytime in the future, without prior notice. +* Some important features have their in-depth documentations in the API Reference. MSAL proposes a clean separation between `public client applications and confidential client applications @@ -109,6 +105,7 @@ PublicClientApplication .. autoclass:: msal.PublicClientApplication :members: + .. autoattribute:: msal.PublicClientApplication.CONSOLE_WINDOW_HANDLE .. automethod:: __init__ ConfidentialClientApplication @@ -134,6 +131,15 @@ See `SerializableTokenCache` for example. .. autoclass:: msal.SerializableTokenCache :members: +Prompt +------ +.. autoclass:: msal.Prompt + :members: + + .. autoattribute:: msal.Prompt.SELECT_ACCOUNT + .. autoattribute:: msal.Prompt.NONE + .. autoattribute:: msal.Prompt.CONSENT + .. autoattribute:: msal.Prompt.LOGIN PopAuthScheme ------------- @@ -146,6 +152,11 @@ New in MSAL Python 1.26 .. autoclass:: msal.PopAuthScheme :members: + .. autoattribute:: msal.PopAuthScheme.HTTP_GET + .. autoattribute:: msal.PopAuthScheme.HTTP_POST + .. autoattribute:: msal.PopAuthScheme.HTTP_PUT + .. autoattribute:: msal.PopAuthScheme.HTTP_DELETE + .. autoattribute:: msal.PopAuthScheme.HTTP_PATCH .. automethod:: __init__ diff --git a/msal/application.py b/msal/application.py index 96277e4a..f82ea2e3 100644 --- a/msal/application.py +++ b/msal/application.py @@ -737,10 +737,11 @@ def initiate_auth_code_flow( maintain state between the request and callback. If absent, this library will automatically generate one internally. :param str prompt: - By default, no prompt value will be sent, not even "none". + By default, no prompt value will be sent, not even string ``"none"``. You will have to specify a value explicitly. - Its valid values are defined in Open ID Connect specs - https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest + Its valid values are the constants defined in + :class:`Prompt `. + :param str login_hint: Optional. Identifier of the user. Generally a User Principal Name (UPN). :param domain_hint: @@ -840,10 +841,10 @@ def get_authorization_request_url( `not recommended `_. :param str prompt: - By default, no prompt value will be sent, not even "none". + By default, no prompt value will be sent, not even string ``"none"``. You will have to specify a value explicitly. - Its valid values are defined in Open ID Connect specs - https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest + Its valid values are the constants defined in + :class:`Prompt `. :param nonce: A cryptographically random value used to mitigate replay attacks. See also `OIDC specs `_. @@ -1819,10 +1820,10 @@ def acquire_token_interactive( :param list scopes: It is a list of case-sensitive strings. :param str prompt: - By default, no prompt value will be sent, not even "none". + By default, no prompt value will be sent, not even string ``"none"``. You will have to specify a value explicitly. - Its valid values are defined in Open ID Connect specs - https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest + Its valid values are the constants defined in + :class:`Prompt `. :param str login_hint: Optional. Identifier of the user. Generally a User Principal Name (UPN). :param domain_hint: @@ -1867,11 +1868,15 @@ def acquire_token_interactive( New in version 1.15. :param int parent_window_handle: - OPTIONAL. If your app is a GUI app running on modern Windows system, - and your app opts in to use broker, + Required if your app is running on Windows and opted in to use broker. + + If your app is a GUI app, you are recommended to also provide its window handle, so that the sign in UI window will properly pop up on top of your window. + If your app is a console app (most Python scripts are console apps), + you can use a placeholder value ``msal.PublicClientApplication.CONSOLE_WINDOW_HANDLE``. + New in version 1.20.0. :param function on_before_launching_ui: From 3b96de605ff4898aaf1f19c919c9574616b3dbed Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 7 Feb 2024 13:11:14 -0800 Subject: [PATCH 127/262] Implement remove_tokens_for_client() --- msal/application.py | 13 +++++++++++++ tests/test_application.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/msal/application.py b/msal/application.py index f82ea2e3..0f86be5b 100644 --- a/msal/application.py +++ b/msal/application.py @@ -2178,6 +2178,19 @@ def _acquire_token_for_client( telemetry_context.update_telemetry(response) return response + def remove_tokens_for_client(self): + """Remove all tokens that were previously acquired via + :func:`~acquire_token_for_client()` for the current client.""" + for env in [self.authority.instance] + self._get_authority_aliases( + self.authority.instance): + for at in self.token_cache.find(TokenCache.CredentialType.ACCESS_TOKEN, query={ + "client_id": self.client_id, + "environment": env, + "home_account_id": None, # These are mostly app-only tokens + }): + self.token_cache.remove_at(at) + # acquire_token_for_client() obtains no RTs, so we have no RT to remove + def acquire_token_on_behalf_of(self, user_assertion, scopes, claims_challenge=None, **kwargs): """Acquires token using on-behalf-of (OBO) flow. diff --git a/tests/test_application.py b/tests/test_application.py index fc529f01..cebc7225 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -662,6 +662,35 @@ def test_organizations_authority_should_emit_warning(self): authority="https://login.microsoftonline.com/organizations") +class TestRemoveTokensForClient(unittest.TestCase): + def test_remove_tokens_for_client_should_remove_client_tokens_only(self): + at_for_user = "AT for user" + cca = msal.ConfidentialClientApplication( + "client_id", client_credential="secret", + authority="https://login.microsoftonline.com/microsoft.onmicrosoft.com") + self.assertEqual( + 0, len(cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN))) + cca.acquire_token_for_client( + ["scope"], + post=lambda url, **kwargs: MinimalResponse( + status_code=200, text=json.dumps({"access_token": "AT for client"}))) + self.assertEqual( + 1, len(cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN))) + cca.acquire_token_by_username_password( + "johndoe", "password", ["scope"], + post=lambda url, **kwargs: MinimalResponse( + status_code=200, text=json.dumps(build_response( + access_token=at_for_user, expires_in=3600, + uid="uid", utid="utid", # This populates home_account_id + )))) + self.assertEqual( + 2, len(cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN))) + cca.remove_tokens_for_client() + remaining_tokens = cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN) + self.assertEqual(1, len(remaining_tokens)) + self.assertEqual(at_for_user, remaining_tokens[0].get("secret")) + + class TestScopeDecoration(unittest.TestCase): def _test_client_id_should_be_a_valid_scope(self, client_id, other_scopes): # B2C needs this https://learn.microsoft.com/en-us/azure/active-directory-b2c/access-tokens#openid-connect-scopes From bb0e24a2e13571d204799820d18089c0974e3709 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 10 Mar 2023 12:14:51 -0800 Subject: [PATCH 128/262] Remove premature int(...) --- msal/throttled_http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index 378cd3df..1e285ff8 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -30,7 +30,7 @@ def _parse_http_429_5xx_retry_after(result=None, **ignored): return 0 # Quick exit default = 60 # Recommended at the end of # https://identitydivision.visualstudio.com/devex/_git/AuthLibrariesApiReview?version=GBdev&path=%2FService%20protection%2FIntial%20set%20of%20protection%20measures.md&_a=preview - retry_after = int(lowercase_headers.get("retry-after", default)) + retry_after = lowercase_headers.get("retry-after", default) try: # AAD's retry_after uses integer format only # https://stackoverflow.microsoft.com/questions/264931/264932 From 0d8b2c201f3d09efd61f23202d051c9433ae0e9d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 5 Jun 2023 12:22:47 -0700 Subject: [PATCH 129/262] MSAL's fallback-from-broker behavior remains a FAQ --- msal/application.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/msal/application.py b/msal/application.py index 0f86be5b..464714d7 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1747,7 +1747,7 @@ def __init__(self, client_id, client_credential=None, **kwargs): You may set enable_broker_on_windows to True. - What is a broker, and why use it? + **What is a broker, and why use it?** A broker is a component installed on your device. Broker implicitly gives your device an identity. By using a broker, @@ -1764,10 +1764,7 @@ def __init__(self, client_id, client_credential=None, **kwargs): so that your broker-enabled apps (even a CLI) could automatically SSO from a previously established signed-in session. - ADFS and B2C do not support broker. - MSAL will automatically fallback to use browser. - - You shall only enable broker when your app: + **You shall only enable broker when your app:** 1. is running on supported platforms, and already registered their corresponding redirect_uri @@ -1780,6 +1777,29 @@ def __init__(self, client_id, client_credential=None, **kwargs): 3. tested with ``acquire_token_interactive()`` and ``acquire_token_silent()``. + **The fallback behaviors of MSAL Python's broker support** + + MSAL will either error out, or silently fallback to non-broker flows. + + 1. MSAL will ignore the `enable_broker_...` and bypass broker + on those auth flows that are known to be NOT supported by broker. + This includes ADFS, B2C, etc.. + For other "could-use-broker" scenarios, please see below. + 2. MSAL errors out when app developer opted-in to use broker + but a direct dependency "mid-tier" package is not installed. + Error message guides app developer to declare the correct dependency + ``msal[broker]``. + We error out here because the error is actionable to app developers. + 3. MSAL silently "deactivates" the broker and fallback to non-broker, + when opted-in, dependency installed yet failed to initialize. + We anticipate this would happen on a device whose OS is too old + or the underlying broker component is somehow unavailable. + There is not much an app developer or the end user can do here. + Eventually, the conditional access policy shall + force the user to switch to a different device. + 4. MSAL errors out when broker is opted in, installed, initialized, + but subsequent token request(s) failed. + :param boolean enable_broker_on_windows: This setting is only effective if your app is running on Windows 10+. This parameter defaults to None, which means MSAL will not utilize a broker. From bf87155e158c360a9047205e5fbe7717345cdf94 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 31 Oct 2023 12:30:35 -0700 Subject: [PATCH 130/262] Change back to use print(result) in error path --- sample/confidential_client_certificate_sample.py | 2 +- sample/confidential_client_secret_sample.py | 2 +- sample/device_flow_sample.py | 2 +- sample/interactive_sample.py | 2 +- sample/username_password_sample.py | 11 ++++++----- sample/vault_jwt_sample.py | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sample/confidential_client_certificate_sample.py b/sample/confidential_client_certificate_sample.py index c8c5f3c6..2c4118a3 100644 --- a/sample/confidential_client_certificate_sample.py +++ b/sample/confidential_client_certificate_sample.py @@ -70,7 +70,7 @@ def acquire_and_use_token(): headers={'Authorization': 'Bearer ' + result['access_token']},).json() print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) else: - print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error while True: # Here we mimic a long-lived daemon diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_secret_sample.py index 48948ff5..9ff6a81b 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_secret_sample.py @@ -69,7 +69,7 @@ def acquire_and_use_token(): headers={'Authorization': 'Bearer ' + result['access_token']},).json() print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) else: - print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error while True: # Here we mimic a long-lived daemon diff --git a/sample/device_flow_sample.py b/sample/device_flow_sample.py index 816bbb18..7d998470 100644 --- a/sample/device_flow_sample.py +++ b/sample/device_flow_sample.py @@ -91,7 +91,7 @@ def acquire_and_use_token(): headers={'Authorization': 'Bearer ' + result['access_token']},).json() print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) else: - print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error while True: # Here we mimic a long-lived daemon diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index f4feb6ec..58d8c9a9 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -86,7 +86,7 @@ def acquire_and_use_token(): headers={'Authorization': 'Bearer ' + result['access_token']},) print("Graph API call result: %s ..." % graph_response.text[:100]) else: - print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error while True: # Here we mimic a long-lived daemon diff --git a/sample/username_password_sample.py b/sample/username_password_sample.py index 25e49ffd..3578e5ef 100644 --- a/sample/username_password_sample.py +++ b/sample/username_password_sample.py @@ -5,7 +5,6 @@ "authority": "https://login.microsoftonline.com/organizations", "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", "username": "your_username@your_tenant.com", - "password": "This is a sample only. You better NOT persist your password.", "scope": ["User.ReadBasic.All"], // You can find the other permission names from this document // https://docs.microsoft.com/en-us/graph/permissions-reference @@ -20,6 +19,7 @@ """ import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] +import getpass import json import logging import time @@ -33,6 +33,7 @@ # logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs config = json.load(open(sys.argv[1])) +config["password"] = getpass.getpass() # If for whatever reason you plan to recreate same ClientApplication periodically, # you shall create one global token cache and reuse it by each ClientApplication @@ -40,9 +41,10 @@ # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache # Create a preferably long-lived app instance, to avoid the overhead of app creation -global_app = msal.PublicClientApplication( - config["client_id"], authority=config["authority"], +global_app = msal.ClientApplication( + config["client_id"], client_credential=config.get("client_secret"), + authority=config["authority"], token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) @@ -73,8 +75,7 @@ def acquire_and_use_token(): headers={'Authorization': 'Bearer ' + result['access_token']},).json() print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) else: - print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error - print(result) + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error if 65001 in result.get("error_codes", []): # Not mean to be coded programatically, but... raise RuntimeError( "AAD requires user consent for U/P flow to succeed. " diff --git a/sample/vault_jwt_sample.py b/sample/vault_jwt_sample.py index e2448fc7..b9c51912 100644 --- a/sample/vault_jwt_sample.py +++ b/sample/vault_jwt_sample.py @@ -132,7 +132,7 @@ def acquire_and_use_token(): headers={'Authorization': 'Bearer ' + result['access_token']},).json() print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) else: - print("Token acquisition failed") # Examine result["error_description"] etc. to diagnose error + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error while True: # Here we mimic a long-lived daemon From 4f0e03d5141d3fe405bc6ad52a2db08ed8287951 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 8 Feb 2024 14:49:15 -0800 Subject: [PATCH 131/262] CCA can be tested by: python -m msal --- msal/__main__.py | 74 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/msal/__main__.py b/msal/__main__.py index 79776b8f..1c09f868 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -60,6 +60,17 @@ def _select_options( if raw_data and accept_nonempty_string: return raw_data +enable_debug_log = _input_boolean("Enable MSAL Python's DEBUG log?") +logging.basicConfig(level=logging.DEBUG if enable_debug_log else logging.INFO) +try: + from dotenv import load_dotenv + load_dotenv() + logging.info("Loaded environment variables from .env file") +except ImportError: + logging.warning( + "python-dotenv is not installed. " + "You may need to set environment variables manually.") + def _input_scopes(): scopes = _select_options([ "https://graph.microsoft.com/.default", @@ -100,6 +111,7 @@ def _acquire_token_silent(app): def _acquire_token_interactive(app, scopes=None, data=None): """acquire_token_interactive() - User will be prompted if app opts to do select_account.""" + assert isinstance(app, msal.PublicClientApplication) scopes = scopes or _input_scopes() # Let user input scope param before less important prompt and login_hint prompt = _select_options([ {"value": None, "description": "Unspecified. Proceed silently with a default account (if any), fallback to prompt."}, @@ -143,6 +155,7 @@ def _acquire_token_by_username_password(app): def _acquire_token_by_device_flow(app): """acquire_token_by_device_flow() - Note that this one does not go through broker""" + assert isinstance(app, msal.PublicClientApplication) flow = app.initiate_device_flow(scopes=_input_scopes()) print(flow["message"]) sys.stdout.flush() # Some terminal needs this to ensure the message is shown @@ -156,6 +169,7 @@ def _acquire_token_by_device_flow(app): def _acquire_ssh_cert_silently(app): """Acquire an SSH Cert silently- This typically only works with Azure CLI""" + assert isinstance(app, msal.PublicClientApplication) account = _select_account(app) if account: result = app.acquire_token_silent( @@ -170,6 +184,7 @@ def _acquire_ssh_cert_silently(app): def _acquire_ssh_cert_interactive(app): """Acquire an SSH Cert interactively - This typically only works with Azure CLI""" + assert isinstance(app, msal.PublicClientApplication) result = _acquire_token_interactive(app, scopes=_SSH_CERT_SCOPE, data=_SSH_CERT_DATA) if result.get("token_type") != "ssh-cert": logging.error("Unable to acquire an ssh-cert") @@ -185,6 +200,7 @@ def _acquire_ssh_cert_interactive(app): def _acquire_pop_token_interactive(app): """Acquire a POP token interactively - This typically only works with Azure CLI""" + assert isinstance(app, msal.PublicClientApplication) POP_SCOPE = ['6256c85f-0aad-4d50-b960-e6e9b21efe35/.default'] # KAP 1P Server App Scope, obtained from https://github.com/Azure/azure-cli-extensions/pull/4468/files#diff-a47efa3186c7eb4f1176e07d0b858ead0bf4a58bfd51e448ee3607a5b4ef47f6R116 result = _acquire_token_interactive(app, scopes=POP_SCOPE, data=_POP_DATA) print_json(result) @@ -198,6 +214,16 @@ def _remove_account(app): app.remove_account(account) print('Account "{}" and/or its token(s) are signed out from MSAL Python'.format(account["username"])) +def _acquire_token_for_client(app): + """CCA.acquire_token_for_client() - Rerun this will get same token from cache.""" + assert isinstance(app, msal.ConfidentialClientApplication) + print_json(app.acquire_token_for_client(scopes=_input_scopes())) + +def _remove_tokens_for_client(app): + """CCA.remove_tokens_for_client() - Run this to evict tokens from cache.""" + assert isinstance(app, msal.ConfidentialClientApplication) + app.remove_tokens_for_client() + def _exit(app): """Exit""" bug_link = ( @@ -235,12 +261,23 @@ def _main(): {"client_id": _AZURE_CLI, "name": "Azure CLI (Correctly configured for MSA-PT)"}, {"client_id": _VISUAL_STUDIO, "name": "Visual Studio (Correctly configured for MSA-PT)"}, {"client_id": "95de633a-083e-42f5-b444-a4295d8e9314", "name": "Whiteboard Services (Non MSA-PT app. Accepts AAD & MSA accounts.)"}, + { + "client_id": os.getenv("CLIENT_ID"), + "client_secret": os.getenv("CLIENT_SECRET"), + "name": "A confidential client app (CCA) whose settings are defined " + "in environment variables CLIENT_ID and CLIENT_SECRET", + }, ], option_renderer=lambda a: a["name"], - header="Impersonate this app (or you can type in the client_id of your own app)", + header="Impersonate this app " + "(or you can type in the client_id of your own public client app)", accept_nonempty_string=True) - enable_broker = _input_boolean("Enable broker? It will error out later if your app has not registered some redirect URI") - enable_debug_log = _input_boolean("Enable MSAL Python's DEBUG log?") + is_cca = isinstance(chosen_app, dict) and "client_secret" in chosen_app + if is_cca and not (chosen_app["client_id"] and chosen_app["client_secret"]): + raise ValueError("You need to set environment variables CLIENT_ID and CLIENT_SECRET") + enable_broker = (not is_cca) and _input_boolean("Enable broker? " + "(It will error out later if your app has not registered some redirect URI)" + ) enable_pii_log = _input_boolean("Enable PII in broker's log?") if enable_broker and enable_debug_log else False authority = _select_options([ "https://login.microsoftonline.com/common", @@ -255,7 +292,8 @@ def _main(): instance_discovery = _input_boolean( "You input an unusual authority which might fail the Instance Discovery. " "Now, do you want to perform Instance Discovery on your input authority?" - ) if not authority.startswith("https://login.microsoftonline.com") else None + ) if authority and not authority.startswith( + "https://login.microsoftonline.com") else None app = msal.PublicClientApplication( chosen_app["client_id"] if isinstance(chosen_app, dict) else chosen_app, authority=authority, @@ -263,21 +301,35 @@ def _main(): enable_broker_on_windows=enable_broker, enable_pii_log=enable_pii_log, token_cache=global_cache, + ) if not is_cca else msal.ConfidentialClientApplication( + chosen_app["client_id"], + client_credential=chosen_app["client_secret"], + authority=authority, + instance_discovery=instance_discovery, + enable_pii_log=enable_pii_log, + token_cache=global_cache, ) - if enable_debug_log: - logging.basicConfig(level=logging.DEBUG) - while True: - func = _select_options([ + methods_to_be_tested = [ _acquire_token_silent, + ] + ([ _acquire_token_interactive, - _acquire_token_by_username_password, _acquire_token_by_device_flow, _acquire_ssh_cert_silently, _acquire_ssh_cert_interactive, _acquire_pop_token_interactive, + ] if isinstance(app, msal.PublicClientApplication) else [] + ) + [ + _acquire_token_by_username_password, _remove_account, - _exit, - ], option_renderer=lambda f: f.__doc__, header="MSAL Python APIs:") + ] + ([ + _acquire_token_for_client, + _remove_tokens_for_client, + ] if isinstance(app, msal.ConfidentialClientApplication) else [] + ) + while True: + func = _select_options( + methods_to_be_tested + [_exit], + option_renderer=lambda f: f.__doc__, header="MSAL Python APIs:") try: func(app) except ValueError as e: From 59c3000192e92a49f483045071b97aa79929d19f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 8 Feb 2024 21:31:13 -0800 Subject: [PATCH 132/262] Pick up latest PyMsalRuntime 0.14.x --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d3b7e40d..2177d2c5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,7 +65,7 @@ broker = # most existing MSAL Python apps do not have the redirect_uri needed by broker. # MSAL Python uses a subset of API from PyMsalRuntime 0.13.0+, # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) - pymsalruntime>=0.13.2,<0.14; python_version>='3.6' and platform_system=='Windows' + pymsalruntime>=0.13.2,<0.15; python_version>='3.6' and platform_system=='Windows' [options.packages.find] exclude = From 9a866ca6c2c960c3412ba613cfb89033bbfa7ca0 Mon Sep 17 00:00:00 2001 From: Ed Singleton Date: Thu, 22 Feb 2024 17:35:54 +0000 Subject: [PATCH 133/262] Don't use bare except when importing (#667) Using a bare except statement when importing hides other errors, which then get lost when the next import fails. Co-authored-by: Ed Singleton --- msal/mex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/mex.py b/msal/mex.py index edecba37..e6f3ed07 100644 --- a/msal/mex.py +++ b/msal/mex.py @@ -27,7 +27,7 @@ try: from urllib.parse import urlparse -except: +except ImportError: from urlparse import urlparse try: from xml.etree import cElementTree as ET From 7e045199798adbe5309034de12c64b1816d23489 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 21 Dec 2023 11:55:15 -0800 Subject: [PATCH 134/262] Releasing 1.27 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 464714d7..7b08c5e2 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.26.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.27.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From c73b7ca471cdfe3a7ca2dcad95302860113d2176 Mon Sep 17 00:00:00 2001 From: Iulian Cociug Date: Fri, 1 Mar 2024 01:16:15 -0800 Subject: [PATCH 135/262] update the default broker redirect uri Ray: I tested it on his Win laptop to successfully acquire normal token and ssh cert from broker --- msal/broker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/msal/broker.py b/msal/broker.py index a0904199..82bc3d87 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -169,8 +169,9 @@ def _signin_interactively( **kwargs): params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) params.set_requested_scopes(scopes) - params.set_redirect_uri("placeholder") # pymsalruntime 0.1 requires non-empty str, - # the actual redirect_uri will be overridden by a value hardcoded by the broker + params.set_redirect_uri("https://login.microsoftonline.com/common/oauth2/nativeclient") + # This default redirect_uri value is not currently used by the broker + # but it is required by the MSAL.cpp to be set to a non-empty valid URI. if prompt: if prompt == "select_account": if login_hint: From c442c780c9777a6f746fed39539a9c42bbb5b0f6 Mon Sep 17 00:00:00 2001 From: Sherman Ouko Date: Mon, 11 Mar 2024 21:36:54 +0300 Subject: [PATCH 136/262] Rebrand from AAD to Microsoft Entra (#655) * Rebrand from AAD to Microsoft Entra * Readme rebranding --- README.md | 8 +++++--- msal/application.py | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9d72fdfe..cda5a1e8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ |:------------:|:--------------:|:--------------------------------------:|:---------------------------------------:|:-----------------:| [![Build status](https://github.com/AzureAD/microsoft-authentication-library-for-python/actions/workflows/python-package.yml/badge.svg?branch=dev)](https://github.com/AzureAD/microsoft-authentication-library-for-python/actions) | [![Documentation Status](https://readthedocs.org/projects/msal-python/badge/?version=latest)](https://msal-python.readthedocs.io/en/latest/?badge=latest) | [![Downloads](https://static.pepy.tech/badge/msal)](https://pypistats.org/packages/msal) | [![Download monthly](https://static.pepy.tech/badge/msal/month)](https://pepy.tech/project/msal) | [📉](https://azuread.github.io/microsoft-authentication-library-for-python/dev/bench/) -The Microsoft Authentication Library for Python enables applications to integrate with the [Microsoft identity platform](https://aka.ms/aaddevv2). It allows you to sign in users or apps with Microsoft identities ([Azure AD](https://azure.microsoft.com/services/active-directory/), [Microsoft Accounts](https://account.microsoft.com) and [Azure AD B2C](https://azure.microsoft.com/services/active-directory-b2c/) accounts) and obtain tokens to call Microsoft APIs such as [Microsoft Graph](https://graph.microsoft.io/) or your own APIs registered with the Microsoft identity platform. It is built using industry standard OAuth2 and OpenID Connect protocols +The Microsoft Authentication Library for Python enables applications to integrate with the [Microsoft identity platform](https://aka.ms/aaddevv2). It allows you to sign in users or apps with Microsoft identities ([Microsoft Entra ID](https://www.microsoft.com/security/business/identity-access/microsoft-entra-id), [External identities](https://www.microsoft.com/security/business/identity-access/microsoft-entra-external-id), [Microsoft Accounts](https://account.microsoft.com) and [Azure AD B2C](https://azure.microsoft.com/services/active-directory-b2c/) accounts) and obtain tokens to call Microsoft APIs such as [Microsoft Graph](https://graph.microsoft.io/) or your own APIs registered with the Microsoft identity platform. It is built using industry standard OAuth2 and OpenID Connect protocols Not sure whether this is the SDK you are looking for your app? There are other Microsoft Identity SDKs [here](https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki/Microsoft-Authentication-Client-Libraries). @@ -23,9 +23,10 @@ Click on the following thumbnail to visit a large map with clickable links to pr ## Installation You can find MSAL Python on [Pypi](https://pypi.org/project/msal/). + 1. If you haven't already, [install and/or upgrade the pip](https://pip.pypa.io/en/stable/installing/) of your Python environment to a recent version. We tested with pip 18.1. -2. As usual, just run `pip install msal`. +1. As usual, just run `pip install msal`. ## Versions @@ -123,7 +124,7 @@ We provide a [full suite of sample applications](https://aka.ms/aaddevsamplesv2) ## Community Help and Support -We leverage Stack Overflow to work with the community on supporting Azure Active Directory and its SDKs, including this one! +We leverage Stack Overflow to work with the community on supporting Microsoft Entra and its SDKs, including this one! We highly recommend you ask your questions on Stack Overflow (we're all on there!) Also browser existing issues to see if someone has had your question before. @@ -132,6 +133,7 @@ Here is the latest Q&A on Stack Overflow for MSAL: [http://stackoverflow.com/questions/tagged/msal](http://stackoverflow.com/questions/tagged/msal) ## Submit Feedback + We'd like your thoughts on this library. Please complete [this short survey.](https://forms.office.com/r/TMjZkDbzjY) ## Security Reporting diff --git a/msal/application.py b/msal/application.py index 7b08c5e2..55a23512 100644 --- a/msal/application.py +++ b/msal/application.py @@ -207,10 +207,10 @@ def __init__( ): """Create an instance of application. - :param str client_id: Your app has a client_id after you register it on AAD. + :param str client_id: Your app has a client_id after you register it on Microsoft Entra admin center. :param Union[str, dict] client_credential: - For :class:`PublicClientApplication`, you simply use `None` here. + For :class:`PublicClientApplication`, you use `None` here. For :class:`ConfidentialClientApplication`, it can be a string containing client secret, or an X509 certificate container in this form:: @@ -916,7 +916,7 @@ def acquire_token_by_auth_code_flow( OAuth2 was designed mostly for singleton services, where tokens are always meant for the same resource and the only changes are in the scopes. - In AAD, tokens can be issued for multiple 3rd party resources. + In Microsoft Entra, tokens can be issued for multiple 3rd party resources. You can ask authorization code for multiple resources, but when you redeem it, the token is for only one intended recipient, called audience. @@ -986,7 +986,7 @@ def acquire_token_by_authorization_code( OAuth2 was designed mostly for singleton services, where tokens are always meant for the same resource and the only changes are in the scopes. - In AAD, tokens can be issued for multiple 3rd party resources. + In Microsoft Entra, tokens can be issued for multiple 3rd party resources. You can ask authorization code for multiple resources, but when you redeem it, the token is for only one intended recipient, called audience. @@ -1004,7 +1004,7 @@ def acquire_token_by_authorization_code( returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. It is a string of a JSON object which contains lists of claims being requested from these locations. - :return: A dict representing the json response from AAD: + :return: A dict representing the json response from Microsoft Entra: - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". @@ -1640,7 +1640,7 @@ def acquire_token_by_username_password( New in version 1.26.0. - :return: A dict representing the json response from AAD: + :return: A dict representing the json response from Microsoft Entra: - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". @@ -1871,7 +1871,7 @@ def acquire_token_interactive( (The rest of the redirect_uri is hard coded as ``http://localhost``.) :param list extra_scopes_to_consent: - "Extra scopes to consent" is a concept only available in AAD. + "Extra scopes to consent" is a concept only available in Microsoft Entra. It refers to other resources you might want to prompt to consent for, in the same interaction, but for which you won't get back a token for in this particular operation. @@ -2114,7 +2114,7 @@ def acquire_token_by_device_flow(self, flow, claims_challenge=None, **kwargs): returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. It is a string of a JSON object which contains lists of claims being requested from these locations. - :return: A dict representing the json response from AAD: + :return: A dict representing the json response from Microsoft Entra: - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". @@ -2159,7 +2159,7 @@ def acquire_token_for_client(self, scopes, claims_challenge=None, **kwargs): returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. It is a string of a JSON object which contains lists of claims being requested from these locations. - :return: A dict representing the json response from AAD: + :return: A dict representing the json response from Microsoft Entra: - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". @@ -2232,7 +2232,7 @@ def acquire_token_on_behalf_of(self, user_assertion, scopes, claims_challenge=No returned from the UserInfo Endpoint and/or in the ID Token and/or Access Token. It is a string of a JSON object which contains lists of claims being requested from these locations. - :return: A dict representing the json response from AAD: + :return: A dict representing the json response from Microsoft Entra: - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". From 70e09fbccb54d793e11ae0d6c5d4749c7d0c1a16 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 26 Feb 2024 23:49:30 -0800 Subject: [PATCH 137/262] Implements a new optional oidc_authority parameter --- msal/application.py | 15 +++++++++- msal/authority.py | 66 +++++++++++++++++++++++++++++------------ tests/test_authority.py | 31 +++++++++++++++++++ 3 files changed, 92 insertions(+), 20 deletions(-) diff --git a/msal/application.py b/msal/application.py index 55a23512..d6256635 100644 --- a/msal/application.py +++ b/msal/application.py @@ -204,6 +204,7 @@ def __init__( instance_discovery=None, allow_broker=None, enable_pii_log=None, + oidc_authority=None, ): """Create an instance of application. @@ -455,6 +456,15 @@ def __init__( The default behavior is False. New in version 1.24.0. + + :param str oidc_authority: + *Added in version 1.28.0*: + It is a URL that identifies an OpenID Connect (OIDC) authority of + the format ``https://contoso.com/tenant``. + MSAL will append ".well-known/openid-configuration" to the authority + and retrieve the OIDC metadata from there, to figure out the endpoints. + + Note: Broker will NOT be used for OIDC authority. """ self.client_id = client_id self.client_credential = client_credential @@ -499,6 +509,8 @@ def __init__( self.app_version = app_version # Here the self.authority will not be the same type as authority in input + if oidc_authority and authority: + raise ValueError("You can not provide both authority and oidc_authority") try: authority_to_use = authority or "https://{}/common/".format(WORLD_WIDE) self.authority = Authority( @@ -506,11 +518,12 @@ def __init__( self.http_client, validate_authority=validate_authority, instance_discovery=self._instance_discovery, + oidc_authority_url=oidc_authority, ) except ValueError: # Those are explicit authority validation errors raise except Exception: # The rest are typically connection errors - if validate_authority and azure_region: + if validate_authority and azure_region and not oidc_authority: # Since caller opts in to use region, here we tolerate connection # errors happened during authority validation at non-region endpoint self.authority = Authority( diff --git a/msal/authority.py b/msal/authority.py index 5e0131f3..de19f963 100644 --- a/msal/authority.py +++ b/msal/authority.py @@ -56,6 +56,7 @@ def __init__( self, authority_url, http_client, validate_authority=True, instance_discovery=None, + oidc_authority_url=None, ): """Creates an authority instance, and also validates it. @@ -65,12 +66,56 @@ def __init__( This parameter only controls whether an instance discovery will be performed. """ + self._http_client = http_client + if oidc_authority_url: + logger.info("Initializing with OIDC authority: %s", oidc_authority_url) + tenant_discovery_endpoint = self._initialize_oidc_authority( + oidc_authority_url) + else: + logger.info("Initializing with Entra authority: %s", authority_url) + tenant_discovery_endpoint = self._initialize_entra_authority( + authority_url, validate_authority, instance_discovery) + try: + openid_config = tenant_discovery( + tenant_discovery_endpoint, + self._http_client) + except ValueError: + error_message = ( + "Unable to get OIDC authority configuration for {url} " + "because its OIDC Discovery endpoint is unavailable at " + "{url}/.well-known/openid-configuration ".format(url=oidc_authority_url) + if oidc_authority_url else + "Unable to get authority configuration for {}. " + "Authority would typically be in a format of " + "https://login.microsoftonline.com/your_tenant " + "or https://tenant_name.ciamlogin.com " + "or https://tenant_name.b2clogin.com/tenant.onmicrosoft.com/policy. " + .format(authority_url) + ) + " Also please double check your tenant name or GUID is correct." + raise ValueError(error_message) + logger.debug( + 'openid_config("%s") = %s', tenant_discovery_endpoint, openid_config) + self.authorization_endpoint = openid_config['authorization_endpoint'] + self.token_endpoint = openid_config['token_endpoint'] + self.device_authorization_endpoint = openid_config.get('device_authorization_endpoint') + _, _, self.tenant = canonicalize(self.token_endpoint) # Usually a GUID + + def _initialize_oidc_authority(self, oidc_authority_url): + authority, self.instance, tenant = canonicalize(oidc_authority_url) + self.is_adfs = tenant.lower() == 'adfs' # As a convention + self._is_b2c = True # Not exactly true, but + # OIDC Authority was designed for CIAM which is the next gen of B2C. + # Besides, application.py uses this to bypass broker. + self._is_known_to_developer = True # Not really relevant, but application.py uses this to bypass authority validation + return oidc_authority_url + "/.well-known/openid-configuration" + + def _initialize_entra_authority( + self, authority_url, validate_authority, instance_discovery): # :param instance_discovery: # By default, the known-to-Microsoft validation will use an # instance discovery endpoint located at ``login.microsoftonline.com``. # You can customize the endpoint by providing a url as a string. # Or you can turn this behavior off by passing in a False here. - self._http_client = http_client if isinstance(authority_url, AuthorityBuilder): authority_url = str(authority_url) authority, self.instance, tenant = canonicalize(authority_url) @@ -111,24 +156,7 @@ def __init__( version="" if self.is_adfs else "/v2.0", ) ).geturl() # Keeping original port and query. Query is useful for test. - try: - openid_config = tenant_discovery( - tenant_discovery_endpoint, - self._http_client) - except ValueError: - raise ValueError( - "Unable to get authority configuration for {}. " - "Authority would typically be in a format of " - "https://login.microsoftonline.com/your_tenant " - "or https://tenant_name.ciamlogin.com " - "or https://tenant_name.b2clogin.com/tenant.onmicrosoft.com/policy. " - "Also please double check your tenant name or GUID is correct.".format( - authority_url)) - logger.debug("openid_config = %s", openid_config) - self.authorization_endpoint = openid_config['authorization_endpoint'] - self.token_endpoint = openid_config['token_endpoint'] - self.device_authorization_endpoint = openid_config.get('device_authorization_endpoint') - _, _, self.tenant = canonicalize(self.token_endpoint) # Usually a GUID + return tenant_discovery_endpoint def user_realm_discovery(self, username, correlation_id=None, response=None): # It will typically return a dict containing "ver", "account_type", diff --git a/tests/test_authority.py b/tests/test_authority.py index 2ced23f8..0d6c790f 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -99,6 +99,37 @@ def test_authority_with_path_should_be_used_as_is(self, oidc_discovery): self.http_client) +@patch("msal.authority._instance_discovery") +@patch("msal.authority.tenant_discovery", return_value={ + "authorization_endpoint": "https://contoso.com/authorize", + "token_endpoint": "https://contoso.com/token", + }) +class TestOidcAuthority(unittest.TestCase): + def test_authority_obj_should_do_oidc_discovery_and_skip_instance_discovery( + self, oidc_discovery, instance_discovery): + c = MinimalHttpClient() + a = Authority(None, c, oidc_authority_url="https://contoso.com/tenant") + instance_discovery.assert_not_called() + oidc_discovery.assert_called_once_with( + "https://contoso.com/tenant/.well-known/openid-configuration", c) + self.assertEqual(a.authorization_endpoint, 'https://contoso.com/authorize') + self.assertEqual(a.token_endpoint, 'https://contoso.com/token') + + def test_application_obj_should_do_oidc_discovery_and_skip_instance_discovery( + self, oidc_discovery, instance_discovery): + app = msal.ClientApplication( + "id", + authority=None, + oidc_authority="https://contoso.com/tenant", + ) + instance_discovery.assert_not_called() + oidc_discovery.assert_called_once_with( + "https://contoso.com/tenant/.well-known/openid-configuration", + app.http_client) + self.assertEqual( + app.authority.authorization_endpoint, 'https://contoso.com/authorize') + self.assertEqual(app.authority.token_endpoint, 'https://contoso.com/token') + class TestAuthorityInternalHelperCanonicalize(unittest.TestCase): def test_canonicalize_tenant_followed_by_extra_paths(self): From e06ca87ca97ef5fac21244b956462e5d86061be5 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 4 Mar 2024 22:45:28 -0800 Subject: [PATCH 138/262] A semi-auto script to test Azure CLI with broker --- tests/broker-test.py | 66 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/broker-test.py diff --git a/tests/broker-test.py b/tests/broker-test.py new file mode 100644 index 00000000..2301096e --- /dev/null +++ b/tests/broker-test.py @@ -0,0 +1,66 @@ +"""This script is used to impersonate Azure CLI +and run 3 pairs of end-to-end tests with broker. +Although not fully automated, it requires only several clicks to finish. + +Each time a new PyMsalRuntime is going to be released, +we can use this script to test it with a given version of MSAL Python. +""" +import msal + +_AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" +SCOPE_ARM = "https://management.azure.com/.default" +placeholder_auth_scheme = msal.PopAuthScheme( + http_method=msal.PopAuthScheme.HTTP_GET, + url="https://example.com/endpoint", + nonce="placeholder", + ) +_JWK1 = """{"kty":"RSA", "n":"2tNr73xwcj6lH7bqRZrFzgSLj7OeLfbn8216uOMDHuaZ6TEUBDN8Uz0ve8jAlKsP9CQFCSVoSNovdE-fs7c15MxEGHjDcNKLWonznximj8pDGZQjVdfK-7mG6P6z-lgVcLuYu5JcWU_PeEqIKg5llOaz-qeQ4LEDS4T1D2qWRGpAra4rJX1-kmrWmX_XIamq30C9EIO0gGuT4rc2hJBWQ-4-FnE1NXmy125wfT3NdotAJGq5lMIfhjfglDbJCwhc8Oe17ORjO3FsB5CLuBRpYmP7Nzn66lRY3Fe11Xz8AEBl3anKFSJcTvlMnFtu3EpD-eiaHfTgRBU7CztGQqVbiQ", "e":"AQAB"}""" +_SSH_CERT_DATA = {"token_type": "ssh-cert", "key_id": "key1", "req_cnf": _JWK1} +_SSH_CERT_SCOPE = "https://pas.windows.net/CheckMyAccess/Linux/.default" + +pca = msal.PublicClientApplication( + _AZURE_CLI, + authority="https://login.microsoftonline.com/organizations", + enable_broker_on_windows=True) + +def interactive_and_silent(scopes, auth_scheme, data, expected_token_type): + print("An account picker shall be pop up, possibly behind this console. Continue from there.") + result = pca.acquire_token_interactive( + scopes, + prompt="select_account", # "az login" does this + parent_window_handle=pca.CONSOLE_WINDOW_HANDLE, # This script is a console app + enable_msa_passthrough=True, # Azure CLI is an MSA-passthrough app + auth_scheme=auth_scheme, + data=data or {}, + ) + _assert(result, expected_token_type) + + accounts = pca.get_accounts() + assert accounts, "The logged in account should have been established by interactive flow" + result = pca.acquire_token_silent( + scopes, + account=accounts[0], + force_refresh=True, # Bypass MSAL Python's token cache to test PyMsalRuntime + auth_scheme=auth_scheme, + data=data or {}, + ) + _assert(result, expected_token_type) + +def _assert(result, expected_token_type): + assert result.get("access_token"), f"We should obtain a token. Got {result} instead." + assert result.get("token_source") == "broker", "Token should be obtained via broker" + assert result.get("token_type").lower() == expected_token_type.lower(), f"{expected_token_type} not found" + +for i in range(2): # Mimic Azure CLI's issue report + interactive_and_silent( + scopes=[SCOPE_ARM], auth_scheme=None, data=None, expected_token_type="bearer") + +interactive_and_silent( + scopes=[SCOPE_ARM], auth_scheme=placeholder_auth_scheme, data=None, expected_token_type="pop") +interactive_and_silent( + scopes=[_SSH_CERT_SCOPE], + data=_SSH_CERT_DATA, + auth_scheme=None, + expected_token_type="ssh-cert", + ) + From 2d03ad97c5ef865b8d9d1a79b61fb08d6fbdea29 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 12 Mar 2024 21:57:14 -0700 Subject: [PATCH 139/262] MSAL Python 1.28.0 broker-test.py shall test Azure CLI in MSA-PT mode --- msal/application.py | 2 +- setup.cfg | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/msal/application.py b/msal/application.py index d6256635..6d5fb3ad 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.27.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.28.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" diff --git a/setup.cfg b/setup.cfg index 2177d2c5..f6b86519 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ # Format https://setuptools.pypa.io/en/latest/userguide/declarative_config.html [bdist_wheel] -universal=1 +universal=0 [metadata] name = msal @@ -16,11 +16,8 @@ url = https://github.com/AzureAD/microsoft-authentication-library-for-python classifiers = Development Status :: 5 - Production/Stable Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -40,7 +37,8 @@ project_urls = [options] include_package_data = False # We used to ship LICENSE, but our __init__.py already mentions MIT packages = find: -python_requires = >=2.7 +# Our test pipeline currently still covers Py37 +python_requires = >=3.7 install_requires = requests>=2.0.0,<3 @@ -56,7 +54,6 @@ install_requires = # https://cryptography.io/en/latest/api-stability/#deprecation cryptography>=0.6,<45 - mock; python_version<'3.3' [options.extras_require] broker = From 98bcfb3c0ccb53e7b63546abe37ff7f85fbc360e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 14 Jul 2023 18:49:44 -0700 Subject: [PATCH 140/262] Convert most built-in samples from json to dotenv --- sample/.env.sample.entra-id | 26 +++++++ sample/.env.sample.external-id | 22 ++++++ sample/.env.sample.external-id-custom-domain | 23 ++++++ sample/confidential_client_secret_sample.py | 72 +++++++++---------- sample/device_flow_sample.py | 56 +++++++++------ sample/interactive_sample.py | 67 ++++++++++------- sample/username_password_sample.py | 75 +++++++++++--------- 7 files changed, 221 insertions(+), 120 deletions(-) create mode 100644 sample/.env.sample.entra-id create mode 100644 sample/.env.sample.external-id create mode 100644 sample/.env.sample.external-id-custom-domain diff --git a/sample/.env.sample.entra-id b/sample/.env.sample.entra-id new file mode 100644 index 00000000..099c609f --- /dev/null +++ b/sample/.env.sample.entra-id @@ -0,0 +1,26 @@ +# This sample can be configured to work with Microsoft Entra ID. +# +# If you are using a Microsoft Entra ID tenant, +# configure the AUTHORITY variable as +# "https://login.microsoftonline.com/TENANT_GUID" +# or "https://login.microsoftonline.com/contoso.onmicrosoft.com". +# +# Alternatively, use "https://login.microsoftonline.com/common" for multi-tenant app. +AUTHORITY= + +# The following variables are required for the app to run. +CLIENT_ID= + +# Leave it empty if you are using a public client which has no client secret. +CLIENT_SECRET= + +# Multiple scopes can be added into the same line, separated by a space. +# Here we use a Microsoft Graph API as an example +# You may need to use your own API's scope. +SCOPE= + +# Required if the sample app wants to call an API. +#ENDPOINT=https://graph.microsoft.com/v1.0/me + +# Required if you are testing the username_password_sample.py +#USERNAME= diff --git a/sample/.env.sample.external-id b/sample/.env.sample.external-id new file mode 100644 index 00000000..350f94de --- /dev/null +++ b/sample/.env.sample.external-id @@ -0,0 +1,22 @@ +# This sample can be configured to work with Microsoft External ID. +# +# If you are using a Microsoft Entra External ID for customers (CIAM) tenant, +# configure AUTHORITY as https://contoso.ciamlogin.com/contoso.onmicrosoft.com +AUTHORITY= + +# The following variables are required for the app to run. +CLIENT_ID= + +# Leave it empty if you are using a public client which has no client secret. +CLIENT_SECRET= + +# Multiple scopes can be added into the same line, separated by a space. +# Here we use a Microsoft Graph API as an example +# You may need to use your own API's scope. +SCOPE= + +# Required if the sample app wants to call an API. +#ENDPOINT=https://graph.microsoft.com/v1.0/me + +# Required if you are testing the username_password_sample.py +#USERNAME= diff --git a/sample/.env.sample.external-id-custom-domain b/sample/.env.sample.external-id-custom-domain new file mode 100644 index 00000000..580edfe8 --- /dev/null +++ b/sample/.env.sample.external-id-custom-domain @@ -0,0 +1,23 @@ +# This sample can be configured to work with Microsoft External ID with custom domain. +# +# If you are using a Microsoft External ID tenant with custom domain, +# configure the OIDC_AUTHORITY variable as +# "https://www.contoso.com/TENANT_GUID/v2.0" +OIDC_AUTHORITY= + +# The following variables are required for the app to run. +CLIENT_ID= + +# Leave it empty if you are using a public client which has no client secret. +CLIENT_SECRET= + +# Multiple scopes can be added into the same line, separated by a space. +# Here we use a Microsoft Graph API as an example +# You may need to use your own API's scope. +SCOPE= + +# Required if the sample app wants to call an API. +#ENDPOINT=https://graph.microsoft.com/v1.0/me + +# Required if you are testing the username_password_sample.py +#USERNAME= diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_secret_sample.py index 9ff6a81b..c6fa35d5 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_secret_sample.py @@ -1,46 +1,37 @@ """ -The configuration file would look like this (sans those // comments): - -{ - "authority": "https://login.microsoftonline.com/Enter_the_Tenant_Name_Here", - "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", - "scope": ["https://graph.microsoft.com/.default"], - // Specific to Client Credentials Grant i.e. acquire_token_for_client(), - // you don't specify, in the code, the individual scopes you want to access. - // Instead, you statically declared them when registering your application. - // Therefore the only possible scope is "resource/.default" - // (here "https://graph.microsoft.com/.default") - // which means "the static permissions defined in the application". - - "secret": "The secret generated by AAD during your confidential app registration", - // For information about generating client secret, refer: - // https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki/Client-Credentials#registering-client-secrets-using-the-application-registration-portal - - "endpoint": "https://graph.microsoft.com/v1.0/users" - // For this resource to work, you need to visit Application Permissions - // page in portal, declare scope User.Read.All, which needs admin consent - // https://github.com/Azure-Samples/ms-identity-python-daemon/blob/master/1-Call-MsGraph-WithSecret/README.md -} - -You can then run this sample with a JSON configuration file: - - python sample.py parameters.json +This sample demonstrates a daemon application that acquires a token using a +client secret and then calls a web API with the token. + +This sample loads its configuration from a .env file. + +To make this sample work, you need to choose one of the following templates: + + .env.sample.entra-id + .env.sample.external-id + .env.sample.external-id-with-custom-domain + +Copy the chosen template to a new file named .env, and fill in the values. + +You can then run this sample: + + python name_of_this_script.py """ -import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import json import logging +import os import time -import requests +from dotenv import load_dotenv # Need "pip install python-dotenv" import msal +import requests # Optional logging # logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script # logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs -config = json.load(open(sys.argv[1])) +load_dotenv() # We use this to load configuration from a .env file # If for whatever reason you plan to recreate same ClientApplication periodically, # you shall create one global token cache and reuse it by each ClientApplication @@ -49,25 +40,32 @@ # Create a preferably long-lived app instance, to avoid the overhead of app creation global_app = msal.ConfidentialClientApplication( - config["client_id"], authority=config["authority"], - client_credential=config["secret"], + os.getenv('CLIENT_ID'), + authority=os.getenv('AUTHORITY'), # For Entra ID or External ID + oidc_authority=os.getenv('OIDC_AUTHORITY'), # For External ID with custom domain + client_credential=os.getenv('CLIENT_SECRET'), token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) +scopes = os.getenv("SCOPE", "").split() def acquire_and_use_token(): # Since MSAL 1.23, acquire_token_for_client(...) will automatically look up # a token from cache, and fall back to acquire a fresh token when needed. - result = global_app.acquire_token_for_client(scopes=config["scope"]) + result = global_app.acquire_token_for_client(scopes=scopes) if "access_token" in result: print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + if os.getenv('ENDPOINT'): + # Calling a web API using the access token + api_result = requests.get( + os.getenv('ENDPOINT'), + headers={'Authorization': 'Bearer ' + result['access_token']}, + ).json() # Assuming the response is JSON + print("Web API call result", json.dumps(api_result, indent=2)) + else: + print("Token acquisition result", json.dumps(result, indent=2)) else: print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error diff --git a/sample/device_flow_sample.py b/sample/device_flow_sample.py index 7d998470..8883d4b2 100644 --- a/sample/device_flow_sample.py +++ b/sample/device_flow_sample.py @@ -1,36 +1,39 @@ """ +This sample demonstrates a headless application that acquires a token using +the device code flow and then calls a web API with the token. The configuration file would look like this: -{ - "authority": "https://login.microsoftonline.com/common", - "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", - "scope": ["User.ReadBasic.All"], - // You can find the other permission names from this document - // https://docs.microsoft.com/en-us/graph/permissions-reference - "endpoint": "https://graph.microsoft.com/v1.0/users" - // You can find more Microsoft Graph API endpoints from Graph Explorer - // https://developer.microsoft.com/en-us/graph/graph-explorer -} +This sample loads its configuration from a .env file. -You can then run this sample with a JSON configuration file: +To make this sample work, you need to choose one of the following templates: - python sample.py parameters.json + .env.sample.entra-id + .env.sample.external-id + .env.sample.external-id-with-custom-domain + +Copy the chosen template to a new file named .env, and fill in the values. + +You can then run this sample: + + python name_of_this_script.py """ -import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import json import logging +import os +import sys import time -import requests +from dotenv import load_dotenv # Need "pip install python-dotenv" import msal +import requests # Optional logging # logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script # logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs -config = json.load(open(sys.argv[1])) +load_dotenv() # We use this to load configuration from a .env file # If for whatever reason you plan to recreate same ClientApplication periodically, # you shall create one global token cache and reuse it by each ClientApplication @@ -39,10 +42,13 @@ # Create a preferably long-lived app instance, to avoid the overhead of app creation global_app = msal.PublicClientApplication( - config["client_id"], authority=config["authority"], + os.getenv('CLIENT_ID'), + authority=os.getenv('AUTHORITY'), # For Entra ID or External ID + oidc_authority=os.getenv('OIDC_AUTHORITY'), # For External ID with custom domain token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) +scopes = os.getenv("SCOPE", "").split() def acquire_and_use_token(): @@ -61,12 +67,12 @@ def acquire_and_use_token(): # Assuming the end user chose this one chosen = accounts[0] # Now let's try to find a token in cache for this account - result = global_app.acquire_token_silent(config["scope"], account=chosen) + result = global_app.acquire_token_silent(scopes, account=chosen) if not result: logging.info("No suitable token exists in cache. Let's get a new one from AAD.") - flow = global_app.initiate_device_flow(scopes=config["scope"]) + flow = global_app.initiate_device_flow(scopes=scopes) if "user_code" not in flow: raise ValueError( "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) @@ -85,11 +91,15 @@ def acquire_and_use_token(): if "access_token" in result: print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + if os.getenv('ENDPOINT'): + # Calling a web API using the access token + api_result = requests.get( + os.getenv('ENDPOINT'), + headers={'Authorization': 'Bearer ' + result['access_token']}, + ).json() # Assuming the response is JSON + print("Web API call result", json.dumps(api_result, indent=2)) + else: + print("Token acquisition result", json.dumps(result, indent=2)) else: print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index 58d8c9a9..3a361fcf 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -1,33 +1,39 @@ """ +This sample demonstrates a desktop application that acquires a token interactively +and then calls a web API with the token. + Prerequisite is documented here: https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_interactive -The configuration file would look like this: +This sample loads its configuration from a .env file. + +To make this sample work, you need to choose one of the following templates: + + .env.sample.entra-id + .env.sample.external-id + .env.sample.external-id-with-custom-domain -{ - "authority": "https://login.microsoftonline.com/organizations", - "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", - "scope": ["User.ReadBasic.All"], - // You can find the other permission names from this document - // https://docs.microsoft.com/en-us/graph/permissions-reference - "username": "your_username@your_tenant.com", // This is optional - "endpoint": "https://graph.microsoft.com/v1.0/users" - // You can find more Microsoft Graph API endpoints from Graph Explorer - // https://developer.microsoft.com/en-us/graph/graph-explorer -} +Copy the chosen template to a new file named .env, and fill in the values. -You can then run this sample with a JSON configuration file: +You can then run this sample: - python sample.py parameters.json + python name_of_this_script.py """ -import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] -import json, logging, time, msal, requests +import json +import logging +import os +import time + +from dotenv import load_dotenv # Need "pip install python-dotenv" +import msal +import requests + # Optional logging # logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script # logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs -config = json.load(open(sys.argv[1])) +load_dotenv() # We use this to load configuration from a .env file # If for whatever reason you plan to recreate same ClientApplication periodically, # you shall create one global token cache and reuse it by each ClientApplication @@ -36,12 +42,15 @@ # Create a preferably long-lived app instance, to avoid the overhead of app creation global_app = msal.PublicClientApplication( - config["client_id"], authority=config["authority"], + os.getenv('CLIENT_ID'), + authority=os.getenv('AUTHORITY'), # For Entra ID or External ID + oidc_authority=os.getenv('OIDC_AUTHORITY'), # For External ID with custom domain #enable_broker_on_windows=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already # See also: https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-desktop-acquire-token-wam#wam-value-proposition token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) +scopes = os.getenv("SCOPE", "").split() def acquire_and_use_token(): @@ -49,7 +58,7 @@ def acquire_and_use_token(): result = None # Firstly, check the cache to see if this end user has signed in before - accounts = global_app.get_accounts(username=config.get("username")) + accounts = global_app.get_accounts(username=os.getenv("USERNAME")) if accounts: logging.info("Account(s) exists in cache, probably with token too. Let's try.") print("Account(s) already signed in:") @@ -58,15 +67,15 @@ def acquire_and_use_token(): chosen = accounts[0] # Assuming the end user chose this one to proceed print("Proceed with account: %s" % chosen["username"]) # Now let's try to find a token in cache for this account - result = global_app.acquire_token_silent(config["scope"], account=chosen) + result = global_app.acquire_token_silent(scopes, account=chosen) if not result: logging.info("No suitable token exists in cache. Let's get a new one from AAD.") print("A local browser window will be open for you to sign in. CTRL+C to cancel.") result = global_app.acquire_token_interactive( # Only works if your app is registered with redirect_uri as http://localhost - config["scope"], + scopes, #parent_window_handle=..., # If broker is enabled, you will be guided to provide a window handle - login_hint=config.get("username"), # Optional. + login_hint=os.getenv("USERNAME"), # Optional. # If you know the username ahead of time, this parameter can pre-fill # the username (or email address) field of the sign-in page for the user, # Often, apps use this parameter during reauthentication, @@ -80,11 +89,15 @@ def acquire_and_use_token(): if "access_token" in result: print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 - # Calling graph using the access token - graph_response = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},) - print("Graph API call result: %s ..." % graph_response.text[:100]) + if os.getenv('ENDPOINT'): + # Calling a web API using the access token + api_result = requests.get( + os.getenv('ENDPOINT'), + headers={'Authorization': 'Bearer ' + result['access_token']}, + ).json() # Assuming the response is JSON + print("Web API call result", json.dumps(api_result, indent=2)) + else: + print("Token acquisition result", json.dumps(result, indent=2)) else: print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error diff --git a/sample/username_password_sample.py b/sample/username_password_sample.py index 3578e5ef..661d2db2 100644 --- a/sample/username_password_sample.py +++ b/sample/username_password_sample.py @@ -1,39 +1,42 @@ """ -The configuration file would look like this: - -{ - "authority": "https://login.microsoftonline.com/organizations", - "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", - "username": "your_username@your_tenant.com", - "scope": ["User.ReadBasic.All"], - // You can find the other permission names from this document - // https://docs.microsoft.com/en-us/graph/permissions-reference - "endpoint": "https://graph.microsoft.com/v1.0/users" - // You can find more Microsoft Graph API endpoints from Graph Explorer - // https://developer.microsoft.com/en-us/graph/graph-explorer -} - -You can then run this sample with a JSON configuration file: - - python sample.py parameters.json +This sample demonstrates a desktop application that acquires a token using a +pair of username and password, and then calls a web API with the token. + +This sample loads its configuration from a .env file. + +To make this sample work, you need to choose one of the following templates: + + .env.sample.entra-id + .env.sample.external-id + .env.sample.external-id-with-custom-domain + +Copy the chosen template to a new file named .env, and fill in the values. + +You can then run this sample: + + python name_of_this_script.py """ -import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] import getpass import json import logging +import os +import sys import time -import requests +from dotenv import load_dotenv # Need "pip install python-dotenv" import msal +import requests # Optional logging # logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script # logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs -config = json.load(open(sys.argv[1])) -config["password"] = getpass.getpass() +load_dotenv() # We use this to load configuration from a .env file +if not os.getenv("USERNAME"): + sys.exit("Please provide a username in the environment variable USERNAME.") +password = getpass.getpass("Password for {}: ".format(os.getenv("USERNAME"))) # If for whatever reason you plan to recreate same ClientApplication periodically, # you shall create one global token cache and reuse it by each ClientApplication @@ -42,12 +45,14 @@ # Create a preferably long-lived app instance, to avoid the overhead of app creation global_app = msal.ClientApplication( - config["client_id"], - client_credential=config.get("client_secret"), - authority=config["authority"], + os.getenv('CLIENT_ID'), + authority=os.getenv('AUTHORITY'), # For Entra ID or External ID + oidc_authority=os.getenv('OIDC_AUTHORITY'), # For External ID with custom domain + client_credential=os.getenv('CLIENT_SECRET') or None, # Treat empty string as None token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) +scopes = os.getenv("SCOPE", "").split() def acquire_and_use_token(): @@ -55,30 +60,34 @@ def acquire_and_use_token(): result = None # Firstly, check the cache to see if this end user has signed in before - accounts = global_app.get_accounts(username=config["username"]) + accounts = global_app.get_accounts(username=os.getenv("USERNAME")) if accounts: logging.info("Account(s) exists in cache, probably with token too. Let's try.") - result = global_app.acquire_token_silent(config["scope"], account=accounts[0]) + result = global_app.acquire_token_silent(scopes, account=accounts[0]) if not result: logging.info("No suitable token exists in cache. Let's get a new one from AAD.") # See this page for constraints of Username Password Flow. # https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki/Username-Password-Authentication result = global_app.acquire_token_by_username_password( - config["username"], config["password"], scopes=config["scope"]) + os.getenv("USERNAME"), password, scopes=scopes) if "access_token" in result: print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + if os.getenv('ENDPOINT'): + # Calling a web API using the access token + api_result = requests.get( + os.getenv('ENDPOINT'), + headers={'Authorization': 'Bearer ' + result['access_token']}, + ).json() # Assuming the response is JSON + print("Web API call result", json.dumps(api_result, indent=2)) + else: + print("Token acquisition result", json.dumps(result, indent=2)) else: print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error if 65001 in result.get("error_codes", []): # Not mean to be coded programatically, but... raise RuntimeError( - "AAD requires user consent for U/P flow to succeed. " + "Microsoft Entra ID requires user consent for U/P flow to succeed. " "Run acquire_token_interactive() instead.") From 52b1fc5a442ff5dd33f48ce717f1032c8002ea9e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 25 Jul 2023 14:56:35 -0700 Subject: [PATCH 141/262] client_credential has its link in RTD now. Finally. --- msal/application.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 6d5fb3ad..1ed36ab8 100644 --- a/msal/application.py +++ b/msal/application.py @@ -210,7 +210,7 @@ def __init__( :param str client_id: Your app has a client_id after you register it on Microsoft Entra admin center. - :param Union[str, dict] client_credential: + :param client_credential: For :class:`PublicClientApplication`, you use `None` here. For :class:`ConfidentialClientApplication`, it can be a string containing client secret, @@ -254,6 +254,8 @@ def __init__( "client_assertion": "...a JWT with claims aud, exp, iss, jti, nbf, and sub..." } + :type client_credential: Union[dict, str] + :param dict client_claims: *Added in version 0.5.0*: It is a dictionary of extra claims that would be signed by From f821b2dd22572d75f9822ce60411c1f94daa47da Mon Sep 17 00:00:00 2001 From: Peter <34331512+pmaytak@users.noreply.github.com> Date: Mon, 15 Apr 2024 10:45:51 -0700 Subject: [PATCH 142/262] Add CODEOWNERS file (#689) * Create CODEOWNERS * Update CODEOWNERS Co-authored-by: Bogdan Gavril --------- Co-authored-by: Ray Luo Co-authored-by: Bogdan Gavril --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..dd24b01a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @AzureAD/id4s-msal-team From 48a7bd3c8e7040c02165e4e12f3a8004f18391a3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 8 Dec 2023 13:14:39 -0800 Subject: [PATCH 143/262] Should have used the constant instead of a raw str --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 1ed36ab8..f6486f88 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1980,7 +1980,7 @@ def acquire_token_interactive( return self._process_broker_response(response, scopes, data) if auth_scheme: - raise ValueError("auth_scheme is currently only available from broker") + raise ValueError(self._AUTH_SCHEME_UNSUPPORTED) on_before_launching_ui(ui="browser") telemetry_context = self._build_telemetry_context( self.ACQUIRE_TOKEN_INTERACTIVE) From 7fffded754c7b0a918f0631c4f556411ee31f048 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 13 Apr 2024 15:07:58 -0700 Subject: [PATCH 144/262] Raise exception rather than returning None for interaction timeout --- oauth2cli/oauth2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/oauth2cli/oauth2.py b/oauth2cli/oauth2.py index 01ac78a9..01b7fc34 100644 --- a/oauth2cli/oauth2.py +++ b/oauth2cli/oauth2.py @@ -28,6 +28,9 @@ string_types = (str,) if sys.version_info[0] >= 3 else (basestring, ) +class BrowserInteractionTimeoutError(RuntimeError): + pass + class BaseClient(object): # This low-level interface works. Yet you'll find its sub-class # more friendly to remind you what parameters are needed in each scenario. @@ -674,6 +677,8 @@ def _obtain_token_by_browser( auth_uri_callback=auth_uri_callback, browser_name=browser_name, ) + if auth_response is None: + raise BrowserInteractionTimeoutError("User did not complete the flow in time") return self.obtain_token_by_auth_code_flow( flow, auth_response, scope=scope, **kwargs) From dd4dbe7091dbe759521d62698c17c58731a3dc3a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 13 Nov 2023 22:21:40 -0800 Subject: [PATCH 145/262] Upgrade action's versions and also enable cache https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows --- .github/workflows/python-package.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0ef8fe32..ed67e2ce 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,11 +31,12 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 # It automatically takes care of pip cache, according to # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#about-caching-workflow-dependencies with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | @@ -61,15 +62,16 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 + cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Setup an updatable cache for Performance Baselines - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: .perf.baseline key: ${{ runner.os }}-performance-${{ hashFiles('tests/test_benchmark.py') }} @@ -101,9 +103,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 + cache: 'pip' - name: Build a package for release run: | python -m pip install build --user From c4152d2920f16c0416ebcfa1b88c34d862db56e3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 18 Apr 2024 12:58:40 -0700 Subject: [PATCH 146/262] Suggests to use XDG_RUNTIME_DIR for token cache --- msal/token_cache.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index f9a55800..444aa2df 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -353,11 +353,14 @@ class SerializableTokenCache(TokenCache): the following simple recipe for file-based persistence may be sufficient:: import os, atexit, msal + cache_filename = os.path.join( # Persist cache into this file + os.getenv("XDG_RUNTIME_DIR", ""), # Automatically wipe out the cache from Linux when user's ssh session ends. See also https://github.com/AzureAD/microsoft-authentication-library-for-python/issues/690 + "my_cache.bin") cache = msal.SerializableTokenCache() - if os.path.exists("my_cache.bin"): - cache.deserialize(open("my_cache.bin", "r").read()) + if os.path.exists(cache_filename): + cache.deserialize(open(cache_filename, "r").read()) atexit.register(lambda: - open("my_cache.bin", "w").write(cache.serialize()) + open(cache_filename, "w").write(cache.serialize()) # Hint: The following optional line persists only when state changed if cache.has_state_changed else None ) From d0693acb62ec6b507939e1fde0039052bcfe8a4f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 2 May 2024 21:43:47 -0700 Subject: [PATCH 147/262] Support reading CCA cert from a pfx file. Tested. --- .github/workflows/python-package.yml | 5 ++ msal/application.py | 73 +++++++++++++++++++++------ setup.cfg | 6 +-- tests/certificate-with-password.pfx | Bin 0 -> 2485 bytes tests/test_cryptography.py | 24 ++++----- tests/test_e2e.py | 15 +++--- 6 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 tests/certificate-with-password.pfx diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ed67e2ce..db891158 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,6 +18,8 @@ jobs: TRAVIS: true LAB_APP_CLIENT_ID: ${{ secrets.LAB_APP_CLIENT_ID }} LAB_APP_CLIENT_SECRET: ${{ secrets.LAB_APP_CLIENT_SECRET }} + LAB_APP_CLIENT_CERT_BASE64: ${{ secrets.LAB_APP_CLIENT_CERT_BASE64 }} + LAB_APP_CLIENT_CERT_PFX_PATH: lab_cert.pfx LAB_OBO_CLIENT_SECRET: ${{ secrets.LAB_OBO_CLIENT_SECRET }} LAB_OBO_CONFIDENTIAL_CLIENT_ID: ${{ secrets.LAB_OBO_CONFIDENTIAL_CLIENT_ID }} LAB_OBO_PUBLIC_CLIENT_ID: ${{ secrets.LAB_OBO_PUBLIC_CLIENT_ID }} @@ -43,6 +45,9 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Populate lab cert.pfx + # https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#storing-base64-binary-blobs-as-secrets + run: echo $LAB_APP_CLIENT_CERT_BASE64 | base64 -d > $LAB_APP_CLIENT_CERT_PFX_PATH - name: Test with pytest run: pytest --benchmark-skip - name: Lint with flake8 diff --git a/msal/application.py b/msal/application.py index f6486f88..2c72f299 100644 --- a/msal/application.py +++ b/msal/application.py @@ -65,6 +65,29 @@ def _str2bytes(raw): return raw +def _load_private_key_from_pfx_path(pfx_path, passphrase_bytes): + # Cert concepts https://security.stackexchange.com/a/226758/125264 + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.serialization import pkcs12 + with open(pfx_path, 'rb') as f: + private_key, cert, _ = pkcs12.load_key_and_certificates( # cryptography 2.5+ + # https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/#cryptography.hazmat.primitives.serialization.pkcs12.load_key_and_certificates + f.read(), passphrase_bytes) + sha1_thumbprint = cert.fingerprint(hashes.SHA1()).hex() # cryptography 0.7+ + # https://cryptography.io/en/latest/x509/reference/#x-509-certificate-object + return private_key, sha1_thumbprint + + +def _load_private_key_from_pem_str(private_key_pem_str, passphrase_bytes): + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.backends import default_backend + return serialization.load_pem_private_key( # cryptography 0.6+ + _str2bytes(private_key_pem_str), + passphrase_bytes, + backend=default_backend(), # It was a required param until 2020 + ) + + def _pii_less_home_account_id(home_account_id): parts = home_account_id.split(".") # It could contain one or two parts parts[0] = "********" @@ -254,6 +277,16 @@ def __init__( "client_assertion": "...a JWT with claims aud, exp, iss, jti, nbf, and sub..." } + .. admonition:: Supporting reading client cerficates from PFX files + + *Added in version 1.29.0*: + Feed in a dictionary containing the path to a PFX file:: + + { + "private_key_pfx_path": "/path/to/your.pfx", + "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", + } + :type client_credential: Union[dict, str] :param dict client_claims: @@ -651,29 +684,37 @@ def _build_client(self, client_credential, authority, skip_regional_client=False default_headers['x-app-ver'] = self.app_version default_body = {"client_info": 1} if isinstance(client_credential, dict): - assert (("private_key" in client_credential - and "thumbprint" in client_credential) or - "client_assertion" in client_credential) client_assertion_type = Client.CLIENT_ASSERTION_TYPE_JWT - if 'client_assertion' in client_credential: + # Use client_credential.get("...") rather than "..." in client_credential + # so that we can ignore an empty string came from an empty ENV VAR. + if client_credential.get("client_assertion"): client_assertion = client_credential['client_assertion'] else: headers = {} - if 'public_certificate' in client_credential: + if client_credential.get('public_certificate'): headers["x5c"] = extract_certs(client_credential['public_certificate']) - if not client_credential.get("passphrase"): - unencrypted_private_key = client_credential['private_key'] + passphrase_bytes = _str2bytes( + client_credential["passphrase"] + ) if client_credential.get("passphrase") else None + if client_credential.get("private_key_pfx_path"): + private_key, sha1_thumbprint = _load_private_key_from_pfx_path( + client_credential["private_key_pfx_path"], passphrase_bytes) + elif ( + client_credential.get("private_key") # PEM blob + and client_credential.get("thumbprint")): + sha1_thumbprint = client_credential["thumbprint"] + if passphrase_bytes: + private_key = _load_private_key_from_pem_str( + client_credential['private_key'], passphrase_bytes) + else: # PEM without passphrase + private_key = client_credential['private_key'] else: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.backends import default_backend - unencrypted_private_key = serialization.load_pem_private_key( - _str2bytes(client_credential["private_key"]), - _str2bytes(client_credential["passphrase"]), - backend=default_backend(), # It was a required param until 2020 - ) + raise ValueError( + "client_credential needs to follow this format " + "https://msal-python.readthedocs.io/en/latest/#msal.ClientApplication.params.client_credential") assertion = JwtAssertionCreator( - unencrypted_private_key, algorithm="RS256", - sha1_thumbprint=client_credential.get("thumbprint"), headers=headers) + private_key, algorithm="RS256", + sha1_thumbprint=sha1_thumbprint, headers=headers) client_assertion = assertion.create_regenerative_assertion( audience=authority.token_endpoint, issuer=self.client_id, additional_claims=self.client_claims or {}) diff --git a/setup.cfg b/setup.cfg index f6b86519..3e890114 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,13 +46,13 @@ install_requires = # therefore is insusceptible to CVE-2022-29217 so no need to bump to PyJWT 2.4+ PyJWT[crypto]>=1.0.0,<3 - # load_pem_private_key() is available since 0.6 - # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#06---2014-09-29 + # load_key_and_certificates() is available since 2.5 + # https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/#cryptography.hazmat.primitives.serialization.pkcs12.load_key_and_certificates # # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=0.6,<45 + cryptography>=2.5,<45 [options.extras_require] diff --git a/tests/certificate-with-password.pfx b/tests/certificate-with-password.pfx new file mode 100644 index 0000000000000000000000000000000000000000..a8d830c937a68de4f82b21d0aa0a84b8cf2ec982 GIT binary patch literal 2485 zcmY+^XE+;*9>;MZi3EuqdsS<;nwT{jTeawj5qtEgU8`EhCe(~lgTYX_c;om37=Rg2goAnj zaZuM^*%^rg^8GV`i*P{5uLNWSFd%VEKmS<(NMSI`zdx`6z(_a{GBlxJ_t`$Vp^Yc4*Nbz2#<>W(n>6^k9nx0UmR{^f)-O>pzNRV+#MMR!fbIjYwOgoF=8S9G_Z z{}36t@53m+aA43*fZ1xegvH|Q%=L>iOXi(8G-mQOr1t$A2<+)sW-s8|NjyBhWn&s& z2|B&2m_h7TQa8lO05ye6KaJzddYEeoGabL%VzFUz&cIJ$59`Si7$9pQ(?x4O@jf90 zJ=W`P*4JU_{s0UU4!)C!k9y!KZTvmnKfdfrY&p-t5d*r{>*1 zWD9m1f59mHr>2@o*D0Y;Op)W2GYeWRpzKHD=_AW~(<3_Aq{vgv?j8R^y7@d37?R0i zM_WE(u0MfuRR&&gv>bbMK@kQkP@=`|YS|nxkcX6Bn<`$xa+EWrGzZvxVG?)=-VZtv zRHp8UAzqZWn_)IB-bi#TG>wb8S}5_h7aq|KqtdOm;&U&P5Ts3G3f{7IZoUc4Z>dt- z-t$6tWrm74;z+x$Toot(dajcVhblayu=q`pT8gOO$glTogDJfdY(+@@ZSu7LxGTSk zwf7J;=jYw-u9FN4={rTv<*4MZ_g>15{<^z3{XX3bzV6sYvJ9-rv`ggDnSi71w&gF9^e33U(TeM`Ts#s_{;UL*;9mYykg zsJ&dLNBfRu*yy}V+T}9b_1=wJyZ{BYLokj^*?<)4n#Ba#h{ zPT+~x1AVV-IEBP)-4X*AsnLpDjRW}bZIVM$67%U{fLal2-}m>C_Bu!$NcVpLD8hjh zfjE%DuPpcLq`<8I;eZ9e_{)v2zuegPKP8xcE1@rC{B<%?e*3o)Kpb!&;%b|;b1G3U z#yrdV&dGBW-KHyzXTz(L&K)GHGj(i*@^D1|n$5Q&p4vjM)RdX5-bY5!*iV=nc#*t4xMM_IZ?@u}PdmhCt)9^)&juy#q!`P>z> zko0`QGTPyP?8SXiU56Q)?Ge zJ3M8>YCuPbzf7YXnPHaZZ!Y# zx)II4G~$`bPA-w$xBqI=)8m~rrJL@nf&6{96_D^xy{r(ACMu)G1-!Rw<7q*o$L$m){x; z{xeF@w9Apw=+4JOvc3K(RD9Cmh2`N%Q_94TEE?cm%&>Kw`{fc4&IfEsRh28G=x!m1 z`?r>*03iJ#J$QKxXLa1H4-6jGd2!5$zTVyKIHWX7!S{*DZ|-F# zzPrSQDO8xBl7aK*i-zJo7rGc);zM(m6jxXQ0eX^?awR@jy(tF+>=tFpSDGdITPNu$ zyTyYd-L4`pdvK48oSF6n2D2Szx%z+5^INQEYF?f5yS7t1<~GpXISMBo%Px9|2@1#V zxl|5Hd|s~a?lo`uo6Q6k;k~29iO?qrWHfaWS1B#V?4w?G#U@yr#}~(ysgJ*8m|rWt zjjKKhnT80$9aQ&+U{&O%LGEfh2(kt=HFIyv}OXcSvpboB0d z$AJAOmxWZZ~IIy`N)%12Xnk5%xHEx6SdC`*(6@$ai2V8#YYaA z`%71G#DLh=aDvD%{Cq|EAuV+*H)eWMYnzC@Me1J1n7w0@2u+7*l@ut;`+sUo+}_Kf zStoan21+QAYgUJYm}G382R5wlVi1OfSF{oQGL>d&+NeCuk_yLk$m-w+`-%Qc(q1`I z&TfdFefG01i)%3VgUGG@raP{86!BIeTBPhI(;Pqd^%{3dDwJfb?MV&O%M< zQMti^NnqBvf(m%RT$F@6OGmx!dHcm7iT9YD><7BVx4XW)PfTd8e0_B~At+#JSF?%E z4bSW%ix7-^Hmd4oBL@4`m+?$3f)cFLZ@ts0DP8(r<2o&JL7S=%-eczsgShrYo%I!U& zKF@lcr*!2jL_i{um*Pr*Qekns8;U9^BM-)|-V1qd{pRR910 literal 0 HcmV?d00001 diff --git a/tests/test_cryptography.py b/tests/test_cryptography.py index 5af5722a..bae06e9b 100644 --- a/tests/test_cryptography.py +++ b/tests/test_cryptography.py @@ -7,7 +7,8 @@ import requests -from msal.application import _str2bytes +from msal.application import ( + _str2bytes, _load_private_key_from_pem_str, _load_private_key_from_pfx_path) latest_cryptography_version = ET.fromstring( @@ -26,6 +27,10 @@ def get_current_ceiling(): raise RuntimeError("Unable to find cryptography info from setup.cfg") +def sibling(filename): + return os.path.join(os.path.dirname(__file__), filename) + + class CryptographyTestCase(TestCase): def test_should_be_run_with_latest_version_of_cryptography(self): @@ -37,18 +42,13 @@ def test_should_be_run_with_latest_version_of_cryptography(self): cryptography.__version__, latest_cryptography_version)) def test_latest_cryptography_should_support_our_usage_without_warnings(self): - with open(os.path.join( - os.path.dirname(__file__), "certificate-with-password.pem")) as f: - cert = f.read() + passphrase_bytes = _str2bytes("password") with warnings.catch_warnings(record=True) as encountered_warnings: - # The usage was copied from application.py - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.backends import default_backend - unencrypted_private_key = serialization.load_pem_private_key( - _str2bytes(cert), - _str2bytes("password"), - backend=default_backend(), # It was a required param until 2020 - ) + with open(sibling("certificate-with-password.pem")) as f: + _load_private_key_from_pem_str(f.read(), passphrase_bytes) + pfx = sibling("certificate-with-password.pfx") # Created by: + # openssl pkcs12 -export -inkey test/certificate-with-password.pem -in tests/certificate-with-password.pem -out tests/certificate-with-password.pfx + _load_private_key_from_pfx_path(pfx, passphrase_bytes) self.assertEqual(0, len(encountered_warnings), "Did cryptography deprecate the functions that we used?") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 33c8cf54..8003dd49 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -446,6 +446,7 @@ def test_device_flow(self): def get_lab_app( env_client_id="LAB_APP_CLIENT_ID", env_name2="LAB_APP_CLIENT_SECRET", # A var name that hopefully avoids false alarm + env_client_cert_path="LAB_APP_CLIENT_CERT_PFX_PATH", authority="https://login.microsoftonline.com/" "72f988bf-86f1-41af-91ab-2d7cd011db47", # Microsoft tenant ID timeout=None, @@ -458,21 +459,23 @@ def get_lab_app( "Reading ENV variables %s and %s for lab app defined at " "https://docs.msidlab.com/accounts/confidentialclient.html", env_client_id, env_name2) - if os.getenv(env_client_id) and os.getenv(env_name2): - # A shortcut mainly for running tests on developer's local development machine - # or it could be setup on Travis CI - # https://docs.travis-ci.com/user/environment-variables/#defining-variables-in-repository-settings + if os.getenv(env_client_id) and os.getenv(env_client_cert_path): + # id came from https://docs.msidlab.com/accounts/confidentialclient.html + client_id = os.getenv(env_client_id) + # Cert came from https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/asset/Microsoft_Azure_KeyVault/Certificate/https://msidlabs.vault.azure.net/certificates/LabVaultAccessCert + client_credential = {"private_key_pfx_path": os.getenv(env_client_cert_path)} + elif os.getenv(env_client_id) and os.getenv(env_name2): # Data came from here # https://docs.msidlab.com/accounts/confidentialclient.html client_id = os.getenv(env_client_id) - client_secret = os.getenv(env_name2) + client_credential = os.getenv(env_name2) else: logger.info("ENV variables are not defined. Fall back to MSI.") # See also https://microsoft.sharepoint-df.com/teams/MSIDLABSExtended/SitePages/Programmatically-accessing-LAB-API's.aspx raise unittest.SkipTest("MSI-based mechanism has not been implemented yet") return msal.ConfidentialClientApplication( client_id, - client_credential=client_secret, + client_credential=client_credential, authority=authority, http_client=MinimalHttpClient(timeout=timeout), **kwargs) From b081f3df2cd3c45cb455a0109e530d7c040d0a8a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 24 May 2024 12:30:42 -0700 Subject: [PATCH 148/262] Lab API changed since May 14, 2024 --- tests/test_e2e.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 8003dd49..8bc7377a 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -963,8 +963,7 @@ class CiamTestCase(LabBasedTestCase): @classmethod def setUpClass(cls): super(CiamTestCase, cls).setUpClass() - cls.user = cls.get_lab_user( - federationProvider="ciam", signinAudience="azureadmyorg", publicClient="No") + cls.user = cls.get_lab_user(federationProvider="ciam") # FYI: Only single- or multi-tenant CIAM app can have other-than-OIDC # delegated permissions on Microsoft Graph. cls.app_config = cls.get_lab_app_object(cls.user["client_id"]) From 3a5990ad504abd3c3cbaa038d15df8963a6284e4 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 3 Aug 2023 15:04:26 -0700 Subject: [PATCH 149/262] No longer need to fake device code flow endpoint --- msal/application.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/msal/application.py b/msal/application.py index 2c72f299..a722b285 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1,10 +1,6 @@ import functools import json import time -try: # Python 2 - from urlparse import urljoin -except: # Python 3 - from urllib.parse import urljoin import logging import sys import warnings @@ -723,9 +719,7 @@ def _build_client(self, client_credential, authority, skip_regional_client=False central_configuration = { "authorization_endpoint": authority.authorization_endpoint, "token_endpoint": authority.token_endpoint, - "device_authorization_endpoint": - authority.device_authorization_endpoint or - urljoin(authority.token_endpoint, "devicecode"), + "device_authorization_endpoint": authority.device_authorization_endpoint, } central_client = _ClientWithCcsRoutingInfo( central_configuration, @@ -749,8 +743,7 @@ def _build_client(self, client_credential, authority, skip_regional_client=False "authorization_endpoint": regional_authority.authorization_endpoint, "token_endpoint": regional_authority.token_endpoint, "device_authorization_endpoint": - regional_authority.device_authorization_endpoint or - urljoin(regional_authority.token_endpoint, "devicecode"), + regional_authority.device_authorization_endpoint, } regional_client = _ClientWithCcsRoutingInfo( regional_configuration, From 41dbf29f4bcc23a54cd3de3141c1caa43455f0bb Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 11 Apr 2024 21:20:53 -0700 Subject: [PATCH 150/262] Promote TokenCache._find() to TokenCache.search() Change all find() in application.py to search() Update msal/token_cache.py Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> Refine inline comments --- msal/application.py | 47 ++++++++++++++++++++++++++++----------------- msal/token_cache.py | 11 ++++++++--- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/msal/application.py b/msal/application.py index a722b285..6a6100fd 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1142,7 +1142,7 @@ def _find_msal_accounts(self, environment): "local_account_id": a.get("local_account_id"), # Tenant-specific "realm": a.get("realm"), # Tenant-specific } - for a in self.token_cache.find( + for a in self.token_cache.search( TokenCache.CredentialType.ACCOUNT, query={"environment": environment}) if a["authority_type"] in interested_authority_types @@ -1188,18 +1188,22 @@ def _sign_out(self, home_account): "home_account_id": home_account["home_account_id"],} # realm-independent app_metadata = self._get_app_metadata(home_account["environment"]) # Remove RTs/FRTs, and they are realm-independent - for rt in [rt for rt in self.token_cache.find( + for rt in [ # Remove RTs from a static list (rather than from a dynamic generator), + # to avoid changing self.token_cache while it is being iterated + rt for rt in self.token_cache.search( TokenCache.CredentialType.REFRESH_TOKEN, query=owned_by_home_account) # Do RT's app ownership check as a precaution, in case family apps # and 3rd-party apps share same token cache, although they should not. if rt["client_id"] == self.client_id or ( app_metadata.get("family_id") # Now let's settle family business and rt.get("family_id") == app_metadata["family_id"]) - ]: + ]: self.token_cache.remove_rt(rt) - for at in self.token_cache.find( # Remove ATs - # Regardless of realm, b/c we've removed realm-independent RTs anyway - TokenCache.CredentialType.ACCESS_TOKEN, query=owned_by_home_account): + for at in list(self.token_cache.search( # Remove ATs from a static list, + # to avoid changing self.token_cache while it is being iterated + TokenCache.CredentialType.ACCESS_TOKEN, query=owned_by_home_account, + # Regardless of realm, b/c we've removed realm-independent RTs anyway + )): # To avoid the complexity of locating sibling family app's AT, # we skip AT's app ownership check. # It means ATs for other apps will also be removed, it is OK because: @@ -1213,11 +1217,15 @@ def _forget_me(self, home_account): owned_by_home_account = { "environment": home_account["environment"], "home_account_id": home_account["home_account_id"],} # realm-independent - for idt in self.token_cache.find( # Remove IDTs, regardless of realm - TokenCache.CredentialType.ID_TOKEN, query=owned_by_home_account): + for idt in list(self.token_cache.search( # Remove IDTs from a static list, + # to avoid changing self.token_cache while it is being iterated + TokenCache.CredentialType.ID_TOKEN, query=owned_by_home_account, # regardless of realm + )): self.token_cache.remove_idt(idt) - for a in self.token_cache.find( # Remove Accounts, regardless of realm - TokenCache.CredentialType.ACCOUNT, query=owned_by_home_account): + for a in list(self.token_cache.search( # Remove Accounts from a static list, + # to avoid changing self.token_cache while it is being iterated + TokenCache.CredentialType.ACCOUNT, query=owned_by_home_account, # regardless of realm + )): self.token_cache.remove_account(a) def _acquire_token_by_cloud_shell(self, scopes, data=None): @@ -1350,12 +1358,12 @@ def _acquire_token_silent_with_error( return result final_result = result for alias in self._get_authority_aliases(self.authority.instance): - if not self.token_cache.find( + if not list(self.token_cache.search( # Need a list to test emptiness self.token_cache.CredentialType.REFRESH_TOKEN, # target=scopes, # MUST NOT filter by scopes, because: # 1. AAD RTs are scope-independent; # 2. therefore target is optional per schema; - query={"environment": alias}): + query={"environment": alias})): # Skip heavy weight logic when RT for this alias doesn't exist continue the_authority = Authority( @@ -1410,11 +1418,13 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( query["key_id"] = key_id now = time.time() refresh_reason = msal.telemetry.AT_ABSENT - for entry in self.token_cache._find( # It returns a generator + for entry in self.token_cache.search( # A generator allows us to + # break early in cache-hit without finding a full list self.token_cache.CredentialType.ACCESS_TOKEN, target=scopes, query=query, - ): # Note that _find() holds a lock during this for loop; + ): # This loop is about token search, not about token deletion. + # Note that search() holds a lock during this loop; # that is fine because this loop is fast expires_in = int(entry["expires_on"]) - now if expires_in < 5*60: # Then consider it expired @@ -1552,10 +1562,10 @@ def _acquire_token_silent_by_finding_specific_refresh_token( rt_remover=None, break_condition=lambda response: False, refresh_reason=None, correlation_id=None, claims_challenge=None, **kwargs): - matches = self.token_cache.find( + matches = list(self.token_cache.search( # We want a list to test emptiness self.token_cache.CredentialType.REFRESH_TOKEN, # target=scopes, # AAD RTs are scope-independent - query=query) + query=query)) logger.debug("Found %d RTs matching %s", len(matches), { k: _pii_less_home_account_id(v) if k == "home_account_id" and v else v for k, v in query.items() @@ -2252,11 +2262,12 @@ def remove_tokens_for_client(self): :func:`~acquire_token_for_client()` for the current client.""" for env in [self.authority.instance] + self._get_authority_aliases( self.authority.instance): - for at in self.token_cache.find(TokenCache.CredentialType.ACCESS_TOKEN, query={ + for at in list(self.token_cache.search( # Remove ATs from a snapshot + TokenCache.CredentialType.ACCESS_TOKEN, query={ "client_id": self.client_id, "environment": env, "home_account_id": None, # These are mostly app-only tokens - }): + })): self.token_cache.remove_at(at) # acquire_token_for_client() obtains no RTs, so we have no RT to remove diff --git a/msal/token_cache.py b/msal/token_cache.py index 444aa2df..ffa0090a 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -2,6 +2,7 @@ import threading import time import logging +import warnings from .authority import canonicalize from .oauth2cli.oidc import decode_part, decode_id_token @@ -117,7 +118,7 @@ def _get(self, credential_type, key, default=None): # O(1) with self._lock: return self._cache.get(credential_type, {}).get(key, default) - def _find(self, credential_type, target=None, query=None): # O(n) generator + def search(self, credential_type, target=None, query=None): # O(n) generator """Returns a generator of matching entries. It is O(1) for AT hits, and O(n) for other types. @@ -150,8 +151,12 @@ def _find(self, credential_type, target=None, query=None): # O(n) generator if entry != preferred_result: # Avoid yielding the same entry twice yield entry - def find(self, credential_type, target=None, query=None): # Obsolete. Use _find() instead. - return list(self._find(credential_type, target=target, query=query)) + def find(self, credential_type, target=None, query=None): + """Equivalent to list(search(...)).""" + warnings.warn( + "Use list(search(...)) instead to explicitly get a list.", + DeprecationWarning) + return list(self.search(credential_type, target=target, query=query)) def add(self, event, now=None): """Handle a token obtaining event, and add tokens into cache.""" From 69874585c2f01c244535a6f10e10481097d648e8 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 28 May 2024 15:54:56 -0700 Subject: [PATCH 151/262] Update the ENV VAR hints in the top of test_e2e.py --- tests/test_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 8bc7377a..5b451558 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1,7 +1,7 @@ """If the following ENV VAR were available, many end-to-end test cases would run. -LAB_APP_CLIENT_SECRET=... LAB_OBO_CLIENT_SECRET=... LAB_APP_CLIENT_ID=... +LAB_APP_CLIENT_CERT_PFX_PATH=... LAB_OBO_PUBLIC_CLIENT_ID=... LAB_OBO_CONFIDENTIAL_CLIENT_ID=... """ From ddb94b213e31ccc5f1f9ba5f174ccf3b62aeec08 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 7 Jun 2024 02:06:12 -0700 Subject: [PATCH 152/262] Pick up PyMsalRuntime 0.16.x --- msal/application.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index 2c72f299..4463362c 100644 --- a/msal/application.py +++ b/msal/application.py @@ -25,7 +25,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.28.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.28.1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" diff --git a/setup.cfg b/setup.cfg index 3e890114..4935d352 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,7 +62,7 @@ broker = # most existing MSAL Python apps do not have the redirect_uri needed by broker. # MSAL Python uses a subset of API from PyMsalRuntime 0.13.0+, # but we still bump the lower bound to 0.13.2+ for its important bugfix (https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/3244) - pymsalruntime>=0.13.2,<0.15; python_version>='3.6' and platform_system=='Windows' + pymsalruntime>=0.13.2,<0.17; python_version>='3.6' and platform_system=='Windows' [options.packages.find] exclude = From cf238926d67b7aa890432d198281fbe450c0f176 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 10 Jun 2024 22:33:00 -0700 Subject: [PATCH 153/262] Adapting to a lab change introduced today --- tests/test_e2e.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 8bc7377a..295da403 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -480,12 +480,17 @@ def get_lab_app( http_client=MinimalHttpClient(timeout=timeout), **kwargs) +class LabTokenError(RuntimeError): + pass + def get_session(lab_app, scopes): # BTW, this infrastructure tests the confidential client flow logger.info("Creating session") result = lab_app.acquire_token_for_client(scopes) - assert result.get("access_token"), \ - "Unable to obtain token for lab. Encountered {}: {}".format( - result.get("error"), result.get("error_description")) + if not result.get("access_token"): + raise LabTokenError( + "Unable to obtain token for lab. Encountered {}: {}".format( + result.get("error"), result.get("error_description") + )) session = requests.Session() session.headers.update({"Authorization": "Bearer %s" % result["access_token"]}) session.hooks["response"].append(lambda r, *args, **kwargs: r.raise_for_status()) @@ -502,7 +507,13 @@ class LabBasedTestCase(E2eTestCase): @classmethod def setUpClass(cls): # https://docs.msidlab.com/accounts/apiaccess.html#code-snippet - cls.session = get_session(get_lab_app(), ["https://msidlab.com/.default"]) + try: + cls.session = get_session(get_lab_app(), ["https://msidlab.com/.default"]) + except LabTokenError: + cls.session = get_session(get_lab_app(), [ + # A lab change since June 10, 2024 which may or may not be reverted + "https://request.msidlab.com/.default", + ]) @classmethod def tearDownClass(cls): From 68a7cb342bef68f498dac3d97618f5d817a42986 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 5 Jun 2024 12:34:56 -0700 Subject: [PATCH 154/262] Read credential from pfx --- msal/application.py | 4 + sample/.env.sample.entra-id | 10 ++- sample/.env.sample.external-id | 10 ++- sample/.env.sample.external-id-custom-domain | 10 ++- .../confidential_client_certificate_sample.py | 80 ------------------- ...ample.py => confidential_client_sample.py} | 3 +- 6 files changed, 27 insertions(+), 90 deletions(-) delete mode 100644 sample/confidential_client_certificate_sample.py rename sample/{confidential_client_secret_sample.py => confidential_client_sample.py} (93%) diff --git a/msal/application.py b/msal/application.py index 3e890cdc..e5e0bd7d 100644 --- a/msal/application.py +++ b/msal/application.py @@ -283,6 +283,10 @@ def __init__( "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", } + The following command will generate a .pfx file from your .key and .pem file:: + + openssl pkcs12 -export -out certificate.pfx -inkey privateKey.key -in certificate.pem + :type client_credential: Union[dict, str] :param dict client_claims: diff --git a/sample/.env.sample.entra-id b/sample/.env.sample.entra-id index 099c609f..ea0e75c4 100644 --- a/sample/.env.sample.entra-id +++ b/sample/.env.sample.entra-id @@ -8,11 +8,15 @@ # Alternatively, use "https://login.microsoftonline.com/common" for multi-tenant app. AUTHORITY= -# The following variables are required for the app to run. CLIENT_ID= -# Leave it empty if you are using a public client which has no client secret. -CLIENT_SECRET= +# Uncomment the following setting if you are using a confidential client +# which has a client secret. Example value: your password +#CLIENT_SECRET= + +# Configure this if you are using a confidential client which has a client credential. +# Example value: {"private_key_pfx_path": "/path/to/your.pfx"} +CLIENT_CREDENTIAL_JSON= # Multiple scopes can be added into the same line, separated by a space. # Here we use a Microsoft Graph API as an example diff --git a/sample/.env.sample.external-id b/sample/.env.sample.external-id index 350f94de..68e739f1 100644 --- a/sample/.env.sample.external-id +++ b/sample/.env.sample.external-id @@ -4,11 +4,15 @@ # configure AUTHORITY as https://contoso.ciamlogin.com/contoso.onmicrosoft.com AUTHORITY= -# The following variables are required for the app to run. CLIENT_ID= -# Leave it empty if you are using a public client which has no client secret. -CLIENT_SECRET= +# Uncomment the following setting if you are using a confidential client +# which has a client secret. Example value: your password +#CLIENT_SECRET= + +# Configure this if you are using a confidential client which has a client credential. +# Example value: {"private_key_pfx_path": "/path/to/your.pfx"} +CLIENT_CREDENTIAL_JSON= # Multiple scopes can be added into the same line, separated by a space. # Here we use a Microsoft Graph API as an example diff --git a/sample/.env.sample.external-id-custom-domain b/sample/.env.sample.external-id-custom-domain index 580edfe8..1d903302 100644 --- a/sample/.env.sample.external-id-custom-domain +++ b/sample/.env.sample.external-id-custom-domain @@ -5,11 +5,15 @@ # "https://www.contoso.com/TENANT_GUID/v2.0" OIDC_AUTHORITY= -# The following variables are required for the app to run. CLIENT_ID= -# Leave it empty if you are using a public client which has no client secret. -CLIENT_SECRET= +# Uncomment the following setting if you are using a confidential client +# which has a client secret. Example value: your password +#CLIENT_SECRET= + +# Configure this if you are using a confidential client which has a client credential. +# Example value: {"private_key_pfx_path": "/path/to/your.pfx"} +CLIENT_CREDENTIAL_JSON= # Multiple scopes can be added into the same line, separated by a space. # Here we use a Microsoft Graph API as an example diff --git a/sample/confidential_client_certificate_sample.py b/sample/confidential_client_certificate_sample.py deleted file mode 100644 index 2c4118a3..00000000 --- a/sample/confidential_client_certificate_sample.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -The configuration file would look like this (sans those // comments): - -{ - "authority": "https://login.microsoftonline.com/Enter_the_Tenant_Name_Here", - "client_id": "your_client_id came from https://learn.microsoft.com/entra/identity-platform/quickstart-register-app", - "scope": ["https://graph.microsoft.com/.default"], - // Specific to Client Credentials Grant i.e. acquire_token_for_client(), - // you don't specify, in the code, the individual scopes you want to access. - // Instead, you statically declared them when registering your application. - // Therefore the only possible scope is "resource/.default" - // (here "https://graph.microsoft.com/.default") - // which means "the static permissions defined in the application". - - "thumbprint": "790E... The thumbprint generated by AAD when you upload your public cert", - "private_key_file": "filename.pem", - // For information about generating thumbprint and private key file, refer: - // https://github.com/AzureAD/microsoft-authentication-library-for-python/wiki/Client-Credentials#client-credentials-with-certificate - - "endpoint": "https://graph.microsoft.com/v1.0/users" - // For this resource to work, you need to visit Application Permissions - // page in portal, declare scope User.Read.All, which needs admin consent - // https://github.com/Azure-Samples/ms-identity-python-daemon/blob/master/2-Call-MsGraph-WithCertificate/README.md -} - -You can then run this sample with a JSON configuration file: - - python sample.py parameters.json -""" - -import sys # For simplicity, we'll read config file from 1st CLI param sys.argv[1] -import json -import logging -import time - -import requests -import msal - - -# Optional logging -# logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script -# logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs - -config = json.load(open(sys.argv[1])) - -# If for whatever reason you plan to recreate same ClientApplication periodically, -# you shall create one global token cache and reuse it by each ClientApplication -global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. - # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache - -# Create a preferably long-lived app instance, to avoid the overhead of app creation -global_app = msal.ConfidentialClientApplication( - config["client_id"], authority=config["authority"], - client_credential={"thumbprint": config["thumbprint"], "private_key": open(config['private_key_file']).read()}, - token_cache=global_token_cache, # Let this app (re)use an existing token cache. - # If absent, ClientApplication will create its own empty token cache - ) - - -def acquire_and_use_token(): - # Since MSAL 1.23, acquire_token_for_client(...) will automatically look up - # a token from cache, and fall back to acquire a fresh token when needed. - result = global_app.acquire_token_for_client(scopes=config["scope"]) - - if "access_token" in result: - print("Token was obtained from:", result["token_source"]) # Since MSAL 1.25 - # Calling graph using the access token - graph_data = requests.get( # Use token to call downstream service - config["endpoint"], - headers={'Authorization': 'Bearer ' + result['access_token']},).json() - print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) - else: - print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error - - -while True: # Here we mimic a long-lived daemon - acquire_and_use_token() - print("Press Ctrl-C to stop.") - time.sleep(5) # Let's say your app would run a workload every X minutes - diff --git a/sample/confidential_client_secret_sample.py b/sample/confidential_client_sample.py similarity index 93% rename from sample/confidential_client_secret_sample.py rename to sample/confidential_client_sample.py index c6fa35d5..e65af15a 100644 --- a/sample/confidential_client_secret_sample.py +++ b/sample/confidential_client_sample.py @@ -43,7 +43,8 @@ os.getenv('CLIENT_ID'), authority=os.getenv('AUTHORITY'), # For Entra ID or External ID oidc_authority=os.getenv('OIDC_AUTHORITY'), # For External ID with custom domain - client_credential=os.getenv('CLIENT_SECRET'), + client_credential=os.getenv('CLIENT_SECRET') # ENV VAR contains a quotation mark-less string + or json.loads(os.getenv('CLIENT_CREDENTIAL_JSON')), # ENV VAR contains a JSON blob as a string token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) From 27fbffc54f9df17d611739e4d25cc2fe33b23a1f Mon Sep 17 00:00:00 2001 From: Feng Gao Date: Mon, 17 Jun 2024 15:52:34 -0700 Subject: [PATCH 155/262] Enable public client ROPC via broker. On Windows, ROPC will call WAM; on macOS, ROPC will call MSAL C++ logic. --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index e5e0bd7d..57446d08 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1710,7 +1710,7 @@ def acquire_token_by_username_password( """ claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) - if False: # Disabled, for now. It was if self._enable_broker: + if self._enable_broker: from .broker import _signin_silently response = _signin_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), From 11f317019b87cdfe051cbbfa677eefea39c7c65d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 19 Jun 2024 16:33:16 -0700 Subject: [PATCH 156/262] Remove a fallback which ends up breaking tests now --- tests/test_e2e.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 2039d035..223117b8 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -507,12 +507,8 @@ class LabBasedTestCase(E2eTestCase): @classmethod def setUpClass(cls): # https://docs.msidlab.com/accounts/apiaccess.html#code-snippet - try: - cls.session = get_session(get_lab_app(), ["https://msidlab.com/.default"]) - except LabTokenError: - cls.session = get_session(get_lab_app(), [ - # A lab change since June 10, 2024 which may or may not be reverted - "https://request.msidlab.com/.default", + cls.session = get_session(get_lab_app(), [ + "https://request.msidlab.com/.default", # A lab change since June 10, 2024 ]) @classmethod From da156fe360be6e8d2c855c401ff3b581d4ec6550 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 14 Jun 2024 13:02:35 -0700 Subject: [PATCH 157/262] Improve error message to fix 710 --- msal/application.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/msal/application.py b/msal/application.py index e5e0bd7d..18cb6df1 100644 --- a/msal/application.py +++ b/msal/application.py @@ -624,8 +624,12 @@ def _decorate_scope( # We could make the developer pass these and then if they do they will # come back asking why they don't see refresh token or user information. raise ValueError( - "API does not accept {} value as user-provided scopes".format( - reserved_scope)) + """You cannot use any scope value that is reserved. +Your input: {} +The reserved list: {}""".format(list(scope_set), list(reserved_scope))) + raise ValueError( + "You cannot use any scope value that is in this reserved list: {}".format( + list(reserved_scope))) # client_id can also be used as a scope in B2C decorated = scope_set | reserved_scope From 95ccef07ca9f806ab51e9a04e40826c9c8e897bb Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 21 Feb 2023 21:47:52 -0800 Subject: [PATCH 158/262] Managed Identity implementation Fix docs Adjusting error message and docs Fix typo --- docs/index.rst | 20 ++ msal/__init__.py | 6 + msal/managed_identity.py | 477 ++++++++++++++++++++++++++++ sample/.env.sample.managed_identity | 17 + sample/managed_identity_sample.py | 77 +++++ tests/test_mi.py | 198 ++++++++++++ 6 files changed, 795 insertions(+) create mode 100644 msal/managed_identity.py create mode 100644 sample/.env.sample.managed_identity create mode 100644 sample/managed_identity_sample.py create mode 100644 tests/test_mi.py diff --git a/docs/index.rst b/docs/index.rst index 15ee4a0a..1c5d79bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -168,3 +168,23 @@ You may want to catch them to provide a better error message to your end users. .. autoclass:: msal.IdTokenError + +Managed Identity +================ +MSAL supports +`Managed Identity `_. + +You can create one of these two kinds of managed identity configuration objects: + +.. autoclass:: msal.SystemAssignedManagedIdentity + :members: + +.. autoclass:: msal.UserAssignedManagedIdentity + :members: + +And then feed the configuration object into a :class:`ManagedIdentityClient` object. + +.. autoclass:: msal.ManagedIdentityClient + :members: + + .. automethod:: __init__ diff --git a/msal/__init__.py b/msal/__init__.py index 87d32019..c5cf9dc2 100644 --- a/msal/__init__.py +++ b/msal/__init__.py @@ -34,8 +34,14 @@ from .oauth2cli.oidc import Prompt, IdTokenError from .token_cache import TokenCache, SerializableTokenCache from .auth_scheme import PopAuthScheme +from .managed_identity import ( + SystemAssignedManagedIdentity, UserAssignedManagedIdentity, + ManagedIdentityClient, + ManagedIdentityError, + ) # Putting module-level exceptions into the package namespace, to make them # 1. officially part of the MSAL public API, and # 2. can still be caught by the user code even if we change the module structure. from .oauth2cli.oauth2 import BrowserInteractionTimeoutError + diff --git a/msal/managed_identity.py b/msal/managed_identity.py new file mode 100644 index 00000000..1a0284ee --- /dev/null +++ b/msal/managed_identity.py @@ -0,0 +1,477 @@ +# Copyright (c) Microsoft Corporation. +# All rights reserved. +# +# This code is licensed under the MIT License. +import json +import logging +import os +import socket +import time +from urllib.parse import urlparse # Python 3+ +from collections import UserDict # Python 3+ +from typing import Union # Needed in Python 3.7 & 3.8 +from .token_cache import TokenCache +from .throttled_http_client import ThrottledHttpClient + + +logger = logging.getLogger(__name__) + + +class ManagedIdentityError(ValueError): + pass + + +class ManagedIdentity(UserDict): + """Feed an instance of this class to :class:`msal.ManagedIdentityClient` + to acquire token for the specified managed identity. + """ + # The key names used in config dict + ID_TYPE = "ManagedIdentityIdType" # Contains keyword ManagedIdentity so its json equivalent will be more readable + ID = "Id" + + # Valid values for key ID_TYPE + CLIENT_ID = "ClientId" + RESOURCE_ID = "ResourceId" + OBJECT_ID = "ObjectId" + SYSTEM_ASSIGNED = "SystemAssigned" + + _types_mapping = { # Maps type name in configuration to type name on wire + CLIENT_ID: "client_id", + RESOURCE_ID: "mi_res_id", + OBJECT_ID: "object_id", + } + + @classmethod + def is_managed_identity(cls, unknown): + return isinstance(unknown, ManagedIdentity) or ( + isinstance(unknown, dict) and cls.ID_TYPE in unknown) + + @classmethod + def is_system_assigned(cls, unknown): + return isinstance(unknown, SystemAssignedManagedIdentity) or ( + isinstance(unknown, dict) + and unknown.get(cls.ID_TYPE) == cls.SYSTEM_ASSIGNED) + + @classmethod + def is_user_assigned(cls, unknown): + return isinstance(unknown, UserAssignedManagedIdentity) or ( + isinstance(unknown, dict) + and unknown.get(cls.ID_TYPE) in cls._types_mapping + and unknown.get(cls.ID)) + + def __init__(self, identifier=None, id_type=None): + # Undocumented. Use subclasses instead. + super(ManagedIdentity, self).__init__({ + self.ID_TYPE: id_type, + self.ID: identifier, + }) + + +class SystemAssignedManagedIdentity(ManagedIdentity): + """Represent a system-assigned managed identity. + + It is equivalent to a Python dict of:: + + {"ManagedIdentityIdType": "SystemAssigned", "Id": None} + + or a JSON blob of:: + + {"ManagedIdentityIdType": "SystemAssigned", "Id": null} + """ + def __init__(self): + super(SystemAssignedManagedIdentity, self).__init__(id_type=self.SYSTEM_ASSIGNED) + + +class UserAssignedManagedIdentity(ManagedIdentity): + """Represent a user-assigned managed identity. + + Depends on the id you provided, the outcome is equivalent to one of the below:: + + {"ManagedIdentityIdType": "ClientId", "Id": "foo"} + {"ManagedIdentityIdType": "ResourceId", "Id": "foo"} + {"ManagedIdentityIdType": "ObjectId", "Id": "foo"} + """ + def __init__(self, *, client_id=None, resource_id=None, object_id=None): + if client_id and not resource_id and not object_id: + super(UserAssignedManagedIdentity, self).__init__( + id_type=self.CLIENT_ID, identifier=client_id) + elif not client_id and resource_id and not object_id: + super(UserAssignedManagedIdentity, self).__init__( + id_type=self.RESOURCE_ID, identifier=resource_id) + elif not client_id and not resource_id and object_id: + super(UserAssignedManagedIdentity, self).__init__( + id_type=self.OBJECT_ID, identifier=object_id) + else: + raise ManagedIdentityError( + "You shall specify one of the three parameters: " + "client_id, resource_id, object_id") + + +class ManagedIdentityClient(object): + """This API encapsulates multiple managed identity back-ends: + VM, App Service, Azure Automation (Runbooks), Azure Function, Service Fabric, + and Azure Arc. + + It also provides token cache support. + + .. note:: + + Cloud Shell support is NOT implemented in this class. + Since MSAL Python 1.18 in May 2022, it has been implemented in + :func:`PublicClientApplication.acquire_token_interactive` via calling pattern + ``PublicClientApplication(...).acquire_token_interactive(scopes=[...], prompt="none")``. + That is appropriate, because Cloud Shell yields a token with + delegated permissions for the end user who has signed in to the Azure Portal + (like what a ``PublicClientApplication`` does), + not a token with application permissions for an app. + """ + _instance, _tenant = socket.getfqdn(), "managed_identity" # Placeholders + + def __init__( + self, + managed_identity: Union[ + dict, + ManagedIdentity, # Could use Type[ManagedIdentity] but it is deprecatred in Python 3.9+ + SystemAssignedManagedIdentity, + UserAssignedManagedIdentity, + ], + *, + http_client, + token_cache=None, + ): + """Create a managed identity client. + + :param managed_identity: + It accepts an instance of :class:`SystemAssignedManagedIdentity` + or :class:`UserAssignedManagedIdentity`. + They are equivalent to a dict with a certain shape, + which may be loaded from a JSON configuration file or an env var. + + :param http_client: + An http client object. For example, you can use ``requests.Session()``, + optionally with exponential backoff behavior demonstrated in this recipe:: + + import msal, requests + from requests.adapters import HTTPAdapter, Retry + s = requests.Session() + retries = Retry(total=3, backoff_factor=0.1, status_forcelist=[ + 429, 500, 501, 502, 503, 504]) + s.mount('https://', HTTPAdapter(max_retries=retries)) + managed_identity = ... + client = msal.ManagedIdentityClient(managed_identity, http_client=s) + + :param token_cache: + Optional. It accepts a :class:`msal.TokenCache` instance to store tokens. + It will use an in-memory token cache by default. + + Recipe 1: Hard code a managed identity for your app:: + + import msal, requests + client = msal.ManagedIdentityClient( + msal.UserAssignedManagedIdentity(client_id="foo"), + http_client=requests.Session(), + ) + token = client.acquire_token_for_client("resource") + + Recipe 2: Write once, run everywhere. + If you use different managed identity on different deployment, + you may use an environment variable (such as MY_MANAGED_IDENTITY_CONFIG) + to store a json blob like + ``{"ManagedIdentityIdType": "ClientId", "Id": "foo"}`` or + ``{"ManagedIdentityIdType": "SystemAssignedManagedIdentity", "Id": null})``. + The following app can load managed identity configuration dynamically:: + + import json, os, msal, requests + config = os.getenv("MY_MANAGED_IDENTITY_CONFIG") + assert config, "An ENV VAR with value should exist" + client = msal.ManagedIdentityClient( + json.loads(config), + http_client=requests.Session(), + ) + token = client.acquire_token_for_client("resource") + """ + self._managed_identity = managed_identity + if isinstance(http_client, ThrottledHttpClient): + raise ValueError( + # It is a precaution to reject application.py's throttled http_client, + # whose cache life on HTTP GET 200 is too long for Managed Identity. + "This class does not currently accept a ThrottledHttpClient.") + self._http_client = http_client + self._token_cache = token_cache or TokenCache() + + def acquire_token_for_client(self, *, resource): # We may support scope in the future + """Acquire token for the managed identity. + + The result will be automatically cached. + Subsequent calls will automatically search from cache first. + + .. note:: + + Known issue: When an Azure VM has only one user-assigned managed identity, + and your app specifies to use system-assigned managed identity, + Azure VM may still return a token for your user-assigned identity. + + This is a service-side behavior that cannot be changed by this library. + `Azure VM docs `_ + """ + access_token_from_cache = None + client_id_in_cache = self._managed_identity.get( + ManagedIdentity.ID, "SYSTEM_ASSIGNED_MANAGED_IDENTITY") + if True: # Does not offer an "if not force_refresh" option, because + # there would be built-in token cache in the service side anyway + matches = self._token_cache.find( + self._token_cache.CredentialType.ACCESS_TOKEN, + target=[resource], + query=dict( + client_id=client_id_in_cache, + environment=self._instance, + realm=self._tenant, + home_account_id=None, + ), + ) + now = time.time() + for entry in matches: + expires_in = int(entry["expires_on"]) - now + if expires_in < 5*60: # Then consider it expired + continue # Removal is not necessary, it will be overwritten + logger.debug("Cache hit an AT") + access_token_from_cache = { # Mimic a real response + "access_token": entry["secret"], + "token_type": entry.get("token_type", "Bearer"), + "expires_in": int(expires_in), # OAuth2 specs defines it as int + } + if "refresh_on" in entry and int(entry["refresh_on"]) < now: # aging + break # With a fallback in hand, we break here to go refresh + return access_token_from_cache # It is still good as new + try: + result = _obtain_token(self._http_client, self._managed_identity, resource) + if "access_token" in result: + expires_in = result.get("expires_in", 3600) + if "refresh_in" not in result and expires_in >= 7200: + result["refresh_in"] = int(expires_in / 2) + self._token_cache.add(dict( + client_id=client_id_in_cache, + scope=[resource], + token_endpoint="https://{}/{}".format(self._instance, self._tenant), + response=result, + params={}, + data={}, + )) + if (result and "error" not in result) or (not access_token_from_cache): + return result + except: # The exact HTTP exception is transportation-layer dependent + # Typically network error. Potential AAD outage? + if not access_token_from_cache: # It means there is no fall back option + raise # We choose to bubble up the exception + return access_token_from_cache + + +def _scope_to_resource(scope): # This is an experimental reasonable-effort approach + u = urlparse(scope) + if u.scheme: + return "{}://{}".format(u.scheme, u.netloc) + return scope # There is no much else we can do here + + +def _obtain_token(http_client, managed_identity, resource): + # A unified low-level API that talks to different Managed Identity + if ("IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ + and "IDENTITY_SERVER_THUMBPRINT" in os.environ + ): + if managed_identity: + logger.debug( + "Ignoring managed_identity parameter. " + "Managed Identity in Service Fabric is configured in the cluster, " + "not during runtime. See also " + "https://learn.microsoft.com/en-us/azure/service-fabric/configure-existing-cluster-enable-managed-identity-token-service") + return _obtain_token_on_service_fabric( + http_client, + os.environ["IDENTITY_ENDPOINT"], + os.environ["IDENTITY_HEADER"], + os.environ["IDENTITY_SERVER_THUMBPRINT"], + resource, + ) + if "IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ: + return _obtain_token_on_app_service( + http_client, + os.environ["IDENTITY_ENDPOINT"], + os.environ["IDENTITY_HEADER"], + managed_identity, + resource, + ) + if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: + if ManagedIdentity.is_user_assigned(managed_identity): + raise ManagedIdentityError( # Note: Azure Identity for Python raised exception too + "Invalid managed_identity parameter. " + "Azure Arc supports only system-assigned managed identity, " + "See also " + "https://learn.microsoft.com/en-us/azure/service-fabric/configure-existing-cluster-enable-managed-identity-token-service") + return _obtain_token_on_arc( + http_client, + os.environ["IDENTITY_ENDPOINT"], + resource, + ) + return _obtain_token_on_azure_vm(http_client, managed_identity, resource) + + +def _adjust_param(params, managed_identity): + id_name = ManagedIdentity._types_mapping.get( + managed_identity.get(ManagedIdentity.ID_TYPE)) + if id_name: + params[id_name] = managed_identity[ManagedIdentity.ID] + +def _obtain_token_on_azure_vm(http_client, managed_identity, resource): + # Based on https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#get-a-token-using-http + logger.debug("Obtaining token via managed identity on Azure VM") + params = { + "api-version": "2018-02-01", + "resource": resource, + } + _adjust_param(params, managed_identity) + resp = http_client.get( + "http://169.254.169.254/metadata/identity/oauth2/token", + params=params, + headers={"Metadata": "true"}, + ) + try: + payload = json.loads(resp.text) + if payload.get("access_token") and payload.get("expires_in"): + return { # Normalizing the payload into OAuth2 format + "access_token": payload["access_token"], + "expires_in": int(payload["expires_in"]), + "resource": payload.get("resource"), + "token_type": payload.get("token_type", "Bearer"), + } + return payload # Typically an error, but it is undefined in the doc above + except json.decoder.JSONDecodeError: + logger.debug("IMDS emits unexpected payload: %s", resp.text) + raise + +def _obtain_token_on_app_service( + http_client, endpoint, identity_header, managed_identity, resource, +): + """Obtains token for + `App Service `_, + Azure Functions, and Azure Automation. + """ + # Prerequisite: Create your app service https://docs.microsoft.com/en-us/azure/app-service/quickstart-python + # Assign it a managed identity https://docs.microsoft.com/en-us/azure/app-service/overview-managed-identity?tabs=portal%2Chttp + # SSH into your container for testing https://docs.microsoft.com/en-us/azure/app-service/configure-linux-open-ssh-session + logger.debug("Obtaining token via managed identity on Azure App Service") + params = { + "api-version": "2019-08-01", + "resource": resource, + } + _adjust_param(params, managed_identity) + resp = http_client.get( + endpoint, + params=params, + headers={ + "X-IDENTITY-HEADER": identity_header, + "Metadata": "true", # Unnecessary yet harmless for App Service, + # It will be needed by Azure Automation + # https://docs.microsoft.com/en-us/azure/automation/enable-managed-identity-for-automation#get-access-token-for-system-assigned-managed-identity-using-http-get + }, + ) + try: + payload = json.loads(resp.text) + if payload.get("access_token") and payload.get("expires_on"): + return { # Normalizing the payload into OAuth2 format + "access_token": payload["access_token"], + "expires_in": int(payload["expires_on"]) - int(time.time()), + "resource": payload.get("resource"), + "token_type": payload.get("token_type", "Bearer"), + } + return { + "error": "invalid_scope", # Empirically, wrong resource ends up with a vague statusCode=500 + "error_description": "{}, {}".format( + payload.get("statusCode"), payload.get("message")), + } + except json.decoder.JSONDecodeError: + logger.debug("IMDS emits unexpected payload: %s", resp.text) + raise + + +def _obtain_token_on_service_fabric( + http_client, endpoint, identity_header, server_thumbprint, resource, +): + """Obtains token for + `Service Fabric `_ + """ + # Deployment https://learn.microsoft.com/en-us/azure/service-fabric/service-fabric-get-started-containers-linux + # See also https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/azure-identity/tests/managed-identity-live/service-fabric/service_fabric.md + # Protocol https://learn.microsoft.com/en-us/azure/service-fabric/how-to-managed-identity-service-fabric-app-code#acquiring-an-access-token-using-rest-api + logger.debug("Obtaining token via managed identity on Azure Service Fabric") + resp = http_client.get( + endpoint, + params={"api-version": "2019-07-01-preview", "resource": resource}, + headers={"Secret": identity_header}, + ) + try: + payload = json.loads(resp.text) + if payload.get("access_token") and payload.get("expires_on"): + return { # Normalizing the payload into OAuth2 format + "access_token": payload["access_token"], + "expires_in": int( # Despite the example in docs shows an integer, + payload["expires_on"] # Azure SDK team's test obtained a string. + ) - int(time.time()), + "resource": payload.get("resource"), + "token_type": payload["token_type"], + } + error = payload.get("error", {}) # https://learn.microsoft.com/en-us/azure/service-fabric/how-to-managed-identity-service-fabric-app-code#error-handling + error_mapping = { # Map Service Fabric errors into OAuth2 errors https://www.rfc-editor.org/rfc/rfc6749#section-5.2 + "SecretHeaderNotFound": "unauthorized_client", + "ManagedIdentityNotFound": "invalid_client", + "ArgumentNullOrEmpty": "invalid_scope", + } + return { + "error": error_mapping.get(payload["error"]["code"], "invalid_request"), + "error_description": resp.text, + } + except json.decoder.JSONDecodeError: + logger.debug("IMDS emits unexpected payload: %s", resp.text) + raise + + +def _obtain_token_on_arc(http_client, endpoint, resource): + # https://learn.microsoft.com/en-us/azure/azure-arc/servers/managed-identity-authentication + logger.debug("Obtaining token via managed identity on Azure Arc") + resp = http_client.get( + endpoint, + params={"api-version": "2020-06-01", "resource": resource}, + headers={"Metadata": "true"}, + ) + www_auth = "www-authenticate" # Header in lower case + challenge = { + # Normalized to lowercase, because header names are case-insensitive + # https://datatracker.ietf.org/doc/html/rfc7230#section-3.2 + k.lower(): v for k, v in resp.headers.items() if k.lower() == www_auth + }.get(www_auth, "").split("=") # Output will be ["Basic realm", "content"] + if not ( # https://datatracker.ietf.org/doc/html/rfc7617#section-2 + len(challenge) == 2 and challenge[0].lower() == "basic realm"): + raise ManagedIdentityError( + "Unrecognizable WWW-Authenticate header: {}".format(resp.headers)) + with open(challenge[1]) as f: + secret = f.read() + response = http_client.get( + endpoint, + params={"api-version": "2020-06-01", "resource": resource}, + headers={"Metadata": "true", "Authorization": "Basic {}".format(secret)}, + ) + try: + payload = json.loads(response.text) + if payload.get("access_token") and payload.get("expires_in"): + # Example: https://learn.microsoft.com/en-us/azure/azure-arc/servers/media/managed-identity-authentication/bash-token-output-example.png + return { + "access_token": payload["access_token"], + "expires_in": int(payload["expires_in"]), + "token_type": payload.get("token_type", "Bearer"), + "resource": payload.get("resource"), + } + except json.decoder.JSONDecodeError: + pass + return { + "error": "invalid_request", + "error_description": response.text, + } + diff --git a/sample/.env.sample.managed_identity b/sample/.env.sample.managed_identity new file mode 100644 index 00000000..8b62f4f1 --- /dev/null +++ b/sample/.env.sample.managed_identity @@ -0,0 +1,17 @@ +# This sample can be configured to work with Microsoft Entra ID's Managed Identity. +# +# A user-assigned managed identity can be represented as a JSON blob. +# Check MSAL Python's API Reference for its syntax. +# https://msal-python.readthedocs.io/en/latest/#managed-identity +# +# Example value when using a user-assigned managed identity: +# {"ManagedIdentityIdType": "ClientId", "Id": "your_managed_identity_id"} +# Leave it empty or absent if you are using a system-assigned managed identity. +MANAGED_IDENTITY= + +# Managed Identity works with resource, not scopes. +RESOURCE= + +# Required if the sample app wants to call an API. +#ENDPOINT=https://graph.microsoft.com/v1.0/me + diff --git a/sample/managed_identity_sample.py b/sample/managed_identity_sample.py new file mode 100644 index 00000000..95bf1b28 --- /dev/null +++ b/sample/managed_identity_sample.py @@ -0,0 +1,77 @@ +""" +This sample demonstrates a daemon application that acquires a token using a +managed identity and then calls a web API with the token. + +This sample loads its configuration from a .env file. + +To make this sample work, you need to choose this template: + + .env.sample.managed_identity + +Copy the chosen template to a new file named .env, and fill in the values. + +You can then run this sample: + + python name_of_this_script.py +""" +import json +import logging +import os +import time + +from dotenv import load_dotenv # Need "pip install python-dotenv" +import msal +import requests + + +# Optional logging +# logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script +# logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs + +load_dotenv() # We use this to load configuration from a .env file + +# If for whatever reason you plan to recreate same ClientApplication periodically, +# you shall create one global token cache and reuse it by each ClientApplication +global_token_cache = msal.TokenCache() # The TokenCache() is in-memory. + # See more options in https://msal-python.readthedocs.io/en/latest/#tokencache + +# Create a managed identity instance based on the environment variable value +if os.getenv('MANAGED_IDENTITY'): + managed_identity = json.loads(os.getenv('MANAGED_IDENTITY')) +else: + managed_identity = msal.SystemAssignedManagedIdentity() + +# Create a preferably long-lived app instance, to avoid the overhead of app creation +global_app = msal.ManagedIdentityClient( + managed_identity, + http_client=requests.Session(), + token_cache=global_token_cache, # Let this app (re)use an existing token cache. + # If absent, ClientApplication will create its own empty token cache + ) +resource = os.getenv("RESOURCE") + + +def acquire_and_use_token(): + # ManagedIdentityClient.acquire_token_for_client(...) will automatically look up + # a token from cache, and fall back to acquire a fresh token when needed. + result = global_app.acquire_token_for_client(resource=resource) + + if "access_token" in result: + if os.getenv('ENDPOINT'): + # Calling a web API using the access token + api_result = requests.get( + os.getenv('ENDPOINT'), + headers={'Authorization': 'Bearer ' + result['access_token']}, + ).json() # Assuming the response is JSON + print("Web API call result", json.dumps(api_result, indent=2)) + else: + print("Token acquisition result", json.dumps(result, indent=2)) + else: + print("Token acquisition failed", result) # Examine result["error_description"] etc. to diagnose error + + +while True: # Here we mimic a long-lived daemon + acquire_and_use_token() + print("Press Ctrl-C to stop.") + time.sleep(5) # Let's say your app would run a workload every X minutes + diff --git a/tests/test_mi.py b/tests/test_mi.py new file mode 100644 index 00000000..9558e737 --- /dev/null +++ b/tests/test_mi.py @@ -0,0 +1,198 @@ +import json +import os +import sys +import time +import unittest +try: + from unittest.mock import patch, ANY, mock_open +except: + from mock import patch, ANY, mock_open +import requests + +from tests.http_client import MinimalResponse +from msal import ( + SystemAssignedManagedIdentity, UserAssignedManagedIdentity, + ManagedIdentityClient, + ManagedIdentityError, +) + + +class ManagedIdentityTestCase(unittest.TestCase): + def test_helper_class_should_be_interchangable_with_dict_which_could_be_loaded_from_file_or_env_var(self): + self.assertEqual( + UserAssignedManagedIdentity(client_id="foo"), + {"ManagedIdentityIdType": "ClientId", "Id": "foo"}) + self.assertEqual( + UserAssignedManagedIdentity(resource_id="foo"), + {"ManagedIdentityIdType": "ResourceId", "Id": "foo"}) + self.assertEqual( + UserAssignedManagedIdentity(object_id="foo"), + {"ManagedIdentityIdType": "ObjectId", "Id": "foo"}) + with self.assertRaises(ManagedIdentityError): + UserAssignedManagedIdentity() + with self.assertRaises(ManagedIdentityError): + UserAssignedManagedIdentity(client_id="foo", resource_id="bar") + self.assertEqual( + SystemAssignedManagedIdentity(), + {"ManagedIdentityIdType": "SystemAssigned", "Id": None}) + + +class ClientTestCase(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.app = ManagedIdentityClient( + { # Here we test it with the raw dict form, to test that + # the client has no hard dependency on ManagedIdentity object + "ManagedIdentityIdType": "SystemAssigned", "Id": None, + }, + http_client=requests.Session(), + ) + + def _test_token_cache(self, app): + cache = app._token_cache._cache + self.assertEqual(1, len(cache.get("AccessToken", [])), "Should have 1 AT") + at = list(cache["AccessToken"].values())[0] + self.assertEqual( + app._managed_identity.get("Id", "SYSTEM_ASSIGNED_MANAGED_IDENTITY"), + at["client_id"], + "Should have expected client_id") + self.assertEqual("managed_identity", at["realm"], "Should have expected realm") + + def _test_happy_path(self, app, mocked_http): + result = app.acquire_token_for_client(resource="R") + mocked_http.assert_called() + self.assertEqual({ + "access_token": "AT", + "expires_in": 1234, + "resource": "R", + "token_type": "Bearer", + }, result, "Should obtain a token response") + self.assertEqual( + result["access_token"], + app.acquire_token_for_client(resource="R").get("access_token"), + "Should hit the same token from cache") + self._test_token_cache(app) + + +class VmTestCase(ClientTestCase): + + def test_happy_path(self): + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_in": "1234", "resource": "R"}', + )) as mocked_method: + self._test_happy_path(self.app, mocked_method) + + def test_vm_error_should_be_returned_as_is(self): + raw_error = '{"raw": "error format is undefined"}' + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=400, + text=raw_error, + )) as mocked_method: + self.assertEqual( + json.loads(raw_error), self.app.acquire_token_for_client(resource="R")) + self.assertEqual({}, self.app._token_cache._cache) + + +@patch.dict(os.environ, {"IDENTITY_ENDPOINT": "http://localhost", "IDENTITY_HEADER": "foo"}) +class AppServiceTestCase(ClientTestCase): + + def test_happy_path(self): + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_on": "%s", "resource": "R"}' % ( + int(time.time()) + 1234), + )) as mocked_method: + self._test_happy_path(self.app, mocked_method) + + def test_app_service_error_should_be_normalized(self): + raw_error = '{"statusCode": 500, "message": "error content is undefined"}' + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=500, + text=raw_error, + )) as mocked_method: + self.assertEqual({ + "error": "invalid_scope", + "error_description": "500, error content is undefined", + }, self.app.acquire_token_for_client(resource="R")) + self.assertEqual({}, self.app._token_cache._cache) + + +@patch.dict(os.environ, { + "IDENTITY_ENDPOINT": "http://localhost", + "IDENTITY_HEADER": "foo", + "IDENTITY_SERVER_THUMBPRINT": "bar", +}) +class ServiceFabricTestCase(ClientTestCase): + + def _test_happy_path(self, app): + with patch.object(app._http_client, "get", return_value=MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_on": %s, "resource": "R", "token_type": "Bearer"}' % ( + int(time.time()) + 1234), + )) as mocked_method: + super(ServiceFabricTestCase, self)._test_happy_path(app, mocked_method) + + def test_happy_path(self): + self._test_happy_path(self.app) + + def test_unified_api_service_should_ignore_unnecessary_client_id(self): + self._test_happy_path(ManagedIdentityClient( + {"ManagedIdentityIdType": "ClientId", "Id": "foo"}, + http_client=requests.Session(), + )) + + def test_sf_error_should_be_normalized(self): + raw_error = ''' +{"error": { + "correlationId": "foo", + "code": "SecretHeaderNotFound", + "message": "Secret is not found in the request headers." +}}''' # https://learn.microsoft.com/en-us/azure/service-fabric/how-to-managed-identity-service-fabric-app-code#error-handling + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=404, + text=raw_error, + )) as mocked_method: + self.assertEqual({ + "error": "unauthorized_client", + "error_description": raw_error, + }, self.app.acquire_token_for_client(resource="R")) + self.assertEqual({}, self.app._token_cache._cache) + + +@patch.dict(os.environ, { + "IDENTITY_ENDPOINT": "http://localhost/token", + "IMDS_ENDPOINT": "http://localhost", +}) +@patch( + "builtins.open" if sys.version_info.major >= 3 else "__builtin__.open", + new=mock_open(read_data="secret"), # `new` requires no extra argument on the decorated function. + # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.patch +) +class ArcTestCase(ClientTestCase): + challenge = MinimalResponse(status_code=401, text="", headers={ + "WWW-Authenticate": "Basic realm=/tmp/foo", + }) + + def test_happy_path(self): + with patch.object(self.app._http_client, "get", side_effect=[ + self.challenge, + MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_in": "1234", "resource": "R"}', + ), + ]) as mocked_method: + super(ArcTestCase, self)._test_happy_path(self.app, mocked_method) + + def test_arc_error_should_be_normalized(self): + with patch.object(self.app._http_client, "get", side_effect=[ + self.challenge, + MinimalResponse(status_code=400, text="undefined"), + ]) as mocked_method: + self.assertEqual({ + "error": "invalid_request", + "error_description": "undefined", + }, self.app.acquire_token_for_client(resource="R")) + self.assertEqual({}, self.app._token_cache._cache) + From d0c20ed66b074daa1e0e2d95751e0bdbd08c2840 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 19 Apr 2024 17:06:22 -0700 Subject: [PATCH 159/262] Refactor throttling and add it to Managed Identity --- msal/managed_identity.py | 45 +++++++++++++++++++++++++++++------ msal/throttled_http_client.py | 43 ++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 1a0284ee..e0eaa131 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -11,7 +11,8 @@ from collections import UserDict # Python 3+ from typing import Union # Needed in Python 3.7 & 3.8 from .token_cache import TokenCache -from .throttled_http_client import ThrottledHttpClient +from .individual_cache import _IndividualCache as IndividualCache +from .throttled_http_client import ThrottledHttpClientBase, _parse_http_429_5xx_retry_after logger = logging.getLogger(__name__) @@ -107,6 +108,22 @@ def __init__(self, *, client_id=None, resource_id=None, object_id=None): "client_id, resource_id, object_id") +class _ThrottledHttpClient(ThrottledHttpClientBase): + def __init__(self, http_client, http_cache): + super(_ThrottledHttpClient, self).__init__(http_client, http_cache) + self.get = IndividualCache( # All MIs (except Cloud Shell) use GETs + mapping=self._expiring_mapping, + key_maker=lambda func, args, kwargs: "POST {} hash={} 429/5xx/Retry-After".format( + args[0], # It is the endpoint, typically a constant per MI type + _hash( + # Managed Identity flavors have inconsistent parameters. + # We simply choose to hash them all. + str(kwargs.get("params")) + str(kwargs.get("data"))), + ), + expires_in=_parse_http_429_5xx_retry_after, + )(http_client.get) + + class ManagedIdentityClient(object): """This API encapsulates multiple managed identity back-ends: VM, App Service, Azure Automation (Runbooks), Azure Function, Service Fabric, @@ -138,6 +155,7 @@ def __init__( *, http_client, token_cache=None, + http_cache=None, ): """Create a managed identity client. @@ -164,6 +182,10 @@ def __init__( Optional. It accepts a :class:`msal.TokenCache` instance to store tokens. It will use an in-memory token cache by default. + :param http_cache: + Optional. It has the same characteristics as the + :paramref:`msal.ClientApplication.http_cache`. + Recipe 1: Hard code a managed identity for your app:: import msal, requests @@ -191,12 +213,21 @@ def __init__( token = client.acquire_token_for_client("resource") """ self._managed_identity = managed_identity - if isinstance(http_client, ThrottledHttpClient): - raise ValueError( - # It is a precaution to reject application.py's throttled http_client, - # whose cache life on HTTP GET 200 is too long for Managed Identity. - "This class does not currently accept a ThrottledHttpClient.") - self._http_client = http_client + self._http_client = _ThrottledHttpClient( + # This class only throttles excess token acquisition requests. + # It does not provide retry. + # Retry is the http_client or caller's responsibility, not MSAL's. + # + # FWIW, here is the inconsistent retry recommendation. + # 1. Only MI on VM defines exotic 404 and 410 retry recommendations + # ( https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling ) + # (especially for 410 which was supposed to be a permanent failure). + # 2. MI on Service Fabric specifically suggests to not retry on 404. + # ( https://learn.microsoft.com/en-us/azure/service-fabric/how-to-managed-cluster-managed-identity-service-fabric-app-code#error-handling ) + http_client.http_client # Patch the raw (unpatched) http client + if isinstance(http_client, ThrottledHttpClientBase) else http_client, + {} if http_cache is None else http_cache, # Default to an in-memory dict + ) self._token_cache = token_cache or TokenCache() def acquire_token_for_client(self, *, resource): # We may support scope in the future diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index 1e285ff8..6af34acb 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -45,25 +45,42 @@ def _extract_data(kwargs, key, default=None): return data.get(key) if isinstance(data, dict) else default -class ThrottledHttpClient(object): - def __init__(self, http_client, http_cache): - """Throttle the given http_client by storing and retrieving data from cache. +class ThrottledHttpClientBase(object): + """Throttle the given http_client by storing and retrieving data from cache. - This wrapper exists so that our patching post() and get() would prevent - re-patching side effect when/if same http_client being reused. - """ - expiring_mapping = ExpiringMapping( # It will automatically clean up + This wrapper exists so that our patching post() and get() would prevent + re-patching side effect when/if same http_client being reused. + + The subclass should implement post() and/or get() + """ + def __init__(self, http_client, http_cache): + self.http_client = http_client + self._expiring_mapping = ExpiringMapping( # It will automatically clean up mapping=http_cache if http_cache is not None else {}, capacity=1024, # To prevent cache blowing up especially for CCA lock=Lock(), # TODO: This should ideally also allow customization ) + def post(self, *args, **kwargs): + return self.http_client.post(*args, **kwargs) + + def get(self, *args, **kwargs): + return self.http_client.get(*args, **kwargs) + + def close(self): + return self.http_client.close() + + +class ThrottledHttpClient(ThrottledHttpClientBase): + def __init__(self, http_client, http_cache): + super(ThrottledHttpClient, self).__init__(http_client, http_cache) + _post = http_client.post # We'll patch _post, and keep original post() intact _post = IndividualCache( # Internal specs requires throttling on at least token endpoint, # here we have a generic patch for POST on all endpoints. - mapping=expiring_mapping, + mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "POST {} client_id={} scope={} hash={} 429/5xx/Retry-After".format( args[0], # It is the url, typically containing authority and tenant @@ -81,7 +98,7 @@ def __init__(self, http_client, http_cache): )(_post) _post = IndividualCache( # It covers the "UI required cache" - mapping=expiring_mapping, + mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "POST {} hash={} 400".format( args[0], # It is the url, typically containing authority and tenant _hash( @@ -120,7 +137,7 @@ def __init__(self, http_client, http_cache): self.post = _post self.get = IndividualCache( # Typically those discovery GETs - mapping=expiring_mapping, + mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "GET {} hash={} 2xx".format( args[0], # It is the url, sometimes containing inline params _hash(kwargs.get("params", "")), @@ -129,13 +146,7 @@ def __init__(self, http_client, http_cache): 3600*24 if 200 <= result.status_code < 300 else 0, )(http_client.get) - self._http_client = http_client - # The following 2 methods have been defined dynamically by __init__() #def post(self, *args, **kwargs): pass #def get(self, *args, **kwargs): pass - def close(self): - """MSAL won't need this. But we allow throttled_http_client.close() anyway""" - return self._http_client.close() - From 25e95605154649f1393c6ee8829846d2e5f05cdc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 22 Apr 2024 12:35:41 -0700 Subject: [PATCH 160/262] Use a short throttling threshold for MI (and CCA) --- msal/application.py | 6 ++- msal/managed_identity.py | 14 +++--- msal/throttled_http_client.py | 68 ++++++++++++++--------------- tests/test_throttled_http_client.py | 18 ++++---- 4 files changed, 52 insertions(+), 54 deletions(-) diff --git a/msal/application.py b/msal/application.py index 65022124..b6a1d9d8 100644 --- a/msal/application.py +++ b/msal/application.py @@ -537,7 +537,11 @@ def __init__( self.http_client.mount("https://", a) self.http_client = ThrottledHttpClient( self.http_client, - {} if http_cache is None else http_cache, # Default to an in-memory dict + http_cache=http_cache, + default_throttle_time=60 + # The default value 60 was recommended mainly for PCA at the end of + # https://identitydivision.visualstudio.com/devex/_git/AuthLibrariesApiReview?version=GBdev&path=%2FService%20protection%2FIntial%20set%20of%20protection%20measures.md&_a=preview + if isinstance(self, PublicClientApplication) else 5, ) self.app_name = app_name diff --git a/msal/managed_identity.py b/msal/managed_identity.py index e0eaa131..12b73081 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -12,7 +12,7 @@ from typing import Union # Needed in Python 3.7 & 3.8 from .token_cache import TokenCache from .individual_cache import _IndividualCache as IndividualCache -from .throttled_http_client import ThrottledHttpClientBase, _parse_http_429_5xx_retry_after +from .throttled_http_client import ThrottledHttpClientBase, RetryAfterParser logger = logging.getLogger(__name__) @@ -109,18 +109,18 @@ def __init__(self, *, client_id=None, resource_id=None, object_id=None): class _ThrottledHttpClient(ThrottledHttpClientBase): - def __init__(self, http_client, http_cache): - super(_ThrottledHttpClient, self).__init__(http_client, http_cache) + def __init__(self, http_client, **kwargs): + super(_ThrottledHttpClient, self).__init__(http_client, **kwargs) self.get = IndividualCache( # All MIs (except Cloud Shell) use GETs mapping=self._expiring_mapping, - key_maker=lambda func, args, kwargs: "POST {} hash={} 429/5xx/Retry-After".format( + key_maker=lambda func, args, kwargs: "REQ {} hash={} 429/5xx/Retry-After".format( args[0], # It is the endpoint, typically a constant per MI type - _hash( + self._hash( # Managed Identity flavors have inconsistent parameters. # We simply choose to hash them all. str(kwargs.get("params")) + str(kwargs.get("data"))), ), - expires_in=_parse_http_429_5xx_retry_after, + expires_in=RetryAfterParser(5).parse, # 5 seconds default for non-PCA )(http_client.get) @@ -226,7 +226,7 @@ def __init__( # ( https://learn.microsoft.com/en-us/azure/service-fabric/how-to-managed-cluster-managed-identity-service-fabric-app-code#error-handling ) http_client.http_client # Patch the raw (unpatched) http client if isinstance(http_client, ThrottledHttpClientBase) else http_client, - {} if http_cache is None else http_cache, # Default to an in-memory dict + http_cache=http_cache, ) self._token_cache = token_cache or TokenCache() diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index 6af34acb..ebad76c7 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -9,35 +9,27 @@ DEVICE_AUTH_GRANT = "urn:ietf:params:oauth:grant-type:device_code" -def _hash(raw): - return sha256(repr(raw).encode("utf-8")).hexdigest() - - -def _parse_http_429_5xx_retry_after(result=None, **ignored): - """Return seconds to throttle""" - assert result is not None, """ - The signature defines it with a default value None, - only because the its shape is already decided by the - IndividualCache's.__call__(). - In actual code path, the result parameter here won't be None. - """ - response = result - lowercase_headers = {k.lower(): v for k, v in getattr( - # Historically, MSAL's HttpResponse does not always have headers - response, "headers", {}).items()} - if not (response.status_code == 429 or response.status_code >= 500 - or "retry-after" in lowercase_headers): - return 0 # Quick exit - default = 60 # Recommended at the end of - # https://identitydivision.visualstudio.com/devex/_git/AuthLibrariesApiReview?version=GBdev&path=%2FService%20protection%2FIntial%20set%20of%20protection%20measures.md&_a=preview - retry_after = lowercase_headers.get("retry-after", default) - try: - # AAD's retry_after uses integer format only - # https://stackoverflow.microsoft.com/questions/264931/264932 - delay_seconds = int(retry_after) - except ValueError: - delay_seconds = default - return min(3600, delay_seconds) +class RetryAfterParser(object): + def __init__(self, default_value=None): + self._default_value = 5 if default_value is None else default_value + + def parse(self, *, result, **ignored): + """Return seconds to throttle""" + response = result + lowercase_headers = {k.lower(): v for k, v in getattr( + # Historically, MSAL's HttpResponse does not always have headers + response, "headers", {}).items()} + if not (response.status_code == 429 or response.status_code >= 500 + or "retry-after" in lowercase_headers): + return 0 # Quick exit + retry_after = lowercase_headers.get("retry-after", self._default_value) + try: + # AAD's retry_after uses integer format only + # https://stackoverflow.microsoft.com/questions/264931/264932 + delay_seconds = int(retry_after) + except ValueError: + delay_seconds = self._default_value + return min(3600, delay_seconds) def _extract_data(kwargs, key, default=None): @@ -53,7 +45,7 @@ class ThrottledHttpClientBase(object): The subclass should implement post() and/or get() """ - def __init__(self, http_client, http_cache): + def __init__(self, http_client, *, http_cache=None): self.http_client = http_client self._expiring_mapping = ExpiringMapping( # It will automatically clean up mapping=http_cache if http_cache is not None else {}, @@ -70,10 +62,14 @@ def get(self, *args, **kwargs): def close(self): return self.http_client.close() + @staticmethod + def _hash(raw): + return sha256(repr(raw).encode("utf-8")).hexdigest() + class ThrottledHttpClient(ThrottledHttpClientBase): - def __init__(self, http_client, http_cache): - super(ThrottledHttpClient, self).__init__(http_client, http_cache) + def __init__(self, http_client, *, default_throttle_time=None, **kwargs): + super(ThrottledHttpClient, self).__init__(http_client, **kwargs) _post = http_client.post # We'll patch _post, and keep original post() intact @@ -86,7 +82,7 @@ def __init__(self, http_client, http_cache): args[0], # It is the url, typically containing authority and tenant _extract_data(kwargs, "client_id"), # Per internal specs _extract_data(kwargs, "scope"), # Per internal specs - _hash( + self._hash( # The followings are all approximations of the "account" concept # to support per-account throttling. # TODO: We may want to disable it for confidential client, though @@ -94,14 +90,14 @@ def __init__(self, http_client, http_cache): _extract_data(kwargs, "code", # "account" of auth code grant _extract_data(kwargs, "username")))), # "account" of ROPC ), - expires_in=_parse_http_429_5xx_retry_after, + expires_in=RetryAfterParser(default_throttle_time or 5).parse, )(_post) _post = IndividualCache( # It covers the "UI required cache" mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "POST {} hash={} 400".format( args[0], # It is the url, typically containing authority and tenant - _hash( + self._hash( # Here we use literally all parameters, even those short-lived # parameters containing timestamps (WS-Trust or POP assertion), # because they will automatically be cleaned up by ExpiringMapping. @@ -140,7 +136,7 @@ def __init__(self, http_client, http_cache): mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "GET {} hash={} 2xx".format( args[0], # It is the url, sometimes containing inline params - _hash(kwargs.get("params", "")), + self._hash(kwargs.get("params", "")), ), expires_in=lambda result=None, **ignored: 3600*24 if 200 <= result.status_code < 300 else 0, diff --git a/tests/test_throttled_http_client.py b/tests/test_throttled_http_client.py index aa20060d..3994719d 100644 --- a/tests/test_throttled_http_client.py +++ b/tests/test_throttled_http_client.py @@ -40,11 +40,10 @@ class CloseMethodCalled(Exception): class TestHttpDecoration(unittest.TestCase): def test_throttled_http_client_should_not_alter_original_http_client(self): - http_cache = {} original_http_client = DummyHttpClient() original_get = original_http_client.get original_post = original_http_client.post - throttled_http_client = ThrottledHttpClient(original_http_client, http_cache) + throttled_http_client = ThrottledHttpClient(original_http_client) goal = """The implementation should wrap original http_client and keep it intact, instead of monkey-patching it""" self.assertNotEqual(throttled_http_client, original_http_client, goal) @@ -54,7 +53,7 @@ def test_throttled_http_client_should_not_alter_original_http_client(self): def _test_RetryAfter_N_seconds_should_keep_entry_for_N_seconds( self, http_client, retry_after): http_cache = {} - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client, http_cache=http_cache) resp1 = http_client.post("https://example.com") # We implemented POST only resp2 = http_client.post("https://example.com") # We implemented POST only logger.debug(http_cache) @@ -90,7 +89,7 @@ def test_one_RetryAfter_request_should_block_a_similar_request(self): http_cache = {} http_client = DummyHttpClient( status_code=429, response_headers={"Retry-After": 2}) - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client, http_cache=http_cache) resp1 = http_client.post("https://example.com", data={ "scope": "one", "claims": "bar", "grant_type": "authorization_code"}) resp2 = http_client.post("https://example.com", data={ @@ -102,7 +101,7 @@ def test_one_RetryAfter_request_should_not_block_a_different_request(self): http_cache = {} http_client = DummyHttpClient( status_code=429, response_headers={"Retry-After": 2}) - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client, http_cache=http_cache) resp1 = http_client.post("https://example.com", data={"scope": "one"}) resp2 = http_client.post("https://example.com", data={"scope": "two"}) logger.debug(http_cache) @@ -112,7 +111,7 @@ def test_one_invalid_grant_should_block_a_similar_request(self): http_cache = {} http_client = DummyHttpClient( status_code=400) # It covers invalid_grant and interaction_required - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client, http_cache=http_cache) resp1 = http_client.post("https://example.com", data={"claims": "foo"}) logger.debug(http_cache) resp1_again = http_client.post("https://example.com", data={"claims": "foo"}) @@ -146,7 +145,7 @@ def test_http_get_200_should_be_cached(self): http_cache = {} http_client = DummyHttpClient( status_code=200) # It covers UserRealm discovery and OIDC discovery - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client, http_cache=http_cache) resp1 = http_client.get("https://example.com?foo=bar") resp2 = http_client.get("https://example.com?foo=bar") logger.debug(http_cache) @@ -156,7 +155,7 @@ def test_device_flow_retry_should_not_be_cached(self): DEVICE_AUTH_GRANT = "urn:ietf:params:oauth:grant-type:device_code" http_cache = {} http_client = DummyHttpClient(status_code=400) - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client, http_cache=http_cache) resp1 = http_client.post( "https://example.com", data={"grant_type": DEVICE_AUTH_GRANT}) resp2 = http_client.post( @@ -165,9 +164,8 @@ def test_device_flow_retry_should_not_be_cached(self): self.assertNotEqual(resp1.text, resp2.text, "Should return a new response") def test_throttled_http_client_should_provide_close(self): - http_cache = {} http_client = DummyHttpClient(status_code=200) - http_client = ThrottledHttpClient(http_client, http_cache) + http_client = ThrottledHttpClient(http_client) with self.assertRaises(CloseMethodCalled): http_client.close() From 234f94207e9a9dd09e45c042ed0011bac4657383 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 23 Apr 2024 12:39:45 -0700 Subject: [PATCH 161/262] Managed Identity for Machine Learning --- msal/managed_identity.py | 43 ++++++++++++++++++++++++++++++++++++++++ tests/test_mi.py | 24 ++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 12b73081..e0a480f7 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -330,6 +330,15 @@ def _obtain_token(http_client, managed_identity, resource): managed_identity, resource, ) + if "MSI_ENDPOINT" in os.environ and "MSI_SECRET" in os.environ: + # Back ported from https://github.com/Azure/azure-sdk-for-python/blob/azure-identity_1.15.0/sdk/identity/azure-identity/azure/identity/_credentials/azure_ml.py + return _obtain_token_on_machine_learning( + http_client, + os.environ["MSI_ENDPOINT"], + os.environ["MSI_SECRET"], + managed_identity, + resource, + ) if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: if ManagedIdentity.is_user_assigned(managed_identity): raise ManagedIdentityError( # Note: Azure Identity for Python raised exception too @@ -346,6 +355,7 @@ def _obtain_token(http_client, managed_identity, resource): def _adjust_param(params, managed_identity): + # Modify the params dict in place id_name = ManagedIdentity._types_mapping.get( managed_identity.get(ManagedIdentity.ID_TYPE)) if id_name: @@ -422,6 +432,39 @@ def _obtain_token_on_app_service( logger.debug("IMDS emits unexpected payload: %s", resp.text) raise +def _obtain_token_on_machine_learning( + http_client, endpoint, secret, managed_identity, resource, +): + # Could not find protocol docs from https://docs.microsoft.com/en-us/azure/machine-learning + # The following implementation is back ported from Azure Identity 1.15.0 + logger.debug("Obtaining token via managed identity on Azure Machine Learning") + params = {"api-version": "2017-09-01", "resource": resource} + _adjust_param(params, managed_identity) + if params["api-version"] == "2017-09-01" and "client_id" in params: + # Workaround for a known bug in Azure ML 2017 API + params["clientid"] = params.pop("client_id") + resp = http_client.get( + endpoint, + params=params, + headers={"secret": secret}, + ) + try: + payload = json.loads(resp.text) + if payload.get("access_token") and payload.get("expires_on"): + return { # Normalizing the payload into OAuth2 format + "access_token": payload["access_token"], + "expires_in": int(payload["expires_on"]) - int(time.time()), + "resource": payload.get("resource"), + "token_type": payload.get("token_type", "Bearer"), + } + return { + "error": "invalid_scope", # TODO: To be tested + "error_description": "{}".format(payload), + } + except json.decoder.JSONDecodeError: + logger.debug("IMDS emits unexpected payload: %s", resp.text) + raise + def _obtain_token_on_service_fabric( http_client, endpoint, identity_header, server_thumbprint, resource, diff --git a/tests/test_mi.py b/tests/test_mi.py index 9558e737..b2c0af28 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -119,6 +119,30 @@ def test_app_service_error_should_be_normalized(self): self.assertEqual({}, self.app._token_cache._cache) +@patch.dict(os.environ, {"MSI_ENDPOINT": "http://localhost", "MSI_SECRET": "foo"}) +class MachineLearningTestCase(ClientTestCase): + + def test_happy_path(self): + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_on": "%s", "resource": "R"}' % ( + int(time.time()) + 1234), + )) as mocked_method: + self._test_happy_path(self.app, mocked_method) + + def test_machine_learning_error_should_be_normalized(self): + raw_error = '{"error": "placeholder", "message": "placeholder"}' + with patch.object(self.app._http_client, "get", return_value=MinimalResponse( + status_code=500, + text=raw_error, + )) as mocked_method: + self.assertEqual({ + "error": "invalid_scope", + "error_description": "{'error': 'placeholder', 'message': 'placeholder'}", + }, self.app.acquire_token_for_client(resource="R")) + self.assertEqual({}, self.app._token_cache._cache) + + @patch.dict(os.environ, { "IDENTITY_ENDPOINT": "http://localhost", "IDENTITY_HEADER": "foo", From 9d41d539acc7e73ca817b64e6c372d3dc9e88fbf Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 23 May 2024 18:30:16 -0700 Subject: [PATCH 162/262] Implementation based on feature requirement Error out on platforms other than Linux and Windows --- msal/__init__.py | 1 + msal/managed_identity.py | 20 +++++++++++++++++++- tests/test_mi.py | 36 +++++++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/msal/__init__.py b/msal/__init__.py index c5cf9dc2..380d584e 100644 --- a/msal/__init__.py +++ b/msal/__init__.py @@ -38,6 +38,7 @@ SystemAssignedManagedIdentity, UserAssignedManagedIdentity, ManagedIdentityClient, ManagedIdentityError, + ArcPlatformNotSupportedError, ) # Putting module-level exceptions into the package namespace, to make them diff --git a/msal/managed_identity.py b/msal/managed_identity.py index e0a480f7..4aaf545f 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -6,6 +6,7 @@ import logging import os import socket +import sys import time from urllib.parse import urlparse # Python 3+ from collections import UserDict # Python 3+ @@ -507,6 +508,14 @@ def _obtain_token_on_service_fabric( raise +_supported_arc_platforms_and_their_prefixes = { + "linux": "/var/opt/azcmagent/tokens", + "win32": os.path.expandvars(r"%ProgramData%\AzureConnectedMachineAgent\Tokens"), +} + +class ArcPlatformNotSupportedError(ManagedIdentityError): + pass + def _obtain_token_on_arc(http_client, endpoint, resource): # https://learn.microsoft.com/en-us/azure/azure-arc/servers/managed-identity-authentication logger.debug("Obtaining token via managed identity on Azure Arc") @@ -525,7 +534,16 @@ def _obtain_token_on_arc(http_client, endpoint, resource): len(challenge) == 2 and challenge[0].lower() == "basic realm"): raise ManagedIdentityError( "Unrecognizable WWW-Authenticate header: {}".format(resp.headers)) - with open(challenge[1]) as f: + if sys.platform not in _supported_arc_platforms_and_their_prefixes: + raise ArcPlatformNotSupportedError( + f"Platform {sys.platform} was undefined and unsupported") + filename = os.path.join( + # This algorithm is documented in an internal doc https://msazure.visualstudio.com/One/_wiki/wikis/One.wiki/233012/VM-Extension-Authoring-for-Arc?anchor=2.-obtaining-tokens + _supported_arc_platforms_and_their_prefixes[sys.platform], + os.path.splitext(os.path.basename(challenge[1]))[0] + ".key") + if os.stat(filename).st_size > 4096: # Check size BEFORE loading its content + raise ManagedIdentityError("Local key file shall not be larger than 4KB") + with open(filename) as f: secret = f.read() response = http_client.get( endpoint, diff --git a/tests/test_mi.py b/tests/test_mi.py index b2c0af28..e8da5d72 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -4,9 +4,9 @@ import time import unittest try: - from unittest.mock import patch, ANY, mock_open + from unittest.mock import patch, ANY, mock_open, Mock except: - from mock import patch, ANY, mock_open + from mock import patch, ANY, mock_open, Mock import requests from tests.http_client import MinimalResponse @@ -14,7 +14,9 @@ SystemAssignedManagedIdentity, UserAssignedManagedIdentity, ManagedIdentityClient, ManagedIdentityError, + ArcPlatformNotSupportedError, ) +from msal.managed_identity import _supported_arc_platforms_and_their_prefixes class ManagedIdentityTestCase(unittest.TestCase): @@ -194,12 +196,13 @@ def test_sf_error_should_be_normalized(self): new=mock_open(read_data="secret"), # `new` requires no extra argument on the decorated function. # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.patch ) +@patch("os.stat", return_value=Mock(st_size=4096)) class ArcTestCase(ClientTestCase): challenge = MinimalResponse(status_code=401, text="", headers={ "WWW-Authenticate": "Basic realm=/tmp/foo", }) - def test_happy_path(self): + def test_happy_path(self, mocked_stat): with patch.object(self.app._http_client, "get", side_effect=[ self.challenge, MinimalResponse( @@ -207,16 +210,27 @@ def test_happy_path(self): text='{"access_token": "AT", "expires_in": "1234", "resource": "R"}', ), ]) as mocked_method: - super(ArcTestCase, self)._test_happy_path(self.app, mocked_method) - - def test_arc_error_should_be_normalized(self): + try: + super(ArcTestCase, self)._test_happy_path(self.app, mocked_method) + mocked_stat.assert_called_with(os.path.join( + _supported_arc_platforms_and_their_prefixes[sys.platform], + "foo.key")) + except ArcPlatformNotSupportedError: + if sys.platform in _supported_arc_platforms_and_their_prefixes: + self.fail("Should not raise ArcPlatformNotSupportedError") + + def test_arc_error_should_be_normalized(self, mocked_stat): with patch.object(self.app._http_client, "get", side_effect=[ self.challenge, MinimalResponse(status_code=400, text="undefined"), ]) as mocked_method: - self.assertEqual({ - "error": "invalid_request", - "error_description": "undefined", - }, self.app.acquire_token_for_client(resource="R")) - self.assertEqual({}, self.app._token_cache._cache) + try: + self.assertEqual({ + "error": "invalid_request", + "error_description": "undefined", + }, self.app.acquire_token_for_client(resource="R")) + self.assertEqual({}, self.app._token_cache._cache) + except ArcPlatformNotSupportedError: + if sys.platform in _supported_arc_platforms_and_their_prefixes: + self.fail("Should not raise ArcPlatformNotSupportedError") From c828a0f2c2bf23a2179624ff37fb02e47592e684 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 30 Apr 2024 17:45:43 -0700 Subject: [PATCH 163/262] get_managed_identity_source() for Azure Identity --- msal/managed_identity.py | 30 +++++++++++++++++++++++ tests/test_mi.py | 52 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 4aaf545f..354fee52 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -14,6 +14,7 @@ from .token_cache import TokenCache from .individual_cache import _IndividualCache as IndividualCache from .throttled_http_client import ThrottledHttpClientBase, RetryAfterParser +from .cloudshell import _is_running_in_cloud_shell logger = logging.getLogger(__name__) @@ -305,6 +306,35 @@ def _scope_to_resource(scope): # This is an experimental reasonable-effort appr return scope # There is no much else we can do here +APP_SERVICE = object() +AZURE_ARC = object() +CLOUD_SHELL = object() # In MSAL Python, token acquisition was done by + # PublicClientApplication(...).acquire_token_interactive(..., prompt="none") +MACHINE_LEARNING = object() +SERVICE_FABRIC = object() +DEFAULT_TO_VM = object() # Unknown environment; default to VM; you may want to probe +def get_managed_identity_source(): + """Detect the current environment and return the likely identity source. + + When this function returns ``CLOUD_SHELL``, you should use + :func:`msal.PublicClientApplication.acquire_token_interactive` with ``prompt="none"`` + to obtain a token. + """ + if ("IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ + and "IDENTITY_SERVER_THUMBPRINT" in os.environ + ): + return SERVICE_FABRIC + if "IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ: + return APP_SERVICE + if "MSI_ENDPOINT" in os.environ and "MSI_SECRET" in os.environ: + return MACHINE_LEARNING + if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: + return AZURE_ARC + if _is_running_in_cloud_shell(): + return CLOUD_SHELL + return DEFAULT_TO_VM + + def _obtain_token(http_client, managed_identity, resource): # A unified low-level API that talks to different Managed Identity if ("IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ diff --git a/tests/test_mi.py b/tests/test_mi.py index e8da5d72..d6dcc159 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -16,7 +16,16 @@ ManagedIdentityError, ArcPlatformNotSupportedError, ) -from msal.managed_identity import _supported_arc_platforms_and_their_prefixes +from msal.managed_identity import ( + _supported_arc_platforms_and_their_prefixes, + get_managed_identity_source, + APP_SERVICE, + AZURE_ARC, + CLOUD_SHELL, + MACHINE_LEARNING, + SERVICE_FABRIC, + DEFAULT_TO_VM, +) class ManagedIdentityTestCase(unittest.TestCase): @@ -234,3 +243,44 @@ def test_arc_error_should_be_normalized(self, mocked_stat): if sys.platform in _supported_arc_platforms_and_their_prefixes: self.fail("Should not raise ArcPlatformNotSupportedError") + +class GetManagedIdentitySourceTestCase(unittest.TestCase): + + @patch.dict(os.environ, { + "IDENTITY_ENDPOINT": "http://localhost", + "IDENTITY_HEADER": "foo", + "IDENTITY_SERVER_THUMBPRINT": "bar", + }) + def test_service_fabric(self): + self.assertEqual(get_managed_identity_source(), SERVICE_FABRIC) + + @patch.dict(os.environ, { + "IDENTITY_ENDPOINT": "http://localhost", + "IDENTITY_HEADER": "foo", + }) + def test_app_service(self): + self.assertEqual(get_managed_identity_source(), APP_SERVICE) + + @patch.dict(os.environ, { + "MSI_ENDPOINT": "http://localhost", + "MSI_SECRET": "foo", + }) + def test_machine_learning(self): + self.assertEqual(get_managed_identity_source(), MACHINE_LEARNING) + + @patch.dict(os.environ, { + "IDENTITY_ENDPOINT": "http://localhost", + "IMDS_ENDPOINT": "http://localhost", + }) + def test_arc(self): + self.assertEqual(get_managed_identity_source(), AZURE_ARC) + + @patch.dict(os.environ, { + "AZUREPS_HOST_ENVIRONMENT": "cloud-shell-foo", + }) + def test_cloud_shell(self): + self.assertEqual(get_managed_identity_source(), CLOUD_SHELL) + + def test_default_to_vm(self): + self.assertEqual(get_managed_identity_source(), DEFAULT_TO_VM) + From 403fed5811658edd53fbd759d1e1540b8963d598 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 17 Jun 2024 17:58:51 -0700 Subject: [PATCH 164/262] MSAL Python 1.29.0 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index b6a1d9d8..0836ec3d 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,7 +21,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.28.1" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.29.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 18174ed8053e3f47ea3f5c75273cac3ba9c71a20 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 25 Jun 2024 16:22:55 -0700 Subject: [PATCH 165/262] The old test app was somehow disabled --- tests/test_broker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index bb7d928e..1b66d0c9 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -41,10 +41,15 @@ def test_signin_interactive_then_acquire_token_silent_then_signout(self): self.assertIn("Status_AccountUnusable", result.get("error_description", "")) def test_unconfigured_app_should_raise_exception(self): - app_without_needed_redirect_uri = "289a413d-284b-4303-9c79-94380abe5d22" + self.skipTest( + "After PyMsalRuntime 0.13.2, " + "AADSTS error codes were removed from error_context; " + "it is not in telemetry either.") + app_without_needed_redirect_uri = "f62c5ae3-bf3a-4af5-afa8-a68b800396e9" # This is the lab app. We repurpose it to be used here with self.assertRaises(RedirectUriError): - _signin_interactively( + result = _signin_interactively( self._authority, app_without_needed_redirect_uri, self._scopes, None) + print(result) # Note: _acquire_token_silently() would raise same exception, # we skip its test here due to the lack of a valid account_id From c1ead1caece73b81f7cb168ca22f8741504ec5bd Mon Sep 17 00:00:00 2001 From: fengga <62267180+fengga@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:38:51 -0700 Subject: [PATCH 166/262] Update ROPC broker related tests (#714) * Update ROPC broker tests * Get test account and password from .env * update --- tests/broker-test.py | 18 ++++++++++++++++++ tests/test_account_source.py | 11 +++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/broker-test.py b/tests/broker-test.py index 2301096e..216d5256 100644 --- a/tests/broker-test.py +++ b/tests/broker-test.py @@ -6,6 +6,13 @@ we can use this script to test it with a given version of MSAL Python. """ import msal +import getpass +import os +try: + from dotenv import load_dotenv # Use this only in local dev machine + load_dotenv() # take environment variables from .env. +except: + pass _AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" SCOPE_ARM = "https://management.azure.com/.default" @@ -46,6 +53,16 @@ def interactive_and_silent(scopes, auth_scheme, data, expected_token_type): ) _assert(result, expected_token_type) +def test_broker_username_password(scopes, expected_token_type): + print("Testing broker username password flows by using accounts in local .env") + username = os.getenv("BROKER_TEST_ACCOUNT") or input("Input test account for broker test: ") + password = os.getenv("BROKER_TEST_ACCOUNT_PASSWORD") or getpass.getpass("Input test account's password: ") + assert(username and password, "You need to provide a test account and its password") + result = pca.acquire_token_by_username_password(username, password, scopes) + _assert(result, expected_token_type) + assert(result.get("token_source") == "broker") + print("Username password test succeeds.") + def _assert(result, expected_token_type): assert result.get("access_token"), f"We should obtain a token. Got {result} instead." assert result.get("token_source") == "broker", "Token should be obtained via broker" @@ -64,3 +81,4 @@ def _assert(result, expected_token_type): expected_token_type="ssh-cert", ) +test_broker_username_password(scopes=[SCOPE_ARM], expected_token_type="bearer") diff --git a/tests/test_account_source.py b/tests/test_account_source.py index b8713992..662f0419 100644 --- a/tests/test_account_source.py +++ b/tests/test_account_source.py @@ -46,20 +46,19 @@ def test_device_flow_and_its_silent_call_should_bypass_broker(self, _, mocked_br mocked_broker_ats.assert_not_called() self.assertEqual(result["token_source"], "identity_provider") - def test_ropc_flow_and_its_silent_call_should_bypass_broker(self, _, mocked_broker_ats): + def test_ropc_flow_and_its_silent_call_should_invoke_broker(self, _, mocked_broker_ats): app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) - with patch.object(app.authority, "user_realm_discovery", return_value={}): + with patch("msal.broker._signin_silently", return_value=dict(TOKEN_RESPONSE, _account_id="placeholder")): result = app.acquire_token_by_username_password( "username", "placeholder", [SCOPE], post=_mock_post) - self.assertEqual(result["token_source"], "identity_provider") + self.assertEqual(result["token_source"], "broker") account = app.get_accounts()[0] - self.assertEqual(account["account_source"], "password") + self.assertEqual(account["account_source"], "broker") result = app.acquire_token_silent_with_error( [SCOPE], account, force_refresh=True, post=_mock_post) - mocked_broker_ats.assert_not_called() - self.assertEqual(result["token_source"], "identity_provider") + self.assertEqual(result["token_source"], "broker") def test_interactive_flow_and_its_silent_call_should_invoke_broker(self, _, mocked_broker_ats): app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) From e80b58f149e3d09e7220a4b5fb0696ff0ab3b59a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 27 Jun 2024 03:05:57 -0700 Subject: [PATCH 167/262] Add the missing token query check --- msal/token_cache.py | 21 ++++++++++++++------- tests/test_e2e.py | 43 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index f9a55800..80f9bce9 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -117,6 +117,12 @@ def _get(self, credential_type, key, default=None): # O(1) with self._lock: return self._cache.get(credential_type, {}).get(key, default) + @staticmethod + def _is_matching(entry: dict, query: dict, target_set: set): + return is_subdict_of(query or {}, entry) and ( + target_set <= set(entry.get("target", "").split()) + if target_set else True) + def _find(self, credential_type, target=None, query=None): # O(n) generator """Returns a generator of matching entries. @@ -125,6 +131,7 @@ def _find(self, credential_type, target=None, query=None): # O(n) generator """ target = sorted(target or []) # Match the order sorted by add() assert isinstance(target, list), "Invalid parameter type" + target_set = set(target) preferred_result = None if (credential_type == self.CredentialType.ACCESS_TOKEN @@ -135,20 +142,20 @@ def _find(self, credential_type, target=None, query=None): # O(n) generator preferred_result = self._get_access_token( query["home_account_id"], query["environment"], query["client_id"], query["realm"], target) - if preferred_result: + if preferred_result and self._is_matching( + preferred_result, query, target_set, + ): yield preferred_result - target_set = set(target) with self._lock: # Since the target inside token cache key is (per schema) unsorted, # there is no point to attempt an O(1) key-value search here. # So we always do an O(n) in-memory search. for entry in self._cache.get(credential_type, {}).values(): - if is_subdict_of(query or {}, entry) and ( - target_set <= set(entry.get("target", "").split()) - if target else True): - if entry != preferred_result: # Avoid yielding the same entry twice - yield entry + if (entry != preferred_result # Avoid yielding the same entry twice + and self._is_matching(entry, query, target_set) + ): + yield entry def find(self, credential_type, target=None, query=None): # Obsolete. Use _find() instead. return list(self._find(credential_type, target=target, query=query)) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 33c8cf54..78441cb5 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -679,11 +679,28 @@ def _test_acquire_token_by_client_secret( class PopWithExternalKeyTestCase(LabBasedTestCase): def _test_service_principal(self): - # Any SP can obtain an ssh-cert. Here we use the lab app. - result = get_lab_app().acquire_token_for_client(self.SCOPE, data=self.DATA1) + app = get_lab_app() # Any SP can obtain an ssh-cert. Here we use the lab app. + result = app.acquire_token_for_client(self.SCOPE, data=self.DATA1) self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( result.get("error"), result.get("error_description"))) self.assertEqual(self.EXPECTED_TOKEN_TYPE, result["token_type"]) + self.assertEqual(result["token_source"], "identity_provider") + + # Test cache hit + cached_result = app.acquire_token_for_client(self.SCOPE, data=self.DATA1) + self.assertIsNotNone( + cached_result.get("access_token"), "Encountered {}: {}".format( + cached_result.get("error"), cached_result.get("error_description"))) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, cached_result["token_type"]) + self.assertEqual(cached_result["token_source"], "cache") + + # refresh_token grant can fetch an ssh-cert bound to a different key + refreshed_result = app.acquire_token_for_client(self.SCOPE, data=self.DATA2) + self.assertIsNotNone( + refreshed_result.get("access_token"), "Encountered {}: {}".format( + refreshed_result.get("error"), refreshed_result.get("error_description"))) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, refreshed_result["token_type"]) + self.assertEqual(refreshed_result["token_source"], "identity_provider") def _test_user_account(self): lab_user = self.get_lab_user(usertype="cloud") @@ -701,16 +718,30 @@ def _test_user_account(self): self.assertIsNotNone(result.get("access_token"), "Encountered {}: {}".format( result.get("error"), result.get("error_description"))) self.assertEqual(self.EXPECTED_TOKEN_TYPE, result["token_type"]) + self.assertEqual(result["token_source"], "identity_provider") logger.debug("%s.cache = %s", self.id(), json.dumps(self.app.token_cache._cache, indent=4)) + # refresh_token grant can hit an ssh-cert bound to the same key + account = self.app.get_accounts()[0] + cached_result = self.app.acquire_token_silent( + self.SCOPE, account=account, data=self.DATA1) + self.assertIsNotNone(cached_result) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, cached_result["token_type"]) + ## Actually, the self._test_acquire_token_interactive() already contained + ## a built-in refresh test, so the token in cache has been refreshed already. + ## Therefore, the following line won't pass, which is expected. + #self.assertEqual(result["access_token"], cached_result['access_token']) + self.assertEqual(cached_result["token_source"], "cache") + # refresh_token grant can fetch an ssh-cert bound to a different key account = self.app.get_accounts()[0] - refreshed_ssh_cert = self.app.acquire_token_silent( + refreshed_result = self.app.acquire_token_silent( self.SCOPE, account=account, data=self.DATA2) - self.assertIsNotNone(refreshed_ssh_cert) - self.assertEqual(self.EXPECTED_TOKEN_TYPE, refreshed_ssh_cert["token_type"]) - self.assertNotEqual(result["access_token"], refreshed_ssh_cert['access_token']) + self.assertIsNotNone(refreshed_result) + self.assertEqual(self.EXPECTED_TOKEN_TYPE, refreshed_result["token_type"]) + self.assertNotEqual(result["access_token"], refreshed_result['access_token']) + self.assertEqual(refreshed_result["token_source"], "identity_provider") class SshCertTestCase(PopWithExternalKeyTestCase): From bf443649fab870cf96e912f504542a722b33c6f3 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 3 Jul 2024 18:00:16 -0700 Subject: [PATCH 168/262] Support SNI via PFX Opt in via client_credential={ "public_certificate": True, "private_key_pfx_path": "/path/to/cert.pfx", } --- msal/application.py | 135 +++++++++++++++++++++++++------------ tests/test_cryptography.py | 4 +- tests/test_e2e.py | 13 ++-- 3 files changed, 101 insertions(+), 51 deletions(-) diff --git a/msal/application.py b/msal/application.py index 0836ec3d..ee3967af 100644 --- a/msal/application.py +++ b/msal/application.py @@ -61,17 +61,23 @@ def _str2bytes(raw): return raw -def _load_private_key_from_pfx_path(pfx_path, passphrase_bytes): +def _parse_pfx(pfx_path, passphrase_bytes): # Cert concepts https://security.stackexchange.com/a/226758/125264 - from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.serialization import pkcs12 with open(pfx_path, 'rb') as f: private_key, cert, _ = pkcs12.load_key_and_certificates( # cryptography 2.5+ # https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/#cryptography.hazmat.primitives.serialization.pkcs12.load_key_and_certificates f.read(), passphrase_bytes) + if not (private_key and cert): + raise ValueError("Your PFX file shall contain both private key and cert") + cert_pem = cert.public_bytes(encoding=serialization.Encoding.PEM).decode() # cryptography 1.0+ + x5c = [ + '\n'.join(cert_pem.splitlines()[1:-1]) # Strip the "--- header ---" and "--- footer ---" + ] sha1_thumbprint = cert.fingerprint(hashes.SHA1()).hex() # cryptography 0.7+ # https://cryptography.io/en/latest/x509/reference/#x-509-certificate-object - return private_key, sha1_thumbprint + return private_key, sha1_thumbprint, x5c def _load_private_key_from_pem_str(private_key_pem_str, passphrase_bytes): @@ -231,47 +237,71 @@ def __init__( :param client_credential: For :class:`PublicClientApplication`, you use `None` here. + For :class:`ConfidentialClientApplication`, - it can be a string containing client secret, - or an X509 certificate container in this form:: + it supports many different input formats for different scenarios. - { - "private_key": "...-----BEGIN PRIVATE KEY-----... in PEM format", - "thumbprint": "A1B2C3D4E5F6...", - "public_certificate": "...-----BEGIN CERTIFICATE-----... (Optional. See below.)", - "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", - } + .. admonition:: Support using a client secret. - MSAL Python requires a "private_key" in PEM format. - If your cert is in a PKCS12 (.pfx) format, you can also - `convert it to PEM and get the thumbprint `_. + Just feed in a string, such as ``"your client secret"``. - The thumbprint is available in your app's registration in Azure Portal. - Alternatively, you can `calculate the thumbprint `_. + .. admonition:: Support using a certificate in X.509 (.pem) format - *Added in version 0.5.0*: - public_certificate (optional) is public key certificate - which will be sent through 'x5c' JWT header only for - subject name and issuer authentication to support cert auto rolls. - - Per `specs `_, - "the certificate containing - the public key corresponding to the key used to digitally sign the - JWS MUST be the first certificate. This MAY be followed by - additional certificates, with each subsequent certificate being the - one used to certify the previous one." - However, your certificate's issuer may use a different order. - So, if your attempt ends up with an error AADSTS700027 - - "The provided signature value did not match the expected signature value", - you may try use only the leaf cert (in PEM/str format) instead. - - *Added in version 1.13.0*: - It can also be a completely pre-signed assertion that you've assembled yourself. - Simply pass a container containing only the key "client_assertion", like this:: + Feed in a dict in this form:: - { - "client_assertion": "...a JWT with claims aud, exp, iss, jti, nbf, and sub..." - } + { + "private_key": "...-----BEGIN PRIVATE KEY-----... in PEM format", + "thumbprint": "A1B2C3D4E5F6...", + "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", + } + + MSAL Python requires a "private_key" in PEM format. + If your cert is in PKCS12 (.pfx) format, + you can convert it to X.509 (.pem) format, + by ``openssl pkcs12 -in file.pfx -out file.pem -nodes``. + + The thumbprint is available in your app's registration in Azure Portal. + Alternatively, you can `calculate the thumbprint `_. + + .. admonition:: Support Subject Name/Issuer Auth with a cert in .pem + + `Subject Name/Issuer Auth + `_ + is an approach to allow easier certificate rotation. + + *Added in version 0.5.0*:: + + { + "private_key": "...-----BEGIN PRIVATE KEY-----... in PEM format", + "thumbprint": "A1B2C3D4E5F6...", + "public_certificate": "...-----BEGIN CERTIFICATE-----...", + "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", + } + + ``public_certificate`` (optional) is public key certificate + which will be sent through 'x5c' JWT header only for + subject name and issuer authentication to support cert auto rolls. + + Per `specs `_, + "the certificate containing + the public key corresponding to the key used to digitally sign the + JWS MUST be the first certificate. This MAY be followed by + additional certificates, with each subsequent certificate being the + one used to certify the previous one." + However, your certificate's issuer may use a different order. + So, if your attempt ends up with an error AADSTS700027 - + "The provided signature value did not match the expected signature value", + you may try use only the leaf cert (in PEM/str format) instead. + + .. admonition:: Supporting raw assertion obtained from elsewhere + + *Added in version 1.13.0*: + It can also be a completely pre-signed assertion that you've assembled yourself. + Simply pass a container containing only the key "client_assertion", like this:: + + { + "client_assertion": "...a JWT with claims aud, exp, iss, jti, nbf, and sub..." + } .. admonition:: Supporting reading client cerficates from PFX files @@ -280,14 +310,26 @@ def __init__( { "private_key_pfx_path": "/path/to/your.pfx", - "passphrase": "Passphrase if the private_key is encrypted (Optional. Added in version 1.6.0)", + "passphrase": "Passphrase if the private_key is encrypted (Optional)", } The following command will generate a .pfx file from your .key and .pem file:: openssl pkcs12 -export -out certificate.pfx -inkey privateKey.key -in certificate.pem - :type client_credential: Union[dict, str] + .. admonition:: Support Subject Name/Issuer Auth with a cert in .pfx + + *Added in version 1.30.0*: + If your .pfx file contains both the private key and public cert, + you can opt in for Subject Name/Issuer Auth like this:: + + { + "private_key_pfx_path": "/path/to/your.pfx", + "public_certificate": True, + "passphrase": "Passphrase if the private_key is encrypted (Optional)", + } + + :type client_credential: Union[dict, str, None] :param dict client_claims: *Added in version 0.5.0*: @@ -699,14 +741,15 @@ def _build_client(self, client_credential, authority, skip_regional_client=False client_assertion = client_credential['client_assertion'] else: headers = {} - if client_credential.get('public_certificate'): - headers["x5c"] = extract_certs(client_credential['public_certificate']) passphrase_bytes = _str2bytes( client_credential["passphrase"] ) if client_credential.get("passphrase") else None if client_credential.get("private_key_pfx_path"): - private_key, sha1_thumbprint = _load_private_key_from_pfx_path( - client_credential["private_key_pfx_path"], passphrase_bytes) + private_key, sha1_thumbprint, x5c = _parse_pfx( + client_credential["private_key_pfx_path"], + passphrase_bytes) + if client_credential.get("public_certificate") is True and x5c: + headers["x5c"] = x5c elif ( client_credential.get("private_key") # PEM blob and client_credential.get("thumbprint")): @@ -720,6 +763,10 @@ def _build_client(self, client_credential, authority, skip_regional_client=False raise ValueError( "client_credential needs to follow this format " "https://msal-python.readthedocs.io/en/latest/#msal.ClientApplication.params.client_credential") + if ("x5c" not in headers # So we did not run the pfx code path + and isinstance(client_credential.get('public_certificate'), str) + ): # Then we treat the public_certificate value as PEM content + headers["x5c"] = extract_certs(client_credential['public_certificate']) assertion = JwtAssertionCreator( private_key, algorithm="RS256", sha1_thumbprint=sha1_thumbprint, headers=headers) diff --git a/tests/test_cryptography.py b/tests/test_cryptography.py index bae06e9b..b119e599 100644 --- a/tests/test_cryptography.py +++ b/tests/test_cryptography.py @@ -8,7 +8,7 @@ import requests from msal.application import ( - _str2bytes, _load_private_key_from_pem_str, _load_private_key_from_pfx_path) + _str2bytes, _load_private_key_from_pem_str, _parse_pfx) latest_cryptography_version = ET.fromstring( @@ -48,7 +48,7 @@ def test_latest_cryptography_should_support_our_usage_without_warnings(self): _load_private_key_from_pem_str(f.read(), passphrase_bytes) pfx = sibling("certificate-with-password.pfx") # Created by: # openssl pkcs12 -export -inkey test/certificate-with-password.pem -in tests/certificate-with-password.pem -out tests/certificate-with-password.pfx - _load_private_key_from_pfx_path(pfx, passphrase_bytes) + _parse_pfx(pfx, passphrase_bytes) self.assertEqual(0, len(encountered_warnings), "Did cryptography deprecate the functions that we used?") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 61870057..68ad2af7 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -80,7 +80,7 @@ def _get_hint(html_mode=None, username=None, lab_name=None, username_uri=None): else "the upn from {}".format(_render( username_uri, description="here" if html_mode else None)), lab=_render( - "https://aka.ms/GetLabUserSecret?Secret=" + (lab_name or "msidlabXYZ"), + "https://aka.ms/GetLabSecret?Secret=" + (lab_name or "msidlabXYZ"), description="this password api" if html_mode else None, ), ) @@ -463,7 +463,10 @@ def get_lab_app( # id came from https://docs.msidlab.com/accounts/confidentialclient.html client_id = os.getenv(env_client_id) # Cert came from https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/asset/Microsoft_Azure_KeyVault/Certificate/https://msidlabs.vault.azure.net/certificates/LabVaultAccessCert - client_credential = {"private_key_pfx_path": os.getenv(env_client_cert_path)} + client_credential = { + "private_key_pfx_path": os.getenv(env_client_cert_path), + "public_certificate": True, # Opt in for SNI + } elif os.getenv(env_client_id) and os.getenv(env_name2): # Data came from here # https://docs.msidlab.com/accounts/confidentialclient.html @@ -529,7 +532,7 @@ def get_lab_user_secret(cls, lab_name="msidlab4"): lab_name = lab_name.lower() if lab_name not in cls._secrets: logger.info("Querying lab user password for %s", lab_name) - url = "https://msidlab.com/api/LabUserSecret?secret=%s" % lab_name + url = "https://msidlab.com/api/LabSecret?secret=%s" % lab_name resp = cls.session.get(url) cls._secrets[lab_name] = resp.json()["value"] return cls._secrets[lab_name] @@ -860,7 +863,7 @@ def test_adfs2019_onprem_acquire_token_by_auth_code(self): # https://msidlab.com/api/user?usertype=onprem&federationprovider=ADFSv2019 username = "..." # The upn from the link above - password="***" # From https://aka.ms/GetLabUserSecret?Secret=msidlabXYZ + password="***" # From https://aka.ms/GetLabSecret?Secret=msidlabXYZ """ config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") config["authority"] = "https://fs.%s.com/adfs" % config["lab_name"] @@ -953,7 +956,7 @@ def test_b2c_acquire_token_by_auth_code(self): username="b2clocal@msidlabb2c.onmicrosoft.com" # This won't work https://msidlab.com/api/user?usertype=b2c - password="***" # From https://aka.ms/GetLabUserSecret?Secret=msidlabb2c + password="***" # From https://aka.ms/GetLabSecret?Secret=msidlabb2c """ config = self.get_lab_app_object(azureenvironment="azureb2ccloud") self._test_acquire_token_by_auth_code( From 3ceb1c82f5e537665fd762cc3a3545bf315ae31e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 5 Jul 2024 12:34:56 -0700 Subject: [PATCH 169/262] Re-optimize the O(1) code path --- msal/token_cache.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index 80f9bce9..715fd055 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -118,7 +118,7 @@ def _get(self, credential_type, key, default=None): # O(1) return self._cache.get(credential_type, {}).get(key, default) @staticmethod - def _is_matching(entry: dict, query: dict, target_set: set): + def _is_matching(entry: dict, query: dict, target_set: set = None) -> bool: return is_subdict_of(query or {}, entry) and ( target_set <= set(entry.get("target", "").split()) if target_set else True) @@ -131,7 +131,6 @@ def _find(self, credential_type, target=None, query=None): # O(n) generator """ target = sorted(target or []) # Match the order sorted by add() assert isinstance(target, list), "Invalid parameter type" - target_set = set(target) preferred_result = None if (credential_type == self.CredentialType.ACCESS_TOKEN @@ -143,17 +142,19 @@ def _find(self, credential_type, target=None, query=None): # O(n) generator query["home_account_id"], query["environment"], query["client_id"], query["realm"], target) if preferred_result and self._is_matching( - preferred_result, query, target_set, + preferred_result, query, + # Needs no target_set here because it is satisfied by dict key ): yield preferred_result + target_set = set(target) with self._lock: # Since the target inside token cache key is (per schema) unsorted, # there is no point to attempt an O(1) key-value search here. # So we always do an O(n) in-memory search. for entry in self._cache.get(credential_type, {}).values(): if (entry != preferred_result # Avoid yielding the same entry twice - and self._is_matching(entry, query, target_set) + and self._is_matching(entry, query, target_set=target_set) ): yield entry From 788b40050fb433b2baa3149afcd211afa08588c6 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 29 Jan 2022 01:10:12 -0800 Subject: [PATCH 170/262] Backport the lazy-loading of dependency --- oauth2cli/assertion.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oauth2cli/assertion.py b/oauth2cli/assertion.py index f01bb2d0..419bb14e 100644 --- a/oauth2cli/assertion.py +++ b/oauth2cli/assertion.py @@ -4,8 +4,6 @@ import uuid import logging -import jwt - logger = logging.getLogger(__name__) @@ -99,6 +97,7 @@ def create_normal_assertion( Parameters are defined in https://tools.ietf.org/html/rfc7523#section-3 Key-value pairs in additional_claims will be added into payload as-is. """ + import jwt # Lazy loading now = time.time() payload = { 'aud': audience, From 2c335d21c61192d04980acc624564391f98a9554 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 22 Jun 2024 02:12:26 -0700 Subject: [PATCH 171/262] Support SHA256 thumbprint --- oauth2cli/assertion.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/oauth2cli/assertion.py b/oauth2cli/assertion.py index 419bb14e..aaab6977 100644 --- a/oauth2cli/assertion.py +++ b/oauth2cli/assertion.py @@ -15,6 +15,8 @@ def _str2bytes(raw): except: # Otherwise we treat it as bytes and return it as-is return raw +def _encode_thumbprint(thumbprint): + return base64.urlsafe_b64encode(binascii.a2b_hex(thumbprint)).decode() class AssertionCreator(object): def create_normal_assertion( @@ -65,7 +67,11 @@ def __call__(self): class JwtAssertionCreator(AssertionCreator): - def __init__(self, key, algorithm, sha1_thumbprint=None, headers=None): + def __init__( + self, key, algorithm, sha1_thumbprint=None, headers=None, + *, + sha256_thumbprint=None, + ): """Construct a Jwt assertion creator. Args: @@ -80,13 +86,15 @@ def __init__(self, key, algorithm, sha1_thumbprint=None, headers=None): RSA and ECDSA algorithms require "pip install cryptography". sha1_thumbprint (str): The x5t aka X.509 certificate SHA-1 thumbprint. headers (dict): Additional headers, e.g. "kid" or "x5c" etc. + sha256_thumbprint (str): The x5t#S256 aka X.509 certificate SHA-256 thumbprint. """ self.key = key self.algorithm = algorithm self.headers = headers or {} + if sha256_thumbprint: # https://datatracker.ietf.org/doc/html/rfc7515#section-4.1.8 + self.headers["x5t#S256"] = _encode_thumbprint(sha256_thumbprint) if sha1_thumbprint: # https://tools.ietf.org/html/rfc7515#section-4.1.7 - self.headers["x5t"] = base64.urlsafe_b64encode( - binascii.a2b_hex(sha1_thumbprint)).decode() + self.headers["x5t"] = _encode_thumbprint(sha1_thumbprint) def create_normal_assertion( self, audience, issuer, subject=None, expires_at=None, expires_in=600, From 57dce47ae3495e546a0913c961c149734566da3a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 11 Jul 2024 12:34:56 -0700 Subject: [PATCH 172/262] Using SHA256 and PSS padding --- msal/application.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/msal/application.py b/msal/application.py index ee3967af..ba10cd39 100644 --- a/msal/application.py +++ b/msal/application.py @@ -75,9 +75,10 @@ def _parse_pfx(pfx_path, passphrase_bytes): x5c = [ '\n'.join(cert_pem.splitlines()[1:-1]) # Strip the "--- header ---" and "--- footer ---" ] + sha256_thumbprint = cert.fingerprint(hashes.SHA256()).hex() # cryptography 0.7+ sha1_thumbprint = cert.fingerprint(hashes.SHA1()).hex() # cryptography 0.7+ # https://cryptography.io/en/latest/x509/reference/#x-509-certificate-object - return private_key, sha1_thumbprint, x5c + return private_key, sha256_thumbprint, sha1_thumbprint, x5c def _load_private_key_from_pem_str(private_key_pem_str, passphrase_bytes): @@ -741,11 +742,12 @@ def _build_client(self, client_credential, authority, skip_regional_client=False client_assertion = client_credential['client_assertion'] else: headers = {} + sha1_thumbprint = sha256_thumbprint = None passphrase_bytes = _str2bytes( client_credential["passphrase"] ) if client_credential.get("passphrase") else None if client_credential.get("private_key_pfx_path"): - private_key, sha1_thumbprint, x5c = _parse_pfx( + private_key, sha256_thumbprint, sha1_thumbprint, x5c = _parse_pfx( client_credential["private_key_pfx_path"], passphrase_bytes) if client_credential.get("public_certificate") is True and x5c: @@ -763,13 +765,22 @@ def _build_client(self, client_credential, authority, skip_regional_client=False raise ValueError( "client_credential needs to follow this format " "https://msal-python.readthedocs.io/en/latest/#msal.ClientApplication.params.client_credential") - if ("x5c" not in headers # So we did not run the pfx code path + if ("x5c" not in headers # So the .pfx file contains no certificate and isinstance(client_credential.get('public_certificate'), str) ): # Then we treat the public_certificate value as PEM content headers["x5c"] = extract_certs(client_credential['public_certificate']) + if sha256_thumbprint and not authority.is_adfs: + assertion_params = { + "algorithm": "PS256", "sha256_thumbprint": sha256_thumbprint, + } + else: # Fall back + if not sha1_thumbprint: + raise ValueError("You shall provide a thumbprint in SHA1.") + assertion_params = { + "algorithm": "RS256", "sha1_thumbprint": sha1_thumbprint, + } assertion = JwtAssertionCreator( - private_key, algorithm="RS256", - sha1_thumbprint=sha1_thumbprint, headers=headers) + private_key, headers=headers, **assertion_params) client_assertion = assertion.create_regenerative_assertion( audience=authority.token_endpoint, issuer=self.client_id, additional_claims=self.client_claims or {}) From 3a4f44fbc599e653cfc45114e9a14b977dfe18c1 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 12 Jul 2024 12:34:56 -0700 Subject: [PATCH 173/262] Expose refresh_on (if any) to fresh or cached response --- msal/application.py | 13 +++++--- msal/managed_identity.py | 8 +++-- tests/test_application.py | 25 +++++++++++--- tests/test_mi.py | 69 ++++++++++++++++++++++++++++----------- 4 files changed, 86 insertions(+), 29 deletions(-) diff --git a/msal/application.py b/msal/application.py index ba10cd39..8f30eb1c 100644 --- a/msal/application.py +++ b/msal/application.py @@ -104,11 +104,14 @@ def _clean_up(result): "msalruntime_telemetry": result.get("_msalruntime_telemetry"), "msal_python_telemetry": result.get("_msal_python_telemetry"), }, separators=(",", ":")) - return { + return_value = { k: result[k] for k in result if k != "refresh_in" # MSAL handled refresh_in, customers need not and not k.startswith('_') # Skim internal properties } + if "refresh_in" in result: # To encourage proactive refresh + return_value["refresh_on"] = int(time.time() + result["refresh_in"]) + return return_value return result # It could be None @@ -1507,9 +1510,11 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( "expires_in": int(expires_in), # OAuth2 specs defines it as int self._TOKEN_SOURCE: self._TOKEN_SOURCE_CACHE, } - if "refresh_on" in entry and int(entry["refresh_on"]) < now: # aging - refresh_reason = msal.telemetry.AT_AGING - break # With a fallback in hand, we break here to go refresh + if "refresh_on" in entry: + access_token_from_cache["refresh_on"] = int(entry["refresh_on"]) + if int(entry["refresh_on"]) < now: # aging + refresh_reason = msal.telemetry.AT_AGING + break # With a fallback in hand, we break here to go refresh self._build_telemetry_context(-1).hit_an_access_token() return access_token_from_cache # It is still good as new else: diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 354fee52..aee57ca3 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -273,8 +273,10 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the "token_type": entry.get("token_type", "Bearer"), "expires_in": int(expires_in), # OAuth2 specs defines it as int } - if "refresh_on" in entry and int(entry["refresh_on"]) < now: # aging - break # With a fallback in hand, we break here to go refresh + if "refresh_on" in entry: + access_token_from_cache["refresh_on"] = int(entry["refresh_on"]) + if int(entry["refresh_on"]) < now: # aging + break # With a fallback in hand, we break here to go refresh return access_token_from_cache # It is still good as new try: result = _obtain_token(self._http_client, self._managed_identity, resource) @@ -290,6 +292,8 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the params={}, data={}, )) + if "refresh_in" in result: + result["refresh_on"] = int(now + result["refresh_in"]) if (result and "error" not in result) or (not access_token_from_cache): return result except: # The exact HTTP exception is transportation-layer dependent diff --git a/tests/test_application.py b/tests/test_application.py index cebc7225..71dc16ea 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -1,6 +1,7 @@ # Note: Since Aug 2019 we move all e2e tests into test_e2e.py, # so this test_application file contains only unit tests without dependency. import sys +import time from msal.application import * from msal.application import _str2bytes import msal @@ -353,10 +354,18 @@ def populate_cache(self, access_token="at", expires_in=86400, refresh_in=43200): uid=self.uid, utid=self.utid, refresh_token=self.rt), }) + def assertRefreshOn(self, result, refresh_in): + refresh_on = int(time.time() + refresh_in) + self.assertTrue( + refresh_on - 1 < result.get("refresh_on", 0) < refresh_on + 1, + "refresh_on should be set properly") + def test_fresh_token_should_be_returned_from_cache(self): # a.k.a. Return unexpired token that is not above token refresh expiration threshold + refresh_in = 450 access_token = "An access token prepopulated into cache" - self.populate_cache(access_token=access_token, expires_in=900, refresh_in=450) + self.populate_cache( + access_token=access_token, expires_in=900, refresh_in=refresh_in) result = self.app.acquire_token_silent( ['s1'], self.account, post=lambda url, *args, **kwargs: # Utilize the undocumented test feature @@ -365,32 +374,38 @@ def test_fresh_token_should_be_returned_from_cache(self): self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_CACHE) self.assertEqual(access_token, result.get("access_token")) self.assertNotIn("refresh_in", result, "Customers need not know refresh_in") + self.assertRefreshOn(result, refresh_in) def test_aging_token_and_available_aad_should_return_new_token(self): # a.k.a. Attempt to refresh unexpired token when AAD available self.populate_cache(access_token="old AT", expires_in=3599, refresh_in=-1) new_access_token = "new AT" + new_refresh_in = 123 def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|84,4|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=200, text=json.dumps({ "access_token": new_access_token, - "refresh_in": 123, + "refresh_in": new_refresh_in, })) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(new_access_token, result.get("access_token")) self.assertNotIn("refresh_in", result, "Customers need not know refresh_in") + self.assertRefreshOn(result, new_refresh_in) def test_aging_token_and_unavailable_aad_should_return_old_token(self): # a.k.a. Attempt refresh unexpired token when AAD unavailable + refresh_in = -1 old_at = "old AT" - self.populate_cache(access_token=old_at, expires_in=3599, refresh_in=-1) + self.populate_cache( + access_token=old_at, expires_in=3599, refresh_in=refresh_in) def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|84,4|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=400, text=json.dumps({"error": "foo"})) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_CACHE) self.assertEqual(old_at, result.get("access_token")) + self.assertRefreshOn(result, refresh_in) def test_expired_token_and_unavailable_aad_should_return_error(self): # a.k.a. Attempt refresh expired token when AAD unavailable @@ -407,16 +422,18 @@ def test_expired_token_and_available_aad_should_return_new_token(self): # a.k.a. Attempt refresh expired token when AAD available self.populate_cache(access_token="expired at", expires_in=-1, refresh_in=-900) new_access_token = "new AT" + new_refresh_in = 123 def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|84,3|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) return MinimalResponse(status_code=200, text=json.dumps({ "access_token": new_access_token, - "refresh_in": 123, + "refresh_in": new_refresh_in, })) result = self.app.acquire_token_silent(['s1'], self.account, post=mock_post) self.assertEqual(result[self.app._TOKEN_SOURCE], self.app._TOKEN_SOURCE_IDP) self.assertEqual(new_access_token, result.get("access_token")) self.assertNotIn("refresh_in", result, "Customers need not know refresh_in") + self.assertRefreshOn(result, new_refresh_in) class TestTelemetryMaintainingOfflineState(unittest.TestCase): diff --git a/tests/test_mi.py b/tests/test_mi.py index d6dcc159..f3182c7b 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -26,6 +26,7 @@ SERVICE_FABRIC, DEFAULT_TO_VM, ) +from msal.token_cache import is_subdict_of class ManagedIdentityTestCase(unittest.TestCase): @@ -60,7 +61,7 @@ def setUp(self): http_client=requests.Session(), ) - def _test_token_cache(self, app): + def assertCacheStatus(self, app): cache = app._token_cache._cache self.assertEqual(1, len(cache.get("AccessToken", [])), "Should have 1 AT") at = list(cache["AccessToken"].values())[0] @@ -70,30 +71,55 @@ def _test_token_cache(self, app): "Should have expected client_id") self.assertEqual("managed_identity", at["realm"], "Should have expected realm") - def _test_happy_path(self, app, mocked_http): - result = app.acquire_token_for_client(resource="R") + def _test_happy_path(self, app, mocked_http, expires_in, resource="R"): + result = app.acquire_token_for_client(resource=resource) mocked_http.assert_called() - self.assertEqual({ + call_count = mocked_http.call_count + expected_result = { "access_token": "AT", - "expires_in": 1234, - "resource": "R", "token_type": "Bearer", - }, result, "Should obtain a token response") + } + self.assertTrue( + is_subdict_of(expected_result, result), # We will test refresh_on later + "Should obtain a token response") + self.assertEqual(expires_in, result["expires_in"], "Should have expected expires_in") + if expires_in >= 7200: + expected_refresh_on = int(time.time() + expires_in / 2) + self.assertTrue( + expected_refresh_on - 1 <= result["refresh_on"] <= expected_refresh_on + 1, + "Should have a refresh_on time around the middle of the token's life") self.assertEqual( result["access_token"], - app.acquire_token_for_client(resource="R").get("access_token"), + app.acquire_token_for_client(resource=resource).get("access_token"), "Should hit the same token from cache") - self._test_token_cache(app) + + self.assertCacheStatus(app) + + result = app.acquire_token_for_client(resource=resource) + self.assertEqual( + call_count, mocked_http.call_count, + "No new call to the mocked http should be made for a cache hit") + self.assertTrue( + is_subdict_of(expected_result, result), # We will test refresh_on later + "Should obtain a token response") + self.assertTrue( + expires_in - 5 < result["expires_in"] <= expires_in, + "Should have similar expires_in") + if expires_in >= 7200: + self.assertTrue( + expected_refresh_on - 5 < result["refresh_on"] <= expected_refresh_on, + "Should have a refresh_on time around the middle of the token's life") class VmTestCase(ClientTestCase): def test_happy_path(self): + expires_in = 7890 # We test a bigger than 7200 value here with patch.object(self.app._http_client, "get", return_value=MinimalResponse( status_code=200, - text='{"access_token": "AT", "expires_in": "1234", "resource": "R"}', + text='{"access_token": "AT", "expires_in": "%s", "resource": "R"}' % expires_in, )) as mocked_method: - self._test_happy_path(self.app, mocked_method) + self._test_happy_path(self.app, mocked_method, expires_in) def test_vm_error_should_be_returned_as_is(self): raw_error = '{"raw": "error format is undefined"}' @@ -110,12 +136,13 @@ def test_vm_error_should_be_returned_as_is(self): class AppServiceTestCase(ClientTestCase): def test_happy_path(self): + expires_in = 1234 with patch.object(self.app._http_client, "get", return_value=MinimalResponse( status_code=200, text='{"access_token": "AT", "expires_on": "%s", "resource": "R"}' % ( - int(time.time()) + 1234), + int(time.time()) + expires_in), )) as mocked_method: - self._test_happy_path(self.app, mocked_method) + self._test_happy_path(self.app, mocked_method, expires_in) def test_app_service_error_should_be_normalized(self): raw_error = '{"statusCode": 500, "message": "error content is undefined"}' @@ -134,12 +161,13 @@ def test_app_service_error_should_be_normalized(self): class MachineLearningTestCase(ClientTestCase): def test_happy_path(self): + expires_in = 1234 with patch.object(self.app._http_client, "get", return_value=MinimalResponse( status_code=200, text='{"access_token": "AT", "expires_on": "%s", "resource": "R"}' % ( - int(time.time()) + 1234), + int(time.time()) + expires_in), )) as mocked_method: - self._test_happy_path(self.app, mocked_method) + self._test_happy_path(self.app, mocked_method, expires_in) def test_machine_learning_error_should_be_normalized(self): raw_error = '{"error": "placeholder", "message": "placeholder"}' @@ -162,12 +190,14 @@ def test_machine_learning_error_should_be_normalized(self): class ServiceFabricTestCase(ClientTestCase): def _test_happy_path(self, app): + expires_in = 1234 with patch.object(app._http_client, "get", return_value=MinimalResponse( status_code=200, text='{"access_token": "AT", "expires_on": %s, "resource": "R", "token_type": "Bearer"}' % ( - int(time.time()) + 1234), + int(time.time()) + expires_in), )) as mocked_method: - super(ServiceFabricTestCase, self)._test_happy_path(app, mocked_method) + super(ServiceFabricTestCase, self)._test_happy_path( + app, mocked_method, expires_in) def test_happy_path(self): self._test_happy_path(self.app) @@ -212,15 +242,16 @@ class ArcTestCase(ClientTestCase): }) def test_happy_path(self, mocked_stat): + expires_in = 1234 with patch.object(self.app._http_client, "get", side_effect=[ self.challenge, MinimalResponse( status_code=200, - text='{"access_token": "AT", "expires_in": "1234", "resource": "R"}', + text='{"access_token": "AT", "expires_in": "%s", "resource": "R"}' % expires_in, ), ]) as mocked_method: try: - super(ArcTestCase, self)._test_happy_path(self.app, mocked_method) + self._test_happy_path(self.app, mocked_method, expires_in) mocked_stat.assert_called_with(os.path.join( _supported_arc_platforms_and_their_prefixes[sys.platform], "foo.key")) From 3279f045dc573d8408ddcdf3565e8b16247627be Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 17 Jul 2024 12:34:56 -0700 Subject: [PATCH 174/262] MSAL 1.30.0 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 8f30eb1c..b3c07a47 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,7 +21,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.29.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.30.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 2b0375127de312a09188ae2b106a29dda71c9a92 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 23 Jul 2024 12:34:56 -0700 Subject: [PATCH 175/262] Bumping cryptography upper bound --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4935d352..490e3ab8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=2.5,<45 + cryptography>=2.5,<46 [options.extras_require] From 95e1bb0ec39e6d589dd3d50e3069e528757bd601 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 22 Jul 2024 12:34:56 -0700 Subject: [PATCH 176/262] Delay getfqdn() from import time to runtime. Fix #715 --- msal/managed_identity.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index aee57ca3..a534fa7a 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -144,7 +144,7 @@ class ManagedIdentityClient(object): (like what a ``PublicClientApplication`` does), not a token with application permissions for an app. """ - _instance, _tenant = socket.getfqdn(), "managed_identity" # Placeholders + __instance, _tenant = None, "managed_identity" # Placeholders def __init__( self, @@ -232,6 +232,11 @@ def __init__( ) self._token_cache = token_cache or TokenCache() + def _get_instance(self): + if self.__instance is None: + self.__instance = socket.getfqdn() # Moved from class definition to here + return self.__instance + def acquire_token_for_client(self, *, resource): # We may support scope in the future """Acquire token for the managed identity. @@ -257,7 +262,7 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the target=[resource], query=dict( client_id=client_id_in_cache, - environment=self._instance, + environment=self._get_instance(), realm=self._tenant, home_account_id=None, ), @@ -287,7 +292,8 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the self._token_cache.add(dict( client_id=client_id_in_cache, scope=[resource], - token_endpoint="https://{}/{}".format(self._instance, self._tenant), + token_endpoint="https://{}/{}".format( + self._get_instance(), self._tenant), response=result, params={}, data={}, From 4e2111cb9418cf8d5cd20b7606b867a3b5d35347 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 26 Jul 2024 12:34:56 -0700 Subject: [PATCH 177/262] Refine inline comment --- msal/managed_identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index a534fa7a..28682de8 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -424,7 +424,7 @@ def _obtain_token_on_azure_vm(http_client, managed_identity, resource): "resource": payload.get("resource"), "token_type": payload.get("token_type", "Bearer"), } - return payload # Typically an error, but it is undefined in the doc above + return payload # It would be {"error": ..., "error_description": ...} according to https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling except json.decoder.JSONDecodeError: logger.debug("IMDS emits unexpected payload: %s", resp.text) raise From ca0877e953200cb9c3e68b572143442349bd1927 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 29 Jul 2024 12:34:56 -0700 Subject: [PATCH 178/262] Mentions MSAL-Extensions --- msal/token_cache.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index 3cf48c4b..e554e118 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -363,11 +363,14 @@ class SerializableTokenCache(TokenCache): This class does NOT actually persist the cache on disk/db/etc.. Depending on your need, - the following simple recipe for file-based persistence may be sufficient:: + the following simple recipe for file-based, unencrypted persistence may be sufficient:: import os, atexit, msal cache_filename = os.path.join( # Persist cache into this file - os.getenv("XDG_RUNTIME_DIR", ""), # Automatically wipe out the cache from Linux when user's ssh session ends. See also https://github.com/AzureAD/microsoft-authentication-library-for-python/issues/690 + os.getenv( + # Automatically wipe out the cache from Linux when user's ssh session ends. + # See also https://github.com/AzureAD/microsoft-authentication-library-for-python/issues/690 + "XDG_RUNTIME_DIR", ""), "my_cache.bin") cache = msal.SerializableTokenCache() if os.path.exists(cache_filename): @@ -380,6 +383,10 @@ class SerializableTokenCache(TokenCache): app = msal.ClientApplication(..., token_cache=cache) ... + Alternatively, you may use a more sophisticated cache persistence library, + `MSAL Extensions `_, + which provides token cache persistence with encryption, and more. + :var bool has_state_changed: Indicates whether the cache state in the memory has changed since last :func:`~serialize` or :func:`~deserialize` call. From fe8f7583a539f9a5c232df31c916368bc12057f7 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 30 Jul 2024 12:34:56 -0700 Subject: [PATCH 179/262] Change arc mi's detection algorithm --- msal/managed_identity.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 28682de8..5636f564 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -316,6 +316,17 @@ def _scope_to_resource(scope): # This is an experimental reasonable-effort appr return scope # There is no much else we can do here +def _get_arc_endpoint(): + if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: + return os.environ["IDENTITY_ENDPOINT"] + if ( # Defined in https://msazure.visualstudio.com/One/_wiki/wikis/One.wiki/233012/VM-Extension-Authoring-for-Arc?anchor=determining-which-endpoint-to-use + sys.platform == "linux" and os.path.exists("/var/opt/azcmagent/bin/himds") + or sys.platform == "win32" and os.path.exists(os.path.expandvars( + r"%ProgramFiles%\AzureConnectedMachineAgent\himds.exe")) + ): + return "http://localhost:40342/metadata/identity/oauth2/token" + + APP_SERVICE = object() AZURE_ARC = object() CLOUD_SHELL = object() # In MSAL Python, token acquisition was done by @@ -338,7 +349,7 @@ def get_managed_identity_source(): return APP_SERVICE if "MSI_ENDPOINT" in os.environ and "MSI_SECRET" in os.environ: return MACHINE_LEARNING - if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: + if _get_arc_endpoint(): return AZURE_ARC if _is_running_in_cloud_shell(): return CLOUD_SHELL @@ -380,18 +391,15 @@ def _obtain_token(http_client, managed_identity, resource): managed_identity, resource, ) - if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: + arc_endpoint = _get_arc_endpoint() + if arc_endpoint: if ManagedIdentity.is_user_assigned(managed_identity): raise ManagedIdentityError( # Note: Azure Identity for Python raised exception too "Invalid managed_identity parameter. " "Azure Arc supports only system-assigned managed identity, " "See also " "https://learn.microsoft.com/en-us/azure/service-fabric/configure-existing-cluster-enable-managed-identity-token-service") - return _obtain_token_on_arc( - http_client, - os.environ["IDENTITY_ENDPOINT"], - resource, - ) + return _obtain_token_on_arc(http_client, arc_endpoint, resource) return _obtain_token_on_azure_vm(http_client, managed_identity, resource) From d51abb4f6c4a9761c3efb885f4bb9408323d4a3d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 5 Aug 2024 12:34:56 -0700 Subject: [PATCH 180/262] CAE for MIv1 CAE team and MSI team are working on turning on CAE by default for MSI v1. So what that means is, App developers will start seeing CAE even without setting the capability - "CP1". Update msal/application.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> Update msal/application.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> Update msal/application.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> Update msal/managed_identity.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> Update msal/managed_identity.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> Update msal/managed_identity.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> Update msal/managed_identity.py Co-authored-by: Den Delimarsky <53200638+localden@users.noreply.github.com> --- msal/application.py | 8 +++++--- msal/managed_identity.py | 33 ++++++++++++++++++++++++++++----- tests/test_mi.py | 15 ++++++++------- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/msal/application.py b/msal/application.py index b3c07a47..75ca6c83 100644 --- a/msal/application.py +++ b/msal/application.py @@ -411,9 +411,11 @@ def __init__( (STS) what this client is capable for, so STS can decide to turn on certain features. For example, if client is capable to handle *claims challenge*, - STS can then issue CAE access tokens to resources - knowing when the resource emits *claims challenge* - the client will be capable to handle. + STS may issue + `Continuous Access Evaluation (CAE) `_ + access tokens to resources, + knowing that when the resource emits a *claims challenge* + the client will be able to handle those challenges. Implementation details: Client capability is implemented using "claims" parameter on the wire, diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 5636f564..181d34c3 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -10,7 +10,7 @@ import time from urllib.parse import urlparse # Python 3+ from collections import UserDict # Python 3+ -from typing import Union # Needed in Python 3.7 & 3.8 +from typing import Optional, Union # Needed in Python 3.7 & 3.8 from .token_cache import TokenCache from .individual_cache import _IndividualCache as IndividualCache from .throttled_http_client import ThrottledHttpClientBase, RetryAfterParser @@ -145,6 +145,9 @@ class ManagedIdentityClient(object): not a token with application permissions for an app. """ __instance, _tenant = None, "managed_identity" # Placeholders + _TOKEN_SOURCE = "token_source" + _TOKEN_SOURCE_IDP = "identity_provider" + _TOKEN_SOURCE_CACHE = "cache" def __init__( self, @@ -237,12 +240,31 @@ def _get_instance(self): self.__instance = socket.getfqdn() # Moved from class definition to here return self.__instance - def acquire_token_for_client(self, *, resource): # We may support scope in the future + def acquire_token_for_client( + self, + *, + resource: str, # If/when we support scope, resource will become optional + claims_challenge: Optional[str] = None, + ): """Acquire token for the managed identity. The result will be automatically cached. Subsequent calls will automatically search from cache first. + :param resource: The resource for which the token is acquired. + + :param claims_challenge: + Optional. + It is a string representation of a JSON object + (which contains lists of claims being requested). + + The tenant admin may choose to revoke all Managed Identity tokens, + and then a *claims challenge* will be returned by the target resource, + as a `claims_challenge` directive in the `www-authenticate` header, + even if the app developer did not opt in for the "CP1" client capability. + Upon receiving a `claims_challenge`, MSAL will skip a token cache read, + and will attempt to acquire a new token. + .. note:: Known issue: When an Azure VM has only one user-assigned managed identity, @@ -255,8 +277,8 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the access_token_from_cache = None client_id_in_cache = self._managed_identity.get( ManagedIdentity.ID, "SYSTEM_ASSIGNED_MANAGED_IDENTITY") - if True: # Does not offer an "if not force_refresh" option, because - # there would be built-in token cache in the service side anyway + now = time.time() + if not claims_challenge: # Then attempt token cache search matches = self._token_cache.find( self._token_cache.CredentialType.ACCESS_TOKEN, target=[resource], @@ -267,7 +289,6 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the home_account_id=None, ), ) - now = time.time() for entry in matches: expires_in = int(entry["expires_on"]) - now if expires_in < 5*60: # Then consider it expired @@ -277,6 +298,7 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the "access_token": entry["secret"], "token_type": entry.get("token_type", "Bearer"), "expires_in": int(expires_in), # OAuth2 specs defines it as int + self._TOKEN_SOURCE: self._TOKEN_SOURCE_CACHE, } if "refresh_on" in entry: access_token_from_cache["refresh_on"] = int(entry["refresh_on"]) @@ -300,6 +322,7 @@ def acquire_token_for_client(self, *, resource): # We may support scope in the )) if "refresh_in" in result: result["refresh_on"] = int(now + result["refresh_in"]) + result[self._TOKEN_SOURCE] = self._TOKEN_SOURCE_IDP if (result and "error" not in result) or (not access_token_from_cache): return result except: # The exact HTTP exception is transportation-layer dependent diff --git a/tests/test_mi.py b/tests/test_mi.py index f3182c7b..2041419d 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -82,20 +82,17 @@ def _test_happy_path(self, app, mocked_http, expires_in, resource="R"): self.assertTrue( is_subdict_of(expected_result, result), # We will test refresh_on later "Should obtain a token response") + self.assertTrue(result["token_source"], "identity_provider") self.assertEqual(expires_in, result["expires_in"], "Should have expected expires_in") if expires_in >= 7200: expected_refresh_on = int(time.time() + expires_in / 2) self.assertTrue( expected_refresh_on - 1 <= result["refresh_on"] <= expected_refresh_on + 1, "Should have a refresh_on time around the middle of the token's life") - self.assertEqual( - result["access_token"], - app.acquire_token_for_client(resource=resource).get("access_token"), - "Should hit the same token from cache") - - self.assertCacheStatus(app) result = app.acquire_token_for_client(resource=resource) + self.assertCacheStatus(app) + self.assertEqual("cache", result["token_source"], "Should hit cache") self.assertEqual( call_count, mocked_http.call_count, "No new call to the mocked http should be made for a cache hit") @@ -110,6 +107,9 @@ def _test_happy_path(self, app, mocked_http, expires_in, resource="R"): expected_refresh_on - 5 < result["refresh_on"] <= expected_refresh_on, "Should have a refresh_on time around the middle of the token's life") + result = app.acquire_token_for_client(resource=resource, claims_challenge="foo") + self.assertEqual("identity_provider", result["token_source"], "Should miss cache") + class VmTestCase(ClientTestCase): @@ -249,7 +249,8 @@ def test_happy_path(self, mocked_stat): status_code=200, text='{"access_token": "AT", "expires_in": "%s", "resource": "R"}' % expires_in, ), - ]) as mocked_method: + ] * 2, # Duplicate a pair of mocks for _test_happy_path()'s CAE check + ) as mocked_method: try: self._test_happy_path(self.app, mocked_method, expires_in) mocked_stat.assert_called_with(os.path.join( From b9d9b05f3562721d72fe7294ed980313fdb1a70e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 25 Jul 2024 12:34:56 -0700 Subject: [PATCH 181/262] Anticipate a PyMsalRuntime 0.17.0 release soon --- setup.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 53e55e83..33ec3f06 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,9 +60,11 @@ broker = # The broker is defined as optional dependency, # so that downstream apps can opt in. The opt-in is needed, partially because # most existing MSAL Python apps do not have the redirect_uri needed by broker. + # # We need pymsalruntime.CallbackData introduced in PyMsalRuntime 0.14 - pymsalruntime>=0.14,<0.17; python_version>='3.6' and platform_system=='Windows' - pymsalruntime>=0.14,<0.17; python_version>='3.8' and platform_system=='Darwin' + pymsalruntime>=0.14,<0.18; python_version>='3.6' and platform_system=='Windows' + # On Mac, PyMsalRuntime 0.17+ is expected to support SSH cert and ROPC + pymsalruntime>=0.17,<0.18; python_version>='3.8' and platform_system=='Darwin' [options.packages.find] exclude = From 4afbd8da6a13a28635265bee33db21a1742f0c53 Mon Sep 17 00:00:00 2001 From: fengga <62267180+fengga@users.noreply.github.com> Date: Fri, 23 Aug 2024 08:10:02 -0700 Subject: [PATCH 182/262] Add an unofficial doc for mac broker integration (#732) * Add/update broker integration doc * Move the doc components from .md file to broker-test.py --- tests/broker-test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/broker-test.py b/tests/broker-test.py index 224a596e..cdcc4817 100644 --- a/tests/broker-test.py +++ b/tests/broker-test.py @@ -4,6 +4,16 @@ Each time a new PyMsalRuntime is going to be released, we can use this script to test it with a given version of MSAL Python. + +1. If you are on a modern Windows device, broker WAM is already built-in; + If you are on a mac device, install CP (Company Portal), login an account in CP and finish the MDM process. +2. For installing MSAL Python from its latest `dev` branch: + `pip install --force-reinstall "git+https://github.com/AzureAD/microsoft-authentication-library-for-python.git[broker]"` +3. (Optional) A proper version of `PyMsalRuntime` has already been installed by the previous command. + But if you want to test a specific version of `PyMsalRuntime`, + you shall manually install that version now. +4. Run this test by `python broker-test.py` and make sure all the tests passed. + """ import msal import getpass From c6595d3a6c674002221e153a8aa5ecff2cd9e8dc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 6 Aug 2024 12:34:56 -0700 Subject: [PATCH 183/262] Switch to the future-proof ciamcud tenant Old ciam2 test tenant is obsolete --- tests/test_e2e.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 74dff31c..fed70ded 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -16,6 +16,7 @@ import json import time import unittest +from urllib.parse import urlparse, parse_qs import sys try: from unittest.mock import patch, ANY @@ -1005,7 +1006,10 @@ class CiamTestCase(LabBasedTestCase): @classmethod def setUpClass(cls): super(CiamTestCase, cls).setUpClass() - cls.user = cls.get_lab_user(federationProvider="ciam") + cls.user = cls.get_lab_user( + #federationProvider="ciam", # This line would return ciam2 tenant + federationProvider="ciamcud", signinAudience="AzureAdMyOrg", # ciam6 + ) # FYI: Only single- or multi-tenant CIAM app can have other-than-OIDC # delegated permissions on Microsoft Graph. cls.app_config = cls.get_lab_app_object(cls.user["client_id"]) @@ -1020,13 +1024,17 @@ def test_ciam_acquire_token_interactive(self): ) def test_ciam_acquire_token_for_client(self): + raw_url = self.app_config["clientSecret"] + secret_url = urlparse(raw_url) + if secret_url.query: # Ciam2 era has a query param Secret=name + secret_name = parse_qs(secret_url.query)["Secret"][0] + else: # Ciam6 era has a URL path that ends with the secret name + secret_name = secret_url.path.split("/")[-1] + logger.info('Detected secret name "%s" from "%s"', secret_name, raw_url) self._test_acquire_token_by_client_secret( client_id=self.app_config["appId"], - client_secret=self.get_lab_user_secret( - self.app_config["clientSecret"].split("=")[-1]), + client_secret=self.get_lab_user_secret(secret_name), authority=self.app_config["authority"], - #scope=["{}/.default".format(self.app_config["appId"])], # AADSTS500207: The account type can't be used for the resource you're trying to access. - #scope=["api://{}/.default".format(self.app_config["appId"])], # AADSTS500011: The resource principal named api://ced781e7-bdb0-4c99-855c-d3bacddea88a was not found in the tenant named MSIDLABCIAM2. This can happen if the application has not been installed by the administrator of the tenant or consented to by any user in the tenant. You might have sent your authentication request to the wrong tenant. scope=self.app_config["scopes"], # It shall ends with "/.default" ) @@ -1046,6 +1054,8 @@ def test_ciam_acquire_token_by_ropc(self): scope=self.app_config["scopes"], ) + @unittest.skip("""As of Aug 2024, in both ciam2 and ciam6, sign-in fails with +AADSTS500208: The domain is not a valid login domain for the account type.""") def test_ciam_device_flow(self): self._test_device_flow( authority=self.app_config["authority"], From 094ce754013a650832fd5a5af8745e93a5bc8485 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 7 Aug 2024 12:34:56 -0700 Subject: [PATCH 184/262] Refactor to reuse CIAM test cases for CIAM CUD --- tests/test_e2e.py | 55 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index fed70ded..ff35a73e 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -172,6 +172,7 @@ def _build_app(cls, client_id, client_credential=None, authority="https://login.microsoftonline.com/common", + oidc_authority=None, scopes=["https://graph.microsoft.com/.default"], # Microsoft Graph http_client=None, azure_region=None, @@ -181,6 +182,7 @@ def _build_app(cls, client_id, client_credential=client_credential, authority=authority, + oidc_authority=oidc_authority, azure_region=azure_region, http_client=http_client or MinimalHttpClient(), ) @@ -194,6 +196,7 @@ def _build_app(cls, return msal.PublicClientApplication( client_id, authority=authority, + oidc_authority=oidc_authority, http_client=http_client or MinimalHttpClient(), enable_broker_on_windows=broker_available, enable_broker_on_mac=broker_available, @@ -201,14 +204,16 @@ def _build_app(cls, def _test_username_password(self, authority=None, client_id=None, username=None, password=None, scope=None, + oidc_authority=None, client_secret=None, # Since MSAL 1.11, confidential client has ROPC too azure_region=None, http_client=None, auth_scheme=None, **ignored): - assert authority and client_id and username and password and scope + assert client_id and username and password and scope and ( + authority or oidc_authority) self.app = self._build_app( - client_id, authority=authority, + client_id, authority=authority, oidc_authority=oidc_authority, http_client=http_client, azure_region=azure_region, # Regional endpoint does not support ROPC. # Here we just use it to test a regional app won't break ROPC. @@ -229,9 +234,14 @@ def _test_username_password(self, os.getenv("TRAVIS"), # It is set when running on TravisCI or Github Actions "Although it is doable, we still choose to skip device flow to save time") def _test_device_flow( - self, client_id=None, authority=None, scope=None, **ignored): - assert client_id and authority and scope - self.app = self._build_app(client_id, authority=authority) + self, + *, + client_id=None, authority=None, oidc_authority=None, scope=None, + **ignored + ): + assert client_id and scope and (authority or oidc_authority) + self.app = self._build_app( + client_id, authority=authority, oidc_authority=oidc_authority) flow = self.app.initiate_device_flow(scopes=scope) assert "user_code" in flow, "DF does not seem to be provisioned: %s".format( json.dumps(flow, indent=4)) @@ -255,7 +265,8 @@ def _test_device_flow( @unittest.skipIf(os.getenv("TRAVIS"), "Browser automation is not yet implemented") def _test_acquire_token_interactive( - self, client_id=None, authority=None, scope=None, port=None, + self, *, client_id=None, authority=None, scope=None, port=None, + oidc_authority=None, username=None, lab_name=None, username_uri="", # Unnecessary if you provided username and lab_name data=None, # Needed by ssh-cert feature @@ -263,8 +274,9 @@ def _test_acquire_token_interactive( enable_msa_passthrough=None, auth_scheme=None, **ignored): - assert client_id and authority and scope - self.app = self._build_app(client_id, authority=authority) + assert client_id and scope and (authority or oidc_authority) + self.app = self._build_app( + client_id, authority=authority, oidc_authority=oidc_authority) logger.info(_get_hint( # Useful when testing broker which shows no welcome_template username=username, lab_name=lab_name, username_uri=username_uri)) result = self.app.acquire_token_interactive( @@ -682,10 +694,13 @@ def _test_acquire_token_obo(self, config_pca, config_cca, def _test_acquire_token_by_client_secret( self, client_id=None, client_secret=None, authority=None, scope=None, + oidc_authority=None, **ignored): - assert client_id and client_secret and authority and scope + assert client_id and client_secret and scope and ( + authority or oidc_authority) self.app = msal.ConfidentialClientApplication( client_id, client_credential=client_secret, authority=authority, + oidc_authority=oidc_authority, http_client=MinimalHttpClient()) result = self.app.acquire_token_for_client(scope) self.assertIsNotNone(result.get("access_token"), "Got %s instead" % result) @@ -1016,7 +1031,8 @@ def setUpClass(cls): def test_ciam_acquire_token_interactive(self): self._test_acquire_token_interactive( - authority=self.app_config["authority"], + authority=self.app_config.get("authority"), + oidc_authority=self.app_config.get("oidc_authority"), client_id=self.app_config["appId"], scope=self.app_config["scopes"], username=self.user["username"], @@ -1034,7 +1050,8 @@ def test_ciam_acquire_token_for_client(self): self._test_acquire_token_by_client_secret( client_id=self.app_config["appId"], client_secret=self.get_lab_user_secret(secret_name), - authority=self.app_config["authority"], + authority=self.app_config.get("authority"), + oidc_authority=self.app_config.get("oidc_authority"), scope=self.app_config["scopes"], # It shall ends with "/.default" ) @@ -1047,7 +1064,8 @@ def test_ciam_acquire_token_by_ropc(self): # and enabling "Allow public client flows". # Otherwise it would hit AADSTS7000218. self._test_username_password( - authority=self.app_config["authority"], + authority=self.app_config.get("authority"), + oidc_authority=self.app_config.get("oidc_authority"), client_id=self.app_config["appId"], username=self.user["username"], password=self.get_lab_user_secret(self.user["lab_name"]), @@ -1058,12 +1076,23 @@ def test_ciam_acquire_token_by_ropc(self): AADSTS500208: The domain is not a valid login domain for the account type.""") def test_ciam_device_flow(self): self._test_device_flow( - authority=self.app_config["authority"], + authority=self.app_config.get("authority"), + oidc_authority=self.app_config.get("oidc_authority"), client_id=self.app_config["appId"], scope=self.app_config["scopes"], ) +class CiamCudTestCase(CiamTestCase): + @classmethod + def setUpClass(cls): + super(CiamCudTestCase, cls).setUpClass() + cls.app_config["authority"] = None + cls.app_config["oidc_authority"] = ( + # Derived from https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.63.0/tests/Microsoft.Identity.Test.Integration.netcore/HeadlessTests/CiamIntegrationTests.cs#L156 + "https://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0") + + class WorldWideRegionalEndpointTestCase(LabBasedTestCase): region = "westus" timeout = 2 # Short timeout makes this test case responsive on non-VM From 95336a4a9216612cac5ab2f9efea85251d17a065 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 19 Jul 2024 12:34:56 -0700 Subject: [PATCH 185/262] Fix typos in doc --- msal/application.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/msal/application.py b/msal/application.py index 75c36d59..95dfde6d 100644 --- a/msal/application.py +++ b/msal/application.py @@ -2050,15 +2050,14 @@ def acquire_token_interactive( - If your app is a GUI app running on modern Windows system, you are required to also provide its window handle, so that the sign-in window will pop up on top of your window. - - If your app is a console app runnong on Windows system, + - If your app is a console app running on Windows system, you can use a placeholder ``PublicClientApplication.CONSOLE_WINDOW_HANDLE``. - If your app is running on Mac, you can use a placeholder ``PublicClientApplication.CONSOLE_WINDOW_HANDLE``. - If your app is a console app (most Python scripts are console apps), - you can use a placeholder value ``msal.PublicClientApplication.CONSOLE_WINDOW_HANDLE``. + Most Python scripts are console apps. New in version 1.20.0. From 828b419521347678f20dbc548732a972cb912178 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 9 Aug 2024 12:34:56 -0700 Subject: [PATCH 186/262] parent_window_handle is also needed on Mac --- msal/application.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/msal/application.py b/msal/application.py index 95dfde6d..7ca4bea1 100644 --- a/msal/application.py +++ b/msal/application.py @@ -2047,13 +2047,10 @@ def acquire_token_interactive( * If your app opts in to use broker, ``parent_window_handle`` is required. - - If your app is a GUI app running on modern Windows system, + - If your app is a GUI app running on Windows or Mac system, you are required to also provide its window handle, so that the sign-in window will pop up on top of your window. - - If your app is a console app running on Windows system, - you can use a placeholder - ``PublicClientApplication.CONSOLE_WINDOW_HANDLE``. - - If your app is running on Mac, + - If your app is a console app running on Windows or Mac system, you can use a placeholder ``PublicClientApplication.CONSOLE_WINDOW_HANDLE``. From fd0335f7cf9811a279e5a8c32340b3d224840c59 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 13 Aug 2024 23:21:08 -0700 Subject: [PATCH 187/262] Explicitly test current broker fallback behaviors --- msal/application.py | 9 +++-- tests/test_application.py | 71 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/msal/application.py b/msal/application.py index 7ca4bea1..48cb8f77 100644 --- a/msal/application.py +++ b/msal/application.py @@ -26,6 +26,11 @@ logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" +def _init_broker(enable_pii_log): # Make it a function to allow mocking + from . import broker # Trigger Broker's initialization, lazily + if enable_pii_log: + broker._enable_pii_log() + def extract_certs(public_cert_content): # Parses raw public certificate file contents and returns a list of strings # Usage: headers = {"x5c": extract_certs(open("my_cert.pem").read())} @@ -655,9 +660,7 @@ def _decide_broker(self, allow_broker, enable_pii_log): if (self._enable_broker and not is_confidential_app and not self.authority.is_adfs and not self.authority._is_b2c): try: - from . import broker # Trigger Broker's initialization - if enable_pii_log: - broker._enable_pii_log() + _init_broker(enable_pii_log) except RuntimeError: self._enable_broker = False logger.exception( diff --git a/tests/test_application.py b/tests/test_application.py index 71dc16ea..d6acaf0b 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -1,11 +1,16 @@ # Note: Since Aug 2019 we move all e2e tests into test_e2e.py, # so this test_application file contains only unit tests without dependency. +import json +import logging import sys import time -from msal.application import * -from msal.application import _str2bytes +from unittest.mock import patch, Mock import msal -from msal.application import _merge_claims_challenge_and_capabilities +from msal.application import ( + extract_certs, + ClientApplication, PublicClientApplication, ConfidentialClientApplication, + _str2bytes, _merge_claims_challenge_and_capabilities, +) from tests import unittest from tests.test_token_cache import build_id_token, build_response from tests.http_client import MinimalHttpClient, MinimalResponse @@ -722,3 +727,63 @@ def test_client_id_should_be_a_valid_scope(self): self._test_client_id_should_be_a_valid_scope("client_id", []) self._test_client_id_should_be_a_valid_scope("client_id", ["foo"]) + +@patch("sys.platform", new="darwin") # Pretend running on Mac. +@patch("msal.authority.tenant_discovery", new=Mock(return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", + })) +@patch("msal.application._init_broker", new=Mock()) # Allow testing without pymsalruntime +class TestBrokerFallback(unittest.TestCase): + + def test_broker_should_be_disabled_by_default(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://login.microsoftonline.com/common", + ) + self.assertFalse(app._enable_broker) + + def test_broker_should_be_enabled_when_opted_in(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://login.microsoftonline.com/common", + enable_broker_on_mac=True, + ) + self.assertTrue(app._enable_broker) + + def test_should_fallback_to_non_broker_when_using_adfs(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://contoso.com/adfs", + #instance_discovery=False, # Automatically skipped when detected ADFS + enable_broker_on_mac=True, + ) + self.assertFalse(app._enable_broker) + + def test_should_fallback_to_non_broker_when_using_b2c(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://contoso.b2clogin.com/contoso/policy", + #instance_discovery=False, # Automatically skipped when detected B2C + enable_broker_on_mac=True, + ) + self.assertFalse(app._enable_broker) + + def test_should_use_broker_when_disabling_instance_discovery(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://contoso.com/path", + instance_discovery=False, # Need this for a generic authority url + enable_broker_on_mac=True, + ) + # TODO: Shall we bypass broker when opted out of instance discovery? + self.assertTrue(app._enable_broker) # Current implementation enables broker + + def test_should_fallback_to_non_broker_when_using_oidc_authority(self): + app = msal.PublicClientApplication( + "client_id", + oidc_authority="https://contoso.com/path", + enable_broker_on_mac=True, + ) + self.assertFalse(app._enable_broker) + From 4ce6646b6af1dca71d0506b0024783c9618e618a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 14 Aug 2024 23:23:22 -0700 Subject: [PATCH 188/262] ADFS and B2C shall not invoke broker --- msal/application.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/msal/application.py b/msal/application.py index 48cb8f77..99f2e285 100644 --- a/msal/application.py +++ b/msal/application.py @@ -221,8 +221,6 @@ class ClientApplication(object): "You can enable broker by following these instructions. " "https://msal-python.readthedocs.io/en/latest/#publicclientapplication") - _enable_broker = False - def __init__( self, client_id, client_credential=None, authority=None, validate_authority=True, @@ -651,14 +649,22 @@ def _decide_broker(self, allow_broker, enable_pii_log): "enable_broker_on_windows=True, " "enable_broker_on_mac=...)", DeprecationWarning) - self._enable_broker = self._enable_broker or ( + opted_in_for_broker = ( + self._enable_broker # True means Opted-in from PCA + or ( # When we started the broker project on Windows platform, # the allow_broker was meant to be cross-platform. Now we realize # that other platforms have different redirect_uri requirements, # so the old allow_broker is deprecated and will only for Windows. allow_broker and sys.platform == "win32") - if (self._enable_broker and not is_confidential_app - and not self.authority.is_adfs and not self.authority._is_b2c): + ) + self._enable_broker = ( # This same variable will also store the state + opted_in_for_broker + and not is_confidential_app + and not self.authority.is_adfs + and not self.authority._is_b2c + ) + if self._enable_broker: try: _init_broker(enable_pii_log) except RuntimeError: From 0a756e96846c113882689a30ea05aff48b07e6ea Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 19 Aug 2024 12:34:56 -0700 Subject: [PATCH 189/262] Error out on invalid ManagedIdentity dict --- msal/managed_identity.py | 8 ++++++-- tests/test_mi.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 181d34c3..bbd4f52f 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -46,8 +46,9 @@ class ManagedIdentity(UserDict): @classmethod def is_managed_identity(cls, unknown): - return isinstance(unknown, ManagedIdentity) or ( - isinstance(unknown, dict) and cls.ID_TYPE in unknown) + return (isinstance(unknown, ManagedIdentity) + or cls.is_system_assigned(unknown) + or cls.is_user_assigned(unknown)) @classmethod def is_system_assigned(cls, unknown): @@ -217,6 +218,9 @@ def __init__( ) token = client.acquire_token_for_client("resource") """ + if not ManagedIdentity.is_managed_identity(managed_identity): + raise ManagedIdentityError( + f"Incorrect managed_identity: {managed_identity}") self._managed_identity = managed_identity self._http_client = _ThrottledHttpClient( # This class only throttles excess token acquisition requests. diff --git a/tests/test_mi.py b/tests/test_mi.py index 2041419d..ec2803ca 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -61,6 +61,14 @@ def setUp(self): http_client=requests.Session(), ) + def test_error_out_on_invalid_input(self): + with self.assertRaises(ManagedIdentityError): + ManagedIdentityClient({"foo": "bar"}, http_client=requests.Session()) + with self.assertRaises(ManagedIdentityError): + ManagedIdentityClient( + {"ManagedIdentityIdType": "undefined", "Id": "foo"}, + http_client=requests.Session()) + def assertCacheStatus(self, app): cache = app._token_cache._cache self.assertEqual(1, len(cache.get("AccessToken", [])), "Should have 1 AT") @@ -241,6 +249,9 @@ class ArcTestCase(ClientTestCase): "WWW-Authenticate": "Basic realm=/tmp/foo", }) + def test_error_out_on_invalid_input(self, mocked_stat): + return super(ArcTestCase, self).test_error_out_on_invalid_input() + def test_happy_path(self, mocked_stat): expires_in = 1234 with patch.object(self.app._http_client, "get", side_effect=[ From 28fbf7cb1fd4ea5e2276cb130acc01104f6f2761 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 30 Aug 2024 12:34:56 -0700 Subject: [PATCH 190/262] Resource id adjustments Resource ID as mi_res_id in App Service, as msi_res_id in other flavors --- msal/managed_identity.py | 13 +++++++++---- tests/test_mi.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index bbd4f52f..608bc1bf 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -40,7 +40,7 @@ class ManagedIdentity(UserDict): _types_mapping = { # Maps type name in configuration to type name on wire CLIENT_ID: "client_id", - RESOURCE_ID: "mi_res_id", + RESOURCE_ID: "msi_res_id", # VM's IMDS prefers msi_res_id https://github.com/Azure/azure-rest-api-specs/blob/dba6ed1f03bda88ac6884c0a883246446cc72495/specification/imds/data-plane/Microsoft.InstanceMetadataService/stable/2018-10-01/imds.json#L233-L239 OBJECT_ID: "object_id", } @@ -430,9 +430,9 @@ def _obtain_token(http_client, managed_identity, resource): return _obtain_token_on_azure_vm(http_client, managed_identity, resource) -def _adjust_param(params, managed_identity): +def _adjust_param(params, managed_identity, types_mapping=None): # Modify the params dict in place - id_name = ManagedIdentity._types_mapping.get( + id_name = (types_mapping or ManagedIdentity._types_mapping).get( managed_identity.get(ManagedIdentity.ID_TYPE)) if id_name: params[id_name] = managed_identity[ManagedIdentity.ID] @@ -479,7 +479,12 @@ def _obtain_token_on_app_service( "api-version": "2019-08-01", "resource": resource, } - _adjust_param(params, managed_identity) + _adjust_param(params, managed_identity, types_mapping={ + ManagedIdentity.CLIENT_ID: "client_id", + ManagedIdentity.RESOURCE_ID: "mi_res_id", # App Service's resource id uses "mi_res_id" + ManagedIdentity.OBJECT_ID: "object_id", + }) + resp = http_client.get( endpoint, params=params, diff --git a/tests/test_mi.py b/tests/test_mi.py index ec2803ca..1f33fe73 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -139,6 +139,22 @@ def test_vm_error_should_be_returned_as_is(self): json.loads(raw_error), self.app.acquire_token_for_client(resource="R")) self.assertEqual({}, self.app._token_cache._cache) + def test_vm_resource_id_parameter_should_be_msi_res_id(self): + app = ManagedIdentityClient( + {"ManagedIdentityIdType": "ResourceId", "Id": "1234"}, + http_client=requests.Session(), + ) + with patch.object(app._http_client, "get", return_value=MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_in": 3600, "resource": "R"}', + )) as mocked_method: + app.acquire_token_for_client(resource="R") + mocked_method.assert_called_with( + 'http://169.254.169.254/metadata/identity/oauth2/token', + params={'api-version': '2018-02-01', 'resource': 'R', 'msi_res_id': '1234'}, + headers={'Metadata': 'true'}, + ) + @patch.dict(os.environ, {"IDENTITY_ENDPOINT": "http://localhost", "IDENTITY_HEADER": "foo"}) class AppServiceTestCase(ClientTestCase): @@ -164,6 +180,22 @@ def test_app_service_error_should_be_normalized(self): }, self.app.acquire_token_for_client(resource="R")) self.assertEqual({}, self.app._token_cache._cache) + def test_app_service_resource_id_parameter_should_be_mi_res_id(self): + app = ManagedIdentityClient( + {"ManagedIdentityIdType": "ResourceId", "Id": "1234"}, + http_client=requests.Session(), + ) + with patch.object(app._http_client, "get", return_value=MinimalResponse( + status_code=200, + text='{"access_token": "AT", "expires_on": 12345, "resource": "R"}', + )) as mocked_method: + app.acquire_token_for_client(resource="R") + mocked_method.assert_called_with( + 'http://localhost', + params={'api-version': '2019-08-01', 'resource': 'R', 'mi_res_id': '1234'}, + headers={'X-IDENTITY-HEADER': 'foo', 'Metadata': 'true'}, + ) + @patch.dict(os.environ, {"MSI_ENDPOINT": "http://localhost", "MSI_SECRET": "foo"}) class MachineLearningTestCase(ClientTestCase): From 866e4ce28d6a87405775d8dbaf2163cd9f706f8f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 16 Aug 2024 12:34:56 -0700 Subject: [PATCH 191/262] Release MSAL Python 1.31.0 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 99f2e285..0869d9e5 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,7 +21,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.30.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.31.0" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 331c16fdd803df6a629cb215a6372dbad93f0ba8 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 22 Aug 2024 12:34:56 -0700 Subject: [PATCH 192/262] Test PyMsalRuntime ImportError and RuntimeError --- tests/test_application.py | 44 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/test_application.py b/tests/test_application.py index d6acaf0b..de916153 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -733,8 +733,48 @@ def test_client_id_should_be_a_valid_scope(self): "authorization_endpoint": "https://contoso.com/placeholder", "token_endpoint": "https://contoso.com/placeholder", })) -@patch("msal.application._init_broker", new=Mock()) # Allow testing without pymsalruntime -class TestBrokerFallback(unittest.TestCase): +class TestMsalBehaviorWithoutPyMsalRuntimeOrBroker(unittest.TestCase): + + @patch("msal.application._init_broker", new=Mock(side_effect=ImportError( + "PyMsalRuntime not installed" + ))) + def test_broker_should_be_disabled_by_default(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://login.microsoftonline.com/common", + ) + self.assertFalse(app._enable_broker) + + @patch("msal.application._init_broker", new=Mock(side_effect=ImportError( + "PyMsalRuntime not installed" + ))) + def test_should_error_out_when_opted_in_yet_pymsalruntime_not_installed(self): + with self.assertRaises(ImportError): + app = msal.PublicClientApplication( + "client_id", + authority="https://login.microsoftonline.com/common", + enable_broker_on_mac=True, + ) + + @patch("msal.application._init_broker", new=Mock(side_effect=RuntimeError( + "PyMsalRuntime raises RuntimeError when broker initialization failed" + ))) + def test_should_fallback_when_pymsalruntime_failed_to_initialize_broker(self): + app = msal.PublicClientApplication( + "client_id", + authority="https://login.microsoftonline.com/common", + enable_broker_on_mac=True, + ) + self.assertFalse(app._enable_broker) + + +@patch("sys.platform", new="darwin") # Pretend running on Mac. +@patch("msal.authority.tenant_discovery", new=Mock(return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", + })) +@patch("msal.application._init_broker", new=Mock()) # Pretend pymsalruntime installed and working +class TestBrokerFallbackWithDifferentAuthorities(unittest.TestCase): def test_broker_should_be_disabled_by_default(self): app = msal.PublicClientApplication( From ede849e0ea6fbc1df2bc8256a248e853b0dacc58 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 12 Sep 2024 12:34:56 -0700 Subject: [PATCH 193/262] Store extra data into access token in cache --- msal/token_cache.py | 7 +++++-- tests/test_token_cache.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index e554e118..136e38b8 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -249,8 +249,11 @@ def __add(self, event, now=None): "expires_on": str(now + expires_in), # Same here "extended_expires_on": str(now + ext_expires_in) # Same here } - if data.get("key_id"): # It happens in SSH-cert or POP scenario - at["key_id"] = data.get("key_id") + at.update({k: data[k] for k in data if k in { + # Also store extra data which we explicitly allow + # So that we won't accidentally store a user's password etc. + "key_id", # It happens in SSH-cert or POP scenario + }}) if "refresh_in" in response: refresh_in = response["refresh_in"] # It is an integer at["refresh_on"] = str(now + refresh_in) # Schema wants a string diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py index 4e301fa3..41547a0a 100644 --- a/tests/test_token_cache.py +++ b/tests/test_token_cache.py @@ -206,6 +206,38 @@ def testAddByAdfs(self): "appmetadata-fs.msidlab8.com-my_client_id") ) + def assertFoundAccessToken(self, *, scopes, query, data=None): + cached_at = None + for cached_at in self.cache.search( + TokenCache.CredentialType.ACCESS_TOKEN, target=scopes, query=query): + for k, v in (data or {}).items(): # The extra data, if any + self.assertEqual(cached_at.get(k), v, f"AT should contain {k}={v}") + self.assertTrue(cached_at, "AT should be cached and searchable") + return cached_at + + def _test_data_should_be_saved_and_searchable_in_access_token(self, data): + scopes = ["s2", "s1", "s3"] # Not in particular order + self.cache.add({ + "data": data, + "client_id": "my_client_id", + "scope": scopes, + "token_endpoint": "https://login.example.com/contoso/v2/token", + "response": build_response( + uid="uid", utid="utid", # client_info + expires_in=3600, access_token="an access token", + refresh_token="a refresh token"), + }, now=1000) + self.assertFoundAccessToken(scopes=scopes, data=data, query=dict( + data, # Also use the extra data as a query criteria + client_id="my_client_id", + environment="login.example.com", + realm="contoso", + home_account_id="uid.utid", + )) + + def test_extra_data_should_also_be_recorded_and_searchable_in_access_token(self): + self._test_data_should_be_saved_and_searchable_in_access_token({"key_id": "1"}) + def test_key_id_is_also_recorded(self): my_key_id = "some_key_id_123" self.cache.add({ @@ -258,7 +290,7 @@ def test_old_rt_data_with_wrong_key_should_still_be_salvaged_into_new_rt(self): ) -class SerializableTokenCacheTestCase(TokenCacheTestCase): +class SerializableTokenCacheTestCase(unittest.TestCase): # Run all inherited test methods, and have extra check in tearDown() def setUp(self): From c9dac6c4d430fa1bddf3777ff2d945431945fd62 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 13 Sep 2024 12:34:56 -0700 Subject: [PATCH 194/262] Change AccessToken key_maker algorithm --- msal/token_cache.py | 14 ++++++---- tests/test_token_cache.py | 58 +++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index 136e38b8..d1d38da0 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -43,6 +43,8 @@ def __init__(self): self._lock = threading.RLock() self._cache = {} self.key_makers = { + # Note: We have changed token key format before when ordering scopes; + # changing token key won't result in cache miss. self.CredentialType.REFRESH_TOKEN: lambda home_account_id=None, environment=None, client_id=None, target=None, **ignored_payload_from_a_real_token: @@ -56,14 +58,18 @@ def __init__(self): ]).lower(), self.CredentialType.ACCESS_TOKEN: lambda home_account_id=None, environment=None, client_id=None, - realm=None, target=None, **ignored_payload_from_a_real_token: - "-".join([ + realm=None, target=None, + # Note: New field(s) can be added here + #key_id=None, + **ignored_payload_from_a_real_token: + "-".join([ # Note: Could use a hash here to shorten key length home_account_id or "", environment or "", self.CredentialType.ACCESS_TOKEN, client_id or "", realm or "", target or "", + #key_id or "", # So ATs of different key_id can coexist ]).lower(), self.CredentialType.ID_TOKEN: lambda home_account_id=None, environment=None, client_id=None, @@ -150,9 +156,7 @@ def search(self, credential_type, target=None, query=None): # O(n) generator target_set = set(target) with self._lock: - # Since the target inside token cache key is (per schema) unsorted, - # there is no point to attempt an O(1) key-value search here. - # So we always do an O(n) in-memory search. + # O(n) search. The key is NOT used in search. for entry in self._cache.get(credential_type, {}).values(): if (entry != preferred_result # Avoid yielding the same entry twice and self._is_matching(entry, query, target_set=target_set) diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py index 41547a0a..4a2a9b56 100644 --- a/tests/test_token_cache.py +++ b/tests/test_token_cache.py @@ -3,7 +3,7 @@ import json import time -from msal.token_cache import * +from msal.token_cache import TokenCache, SerializableTokenCache from tests import unittest @@ -51,6 +51,8 @@ class TokenCacheTestCase(unittest.TestCase): def setUp(self): self.cache = TokenCache() + self.at_key_maker = self.cache.key_makers[ + TokenCache.CredentialType.ACCESS_TOKEN] def testAddByAad(self): client_id = "my_client_id" @@ -78,11 +80,8 @@ def testAddByAad(self): 'target': 's1 s2 s3', # Sorted 'token_type': 'some type', } - self.assertEqual( - access_token_entry, - self.cache._cache["AccessToken"].get( - 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3') - ) + self.assertEqual(access_token_entry, self.cache._cache["AccessToken"].get( + self.at_key_maker(**access_token_entry))) self.assertIn( access_token_entry, self.cache.find(self.cache.CredentialType.ACCESS_TOKEN), @@ -144,8 +143,7 @@ def testAddByAdfs(self): expires_in=3600, access_token="an access token", id_token=id_token, refresh_token="a refresh token"), }, now=1000) - self.assertEqual( - { + access_token_entry = { 'cached_at': "1000", 'client_id': 'my_client_id', 'credential_type': 'AccessToken', @@ -157,10 +155,9 @@ def testAddByAdfs(self): 'secret': 'an access token', 'target': 's1 s2 s3', # Sorted 'token_type': 'some type', - }, - self.cache._cache["AccessToken"].get( - 'subject-fs.msidlab8.com-accesstoken-my_client_id-adfs-s1 s2 s3') - ) + } + self.assertEqual(access_token_entry, self.cache._cache["AccessToken"].get( + self.at_key_maker(**access_token_entry))) self.assertEqual( { 'client_id': 'my_client_id', @@ -238,37 +235,32 @@ def _test_data_should_be_saved_and_searchable_in_access_token(self, data): def test_extra_data_should_also_be_recorded_and_searchable_in_access_token(self): self._test_data_should_be_saved_and_searchable_in_access_token({"key_id": "1"}) - def test_key_id_is_also_recorded(self): - my_key_id = "some_key_id_123" - self.cache.add({ - "data": {"key_id": my_key_id}, - "client_id": "my_client_id", - "scope": ["s2", "s1", "s3"], # Not in particular order - "token_endpoint": "https://login.example.com/contoso/v2/token", - "response": build_response( - uid="uid", utid="utid", # client_info - expires_in=3600, access_token="an access token", - refresh_token="a refresh token"), - }, now=1000) - cached_key_id = self.cache._cache["AccessToken"].get( - 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3', - {}).get("key_id") - self.assertEqual(my_key_id, cached_key_id, "AT should be bound to the key") + def test_access_tokens_with_different_key_id(self): + self._test_data_should_be_saved_and_searchable_in_access_token({"key_id": "1"}) + self._test_data_should_be_saved_and_searchable_in_access_token({"key_id": "2"}) + self.assertEqual( + len(self.cache._cache["AccessToken"]), + 1, """Historically, tokens are not keyed by key_id, +so a new token overwrites the old one, and we would end up with 1 token in cache""") def test_refresh_in_should_be_recorded_as_refresh_on(self): # Sounds weird. Yep. + scopes = ["s2", "s1", "s3"] # Not in particular order self.cache.add({ "client_id": "my_client_id", - "scope": ["s2", "s1", "s3"], # Not in particular order + "scope": scopes, "token_endpoint": "https://login.example.com/contoso/v2/token", "response": build_response( uid="uid", utid="utid", # client_info expires_in=3600, refresh_in=1800, access_token="an access token", ), #refresh_token="a refresh token"), }, now=1000) - refresh_on = self.cache._cache["AccessToken"].get( - 'uid.utid-login.example.com-accesstoken-my_client_id-contoso-s1 s2 s3', - {}).get("refresh_on") - self.assertEqual("2800", refresh_on, "Should save refresh_on") + at = self.assertFoundAccessToken(scopes=scopes, query=dict( + client_id="my_client_id", + environment="login.example.com", + realm="contoso", + home_account_id="uid.utid", + )) + self.assertEqual("2800", at.get("refresh_on"), "Should save refresh_on") def test_old_rt_data_with_wrong_key_should_still_be_salvaged_into_new_rt(self): sample = { From 60144d5115144e964bb8acd4b555b8e2828a0afc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 16 Sep 2024 12:34:56 -0700 Subject: [PATCH 195/262] TokenCache.search() also wipes stale access tokens --- msal/token_cache.py | 20 +++++++++++++++++--- tests/test_application.py | 7 +++++-- tests/test_token_cache.py | 16 ++++++++++------ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index d1d38da0..66be5c9f 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -130,7 +130,7 @@ def _is_matching(entry: dict, query: dict, target_set: set = None) -> bool: target_set <= set(entry.get("target", "").split()) if target_set else True) - def search(self, credential_type, target=None, query=None): # O(n) generator + def search(self, credential_type, target=None, query=None, *, now=None): # O(n) generator """Returns a generator of matching entries. It is O(1) for AT hits, and O(n) for other types. @@ -157,18 +157,32 @@ def search(self, credential_type, target=None, query=None): # O(n) generator target_set = set(target) with self._lock: # O(n) search. The key is NOT used in search. + now = int(time.time() if now is None else now) + expired_access_tokens = [ + # Especially when/if we key ATs by ephemeral fields such as key_id, + # stale ATs keyed by an old key_id would stay forever. + # Here we collect them for their removal. + ] for entry in self._cache.get(credential_type, {}).values(): + if ( # Automatically delete expired access tokens + credential_type == self.CredentialType.ACCESS_TOKEN + and int(entry["expires_on"]) < now + ): + expired_access_tokens.append(entry) # Can't delete them within current for-loop + continue if (entry != preferred_result # Avoid yielding the same entry twice and self._is_matching(entry, query, target_set=target_set) ): yield entry + for at in expired_access_tokens: + self.remove_at(at) - def find(self, credential_type, target=None, query=None): + def find(self, credential_type, target=None, query=None, *, now=None): """Equivalent to list(search(...)).""" warnings.warn( "Use list(search(...)) instead to explicitly get a list.", DeprecationWarning) - return list(self.search(credential_type, target=target, query=query)) + return list(self.search(credential_type, target=target, query=query, now=now)) def add(self, event, now=None): """Handle a token obtaining event, and add tokens into cache.""" diff --git a/tests/test_application.py b/tests/test_application.py index de916153..0736164c 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -340,6 +340,7 @@ class TestApplicationForRefreshInBehaviors(unittest.TestCase): account = {"home_account_id": "{}.{}".format(uid, utid)} rt = "this is a rt" client_id = "my_app" + soon = 60 # application.py considers tokens within 5 minutes as expired @classmethod def setUpClass(cls): # Initialization at runtime, not interpret-time @@ -414,7 +415,8 @@ def mock_post(url, headers=None, *args, **kwargs): def test_expired_token_and_unavailable_aad_should_return_error(self): # a.k.a. Attempt refresh expired token when AAD unavailable - self.populate_cache(access_token="expired at", expires_in=-1, refresh_in=-900) + self.populate_cache( + access_token="expired at", expires_in=self.soon, refresh_in=-900) error = "something went wrong" def mock_post(url, headers=None, *args, **kwargs): self.assertEqual("4|84,3|", (headers or {}).get(CLIENT_CURRENT_TELEMETRY)) @@ -425,7 +427,8 @@ def mock_post(url, headers=None, *args, **kwargs): def test_expired_token_and_available_aad_should_return_new_token(self): # a.k.a. Attempt refresh expired token when AAD available - self.populate_cache(access_token="expired at", expires_in=-1, refresh_in=-900) + self.populate_cache( + access_token="expired at", expires_in=self.soon, refresh_in=-900) new_access_token = "new AT" new_refresh_in = 123 def mock_post(url, headers=None, *args, **kwargs): diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py index 4a2a9b56..494d6daf 100644 --- a/tests/test_token_cache.py +++ b/tests/test_token_cache.py @@ -58,6 +58,7 @@ def testAddByAad(self): client_id = "my_client_id" id_token = build_id_token( oid="object1234", preferred_username="John Doe", aud=client_id) + now = 1000 self.cache.add({ "client_id": client_id, "scope": ["s2", "s1", "s3"], # Not in particular order @@ -66,7 +67,7 @@ def testAddByAad(self): uid="uid", utid="utid", # client_info expires_in=3600, access_token="an access token", id_token=id_token, refresh_token="a refresh token"), - }, now=1000) + }, now=now) access_token_entry = { 'cached_at': "1000", 'client_id': 'my_client_id', @@ -84,7 +85,7 @@ def testAddByAad(self): self.at_key_maker(**access_token_entry))) self.assertIn( access_token_entry, - self.cache.find(self.cache.CredentialType.ACCESS_TOKEN), + self.cache.find(self.cache.CredentialType.ACCESS_TOKEN, now=now), "find(..., query=None) should not crash, even though MSAL does not use it") self.assertEqual( { @@ -203,10 +204,12 @@ def testAddByAdfs(self): "appmetadata-fs.msidlab8.com-my_client_id") ) - def assertFoundAccessToken(self, *, scopes, query, data=None): + def assertFoundAccessToken(self, *, scopes, query, data=None, now=None): cached_at = None for cached_at in self.cache.search( - TokenCache.CredentialType.ACCESS_TOKEN, target=scopes, query=query): + TokenCache.CredentialType.ACCESS_TOKEN, + target=scopes, query=query, now=now, + ): for k, v in (data or {}).items(): # The extra data, if any self.assertEqual(cached_at.get(k), v, f"AT should contain {k}={v}") self.assertTrue(cached_at, "AT should be cached and searchable") @@ -214,6 +217,7 @@ def assertFoundAccessToken(self, *, scopes, query, data=None): def _test_data_should_be_saved_and_searchable_in_access_token(self, data): scopes = ["s2", "s1", "s3"] # Not in particular order + now = 1000 self.cache.add({ "data": data, "client_id": "my_client_id", @@ -223,8 +227,8 @@ def _test_data_should_be_saved_and_searchable_in_access_token(self, data): uid="uid", utid="utid", # client_info expires_in=3600, access_token="an access token", refresh_token="a refresh token"), - }, now=1000) - self.assertFoundAccessToken(scopes=scopes, data=data, query=dict( + }, now=now) + self.assertFoundAccessToken(scopes=scopes, data=data, now=now, query=dict( data, # Also use the extra data as a query criteria client_id="my_client_id", environment="login.example.com", From bb6d8723e59af7019a420f944d955cab9abb9cae Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 10 Sep 2024 12:34:56 -0700 Subject: [PATCH 196/262] Warning when obsolete msal-extensions is detected --- msal/application.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/msal/application.py b/msal/application.py index 0869d9e5..5cf7e32a 100644 --- a/msal/application.py +++ b/msal/application.py @@ -194,6 +194,21 @@ def obtain_token_by_username_password(self, username, password, **kwargs): username, password, headers=headers, **kwargs) +def _msal_extension_check(): + # Can't run this in module or class level otherwise you'll get circular import error + try: + from msal_extensions import __version__ as v + major, minor, _ = v.split(".", maxsplit=3) + if not (int(major) >= 1 and int(minor) >= 2): + warnings.warn( + "Please upgrade msal-extensions. " + "Only msal-extensions 1.2+ can work with msal 1.30+") + except ImportError: + pass # The optional msal_extensions is not installed. Business as usual. + except ValueError: + logger.exception(f"msal_extensions version {v} not in major.minor.patch format") + + class ClientApplication(object): """You do not usually directly use this class. Use its subclasses instead: :class:`PublicClientApplication` and :class:`ConfidentialClientApplication`. @@ -635,6 +650,8 @@ def __init__( self.authority_groups = None self._telemetry_buffer = {} self._telemetry_lock = Lock() + _msal_extension_check() + def _decide_broker(self, allow_broker, enable_pii_log): is_confidential_app = self.client_credential or isinstance( From ee5b310b001cdfb63a44375b88fa3987359df9f1 Mon Sep 17 00:00:00 2001 From: Dharshan Birur Jayaprabhu Date: Fri, 4 Oct 2024 10:09:39 -0700 Subject: [PATCH 197/262] update .gitignore file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 36b43713..58868119 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,8 @@ docs/_build/ # The test configuration file(s) could potentially contain credentials tests/config.json +# Token Cache files +msal_cache.bin .env .perf.baseline From 5ad506dfff8788fba6ca4c7e6a4bce78d022b1d9 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 2 Sep 2024 12:34:56 -0700 Subject: [PATCH 198/262] Provide accurate msal[broker] version hint on Mac --- msal/broker.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/msal/broker.py b/msal/broker.py index e16e6102..775475a7 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -23,7 +23,15 @@ except (ImportError, AttributeError): # AttributeError happens when a prior pymsalruntime uninstallation somehow leaved an empty folder behind # PyMsalRuntime currently supports these Windows versions, listed in this MSFT internal link # https://github.com/AzureAD/microsoft-authentication-library-for-cpp/pull/2406/files - raise ImportError('You need to install dependency by: pip install "msal[broker]>=1.20,<2"') + min_ver = { + "win32": "1.20", + "darwin": "1.31", + }.get(sys.platform) + if min_ver: + raise ImportError( + f'You must install dependency by: pip install "msal[broker]>={min_ver},<2"') + else: # Unsupported platform + raise ImportError("Dependency pymsalruntime unavailable on current platform") # It could throw RuntimeError when running on ancient versions of Windows From 95a63a7fe97d91b99979e5bf78e03f6acf40a286 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 27 Sep 2024 12:34:56 -0700 Subject: [PATCH 199/262] Testcase for app without redirect_uri using broker --- tests/test_application.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_application.py b/tests/test_application.py index 0736164c..e565e105 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -751,7 +751,8 @@ def test_broker_should_be_disabled_by_default(self): @patch("msal.application._init_broker", new=Mock(side_effect=ImportError( "PyMsalRuntime not installed" ))) - def test_should_error_out_when_opted_in_yet_pymsalruntime_not_installed(self): + def test_opt_in_should_error_out_when_pymsalruntime_not_installed(self): + """Because it is actionable to app developer to add dependency declaration""" with self.assertRaises(ImportError): app = msal.PublicClientApplication( "client_id", @@ -830,3 +831,28 @@ def test_should_fallback_to_non_broker_when_using_oidc_authority(self): ) self.assertFalse(app._enable_broker) + def test_app_did_not_register_redirect_uri_should_error_out(self): + """Because it is actionable to app developer to add redirect URI""" + app = msal.PublicClientApplication( + "client_id", + authority="https://login.microsoftonline.com/common", + enable_broker_on_mac=True, + ) + self.assertTrue(app._enable_broker) + with patch.object( + # Note: We tried @patch("msal.broker.foo", ...) but it ended up with + # "module msal does not have attribute broker" + app, "_acquire_token_interactive_via_broker", return_value={ + "error": "broker_error", + "error_description": + "(pii). " # pymsalruntime no longer surfaces AADSTS error, + # So MSAL Python can't raise RedirectUriError. + "Status: Response_Status.Status_ApiContractViolation, " + "Error code: 3399614473, Tag 557973642", + }): + result = app.acquire_token_interactive( + ["scope"], + parent_window_handle=app.CONSOLE_WINDOW_HANDLE, + ) + self.assertEqual(result.get("error"), "broker_error") + From 33dbe3eabe98b79671560c88f12a717c201bde47 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 4 Oct 2024 12:34:56 -0700 Subject: [PATCH 200/262] Default to env var MSAL_FORCE_REGION Add an alias DISABLE_MSAL_FORCE_REFRESH for False --- msal/application.py | 24 +++++++++++++++++------- tests/test_e2e.py | 14 +++++++++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/msal/application.py b/msal/application.py index 5cf7e32a..260d80e0 100644 --- a/msal/application.py +++ b/msal/application.py @@ -5,6 +5,7 @@ import sys import warnings from threading import Lock +from typing import Optional # Needed in Python 3.7 & 3.8 import os from .oauth2cli import Client, JwtAssertionCreator @@ -225,6 +226,7 @@ class ClientApplication(object): REMOVE_ACCOUNT_ID = "903" ATTEMPT_REGION_DISCOVERY = True # "TryAutoDetect" + DISABLE_MSAL_FORCE_REGION = False # Used in azure_region to disable MSAL_FORCE_REGION behavior _TOKEN_SOURCE = "token_source" _TOKEN_SOURCE_IDP = "identity_provider" _TOKEN_SOURCE_CACHE = "cache" @@ -448,11 +450,14 @@ def __init__( Instructs MSAL to use the Entra regional token service. This legacy feature is only available to first-party applications. Only ``acquire_token_for_client()`` is supported. - Supports 3 values: + Supports 4 values: - ``azure_region=None`` - meaning no region is used. This is the default value. - ``azure_region="some_region"`` - meaning the specified region is used. - ``azure_region=True`` - meaning MSAL will try to auto-detect the region. This is not recommended. + 1. ``azure_region=None`` - This default value means no region is configured. + MSAL will use the region defined in env var ``MSAL_FORCE_REGION``. + 2. ``azure_region="some_region"`` - meaning the specified region is used. + 3. ``azure_region=True`` - meaning + MSAL will try to auto-detect the region. This is not recommended. + 4. ``azure_region=False`` - meaning MSAL will use no region. .. note:: Region auto-discovery has been tested on VMs and on Azure Functions. It is unreliable. @@ -630,7 +635,10 @@ def __init__( except ValueError: # Those are explicit authority validation errors raise except Exception: # The rest are typically connection errors - if validate_authority and azure_region and not oidc_authority: + if validate_authority and not oidc_authority and ( + azure_region # Opted in to use region + or (azure_region is None and os.getenv("MSAL_FORCE_REGION")) # Will use region + ): # Since caller opts in to use region, here we tolerate connection # errors happened during authority validation at non-region endpoint self.authority = Authority( @@ -724,9 +732,11 @@ def _build_telemetry_context( self._telemetry_buffer, self._telemetry_lock, api_id, correlation_id=correlation_id, refresh_reason=refresh_reason) - def _get_regional_authority(self, central_authority): - if not self._region_configured: # User did not opt-in to ESTS-R + def _get_regional_authority(self, central_authority) -> Optional[Authority]: + if self._region_configured is False: # User opts out of ESTS-R return None # Short circuit to completely bypass region detection + if self._region_configured is None: # User did not make an ESTS-R choice + self._region_configured = os.getenv("MSAL_FORCE_REGION") or None self._region_detected = self._region_detected or _detect_region( self.http_client if self._region_configured is not None else None) if (self._region_configured != self.ATTEMPT_REGION_DISCOVERY diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ff35a73e..a0796547 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1130,11 +1130,23 @@ def _test_acquire_token_for_client(self, configured_region, expected_region): def test_acquire_token_for_client_should_hit_global_endpoint_by_default(self): self._test_acquire_token_for_client(None, None) - def test_acquire_token_for_client_should_ignore_env_var_by_default(self): + def test_acquire_token_for_client_should_ignore_env_var_region_name_by_default(self): os.environ["REGION_NAME"] = "eastus" self._test_acquire_token_for_client(None, None) del os.environ["REGION_NAME"] + @patch.dict(os.environ, {"MSAL_FORCE_REGION": "eastus"}) + def test_acquire_token_for_client_should_use_env_var_msal_force_region_by_default(self): + self._test_acquire_token_for_client(None, "eastus") + + @patch.dict(os.environ, {"MSAL_FORCE_REGION": "eastus"}) + def test_acquire_token_for_client_should_prefer_the_explicit_region(self): + self._test_acquire_token_for_client("westus", "westus") + + @patch.dict(os.environ, {"MSAL_FORCE_REGION": "eastus"}) + def test_acquire_token_for_client_should_allow_opt_out_env_var_msal_force_region(self): + self._test_acquire_token_for_client(False, None) + def test_acquire_token_for_client_should_use_a_specified_region(self): self._test_acquire_token_for_client("westus", "westus") From 7db6c2ccdd121ad730517ebd5e6195feec06ed41 Mon Sep 17 00:00:00 2001 From: Alexander Clouter Date: Thu, 17 Oct 2024 21:46:19 +0100 Subject: [PATCH 201/262] allow MI endpoint changing through environment variable (#754) useful during development where you are using SSH tunnelling to utilise the credentials assigned to an actual instance --- msal/managed_identity.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 608bc1bf..14e24dca 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -134,6 +134,23 @@ class ManagedIdentityClient(object): It also provides token cache support. + .. admonition:: Special case when your local development wants to use a managed identity on Azure VM. + + By setting the environment variable ``MSAL_MANAGED_IDENTITY_ENDPOINT`` + you override the default identity URL used in MSAL's Azure VM managed identity + code path. + + This is useful during local development where it may be desirable to + utilise the credentials assigned to an actual VM instance via SSH tunnelling. + + For example, if you create your SSH tunnel this way (assuming your VM is on ``192.0.2.1``):: + + ssh -L 8000:169.254.169.254:80 192.0.2.1 + + Then your code could run locally using:: + + env MSAL_MANAGED_IDENTITY_ENDPOINT=http://localhost:8000/metadata/identity/oauth2/token python your_script.py + .. note:: Cloud Shell support is NOT implemented in this class. @@ -446,7 +463,7 @@ def _obtain_token_on_azure_vm(http_client, managed_identity, resource): } _adjust_param(params, managed_identity) resp = http_client.get( - "http://169.254.169.254/metadata/identity/oauth2/token", + os.getenv('MSAL_MANAGED_IDENTITY_ENDPOINT', 'http://169.254.169.254/metadata/identity/oauth2/token'), params=params, headers={"Metadata": "true"}, ) From e9b3913703c1b9725ab1e68a30e8acda0a6dfffc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 29 Oct 2024 12:34:56 -0700 Subject: [PATCH 202/262] Fix incorrect arc detection and add test cases --- msal/managed_identity.py | 8 +++++--- tests/test_mi.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 5636f564..755c9bd8 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -319,10 +319,12 @@ def _scope_to_resource(scope): # This is an experimental reasonable-effort appr def _get_arc_endpoint(): if "IDENTITY_ENDPOINT" in os.environ and "IMDS_ENDPOINT" in os.environ: return os.environ["IDENTITY_ENDPOINT"] - if ( # Defined in https://msazure.visualstudio.com/One/_wiki/wikis/One.wiki/233012/VM-Extension-Authoring-for-Arc?anchor=determining-which-endpoint-to-use - sys.platform == "linux" and os.path.exists("/var/opt/azcmagent/bin/himds") + if ( # Defined in https://eng.ms/docs/cloud-ai-platform/azure-core/azure-management-and-platforms/control-plane-bburns/hybrid-resource-provider/azure-arc-for-servers/specs/extension_authoring + sys.platform == "linux" and os.path.exists("/opt/azcmagent/bin/himds") or sys.platform == "win32" and os.path.exists(os.path.expandvars( - r"%ProgramFiles%\AzureConnectedMachineAgent\himds.exe")) + # Avoid Windows-only "%EnvVar%" syntax so that tests can be run on Linux + r"${ProgramFiles}\AzureConnectedMachineAgent\himds.exe" + )) ): return "http://localhost:40342/metadata/identity/oauth2/token" diff --git a/tests/test_mi.py b/tests/test_mi.py index f3182c7b..d3a83a0c 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -303,9 +303,23 @@ def test_machine_learning(self): "IDENTITY_ENDPOINT": "http://localhost", "IMDS_ENDPOINT": "http://localhost", }) - def test_arc(self): + def test_arc_by_env_var(self): self.assertEqual(get_managed_identity_source(), AZURE_ARC) + @patch("msal.managed_identity.os.path.exists", return_value=True) + @patch("msal.managed_identity.sys.platform", new="linux") + def test_arc_by_file_existence_on_linux(self, mocked_exists): + self.assertEqual(get_managed_identity_source(), AZURE_ARC) + mocked_exists.assert_called_with("/opt/azcmagent/bin/himds") + + @patch("msal.managed_identity.os.path.exists", return_value=True) + @patch("msal.managed_identity.sys.platform", new="win32") + @patch.dict(os.environ, {"ProgramFiles": "C:\Program Files"}) + def test_arc_by_file_existence_on_windows(self, mocked_exists): + self.assertEqual(get_managed_identity_source(), AZURE_ARC) + mocked_exists.assert_called_with( + r"C:\Program Files\AzureConnectedMachineAgent\himds.exe") + @patch.dict(os.environ, { "AZUREPS_HOST_ENVIRONMENT": "cloud-shell-foo", }) From 69a96fe17b49d4ba698108d70f93fecbdff2117e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 1 Nov 2024 12:34:56 -0700 Subject: [PATCH 203/262] Release MSAL Python 1.31.1 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 0869d9e5..bf55e5e9 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,7 +21,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.31.0" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.31.1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From 7826ea8ffc998ae66b0f340ebd53e52ddd20d1f0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 19 Nov 2024 15:02:15 -0800 Subject: [PATCH 204/262] Revert "allow MI endpoint changing through environment variable (#754)" (#769) This reverts commit 7db6c2ccdd121ad730517ebd5e6195feec06ed41. --- msal/managed_identity.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 14e24dca..608bc1bf 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -134,23 +134,6 @@ class ManagedIdentityClient(object): It also provides token cache support. - .. admonition:: Special case when your local development wants to use a managed identity on Azure VM. - - By setting the environment variable ``MSAL_MANAGED_IDENTITY_ENDPOINT`` - you override the default identity URL used in MSAL's Azure VM managed identity - code path. - - This is useful during local development where it may be desirable to - utilise the credentials assigned to an actual VM instance via SSH tunnelling. - - For example, if you create your SSH tunnel this way (assuming your VM is on ``192.0.2.1``):: - - ssh -L 8000:169.254.169.254:80 192.0.2.1 - - Then your code could run locally using:: - - env MSAL_MANAGED_IDENTITY_ENDPOINT=http://localhost:8000/metadata/identity/oauth2/token python your_script.py - .. note:: Cloud Shell support is NOT implemented in this class. @@ -463,7 +446,7 @@ def _obtain_token_on_azure_vm(http_client, managed_identity, resource): } _adjust_param(params, managed_identity) resp = http_client.get( - os.getenv('MSAL_MANAGED_IDENTITY_ENDPOINT', 'http://169.254.169.254/metadata/identity/oauth2/token'), + "http://169.254.169.254/metadata/identity/oauth2/token", params=params, headers={"Metadata": "true"}, ) From 1d85e8a20b1fd1b28ae9bd8b77bb93fdade6f8b7 Mon Sep 17 00:00:00 2001 From: jiasli <4003950+jiasli@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:45:53 +0800 Subject: [PATCH 205/262] SystemAssigned --- msal/managed_identity.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index bad96a08..ec032ca7 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -154,7 +154,7 @@ def __init__( self, managed_identity: Union[ dict, - ManagedIdentity, # Could use Type[ManagedIdentity] but it is deprecatred in Python 3.9+ + ManagedIdentity, # Could use Type[ManagedIdentity] but it is deprecated in Python 3.9+ SystemAssignedManagedIdentity, UserAssignedManagedIdentity, ], @@ -206,7 +206,7 @@ def __init__( you may use an environment variable (such as MY_MANAGED_IDENTITY_CONFIG) to store a json blob like ``{"ManagedIdentityIdType": "ClientId", "Id": "foo"}`` or - ``{"ManagedIdentityIdType": "SystemAssignedManagedIdentity", "Id": null})``. + ``{"ManagedIdentityIdType": "SystemAssigned", "Id": null}``. The following app can load managed identity configuration dynamically:: import json, os, msal, requests @@ -648,4 +648,3 @@ def _obtain_token_on_arc(http_client, endpoint, resource): "error": "invalid_request", "error_description": response.text, } - From 60007ac58d94947287ba15be36e83ea1978b4cc1 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 13 Nov 2024 12:34:56 -0800 Subject: [PATCH 206/262] Some helpers to test Python 3.14 via docker --- Dockerfile | 11 +++++++++++ docker_run.sh | 15 +++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 Dockerfile create mode 100755 docker_run.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..dd28e913 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +# This file is only needed for testing Python 3.14 whose image contains no cffi. +# (As a comparison, the official python:3.13-slim works out of the box.) + +# TODO: Can this Dockerfile use multi-stage build? +# https://testdriven.io/tips/6da2d9c9-8849-4386-b7f9-13b28514ded8/ +FROM python:3.14.0a2-slim + +RUN apt-get update && apt-get install -y \ + gcc \ + libffi-dev + diff --git a/docker_run.sh b/docker_run.sh new file mode 100755 index 00000000..f7cbfb98 --- /dev/null +++ b/docker_run.sh @@ -0,0 +1,15 @@ +#!/usr/bin/bash +IMAGE_NAME=msal:latest + +docker build -t $IMAGE_NAME - < Dockerfile + +echo "=== Integration Test for Python 3.14 which has no AppImage yet ===" +echo "After seeing the bash prompt, run the following to test:" +echo " pip install -e ." +echo " pytest --capture=no -s tests/chosen_test_file.py" +docker run --rm -it \ + --privileged \ + -w /home -v $PWD:/home \ + $IMAGE_NAME \ + $@ + From a5f8b67b7496effdae0da79cd62e1b6d7fb36997 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 29 Nov 2024 12:34:56 -0800 Subject: [PATCH 207/262] Remove Dockerfile containing 3rd party image --- Dockerfile | 11 ----------- docker_run.sh | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 15 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index dd28e913..00000000 --- a/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# This file is only needed for testing Python 3.14 whose image contains no cffi. -# (As a comparison, the official python:3.13-slim works out of the box.) - -# TODO: Can this Dockerfile use multi-stage build? -# https://testdriven.io/tips/6da2d9c9-8849-4386-b7f9-13b28514ded8/ -FROM python:3.14.0a2-slim - -RUN apt-get update && apt-get install -y \ - gcc \ - libffi-dev - diff --git a/docker_run.sh b/docker_run.sh index f7cbfb98..12747db4 100755 --- a/docker_run.sh +++ b/docker_run.sh @@ -1,15 +1,24 @@ #!/usr/bin/bash -IMAGE_NAME=msal:latest -docker build -t $IMAGE_NAME - < Dockerfile +# Error out if there is less than 1 argument +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [command]" + echo "Example: $0 python:3.14.0a2-slim bash" + exit 1 +fi -echo "=== Integration Test for Python 3.14 which has no AppImage yet ===" +# We will get a standard Python image from the input, +# so that we don't need to hard code one in a Dockerfile +IMAGE_NAME=$1 + +echo "=== Starting $IMAGE_NAME (especially those which have no AppImage yet) ===" echo "After seeing the bash prompt, run the following to test:" +echo " apt update && apt install -y gcc libffi-dev # Needed in Python 3.14.0a2-slim" echo " pip install -e ." echo " pytest --capture=no -s tests/chosen_test_file.py" docker run --rm -it \ --privileged \ -w /home -v $PWD:/home \ $IMAGE_NAME \ - $@ + $2 From f5196d84f2aaf6d9927259d78815d5c1c21f9cc8 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 28 Nov 2024 11:58:21 -0800 Subject: [PATCH 208/262] Relax the upper bound after testing the latest one --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 33ec3f06..6dfcfc7b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=2.5,<46 + cryptography>=2.5,<47 [options.extras_require] From 7b3175631c03c5b5476aaf29239c31878ab80b23 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 8 Oct 2024 12:34:56 -0700 Subject: [PATCH 209/262] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..48147adb --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,37 @@ +# Derived from the default YAML generated by Azure DevOps for a Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- dev +- azure-pipelines + +pool: + vmImage: ubuntu-latest +strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + Python311: + python.version: '3.11' + Python312: + python.version: '3.12' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From 37badb876e1dc7c23022445f241bc8bf954c49f2 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 19 Dec 2024 12:34:56 -0800 Subject: [PATCH 210/262] Py3.7 is unavailable on ubuntu 24.04 --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index db891158..e40950b2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -25,10 +25,10 @@ jobs: LAB_OBO_PUBLIC_CLIENT_ID: ${{ secrets.LAB_OBO_PUBLIC_CLIENT_ID }} # Derived from https://docs.github.com/en/actions/guides/building-and-testing-python#starting-with-the-python-workflow-template - runs-on: ubuntu-latest # It switched to 22.04 shortly after 2022-Nov-8 + runs-on: ubuntu-22.04 # ubuntu-latest switched to 22.04 shortly after 2022-Nov-8 strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12", 3.13] steps: - uses: actions/checkout@v4 From 3803ae22c8c712fd6f74db8576f219e31c94039d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 10 Jan 2025 12:34:56 -0800 Subject: [PATCH 211/262] Suppress a false positive CodeQL alarm --- tests/test_e2e.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index a0796547..d302545c 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1333,6 +1333,9 @@ def test_at_pop_calling_pattern(self): nonce=self._extract_pop_nonce(resp.headers.get("WWW-Authenticate")), ), )) + # The api_endpoint is for test only and has no proper SSL certificate, + # so we suppress the CodeQL warning for disabling SSL certificate checks + # @suppress py/bandit/requests-ssl-verify-disabled resp = requests.get(api_endpoint, verify=False, headers={ "Authorization": "pop {}".format(result["access_token"]), }) From 3f3d1336751a19cb8a7c0575e68ffbea66581008 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 17 Dec 2024 12:34:56 -0800 Subject: [PATCH 212/262] Update the hint of where to get a lab certificate --- tests/test_e2e.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d302545c..048e1840 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -476,9 +476,10 @@ def get_lab_app( if os.getenv(env_client_id) and os.getenv(env_client_cert_path): # id came from https://docs.msidlab.com/accounts/confidentialclient.html client_id = os.getenv(env_client_id) - # Cert came from https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/asset/Microsoft_Azure_KeyVault/Certificate/https://msidlabs.vault.azure.net/certificates/LabVaultAccessCert client_credential = { - "private_key_pfx_path": os.getenv(env_client_cert_path), + "private_key_pfx_path": + # Cert came from https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/asset/Microsoft_Azure_KeyVault/Certificate/https://msidlabs.vault.azure.net/certificates/LabAuth + os.getenv(env_client_cert_path), "public_certificate": True, # Opt in for SNI } elif os.getenv(env_client_id) and os.getenv(env_name2): From 5420c59aaa9b1fd657ce0c7b5f3168ce732c6045 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 28 Nov 2024 11:58:21 -0800 Subject: [PATCH 213/262] Relax the upper bound after testing the latest one --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 33ec3f06..6dfcfc7b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=2.5,<46 + cryptography>=2.5,<47 [options.extras_require] From f27f4b227e6c7ed336a8ce6fcd606c581f806cd2 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 20 Jan 2025 12:34:56 -0800 Subject: [PATCH 214/262] Supports GUID/scope and https://contoso.com//scope --- msal/cloudshell.py | 6 +++++- tests/test_cloudshell.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tests/test_cloudshell.py diff --git a/msal/cloudshell.py b/msal/cloudshell.py index f4feaf44..1a25dea4 100644 --- a/msal/cloudshell.py +++ b/msal/cloudshell.py @@ -32,8 +32,12 @@ def _scope_to_resource(scope): # This is an experimental reasonable-effort appr if scope.startswith(a): return a u = urlparse(scope) + if not u.scheme and not u.netloc: # Typically the "GUID/scope" case + return u.path.split("/")[0] if u.scheme: - return "{}://{}".format(u.scheme, u.netloc) + trailer = ( # https://learn.microsoft.com/en-us/entra/identity-platform/scopes-oidc#trailing-slash-and-default + "/" if u.path.startswith("//") else "") + return "{}://{}{}".format(u.scheme, u.netloc, trailer) return scope # There is no much else we can do here diff --git a/tests/test_cloudshell.py b/tests/test_cloudshell.py new file mode 100644 index 00000000..1b0277b2 --- /dev/null +++ b/tests/test_cloudshell.py @@ -0,0 +1,21 @@ +import unittest +from msal.cloudshell import _scope_to_resource + +class TestScopeToResource(unittest.TestCase): + + def test_expected_behaviors(self): + for scope, expected_resource in { + "https://analysis.windows.net/powerbi/api/foo": + "https://analysis.windows.net/powerbi/api", # A special case + "https://pas.windows.net/CheckMyAccess/Linux/.default": + "https://pas.windows.net/CheckMyAccess/Linux/.default", # Special case + "https://double-slash.com//scope": "https://double-slash.com/", + "https://single-slash.com/scope": "https://single-slash.com", + "guid/some/scope": "guid", + "797f4846-ba00-4fd7-ba43-dac1f8f63013/.default": # Realistic GUID + "797f4846-ba00-4fd7-ba43-dac1f8f63013" + }.items(): + self.assertEqual(_scope_to_resource(scope), expected_resource) + +if __name__ == '__main__': + unittest.main() From 388b86d31559cda8be9b797fefe61dc11dc7c757 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 6 Jan 2025 12:34:56 -0800 Subject: [PATCH 215/262] Pin ubuntu 22.04 so that we can still test on Py37 --- .github/workflows/python-package.yml | 4 ++-- tests/test_cloudshell.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index db891158..8ed0073c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -25,10 +25,10 @@ jobs: LAB_OBO_PUBLIC_CLIENT_ID: ${{ secrets.LAB_OBO_PUBLIC_CLIENT_ID }} # Derived from https://docs.github.com/en/actions/guides/building-and-testing-python#starting-with-the-python-workflow-template - runs-on: ubuntu-latest # It switched to 22.04 shortly after 2022-Nov-8 + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 diff --git a/tests/test_cloudshell.py b/tests/test_cloudshell.py index 1b0277b2..9a0e5709 100644 --- a/tests/test_cloudshell.py +++ b/tests/test_cloudshell.py @@ -12,8 +12,10 @@ def test_expected_behaviors(self): "https://double-slash.com//scope": "https://double-slash.com/", "https://single-slash.com/scope": "https://single-slash.com", "guid/some/scope": "guid", - "797f4846-ba00-4fd7-ba43-dac1f8f63013/.default": # Realistic GUID - "797f4846-ba00-4fd7-ba43-dac1f8f63013" + "6dae42f8-4368-4678-94ff-3960e28e3630/.default": + # The real guid of AKS resource + # https://learn.microsoft.com/en-us/azure/aks/kubelogin-authentication#how-to-use-kubelogin-with-aks + "6dae42f8-4368-4678-94ff-3960e28e3630", }.items(): self.assertEqual(_scope_to_resource(scope), expected_resource) From bf27fc6d28320dc1384b98ca7bd9844a3a6f809f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 28 Jan 2025 12:34:56 -0800 Subject: [PATCH 216/262] MSAL Python 1.31.2b1 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index bf55e5e9..0b66f03e 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,7 +21,7 @@ # The __init__.py will import this. Not the other way around. -__version__ = "1.31.1" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.31.2b1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" From a37d3a329f7329fe0761579d7a52240ff1cd2fa3 Mon Sep 17 00:00:00 2001 From: Ugonna Akali <137432604+Ugonnaak1@users.noreply.github.com> Date: Thu, 6 Feb 2025 17:08:32 -0800 Subject: [PATCH 217/262] Pass Sku and Ver to MsalRuntime (#786) * Pass Sku and Ver to MsalRuntime * edit suggestions * suggestion edits * whitespace --- msal/__init__.py | 2 +- msal/application.py | 5 ++--- msal/broker.py | 12 +++++++++--- msal/sku.py | 6 ++++++ 4 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 msal/sku.py diff --git a/msal/__init__.py b/msal/__init__.py index 380d584e..295e9756 100644 --- a/msal/__init__.py +++ b/msal/__init__.py @@ -26,12 +26,12 @@ #------------------------------------------------------------------------------ from .application import ( - __version__, ClientApplication, ConfidentialClientApplication, PublicClientApplication, ) from .oauth2cli.oidc import Prompt, IdTokenError +from .sku import __version__ from .token_cache import TokenCache, SerializableTokenCache from .auth_scheme import PopAuthScheme from .managed_identity import ( diff --git a/msal/application.py b/msal/application.py index e46d7484..c3d23f25 100644 --- a/msal/application.py +++ b/msal/application.py @@ -19,10 +19,9 @@ from .region import _detect_region from .throttled_http_client import ThrottledHttpClient from .cloudshell import _is_running_in_cloud_shell +from .sku import SKU, __version__ -# The __init__.py will import this. Not the other way around. -__version__ = "1.31.1" # When releasing, also check and bump our dependencies's versions if needed logger = logging.getLogger(__name__) _AUTHORITY_TYPE_CLOUDSHELL = "CLOUDSHELL" @@ -770,7 +769,7 @@ def _build_client(self, client_credential, authority, skip_regional_client=False client_assertion = None client_assertion_type = None default_headers = { - "x-client-sku": "MSAL.Python", "x-client-ver": __version__, + "x-client-sku": SKU, "x-client-ver": __version__, "x-client-os": sys.platform, "x-ms-lib-capability": "retry-after, h429", } diff --git a/msal/broker.py b/msal/broker.py index 775475a7..f4d71e11 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -7,6 +7,7 @@ import time import uuid +from .sku import __version__, SKU logger = logging.getLogger(__name__) try: @@ -135,13 +136,18 @@ def _get_new_correlation_id(): def _enable_msa_pt(params): params.set_additional_parameter("msal_request_type", "consumer_passthrough") # PyMsalRuntime 0.8+ +def _build_msal_runtime_auth_params(client_id, authority): + params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params.set_additional_parameter("msal_client_sku", SKU) + params.set_additional_parameter("msal_client_ver", __version__) + return params def _signin_silently( authority, client_id, scopes, correlation_id=None, claims=None, enable_msa_pt=False, auth_scheme=None, **kwargs): - params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params = _build_msal_runtime_auth_params(client_id, authority) params.set_requested_scopes(scopes) if claims: params.set_decoded_claims(claims) @@ -174,7 +180,7 @@ def _signin_interactively( enable_msa_pt=False, auth_scheme=None, **kwargs): - params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params = _build_msal_runtime_auth_params(client_id, authority) params.set_requested_scopes(scopes) params.set_redirect_uri( _redirect_uri_on_mac if sys.platform == "darwin" else @@ -230,7 +236,7 @@ def _acquire_token_silently( account = _read_account_by_id(account_id, correlation_id) if account is None: return - params = pymsalruntime.MSALRuntimeAuthParameters(client_id, authority) + params = _build_msal_runtime_auth_params(client_id, authority) params.set_requested_scopes(scopes) if claims: params.set_decoded_claims(claims) diff --git a/msal/sku.py b/msal/sku.py new file mode 100644 index 00000000..e383e151 --- /dev/null +++ b/msal/sku.py @@ -0,0 +1,6 @@ +"""This module is from where we recieve the client sku name and version. +""" + +# The __init__.py will import this. Not the other way around. +__version__ = "1.31.1" # When releasing, also check and bump our dependencies's versions if needed +SKU = "MSAL.Python" From 6508d992f4bbfcae21c7f1175c24e4076ade0ad5 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 16 Jan 2025 12:34:56 -0800 Subject: [PATCH 218/262] Try to suppress another verify=False --- tests/test_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 048e1840..ad29e808 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1323,7 +1323,7 @@ def test_at_pop_calling_pattern(self): # We skip it here because this test case has not yet initialize self.app # assert self.app.is_pop_supported() api_endpoint = "https://20.190.132.47/beta/me" - resp = requests.get(api_endpoint, verify=False) + resp = requests.get(api_endpoint, verify=False) # @suppress py/bandit/requests-ssl-verify-disabled self.assertEqual(resp.status_code, 401, "Initial call should end with an http 401 error") result = self._get_shr_pop(**dict( self.get_lab_user(usertype="cloud"), # This is generally not the current laptop's default AAD account From 6c84adf6723491edd4905a477e6e8da4df266313 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 5 Dec 2024 12:34:56 -0800 Subject: [PATCH 219/262] Adds dSTS authority (as if it were an oidc_authority) --- msal/application.py | 4 ++++ tests/test_authority.py | 51 +++++++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/msal/application.py b/msal/application.py index c3d23f25..7f7615de 100644 --- a/msal/application.py +++ b/msal/application.py @@ -6,6 +6,7 @@ import warnings from threading import Lock from typing import Optional # Needed in Python 3.7 & 3.8 +from urllib.parse import urlparse import os from .oauth2cli import Client, JwtAssertionCreator @@ -622,6 +623,9 @@ def __init__( # Here the self.authority will not be the same type as authority in input if oidc_authority and authority: raise ValueError("You can not provide both authority and oidc_authority") + if isinstance(authority, str) and urlparse(authority).path.startswith( + "/dstsv2"): # dSTS authority's path always starts with "/dstsv2" + oidc_authority = authority # So we treat it as if an oidc_authority try: authority_to_use = authority or "https://{}/common/".format(WORLD_WIDE) self.authority = Authority( diff --git a/tests/test_authority.py b/tests/test_authority.py index 0d6c790f..3fd1fce1 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -104,32 +104,63 @@ def test_authority_with_path_should_be_used_as_is(self, oidc_discovery): "authorization_endpoint": "https://contoso.com/authorize", "token_endpoint": "https://contoso.com/token", }) -class TestOidcAuthority(unittest.TestCase): +class OidcAuthorityTestCase(unittest.TestCase): + authority = "https://contoso.com/tenant" + + def setUp(self): + # setUp() gives subclass a dynamic setup based on their authority + self.oidc_discovery_endpoint = ( + # MSAL Python always does OIDC Discovery, + # not to be confused with Instance Discovery + # Here the test is to confirm the OIDC endpoint contains no "/v2.0" + self.authority + "/.well-known/openid-configuration") + def test_authority_obj_should_do_oidc_discovery_and_skip_instance_discovery( self, oidc_discovery, instance_discovery): c = MinimalHttpClient() - a = Authority(None, c, oidc_authority_url="https://contoso.com/tenant") + a = Authority(None, c, oidc_authority_url=self.authority) instance_discovery.assert_not_called() - oidc_discovery.assert_called_once_with( - "https://contoso.com/tenant/.well-known/openid-configuration", c) + oidc_discovery.assert_called_once_with(self.oidc_discovery_endpoint, c) self.assertEqual(a.authorization_endpoint, 'https://contoso.com/authorize') self.assertEqual(a.token_endpoint, 'https://contoso.com/token') def test_application_obj_should_do_oidc_discovery_and_skip_instance_discovery( self, oidc_discovery, instance_discovery): app = msal.ClientApplication( - "id", - authority=None, - oidc_authority="https://contoso.com/tenant", - ) + "id", authority=None, oidc_authority=self.authority) instance_discovery.assert_not_called() oidc_discovery.assert_called_once_with( - "https://contoso.com/tenant/.well-known/openid-configuration", - app.http_client) + self.oidc_discovery_endpoint, app.http_client) self.assertEqual( app.authority.authorization_endpoint, 'https://contoso.com/authorize') self.assertEqual(app.authority.token_endpoint, 'https://contoso.com/token') + +class DstsAuthorityTestCase(OidcAuthorityTestCase): + # Inherits OidcAuthority's test cases and run them with a dSTS authority + authority = ( # dSTS is single tenanted with a tenant placeholder + 'https://test-instance1-dsts.dsts.core.azure-test.net/dstsv2/common') + authorization_endpoint = ( + "https://some.url.dsts.core.azure-test.net/dstsv2/common/oauth2/authorize") + token_endpoint = ( + "https://some.url.dsts.core.azure-test.net/dstsv2/common/oauth2/token") + + @patch("msal.authority._instance_discovery") + @patch("msal.authority.tenant_discovery", return_value={ + "authorization_endpoint": authorization_endpoint, + "token_endpoint": token_endpoint, + }) # We need to create new patches (i.e. mocks) for non-inherited test cases + def test_application_obj_should_accept_dsts_url_as_an_authority( + self, oidc_discovery, instance_discovery): + app = msal.ClientApplication("id", authority=self.authority) + instance_discovery.assert_not_called() + oidc_discovery.assert_called_once_with( + self.oidc_discovery_endpoint, app.http_client) + self.assertEqual( + app.authority.authorization_endpoint, self.authorization_endpoint) + self.assertEqual(app.authority.token_endpoint, self.token_endpoint) + + class TestAuthorityInternalHelperCanonicalize(unittest.TestCase): def test_canonicalize_tenant_followed_by_extra_paths(self): From 971f110cddaf1a98f4021340282fa5fa1d4d699a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 7 Apr 2022 19:50:41 -0700 Subject: [PATCH 220/262] Add test case to show that OBO supports SP --- tests/test_e2e.py | 89 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ad29e808..b2701569 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -649,19 +649,27 @@ def _test_acquire_token_obo(self, config_pca, config_cca, # Here we just test regional apps won't adversely break OBO http_client=None, ): - # 1. An app obtains a token representing a user, for our mid-tier service - pca = msal.PublicClientApplication( - config_pca["client_id"], authority=config_pca["authority"], - azure_region=azure_region, - http_client=http_client or MinimalHttpClient()) - pca_result = pca.acquire_token_by_username_password( - config_pca["username"], - config_pca["password"], - scopes=config_pca["scope"], - ) - self.assertIsNotNone( - pca_result.get("access_token"), - "PCA failed to get AT because %s" % json.dumps(pca_result, indent=2)) + if "client_secret" not in config_pca: + # 1.a An app obtains a token representing a user, for our mid-tier service + result = msal.PublicClientApplication( + config_pca["client_id"], authority=config_pca["authority"], + azure_region=azure_region, + http_client=http_client or MinimalHttpClient(), + ).acquire_token_by_username_password( + config_pca["username"], config_pca["password"], + scopes=config_pca["scope"], + ) + else: # We repurpose the config_pca to contain client_secret for cca app 1 + # 1.b An app obtains a token representing itself, for our mid-tier service + result = msal.ConfidentialClientApplication( + config_pca["client_id"], authority=config_pca["authority"], + client_credential=config_pca["client_secret"], + azure_region=azure_region, + http_client=http_client or MinimalHttpClient(), + ).acquire_token_for_client(scopes=config_pca["scope"]) + assertion = result.get("access_token") + self.assertIsNotNone(assertion, "First app failed to get AT. {}".format( + json.dumps(result, indent=2))) # 2. Our mid-tier service uses OBO to obtain a token for downstream service cca = msal.ConfidentialClientApplication( @@ -674,9 +682,9 @@ def _test_acquire_token_obo(self, config_pca, config_cca, # That's fine if OBO app uses short-lived msal instance per session. # Otherwise, the OBO app need to implement a one-cache-per-user setup. ) - cca_result = cca.acquire_token_on_behalf_of( - pca_result['access_token'], config_cca["scope"]) - self.assertNotEqual(None, cca_result.get("access_token"), str(cca_result)) + cca_result = cca.acquire_token_on_behalf_of(assertion, config_cca["scope"]) + self.assertIsNotNone(cca_result.get("access_token"), "OBO call failed: {}".format( + json.dumps(cca_result, indent=2))) # 3. Now the OBO app can simply store downstream token(s) in same session. # Alternatively, if you want to persist the downstream AT, and possibly @@ -685,13 +693,27 @@ def _test_acquire_token_obo(self, config_pca, config_cca, # Assuming you already did that (which is not shown in this test case), # the following part shows one of the ways to obtain an AT from cache. username = cca_result.get("id_token_claims", {}).get("preferred_username") - if username: # It means CCA have requested an IDT w/ "profile" scope - self.assertEqual(config_cca["username"], username) accounts = cca.get_accounts(username=username) - assert len(accounts) == 1, "App is expected to partition token cache per user" - account = accounts[0] + if username is not None: # It means CCA have requested an IDT w/ "profile" scope + assert config_cca["username"] == username, "Incorrect test case configuration" + self.assertEqual(1, len(accounts), "App is supposed to partition token cache per user") + account = accounts[0] # Alternatively, cca app could just loop through each account result = cca.acquire_token_silent(config_cca["scope"], account) - self.assertEqual(cca_result["access_token"], result["access_token"]) + self.assertTrue( + result and result.get("access_token") == cca_result["access_token"], + "CCA should hit an access token from cache: {}".format( + json.dumps(cca.token_cache._cache, indent=2))) + if "refresh_token" in cca_result: + result = cca.acquire_token_silent( + config_cca["scope"], account=account, force_refresh=True) + self.assertTrue( + result and "access_token" in result, + "CCA should get an AT silently, but we got this instead: {}".format(result)) + self.assertNotEqual( + result["access_token"], cca_result["access_token"], + "CCA should get a new AT") + else: + logger.info("AAD did not issue a RT for OBO flow") def _test_acquire_token_by_client_secret( self, client_id=None, client_secret=None, authority=None, scope=None, @@ -933,6 +955,31 @@ def test_acquire_token_obo(self): self._test_acquire_token_obo(config_pca, config_cca) + @unittest.skipUnless( + os.path.exists("tests/sp_obo.pem"), + "Need a 'tests/sp_obo.pem' private to run OBO for SP test") + def test_acquire_token_obo_for_sp(self): + authority = "https://login.windows-ppe.net/f686d426-8d16-42db-81b7-ab578e110ccd" + with open("tests/sp_obo.pem") as pem: + client_secret = { + "private_key": pem.read(), + "thumbprint": "378938210C976692D7F523B8C4FFBB645D17CE92", + } + midtier_app = { + "authority": authority, + "client_id": "c84e9c32-0bc9-4a73-af05-9efe9982a322", + "client_secret": client_secret, + "scope": ["23d08a1e-1249-4f7c-b5a5-cb11f29b6923/.default"], + #"username": "OBO-Client-PPE", # We do NOT attempt locating initial_app by name + } + initial_app = { + "authority": authority, + "client_id": "9793041b-9078-4942-b1d2-babdc472cc0c", + "client_secret": client_secret, + "scope": [midtier_app["client_id"] + "/.default"], + } + self._test_acquire_token_obo(initial_app, midtier_app) + def test_acquire_token_by_client_secret(self): # Vastly different than ArlingtonCloudTestCase.test_acquire_token_by_client_secret() _app = self.get_lab_app_object( From 0340f5e78d49195917c5682f082e461e9825a2a9 Mon Sep 17 00:00:00 2001 From: Dharshan BJ Date: Tue, 11 Feb 2025 13:33:30 -0800 Subject: [PATCH 221/262] Enable Issue-Sentinel to scan for similar issues (#790) --- .github/workflows/RunIssueSentinel.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/RunIssueSentinel.yml diff --git a/.github/workflows/RunIssueSentinel.yml b/.github/workflows/RunIssueSentinel.yml new file mode 100644 index 00000000..2bb393e1 --- /dev/null +++ b/.github/workflows/RunIssueSentinel.yml @@ -0,0 +1,17 @@ +name: Run issue sentinel +on: + issues: + types: [opened, edited, closed] + +jobs: + Issue: + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Run Issue Sentinel + uses: Azure/issue-sentinel@v1 + with: + password: ${{secrets.ISSUE_SENTINEL_PASSWORD}} + enable-similar-issues-scanning: true # Scan for similar issues + enable-security-issues-scanning: true # Scan for security issues \ No newline at end of file From 4d168cfd77e0f28bc16302698344db07c1f9e5d8 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 13 Feb 2025 11:41:06 -0800 Subject: [PATCH 222/262] We have long been switched to use Github actions --- .travis.yml | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 85917242..00000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -sudo: false -language: python -python: - - "2.7" - - "3.5" - - "3.6" -# Borrowed from https://github.com/travis-ci/travis-ci/issues/9815 -# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true - - python: 3.8 - dist: xenial - sudo: true - -install: - - pip install -r requirements.txt -script: - - python -m unittest discover -s tests - -deploy: - - # test pypi - provider: pypi - distributions: "sdist bdist_wheel" - server: https://test.pypi.org/legacy/ - user: "nugetaad" - password: - secure: KkjKySJujYxx31B15mlAZr2Jo4P99LcrMj3uON/X/WMXAqYVcVsYJ6JSzUvpNnCAgk+1hc24Qp6nibQHV824yiK+eG4qV+lpzkEEedkRx6NOW/h09OkT+pOSVMs0kcIhz7FzqChpl+jf6ZZpb13yJpQg2LoZIA4g8UdYHHFidWt4m5u1FZ9LPCqQ0OT3gnKK4qb0HIDaECfz5GYzrelLLces0PPwj1+X5eb38xUVtbkA1UJKLGKI882D8Rq5eBdbnDGsfDnF6oU+EBnGZ7o6HVQLdBgagDoVdx7yoXyntULeNxTENMTOZJEJbncQwxRgeEqJWXTTEW57O6Jo5uiHEpJA9lAePlRbS+z6BPDlnQogqOdTsYS0XMfOpYE0/r3cbtPUjETOmGYQxjQzfrFBfM7jaWnUquymZRYqCQ66VDo3I/ykNOCoM9qTmWt5L/MFfOZyoxLHnDThZBdJ3GXHfbivg+v+vOfY1gG8e2H2lQY+/LIMIJibF+MS4lJgrB81dcNdBzyxMNByuWQjSL1TY7un0QzcRcZz2NLrFGg8+9d67LQq4mK5ySimc6zdgnanuROU02vGr1EApT6D/qUItiulFgWqInNKrFXE9q74UP/WSooZPoLa3Du8y5s4eKerYYHQy5eSfIC8xKKDU8MSgoZhwQhCUP46G9Nsty0PYQc= - on: - branch: master - tags: false - condition: $TRAVIS_PYTHON_VERSION = "2.7" - - - # production pypi - provider: pypi - distributions: "sdist bdist_wheel" - user: "nugetaad" - password: - secure: KkjKySJujYxx31B15mlAZr2Jo4P99LcrMj3uON/X/WMXAqYVcVsYJ6JSzUvpNnCAgk+1hc24Qp6nibQHV824yiK+eG4qV+lpzkEEedkRx6NOW/h09OkT+pOSVMs0kcIhz7FzqChpl+jf6ZZpb13yJpQg2LoZIA4g8UdYHHFidWt4m5u1FZ9LPCqQ0OT3gnKK4qb0HIDaECfz5GYzrelLLces0PPwj1+X5eb38xUVtbkA1UJKLGKI882D8Rq5eBdbnDGsfDnF6oU+EBnGZ7o6HVQLdBgagDoVdx7yoXyntULeNxTENMTOZJEJbncQwxRgeEqJWXTTEW57O6Jo5uiHEpJA9lAePlRbS+z6BPDlnQogqOdTsYS0XMfOpYE0/r3cbtPUjETOmGYQxjQzfrFBfM7jaWnUquymZRYqCQ66VDo3I/ykNOCoM9qTmWt5L/MFfOZyoxLHnDThZBdJ3GXHfbivg+v+vOfY1gG8e2H2lQY+/LIMIJibF+MS4lJgrB81dcNdBzyxMNByuWQjSL1TY7un0QzcRcZz2NLrFGg8+9d67LQq4mK5ySimc6zdgnanuROU02vGr1EApT6D/qUItiulFgWqInNKrFXE9q74UP/WSooZPoLa3Du8y5s4eKerYYHQy5eSfIC8xKKDU8MSgoZhwQhCUP46G9Nsty0PYQc= - on: - branch: master - tags: true - condition: $TRAVIS_PYTHON_VERSION = "2.7" - From 7f15c399b6c34f1dbf838ccf077286290b3ee4d0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 27 Aug 2024 12:34:56 -0700 Subject: [PATCH 223/262] Fix incorrect login_hint detection --- msal/application.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 7f7615de..684b821b 100644 --- a/msal/application.py +++ b/msal/application.py @@ -2241,7 +2241,8 @@ def _acquire_token_interactive_via_broker( # _signin_silently() only gets tokens for default account, # but this seems to have been fixed in PyMsalRuntime 0.11.2 "access_token" in response and login_hint - and response.get("id_token_claims", {}) != login_hint) + and login_hint != response.get( + "id_token_claims", {}).get("preferred_username")) wrong_account_error_message = ( 'prompt="none" will not work for login_hint="non-default-user"') if is_wrong_account: From a565ba20e5bc83b65219086f3e55302d59b0ea00 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 20 Sep 2024 12:34:56 -0700 Subject: [PATCH 224/262] E2e attempts broker albeit PyMsalRuntime init fails --- msal/application.py | 5 ++-- tests/broker_util.py | 21 ++++++++++++++++ tests/test_account_source.py | 46 ++++++++++++++++++++---------------- tests/test_e2e.py | 21 ++++++---------- 4 files changed, 56 insertions(+), 37 deletions(-) create mode 100644 tests/broker_util.py diff --git a/msal/application.py b/msal/application.py index 684b821b..e436b4f5 100644 --- a/msal/application.py +++ b/msal/application.py @@ -675,7 +675,8 @@ def _decide_broker(self, allow_broker, enable_pii_log): "allow_broker is deprecated. " "Please use PublicClientApplication(..., " "enable_broker_on_windows=True, " - "enable_broker_on_mac=...)", + # No need to mention non-Windows platforms, because allow_broker is only for Windows + "...)", DeprecationWarning) opted_in_for_broker = ( self._enable_broker # True means Opted-in from PCA @@ -697,7 +698,7 @@ def _decide_broker(self, allow_broker, enable_pii_log): _init_broker(enable_pii_log) except RuntimeError: self._enable_broker = False - logger.exception( + logger.warning( # It is common on Mac and Linux where broker is not built-in "Broker is unavailable on this platform. " "We will fallback to non-broker.") logger.debug("Broker enabled? %s", self._enable_broker) diff --git a/tests/broker_util.py b/tests/broker_util.py new file mode 100644 index 00000000..e9722358 --- /dev/null +++ b/tests/broker_util.py @@ -0,0 +1,21 @@ +import logging + + +logger = logging.getLogger(__name__) + + +def is_pymsalruntime_installed() -> bool: + try: + import pymsalruntime + logger.info("PyMsalRuntime installed and initialized") + return True + except ImportError: + logger.info("PyMsalRuntime not installed") + return False + except RuntimeError: + logger.warning( + "PyMsalRuntime installed but failed to initialize the real broker. " + "This may happen on Mac and Linux where broker is not built-in. " + "Test cases shall attempt broker and test its fallback behavior." + ) + return True diff --git a/tests/test_account_source.py b/tests/test_account_source.py index 662f0419..7b449ef3 100644 --- a/tests/test_account_source.py +++ b/tests/test_account_source.py @@ -3,15 +3,11 @@ from unittest.mock import patch except: from mock import patch -try: - import pymsalruntime - broker_available = True -except ImportError: - broker_available = False import msal from tests import unittest from tests.test_token_cache import build_response from tests.http_client import MinimalResponse +from tests.broker_util import is_pymsalruntime_installed SCOPE = "scope_foo" @@ -24,54 +20,62 @@ def _mock_post(url, headers=None, *args, **kwargs): return MinimalResponse(status_code=200, text=json.dumps(TOKEN_RESPONSE)) -@unittest.skipUnless(broker_available, "These test cases need pip install msal[broker]") +@unittest.skipUnless(is_pymsalruntime_installed(), "These test cases need pip install msal[broker]") @patch("msal.broker._acquire_token_silently", return_value=dict( - TOKEN_RESPONSE, _account_id="placeholder")) + TOKEN_RESPONSE, _account_id="placeholder")) @patch.object(msal.authority, "tenant_discovery", return_value={ "authorization_endpoint": "https://contoso.com/placeholder", "token_endpoint": "https://contoso.com/placeholder", }) # Otherwise it would fail on OIDC discovery class TestAccountSourceBehavior(unittest.TestCase): + def setUp(self): + self.app = msal.PublicClientApplication( + "client_id", + enable_broker_on_windows=True, + ) + if not self.app._enable_broker: + self.skipTest( + "These test cases require patching msal.broker which is only possible " + "when broker enabled successfully i.e. no RuntimeError") + return super().setUp() + def test_device_flow_and_its_silent_call_should_bypass_broker(self, _, mocked_broker_ats): - app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) - result = app.acquire_token_by_device_flow({"device_code": "123"}, post=_mock_post) + result = self.app.acquire_token_by_device_flow({"device_code": "123"}, post=_mock_post) self.assertEqual(result["token_source"], "identity_provider") - account = app.get_accounts()[0] + account = self.app.get_accounts()[0] self.assertEqual(account["account_source"], "urn:ietf:params:oauth:grant-type:device_code") - result = app.acquire_token_silent_with_error( + result = self.app.acquire_token_silent_with_error( [SCOPE], account, force_refresh=True, post=_mock_post) mocked_broker_ats.assert_not_called() self.assertEqual(result["token_source"], "identity_provider") def test_ropc_flow_and_its_silent_call_should_invoke_broker(self, _, mocked_broker_ats): - app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) with patch("msal.broker._signin_silently", return_value=dict(TOKEN_RESPONSE, _account_id="placeholder")): - result = app.acquire_token_by_username_password( + result = self.app.acquire_token_by_username_password( "username", "placeholder", [SCOPE], post=_mock_post) self.assertEqual(result["token_source"], "broker") - account = app.get_accounts()[0] + account = self.app.get_accounts()[0] self.assertEqual(account["account_source"], "broker") - result = app.acquire_token_silent_with_error( + result = self.app.acquire_token_silent_with_error( [SCOPE], account, force_refresh=True, post=_mock_post) self.assertEqual(result["token_source"], "broker") def test_interactive_flow_and_its_silent_call_should_invoke_broker(self, _, mocked_broker_ats): - app = msal.PublicClientApplication("client_id", enable_broker_on_windows=True) - with patch.object(app, "_acquire_token_interactive_via_broker", return_value=dict( + with patch.object(self.app, "_acquire_token_interactive_via_broker", return_value=dict( TOKEN_RESPONSE, _account_id="placeholder")): - result = app.acquire_token_interactive( - [SCOPE], parent_window_handle=app.CONSOLE_WINDOW_HANDLE) + result = self.app.acquire_token_interactive( + [SCOPE], parent_window_handle=self.app.CONSOLE_WINDOW_HANDLE) self.assertEqual(result["token_source"], "broker") - account = app.get_accounts()[0] + account = self.app.get_accounts()[0] self.assertEqual(account["account_source"], "broker") - result = app.acquire_token_silent_with_error( + result = self.app.acquire_token_silent_with_error( [SCOPE], account, force_refresh=True, post=_mock_post) mocked_broker_ats.assert_called_once() self.assertEqual(result["token_source"], "broker") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b2701569..b1eb0b12 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -29,12 +29,9 @@ from tests.http_client import MinimalHttpClient, MinimalResponse from msal.oauth2cli import AuthCodeReceiver from msal.oauth2cli.oidc import decode_part +from tests.broker_util import is_pymsalruntime_installed + -try: - import pymsalruntime - broker_available = True -except ImportError: - broker_available = False logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG if "-v" in sys.argv else logging.INFO) @@ -44,6 +41,7 @@ except ImportError: logger.warn("Run pip install -r requirements.txt for optional dependency") +_PYMSALRUNTIME_INSTALLED = is_pymsalruntime_installed() _AZURE_CLI = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" def _get_app_and_auth_code( @@ -187,19 +185,14 @@ def _build_app(cls, http_client=http_client or MinimalHttpClient(), ) else: - # Reuse same test cases, by run them with and without broker - try: - import pymsalruntime - broker_available = True - except ImportError: - broker_available = False + # Reuse same test cases, by running them with and without PyMsalRuntime installed return msal.PublicClientApplication( client_id, authority=authority, oidc_authority=oidc_authority, http_client=http_client or MinimalHttpClient(), - enable_broker_on_windows=broker_available, - enable_broker_on_mac=broker_available, + enable_broker_on_windows=_PYMSALRUNTIME_INSTALLED, + enable_broker_on_mac=_PYMSALRUNTIME_INSTALLED, ) def _test_username_password(self, @@ -1307,7 +1300,7 @@ def test_acquire_token_silent_with_an_empty_cache_should_return_none(self): # it means MSAL Python is not affected by that. -@unittest.skipUnless(broker_available, "AT POP feature is only supported by using broker") +@unittest.skipUnless(_PYMSALRUNTIME_INSTALLED, "AT POP feature is only supported by using broker") class PopTestCase(LabBasedTestCase): def test_at_pop_should_contain_pop_scheme_content(self): auth_scheme = msal.PopAuthScheme( From 4e2a4ec0d4af3bf44c1f1608903440e5f6e75aa9 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 12 Feb 2025 12:34:56 -0800 Subject: [PATCH 225/262] Specify verify=True to hopefully satisfy CodeQL --- tests/test_e2e.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b1eb0b12..3da02300 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1362,8 +1362,19 @@ def test_at_pop_calling_pattern(self): # and then fallback to bearer token code path. # We skip it here because this test case has not yet initialize self.app # assert self.app.is_pop_supported() + api_endpoint = "https://20.190.132.47/beta/me" - resp = requests.get(api_endpoint, verify=False) # @suppress py/bandit/requests-ssl-verify-disabled + verify = True # Hopefully this will make CodeQL happy + if verify: + self.skipTest(""" + The api_endpoint is for test only and has no proper SSL certificate, + so you would have to disable SSL certificate checks and run this test case manually. + We tried suppressing the CodeQL warning by adding this in the proper places + @suppress py/bandit/requests-ssl-verify-disabled + but it did not work. + """) + # @suppress py/bandit/requests-ssl-verify-disabled + resp = requests.get(api_endpoint, verify=verify) # CodeQL [SM03157] self.assertEqual(resp.status_code, 401, "Initial call should end with an http 401 error") result = self._get_shr_pop(**dict( self.get_lab_user(usertype="cloud"), # This is generally not the current laptop's default AAD account @@ -1374,10 +1385,11 @@ def test_at_pop_calling_pattern(self): nonce=self._extract_pop_nonce(resp.headers.get("WWW-Authenticate")), ), )) - # The api_endpoint is for test only and has no proper SSL certificate, - # so we suppress the CodeQL warning for disabling SSL certificate checks - # @suppress py/bandit/requests-ssl-verify-disabled - resp = requests.get(api_endpoint, verify=False, headers={ + resp = requests.get( + api_endpoint, + # CodeQL [SM03157] + verify=verify, # @suppress py/bandit/requests-ssl-verify-disabled + headers={ "Authorization": "pop {}".format(result["access_token"]), }) self.assertEqual(resp.status_code, 200, "POP resource should be accessible") From f22994f93e1f49d6fb6234bbc8a7a1c90604b247 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 9 Oct 2024 12:34:56 -0700 Subject: [PATCH 226/262] enable_broker_on_windows and enable_broker_on_mac present as keyword-only arguments --- msal/application.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/msal/application.py b/msal/application.py index e436b4f5..25a0db2b 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1919,7 +1919,12 @@ class PublicClientApplication(ClientApplication): # browser app or mobile app DEVICE_FLOW_CORRELATION_ID = "_correlation_id" CONSOLE_WINDOW_HANDLE = object() - def __init__(self, client_id, client_credential=None, **kwargs): + def __init__( + self, client_id, client_credential=None, + *, + enable_broker_on_windows=None, + enable_broker_on_mac=None, + **kwargs): """Same as :func:`ClientApplication.__init__`, except that ``client_credential`` parameter shall remain ``None``. @@ -1996,9 +2001,6 @@ def __init__(self, client_id, client_credential=None, **kwargs): """ if client_credential is not None: raise ValueError("Public Client should not possess credentials") - # Using kwargs notation for now. We will switch to keyword-only arguments. - enable_broker_on_windows = kwargs.pop("enable_broker_on_windows", False) - enable_broker_on_mac = kwargs.pop("enable_broker_on_mac", False) self._enable_broker = bool( enable_broker_on_windows and sys.platform == "win32" or enable_broker_on_mac and sys.platform == "darwin") From a6d3d0d22a61b5b71a4a501dfb57aaa21462c4e6 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 15 Oct 2024 12:34:56 -0700 Subject: [PATCH 227/262] Skip authority discovery in test_application.py --- tests/test_application.py | 38 ++++++++++++++++++++++++++++---------- tests/test_e2e.py | 7 ++++++- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/tests/test_application.py b/tests/test_application.py index e565e105..0c7f2d29 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -20,6 +20,12 @@ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) +_OIDC_DISCOVERY = "msal.authority.tenant_discovery" +_OIDC_DISCOVERY_MOCK = Mock(return_value={ + "authorization_endpoint": "https://contoso.com/placeholder", + "token_endpoint": "https://contoso.com/placeholder", +}) + class TestHelperExtractCerts(unittest.TestCase): # It is used by SNI scenario @@ -58,10 +64,9 @@ def test_bytes_to_bytes(self): class TestClientApplicationAcquireTokenSilentErrorBehaviors(unittest.TestCase): + @patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) def setUp(self): self.authority_url = "https://login.microsoftonline.com/common" - self.authority = msal.authority.Authority( - self.authority_url, MinimalHttpClient()) self.scopes = ["s1", "s2"] self.uid = "my_uid" self.utid = "my_utid" @@ -116,12 +121,11 @@ def tester(url, **kwargs): self.assertEqual("", result.get("classification")) +@patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) class TestClientApplicationAcquireTokenSilentFociBehaviors(unittest.TestCase): def setUp(self): self.authority_url = "https://login.microsoftonline.com/common" - self.authority = msal.authority.Authority( - self.authority_url, MinimalHttpClient()) self.scopes = ["s1", "s2"] self.uid = "my_uid" self.utid = "my_utid" @@ -148,7 +152,7 @@ def tester(url, data=None, **kwargs): self.assertEqual(self.frt, data.get("refresh_token"), "Should attempt the FRT") return MinimalResponse(status_code=400, text=error_response) app._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( - self.authority, self.scopes, self.account, post=tester) + app.authority, self.scopes, self.account, post=tester) self.assertNotEqual([], app.token_cache.find( msal.TokenCache.CredentialType.REFRESH_TOKEN, query={"secret": self.frt}), "The FRT should not be removed from the cache") @@ -168,7 +172,7 @@ def tester(url, data=None, **kwargs): self.assertEqual(rt, data.get("refresh_token"), "Should attempt the RT") return MinimalResponse(status_code=200, text='{}') app._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( - self.authority, self.scopes, self.account, post=tester) + app.authority, self.scopes, self.account, post=tester) def test_unknown_family_app_will_attempt_frt_and_join_family(self): def tester(url, data=None, **kwargs): @@ -180,7 +184,7 @@ def tester(url, data=None, **kwargs): app = ClientApplication( "unknown_family_app", authority=self.authority_url, token_cache=self.cache) at = app._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( - self.authority, self.scopes, self.account, post=tester) + app.authority, self.scopes, self.account, post=tester) logger.debug("%s.cache = %s", self.id(), self.cache.serialize()) self.assertEqual("at", at.get("access_token"), "New app should get a new AT") app_metadata = app.token_cache.find( @@ -202,7 +206,7 @@ def tester(url, data=None, **kwargs): app = ClientApplication( "preexisting_family_app", authority=self.authority_url, token_cache=self.cache) resp = app._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( - self.authority, self.scopes, self.account, post=tester) + app.authority, self.scopes, self.account, post=tester) logger.debug("%s.cache = %s", self.id(), self.cache.serialize()) self.assertEqual(json.loads(error_response), resp, "Error raised will be returned") @@ -237,7 +241,7 @@ def test_family_app_remove_account(self): class TestClientApplicationForAuthorityMigration(unittest.TestCase): - @classmethod + # Chose to not mock oidc discovery, because AuthorityMigration might rely on real data def setUp(self): self.environment_in_cache = "sts.windows.net" self.authority_url_in_app = "https://login.microsoftonline.com/common" @@ -444,6 +448,7 @@ def mock_post(url, headers=None, *args, **kwargs): self.assertRefreshOn(result, new_refresh_in) +# TODO Patching oidc discovery ends up failing. But we plan to remove offline telemetry anyway. class TestTelemetryMaintainingOfflineState(unittest.TestCase): authority_url = "https://login.microsoftonline.com/common" scopes = ["s1", "s2"] @@ -524,6 +529,7 @@ def mock_post(url, headers=None, *args, **kwargs): class TestTelemetryOnClientApplication(unittest.TestCase): @classmethod + @patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) def setUpClass(cls): # Initialization at runtime, not interpret-time cls.app = ClientApplication( "client_id", authority="https://login.microsoftonline.com/common") @@ -552,6 +558,7 @@ def mock_post(url, headers=None, *args, **kwargs): class TestTelemetryOnPublicClientApplication(unittest.TestCase): @classmethod + @patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) def setUpClass(cls): # Initialization at runtime, not interpret-time cls.app = PublicClientApplication( "client_id", authority="https://login.microsoftonline.com/common") @@ -581,6 +588,7 @@ def mock_post(url, headers=None, *args, **kwargs): class TestTelemetryOnConfidentialClientApplication(unittest.TestCase): @classmethod + @patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) def setUpClass(cls): # Initialization at runtime, not interpret-time cls.app = ConfidentialClientApplication( "client_id", client_credential="secret", @@ -626,6 +634,7 @@ def mock_post(url, headers=None, *args, **kwargs): self.assertEqual(at, result.get("access_token")) +@patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) class TestClientApplicationWillGroupAccounts(unittest.TestCase): def test_get_accounts(self): client_id = "my_app" @@ -678,15 +687,24 @@ def mock_post(url, headers=None, *args, **kwargs): with self.assertWarns(DeprecationWarning): app.acquire_token_for_client(["scope"], post=mock_post) + @patch(_OIDC_DISCOVERY, new=Mock(return_value={ + "authorization_endpoint": "https://contoso.com/common", + "token_endpoint": "https://contoso.com/common", + })) def test_common_authority_should_emit_warning(self): self._test_certain_authority_should_emit_warning( authority="https://login.microsoftonline.com/common") + @patch(_OIDC_DISCOVERY, new=Mock(return_value={ + "authorization_endpoint": "https://contoso.com/organizations", + "token_endpoint": "https://contoso.com/organizations", + })) def test_organizations_authority_should_emit_warning(self): self._test_certain_authority_should_emit_warning( authority="https://login.microsoftonline.com/organizations") +@patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) class TestRemoveTokensForClient(unittest.TestCase): def test_remove_tokens_for_client_should_remove_client_tokens_only(self): at_for_user = "AT for user" @@ -716,6 +734,7 @@ def test_remove_tokens_for_client_should_remove_client_tokens_only(self): self.assertEqual(at_for_user, remaining_tokens[0].get("secret")) +@patch(_OIDC_DISCOVERY, new=_OIDC_DISCOVERY_MOCK) class TestScopeDecoration(unittest.TestCase): def _test_client_id_should_be_a_valid_scope(self, client_id, other_scopes): # B2C needs this https://learn.microsoft.com/en-us/azure/active-directory-b2c/access-tokens#openid-connect-scopes @@ -855,4 +874,3 @@ def test_app_did_not_register_redirect_uri_should_error_out(self): parent_window_handle=app.CONSOLE_WINDOW_HANDLE, ) self.assertEqual(result.get("error"), "broker_error") - diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 3da02300..533d44ca 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -310,8 +310,13 @@ def _test_acquire_token_interactive( msal.application._is_running_in_cloud_shell(), "Manually run this test case from inside Cloud Shell") class CloudShellTestCase(E2eTestCase): - app = msal.PublicClientApplication("client_id") scope_that_requires_no_managed_device = "https://management.core.windows.net/" # Scopes came from https://msazure.visualstudio.com/One/_git/compute-CloudShell?path=/src/images/agent/env/envconfig.PROD.json&version=GBmaster&_a=contents + + def setUpClass(cls): + # Doing it here instead of as a class member, + # otherwise its overhead incurs even when running other cases + cls.app = msal.PublicClientApplication("client_id") + def test_access_token_should_be_obtained_for_a_supported_scope(self): result = self.app.acquire_token_interactive( [self.scope_that_requires_no_managed_device], prompt="none") From fb3e21cf501587dccb71676a3d06f9ea24476102 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 10 Mar 2025 16:54:04 -0700 Subject: [PATCH 228/262] ADFSv2 and ADFSv3 labs are decommissioned --- tests/test_e2e.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 533d44ca..d2e66c88 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -850,11 +850,13 @@ def test_adfs4_fed_user(self): config["password"] = self.get_lab_user_secret(config["lab_name"]) self._test_username_password(**config) + @unittest.skip("ADFSv3 is decommissioned in our test environment") def test_adfs3_fed_user(self): config = self.get_lab_user(usertype="federated", federationProvider="ADFSv3") config["password"] = self.get_lab_user_secret(config["lab_name"]) self._test_username_password(**config) + @unittest.skip("ADFSv2 is decommissioned in our test environment") def test_adfs2_fed_user(self): config = self.get_lab_user(usertype="federated", federationProvider="ADFSv2") config["password"] = self.get_lab_user_secret(config["lab_name"]) From 689e8624b5987be529ab91f8786a1105ee505f23 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 27 Feb 2025 12:34:56 -0800 Subject: [PATCH 229/262] Support pod identity --- msal/managed_identity.py | 4 +++- tests/test_mi.py | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index ec032ca7..6f85571d 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -448,7 +448,9 @@ def _obtain_token_on_azure_vm(http_client, managed_identity, resource): } _adjust_param(params, managed_identity) resp = http_client.get( - "http://169.254.169.254/metadata/identity/oauth2/token", + os.getenv( + "AZURE_POD_IDENTITY_AUTHORITY_HOST", "http://169.254.169.254" + ).strip("/") + "/metadata/identity/oauth2/token", params=params, headers={"Metadata": "true"}, ) diff --git a/tests/test_mi.py b/tests/test_mi.py index c5a99ae3..a7c2cb6c 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -121,13 +121,29 @@ def _test_happy_path(self, app, mocked_http, expires_in, resource="R"): class VmTestCase(ClientTestCase): - def test_happy_path(self): + def _test_happy_path(self) -> callable: expires_in = 7890 # We test a bigger than 7200 value here with patch.object(self.app._http_client, "get", return_value=MinimalResponse( status_code=200, text='{"access_token": "AT", "expires_in": "%s", "resource": "R"}' % expires_in, )) as mocked_method: - self._test_happy_path(self.app, mocked_method, expires_in) + super(VmTestCase, self)._test_happy_path(self.app, mocked_method, expires_in) + return mocked_method + + def test_happy_path_of_vm(self): + self._test_happy_path().assert_called_with( + 'http://169.254.169.254/metadata/identity/oauth2/token', + params={'api-version': '2018-02-01', 'resource': 'R'}, + headers={'Metadata': 'true'}, + ) + + @patch.dict(os.environ, {"AZURE_POD_IDENTITY_AUTHORITY_HOST": "http://localhost:1234//"}) + def test_happy_path_of_pod_identity(self): + self._test_happy_path().assert_called_with( + 'http://localhost:1234/metadata/identity/oauth2/token', + params={'api-version': '2018-02-01', 'resource': 'R'}, + headers={'Metadata': 'true'}, + ) def test_vm_error_should_be_returned_as_is(self): raw_error = '{"raw": "error format is undefined"}' From b92b4f18cb176231ae494639d24aeb19bd457d73 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 31 Jan 2025 12:34:56 -0800 Subject: [PATCH 230/262] Bumping version number to 1.32.0 --- msal/sku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/sku.py b/msal/sku.py index e383e151..2a3172aa 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.31.1" # When releasing, also check and bump our dependencies's versions if needed +__version__ = "1.32.0" SKU = "MSAL.Python" From 137dee46c9189b9d02105d92f8dcac5fd39a9266 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 22 Oct 2024 12:34:56 -0700 Subject: [PATCH 231/262] Update error hints for where to get OBO test apps --- tests/test_e2e.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d2e66c88..28c73abc 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -934,10 +934,10 @@ def test_adfs2019_onprem_acquire_token_interactive(self): "Need LAB_OBO_CLIENT_SECRET from https://aka.ms/GetLabSecret?Secret=TodoListServiceV2-OBO") @unittest.skipUnless( os.getenv("LAB_OBO_CONFIDENTIAL_CLIENT_ID"), - "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://docs.msidlab.com/flows/onbehalfofflow.html") + "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://identitydivision.visualstudio.com/Engineering/_git/IDLABS?path=/src/DocFX/LabDocs/flows/onbehalfofflow.md&_a=preview") @unittest.skipUnless( os.getenv("LAB_OBO_PUBLIC_CLIENT_ID"), - "Need LAB_OBO_PUBLIC_CLIENT_ID from https://docs.msidlab.com/flows/onbehalfofflow.html") + "Need LAB_OBO_PUBLIC_CLIENT_ID from https://identitydivision.visualstudio.com/Engineering/_git/IDLABS?path=/src/DocFX/LabDocs/flows/onbehalfofflow.md&_a=preview") def test_acquire_token_obo(self): config = self.get_lab_user(usertype="cloud") @@ -998,7 +998,7 @@ def test_acquire_token_by_client_secret(self): "Need LAB_OBO_CLIENT_SECRET from https://aka.ms/GetLabSecret?Secret=TodoListServiceV2-OBO") @unittest.skipUnless( os.getenv("LAB_OBO_CONFIDENTIAL_CLIENT_ID"), - "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://docs.msidlab.com/flows/onbehalfofflow.html") + "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://identitydivision.visualstudio.com/Engineering/_git/IDLABS?path=/src/DocFX/LabDocs/flows/onbehalfofflow.md&_a=preview") def test_confidential_client_acquire_token_by_username_password(self): # This approach won't work: # config = self.get_lab_user(usertype="cloud", publicClient="no") @@ -1215,10 +1215,10 @@ def test_acquire_token_for_client_should_use_an_env_var_with_long_region_name(se "Need LAB_OBO_CLIENT_SECRET from https://aka.ms/GetLabSecret?Secret=TodoListServiceV2-OBO") @unittest.skipUnless( os.getenv("LAB_OBO_CONFIDENTIAL_CLIENT_ID"), - "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://docs.msidlab.com/flows/onbehalfofflow.html") + "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://identitydivision.visualstudio.com/Engineering/_git/IDLABS?path=/src/DocFX/LabDocs/flows/onbehalfofflow.md&_a=preview") @unittest.skipUnless( os.getenv("LAB_OBO_PUBLIC_CLIENT_ID"), - "Need LAB_OBO_PUBLIC_CLIENT_ID from https://docs.msidlab.com/flows/onbehalfofflow.html") + "Need LAB_OBO_PUBLIC_CLIENT_ID from https://identitydivision.visualstudio.com/Engineering/_git/IDLABS?path=/src/DocFX/LabDocs/flows/onbehalfofflow.md&_a=preview") def test_cca_obo_should_bypass_regional_endpoint_therefore_still_work(self): """We test OBO because it is implemented in sub class ConfidentialClientApplication""" config = self.get_lab_user(usertype="cloud") @@ -1246,7 +1246,7 @@ def test_cca_obo_should_bypass_regional_endpoint_therefore_still_work(self): "Need LAB_OBO_CLIENT_SECRET from https://aka.ms/GetLabSecret?Secret=TodoListServiceV2-OBO") @unittest.skipUnless( os.getenv("LAB_OBO_CONFIDENTIAL_CLIENT_ID"), - "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://docs.msidlab.com/flows/onbehalfofflow.html") + "Need LAB_OBO_CONFIDENTIAL_CLIENT_ID from https://identitydivision.visualstudio.com/Engineering/_git/IDLABS?path=/src/DocFX/LabDocs/flows/onbehalfofflow.md&_a=preview") def test_cca_ropc_should_bypass_regional_endpoint_therefore_still_work(self): """We test ROPC because it is implemented in base class ClientApplication""" config = self.get_lab_user(usertype="cloud") From 4db4a8c2d740c5315f60f07a7e1ee58667b2a759 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 4 Nov 2024 12:34:56 -0800 Subject: [PATCH 232/262] This typo fix will probably resolve https://github.com/MicrosoftDocs/microsoft-authentication-library-for-python/issues/97 --- msal/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/application.py b/msal/application.py index 25a0db2b..dac7aa19 100644 --- a/msal/application.py +++ b/msal/application.py @@ -329,7 +329,7 @@ def __init__( "client_assertion": "...a JWT with claims aud, exp, iss, jti, nbf, and sub..." } - .. admonition:: Supporting reading client cerficates from PFX files + .. admonition:: Supporting reading client certificates from PFX files *Added in version 1.29.0*: Feed in a dictionary containing the path to a PFX file:: From 30dce4ecc63d93ef34b89c052aab1a1231395ce6 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 25 Mar 2025 12:34:56 -0700 Subject: [PATCH 233/262] Swallow exception during optional check This used to cause issue with MSAL 1.32 and MSAL EX <= 1.2 running inside read-only container --- msal/application.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/msal/application.py b/msal/application.py index dac7aa19..31e1105d 100644 --- a/msal/application.py +++ b/msal/application.py @@ -208,6 +208,11 @@ def _msal_extension_check(): pass # The optional msal_extensions is not installed. Business as usual. except ValueError: logger.exception(f"msal_extensions version {v} not in major.minor.patch format") + except: + logger.exception( + "Unable to import msal_extensions during an optional check. " + "This exception can be safely ignored." + ) class ClientApplication(object): From 2f9747fa995e55fe4034308d9d23133523827e09 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 21 Feb 2025 12:34:56 -0800 Subject: [PATCH 234/262] ManagedIdentityClient sends xms_cc and token_sha256_to_refresh to SF --- msal/managed_identity.py | 58 ++++++++++++++++++++++++++++------ tests/test_mi.py | 67 ++++++++++++++++++++++++++++++++++------ 2 files changed, 106 insertions(+), 19 deletions(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 6f85571d..230ff2f9 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -2,6 +2,7 @@ # All rights reserved. # # This code is licensed under the MIT License. +import hashlib import json import logging import os @@ -10,7 +11,7 @@ import time from urllib.parse import urlparse # Python 3+ from collections import UserDict # Python 3+ -from typing import Optional, Union # Needed in Python 3.7 & 3.8 +from typing import List, Optional, Union # Needed in Python 3.7 & 3.8 from .token_cache import TokenCache from .individual_cache import _IndividualCache as IndividualCache from .throttled_http_client import ThrottledHttpClientBase, RetryAfterParser @@ -162,6 +163,7 @@ def __init__( http_client, token_cache=None, http_cache=None, + client_capabilities: Optional[List[str]] = None, ): """Create a managed identity client. @@ -192,6 +194,17 @@ def __init__( Optional. It has the same characteristics as the :paramref:`msal.ClientApplication.http_cache`. + :param list[str] client_capabilities: (optional) + Allows configuration of one or more client capabilities, e.g. ["CP1"]. + + Client capability is meant to inform the Microsoft identity platform + (STS) what this client is capable for, + so STS can decide to turn on certain features. + + Implementation details: + Client capability in Managed Identity is relayed as-is + via ``xms_cc`` parameter on the wire. + Recipe 1: Hard code a managed identity for your app:: import msal, requests @@ -238,6 +251,7 @@ def __init__( http_cache=http_cache, ) self._token_cache = token_cache or TokenCache() + self._client_capabilities = client_capabilities def _get_instance(self): if self.__instance is None: @@ -266,8 +280,7 @@ def acquire_token_for_client( and then a *claims challenge* will be returned by the target resource, as a `claims_challenge` directive in the `www-authenticate` header, even if the app developer did not opt in for the "CP1" client capability. - Upon receiving a `claims_challenge`, MSAL will skip a token cache read, - and will attempt to acquire a new token. + Upon receiving a `claims_challenge`, MSAL will attempt to acquire a new token. .. note:: @@ -278,11 +291,13 @@ def acquire_token_for_client( This is a service-side behavior that cannot be changed by this library. `Azure VM docs `_ """ + access_token_to_refresh = None # This could become a public parameter in the future access_token_from_cache = None client_id_in_cache = self._managed_identity.get( ManagedIdentity.ID, "SYSTEM_ASSIGNED_MANAGED_IDENTITY") now = time.time() - if not claims_challenge: # Then attempt token cache search + if True: # Attempt cache search even if receiving claims_challenge, + # because we want to locate the existing token (if any) and refresh it matches = self._token_cache.find( self._token_cache.CredentialType.ACCESS_TOKEN, target=[resource], @@ -297,6 +312,11 @@ def acquire_token_for_client( expires_in = int(entry["expires_on"]) - now if expires_in < 5*60: # Then consider it expired continue # Removal is not necessary, it will be overwritten + if claims_challenge and not access_token_to_refresh: + # Since caller did not pinpoint the token causing claims challenge, + # we have to assume it is the first token we found in cache. + access_token_to_refresh = entry["secret"] + break logger.debug("Cache hit an AT") access_token_from_cache = { # Mimic a real response "access_token": entry["secret"], @@ -310,7 +330,13 @@ def acquire_token_for_client( break # With a fallback in hand, we break here to go refresh return access_token_from_cache # It is still good as new try: - result = _obtain_token(self._http_client, self._managed_identity, resource) + result = _obtain_token( + self._http_client, self._managed_identity, resource, + access_token_sha256_to_refresh=hashlib.sha256( + access_token_to_refresh.encode("utf-8")).hexdigest() + if access_token_to_refresh else None, + client_capabilities=self._client_capabilities, + ) if "access_token" in result: expires_in = result.get("expires_in", 3600) if "refresh_in" not in result and expires_in >= 7200: @@ -385,8 +411,12 @@ def get_managed_identity_source(): return DEFAULT_TO_VM -def _obtain_token(http_client, managed_identity, resource): - # A unified low-level API that talks to different Managed Identity +def _obtain_token( + http_client, managed_identity, resource, + *, + access_token_sha256_to_refresh: Optional[str] = None, + client_capabilities: Optional[List[str]] = None, +): if ("IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ and "IDENTITY_SERVER_THUMBPRINT" in os.environ ): @@ -402,6 +432,8 @@ def _obtain_token(http_client, managed_identity, resource): os.environ["IDENTITY_HEADER"], os.environ["IDENTITY_SERVER_THUMBPRINT"], resource, + access_token_sha256_to_refresh=access_token_sha256_to_refresh, + client_capabilities=client_capabilities, ) if "IDENTITY_ENDPOINT" in os.environ and "IDENTITY_HEADER" in os.environ: return _obtain_token_on_app_service( @@ -553,6 +585,9 @@ def _obtain_token_on_machine_learning( def _obtain_token_on_service_fabric( http_client, endpoint, identity_header, server_thumbprint, resource, + *, + access_token_sha256_to_refresh: str = None, + client_capabilities: Optional[List[str]] = None, ): """Obtains token for `Service Fabric `_ @@ -563,7 +598,12 @@ def _obtain_token_on_service_fabric( logger.debug("Obtaining token via managed identity on Azure Service Fabric") resp = http_client.get( endpoint, - params={"api-version": "2019-07-01-preview", "resource": resource}, + params={k: v for k, v in { + "api-version": "2019-07-01-preview", + "resource": resource, + "token_sha256_to_refresh": access_token_sha256_to_refresh, + "xms_cc": ",".join(client_capabilities) if client_capabilities else None, + }.items() if v is not None}, headers={"Secret": identity_header}, ) try: @@ -584,7 +624,7 @@ def _obtain_token_on_service_fabric( "ArgumentNullOrEmpty": "invalid_scope", } return { - "error": error_mapping.get(payload["error"]["code"], "invalid_request"), + "error": error_mapping.get(error.get("code"), "invalid_request"), "error_description": resp.text, } except json.decoder.JSONDecodeError: diff --git a/tests/test_mi.py b/tests/test_mi.py index a7c2cb6c..0fd432a5 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -1,7 +1,9 @@ +import hashlib import json import os import sys import time +from typing import List, Optional import unittest try: from unittest.mock import patch, ANY, mock_open, Mock @@ -52,15 +54,23 @@ def test_helper_class_should_be_interchangable_with_dict_which_could_be_loaded_f class ClientTestCase(unittest.TestCase): maxDiff = None - def setUp(self): - self.app = ManagedIdentityClient( + def _build_app( + self, + *, + client_capabilities: Optional[List[str]] = None, + ): + return ManagedIdentityClient( { # Here we test it with the raw dict form, to test that # the client has no hard dependency on ManagedIdentity object "ManagedIdentityIdType": "SystemAssigned", "Id": None, }, http_client=requests.Session(), + client_capabilities=client_capabilities, ) + def setUp(self): + self.app = self._build_app() + def test_error_out_on_invalid_input(self): with self.assertRaises(ManagedIdentityError): ManagedIdentityClient({"foo": "bar"}, http_client=requests.Session()) @@ -79,7 +89,13 @@ def assertCacheStatus(self, app): "Should have expected client_id") self.assertEqual("managed_identity", at["realm"], "Should have expected realm") - def _test_happy_path(self, app, mocked_http, expires_in, resource="R"): + def _test_happy_path( + self, app, mocked_http, expires_in, *, resource="R", claims_challenge=None, + ): + """It tests a normal token request that is expected to hit IdP, + a subsequent same token request that is expected to hit cache, + and then a request with claims_challenge that shall hit IdP again. + """ result = app.acquire_token_for_client(resource=resource) mocked_http.assert_called() call_count = mocked_http.call_count @@ -115,7 +131,8 @@ def _test_happy_path(self, app, mocked_http, expires_in, resource="R"): expected_refresh_on - 5 < result["refresh_on"] <= expected_refresh_on, "Should have a refresh_on time around the middle of the token's life") - result = app.acquire_token_for_client(resource=resource, claims_challenge="foo") + result = app.acquire_token_for_client( + resource=resource, claims_challenge=claims_challenge or "placeholder") self.assertEqual("identity_provider", result["token_source"], "Should miss cache") @@ -132,6 +149,9 @@ def _test_happy_path(self) -> callable: def test_happy_path_of_vm(self): self._test_happy_path().assert_called_with( + # The last call contained claims_challenge + # but since IMDS doesn't support token_sha256_to_refresh, + # the request shall remain the same as before 'http://169.254.169.254/metadata/identity/oauth2/token', params={'api-version': '2018-02-01', 'resource': 'R'}, headers={'Metadata': 'true'}, @@ -244,19 +264,46 @@ def test_machine_learning_error_should_be_normalized(self): "IDENTITY_SERVER_THUMBPRINT": "bar", }) class ServiceFabricTestCase(ClientTestCase): + access_token = "AT" + access_token_sha256 = hashlib.sha256(access_token.encode()).hexdigest() - def _test_happy_path(self, app): + def _test_happy_path(self, app, *, claims_challenge=None) -> callable: expires_in = 1234 with patch.object(app._http_client, "get", return_value=MinimalResponse( status_code=200, - text='{"access_token": "AT", "expires_on": %s, "resource": "R", "token_type": "Bearer"}' % ( - int(time.time()) + expires_in), + text='{"access_token": "%s", "expires_on": %s, "resource": "R", "token_type": "Bearer"}' % ( + self.access_token, int(time.time()) + expires_in), )) as mocked_method: super(ServiceFabricTestCase, self)._test_happy_path( - app, mocked_method, expires_in) + app, mocked_method, expires_in, claims_challenge=claims_challenge) + return mocked_method - def test_happy_path(self): - self._test_happy_path(self.app) + def test_happy_path_with_client_capabilities_should_relay_capabilities(self): + self._test_happy_path(self._build_app(client_capabilities=["foo", "bar"])).assert_called_with( + 'http://localhost', + params={ + 'api-version': '2019-07-01-preview', + 'resource': 'R', + 'token_sha256_to_refresh': self.access_token_sha256, + "xms_cc": "foo,bar", + }, + headers={'Secret': 'foo'}, + ) + + def test_happy_path_with_claim_challenge_should_send_sha256_to_provider(self): + self._test_happy_path( + self._build_app(client_capabilities=[]), # Test empty client_capabilities + claims_challenge='{"access_token": {"nbf": {"essential": true, "value": "1563308371"}}}', + ).assert_called_with( + 'http://localhost', + params={ + 'api-version': '2019-07-01-preview', + 'resource': 'R', + 'token_sha256_to_refresh': self.access_token_sha256, + # There is no xms_cc in this case + }, + headers={'Secret': 'foo'}, + ) def test_unified_api_service_should_ignore_unnecessary_client_id(self): self._test_happy_path(ManagedIdentityClient( From 45e39669ca8e3fd768610d748235c3775ca4db84 Mon Sep 17 00:00:00 2001 From: Paul Van Eck Date: Thu, 3 Apr 2025 01:20:16 +0000 Subject: [PATCH 235/262] Update deprecated TokenCache API usage There was a place in `ManagedIdentityClient` where the `TokenCache.find` method was still being used, leading to some deprecation warnings being printed. This updates that to use `TokenCache.search`. Signed-off-by: Paul Van Eck --- msal/managed_identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 230ff2f9..866d379a 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -298,7 +298,7 @@ def acquire_token_for_client( now = time.time() if True: # Attempt cache search even if receiving claims_challenge, # because we want to locate the existing token (if any) and refresh it - matches = self._token_cache.find( + matches = self._token_cache.search( self._token_cache.CredentialType.ACCESS_TOKEN, target=[resource], query=dict( From 28a67a7ab551e6010a7b69c7c9d7b68aa69f69fc Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 5 Nov 2024 12:34:56 -0800 Subject: [PATCH 236/262] Use v1.29+ TokenCache.search() instead of find() --- tests/test_application.py | 59 ++++++++++++++++++++------------------- tests/test_token_cache.py | 10 ++++--- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/tests/test_application.py b/tests/test_application.py index 0c7f2d29..16e512c4 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -153,8 +153,8 @@ def tester(url, data=None, **kwargs): return MinimalResponse(status_code=400, text=error_response) app._acquire_token_silent_by_finding_rt_belongs_to_me_or_my_family( app.authority, self.scopes, self.account, post=tester) - self.assertNotEqual([], app.token_cache.find( - msal.TokenCache.CredentialType.REFRESH_TOKEN, query={"secret": self.frt}), + self.assertIsNotNone(next(app.token_cache.search( + msal.TokenCache.CredentialType.REFRESH_TOKEN, query={"secret": self.frt}), None), "The FRT should not be removed from the cache") def test_known_orphan_app_will_skip_frt_and_only_use_its_own_rt(self): @@ -187,11 +187,11 @@ def tester(url, data=None, **kwargs): app.authority, self.scopes, self.account, post=tester) logger.debug("%s.cache = %s", self.id(), self.cache.serialize()) self.assertEqual("at", at.get("access_token"), "New app should get a new AT") - app_metadata = app.token_cache.find( + app_metadata = next(app.token_cache.search( msal.TokenCache.CredentialType.APP_METADATA, - query={"client_id": app.client_id}) - self.assertNotEqual([], app_metadata, "Should record new app's metadata") - self.assertEqual("1", app_metadata[0].get("family_id"), + query={"client_id": app.client_id}), None) + self.assertIsNotNone(app_metadata, "Should record new app's metadata") + self.assertEqual("1", app_metadata.get("family_id"), "The new family app should be recorded as in the same family") # Known family app will simply use FRT, which is largely the same as this one @@ -218,25 +218,25 @@ def test_family_app_remove_account(self): account = app.get_accounts()[0] mine = {"home_account_id": account["home_account_id"]} - self.assertNotEqual([], self.cache.find( - self.cache.CredentialType.ACCESS_TOKEN, query=mine)) - self.assertNotEqual([], self.cache.find( - self.cache.CredentialType.REFRESH_TOKEN, query=mine)) - self.assertNotEqual([], self.cache.find( - self.cache.CredentialType.ID_TOKEN, query=mine)) - self.assertNotEqual([], self.cache.find( - self.cache.CredentialType.ACCOUNT, query=mine)) + self.assertIsNotNone(next(self.cache.search( + self.cache.CredentialType.ACCESS_TOKEN, query=mine), None)) + self.assertIsNotNone(next(self.cache.search( + self.cache.CredentialType.REFRESH_TOKEN, query=mine), None)) + self.assertIsNotNone(next(self.cache.search( + self.cache.CredentialType.ID_TOKEN, query=mine), None)) + self.assertIsNotNone(next(self.cache.search( + self.cache.CredentialType.ACCOUNT, query=mine), None)) app.remove_account(account) - self.assertEqual([], self.cache.find( - self.cache.CredentialType.ACCESS_TOKEN, query=mine)) - self.assertEqual([], self.cache.find( - self.cache.CredentialType.REFRESH_TOKEN, query=mine)) - self.assertEqual([], self.cache.find( - self.cache.CredentialType.ID_TOKEN, query=mine)) - self.assertEqual([], self.cache.find( - self.cache.CredentialType.ACCOUNT, query=mine)) + self.assertIsNone(next(self.cache.search( + self.cache.CredentialType.ACCESS_TOKEN, query=mine), None)) + self.assertIsNone(next(self.cache.search( + self.cache.CredentialType.REFRESH_TOKEN, query=mine), None)) + self.assertIsNone(next(self.cache.search( + self.cache.CredentialType.ID_TOKEN, query=mine), None)) + self.assertIsNone(next(self.cache.search( + self.cache.CredentialType.ACCOUNT, query=mine), None)) class TestClientApplicationForAuthorityMigration(unittest.TestCase): @@ -711,14 +711,14 @@ def test_remove_tokens_for_client_should_remove_client_tokens_only(self): cca = msal.ConfidentialClientApplication( "client_id", client_credential="secret", authority="https://login.microsoftonline.com/microsoft.onmicrosoft.com") - self.assertEqual( - 0, len(cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN))) + self.assertIsNone(next(cca.token_cache.search( + msal.TokenCache.CredentialType.ACCESS_TOKEN), None)) cca.acquire_token_for_client( ["scope"], post=lambda url, **kwargs: MinimalResponse( status_code=200, text=json.dumps({"access_token": "AT for client"}))) - self.assertEqual( - 1, len(cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN))) + self.assertEqual(1, len(list(cca.token_cache.search( + msal.TokenCache.CredentialType.ACCESS_TOKEN)))) cca.acquire_token_by_username_password( "johndoe", "password", ["scope"], post=lambda url, **kwargs: MinimalResponse( @@ -726,10 +726,11 @@ def test_remove_tokens_for_client_should_remove_client_tokens_only(self): access_token=at_for_user, expires_in=3600, uid="uid", utid="utid", # This populates home_account_id )))) - self.assertEqual( - 2, len(cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN))) + self.assertEqual(2, len(list(cca.token_cache.search( + msal.TokenCache.CredentialType.ACCESS_TOKEN)))) cca.remove_tokens_for_client() - remaining_tokens = cca.token_cache.find(msal.TokenCache.CredentialType.ACCESS_TOKEN) + remaining_tokens = list(cca.token_cache.search( + msal.TokenCache.CredentialType.ACCESS_TOKEN)) self.assertEqual(1, len(remaining_tokens)) self.assertEqual(at_for_user, remaining_tokens[0].get("secret")) diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py index 494d6daf..5310b789 100644 --- a/tests/test_token_cache.py +++ b/tests/test_token_cache.py @@ -2,6 +2,7 @@ import base64 import json import time +import warnings from msal.token_cache import TokenCache, SerializableTokenCache from tests import unittest @@ -83,10 +84,11 @@ def testAddByAad(self): } self.assertEqual(access_token_entry, self.cache._cache["AccessToken"].get( self.at_key_maker(**access_token_entry))) - self.assertIn( - access_token_entry, - self.cache.find(self.cache.CredentialType.ACCESS_TOKEN, now=now), - "find(..., query=None) should not crash, even though MSAL does not use it") + with warnings.catch_warnings(): + self.assertIn( + access_token_entry, + self.cache.find(self.cache.CredentialType.ACCESS_TOKEN, now=now), + "find(..., query=None) should not crash, even though MSAL does not use it") self.assertEqual( { 'client_id': 'my_client_id', From 4eb7bd105b2281a253ab94d4b81f56ca1504f500 Mon Sep 17 00:00:00 2001 From: Dharshan BJ Date: Mon, 7 Apr 2025 12:48:35 -0700 Subject: [PATCH 237/262] Enable broker support on Linux for WSL (#766) * Enable broker support on Linux * update version number * Update sample/interactive_sample.py Co-authored-by: Ray Luo * Update msal/application.py Co-authored-by: Ray Luo * Update tests/broker-test.py Co-authored-by: Ray Luo * revert back release version bump * address comments * address comment * update approximate version hint * update * Update msal/application.py Co-authored-by: Ray Luo * Address comments * Update * Add enable_broker_on_wsl flag * Address comments * Update msal/__main__.py Co-authored-by: Ray Luo * Update tests/test_e2e.py Co-authored-by: Ray Luo * Update msal/application.py Co-authored-by: Ray Luo * Update msal/application.py Co-authored-by: Ray Luo * Bump up msal py version to 1.33 * Update msal/application.py Co-authored-by: Ray Luo --------- Co-authored-by: Ray Luo --- msal/__main__.py | 2 ++ msal/application.py | 47 +++++++++++++++++++++++++++++------- msal/broker.py | 1 + sample/interactive_sample.py | 3 ++- setup.cfg | 6 +++-- tests/broker-test.py | 5 +++- tests/test_e2e.py | 2 ++ 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/msal/__main__.py b/msal/__main__.py index 0c6c59f7..4107fd89 100644 --- a/msal/__main__.py +++ b/msal/__main__.py @@ -300,6 +300,8 @@ def _main(): instance_discovery=instance_discovery, enable_broker_on_windows=enable_broker, enable_broker_on_mac=enable_broker, + enable_broker_on_linux=enable_broker, + enable_broker_on_wsl=enable_broker, enable_pii_log=enable_pii_log, token_cache=global_cache, ) if not is_cca else msal.ConfidentialClientApplication( diff --git a/msal/application.py b/msal/application.py index 31e1105d..a335e3f2 100644 --- a/msal/application.py +++ b/msal/application.py @@ -21,7 +21,7 @@ from .throttled_http_client import ThrottledHttpClient from .cloudshell import _is_running_in_cloud_shell from .sku import SKU, __version__ - +from .oauth2cli.authcode import is_wsl logger = logging.getLogger(__name__) @@ -164,6 +164,8 @@ def _preferred_browser(): pass # We may still proceed return None +def _is_ssh_cert_or_pop_request(token_type, auth_scheme) -> bool: + return token_type == "ssh-cert" or token_type == "pop" or isinstance(auth_scheme, msal.auth_scheme.PopAuthScheme) class _ClientWithCcsRoutingInfo(Client): @@ -710,7 +712,7 @@ def _decide_broker(self, allow_broker, enable_pii_log): def is_pop_supported(self): """Returns True if this client supports Proof-of-Possession Access Token.""" - return self._enable_broker + return self._enable_broker and sys.platform in ("win32", "darwin") def _decorate_scope( self, scopes, @@ -1582,10 +1584,12 @@ def _acquire_token_silent_from_cache_and_possibly_refresh_it( raise ValueError("auth_scheme is not supported in Cloud Shell") return self._acquire_token_by_cloud_shell(scopes, data=data) + is_ssh_cert_or_pop_request = _is_ssh_cert_or_pop_request(data.get("token_type"), auth_scheme) + if self._enable_broker and account and account.get("account_source") in ( _GRANT_TYPE_BROKER, # Broker successfully established this account previously. None, # Unknown data from older MSAL. Broker might still work. - ): + ) and (sys.platform in ("win32", "darwin") or not is_ssh_cert_or_pop_request): from .broker import _acquire_token_silently response = _acquire_token_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), @@ -1832,7 +1836,7 @@ def acquire_token_by_username_password( """ claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) - if self._enable_broker: + if self._enable_broker and sys.platform in ("win32", "darwin"): from .broker import _signin_silently response = _signin_silently( "https://{}/{}".format(self.authority.instance, self.authority.tenant), @@ -1929,13 +1933,15 @@ def __init__( *, enable_broker_on_windows=None, enable_broker_on_mac=None, + enable_broker_on_linux=None, + enable_broker_on_wsl=None, **kwargs): """Same as :func:`ClientApplication.__init__`, except that ``client_credential`` parameter shall remain ``None``. .. note:: - You may set enable_broker_on_windows and/or enable_broker_on_mac to True. + You may set enable_broker_on_windows and/or enable_broker_on_mac and/or enable_broker_on_linux and/or enable_broker_on_wsl to True. **What is a broker, and why use it?** @@ -1963,9 +1969,11 @@ def __init__( if your app is expected to run on Windows 10+ * ``msauth.com.msauth.unsignedapp://auth`` if your app is expected to run on Mac + * ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` + if your app is expected to run on Linux, especially WSL 2. installed broker dependency, - e.g. ``pip install msal[broker]>=1.31,<2``. + e.g. ``pip install msal[broker]>=1.33,<2``. 3. tested with ``acquire_token_interactive()`` and ``acquire_token_silent()``. @@ -2003,12 +2011,29 @@ def __init__( This parameter defaults to None, which means MSAL will not utilize a broker. New in MSAL Python 1.31.0. + + :param boolean enable_broker_on_linux: + This setting is only effective if your app is running on Linux, including WSL. + This parameter defaults to None, which means MSAL will not utilize a broker. + + New in MSAL Python 1.33.0. + + :param boolean enable_broker_on_wsl: + This setting is only effective if your app is running on WSL. + This parameter defaults to None, which means MSAL will not utilize a broker. + + New in MSAL Python 1.33.0. """ if client_credential is not None: raise ValueError("Public Client should not possess credentials") + self._enable_broker = bool( enable_broker_on_windows and sys.platform == "win32" - or enable_broker_on_mac and sys.platform == "darwin") + or enable_broker_on_mac and sys.platform == "darwin" + or enable_broker_on_linux and sys.platform == "linux" + or enable_broker_on_wsl and is_wsl() + ) + super(PublicClientApplication, self).__init__( client_id, client_credential=None, **kwargs) @@ -2137,6 +2162,8 @@ def acquire_token_interactive( False ) and data.get("token_type") != "ssh-cert" # Work around a known issue as of PyMsalRuntime 0.8 self._validate_ssh_cert_input_data(data) + is_ssh_cert_or_pop_request = _is_ssh_cert_or_pop_request(data.get("token_type"), auth_scheme) + if not on_before_launching_ui: on_before_launching_ui = lambda **kwargs: None if _is_running_in_cloud_shell() and prompt == "none": @@ -2145,7 +2172,7 @@ def acquire_token_interactive( return self._acquire_token_by_cloud_shell(scopes, data=data) claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) - if self._enable_broker: + if self._enable_broker and (sys.platform in ("win32", "darwin") or not is_ssh_cert_or_pop_request): if parent_window_handle is None: raise ValueError( "parent_window_handle is required when you opted into using broker. " @@ -2170,7 +2197,9 @@ def acquire_token_interactive( ) return self._process_broker_response(response, scopes, data) - if auth_scheme: + if isinstance(auth_scheme, msal.auth_scheme.PopAuthScheme) and sys.platform == "linux": + raise ValueError("POP is not supported on Linux") + elif auth_scheme: raise ValueError(self._AUTH_SCHEME_UNSUPPORTED) on_before_launching_ui(ui="browser") telemetry_context = self._build_telemetry_context( diff --git a/msal/broker.py b/msal/broker.py index f4d71e11..1e794b36 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -27,6 +27,7 @@ min_ver = { "win32": "1.20", "darwin": "1.31", + "linux": "1.33", }.get(sys.platform) if min_ver: raise ImportError( diff --git a/sample/interactive_sample.py b/sample/interactive_sample.py index 8c3f2df9..11f823f0 100644 --- a/sample/interactive_sample.py +++ b/sample/interactive_sample.py @@ -47,7 +47,8 @@ oidc_authority=os.getenv('OIDC_AUTHORITY'), # For External ID with custom domain #enable_broker_on_windows=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already #enable_broker_on_mac=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already - + #enable_broker_on_linux=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already + #enable_broker_on_wsl=True, # Opted in. You will be guided to meet the prerequisites, if your app hasn't already token_cache=global_token_cache, # Let this app (re)use an existing token cache. # If absent, ClientApplication will create its own empty token cache ) diff --git a/setup.cfg b/setup.cfg index 6dfcfc7b..16f7cca3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,9 +62,11 @@ broker = # most existing MSAL Python apps do not have the redirect_uri needed by broker. # # We need pymsalruntime.CallbackData introduced in PyMsalRuntime 0.14 - pymsalruntime>=0.14,<0.18; python_version>='3.6' and platform_system=='Windows' + pymsalruntime>=0.14,<0.19; python_version>='3.6' and platform_system=='Windows' # On Mac, PyMsalRuntime 0.17+ is expected to support SSH cert and ROPC - pymsalruntime>=0.17,<0.18; python_version>='3.8' and platform_system=='Darwin' + pymsalruntime>=0.17,<0.19; python_version>='3.8' and platform_system=='Darwin' + # PyMsalRuntime 0.18+ is expected to support broker on Linux + pymsalruntime>=0.18,<0.19; python_version>='3.8' and platform_system=='Linux' [options.packages.find] exclude = diff --git a/tests/broker-test.py b/tests/broker-test.py index cdcc4817..fa4d916c 100644 --- a/tests/broker-test.py +++ b/tests/broker-test.py @@ -39,7 +39,10 @@ _AZURE_CLI, authority="https://login.microsoftonline.com/organizations", enable_broker_on_mac=True, - enable_broker_on_windows=True) + enable_broker_on_windows=True, + enable_broker_on_linux=True, + enable_broker_on_wsl=True, + ) def interactive_and_silent(scopes, auth_scheme, data, expected_token_type): print("An account picker shall be pop up, possibly behind this console. Continue from there.") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 28c73abc..dde26f0d 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -193,6 +193,8 @@ def _build_app(cls, http_client=http_client or MinimalHttpClient(), enable_broker_on_windows=_PYMSALRUNTIME_INSTALLED, enable_broker_on_mac=_PYMSALRUNTIME_INSTALLED, + enable_broker_on_linux=_PYMSALRUNTIME_INSTALLED, + enable_broker_on_wsl=_PYMSALRUNTIME_INSTALLED, ) def _test_username_password(self, From 1321e3793eebd35aac7b80b64e3a6aa49a32a51a Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Tue, 15 Apr 2025 01:14:14 +0300 Subject: [PATCH 238/262] Fix username/password validation in broker test (#807) * Fix username/password assert in broker test Signed-off-by: Emmanuel Ferdman * Update tests/broker-test.py Co-authored-by: Ray Luo --------- Signed-off-by: Emmanuel Ferdman Co-authored-by: Ray Luo --- tests/broker-test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/broker-test.py b/tests/broker-test.py index fa4d916c..8c2d5503 100644 --- a/tests/broker-test.py +++ b/tests/broker-test.py @@ -71,10 +71,10 @@ def test_broker_username_password(scopes, expected_token_type): print("Testing broker username password flows by using accounts in local .env") username = os.getenv("BROKER_TEST_ACCOUNT") or input("Input test account for broker test: ") password = os.getenv("BROKER_TEST_ACCOUNT_PASSWORD") or getpass.getpass("Input test account's password: ") - assert(username and password, "You need to provide a test account and its password") + assert username and password, "You need to provide a test account and its password" result = pca.acquire_token_by_username_password(username, password, scopes) _assert(result, expected_token_type) - assert(result.get("token_source") == "broker") + assert result.get("token_source") == "broker" print("Username password test succeeds.") def _assert(result, expected_token_type): From 29d1ac1558d6f87754fef71ed950bd806919d803 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 17 Apr 2025 12:34:56 -0700 Subject: [PATCH 239/262] Only cache desirable data in http cache --- msal/application.py | 1 + msal/managed_identity.py | 9 ++-- msal/throttled_http_client.py | 72 +++++++++++++++++---------- tests/test_mi.py | 35 +++++++++++++- tests/test_throttled_http_client.py | 75 ++++++++++++++++++++++++++--- tox.ini | 28 +++++++++++ 6 files changed, 180 insertions(+), 40 deletions(-) create mode 100644 tox.ini diff --git a/msal/application.py b/msal/application.py index 25a0db2b..4310a1f6 100644 --- a/msal/application.py +++ b/msal/application.py @@ -499,6 +499,7 @@ def __init__( except ( FileNotFoundError, # Or IOError in Python 2 pickle.UnpicklingError, # A corrupted http cache file + AttributeError, # Cache created by a different version of MSAL ): persisted_http_cache = {} # Recover by starting afresh atexit.register(lambda: pickle.dump( diff --git a/msal/managed_identity.py b/msal/managed_identity.py index 6f85571d..692bb7ad 100644 --- a/msal/managed_identity.py +++ b/msal/managed_identity.py @@ -112,8 +112,8 @@ def __init__(self, *, client_id=None, resource_id=None, object_id=None): class _ThrottledHttpClient(ThrottledHttpClientBase): - def __init__(self, http_client, **kwargs): - super(_ThrottledHttpClient, self).__init__(http_client, **kwargs) + def __init__(self, *args, **kwargs): + super(_ThrottledHttpClient, self).__init__(*args, **kwargs) self.get = IndividualCache( # All MIs (except Cloud Shell) use GETs mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "REQ {} hash={} 429/5xx/Retry-After".format( @@ -124,7 +124,7 @@ def __init__(self, http_client, **kwargs): str(kwargs.get("params")) + str(kwargs.get("data"))), ), expires_in=RetryAfterParser(5).parse, # 5 seconds default for non-PCA - )(http_client.get) + )(self.get) # Note: Decorate the parent get(), not the http_client.get() class ManagedIdentityClient(object): @@ -233,8 +233,7 @@ def __init__( # (especially for 410 which was supposed to be a permanent failure). # 2. MI on Service Fabric specifically suggests to not retry on 404. # ( https://learn.microsoft.com/en-us/azure/service-fabric/how-to-managed-cluster-managed-identity-service-fabric-app-code#error-handling ) - http_client.http_client # Patch the raw (unpatched) http client - if isinstance(http_client, ThrottledHttpClientBase) else http_client, + http_client, http_cache=http_cache, ) self._token_cache = token_cache or TokenCache() diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index ebad76c7..fd0c9ad5 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -3,6 +3,8 @@ from .individual_cache import _IndividualCache as IndividualCache from .individual_cache import _ExpiringMapping as ExpiringMapping +from .oauth2cli.http import Response +from .exceptions import MsalServiceError # https://datatracker.ietf.org/doc/html/rfc8628#section-3.4 @@ -10,6 +12,7 @@ class RetryAfterParser(object): + FIELD_NAME_LOWER = "Retry-After".lower() def __init__(self, default_value=None): self._default_value = 5 if default_value is None else default_value @@ -20,9 +23,9 @@ def parse(self, *, result, **ignored): # Historically, MSAL's HttpResponse does not always have headers response, "headers", {}).items()} if not (response.status_code == 429 or response.status_code >= 500 - or "retry-after" in lowercase_headers): + or self.FIELD_NAME_LOWER in lowercase_headers): return 0 # Quick exit - retry_after = lowercase_headers.get("retry-after", self._default_value) + retry_after = lowercase_headers.get(self.FIELD_NAME_LOWER, self._default_value) try: # AAD's retry_after uses integer format only # https://stackoverflow.microsoft.com/questions/264931/264932 @@ -37,16 +40,41 @@ def _extract_data(kwargs, key, default=None): return data.get(key) if isinstance(data, dict) else default +class NormalizedResponse(Response): + """A http response with the shape defined in Response, + but contains only the data we will store in cache. + """ + def __init__(self, raw_response): + super().__init__() + self.status_code = raw_response.status_code + self.text = raw_response.text + self.headers = { # Only keep the headers which ThrottledHttpClient cares about + k: v for k, v in raw_response.headers.items() + if k.lower() == RetryAfterParser.FIELD_NAME_LOWER + } + + ## Note: Don't use the following line, + ## because when being pickled, it will indirectly pickle the whole raw_response + # self.raise_for_status = raw_response.raise_for_status + def raise_for_status(self): + if self.status_code >= 400: + raise MsalServiceError("HTTP Error: {}".format(self.status_code)) + + class ThrottledHttpClientBase(object): """Throttle the given http_client by storing and retrieving data from cache. - This wrapper exists so that our patching post() and get() would prevent - re-patching side effect when/if same http_client being reused. + This base exists so that: + 1. These base post() and get() will return a NormalizedResponse + 2. The base __init__() will NOT re-throttle even if caller accidentally nested ThrottledHttpClient. - The subclass should implement post() and/or get() + Subclasses shall only need to dynamically decorate their post() and get() methods + in their __init__() method. """ def __init__(self, http_client, *, http_cache=None): - self.http_client = http_client + self.http_client = http_client.http_client if isinstance( + # If it is already a ThrottledHttpClientBase, we use its raw (unthrottled) http client + http_client, ThrottledHttpClientBase) else http_client self._expiring_mapping = ExpiringMapping( # It will automatically clean up mapping=http_cache if http_cache is not None else {}, capacity=1024, # To prevent cache blowing up especially for CCA @@ -54,10 +82,10 @@ def __init__(self, http_client, *, http_cache=None): ) def post(self, *args, **kwargs): - return self.http_client.post(*args, **kwargs) + return NormalizedResponse(self.http_client.post(*args, **kwargs)) def get(self, *args, **kwargs): - return self.http_client.get(*args, **kwargs) + return NormalizedResponse(self.http_client.get(*args, **kwargs)) def close(self): return self.http_client.close() @@ -68,12 +96,11 @@ def _hash(raw): class ThrottledHttpClient(ThrottledHttpClientBase): - def __init__(self, http_client, *, default_throttle_time=None, **kwargs): - super(ThrottledHttpClient, self).__init__(http_client, **kwargs) - - _post = http_client.post # We'll patch _post, and keep original post() intact - - _post = IndividualCache( + """A throttled http client that is used by MSAL's non-managed identity clients.""" + def __init__(self, *args, default_throttle_time=None, **kwargs): + """Decorate self.post() and self.get() dynamically""" + super(ThrottledHttpClient, self).__init__(*args, **kwargs) + self.post = IndividualCache( # Internal specs requires throttling on at least token endpoint, # here we have a generic patch for POST on all endpoints. mapping=self._expiring_mapping, @@ -91,9 +118,9 @@ def __init__(self, http_client, *, default_throttle_time=None, **kwargs): _extract_data(kwargs, "username")))), # "account" of ROPC ), expires_in=RetryAfterParser(default_throttle_time or 5).parse, - )(_post) + )(self.post) - _post = IndividualCache( # It covers the "UI required cache" + self.post = IndividualCache( # It covers the "UI required cache" mapping=self._expiring_mapping, key_maker=lambda func, args, kwargs: "POST {} hash={} 400".format( args[0], # It is the url, typically containing authority and tenant @@ -125,12 +152,10 @@ def __init__(self, http_client, *, default_throttle_time=None, **kwargs): isinstance(kwargs.get("data"), dict) and kwargs["data"].get("grant_type") == DEVICE_AUTH_GRANT ) - and "retry-after" not in set( # Leave it to the Retry-After decorator + and RetryAfterParser.FIELD_NAME_LOWER not in set( # Otherwise leave it to the Retry-After decorator h.lower() for h in getattr(result, "headers", {}).keys()) else 0, - )(_post) - - self.post = _post + )(self.post) self.get = IndividualCache( # Typically those discovery GETs mapping=self._expiring_mapping, @@ -140,9 +165,4 @@ def __init__(self, http_client, *, default_throttle_time=None, **kwargs): ), expires_in=lambda result=None, **ignored: 3600*24 if 200 <= result.status_code < 300 else 0, - )(http_client.get) - - # The following 2 methods have been defined dynamically by __init__() - #def post(self, *args, **kwargs): pass - #def get(self, *args, **kwargs): pass - + )(self.get) diff --git a/tests/test_mi.py b/tests/test_mi.py index a7c2cb6c..e065899c 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -9,7 +9,8 @@ from mock import patch, ANY, mock_open, Mock import requests -from tests.http_client import MinimalResponse +from tests.test_throttled_http_client import ( + MinimalResponse, ThrottledHttpClientBaseTestCase, DummyHttpClient) from msal import ( SystemAssignedManagedIdentity, UserAssignedManagedIdentity, ManagedIdentityClient, @@ -17,6 +18,7 @@ ArcPlatformNotSupportedError, ) from msal.managed_identity import ( + _ThrottledHttpClient, _supported_arc_platforms_and_their_prefixes, get_managed_identity_source, APP_SERVICE, @@ -49,6 +51,37 @@ def test_helper_class_should_be_interchangable_with_dict_which_could_be_loaded_f {"ManagedIdentityIdType": "SystemAssigned", "Id": None}) +class ThrottledHttpClientTestCase(ThrottledHttpClientBaseTestCase): + def test_throttled_http_client_should_not_alter_original_http_client(self): + self.assertNotAlteringOriginalHttpClient(_ThrottledHttpClient) + + def test_throttled_http_client_should_not_cache_successful_http_response(self): + http_cache = {} + http_client=DummyHttpClient( + status_code=200, + response_text='{"access_token": "AT", "expires_in": "1234", "resource": "R"}', + ) + app = ManagedIdentityClient( + SystemAssignedManagedIdentity(), http_client=http_client, http_cache=http_cache) + result = app.acquire_token_for_client(resource="R") + self.assertEqual("AT", result["access_token"]) + self.assertEqual({}, http_cache, "Should not cache successful http response") + + def test_throttled_http_client_should_cache_unsuccessful_http_response(self): + http_cache = {} + http_client=DummyHttpClient( + status_code=400, + response_headers={"Retry-After": "1"}, + response_text='{"error": "invalid_request"}', + ) + app = ManagedIdentityClient( + SystemAssignedManagedIdentity(), http_client=http_client, http_cache=http_cache) + result = app.acquire_token_for_client(resource="R") + self.assertEqual("invalid_request", result["error"]) + self.assertNotEqual({}, http_cache, "Should cache unsuccessful http response") + self.assertCleanPickle(http_cache) + + class ClientTestCase(unittest.TestCase): maxDiff = None diff --git a/tests/test_throttled_http_client.py b/tests/test_throttled_http_client.py index 3994719d..c15a7877 100644 --- a/tests/test_throttled_http_client.py +++ b/tests/test_throttled_http_client.py @@ -1,27 +1,43 @@ # Test cases for https://identitydivision.visualstudio.com/devex/_git/AuthLibrariesApiReview?version=GBdev&path=%2FService%20protection%2FIntial%20set%20of%20protection%20measures.md&_a=preview&anchor=common-test-cases +import pickle from time import sleep from random import random import logging -from msal.throttled_http_client import ThrottledHttpClient + +from msal.throttled_http_client import ( + ThrottledHttpClientBase, ThrottledHttpClient, NormalizedResponse) + from tests import unittest -from tests.http_client import MinimalResponse +from tests.http_client import MinimalResponse as _MinimalResponse logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) +class MinimalResponse(_MinimalResponse): + SIGNATURE = str(random()).encode("utf-8") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._ = ( # Only an instance attribute will be stored in pickled instance + self.__class__.SIGNATURE) # Useful for testing its presence in pickled instance + + class DummyHttpClient(object): - def __init__(self, status_code=None, response_headers=None): + def __init__(self, status_code=None, response_headers=None, response_text=None): self._status_code = status_code self._response_headers = response_headers + self._response_text = response_text def _build_dummy_response(self): return MinimalResponse( status_code=self._status_code, headers=self._response_headers, - text=random(), # So that we'd know whether a new response is received - ) + text=self._response_text if self._response_text is not None else str( + random() # So that we'd know whether a new response is received + ), + ) def post(self, url, params=None, data=None, headers=None, **kwargs): return self._build_dummy_response() @@ -37,19 +53,54 @@ class CloseMethodCalled(Exception): pass -class TestHttpDecoration(unittest.TestCase): +class ThrottledHttpClientBaseTestCase(unittest.TestCase): - def test_throttled_http_client_should_not_alter_original_http_client(self): + def assertCleanPickle(self, obj): + self.assertTrue(bool(obj), "The object should not be empty") + self.assertNotIn( + MinimalResponse.SIGNATURE, pickle.dumps(obj), + "A pickled object should not contain undesirable data") + + def assertValidResponse(self, response): + self.assertIsInstance(response, NormalizedResponse) + self.assertCleanPickle(response) + + def test_pickled_minimal_response_should_contain_signature(self): + self.assertIn(MinimalResponse.SIGNATURE, pickle.dumps(MinimalResponse( + status_code=200, headers={}, text="foo"))) + + def test_throttled_http_client_base_response_should_not_contain_signature(self): + http_client = ThrottledHttpClientBase(DummyHttpClient(status_code=200)) + response = http_client.post("https://example.com") + self.assertValidResponse(response) + + def assertNotAlteringOriginalHttpClient(self, ThrottledHttpClientClass): original_http_client = DummyHttpClient() original_get = original_http_client.get original_post = original_http_client.post - throttled_http_client = ThrottledHttpClient(original_http_client) + throttled_http_client = ThrottledHttpClientClass(original_http_client) goal = """The implementation should wrap original http_client and keep it intact, instead of monkey-patching it""" self.assertNotEqual(throttled_http_client, original_http_client, goal) self.assertEqual(original_post, original_http_client.post) self.assertEqual(original_get, original_http_client.get) + def test_throttled_http_client_base_should_not_alter_original_http_client(self): + self.assertNotAlteringOriginalHttpClient(ThrottledHttpClientBase) + + def test_throttled_http_client_base_should_not_nest_http_client(self): + original_http_client = DummyHttpClient() + throttled_http_client = ThrottledHttpClientBase(original_http_client) + self.assertIs(original_http_client, throttled_http_client.http_client) + nested_throttled_http_client = ThrottledHttpClientBase(throttled_http_client) + self.assertIs(original_http_client, nested_throttled_http_client.http_client) + + +class ThrottledHttpClientTestCase(ThrottledHttpClientBaseTestCase): + + def test_throttled_http_client_should_not_alter_original_http_client(self): + self.assertNotAlteringOriginalHttpClient(ThrottledHttpClient) + def _test_RetryAfter_N_seconds_should_keep_entry_for_N_seconds( self, http_client, retry_after): http_cache = {} @@ -112,15 +163,23 @@ def test_one_invalid_grant_should_block_a_similar_request(self): http_client = DummyHttpClient( status_code=400) # It covers invalid_grant and interaction_required http_client = ThrottledHttpClient(http_client, http_cache=http_cache) + resp1 = http_client.post("https://example.com", data={"claims": "foo"}) logger.debug(http_cache) + self.assertValidResponse(resp1) resp1_again = http_client.post("https://example.com", data={"claims": "foo"}) + self.assertValidResponse(resp1_again) self.assertEqual(resp1.text, resp1_again.text, "Should return a cached response") + resp2 = http_client.post("https://example.com", data={"claims": "bar"}) + self.assertValidResponse(resp2) self.assertNotEqual(resp1.text, resp2.text, "Should return a new response") resp2_again = http_client.post("https://example.com", data={"claims": "bar"}) + self.assertValidResponse(resp2_again) self.assertEqual(resp2.text, resp2_again.text, "Should return a cached response") + self.assertCleanPickle(http_cache) + def test_one_foci_app_recovering_from_invalid_grant_should_also_unblock_another(self): """ Need not test multiple FOCI app's acquire_token_silent() here. By design, diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..dffd5110 --- /dev/null +++ b/tox.ini @@ -0,0 +1,28 @@ +[tox] +env_list = + py3 +minversion = 4.21.2 + +[testenv] +description = run the tests with pytest +package = wheel +wheel_build_env = .pkg +passenv = + # This allows tox environment on a DevBox to trigger host browser + DISPLAY +deps = + pytest>=6 + -r requirements.txt +commands = + pip list + {posargs:pytest --color=yes} + +[testenv:azcli] +deps = + azure-cli +commands_pre = + # It will unfortunately be run every time but luckily subsequent runs are fast. + pip install -e . +commands = + pip list + {posargs:az --version} From 2c3cd281736c8a2aa436b71ed2c260662fd35b3d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 18 Apr 2025 12:34:56 -0700 Subject: [PATCH 240/262] Improve IndividualCache test cases --- msal/individual_cache.py | 14 +++++++++----- tests/test_individual_cache.py | 27 ++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/msal/individual_cache.py b/msal/individual_cache.py index 4c6fa00e..34f275dd 100644 --- a/msal/individual_cache.py +++ b/msal/individual_cache.py @@ -59,6 +59,10 @@ def __init__(self, mapping=None, capacity=None, expires_in=None, lock=None, self._expires_in = expires_in self._lock = Lock() if lock is None else lock + def _peek(self): + # Returns (sequence, timestamps) without triggering maintenance + return self._mapping.get(self._INDEX, ([], {})) + def _validate_key(self, key): if key == self._INDEX: raise ValueError("key {} is a reserved keyword in {}".format( @@ -85,7 +89,7 @@ def _set(self, key, value, expires_in): # This internal implementation powers both set() and __setitem__(), # so that they don't depend on each other. self._validate_key(key) - sequence, timestamps = self._mapping.get(self._INDEX, ([], {})) + sequence, timestamps = self._peek() self._maintenance(sequence, timestamps) # O(logN) now = int(time.time()) expires_at = now + expires_in @@ -136,7 +140,7 @@ def __getitem__(self, key): # O(1) self._validate_key(key) with self._lock: # Skip self._maintenance(), because it would need O(logN) time - sequence, timestamps = self._mapping.get(self._INDEX, ([], {})) + sequence, timestamps = self._peek() expires_at, created_at = timestamps[key] # Would raise KeyError accordingly now = int(time.time()) if not created_at <= now < expires_at: @@ -155,14 +159,14 @@ def __delitem__(self, key): # O(1) with self._lock: # Skip self._maintenance(), because it would need O(logN) time self._mapping.pop(key, None) # O(1) - sequence, timestamps = self._mapping.get(self._INDEX, ([], {})) + sequence, timestamps = self._peek() del timestamps[key] # O(1) self._mapping[self._INDEX] = sequence, timestamps def __len__(self): # O(logN) """Drop all expired items and return the remaining length""" with self._lock: - sequence, timestamps = self._mapping.get(self._INDEX, ([], {})) + sequence, timestamps = self._peek() self._maintenance(sequence, timestamps) # O(logN) self._mapping[self._INDEX] = sequence, timestamps return len(timestamps) # Faster than iter(self._mapping) when it is on disk @@ -170,7 +174,7 @@ def __len__(self): # O(logN) def __iter__(self): """Drop all expired items and return an iterator of the remaining items""" with self._lock: - sequence, timestamps = self._mapping.get(self._INDEX, ([], {})) + sequence, timestamps = self._peek() self._maintenance(sequence, timestamps) # O(logN) self._mapping[self._INDEX] = sequence, timestamps return iter(timestamps) # Faster than iter(self._mapping) when it is on disk diff --git a/tests/test_individual_cache.py b/tests/test_individual_cache.py index 38bd572d..ce4aa993 100644 --- a/tests/test_individual_cache.py +++ b/tests/test_individual_cache.py @@ -8,7 +8,13 @@ class TestExpiringMapping(unittest.TestCase): def setUp(self): self.mapping = {} - self.m = ExpiringMapping(mapping=self.mapping, capacity=2, expires_in=1) + self.expires_in = 1 + self.m = ExpiringMapping( + mapping=self.mapping, capacity=2, expires_in=self.expires_in) + + def how_many(self): + # This helper checks how many items are in the mapping, WITHOUT triggering purge + return len(self.m._peek()[1]) def test_should_disallow_accessing_reserved_keyword(self): with self.assertRaises(ValueError): @@ -40,11 +46,21 @@ def test_iter_should_purge(self): sleep(1) self.assertEqual([], list(self.m)) - def test_get_should_purge(self): + def test_get_should_not_purge_and_should_return_only_when_the_item_is_still_valid(self): self.m["thing one"] = "one" + self.m["thing two"] = "two" sleep(1) + self.assertEqual(2, self.how_many(), "We begin with 2 items") with self.assertRaises(KeyError): self.m["thing one"] + self.assertEqual(1, self.how_many(), "get() should not purge the remaining items") + + def test_setitem_should_purge(self): + self.m["thing one"] = "one" + sleep(1) + self.m["thing two"] = "two" + self.assertEqual(1, self.how_many(), "setitem() should purge all expired items") + self.assertEqual("two", self.m["thing two"], "The remaining item should be thing two") def test_various_expiring_time(self): self.assertEqual(0, len(self.m)) @@ -57,12 +73,13 @@ def test_various_expiring_time(self): def test_old_item_can_be_updated_with_new_expiry_time(self): self.assertEqual(0, len(self.m)) self.m["thing"] = "one" - self.m.set("thing", "two", 2) + new_lifetime = 3 # 2-second seems too short and causes flakiness + self.m.set("thing", "two", new_lifetime) self.assertEqual(1, len(self.m), "It contains 1 item") self.assertEqual("two", self.m["thing"], 'Already been updated to "two"') - sleep(1) + sleep(self.expires_in) self.assertEqual("two", self.m["thing"], "Not yet expires") - sleep(1) + sleep(new_lifetime - self.expires_in) self.assertEqual(0, len(self.m)) def test_oversized_input_should_purge_most_aging_item(self): From 9156a30f6fded731ac48ad7f7e0d63246198ba74 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 23 Apr 2025 12:34:56 -0700 Subject: [PATCH 241/262] MSAL Python 1.32.1 --- msal/sku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/sku.py b/msal/sku.py index 2a3172aa..1f38aeb7 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.32.0" +__version__ = "1.32.1" SKU = "MSAL.Python" From 04b5245c3db12390abf59cc58493f3b8f8ed725b Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 24 Apr 2025 17:09:41 -0700 Subject: [PATCH 242/262] Tolerate an http response object with no headers --- msal/throttled_http_client.py | 15 ++++++++++----- tests/test_throttled_http_client.py | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index fd0c9ad5..3939db91 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -11,6 +11,13 @@ DEVICE_AUTH_GRANT = "urn:ietf:params:oauth:grant-type:device_code" +def _get_headers(response): + # MSAL's HttpResponse did not have headers until 1.23.0 + # https://github.com/AzureAD/microsoft-authentication-library-for-python/pull/581/files#diff-28866b706bc3830cd20485685f20fe79d45b58dce7050e68032e9d9372d68654R61 + # This helper ensures graceful degradation to {} without exception + return getattr(response, "headers", {}) + + class RetryAfterParser(object): FIELD_NAME_LOWER = "Retry-After".lower() def __init__(self, default_value=None): @@ -19,9 +26,7 @@ def __init__(self, default_value=None): def parse(self, *, result, **ignored): """Return seconds to throttle""" response = result - lowercase_headers = {k.lower(): v for k, v in getattr( - # Historically, MSAL's HttpResponse does not always have headers - response, "headers", {}).items()} + lowercase_headers = {k.lower(): v for k, v in _get_headers(response).items()} if not (response.status_code == 429 or response.status_code >= 500 or self.FIELD_NAME_LOWER in lowercase_headers): return 0 # Quick exit @@ -49,7 +54,7 @@ def __init__(self, raw_response): self.status_code = raw_response.status_code self.text = raw_response.text self.headers = { # Only keep the headers which ThrottledHttpClient cares about - k: v for k, v in raw_response.headers.items() + k: v for k, v in _get_headers(raw_response).items() if k.lower() == RetryAfterParser.FIELD_NAME_LOWER } @@ -153,7 +158,7 @@ def __init__(self, *args, default_throttle_time=None, **kwargs): and kwargs["data"].get("grant_type") == DEVICE_AUTH_GRANT ) and RetryAfterParser.FIELD_NAME_LOWER not in set( # Otherwise leave it to the Retry-After decorator - h.lower() for h in getattr(result, "headers", {}).keys()) + h.lower() for h in _get_headers(result)) else 0, )(self.post) diff --git a/tests/test_throttled_http_client.py b/tests/test_throttled_http_client.py index c15a7877..fecb98d2 100644 --- a/tests/test_throttled_http_client.py +++ b/tests/test_throttled_http_client.py @@ -49,6 +49,18 @@ def close(self): raise CloseMethodCalled("Not used by MSAL, but our customers may use it") +class DummyHttpClientWithoutResponseHeaders(DummyHttpClient): + def post(self, url, params=None, data=None, headers=None, **kwargs): + response = super().post(url, params, data, headers, **kwargs) + del response.headers # Early versions of MSAL did not require http client to return headers + return response + + def get(self, url, params=None, headers=None, **kwargs): + response = super().get(url, params, headers, **kwargs) + del response.headers # Early versions of MSAL did not require http client to return headers + return response + + class CloseMethodCalled(Exception): pass @@ -69,6 +81,12 @@ def test_pickled_minimal_response_should_contain_signature(self): self.assertIn(MinimalResponse.SIGNATURE, pickle.dumps(MinimalResponse( status_code=200, headers={}, text="foo"))) + def test_throttled_http_client_base_response_should_tolerate_headerless_response(self): + http_client = ThrottledHttpClientBase(DummyHttpClientWithoutResponseHeaders( + status_code=200, response_text="foo")) + response = http_client.post("https://example.com") + self.assertEqual(response.text, "foo", "Should return the same response text") + def test_throttled_http_client_base_response_should_not_contain_signature(self): http_client = ThrottledHttpClientBase(DummyHttpClient(status_code=200)) response = http_client.post("https://example.com") From e60467c22a79bc0e613283024216e553230b4fda Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 25 Apr 2025 12:34:56 -0700 Subject: [PATCH 243/262] MSAL 1.32.2 --- msal/sku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/sku.py b/msal/sku.py index 1f38aeb7..ca7c81cc 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.32.1" +__version__ = "1.32.2" SKU = "MSAL.Python" From 3c87ba934f24471f9642a58588dff8650192117f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 25 Apr 2025 05:11:57 -0700 Subject: [PATCH 244/262] Allow more http response headers --- msal/throttled_http_client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index 3939db91..2e79a609 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -53,9 +53,12 @@ def __init__(self, raw_response): super().__init__() self.status_code = raw_response.status_code self.text = raw_response.text - self.headers = { # Only keep the headers which ThrottledHttpClient cares about - k: v for k, v in _get_headers(raw_response).items() - if k.lower() == RetryAfterParser.FIELD_NAME_LOWER + self.headers = { + k.lower(): v for k, v in _get_headers(raw_response).items() + # Attempted storing only a small set of headers (such as Retry-After), + # but it tends to lead to missing information (such as WWW-Authenticate). + # So we store all headers, which are expected to contain only public info, + # because we throttle only error responses and public responses. } ## Note: Don't use the following line, From dd4fe699256bd4b16ecc3e1b36bf04b511814e23 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 25 Apr 2025 05:39:11 -0700 Subject: [PATCH 245/262] MSAL Python 1.32.3 --- msal/sku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/sku.py b/msal/sku.py index ca7c81cc..36be3138 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.32.2" +__version__ = "1.32.3" SKU = "MSAL.Python" From 5a1a0ff6570137e18ac8f3ed6f1e5d21d9fbf2e9 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 29 Apr 2025 12:34:56 -0700 Subject: [PATCH 246/262] Add dependency management suggestions --- RELEASES.md | 41 +++++++++++++++++++++++++ dependency-management.md | 66 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 RELEASES.md create mode 100644 dependency-management.md diff --git a/RELEASES.md b/RELEASES.md new file mode 100644 index 00000000..3e65b976 --- /dev/null +++ b/RELEASES.md @@ -0,0 +1,41 @@ +# Microsoft Identity SDK Versioning and Servicing FAQ + +We have adopted the semantic versioning flow that is industry standard for OSS projects. It gives the maximum amount of control on what risk you take with what versions. If you know how semantic versioning works with node.js, java, and ruby none of this will be new. + +## Semantic Versioning and API stability promises + +Microsoft Authentication libraries are independent open source libraries that are used by partners both internal and external to Microsoft. As with the rest of Microsoft, we have moved to a rapid iteration model where bugs are fixed daily and new versions are produced as required. To communicate these frequent changes to external partners and customers, we use semantic versioning for all our public Microsoft Authentication SDK libraries. This follows the practices of other open source libraries on the internet. This allows us to support our downstream partners which will lock on certain versions for stability purposes, as well as providing for the distribution over NuGet, CocoaPods, and Maven. + +The semantics are: MAJOR.MINOR.PATCH (example 1.1.5) + +We will update our code distributions to use the latest PATCH semantic version number in order to make sure our customers and partners get the latest bug fixes. Downstream partner needs to pull the latest PATCH version. Most partners should try lock on the latest MINOR version number in their builds and accept any updates in the PATCH number. + +Example: +Using NuGet, this ensures all 1.1.0 to 1.1.x updates are included when building your code, but not 1.2. + +``` + +``` + +| Version | Description | Example | +|:-------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------:| +| x.x.x | PATCH version number. Incrementing these numbers is for bug fixes and updates but do not introduce new features. This is used for close partners who build on our platform release (ex. Azure AD Fabric, Office, etc.),In addition, Cocoapods, NuGet, and Maven use this number to deliver the latest release to customers. This will update frequently (sometimes within the same day),There is no new features, and no regressions or API surface changes. Code will continue to work unless affected by a particular code fix. | MSAL for iOS 1.0.10,(this was a fix for the Storyboard display that was fixed for a specific Office team) | +| x.x | MINOR version numbers. Incrementing these second numbers are for new feature additions that do not impact existing features or introduce regressions. They are purely additive, but may require testing to ensure nothing is impacted.,All x.x.x bug fixes will also roll up in to this number.,There is no regressions or API surface changes. Code will continue to work unless affected by a particular code fix or needs this new feature. | MSAL for iOS 1.1.0,(this added WPJ capability to MSAL, and rolled all the updates from 1.0.0 to 1.0.12) | +| x | MAJOR version numbers. This should be considered a new, supported version of Microsoft Authentication SDK and begins the Azure two year support cycle anew. Major new features are introduced and API changes can occur.,This should only be used after a large amount of testing and used only if those features are needed.,We will continue to service MAJOR version numbers with bug fixes up to the two year support cycle. | MSAL for iOS 1.0,(our first official release of MSAL) | + +## Serviceability + +When we release a new MINOR version, the previous MINOR version is abandoned. + +When we release a new MAJOR version, we will continue to apply bug fixes to the existing features in the previous MAJOR version for up to the 2 year support cycle for Azure. +Example: We release MSALiOS 2.0 in the future which supports unified Auth for AAD and MSA. Later, we then have a fix in Conditional Access for MSALiOS. Since that feature exists both in MSALiOS 1.1 and MSALiOS 2.0, we will fix both. It will roll up in a PATCH number for each. Customers that are still locked down on MSALiOS 1.1 will receive the benefit of this fix. + +## Microsoft Authentication SDKs and Azure Active Directory + +Microsoft Authentication SDKs major versions will maintain backwards compatibility with Azure Active Directory web services through the support period. This means that the API surface area defined in a MAJOR version will continue to work for 2 years after release. + +We will respond to bugs quickly from our partners and customers submitted through GitHub and through our private alias (tellaad@microsoft.com) for security issues and update the PATCH version number. We will also submit a change summary for each PATCH number. +Occasionally, there will be security bugs or breaking bugs from our partners that will require an immediate fix and a publish of an update to all partners and customers. When this occurs, we will do an emergency roll up to a PATCH version number and update all our distribution methods to the latest. diff --git a/dependency-management.md b/dependency-management.md new file mode 100644 index 00000000..f2a1bb41 --- /dev/null +++ b/dependency-management.md @@ -0,0 +1,66 @@ +# Best Practices for Managing Dependency Versions + +The best practices for managing dependency versions depend on +whether you are maintaining an application or a library. + +## If you are maintaining an application + +There are two suggestions that you shall follow. + +1. Pinning dependencies in a production environment. + + This refers to explicitly declaring the exact version of your application's dependencies, + including those of its dependencies (a.k.a. transitive dependencies). + This ensures consistent and predictable deployments by preventing accidental upgrades that might introduce errors or regressions. + + To achieve this, the steps can be different, based on the tooling you choose. + For example, if you are using the common `pip` tool, + you shall specify the version like this `pip install PackageName==x.y.z`, + where x.y.z is the latest stable version that you have tested with your app. + + Another approach is to start from a clean virtual environment, + install all the direct dependencies using the method described above, + and then run a `pip freeze > requirements.txt`. + It will generate a requirements.txt file containing all your dependencies + (including the transitive dependencies) pinned to their current versions. + Now you commit that requirements.txt. + From now on, you can consistently recreate your production environment by + `pip install -r requirements.txt`. + + You may also use advanced dependency management tools to simplify the work flow. + For example, + [pip-tools](https://pypi.org/project/pip-tools). + +2. Keep the dependencies updated. + + The purpose of pinning dependencies in the production is not about + sticking with old versions forever. + It is about allowing you to control when to update what. + In the long run, you are still recommended to keep the dependencies updated because: + + * The digital landscape is rife with threats. + Outdated dependencies are prime targets for malicious actors. + By staying updated, you fortify your project against known vulnerabilities + and significantly reduce the risk of exploitation. + * Software is rarely perfect. New versions of dependencies often include bug fixes, + improving the stability and performance of your application. + Ignoring updates means missing out on these enhancements. + + You shall maintain a non-production environment, + periodically upgrade its dependencies to latest versions, test it out. + If the test passes, you can update the requirements.txt accordingly, + and deploy it to your production environment. + + + +## If you are maintaining a library + +If you are maintaining a library (meaning it will be used by its downstream applications), +you shall avoid pinning your dependencies. +Instead, you shall declare your dependencies by range, such as `msal>=1.33, <2.0`. +The lower bound is the smallest version that started to provide the API you are using, +the upper bound is the version that is expected to contain breaking changes. + From db889c7da3226ea7717f26b6f0613b19c635f507 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Thu, 1 May 2025 12:34:56 -0700 Subject: [PATCH 247/262] Remind developers about http_cache's unstable format --- msal/application.py | 12 +++++++++--- tox.ini | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/msal/application.py b/msal/application.py index 5ed55c28..0d6fe47a 100644 --- a/msal/application.py +++ b/msal/application.py @@ -495,10 +495,13 @@ def __init__( If your app is a command-line app (CLI), you would want to persist your http_cache across different CLI runs. + The persisted file's format may change due to, but not limited to, + `unstable protocol `_, + so your implementation shall tolerate unexpected loading errors. The following recipe shows a way to do so:: # Just add the following lines at the beginning of your CLI script - import sys, atexit, pickle + import sys, atexit, pickle, logging http_cache_filename = sys.argv[0] + ".http_cache" try: with open(http_cache_filename, "rb") as f: @@ -509,6 +512,9 @@ def __init__( AttributeError, # Cache created by a different version of MSAL ): persisted_http_cache = {} # Recover by starting afresh + except: # Unexpected exceptions + logging.exception("You may want to debug this") + persisted_http_cache = {} # Recover by starting afresh atexit.register(lambda: pickle.dump( # When exit, flush it back to the file. # It may occasionally overwrite another process's concurrent write, @@ -2012,12 +2018,12 @@ def __init__( This parameter defaults to None, which means MSAL will not utilize a broker. New in MSAL Python 1.31.0. - + :param boolean enable_broker_on_linux: This setting is only effective if your app is running on Linux, including WSL. This parameter defaults to None, which means MSAL will not utilize a broker. - New in MSAL Python 1.33.0. + New in MSAL Python 1.33.0. :param boolean enable_broker_on_wsl: This setting is only effective if your app is running on WSL. diff --git a/tox.ini b/tox.ini index dffd5110..a66d58e0 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,12 @@ commands = pip list {posargs:pytest --color=yes} +[testenv:docs] +deps = + -r docs/requirements.txt +commands = + sphinx-build docs docs/_build + [testenv:azcli] deps = azure-cli From 62855d4a3ffe7b7dcc0b3d08e1e4896be66701a7 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 30 Apr 2025 12:34:56 -0700 Subject: [PATCH 248/262] Properly throw MsalServiceError exception --- msal/exceptions.py | 24 +++++++++++++++++++----- msal/throttled_http_client.py | 5 ++++- tests/test_throttled_http_client.py | 25 +++++++++++++++++++++---- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/msal/exceptions.py b/msal/exceptions.py index 5e9ee151..a6da67b1 100644 --- a/msal/exceptions.py +++ b/msal/exceptions.py @@ -27,12 +27,26 @@ class MsalError(Exception): # Define the template in Unicode to accommodate possible Unicode variables - msg = u'An unspecified error' + msg = u'An unspecified error' # Keeping for backward compatibility - def __init__(self, *args, **kwargs): - super(MsalError, self).__init__(self.msg.format(**kwargs), *args) - self.kwargs = kwargs class MsalServiceError(MsalError): - msg = u"{error}: {error_description}" + msg = u"{error}: {error_description}" # Keeping for backward compatibility + def __init__( + self, + *args, + error: str, error_description: str, # Historically required, keeping them for now + # 1. We can't simply remove them, or else it will be a breaking change + # 2. We may change them to optional without breaking anyone. However, + # such a change will be a one-way change, because once being optional, + # we will never be able to change them (back) to be required. + # 3. Since they were required and already exist anyway, + # now we just keep them required "for now", + # just in case that we would use them again. + # There is no plan to do #1; and we keep option #2 open; we go with #3. + **kwargs, + ): + super().__init__(*args, **kwargs) + self._error = error + self._error_description = error_description diff --git a/msal/throttled_http_client.py b/msal/throttled_http_client.py index 2e79a609..7c64fbf6 100644 --- a/msal/throttled_http_client.py +++ b/msal/throttled_http_client.py @@ -66,7 +66,10 @@ def __init__(self, raw_response): # self.raise_for_status = raw_response.raise_for_status def raise_for_status(self): if self.status_code >= 400: - raise MsalServiceError("HTTP Error: {}".format(self.status_code)) + raise MsalServiceError( + "HTTP Error: {}".format(self.status_code), + error=None, error_description=None, # Historically required, keeping them for now + ) class ThrottledHttpClientBase(object): diff --git a/tests/test_throttled_http_client.py b/tests/test_throttled_http_client.py index fecb98d2..6fbd0ed8 100644 --- a/tests/test_throttled_http_client.py +++ b/tests/test_throttled_http_client.py @@ -6,6 +6,7 @@ from msal.throttled_http_client import ( ThrottledHttpClientBase, ThrottledHttpClient, NormalizedResponse) +from msal.exceptions import MsalServiceError from tests import unittest from tests.http_client import MinimalResponse as _MinimalResponse @@ -65,6 +66,26 @@ class CloseMethodCalled(Exception): pass +class NormalizedResponseTestCase(unittest.TestCase): + def test_pickled_minimal_response_should_contain_signature(self): + self.assertIn(MinimalResponse.SIGNATURE, pickle.dumps(MinimalResponse( + status_code=200, headers={}, text="foo"))) + + def test_normalized_response_should_not_contain_signature(self): + response = NormalizedResponse(MinimalResponse( + status_code=200, headers={}, text="foo")) + self.assertNotIn( + MinimalResponse.SIGNATURE, pickle.dumps(response), + "A pickled object should not contain undesirable data") + self.assertEqual(response.text, "foo", "Should return the same response text") + + def test_normalized_response_raise_for_status_should_raise(self): + response = NormalizedResponse(MinimalResponse( + status_code=400, headers={}, text="foo")) + with self.assertRaises(MsalServiceError): + response.raise_for_status() + + class ThrottledHttpClientBaseTestCase(unittest.TestCase): def assertCleanPickle(self, obj): @@ -77,10 +98,6 @@ def assertValidResponse(self, response): self.assertIsInstance(response, NormalizedResponse) self.assertCleanPickle(response) - def test_pickled_minimal_response_should_contain_signature(self): - self.assertIn(MinimalResponse.SIGNATURE, pickle.dumps(MinimalResponse( - status_code=200, headers={}, text="foo"))) - def test_throttled_http_client_base_response_should_tolerate_headerless_response(self): http_client = ThrottledHttpClientBase(DummyHttpClientWithoutResponseHeaders( status_code=200, response_text="foo")) From deb61fa1f67d6863c4137a3a0c04db791b0d3a72 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 2 May 2025 12:34:56 -0700 Subject: [PATCH 249/262] Improve test cases to test header-less response --- tests/http_client.py | 7 ++++++- tests/test_throttled_http_client.py | 23 ++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/http_client.py b/tests/http_client.py index f6f24739..34d430f0 100644 --- a/tests/http_client.py +++ b/tests/http_client.py @@ -28,7 +28,12 @@ class MinimalResponse(object): # Not for production use def __init__(self, requests_resp=None, status_code=None, text=None, headers=None): self.status_code = status_code or requests_resp.status_code self.text = text if text is not None else requests_resp.text - self.headers = {} if headers is None else headers + if headers: + # Early versions of MSAL did not require http response to contain headers. + # As of April 2025, some Azure Identity code paths still yield response without headers. + # Here we mimic the behavior of header-less response by default, + # so that test cases can cover header-less response scenarios. + self.headers = headers self._raw_resp = requests_resp def raise_for_status(self): diff --git a/tests/test_throttled_http_client.py b/tests/test_throttled_http_client.py index 6fbd0ed8..f8fa6551 100644 --- a/tests/test_throttled_http_client.py +++ b/tests/test_throttled_http_client.py @@ -50,18 +50,6 @@ def close(self): raise CloseMethodCalled("Not used by MSAL, but our customers may use it") -class DummyHttpClientWithoutResponseHeaders(DummyHttpClient): - def post(self, url, params=None, data=None, headers=None, **kwargs): - response = super().post(url, params, data, headers, **kwargs) - del response.headers # Early versions of MSAL did not require http client to return headers - return response - - def get(self, url, params=None, headers=None, **kwargs): - response = super().get(url, params, headers, **kwargs) - del response.headers # Early versions of MSAL did not require http client to return headers - return response - - class CloseMethodCalled(Exception): pass @@ -99,9 +87,14 @@ def assertValidResponse(self, response): self.assertCleanPickle(response) def test_throttled_http_client_base_response_should_tolerate_headerless_response(self): - http_client = ThrottledHttpClientBase(DummyHttpClientWithoutResponseHeaders( - status_code=200, response_text="foo")) - response = http_client.post("https://example.com") + # MSAL Python 1.32.1 had a regression that caused it to require headers in the response. + # This was fixed in 1.32.2 + # https://github.com/AzureAD/microsoft-authentication-library-for-python/compare/1.32.1...1.32.2 + # This test case is to ensure that we can tolerate headerless response. + http_client = DummyHttpClient(status_code=200, response_text="foo") + raw_response = http_client.post("https://example.com") + self.assertFalse(hasattr(raw_response, "headers"), "Should not contain headers") + response = ThrottledHttpClientBase(http_client).post("https://example.com") self.assertEqual(response.text, "foo", "Should return the same response text") def test_throttled_http_client_base_response_should_not_contain_signature(self): From db1c3848456696920fb5fb21c1a25a5e9323e443 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 19 May 2025 12:34:56 -0700 Subject: [PATCH 250/262] Cryptography shipped a new version during weekend --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 16f7cca3..10e8be55 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=2.5,<47 + cryptography>=2.5,<48 [options.extras_require] From 4c632c82808e7a3a73635718d92a8c3fc09dc871 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 26 May 2025 12:34:56 -0700 Subject: [PATCH 251/262] Linux broker needs a specific redirect_uri A recent customer troubleshooting case reveals that the Linux broker needs a specific redirect_uri as its prerequisite --- msal/application.py | 28 +++++++++++++++------------- msal/broker.py | 6 ++++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/msal/application.py b/msal/application.py index 0d6fe47a..24ef91d7 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1948,8 +1948,6 @@ def __init__( .. note:: - You may set enable_broker_on_windows and/or enable_broker_on_mac and/or enable_broker_on_linux and/or enable_broker_on_wsl to True. - **What is a broker, and why use it?** A broker is a component installed on your device. @@ -1967,22 +1965,26 @@ def __init__( so that your broker-enabled apps (even a CLI) could automatically SSO from a previously established signed-in session. - **You shall only enable broker when your app:** + **How to opt in to use broker?** - 1. is running on supported platforms, - and already registered their corresponding redirect_uri + 1. You can set any combination of the following opt-in parameters to true: - * ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` - if your app is expected to run on Windows 10+ - * ``msauth.com.msauth.unsignedapp://auth`` - if your app is expected to run on Mac - * ``ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id`` - if your app is expected to run on Linux, especially WSL + +--------------------------+-----------------------------------+------------------------------------------------------------------------------------+ + | Opt-in flag | If app will run on | App has registered this as a Desktop platform redirect URI in Azure Portal | + +==========================+===================================+====================================================================================+ + | enable_broker_on_windows | Windows 10+ | ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id | + +--------------------------+-----------------------------------+------------------------------------------------------------------------------------+ + | enable_broker_on_wsl | WSL | ms-appx-web://Microsoft.AAD.BrokerPlugin/your_client_id | + +--------------------------+-----------------------------------+------------------------------------------------------------------------------------+ + | enable_broker_on_mac | Mac with Company Portal installed | msauth.com.msauth.unsignedapp://auth | + +--------------------------+-----------------------------------+------------------------------------------------------------------------------------+ + | enable_broker_on_linux | Linux with Intune installed | ``https://login.microsoftonline.com/common/oauth2/nativeclient`` (MUST be enabled) | + +--------------------------+-----------------------------------+------------------------------------------------------------------------------------+ - 2. installed broker dependency, + 2. Install broker dependency, e.g. ``pip install msal[broker]>=1.33,<2``. - 3. tested with ``acquire_token_interactive()`` and ``acquire_token_silent()``. + 3. Test with ``acquire_token_interactive()`` and ``acquire_token_silent()``. **The fallback behaviors of MSAL Python's broker support** diff --git a/msal/broker.py b/msal/broker.py index 1e794b36..5ce25bcd 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -60,8 +60,10 @@ def _convert_error(error, client_id): or "AADSTS7000218" in context # This "request body must contain ... client_secret" is just a symptom of current app has no WAM redirect_uri ): raise RedirectUriError( # This would be seen by either the app developer or end user - "MsalRuntime needs the current app to register these redirect_uri " - "(1) ms-appx-web://Microsoft.AAD.BrokerPlugin/{} (2) {}".format( + """MsalRuntime needs the current app to register these redirect_uri +(1) ms-appx-web://Microsoft.AAD.BrokerPlugin/{} +(2) {} +(3) https://login.microsoftonline.com/common/oauth2/nativeclient""".format( client_id, _redirect_uri_on_mac)) # OTOH, AAD would emit other errors when other error handling branch was hit first, # so, the AADSTS50011/RedirectUriError is not guaranteed to happen. From d49296c1b2a929a6ab11380e237daa89a5298512 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 4 Jun 2025 05:02:12 -0700 Subject: [PATCH 252/262] Bump version number (#827) --- msal/sku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/sku.py b/msal/sku.py index 36be3138..4cbd6310 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.32.3" +__version__ = "1.33.0b1" SKU = "MSAL.Python" From b1d8cd71145a8b1889b490f9b0dfbe4b1ac3a7f1 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 11 Jun 2025 12:34:56 -0700 Subject: [PATCH 253/262] Use lowercase environment value during searching --- msal/token_cache.py | 8 +++++++- tests/test_mi.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/msal/token_cache.py b/msal/token_cache.py index 66be5c9f..846c8132 100644 --- a/msal/token_cache.py +++ b/msal/token_cache.py @@ -126,7 +126,13 @@ def _get(self, credential_type, key, default=None): # O(1) @staticmethod def _is_matching(entry: dict, query: dict, target_set: set = None) -> bool: - return is_subdict_of(query or {}, entry) and ( + query_with_lowercase_environment = { + # __add() canonicalized entry's environment value to lower case, + # so we do the same here. + k: v.lower() if k == "environment" and isinstance(v, str) else v + for k, v in query.items() + } if query else {} + return is_subdict_of(query_with_lowercase_environment, entry) and ( target_set <= set(entry.get("target", "").split()) if target_set else True) diff --git a/tests/test_mi.py b/tests/test_mi.py index 32f9efae..864d258c 100644 --- a/tests/test_mi.py +++ b/tests/test_mi.py @@ -190,6 +190,10 @@ def test_happy_path_of_vm(self): headers={'Metadata': 'true'}, ) + @patch("msal.managed_identity.socket.getfqdn", new=lambda: "MixedCaseHostName") + def test_happy_path_of_windows_vm(self): + self.test_happy_path_of_vm() + @patch.dict(os.environ, {"AZURE_POD_IDENTITY_AUTHORITY_HOST": "http://localhost:1234//"}) def test_happy_path_of_pod_identity(self): self._test_happy_path().assert_called_with( From 70fd4d1599fc15c876c8eaccd29b9f7ae73fecd6 Mon Sep 17 00:00:00 2001 From: Ashok Kumar Ramakrishnan <83938949+ashok672@users.noreply.github.com> Date: Fri, 18 Jul 2025 00:45:02 -0700 Subject: [PATCH 254/262] Add claims challenge parameter in initiate_device_flow (#839) * Add claims challenge parameter in initiate_device_flow * Update msal/application.py Co-authored-by: Ray Luo * Update msal/oauth2cli/oauth2.py Co-authored-by: Ray Luo * Update msal/application.py Co-authored-by: Ray Luo * Update oauth2.py * Update oauth2.py --------- Co-authored-by: Ray Luo --- msal/application.py | 4 +++- msal/oauth2cli/oauth2.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/msal/application.py b/msal/application.py index 24ef91d7..c58678ca 100644 --- a/msal/application.py +++ b/msal/application.py @@ -2326,7 +2326,7 @@ def _acquire_token_interactive_via_broker( auth_scheme=auth_scheme, **data) - def initiate_device_flow(self, scopes=None, **kwargs): + def initiate_device_flow(self, scopes=None, *, claims_challenge=None, **kwargs): """Initiate a Device Flow instance, which will be used in :func:`~acquire_token_by_device_flow`. @@ -2341,6 +2341,8 @@ def initiate_device_flow(self, scopes=None, **kwargs): flow = self.client.initiate_device_flow( scope=self._decorate_scope(scopes or []), headers={msal.telemetry.CLIENT_REQUEST_ID: correlation_id}, + data={"claims": _merge_claims_challenge_and_capabilities( + self._client_capabilities, claims_challenge)}, **kwargs) flow[self.DEVICE_FLOW_CORRELATION_ID] = correlation_id return flow diff --git a/msal/oauth2cli/oauth2.py b/msal/oauth2cli/oauth2.py index 01b7fc34..ef32ceaa 100644 --- a/msal/oauth2cli/oauth2.py +++ b/msal/oauth2cli/oauth2.py @@ -305,7 +305,7 @@ class Client(BaseClient): # We choose to implement all 4 grants in 1 class grant_assertion_encoders = {GRANT_TYPE_SAML2: BaseClient.encode_saml_assertion} - def initiate_device_flow(self, scope=None, **kwargs): + def initiate_device_flow(self, scope=None, *, data=None, **kwargs): # type: (list, **dict) -> dict # The naming of this method is following the wording of this specs # https://tools.ietf.org/html/draft-ietf-oauth-device-flow-12#section-3.1 @@ -323,8 +323,11 @@ def initiate_device_flow(self, scope=None, **kwargs): DAE = "device_authorization_endpoint" if not self.configuration.get(DAE): raise ValueError("You need to provide device authorization endpoint") + _data = {"client_id": self.client_id, "scope": self._stringify(scope or [])} + if isinstance(data, dict): + _data.update(data) resp = self._http_client.post(self.configuration[DAE], - data={"client_id": self.client_id, "scope": self._stringify(scope or [])}, + data=_data, headers=dict(self.default_headers, **kwargs.pop("headers", {})), **kwargs) flow = json.loads(resp.text) From 923a7321a53df28e6acce1c5c182b59262bd9ca0 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Tue, 22 Jul 2025 12:39:22 -0700 Subject: [PATCH 255/262] MSAL Python 1.33.0 (#841) --- msal/sku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msal/sku.py b/msal/sku.py index 4cbd6310..4dc5c138 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.33.0b1" +__version__ = "1.33.0" SKU = "MSAL.Python" From bebbdd4b61b5d858980bf34eae2825dfec30a54d Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 30 Jul 2025 12:34:56 -0700 Subject: [PATCH 256/262] ADFS labs were decommissioned since late July 2025 --- tests/test_e2e.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index dde26f0d..65b90aa2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -841,12 +841,14 @@ def test_user_account(self): class WorldWideTestCase(LabBasedTestCase): + _ADFS_LABS_UNAVAILABLE = "ADFS labs were temporarily down since July 2025 until further notice" def test_aad_managed_user(self): # Pure cloud config = self.get_lab_user(usertype="cloud") config["password"] = self.get_lab_user_secret(config["lab_name"]) self._test_username_password(**config) + @unittest.skip(_ADFS_LABS_UNAVAILABLE) def test_adfs4_fed_user(self): config = self.get_lab_user(usertype="federated", federationProvider="ADFSv4") config["password"] = self.get_lab_user_secret(config["lab_name"]) @@ -864,6 +866,7 @@ def test_adfs2_fed_user(self): config["password"] = self.get_lab_user_secret(config["lab_name"]) self._test_username_password(**config) + @unittest.skip(_ADFS_LABS_UNAVAILABLE) def test_adfs2019_fed_user(self): try: config = self.get_lab_user(usertype="federated", federationProvider="ADFSv2019") @@ -892,6 +895,7 @@ def test_msa_pt_app_signin_via_organizations_authority_without_login_hint(self): prompt="select_account", # In MSAL Python, this resets login_hint )) + @unittest.skip(_ADFS_LABS_UNAVAILABLE) def test_ropc_adfs2019_onprem(self): # Configuration is derived from https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/4.7.0/tests/Microsoft.Identity.Test.Common/TestConstants.cs#L250-L259 config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") @@ -900,6 +904,7 @@ def test_ropc_adfs2019_onprem(self): config["password"] = self.get_lab_user_secret(config["lab_name"]) self._test_username_password(**config) + @unittest.skip(_ADFS_LABS_UNAVAILABLE) def test_adfs2019_onprem_acquire_token_by_auth_code(self): """When prompted, you can manually login using this account: @@ -913,6 +918,7 @@ def test_adfs2019_onprem_acquire_token_by_auth_code(self): config["port"] = 8080 self._test_acquire_token_by_auth_code(**config) + @unittest.skip(_ADFS_LABS_UNAVAILABLE) def test_adfs2019_onprem_acquire_token_by_auth_code_flow(self): config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") self._test_acquire_token_by_auth_code_flow(**dict( @@ -922,6 +928,7 @@ def test_adfs2019_onprem_acquire_token_by_auth_code_flow(self): port=8080, )) + @unittest.skip(_ADFS_LABS_UNAVAILABLE) def test_adfs2019_onprem_acquire_token_interactive(self): config = self.get_lab_user(usertype="onprem", federationProvider="ADFSv2019") self._test_acquire_token_interactive(**dict( From cff139e0d84038f43cfb7d285fb1adbc16fc900f Mon Sep 17 00:00:00 2001 From: xinyuxu1026 <93947208+xinyuxu1026@users.noreply.github.com> Date: Fri, 29 Aug 2025 11:49:14 -0700 Subject: [PATCH 257/262] Set Redirect Uri for broker silent flow on Linux platform (#846) --- msal/broker.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/msal/broker.py b/msal/broker.py index 5ce25bcd..b595608b 100644 --- a/msal/broker.py +++ b/msal/broker.py @@ -145,12 +145,20 @@ def _build_msal_runtime_auth_params(client_id, authority): params.set_additional_parameter("msal_client_ver", __version__) return params +def _set_redirect_uri_for_linux(params): + if sys.platform == "linux": + # This is required by Linux Java Broker to set a non-empty valid redirect_uri + params.set_redirect_uri( + "https://login.microsoftonline.com/common/oauth2/nativeclient" + ) + def _signin_silently( authority, client_id, scopes, correlation_id=None, claims=None, enable_msa_pt=False, auth_scheme=None, **kwargs): params = _build_msal_runtime_auth_params(client_id, authority) + _set_redirect_uri_for_linux(params) params.set_requested_scopes(scopes) if claims: params.set_decoded_claims(claims) @@ -240,6 +248,7 @@ def _acquire_token_silently( if account is None: return params = _build_msal_runtime_auth_params(client_id, authority) + _set_redirect_uri_for_linux(params) params.set_requested_scopes(scopes) if claims: params.set_decoded_claims(claims) From 2337a69a680139cac54521cfbe4fedef137a2b06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 23:57:01 -0700 Subject: [PATCH 258/262] Bump pypa/gh-action-pypi-publish in /.github/workflows (#849) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.4.2 to 1.13.0. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.4.2...v1.13.0) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.13.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8ed0073c..32ab3924 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -119,7 +119,7 @@ jobs: - name: | Publish to TestPyPI when pushing to release-* branch. You better test with a1, a2, b1, b2 releases first. - uses: pypa/gh-action-pypi-publish@v1.4.2 + uses: pypa/gh-action-pypi-publish@v1.13.0 if: startsWith(github.ref, 'refs/heads/release-') with: user: __token__ @@ -127,7 +127,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI when tagged if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@v1.4.2 + uses: pypa/gh-action-pypi-publish@v1.13.0 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} From f803aec300d13f4a55c78941409c60befb2c8f1f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Mon, 22 Sep 2025 16:49:01 -0700 Subject: [PATCH 259/262] Release 1.34.0 (#853) * ADFS labs were decommissioned since late July 2025 * MSAL Python 1.34.0b1 * Declare support for Python 3.13 * Bumping cryptography which also drops Python 3.7 * 1.34.0b1 + minor changes = 1.34.0 --- .github/workflows/python-package.yml | 2 +- msal/sku.py | 2 +- setup.cfg | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 32ab3924..eb6ebc50 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 diff --git a/msal/sku.py b/msal/sku.py index 4dc5c138..3069da43 100644 --- a/msal/sku.py +++ b/msal/sku.py @@ -2,5 +2,5 @@ """ # The __init__.py will import this. Not the other way around. -__version__ = "1.33.0" +__version__ = "1.34.0" SKU = "MSAL.Python" diff --git a/setup.cfg b/setup.cfg index 10e8be55..373524f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,12 +18,12 @@ classifiers = Programming Language :: Python Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 License :: OSI Approved :: MIT License Operating System :: OS Independent @@ -38,7 +38,7 @@ project_urls = include_package_data = False # We used to ship LICENSE, but our __init__.py already mentions MIT packages = find: # Our test pipeline currently still covers Py37 -python_requires = >=3.7 +python_requires = >=3.8 install_requires = requests>=2.0.0,<3 @@ -52,7 +52,7 @@ install_requires = # And we will use the cryptography (X+3).0.0 as the upper bound, # based on their latest deprecation policy # https://cryptography.io/en/latest/api-stability/#deprecation - cryptography>=2.5,<48 + cryptography>=2.5,<49 [options.extras_require] From 562d72e295f807bba99d5eee9029591d9ccee60c Mon Sep 17 00:00:00 2001 From: Ugonna Akali <137432604+Ugonnaak1@users.noreply.github.com> Date: Tue, 30 Sep 2025 10:47:44 -0700 Subject: [PATCH 260/262] ROPC deprecation (#855) * ROPC deprecation * sample edit * reenable e2e tests * reenable tests * edit * remove import * fix wording Co-authored-by: Bogdan Gavril * edits * add comment * format * docstring changes --------- Co-authored-by: Bogdan Gavril --- msal/application.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/msal/application.py b/msal/application.py index c58678ca..b4371c63 100644 --- a/msal/application.py +++ b/msal/application.py @@ -1840,7 +1840,17 @@ def acquire_token_by_username_password( - A successful response would contain "access_token" key, - an error response would contain "error" and usually "error_description". + + [Deprecated] This API is deprecated for public client flows and will be + removed in a future release. Use a more secure flow instead. + Migration guide: https://aka.ms/msal-ropc-migration + """ + is_confidential_app = self.client_credential or isinstance( + self, ConfidentialClientApplication) + if not is_confidential_app: + warnings.warn("""This API has been deprecated for public client flows, please use a more secure flow. + See https://aka.ms/msal-ropc-migration for migration guidance""", DeprecationWarning) claims = _merge_claims_challenge_and_capabilities( self._client_capabilities, claims_challenge) if self._enable_broker and sys.platform in ("win32", "darwin"): From 2a138a5a56444f3a52c5bf33751cc32ad006d126 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Wed, 24 Sep 2025 12:34:56 -0700 Subject: [PATCH 261/262] Demonstrates the behavior of mismatching scope --- tests/test_application.py | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_application.py b/tests/test_application.py index 16e512c4..556750fa 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -875,3 +875,50 @@ def test_app_did_not_register_redirect_uri_should_error_out(self): parent_window_handle=app.CONSOLE_WINDOW_HANDLE, ) self.assertEqual(result.get("error"), "broker_error") + + +class MismatchingScopeTestCase(unittest.TestCase): + """Test cache behavior when HTTP response scope differs from requested scope""" + + def test_token_should_be_cached_with_response_scope(self): + """Based on https://datatracker.ietf.org/doc/html/rfc6749#section-3.3 + authorization server may issue an access token with different scope. + For example, eSTS normalizes scopes by adding or removing trailing slash. + Calling app is supposed to use the normalized scope for subsequent calls. + """ + + # Create a fresh app instance + app = ConfidentialClientApplication( + "client_id", client_credential="secret", + authority="https://login.microsoftonline.com/common") + + # Mocked request: ask for "invalid_scope" scope but receive "valid_scope1 valid_scope2" scope in response + def mock_post(url, headers=None, *args, **kwargs): + return MinimalResponse(status_code=200, text=json.dumps({ + "access_token": "AT_with_valid_scope1_valid_scope2_scopes", + "expires_in": 3600, + "scope": "valid_scope1 valid_scope2", # Response scope differs from requested scope + "token_type": "Bearer" + })) + + result1 = app.acquire_token_for_client(["invalid_scope"], post=mock_post) + self.assertEqual(result1[app._TOKEN_SOURCE], app._TOKEN_SOURCE_IDP) + self.assertEqual("AT_with_valid_scope1_valid_scope2_scopes", result1.get("access_token")) + self.assertEqual(["valid_scope1", "valid_scope2"], result1.get("scope").split()) # Scope from response + + # Second request: ask for same "invalid_scope" scope again + # Since cached token has "valid_scope1 valid_scope2" scopes, it shouldn't match the "invalid_scope" request + # This should go to IDP again and receive the same response + result2 = app.acquire_token_for_client(["invalid_scope"], post=mock_post) + # Should get a new token from IDP, not from cache + self.assertEqual(result2[app._TOKEN_SOURCE], app._TOKEN_SOURCE_IDP) + self.assertEqual("AT_with_valid_scope1_valid_scope2_scopes", result2.get("access_token")) + self.assertEqual(["valid_scope1", "valid_scope2"], result2.get("scope").split()) + + # Third and fourth requests: ask for individual valid scopes + # Should hit cache for the token that has "valid_scope1 valid_scope2" scopes + for scope in ["valid_scope1", "valid_scope2"]: + result = app.acquire_token_for_client([scope]) + self.assertEqual(result[app._TOKEN_SOURCE], app._TOKEN_SOURCE_CACHE) + self.assertEqual("AT_with_valid_scope1_valid_scope2_scopes", result.get("access_token")) + self.assertIsNone(result.get("scope"), "scope field is not returned when token comes from cache") \ No newline at end of file From 59fa195804532bbc5708e13fa5b6245827d469b6 Mon Sep 17 00:00:00 2001 From: Dharshan BJ Date: Mon, 13 Oct 2025 17:15:23 -0700 Subject: [PATCH 262/262] Update pymsalruntime version range to handle the latest 0.20.0 release --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 373524f0..8fbf015f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,11 +62,11 @@ broker = # most existing MSAL Python apps do not have the redirect_uri needed by broker. # # We need pymsalruntime.CallbackData introduced in PyMsalRuntime 0.14 - pymsalruntime>=0.14,<0.19; python_version>='3.6' and platform_system=='Windows' + pymsalruntime>=0.14,<0.21; python_version>='3.8' and platform_system=='Windows' # On Mac, PyMsalRuntime 0.17+ is expected to support SSH cert and ROPC - pymsalruntime>=0.17,<0.19; python_version>='3.8' and platform_system=='Darwin' + pymsalruntime>=0.17,<0.21; python_version>='3.8' and platform_system=='Darwin' # PyMsalRuntime 0.18+ is expected to support broker on Linux - pymsalruntime>=0.18,<0.19; python_version>='3.8' and platform_system=='Linux' + pymsalruntime>=0.18,<0.21; python_version>='3.8' and platform_system=='Linux' [options.packages.find] exclude =