Skip to content

update mgmt docs build to not rely on hardcoded mappings #34482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kristapratico merged 4 commits into from
Mar 6, 2024
Merged

update mgmt docs build to not rely on hardcoded mappings #34482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3b99f14
hardcoding everything wip, should able to compute automatically
kristapratico Feb 12, 2024
5b87b8b
update the code to not use hardcoded package_service_mapping
kristapratico Feb 19, 2024
62d053a
make sure we preserve ordering of latest --> oldest
kristapratico Feb 19, 2024
9324e6c
update copyright year
kristapratico Feb 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions doc/sphinx/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@

intersphinx_mapping = {
# Dependencies
'python': ('https://docs.python.org/3.11', ('/usr/share/doc/python3-doc/html/objects.inv', None)),
'requests': ('https://requests.kennethreitz.org/en/master/', None),
'aiohttp': ('https://aiohttp.readthedocs.io/en/stable/', None),
'python': ('https://docs.python.org/3.11', None),
'requests': ('https://requests.kennethreitz.org/en/latest/', None),
'aiohttp': ('https://docs.aiohttp.org/en/stable/', None),
'trio': ('https://trio.readthedocs.io/en/stable/', None),
'msal': ('https://msal-python.readthedocs.io/en/latest/', None),
# Azure packages
Expand All @@ -87,7 +87,7 @@

# General information about the project.
project = u'Azure SDK for Python'
copyright = u'2019, Microsoft'
copyright = u'2024, Microsoft'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
Expand Down
176 changes: 81 additions & 95 deletions doc/sphinx/generate_doc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,8 @@
import os
import glob

import typing
from typing import Dict, List

CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "package_service_mapping.json")
GENERATED_PACKAGES_LIST_FILE = "toc_tree.rst"

_LOGGER = logging.getLogger(__name__)
Expand Down Expand Up @@ -73,117 +71,112 @@ def make_title(title):

"""

# Update the code to compute this list automatically
MULTIAPI_VERSION_NAMESPACE = [
"azure.mgmt.storage",
"azure.mgmt.web",
"azure.mgmt.network",
"azure.mgmt.compute",
"azure.mgmt.containerregistry",
"azure.mgmt.containerservice",
"azure.mgmt.dns",
"azure.mgmt.eventhub",
"azure.mgmt.resource.resources",
"azure.mgmt.resource.features",
"azure.mgmt.resource.links",
"azure.mgmt.resource.locks",
"azure.mgmt.resource.policy",
"azure.mgmt.resource.subscriptions",
]


def get_updated_config(config_path: str, package_root: str) -> Dict[str, Dict[str, typing.Union[str, List[str]]]]:
with Path(config_path).open() as config_fd:
config = json.load(config_fd)

namespace_folders = os.path.basename(package_root).split("-")
package_name = "-".join(namespace_folders)
namespace = ".".join(namespace_folders)

api_directory = os.path.join(package_root, *namespace_folders)
def get_valid_versions(api_directory: str) -> List[str]:
glob_path = os.path.join(api_directory, "v20*/")
valid_versions = glob.glob(glob_path)
return glob.glob(glob_path)

for version_path in valid_versions:
api_version = os.path.relpath(version_path, start=api_directory)
full_namespace = namespace + f".{api_version}"

if package_name in config:
if "namespaces" in config[package_name]:
ns = config[package_name]["namespaces"]
if ns:
if full_namespace not in ns:
ns.append(full_namespace)
def is_subnamespace(version: str) -> bool:
return not version.startswith("v20")

return config

def get_package_namespaces(package_root: str) -> List[str]:
namespace_folders = os.path.basename(package_root).split("-")
namespace = ".".join(namespace_folders)

def generate_doc(config_path: str, output_directory: str = "./ref/", package_root: str = None) -> None:
multiapi_found_apiversion = {}
# we are handed a directory that looks like <path-to-repo>/sdk/containerservice/azure-mgmt-containerservice/
project_pattern = os.path.basename(package_root).replace("-", ".")
rst_path_template = os.path.join(output_directory, "{}.rst")
rst_namespace_template = os.path.join(output_directory, "{}.{}.rst")
# add top namespace
namespaces = {namespace: []}

config = get_updated_config(config_path, package_root)
api_directory = os.path.join(package_root, *namespace_folders)
valid_versions = get_valid_versions(api_directory)

package_list_path = []
# check for subnamespaces like azure.mgmt.resource.locks
if not valid_versions:
subnamespaces = glob.glob(f"{api_directory}/*/")
for snp_path in subnamespaces:
versions = get_valid_versions(snp_path)
valid_versions.extend(versions)

namespaces = [n for pack in config.values() for n in pack.get("namespaces", {})]
for version_path in valid_versions:
version = os.path.relpath(version_path, start=api_directory)
if is_subnamespace(version):
subnamespace_name, api_version = version.split("/")
full_namespace = ".".join([namespace, subnamespace_name, api_version])
subnamespace = ".".join([namespace, subnamespace_name])
if subnamespace not in namespaces[namespace]:
namespaces[namespace].append(subnamespace)
namespaces.setdefault(subnamespace, []).append(full_namespace)
else:
full_namespace = ".".join([namespace, version])
namespaces[namespace].append(full_namespace)

for namespace in namespaces:
if project_pattern and not namespace.startswith(project_pattern):
continue
return namespaces

_LOGGER.info("Working on %s", namespace)

rst_path = rst_path_template.format(namespace)
with Path(rst_path).open("w") as rst_file:
rst_file.write(PACKAGE_TEMPLATE.format(title=make_title(namespace + " package"), namespace=namespace))
def write_rst(version: str, rst_path_template: str, rst_namespace_template: str, package_list_path: List[str]) -> None:
rst_path = rst_path_template.format(version)
with Path(rst_path).open("w") as rst_file:
rst_file.write(PACKAGE_TEMPLATE.format(title=make_title(version + " package"), namespace=version))

for module in ["operations", "models"]:
with Path(rst_namespace_template.format(namespace, module)).open("w") as rst_file:
rst_file.write(
SUBMODULE_TEMPLATE.format(
title=make_title(namespace + "." + module + " module"), namespace=namespace, submodule=module
)
for module in ["operations", "models"]:
with Path(rst_namespace_template.format(version, module)).open("w") as rst_file:
rst_file.write(
SUBMODULE_TEMPLATE.format(
title=make_title(version + "." + module + " module"), namespace=version, submodule=module
)
)
package_list_path.append(rst_path)

for multiapi_namespace in MULTIAPI_VERSION_NAMESPACE:
length = len(multiapi_namespace.split("."))
if namespace.split(".")[0:length] == multiapi_namespace.split(".")[0:length]:
_LOGGER.info("MultiAPI namespace on %s", multiapi_namespace)
api_package = namespace.split(multiapi_namespace + ".")[1]
multiapi_found_apiversion.setdefault(multiapi_namespace, []).append(api_package)
break
else:
package_list_path.append(rst_path)

for multiapi_namespace, apilist in multiapi_found_apiversion.items():
apilist.sort()
apilist.reverse()
rst_path = rst_path_template.format(multiapi_namespace)
with Path(rst_path).open("w") as rst_file:
rst_file.write(
MULTIAPI_VERSION_PACKAGE_TEMPLATE.format(
title=make_title(multiapi_namespace + " package"), namespace=multiapi_namespace
)
def write_multiapi_rst(namespace: str, versions: List[str], rst_path_template: str, package_list_path: List[str]) -> None:
rst_path = rst_path_template.format(namespace)
with Path(rst_path).open("w") as rst_file:
rst_file.write(
MULTIAPI_VERSION_PACKAGE_TEMPLATE.format(
title=make_title(namespace + " package"), namespace=namespace
)
for version in apilist:
rst_file.write(" {namespace}.{version}\n".format(namespace=multiapi_namespace, version=version))
package_list_path.append(rst_path)
)
for version in versions:
rst_file.write(" {version}\n".format(version=version))
package_list_path.append(rst_path)

# now handle the packages from data plane that we want to be present properly sorted in the list of packages
for package in config.keys():
if "manually_generated" in config[package] and config[package]["manually_generated"] == True:
package_list_path.append(rst_path_template.format(package.replace("-", ".")))

def write_toc_tree(package_list_path: List[str]) -> None:
package_list_path.sort()
with Path(GENERATED_PACKAGES_LIST_FILE).open("w") as generate_file_list_fd:
lines_to_write = "\n".join([" " + package for package in package_list_path])
generate_file_list_fd.write(RST_AUTODOC_TOCTREE.format(generated_packages=lines_to_write))


def generate_doc(output_directory: str = "./ref/", package_root: str = None) -> None:
# we are handed a directory that looks like <path-to-repo>/sdk/containerservice/azure-mgmt-containerservice/
rst_path_template = os.path.join(output_directory, "{}.rst")
rst_namespace_template = os.path.join(output_directory, "{}.{}.rst")

namespaces = get_package_namespaces(package_root)

package_list_path = []

for namespace, multi_api_versions in namespaces.items():
_LOGGER.info("Working on %s", namespace)
if not multi_api_versions:
write_rst(namespace, rst_path_template, rst_namespace_template, package_list_path)
continue

multi_api_versions.sort(reverse=True)
write_multiapi_rst(namespace, multi_api_versions, rst_path_template, package_list_path)

for version in multi_api_versions:
if is_subnamespace(version.split(".")[-1]):
# subnamespace already handled in write_multiapi_rst
continue
_LOGGER.info("Working on %s", version)
write_rst(version, rst_path_template, rst_namespace_template, package_list_path)

write_toc_tree(package_list_path)


def main():
parser = argparse.ArgumentParser(description="Generate sphinx api stubs for one or multiple management packages.")
parser.add_argument(
Expand All @@ -192,13 +185,6 @@ def main():
dest="project",
help="Want to target a specific management package? Pass the targeted package root to this argument.",
)
parser.add_argument(
"--config",
"-c",
dest="config_path",
default=CONFIG_FILE,
help="The JSON configuration format path [default: %(default)s]",
)
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbosity in INFO mode")
parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode")
parser.add_argument(
Expand All @@ -212,7 +198,7 @@ def main():
logging.basicConfig()
main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)

generate_doc(args.config_path, args.output_directory, args.project)
generate_doc(args.output_directory, args.project)


if __name__ == "__main__":
Expand Down
Loading