Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Run black on scripts folder
  • Loading branch information
parthea committed Apr 25, 2021
commit 8878c01b6c749a1a91799fa366618d626aa4ddbc
42 changes: 25 additions & 17 deletions scripts/changesummary.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,18 @@
import numpy as np

BRANCH_ARTIFACTS_DIR = (
pathlib.Path(__file__).parent.resolve() / "googleapiclient" / "discovery_cache" / "documents"
pathlib.Path(__file__).parent.resolve()
/ "googleapiclient"
/ "discovery_cache"
/ "documents"
)
MAIN_ARTIFACTS_DIR = (
pathlib.Path(__file__).parent.resolve() / ".." / "main" / "googleapiclient" / "discovery_cache" / "documents"
pathlib.Path(__file__).parent.resolve()
/ ".."
/ "main"
/ "googleapiclient"
/ "discovery_cache"
/ "documents"
)

MULTIPROCESSING_NUM_PER_BATCH = 5
Expand Down Expand Up @@ -124,11 +132,10 @@ def _get_discovery_differences(self, filename):
combined_docs = (
pd.concat([current_doc, new_doc], keys=["CurrentValue", "NewValue"])
# Drop the index column
.reset_index(drop=True,level=1)
.reset_index(drop=True, level=1)
# Transpose the DataFrame, Resulting Columns should be
# ["Key", "CurrentValue", "New Value"]
.rename_axis(['Key'], axis=1)
.transpose()
.rename_axis(["Key"], axis=1).transpose()
# Drop the index column
.reset_index()
)
Expand Down Expand Up @@ -199,7 +206,8 @@ def _get_discovery_differences(self, filename):
# children keys have been added.
all_added = (
parent_added_agg[
(parent_added_agg["Proportion"] == 1) & (parent_added_agg["Added"] == True)
(parent_added_agg["Proportion"] == 1)
& (parent_added_agg["Added"] == True)
][["Parent", "NumLevels"]]
.sort_values("NumLevels", ascending=True)
.Parent.to_list()
Expand Down Expand Up @@ -327,7 +335,7 @@ def _get_keys_to_ignore(self):
return keys_to_ignore

def _get_stable_versions(self, versions):
""" Returns a pandas series `pd.Series()` of boolean values,
"""Returns a pandas series `pd.Series()` of boolean values,
corresponding to the given series, indicating whether the version is
considered stable or not.
args:
Expand Down Expand Up @@ -374,8 +382,7 @@ def _get_summary_and_write_to_disk(self, dataframe, directory):
# Create a new column `Summary`, which will contain a string with the
# conventional commit message.
dataframe["Summary"] = np.vectorize(self._build_summary_message)(
dataframe["Name"],
dataframe["IsFeatureAggregate"]
dataframe["Name"], dataframe["IsFeatureAggregate"]
)

# Write the final dataframe to disk as it will be used in the
Expand All @@ -384,7 +391,7 @@ def _get_summary_and_write_to_disk(self, dataframe, directory):
return dataframe

def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
""" Writes verbose information to file about changes made to discovery
"""Writes verbose information to file about changes made to discovery
artifacts based on the provided dataframe. A separate file is saved
for each api in the current working directory. The extension of the
files will be `'.verbose'`.
Expand All @@ -400,8 +407,9 @@ def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
verbose_changes = []

# Sort the dataframe to minimize file operations below.
dataframe.sort_values(by=["Name","Version","ChangeType"],
ascending=True, inplace=True)
dataframe.sort_values(
by=["Name", "Version", "ChangeType"], ascending=True, inplace=True
)

# Select only the relevant columns. We need to create verbose output
# by Api Name, Version and ChangeType so we need to group by these
Expand Down Expand Up @@ -433,7 +441,7 @@ def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
# Clear the array of strings with information from the previous
# api and reset the last version
verbose_changes = []
lastVersion = ''
lastVersion = ""
# Create a file which contains verbose changes for the current
# API being processed
filename = "{0}.verbose".format(currentApi)
Expand All @@ -450,12 +458,13 @@ def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
# summary column are the same for a given API.
verbose_changes.append(summary_df[current_api_filter].Summary.iloc[0])


# If the version has changed, we need to create append a new heading
# in the verbose summary which contains the api and version.
if lastVersion != currentVersion:
# Append a header string with the API and version
verbose_changes.append("\n\n#### {0}:{1}\n\n".format(currentApi, currentVersion))
verbose_changes.append(
"\n\n#### {0}:{1}\n\n".format(currentApi, currentVersion)
)

lastVersion = currentVersion
lastType = ChangeType.UNKNOWN
Expand All @@ -476,7 +485,7 @@ def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
# type group.
verbose_changes.extend(
[
"- {0} (Total Keys: {1})\n".format(row['Key'], row['Count'])
"- {0} (Total Keys: {1})\n".format(row["Key"], row["Count"])
for index, row in group[["Key", "Count"]].iterrows()
]
)
Expand Down Expand Up @@ -521,4 +530,3 @@ def detect_discovery_changes(self):
# Create verbose change information for each API which contains
# a list of changes by key and write it to disk.
self._write_verbose_changes_to_disk(result, self._temp_dir, summary_df)

9 changes: 3 additions & 6 deletions scripts/changesummary_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,20 +220,17 @@ def test_detect_discovery_changes(self):
# drive was updated
# 74 key changes in total. All 74 key changes should be additions
self.assertEqual(len(result[result["Name"] == "drive"]), 74)
self.assertEqual(
len(result[(result["Name"] == "drive") & result["Added"]]), 74
)
self.assertEqual(len(result[(result["Name"] == "drive") & result["Added"]]), 74)
self.assertEqual(
len(result[(result["Name"] == "drive") & result["Deleted"]]), 0
)
self.assertTrue(result[(result["Name"] == "drive")].IsStable.all())
self.assertTrue(
result[(result["Name"] == "drive")].IsFeatureAggregate.all()
)
self.assertTrue(result[(result["Name"] == "drive")].IsFeatureAggregate.all())
self.assertEqual(
result[(result["Name"] == "drive")].Summary.iloc[0],
"feat(drive): update the api",
)


if __name__ == "__main__":
unittest.main()
41 changes: 25 additions & 16 deletions scripts/updatediscoveryartifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@


SCRIPTS_DIR = pathlib.Path(__file__).parent.resolve()
DISCOVERY_DOC_DIR = SCRIPTS_DIR / ".." / "googleapiclient" / "discovery_cache" / "documents"
DISCOVERY_DOC_DIR = (
SCRIPTS_DIR / ".." / "googleapiclient" / "discovery_cache" / "documents"
)
REFERENCE_DOC_DIR = SCRIPTS_DIR / ".." / "docs" / "dyn"
TEMP_DIR = SCRIPTS_DIR / "temp"

Expand All @@ -35,8 +37,8 @@
shutil.rmtree(TEMP_DIR, ignore_errors=True)

# Check out a fresh copy
subprocess.call(['git', 'checkout', DISCOVERY_DOC_DIR])
subprocess.call(['git', 'checkout', REFERENCE_DOC_DIR])
subprocess.call(["git", "checkout", DISCOVERY_DOC_DIR])
subprocess.call(["git", "checkout", REFERENCE_DOC_DIR])

# Snapshot current discovery artifacts to a temporary directory
with tempfile.TemporaryDirectory() as current_discovery_doc_dir:
Expand All @@ -46,27 +48,34 @@
describe.generate_all_api_documents()

# Get a list of files changed using `git diff`
git_diff_output = subprocess.check_output(['git',
'diff',
'origin/master',
'--name-only',
'--',
DISCOVERY_DOC_DIR / '*.json',
REFERENCE_DOC_DIR / '*.html',
REFERENCE_DOC_DIR / '*.md',
],
universal_newlines=True)
git_diff_output = subprocess.check_output(
[
"git",
"diff",
"origin/master",
"--name-only",
"--",
DISCOVERY_DOC_DIR / "*.json",
REFERENCE_DOC_DIR / "*.html",
REFERENCE_DOC_DIR / "*.md",
],
universal_newlines=True,
)

# Create lists of the changed files
all_changed_files = [pathlib.Path(file_name).name for file_name in git_diff_output.split('\n')]
all_changed_files = [
pathlib.Path(file_name).name for file_name in git_diff_output.split("\n")
]
json_changed_files = [file for file in all_changed_files if file.endswith(".json")]

# Create temporary directory
pathlib.Path(TEMP_DIR).mkdir()

# Analyze the changes in discovery artifacts using the changesummary module
changesummary.ChangeSummary(DISCOVERY_DOC_DIR, current_discovery_doc_dir, TEMP_DIR, json_changed_files).detect_discovery_changes()
changesummary.ChangeSummary(
DISCOVERY_DOC_DIR, current_discovery_doc_dir, TEMP_DIR, json_changed_files
).detect_discovery_changes()

# Write a list of the files changed to a file called `changed files` which will be used in the `createcommits.sh` script.
with open(TEMP_DIR / "changed_files", "w") as f:
f.writelines('\n'.join(all_changed_files))
f.writelines("\n".join(all_changed_files))