Skip to content

Commit a69ce1c

Browse files
averikitschaverikitsch
authored
chore(ci): Add kokoro pipeline for release and docs (#12)
* chore(ci): Add kokoro pipeline for release and docs * lint * Update publish-docs.sh
1 parent a97905c commit a69ce1c

File tree

15 files changed

+1649
-171
lines changed

15 files changed

+1649
-171
lines changed

‎.kokoro/build.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
# Copyright 2024 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
if [[ -z "${PROJECT_ROOT:-}" ]]; then
19+
PROJECT_ROOT="github/langchain-google-spanner-python"
20+
fi
21+
22+
cd "${PROJECT_ROOT}"
23+
24+
# Disable buffering, so that the logs stream through.
25+
export PYTHONUNBUFFERED=1
26+
27+
# Debug: show build environment
28+
env | grep KOKORO
29+
30+
# Setup service account credentials.
31+
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
32+
33+
# Setup project id.
34+
export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
35+
36+
# Remove old nox
37+
python3 -m pip uninstall --yes --quiet nox-automation
38+
39+
# Install nox
40+
python3 -m pip install --upgrade --quiet nox
41+
python3 -m nox --version
42+
43+
# If this is a continuous build, send the test log to the FlakyBot.
44+
# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
45+
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
46+
cleanup() {
47+
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
48+
$KOKORO_GFILE_DIR/linux_amd64/flakybot
49+
}
50+
trap cleanup EXIT HUP
51+
fi
52+
53+
# If NOX_SESSION is set, it only runs the specified session,
54+
# otherwise run all the sessions.
55+
if [[ -n "${NOX_SESSION:-}" ]]; then
56+
python3 -m nox -s ${NOX_SESSION:-}
57+
else
58+
python3 -m nox
59+
fi

‎.kokoro/docker/docs/Dockerfile‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from ubuntu:22.04
16+
17+
ENV DEBIAN_FRONTEND noninteractive
18+
19+
# Ensure local Python is preferred over distribution Python.
20+
ENV PATH /usr/local/bin:$PATH
21+
22+
# Install dependencies.
23+
RUN apt-get update \
24+
&& apt-get install -y --no-install-recommends \
25+
apt-transport-https \
26+
build-essential \
27+
ca-certificates \
28+
curl \
29+
dirmngr \
30+
git \
31+
gpg-agent \
32+
graphviz \
33+
libbz2-dev \
34+
libdb5.3-dev \
35+
libexpat1-dev \
36+
libffi-dev \
37+
liblzma-dev \
38+
libreadline-dev \
39+
libsnappy-dev \
40+
libssl-dev \
41+
libsqlite3-dev \
42+
portaudio19-dev \
43+
python3-distutils \
44+
redis-server \
45+
software-properties-common \
46+
ssh \
47+
sudo \
48+
tcl \
49+
tcl-dev \
50+
tk \
51+
tk-dev \
52+
uuid-dev \
53+
wget \
54+
zlib1g-dev \
55+
&& add-apt-repository universe \
56+
&& apt-get update \
57+
&& apt-get -y install jq \
58+
&& apt-get clean autoclean \
59+
&& apt-get autoremove -y \
60+
&& rm -rf /var/lib/apt/lists/* \
61+
&& rm -f /var/cache/apt/archives/*.deb
62+
63+
ENV VERSION v4.2.0
64+
ENV BINARY yq_linux_amd64
65+
RUN wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && \
66+
chmod +x /usr/bin/yq
67+
68+
###################### Install python 3.9.13
69+
70+
# Download python 3.9.13
71+
RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz
72+
73+
# Extract files
74+
RUN tar -xvf Python-3.9.13.tgz
75+
76+
# Install python 3.9.13
77+
RUN ./Python-3.9.13/configure --enable-optimizations
78+
RUN make altinstall
79+
80+
###################### Install pip
81+
RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
82+
&& python3 /tmp/get-pip.py \
83+
&& rm /tmp/get-pip.py
84+
85+
# Test pip
86+
RUN python3 -m pip
87+
88+
CMD ["python3.8"]

‎.kokoro/docs/common.cfg‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
# Build logs will be here
4+
action {
5+
define_artifacts {
6+
regex: "**/*sponge_log.xml"
7+
}
8+
}
9+
10+
# Download trampoline resources.
11+
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
12+
13+
# Use the trampoline script to run in docker.
14+
build_file: "langchain-google-spanner-python/.kokoro/trampoline_v2.sh"
15+
16+
# Configure the docker image for kokoro-trampoline.
17+
env_vars: {
18+
key: "TRAMPOLINE_IMAGE"
19+
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
20+
}
21+
env_vars: {
22+
key: "TRAMPOLINE_BUILD_FILE"
23+
value: "github/langchain-google-spanner-python/.kokoro/publish-docs.sh"
24+
}
25+
26+
env_vars: {
27+
key: "STAGING_BUCKET"
28+
value: "docs-staging"
29+
}
30+
31+
env_vars: {
32+
key: "V2_STAGING_BUCKET"
33+
# Push non-cloud library docs to `docs-staging-v2-staging` instead of the
34+
# Cloud RAD bucket `docs-staging-v2`
35+
value: "docs-staging-v2-staging"
36+
}
37+
38+
# It will upload the docker image after successful builds.
39+
env_vars: {
40+
key: "TRAMPOLINE_IMAGE_UPLOAD"
41+
value: "true"
42+
}
43+
44+
# It will always build the docker image.
45+
env_vars: {
46+
key: "TRAMPOLINE_DOCKERFILE"
47+
value: ".kokoro/docker/docs/Dockerfile"
48+
}
49+
50+
# Fetch the token needed for reporting release status to GitHub
51+
before_action {
52+
fetch_keystore {
53+
keystore_resource {
54+
keystore_config_id: 73713
55+
keyname: "yoshi-automation-github-key"
56+
}
57+
}
58+
}
59+
60+
before_action {
61+
fetch_keystore {
62+
keystore_resource {
63+
keystore_config_id: 73713
64+
keyname: "docuploader_service_account"
65+
}
66+
}
67+
}

‎.kokoro/docs/docs-presubmit.cfg‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
env_vars: {
4+
key: "STAGING_BUCKET"
5+
value: "gcloud-python-test"
6+
}
7+
8+
env_vars: {
9+
key: "V2_STAGING_BUCKET"
10+
value: "gcloud-python-test"
11+
}
12+
13+
# We only upload the image in the main `docs` build.
14+
env_vars: {
15+
key: "TRAMPOLINE_IMAGE_UPLOAD"
16+
value: "false"
17+
}
18+
19+
env_vars: {
20+
key: "TRAMPOLINE_BUILD_FILE"
21+
value: "github/langchain-google-spanner-python/.kokoro/build.sh"
22+
}
23+
24+
# Only run this nox session.
25+
env_vars: {
26+
key: "NOX_SESSION"
27+
value: "docs docfx"
28+
}

‎.kokoro/docs/docs.cfg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto

‎.kokoro/populate-secrets.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
# Copyright 2024 Google LLC.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
19+
function msg { println "$*" >&2 ;}
20+
function println { printf '%s\n' "$(now) $*" ;}
21+
22+
23+
# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
24+
# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
25+
SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
26+
msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
27+
mkdir -p ${SECRET_LOCATION}
28+
for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
29+
do
30+
msg "Retrieving secret ${key}"
31+
docker run --entrypoint=gcloud \
32+
--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
33+
gcr.io/google.com/cloudsdktool/cloud-sdk \
34+
secrets versions access latest \
35+
--project cloud-devrel-kokoro-resources \
36+
--secret ${key} > \
37+
"${SECRET_LOCATION}/${key}"
38+
if [[ $? == 0 ]]; then
39+
msg "Secret written to ${SECRET_LOCATION}/${key}"
40+
else
41+
msg "Error retrieving secret ${key}"
42+
fi
43+
done

‎.kokoro/publish-docs.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Copyright 2024 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
# Disable buffering, so that the logs stream through.
19+
export PYTHONUNBUFFERED=1
20+
21+
export PATH="${HOME}/.local/bin:${PATH}"
22+
23+
# Install nox
24+
python3 -m pip install --require-hashes -r .kokoro/requirements.txt
25+
python3 -m nox --version
26+
27+
# build docs
28+
nox -s docs
29+
30+
VERSION=$(python -c "import langchain_google_spanner;print(langchain_google_spanner.__version__)")
31+
32+
# create metadata
33+
python3 -m docuploader create-metadata \
34+
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
35+
--version=$VERSION \
36+
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
37+
--distribution-name=$(yq -oy '.project.name' pyproject.toml) \
38+
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
39+
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
40+
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
41+
42+
cat docs.metadata
43+
44+
# upload docs
45+
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
46+
47+
48+
# docfx yaml files
49+
nox -s docfx
50+
51+
# create metadata.
52+
python3 -m docuploader create-metadata \
53+
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
54+
--version=$VERSION \
55+
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
56+
--distribution-name=$(yq -oy '.project.name' pyproject.toml) \
57+
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
58+
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
59+
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
60+
61+
cat docs.metadata
62+
63+
# upload docs
64+
python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"

‎.kokoro/release.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# Copyright 2024 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
# Start the releasetool reporter
19+
python3 -m pip install --require-hashes -r github/langchain-google-spanner-python/.kokoro/requirements.txt
20+
python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
21+
22+
# Disable buffering, so that the logs stream through.
23+
export PYTHONUNBUFFERED=1
24+
25+
# Move into the package, build the distribution and upload.
26+
TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
27+
cd github/langchain-google-spanner-python
28+
python3 -m build --wheel
29+
twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*

0 commit comments

Comments
 (0)