Add GHA workflow to build CLI API docs (#6187)

.changes/unreleased/Under the Hood-20221116-130037.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Add github actions workflow to generate high level CLI API docs
+time: 2022-11-16T13:00:37.916202-06:00
+custom:
+  Author: stu-k
+  Issue: "5942"
+  PR: "6187"

.github/workflows/generate-cli-api-docs.yml

@@ -0,0 +1,166 @@
+# **what?**
+# On push, if anything in core/dbt/docs or core/dbt/cli has been
+# created or modified, regenerate the CLI API docs using sphinx.
+
+# **why?**
+# We watch for changes in core/dbt/cli because the CLI API docs rely on click
+# and all supporting flags/params to be generated. We watch for changes in
+# core/dbt/docs since any changes to sphinx configuration or any of the
+# .rst files there could result in a differently build final index.html file.
+
+# **when?**
+# Whenever a change has been pushed to a branch, and only if there is a diff
+# between the PR branch and main's core/dbt/cli and or core/dbt/docs dirs.
+
+# TODO: add bot comment to PR informing contributor that the docs have been committed
+# TODO: figure out why github action triggered pushes cause github to fail to report
+#       the status of jobs
+
+name: Generate CLI API docs
+
+on:
+  pull_request:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  CLI_DIR: ${{ github.workspace }}/core/dbt/cli
+  DOCS_DIR: ${{ github.workspace }}/core/dbt/docs
+  DOCS_BUILD_DIR: ${{ github.workspace }}/core/dbt/docs/build
+
+jobs:
+  check_gen:
+    name: check if generation needed
+    runs-on: ubuntu-latest
+    outputs:
+      cli_dir_changed: ${{ steps.check_cli.outputs.cli_dir_changed }}
+      docs_dir_changed: ${{ steps.check_docs.outputs.docs_dir_changed }}
+
+    steps:
+    - name: "[DEBUG] print variables"
+      run: |
+        echo "env.CLI_DIR:        ${{ env.CLI_DIR }}"
+        echo "env.DOCS_BUILD_DIR: ${{ env.DOCS_BUILD_DIR }}"
+        echo "env.DOCS_DIR:       ${{ env.DOCS_DIR }}"
+        echo ">>>>> git log"
+        git log --pretty=oneline | head -5
+
+    - name: git checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        ref: ${{ github.head_ref }}
+
+    - name: set shas
+      id: set_shas
+      run: |
+        THIS_SHA=$(git rev-parse @)
+        LAST_SHA=$(git rev-parse @~1)
+
+        echo "this sha: $THIS_SHA"
+        echo "last sha: $LAST_SHA"
+
+        echo "this_sha=$THIS_SHA" >> $GITHUB_OUTPUT
+        echo "last_sha=$LAST_SHA" >> $GITHUB_OUTPUT
+
+    - name: check for changes in core/dbt/cli
+      id: check_cli
+      run: |
+        CLI_DIR_CHANGES=$(git diff \
+          ${{ steps.set_shas.outputs.last_sha }} \
+          ${{ steps.set_shas.outputs.this_sha }} \
+          -- ${{ env.CLI_DIR }})
+
+        if [ -n "$CLI_DIR_CHANGES" ]; then
+          echo "changes found"
+          echo $CLI_DIR_CHANGES
+          echo "cli_dir_changed=true" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+        echo "cli_dir_changed=false" >> $GITHUB_OUTPUT
+        echo "no changes found"
+
+    - name: check for changes in core/dbt/docs
+      id: check_docs
+      if: steps.check_cli.outputs.cli_dir_changed == 'false'
+      run: |
+        DOCS_DIR_CHANGES=$(git diff --name-only \
+          ${{ steps.set_shas.outputs.last_sha }} \
+          ${{ steps.set_shas.outputs.this_sha }} \
+          -- ${{ env.DOCS_DIR }} ':!${{ env.DOCS_BUILD_DIR }}')
+
+        DOCS_BUILD_DIR_CHANGES=$(git diff --name-only \
+          ${{ steps.set_shas.outputs.last_sha }} \
+          ${{ steps.set_shas.outputs.this_sha }} \
+          -- ${{ env.DOCS_BUILD_DIR }})
+
+        if [ -n "$DOCS_DIR_CHANGES" ] && [ -z "$DOCS_BUILD_DIR_CHANGES" ]; then
+          echo "changes found"
+          echo $DOCS_DIR_CHANGES
+          echo "docs_dir_changed=true" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+        echo "docs_dir_changed=false" >> $GITHUB_OUTPUT
+        echo "no changes found"
+
+  gen_docs:
+    name: generate docs
+    runs-on: ubuntu-latest
+    needs: [check_gen]
+    if: |
+      needs.check_gen.outputs.cli_dir_changed == 'true'
+      || needs.check_gen.outputs.docs_dir_changed == 'true'
+
+    steps:
+      - name: "[DEBUG] print variables"
+        run: |
+          echo "env.DOCS_DIR:    ${{ env.DOCS_DIR }}"
+          echo "github head_ref: ${{ github.head_ref }}"
+
+      - name: git checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: install python
+        uses: actions/setup-python@v4.3.0
+        with:
+          python-version: 3.8
+
+      - name: install dev requirements
+        run: |
+          python3 -m venv env
+          source env/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt -r dev-requirements.txt
+
+      - name: generate docs
+        run: |
+          source env/bin/activate
+          cd ${{ env.DOCS_DIR }}
+
+          echo "cleaning existing docs"
+          make clean
+
+          echo "creating docs"
+          make html
+
+      - name: debug
+        run: |
+          echo ">>>>> status"
+          git status
+          echo ">>>>> remotes"
+          git remote -v
+          echo ">>>>> branch"
+          git branch -v
+          echo ">>>>> log"
+          git log --pretty=oneline | head -5
+
+      - name: commit docs
+        run: |
+          git config user.name 'Github Build Bot'
+          git config user.email 'buildbot@fishtownanalytics.com'
+          git commit -am "Add generated CLI API docs"
+          git push -u origin ${{ github.head_ref }}

.gitignore

@@ -11,6 +11,7 @@ __pycache__/
 env*/
 dbt_env/
 build/
+!core/dbt/docs/build
 develop-eggs/
 dist/
 downloads/

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # Eventually the hooks described here will be run as tests before merging each PR.
 
 # TODO: remove global exclusion of tests when testing overhaul is complete
-exclude: ^test/
+exclude: ^(test/|core/dbt/docs/build/)
 
 # Force all unspecified python hooks to run python 3.8
 default_language_version:

core/dbt/docs/build/html/.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 1ee31fc16e025fb98598189ba2cb5fcb
+tags: 645f666f9bcd5a90fca523b33c5a78b7

core/dbt/docs/build/html/_sources/index.rst.txt

@@ -0,0 +1,4 @@
+dbt-core's API documentation
+============================
+
+.. dbt_click:: dbt.cli.main:cli

core/dbt/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js

@@ -0,0 +1,134 @@
+/*
+ * _sphinx_javascript_frameworks_compat.js
+ * ~~~~~~~~~~
+ *
+ * Compatability shim for jQuery and underscores.js.
+ *
+ * WILL BE REMOVED IN Sphinx 6.0
+ * xref RemovedInSphinx60Warning
+ *
+ */
+
+/**
+ * select a different prefix for underscore
+ */
+$u = _.noConflict();
+
+
+/**
+ * small helper function to urldecode strings
+ *
+ * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#Decoding_query_parameters_from_a_URL
+ */
+jQuery.urldecode = function(x) {
+    if (!x) {
+        return x
+    }
+    return decodeURIComponent(x.replace(/\+/g, ' '));
+};
+
+/**
+ * small helper function to urlencode strings
+ */
+jQuery.urlencode = encodeURIComponent;
+
+/**
+ * This function returns the parsed url parameters of the
+ * current request. Multiple values per key are supported,
+ * it will always return arrays of strings for the value parts.
+ */
+jQuery.getQueryParameters = function(s) {
+    if (typeof s === 'undefined')
+        s = document.location.search;
+    var parts = s.substr(s.indexOf('?') + 1).split('&');
+    var result = {};
+    for (var i = 0; i < parts.length; i++) {
+        var tmp = parts[i].split('=', 2);
+        var key = jQuery.urldecode(tmp[0]);
+        var value = jQuery.urldecode(tmp[1]);
+        if (key in result)
+            result[key].push(value);
+        else
+            result[key] = [value];
+    }
+    return result;
+};
+
+/**
+ * highlight a given string on a jquery object by wrapping it in
+ * span elements with the given class name.
+ */
+jQuery.fn.highlightText = function(text, className) {
+    function highlight(node, addItems) {
+        if (node.nodeType === 3) {
+            var val = node.nodeValue;
+            var pos = val.toLowerCase().indexOf(text);
+            if (pos >= 0 &&
+                !jQuery(node.parentNode).hasClass(className) &&
+                !jQuery(node.parentNode).hasClass("nohighlight")) {
+                var span;
+                var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
+                if (isInSVG) {
+                    span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
+                } else {
+                    span = document.createElement("span");
+                    span.className = className;
+                }
+                span.appendChild(document.createTextNode(val.substr(pos, text.length)));
+                node.parentNode.insertBefore(span, node.parentNode.insertBefore(
+                    document.createTextNode(val.substr(pos + text.length)),
+                    node.nextSibling));
+                node.nodeValue = val.substr(0, pos);
+                if (isInSVG) {
+                    var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
+                    var bbox = node.parentElement.getBBox();
+                    rect.x.baseVal.value = bbox.x;
+                    rect.y.baseVal.value = bbox.y;
+                    rect.width.baseVal.value = bbox.width;
+                    rect.height.baseVal.value = bbox.height;
+                    rect.setAttribute('class', className);
+                    addItems.push({
+                        "parent": node.parentNode,
+                        "target": rect});
+                }
+            }
+        }
+        else if (!jQuery(node).is("button, select, textarea")) {
+            jQuery.each(node.childNodes, function() {
+                highlight(this, addItems);
+            });
+        }
+    }
+    var addItems = [];
+    var result = this.each(function() {
+        highlight(this, addItems);
+    });
+    for (var i = 0; i < addItems.length; ++i) {
+        jQuery(addItems[i].parent).before(addItems[i].target);
+    }
+    return result;
+};
+
+/*
+ * backward compatibility for jQuery.browser
+ * This will be supported until firefox bug is fixed.
+ */
+if (!jQuery.browser) {
+    jQuery.uaMatch = function(ua) {
+        ua = ua.toLowerCase();
+
+        var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
+            /(webkit)[ \/]([\w.]+)/.exec(ua) ||
+            /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
+            /(msie) ([\w.]+)/.exec(ua) ||
+            ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
+            [];
+
+        return {
+            browser: match[ 1 ] || "",
+            version: match[ 2 ] || "0"
+        };
+    };
+    jQuery.browser = {};
+    jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
+}