New changelog scripts

.github/workflows/publish-draft-release.yml

@@ -28,6 +28,7 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v2
+
       - name: Cache target dir
         uses: actions/cache@v2
         with:
@@ -36,21 +37,25 @@ jobs:
           restore-keys: |
             srtool-target-${{ matrix.runtime }}-
             srtool-target-
+
       - name: Build ${{ matrix.runtime }} runtime
         id: srtool_build
         uses: chevdor/[email protected]
         with:
           # This is the default with chevdor/[email protected] but we make it clear
           image: paritytech/srtool
           chain: ${{ matrix.runtime }}
+
       - name: Store srtool digest to disk
         run: |
           echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.runtime }}_srtool_output.json
+
       - name: Upload ${{ matrix.runtime }} srtool json
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.runtime }}-srtool-json
           path: ${{ matrix.runtime }}_srtool_output.json
+
       - name: Upload ${{ matrix.runtime }} runtime
         uses: actions/upload-artifact@v2
         with:
@@ -70,20 +75,59 @@ jobs:
         with:
           fetch-depth: 0
           path: polkadot
-      - name: Set up Ruby 2.7
-        uses: actions/setup-ruby@v1
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: 3.0.0
+
       - name: Download srtool json output
         uses: actions/download-artifact@v2
-      - name: Generate release text
+
+      - name: Prepare tooling
+        run: |
+          cd polkadot/scripts/changelog
+          gem install bundler changelogerator:0.9.1
+          bundle install
+          changelogerator --help
+
+          URL=https://github.com/chevdor/tera-cli/releases/download/v0.2.1/tera-cli_linux_amd64.deb
+          wget $URL -O tera.deb
+          sudo dpkg -i tera.deb
+          tera --version
+
+      - name: Generate release notes
         env:
           RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
           RUSTC_NIGHTLY: ${{ needs.get-rust-versions.outputs.rustc-nightly }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NO_CACHE: 1
+          DEBUG: 1
+          WESTEND_DIGEST: ${{ github.workspace}}/westend-srtool-json/westend_srtool_output.json
+          KUSAMA_DIGEST: ${{ github.workspace}}/kusama-srtool-json/kusama_srtool_output.json
+          POLKADOT_DIGEST: ${{ github.workspace}}/polkadot-srtool-json/polkadot_srtool_output.json
+          PRE_RELEASE: ${{ github.event.inputs.pre_release }}
+          HIDE_SRTOOL_ROCOCO: false
         run: |
-          gem install changelogerator:0.0.16 git toml
-          ruby $GITHUB_WORKSPACE/polkadot/scripts/github/generate_release_text.rb | tee release_text.md
+          find ${{env.GITHUB_WORKSPACE}} -type f -name "*_srtool_output.json"
+          ls -al $WESTEND_DIGEST
+          ls -al $KUSAMA_DIGEST
+          ls -al $POLKADOT_DIGEST
+
+          cd polkadot/scripts/changelog
+
+          ./bin/changelog
+          ls -al release-notes.md
+          ls -al context.json
+
+      - name: Archive artifact context.json
+        uses: actions/upload-artifact@v2
+        with:
+          name: release-notes-context
+          path: |
+            polkadot/scripts/changelog/context.json
+            **/*_srtool_output.json
+
       - name: Create draft release
         id: create-release
         uses: actions/create-release@v1
@@ -92,12 +136,14 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: Polkadot ${{ github.ref }}
-          body_path: ./release_text.md
+          body_path: ./polkadot/scripts/changelog/release-notes.md
           draft: true
 
   publish-runtimes:
     runs-on: ubuntu-latest
     needs: ["publish-draft-release"]
+    env:
+      RUNTIME_DIR: runtime
     strategy:
       matrix:
         runtime: ["polkadot", "kusama", "westend"]
@@ -107,16 +153,21 @@ jobs:
       - name: Download artifacts
         uses: actions/download-artifact@v2
       - name: Set up Ruby 2.7
-        uses: actions/setup-ruby@v1
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: 3.0.0
       - name: Get runtime version
         id: get-runtime-ver
         run: |
-          ls
-          ls "${{ matrix.runtime }}-runtime"
-          runtime_ver="$(ruby -e 'require "./scripts/github/lib.rb"; puts get_runtime("${{ matrix.runtime }}")')"
+          echo "require './scripts/github/lib.rb'" > script.rb
+          echo "puts get_runtime(runtime: \"${{ matrix.runtime }}\", runtime_dir: \"$RUNTIME_DIR\")" >> script.rb
+
+          echo "Current folder: $PWD"
+          ls "$RUNTIME_DIR/${{ matrix.runtime }}"
+          runtime_ver=$(ruby script.rb)
+          echo "Found version: >$runtime_ver<"
           echo "::set-output name=runtime_ver::$runtime_ver"
+
       - name: Upload compressed ${{ matrix.runtime }} wasm
         uses: actions/upload-release-asset@v1
         env:

scripts/changelog/.gitignore

@@ -0,0 +1,4 @@
+changelog.md
+*.json
+release*.md
+.env

scripts/changelog/Gemfile

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gem 'octokit', '~> 4'
+
+gem 'git_diff_parser', '~> 3'
+
+gem 'toml', '~> 0.3.0'
+
+gem 'rake', group: :dev
+
+gem 'optparse', '~> 0.1.1'
+
+gem 'logger', '~> 1.4'
+
+gem 'test-unit', group: :dev
+
+gem 'rubocop', group: :dev, require: false

scripts/changelog/Gemfile.lock

@@ -0,0 +1,79 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    git_diff_parser (3.2.0)
+    logger (1.4.4)
+    multipart-post (2.1.1)
+    octokit (4.21.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    optparse (0.1.1)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    parslet (2.0.0)
+    power_assert (2.0.1)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.23.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.13.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    test-unit (3.5.1)
+      power_assert
+    toml (0.3.0)
+      parslet (>= 1.8.0, < 3.0.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  git_diff_parser (~> 3)
+  logger (~> 1.4)
+  octokit (~> 4)
+  optparse (~> 0.1.1)
+  rake
+  rubocop
+  test-unit
+  toml (~> 0.3.0)
+
+BUNDLED WITH
+   2.2.22

scripts/changelog/README.md

@@ -0,0 +1,71 @@
+# Changelog
+
+Currently, the changelog is built locally. It will be moved to CI once labels stabilize.
+
+For now, a bit of preparation is required before you can run the script:
+- fetch the srtool digests
+- store them under the `digests` folder as `<chain>-srtool-digest.json`
+- ensure the `.env` file is up to date with correct information
+
+The content of the release notes is generated from the template files under the `scripts/changelog/templates` folder. For readability and maintenance, the template is split into several small snippets.
+
+Run:
+```
+./bin/changelog <ref_since> [<ref_until>=HEAD]
+```
+
+For instance:
+```
+./bin/changelog statemine-v5.0.0
+```
+
+A file called `release-notes.md` will be generated and can be used for the release.
+
+## ENV
+
+You may use the following ENV for testing:
+
+```
+RUSTC_STABLE="rustc 1.56.1 (59eed8a2a 2021-11-01)"
+RUSTC_NIGHTLY="rustc 1.57.0-nightly (51e514c0f 2021-09-12)"
+PRE_RELEASE=true
+HIDE_SRTOOL_ROCOCO=true
+HIDE_SRTOOL_SHELL=true
+REF1=statemine-v5.0.0
+REF2=HEAD
+DEBUG=1
+NO_CACHE=1
+```
+## Considered labels
+
+The following list will likely evolve over time and it will be hard to keep it in sync.
+In any case, if you want to find all the labels that are used, search for `meta` in the templates.
+Currently, the considered labels are:
+
+- Priority: C<N> labels
+- Audit: D<N> labels
+- E4 => new host function
+- B0 => silent, not showing up
+- B1-releasenotes (misc unless other labels)
+- B5-client (client changes)
+- B7-runtimenoteworthy (runtime changes)
+- T6-XCM
+
+Note that labels with the same letter are mutually exclusive.
+A PR should not have both `B0` and `B5`, or both `C1` and `C9`. In case of conflicts, the template will
+decide which label will be considered.
+
+## Dev and debuggin
+
+### Hot Reload
+
+The following command allows **Hot Reload**:
+```
+fswatch templates -e ".*\.md$" | xargs -n1 -I{} ./bin/changelog statemine-v5.0.0
+```
+### Caching
+
+By default, if the changelog data from Github is already present, the calls to the Github API will be skipped
+and the local version of the data will be used. This is much faster.
+If you know that some labels have changed in Github, you probably want to refresh the data.
+You can then either delete manually the `cumulus.json` file or `export NO_CACHE=1` to force refreshing the data.