diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 1c0f786..04c5bfb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,6 +29,10 @@ jobs: - name: Lint check run: rubocop + - name: Set up tests + if: ${{ github.event_name == 'pull_request' }} + run: echo "GITHUB_TOKEN=disabled" >> "$GITHUB_ENV" + - name: Run tests run: bundle exec rake test diff --git a/.rubocop.yml b/.rubocop.yml index 1868b51..27c0ec6 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,6 +10,8 @@ Metrics/AbcSize: Enabled: false Metrics/ClassLength: Enabled: false +Metrics/BlockLength: + Enabled: false AllCops: TargetRubyVersion: 2.7 NewCops: enable diff --git a/.ruby-version b/.ruby-version index b502146..eca690e 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.0.2 +3.0.5 diff --git a/README.md b/README.md index 580bd07..4dd9cd6 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,86 @@ # Changelog-erator :) -This utility is written in Ruby. It uses the Octokit gem to connect to: +This utility is written in Ruby. It uses the [Octokit](https://github.com/octokit) gem to: - connect to Github - retrieve the list of changes between 2 references -While 0.x versions were rather specific to the [Polkadot](https://github.com/paritytech/polkadot) repository, those limitations have been removed starting with 0.9.x versions. +While `0.x` versions were rather specific to the [Polkadot](https://github.com/paritytech/polkadot) repository, those limitations have been removed starting with `0.9.x` versions. The only requirement if you want to benefit from filtering labels in a better way that just comparing label names if to name your labels using the following generic pattern as see [here](https://github.com/paritytech/polkadot/labels): -`-` +`[-]` For instance: - `B0-Silent 😎` - `C1-Important ❗️` +- `Z12` -Each of your issues or PR can have any number of labels but you should avoid having multiple labels of the same category. For instance, you do not want to have both `B0` and `B7`. +Each of your issues or PR can have any number of labels. Advanced rules for labels can be defined and enforced using [ruled_labels](https://github.com/chevdor/ruled_labels). -Running the `changelogerator` will fetch all your changes from Github. You should set a value for the ENV `GITHUB_TOKEN` if you want to avoid being rate limited. +Running `changelogerator` will fetch all the changes for a given repository from Github. You should set a value for the ENV `GITHUB_TOKEN` if you want to avoid being rate limited. The produce json will be formatted as: -``` + +```jsonc { - "cl": { - "meta": { + "polkadot": { + "meta": { // top-level metadata which aggregates the info for .changes[].meta "C": { - "min": 3, - "max": 9, - "count": 3 - }, ... + "min": 1, + "max": 3, + "count": 2 + }, + "D": { + "min": 2, + "max": 2, + "count": 1 + } }, - "repository" : { - "id" : 42, - "node_id" : "...", - "name" : "polkadot", - ... + "repository": { + /* + Includes the fields from the GitHub Repositories REST API. + See: https://docs.github.com/en/rest/repos/repos#get-a-repository + */ }, "changes": [ + /* + Each entry corresponds to a pull request with the fields from the + GitHub PR Rest API. + See: https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request + In addition to the API fields, each entry has a "meta" field with + information computed from its labels. + */ { "meta": { "C": { - "value": 7 + "C1": { // parsed from C1-needs_audit + "value": 1, + "text": "needs_audit" + }, + "C3": { // parsed from C3-urgent + "value": 3, + "text": "urgent" + }, + "agg": { + "min": 1, // related to the value 1 of C1 + "max": 3, // related to the value 3 of C3 + "count": 2 // two C-labels were parsed (C1 and C3) + } }, - "D": { "value": 8 } - }, - ... Github information ... - "number": 1234, - ... way more information about the commit - ... coming from Github + "D": { + "D2": { // parsed from D2-runtime + "value": 2, + "text": "runtime" + }, + "agg": { + "min": 2, // related to the value 2 of D2 + "max": 2, // related to the value 2 of D2 + "count": 1 // one D-label was parsed (D2) + } + } + } + // ... other fields from the REST API } ] } @@ -56,6 +89,12 @@ The produce json will be formatted as: The produced json output can then easily reprocessed into a beautiful changelog using tools such as [`tera`](https://github.com/chevdor/tera-cli). +## Testing + +Some of the tests fetch data from GitHub. In case you do not have a +`$GITHUB_TOKEN` or simply do not wish to run them, set `GITHUB_TOKEN=disabled` +in your environment variables. + ## Usage You can check out the [tests](./test) to see it in action or query the help: @@ -81,15 +120,20 @@ Here is an example: ``` jq \ - --slurpfile srtool_kusama kusama-srtool-digest.json \ - --slurpfile srtool_polkadot polkadot-srtool-digest.json \ - --slurpfile polkadot polkadot.json \ - --slurpfile substrate substrate.json \ + --slurpfile foo.json \ + --slurpfile bar.json \ -n '{ - polkadot: $polkadot[0], - substrate: $substrate[0], - srtool: [ - { name: "kusama", data: $srtool_kusama[0] }, - { name: "polkadot", data: $srtool_polkadot[0] } + foo: $foo[0], + bar: $bar[0], + foobar: [ + { age: 12 }, + { name: "bob" } ] }' | tee context.json ``` + +## Dev notes + +- install: `bundle install` +- test: `rake test` (some of the test appear to be UN responding for a good minute, see how to set `GITHUB_TOKEN` above to speed things up) +- run single test: `ruby -I test test/test_with_token.rb -n test_polkadot_1_commit` +- linting/formatting: `rubocop` diff --git a/Rakefile b/Rakefile index b34baa9..37d8018 100644 --- a/Rakefile +++ b/Rakefile @@ -26,11 +26,12 @@ end # Build the gem task :build do sh 'gem build changelogerator.gemspec' + sh 'mv *.gem build/' end # Install the gem task install: :build do - sh format('gem install %s', gem_name) + sh format('gem install build/%s', gem_name) end # Publish the gem diff --git a/bin/changelogerator b/bin/changelogerator index 7fea340..74174d3 100755 --- a/bin/changelogerator +++ b/bin/changelogerator @@ -44,6 +44,13 @@ of the changes between 2 references on your Github project. puts opts exit end + + opts.on('-V', '--version', 'Show the version') do + gemspec = "#{__dir__}/../changelogerator.gemspec" + gem = Gem::Specification.load(gemspec) + puts format('%s v%s', { v: gem.version, n: gem.name }) + exit + end end.parse! @options[:repo] = ARGV[0] diff --git a/build/.gitkeep b/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/changelogerator.gemspec b/changelogerator.gemspec index ccdcee0..408dee1 100644 --- a/changelogerator.gemspec +++ b/changelogerator.gemspec @@ -1,11 +1,13 @@ # frozen_string_literal: true +require 'rake' + Gem::Specification.new do |s| s.name = 'changelogerator' - s.version = '0.9.1' + s.version = '0.10.0' s.summary = 'Changelog generation/management' s.authors = ['Martin Pugh', 'Wilfried Kopp'] - s.files = ['lib/changelogerator.rb', 'bin/changelogerator'] + s.files = FileList['lib/**/*.rb', 'bin/changelogerator'] s.test_files = s.files.grep(%r{^(test|spec|features)/}) s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) } s.description = 'A utility to fetch the data required to generate a changelog based on change in Github and formatted labels.' @@ -15,4 +17,5 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'git_diff_parser', '~> 3' s.add_runtime_dependency 'octokit', '~> 4' s.required_ruby_version = '>= 2.7' + s.metadata = { 'rubygems_mfa_required' => 'true' } end diff --git a/lib/change.rb b/lib/change.rb new file mode 100644 index 0000000..7fbbb40 --- /dev/null +++ b/lib/change.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'label' + +### A class describe one change that can potentially have several labels +class Change + attr_reader :labels + + def initialize(labels) + # Below we test if we got the full data from Octokit or + # only some fake data (label names only) from our tests. + @labels = labels.map do |label| + if label.respond_to?(:name) + from_octokit(label) + else + from_str(label) + end + end + + @extra = {} + end + + def []=(key, value) + @extra[key] = value + end + + def meta + @extra['meta'] + end + + private + + def from_octokit(label) + Label.new(label.name) + end + + def from_str(label_name) + Label.new(label_name) + end +end diff --git a/lib/changelogerator.rb b/lib/changelogerator.rb index 0b828a3..62f687b 100644 --- a/lib/changelogerator.rb +++ b/lib/changelogerator.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'label' +require 'change' + # A small wrapper class for more easily generating and manipulating Github/Git # changelogs. Given two different git objects (sha, tag, whatever), it will # find all PRs that made up that diff and store them as a list. Also allows @@ -32,13 +35,19 @@ def compute_global_meta change._links = nil change[:meta].each_key do |meta_key| - current = change[:meta][meta_key] - - meta[meta_key] = {} unless meta[meta_key] - meta[meta_key][:min] = current[:value] if !meta[meta_key][:min] || current[:value] < meta[meta_key][:min] - meta[meta_key][:max] = current[:value] if !meta[meta_key][:max] || current[:value] > meta[meta_key][:max] - meta[meta_key][:count] = 0 unless meta[meta_key][:count] - meta[meta_key][:count] += 1 + aggregate = change[:meta][meta_key]['agg'] + + if meta[meta_key] + meta[meta_key][:min] = aggregate['min'] if aggregate['min'] < meta[meta_key][:min] + meta[meta_key][:max] = aggregate['max'] if aggregate['max'] > meta[meta_key][:max] + meta[meta_key][:count] += aggregate['count'] + else + meta[meta_key] = { + min: aggregate['min'], + max: aggregate['max'], + count: aggregate['count'] + } + end end end end @@ -54,18 +63,6 @@ def self.changes_files_in_paths?(change, paths) nil end - # Return the label code for a change - # if the label name matches the expected pattern. - # nil otherwise. - def self.get_label_code(name) - m = match = name.match(/^([a-z])(\d+)-(.*)$/i) - if m - letter, number, text = match.captures - return [letter, number, text] - end - nil - end - ## End of class methods # github_repo: 'paritytech/polkadot' @@ -83,16 +80,17 @@ def initialize(github_repo, from, to, token: '', prefix: nil) @repository = @gh.repository(@repo) @prefix = prefix ids = pr_ids_from_git_diff(from, to) + # The following takes very long time @changes = prs_from_ids(ids) @changes.map do |c| - compute_change_meta(c) + self.class.compute_change_meta(c) end compute_global_meta end def add(change) - compute_change_meta(change) + self.class.compute_change_meta(change) prettify_title(change) changes.prepend(change) @meta = compute_global_meta @@ -123,25 +121,41 @@ def to_json(*_args) JSON.fast_generate(commits, opts) end - private - # Compute and attach metadata about one change - def compute_change_meta(change) + def self.compute_change_meta(change) meta = {} - change.labels.each do |label| - letter, number, text = self.class.get_label_code(label.name) - next unless letter && number + change.labels.each do |lbl| + label = Label.new(lbl.name) + + next unless label + + if meta.key?(label.letter) + aggregate = meta[label.letter]['agg'] + aggregate['max'] = label.number if label.number > aggregate['max'] + aggregate['min'] = label.number if label.number < aggregate['min'] + aggregate['count'] += 1 + else + meta[label.letter] = { + 'agg' => { + 'count' => 1, + 'max' => label.number, + 'min' => label.number + } + } + end - meta[letter] = { - value: number.to_i, - text: text + meta[label.letter]["#{label.letter}#{label.number}"] = { + 'value' => label.number, + 'text' => label.description } end change['meta'] = meta end + private + # Prepend the repo if @prefix is true def prettify_title(pull) pull[:pretty_title] = if @prefix @@ -163,6 +177,7 @@ def pr_ids_from_git_diff(from, to) end.compact.map(&:to_i) end + # TODO: See if we can make this quicker def prs_from_ids(ids) batch_size = 100 prs = [] diff --git a/lib/label.rb b/lib/label.rb new file mode 100644 index 0000000..65c4e31 --- /dev/null +++ b/lib/label.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +### Label = Letter + Number [+ Description] +class Label + attr_accessor :letter, :number, :description + + # Return the label letter for a change if the label name matches the expected pattern. + # nil otherwise. + def parse(label) + m = match = label.match(/^([a-z])(\d+)\s*-?\s*(.*)$/i) + return nil unless m + + letter, digits, text = match.captures + number = digits.to_i + [letter, number, text] + end + + def initialize(input) + raise InvalidInput, 'Invalid, it must be a non-empty string' unless input + + p = parse(input) + raise InvalidLabel, format('Invalid label "%s"', { input: input }) unless p + + @letter = p[0].upcase + @number = p[1] + @description = p[2] unless p[2].empty? + end + + ### Implemented for compatibility reasons + def name + to_str + end + + def to_str + format('%s%d - %s', { l: @letter, n: @number, d: @description }) + end +end + +class InvalidLabel < StandardError; end + +class InvalidInput < StandardError; end diff --git a/test/test_helpers.rb b/test/test_helpers.rb deleted file mode 100644 index 520d8c4..0000000 --- a/test/test_helpers.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require 'json' -require_relative '../lib/changelogerator' -require 'test/unit' - -class TestChangelogerator < Test::Unit::TestCase - def test_helpers - letter, number, text = Changelog.get_label_code('B2-test') - assert_equal('B', letter) - assert_equal('2', number) - assert_equal('test', text) - - letter, number, text = Changelog.get_label_code('Z44-test 😁') - assert_equal('Z', letter) - assert_equal('44', number) - assert_equal('test 😁', text) - - letter, number, text = Changelog.get_label_code('123-foo') - assert_equal(nil, letter) - assert_equal(nil, number) - assert_equal(nil, text) - end -end diff --git a/test/test_local.rb b/test/test_local.rb new file mode 100644 index 0000000..bfc0101 --- /dev/null +++ b/test/test_local.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'json' +require_relative '../lib/changelogerator' +require_relative '../lib/change' +require 'test/unit' + +class TestLocal < Test::Unit::TestCase + def test_meta_1_commit + change = Change.new(%w[A1-foo A2-foo B0-foo B1-foo B2-foo]) + Changelog.compute_change_meta(change) + + assert_equal(%w[A B], change.meta.keys) + + assert_equal(change.meta['A']['agg']['min'], 1) # A(1) + assert_equal(change.meta['A']['agg']['max'], 2) # A(2) + assert_equal(change.meta['A']['agg']['count'], 2) # A1 + A2 + + assert_equal(change.meta['B']['agg']['min'], 0) # B(0) + assert_equal(change.meta['B']['agg']['max'], 2) # B(2) + assert_equal(change.meta['B']['agg']['count'], 3) # B0 + B1 + B2 + + assert_equal(JSON.pretty_generate(change.meta), '{ + "A": { + "agg": { + "count": 2, + "max": 2, + "min": 1 + }, + "A1": { + "value": 1, + "text": "foo" + }, + "A2": { + "value": 2, + "text": "foo" + } + }, + "B": { + "agg": { + "count": 3, + "max": 2, + "min": 0 + }, + "B0": { + "value": 0, + "text": "foo" + }, + "B1": { + "value": 1, + "text": "foo" + }, + "B2": { + "value": 2, + "text": "foo" + } + } +}') + end +end diff --git a/test/test_meta.rb b/test/test_meta.rb deleted file mode 100644 index 6a9cfdc..0000000 --- a/test/test_meta.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require 'json' -require_relative '../lib/changelogerator' -require 'test/unit' - -class TestChangelogerator < Test::Unit::TestCase - def test_meta_1_commit - ref1 = '2ebabcec7fcbb3d13a965a852df0559a4aa12a5e' - ref2 = '9c05f9753b2f939ccf5ba18c08dd4c83c3ab9e0b' - - cl = Changelog.new( - 'paritytech/polkadot', ref1, ref2, - token: ENV['GITHUB_TOKEN'], - prefix: true - ) - - j = cl.to_json - assert_equal(1, cl.changes.length) - assert_equal(%w[A B C], cl.changes[0].meta.keys) # A2 + B0 + C1 - - p cl.meta - - puts format('JSON Length: %d', j.length) - assert(j.length > 11_000) - assert(j.length < 13_000) - end -end diff --git a/test/test_parser.rb b/test/test_parser.rb new file mode 100644 index 0000000..2607f5d --- /dev/null +++ b/test/test_parser.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require_relative '../lib/label' +require 'test/unit' + +class TestParser < Test::Unit::TestCase + def test1 + lbl = Label.new('B2-foo') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo', lbl.description) + end + + def test_no_desc + lbl = Label.new('B2-') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal(nil, lbl.description) + end + + def test_no_dash + lbl = Label.new('B2') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal(nil, lbl.description) + end + + def test_double_digits + lbl = Label.new('B12-foo') + assert_equal('B', lbl.letter) + assert_equal(12, lbl.number) + assert_equal('foo', lbl.description) + end + + def test_spacing1 + lbl = Label.new('B2 -foo') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo', lbl.description) + end + + def test_spacing2 + lbl = Label.new('B2- foo') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo', lbl.description) + end + + def test_spacing3 + lbl = Label.new('B2 - foo') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo', lbl.description) + end + + def test_description_with_spaces1 + lbl = Label.new('B2-foo bar') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo bar', lbl.description) + end + + def test_description_with_spaces2 + lbl = Label.new('B2- foo bar') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo bar', lbl.description) + end + + def test_description_with_emojis + lbl = Label.new('B2-foo bar 😄😄😄') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo bar 😄😄😄', lbl.description) + end + + def test_case1 + lbl = Label.new('b2-foo Bar') + assert_equal('B', lbl.letter) + assert_equal(2, lbl.number) + assert_equal('foo Bar', lbl.description) + end + + def test_case_no_label + assert_raises InvalidLabel, 'xxx' do + Label.new('123-foo') + end + end + + def test_leading_zeroes + lbl = Label.new('A00099-foo') + assert_equal('A', lbl.letter) + assert_equal(99, lbl.number) + assert_equal('foo', lbl.description) + end +end diff --git a/test/test_basic.rb b/test/test_with_token.rb similarity index 84% rename from test/test_basic.rb rename to test/test_with_token.rb index 460ae0f..51f6be3 100644 --- a/test/test_basic.rb +++ b/test/test_with_token.rb @@ -3,8 +3,15 @@ require 'json' require_relative '../lib/changelogerator' require 'test/unit' +# require 'pry' class TestChangelogerator < Test::Unit::TestCase + def setup + gh_token = ENV['GITHUB_TOKEN'] + omit("Skipping because $GITHUB_TOKEN=#{gh_token}") if gh_token == 'disabled' + raise 'Missing $GITHUB_TOKEN token' if gh_token.nil? || gh_token.empty? + end + def test_polkadot_1_commit ref1 = '2ebabcec7fcbb3d13a965a852df0559a4aa12a5e' ref2 = '9c05f9753b2f939ccf5ba18c08dd4c83c3ab9e0b' @@ -35,7 +42,7 @@ def test_polkadot_many_commits assert_equal(244, cl.changes.length) puts format('JSON Length: %d', j.length) assert(j.length > 1_600_000) - assert(j.length < 1_700_000) + assert(j.length < 1_750_000) end def test_cumulus_many_commits