diff --git a/README.md b/README.md index cc8fa57b..2abfe1e0 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,9 @@ Parameters: * `ignorePatterns` an array of objects holding regular expressions which a link is checked against and skipped for checking in case of a match. Example: `[{ pattern: /foo/ }]` * `replacementPatterns` an array of objects holding regular expressions which are replaced in a link with their corresponding replacement string. This behavior allows (for example) to adapt to certain platform conventions hosting the Markdown. Example: `[{ pattern: /^.attachments/, replacement: "file://some/conventional/folder/.attachments" }]` * `ignoreDisable` if this is `true` then disable comments are ignored. + * `retryOn429` if this is `true` then retry request when response is an HTTP code 429 after the duration indicated by `retry-after` header. + * `aliveStatusCodes` a list of HTTP codes to consider as alive. + Example: `[200,206]` * `callback` function which accepts `(err, results)`. * `err` an Error object when the operation cannot be completed, otherwise `null`. * `results` an array of objects with the following properties: @@ -129,17 +132,16 @@ If not supplied, the tool reads from standard input. #### Usage ``` - - Usage: markdown-link-check [options] [filenameOrUrl] - - Options: - - -p, --progress show progress bar - -c, --config [config] apply a configuration file (JSON) - -q, --quiet display errors only - -v, --verbose displays detailed error information - -h, --help output usage information - +Usage: markdown-link-check [options] [filenameOrUrl] + +Options: + -p, --progress show progress bar + -c, --config [config] apply a config file (JSON), holding e.g. url specific header configuration + -q, --quiet displays errors only + -v, --verbose displays detailed error information + -a, --alive comma separated list of HTTP code to be considered as alive + -r, --retry retry after the duration indicated in 'retry-after' header when HTTP code is 429 + -h, --help display help for command ``` ##### Config file format @@ -149,6 +151,8 @@ If not supplied, the tool reads from standard input. * `ignorePatterns`: An array of objects holding regular expressions which a link is checked against and skipped for checking in case of a match. * `replacementPatterns`: An array of objects holding regular expressions which are replaced in a link with their corresponding replacement string. This behavior allows (for example) to adapt to certain platform conventions hosting the Markdown. * `httpHeaders`: The headers are only applied to links where the link **starts with** one of the supplied URLs in the `urls` section. +* `retryOn429` if this is `true` then retry request when response is an HTTP code 429 after the duration indicated by `retry-after` header. +* `aliveStatusCodes` a list of HTTP codes to consider as alive. **Example:** @@ -175,7 +179,9 @@ If not supplied, the tool reads from standard input. "Foo": "Bar" } } - ] + ], + "retryOn429":true, + "aliveStatusCodes":[200, 206] } ``` diff --git a/index.js b/index.js index b4db1e80..e1602ec3 100644 --- a/index.js +++ b/index.js @@ -72,6 +72,7 @@ module.exports = function markdownLinkCheck(markdown, opts, callback) { } linkCheck(link, opts, function (err, result) { + if (opts.showProgressBar) { bar.tick(); } diff --git a/markdown-link-check b/markdown-link-check index fbe95200..63d34de9 100755 --- a/markdown-link-check +++ b/markdown-link-check @@ -21,11 +21,19 @@ const opts = {}; let filenameForOutput = ''; let stream = process.stdin; // read from stdin unless a filename is given +function commaSeparatedCodesList(value, dummyPrevious) { + return value.split(',').map(function(item) { + return parseInt(item, 10); + }); +} + program .option('-p, --progress', 'show progress bar') .option('-c, --config [config]', 'apply a config file (JSON), holding e.g. url specific header configuration') .option('-q, --quiet', 'displays errors only') .option('-v, --verbose', 'displays detailed error information') + .option('-a, --alive ', 'comma separated list of HTTP codes to be considered as alive', commaSeparatedCodesList) + .option('-r, --retry', 'retry after the duration indicated in \'retry-after\' header when HTTP code is 429') .arguments('[filenameOrUrl]') .action(function (filenameOrUrl) { filenameForOutput = filenameOrUrl; @@ -69,6 +77,8 @@ program opts.showProgressBar = (program.progress === true); // force true or undefined to be true or false. opts.quiet = (program.quiet === true); opts.verbose = (program.verbose === true); +opts.retryOn429 = (program.retry === true); +opts.aliveStatusCodes = program.alive; let markdown = ''; // collect the markdown data, then process it @@ -104,6 +114,8 @@ stream opts.replacementPatterns = config.replacementPatterns; opts.httpHeaders = config.httpHeaders; opts.ignoreDisable = config.ignoreDisable; + opts.retryOn429 = config.retryOn429; + opts.aliveStatusCodes = config.aliveStatusCodes; runMarkdownLinkCheck(markdown, opts); }); diff --git a/test/markdown-link-check.test.js b/test/markdown-link-check.test.js index bf3442c0..3f791da7 100644 --- a/test/markdown-link-check.test.js +++ b/test/markdown-link-check.test.js @@ -9,11 +9,17 @@ const markdownLinkCheck = require('../'); describe('markdown-link-check', function () { + // add a longer timeout on tests so we can really test real cases. + // Mocha default is 2s, make it 5s here. + this.timeout(5000); + let baseUrl; before(function (done) { const app = express(); + var laterRetryCount = 0; + app.head('/nohead', function (req, res) { res.sendStatus(405); // method not allowed }); @@ -21,6 +27,20 @@ describe('markdown-link-check', function () { res.sendStatus(200); }); + app.get('/partial', function (req, res) { + res.sendStatus(206); + }); + + app.get('/later', function (req, res) { + if(laterRetryCount<2){ + laterRetryCount++; + res.append('retry-after', '2s'); + res.sendStatus(429); + }else{ + res.sendStatus(200); + } + }); + app.get('/foo/redirect', function (req, res) { res.redirect('/foo/bar'); }); @@ -75,7 +95,9 @@ describe('markdown-link-check', function () { urls: [baseUrl + '/basic-auth'], headers: { 'Authorization': 'Basic Zm9vOmJhcg==', 'Foo': 'Bar' } } - ] + ], + "aliveStatusCodes":[200, 206], + "retryOn429":true }, function (err, results) { expect(err).to.be(null); expect(results).to.be.an('array'); @@ -108,6 +130,12 @@ describe('markdown-link-check', function () { // replaced { statusCode: 200, status: 'alive' }, + // request rate limit return 429, retry later and get 200 + { statusCode: 200, status: 'alive' }, + + // partial + { statusCode: 206, status: 'alive' }, + // hello image { statusCode: 200, status: 'alive' }, diff --git a/test/sample.md b/test/sample.md index ea8b76b3..ddc7d393 100644 --- a/test/sample.md +++ b/test/sample.md @@ -11,6 +11,8 @@ This is a test file: * [basic-auth](%%BASE_URL%%/basic-auth) (alive) * [ignored](%%BASE_URL%%/something/not-working-and-ignored/something) (ignored) * [replaced](%%BASE_URL%%/boo/bar) +* [later](%%BASE_URL%%/later) +* [partial](%%BASE_URL%%/partial) ![img](%%BASE_URL%%/hello.jpg) (alive) ![img](hello.jpg) (alive)