From d011491aa1cf0d99ac1ae6dfc044f52520a5052c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 15:55:19 -0500 Subject: [PATCH 001/281] save --- .gitignore | 2 + design.md | 37 ++++++++ lib/dsl.js | 15 +++ package-lock.json | 200 +++++++++++++++++++++++++++++++++++++++ package.json | 18 ++++ test/dsl-test.js | 13 +++ test/fixtures/example.js | 80 ++++++++++++++++ test/fixtures/index.js | 12 +++ 8 files changed, 377 insertions(+) create mode 100644 .gitignore create mode 100644 design.md create mode 100644 lib/dsl.js create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 test/dsl-test.js create mode 100644 test/fixtures/example.js create mode 100644 test/fixtures/index.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1ca9571 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +npm-debug.log diff --git a/design.md b/design.md new file mode 100644 index 0000000..01a97bf --- /dev/null +++ b/design.md @@ -0,0 +1,37 @@ +# Design + +```js +const state = { + type: i32() +}; + +const settings = { + message_begin: notify() +}; + +const HTTP_REQUEST = 1; +const HTTP_RESPONSE = 2; + +start_req_or_res = (c) => { + switch (c) { + (0x0a, 0x0d): + skip(); + + 'H': + next = res_or_resp_H; + settings.message_begin(); + + default: + state.type = HTTP_REQUEST; + goto start_req; + } +} + +res_or_resp_H = (c) => { + switch (c) { + 'T': + state.type = HTTP_RESPONSE; + next = s_res_HT; + } +}; +``` diff --git a/lib/dsl.js b/lib/dsl.js new file mode 100644 index 0000000..bcc2c0d --- /dev/null +++ b/lib/dsl.js @@ -0,0 +1,15 @@ +'use strict'; + +const esprima = require('esprima'); + +class DSL { + constructor(source) { + this.source = source; + this.ast = null; + } + + compile() { + this.ast = esprima.parse(this.source); + } +} +module.exports = DSL; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..163812c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,200 @@ +{ + "name": "parser-dsl", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "1.0.0", + "concat-map": "0.0.1" + } + }, + "browser-stdout": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", + "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", + "dev": true + }, + "commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "diff": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", + "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "esprima": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", + "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==" + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "glob": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "dev": true, + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } + }, + "growl": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", + "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==", + "dev": true + }, + "has-flag": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", + "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", + "dev": true + }, + "he": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "1.4.0", + "wrappy": "1.0.2" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "1.1.11" + } + }, + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } + }, + "mocha": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.0.0.tgz", + "integrity": "sha512-ukB2dF+u4aeJjc6IGtPNnJXfeby5d4ZqySlIBT0OEyva/DrMjVm5HkQxKnHDLKEfEQBsEnwTg9HHhtPHJdTd8w==", + "dev": true, + "requires": { + "browser-stdout": "1.3.0", + "commander": "2.11.0", + "debug": "3.1.0", + "diff": "3.3.1", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.3", + "he": "1.1.1", + "mkdirp": "0.5.1", + "supports-color": "4.4.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1.0.2" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "supports-color": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", + "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "dev": true, + "requires": { + "has-flag": "2.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..5f59bcb --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "parser-dsl", + "version": "1.0.0", + "description": "", + "main": "lib/dsl.js", + "scripts": { + "test": "mocha --reporter=spec test/*-test.js" + }, + "keywords": [], + "author": "Fedor Indutny (http://darksi.de/)", + "license": "MIT", + "dependencies": { + "esprima": "^4.0.0" + }, + "devDependencies": { + "mocha": "^5.0.0" + } +} diff --git a/test/dsl-test.js b/test/dsl-test.js new file mode 100644 index 0000000..351e1fa --- /dev/null +++ b/test/dsl-test.js @@ -0,0 +1,13 @@ +'use strict'; + +const DSL = require('../'); + +const fixtures = require('./fixtures'); + +describe('LLParse', () => { + it('should compile `example.js`', () => { + const dsl = new DSL(fixtures.source.example); + + dsl.compile(); + }); +}); diff --git a/test/fixtures/example.js b/test/fixtures/example.js new file mode 100644 index 0000000..1bd4152 --- /dev/null +++ b/test/fixtures/example.js @@ -0,0 +1,80 @@ +'use strict'; + +const state = { + type: i32(), + method: i32() +}; + +const settings = { + on_message_begin: notify(), + on_url: data('url') +}; + +const METHODS = { + 'GET': 1, + 'HEAD': 2, + 'POST': 3, + 'PUT': 4 +}; + +const HTTP_REQUEST = 1; +const HTTP_RESPONSE = 2; + +const INVALID_METHOD = 1; +const INVALID_URL_CHARACTER = 2; + +start_req_or_res = () => { + switch (_) { + case [ 0x0a, 0x0d ]: + skip(); + + '@notify-on-start(on_message_begin)'; + case METHODS: + state.type = HTTP_REQUEST; + state.method = match(); + next(request_after_method); + + '@notify-on-start(on_message_begin)'; + case 'HTTP': + state.type = HTTP_RESPONSE; + next(response_slash); + + '@unlikely'; + default: + error(INVALID_METHOD, 'Unknown method'); + } +}; + +request_after_method = () => { + switch (_) { + case ' ': + skip(); + + '@unlikely'; + case 0x0: + error(INVALID_METHOD, '`\0` after method'); + + default: + settings.on_url.start(); + rerun(url); + } +}; + +url = () => { + switch (_) { + case ' ': + next(req_http_start); + settings.on_url.end(); + + '@unlikely'; + case [ '\r', '\n' ]: + error(INVALID_URL_CHARACTER, + 'URL can\'t have newline chars in it'); + + '@unlikely'; + '@mode(strict)'; + case [ '\t', '\f' ]: + error(INVALID_URL_CHARACTER, + 'URL can\'t have "\\t" or "\\f" chars in it'); + } +}; diff --git a/test/fixtures/index.js b/test/fixtures/index.js new file mode 100644 index 0000000..fafc287 --- /dev/null +++ b/test/fixtures/index.js @@ -0,0 +1,12 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const file = (name) => { + return fs.readFileSync(path.join(__dirname, name)).toString(); +}; + +exports.source = { + example: file('example.js') +}; From 2d312fec0a7572419840b81ab040920995110f12 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 15:55:42 -0500 Subject: [PATCH 002/281] save --- lib/dsl.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/dsl.js b/lib/dsl.js index bcc2c0d..f64457d 100644 --- a/lib/dsl.js +++ b/lib/dsl.js @@ -10,6 +10,7 @@ class DSL { compile() { this.ast = esprima.parse(this.source); + console.log(this.ast); } } module.exports = DSL; From 074d18762d578e1c5e2f8ad3c1a657b0fb08eb49 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 16:47:46 -0500 Subject: [PATCH 003/281] save --- .gitignore | 2 + lib/dsl.js | 179 ++++++++++++++++++++++++++++++++++++++++++++++- lib/dsl/types.js | 20 ++++++ test/dsl-test.js | 3 +- 4 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 lib/dsl/types.js diff --git a/.gitignore b/.gitignore index 1ca9571..bc4c847 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ node_modules/ npm-debug.log +*.ll +*.o diff --git a/lib/dsl.js b/lib/dsl.js index f64457d..4d4e5cf 100644 --- a/lib/dsl.js +++ b/lib/dsl.js @@ -1,16 +1,191 @@ 'use strict'; +const assert = require('assert'); const esprima = require('esprima'); +const kError = Symbol('error'); +const kParseGlobals = Symbol('parseGlobals'); +const kParseConfigObject = Symbol('parseConfigObject'); +const kCompileState = Symbol('compileState'); +const kCompileSettings = Symbol('compileSettings'); + +const types = require('./dsl/types'); + +const STATE_TYPES = types.STATE_TYPES; +const SETTINGS_TYPES = types.SETTINGS_TYPES; + class DSL { constructor(source) { this.source = source; + this.ast = null; + this.globals = null; + this.state = null; } compile() { - this.ast = esprima.parse(this.source); - console.log(this.ast); + assert.strictEqual(this.ast, null, + 'Can\'t run twice without reinstantiating'); + + this.ast = esprima.parse(this.source, { loc: true }); + assert.equal(this.ast.type, 'Program'); + + this.globals = this[kParseGlobals](this.ast.body); + + let out = ''; + + out += ';;; State Start\n\n'; + out += this[kCompileState](); + out += '\n;;; State End\n\n'; + + out += ';;; Settings Start\n\n'; + out += this[kCompileSettings](); + out += '\n;;; Settings End \n\n'; + + return out; + } + + [kError](node, string) { + if (!node || !node.loc || !node.loc.start) + throw new Error(string + ' at unknown position'); + + const start = node.loc.start; + const err = new Error(string + ` at ${start.line}:${start.column}`); + err.node = node; + throw err; + } + + [kParseGlobals](body) { + const globals = new Map(); + + const declaration = (decl) => { + if (globals.has(decl.id.name)) + this[kError](decl.id, `Duplicate definition of ${decl.id.name}`); + + if (!decl.init) + this[kError](decl, `Missing init value for ${decl.id.name}`); + + globals.set(decl.id.name, decl.init); + }; + + body.forEach((node) => { + if (node.type !== 'VariableDeclaration') + return; + + node.declarations.forEach(declaration); + }); + + return globals; + } + + [kParseConfigObject](node, types) { + if (node.type !== 'ObjectExpression') { + this[kError](node, + 'Invalid configuration value type, must be ObjectExpression'); + } + + const result = []; + node.properties.forEach((prop) => { + if (prop.computed) + this[kError](prop, 'Configuration properties can\'t be computed'); + + if (prop.key.type !== 'Identifier') { + this[kError](prop.key, + 'Configuration property keys must be Identifiers'); + } + + if (prop.value.type !== 'CallExpression') { + this[kError](prop.key, + 'Configuration property values must be CallExpressions'); + } + + const key = prop.key.name; + + if (prop.value.callee.type !== 'Identifier') + this[kError](value.callee, 'Unknown configuration property type'); + + const type = prop.value.callee.name; + const args = prop.value.arguments.map((arg) => { + if (arg.type !== 'Literal') + this[kError](args, 'Invalid argument type, must be Literal'); + + return arg.value; + }); + + if (!types.hasOwnProperty(type)) + this[kError](value.callee, 'Unknown `state` property type'); + + const validation = types[type](args); + if (validation !== true) { + this[kError](value.callee, + `Invalid arguments for "${type}": ${validation}`); + } + + result.push({ key, type, args }); + }); + return result; + } + + [kCompileState]() { + assert(this.globals.has('state'), 'Missing `state` global object'); + + const props = this[kParseConfigObject]( + this.globals.get('state'), STATE_TYPES); + + const state = new Map(); + this.state = state; + + let out = '%state = type {\n'; + + props.forEach((prop, index) => { + const isLast = index === props.length - 1; + + out += ` ${prop.type}${isLast ? '' : ','}`; + out += ` ; ${index} => ${prop.key}`; + if (prop.args.length !== 0) + out += ` ${JSON.stringify(prop.args)}`; + out += '\n'; + + state.set(prop.key, { index, type: prop.type, args: prop.args }); + }); + + out += '}\n'; + + return out; + } + + [kCompileSettings]() { + assert(this.globals.has('settings'), 'Missing `settings` global object'); + + const props = this[kParseConfigObject]( + this.globals.get('settings'), SETTINGS_TYPES); + + const settings = new Map(); + this.settings = settings; + + let out = '%settings = type {\n'; + + props.forEach((prop, index) => { + const isLast = index === props.length - 1; + + let type; + if (prop.type === 'notify') + type = 'i32 (%state*)*'; + else + type = 'i32 (%state*, i8*, i64)*'; + + out += ` ${type}${isLast ? '' : ','}`; + out += ` ; ${index} => ${prop.key}`; + if (prop.args.length !== 0) + out += ` ${JSON.stringify(prop.args)}`; + out += '\n'; + + settings.set(prop.key, { index, type: prop.type, args: prop.args }); + }); + + out += '}\n'; + + return out; } } module.exports = DSL; diff --git a/lib/dsl/types.js b/lib/dsl/types.js new file mode 100644 index 0000000..4fd1678 --- /dev/null +++ b/lib/dsl/types.js @@ -0,0 +1,20 @@ +'use strict'; + +const disallowArgs = args => args.length === 0; + +exports.STATE_TYPES = { + i8: disallowArgs, + i16: disallowArgs, + i32: disallowArgs, + i64: disallowArgs +}; + +exports.SETTINGS_TYPES = { + notify: disallowArgs, + data: (args) => { + if (args.length !== 1) + return false; + + return typeof args[0] === 'string'; + } +}; diff --git a/test/dsl-test.js b/test/dsl-test.js index 351e1fa..3126552 100644 --- a/test/dsl-test.js +++ b/test/dsl-test.js @@ -8,6 +8,7 @@ describe('LLParse', () => { it('should compile `example.js`', () => { const dsl = new DSL(fixtures.source.example); - dsl.compile(); + const out = dsl.compile(); + console.log(out); }); }); From 0ead44ad0c1e03e22827bbfe325a2914afcb9f6e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 16:55:18 -0500 Subject: [PATCH 004/281] save --- lib/dsl.js | 185 ++------------------------------------------ lib/dsl/compiler.js | 183 +++++++++++++++++++++++++++++++++++++++++++ test/dsl-test.js | 2 +- 3 files changed, 190 insertions(+), 180 deletions(-) create mode 100644 lib/dsl/compiler.js diff --git a/lib/dsl.js b/lib/dsl.js index 4d4e5cf..89db6bf 100644 --- a/lib/dsl.js +++ b/lib/dsl.js @@ -1,191 +1,18 @@ 'use strict'; -const assert = require('assert'); -const esprima = require('esprima'); +const Compiler = require('./dsl/compiler'); -const kError = Symbol('error'); -const kParseGlobals = Symbol('parseGlobals'); -const kParseConfigObject = Symbol('parseConfigObject'); -const kCompileState = Symbol('compileState'); -const kCompileSettings = Symbol('compileSettings'); - -const types = require('./dsl/types'); - -const STATE_TYPES = types.STATE_TYPES; -const SETTINGS_TYPES = types.SETTINGS_TYPES; +// API, really class DSL { - constructor(source) { + constructor(prefix, source) { + this.prefix = prefix; this.source = source; - - this.ast = null; - this.globals = null; - this.state = null; } compile() { - assert.strictEqual(this.ast, null, - 'Can\'t run twice without reinstantiating'); - - this.ast = esprima.parse(this.source, { loc: true }); - assert.equal(this.ast.type, 'Program'); - - this.globals = this[kParseGlobals](this.ast.body); - - let out = ''; - - out += ';;; State Start\n\n'; - out += this[kCompileState](); - out += '\n;;; State End\n\n'; - - out += ';;; Settings Start\n\n'; - out += this[kCompileSettings](); - out += '\n;;; Settings End \n\n'; - - return out; - } - - [kError](node, string) { - if (!node || !node.loc || !node.loc.start) - throw new Error(string + ' at unknown position'); - - const start = node.loc.start; - const err = new Error(string + ` at ${start.line}:${start.column}`); - err.node = node; - throw err; - } - - [kParseGlobals](body) { - const globals = new Map(); - - const declaration = (decl) => { - if (globals.has(decl.id.name)) - this[kError](decl.id, `Duplicate definition of ${decl.id.name}`); - - if (!decl.init) - this[kError](decl, `Missing init value for ${decl.id.name}`); - - globals.set(decl.id.name, decl.init); - }; - - body.forEach((node) => { - if (node.type !== 'VariableDeclaration') - return; - - node.declarations.forEach(declaration); - }); - - return globals; - } - - [kParseConfigObject](node, types) { - if (node.type !== 'ObjectExpression') { - this[kError](node, - 'Invalid configuration value type, must be ObjectExpression'); - } - - const result = []; - node.properties.forEach((prop) => { - if (prop.computed) - this[kError](prop, 'Configuration properties can\'t be computed'); - - if (prop.key.type !== 'Identifier') { - this[kError](prop.key, - 'Configuration property keys must be Identifiers'); - } - - if (prop.value.type !== 'CallExpression') { - this[kError](prop.key, - 'Configuration property values must be CallExpressions'); - } - - const key = prop.key.name; - - if (prop.value.callee.type !== 'Identifier') - this[kError](value.callee, 'Unknown configuration property type'); - - const type = prop.value.callee.name; - const args = prop.value.arguments.map((arg) => { - if (arg.type !== 'Literal') - this[kError](args, 'Invalid argument type, must be Literal'); - - return arg.value; - }); - - if (!types.hasOwnProperty(type)) - this[kError](value.callee, 'Unknown `state` property type'); - - const validation = types[type](args); - if (validation !== true) { - this[kError](value.callee, - `Invalid arguments for "${type}": ${validation}`); - } - - result.push({ key, type, args }); - }); - return result; - } - - [kCompileState]() { - assert(this.globals.has('state'), 'Missing `state` global object'); - - const props = this[kParseConfigObject]( - this.globals.get('state'), STATE_TYPES); - - const state = new Map(); - this.state = state; - - let out = '%state = type {\n'; - - props.forEach((prop, index) => { - const isLast = index === props.length - 1; - - out += ` ${prop.type}${isLast ? '' : ','}`; - out += ` ; ${index} => ${prop.key}`; - if (prop.args.length !== 0) - out += ` ${JSON.stringify(prop.args)}`; - out += '\n'; - - state.set(prop.key, { index, type: prop.type, args: prop.args }); - }); - - out += '}\n'; - - return out; - } - - [kCompileSettings]() { - assert(this.globals.has('settings'), 'Missing `settings` global object'); - - const props = this[kParseConfigObject]( - this.globals.get('settings'), SETTINGS_TYPES); - - const settings = new Map(); - this.settings = settings; - - let out = '%settings = type {\n'; - - props.forEach((prop, index) => { - const isLast = index === props.length - 1; - - let type; - if (prop.type === 'notify') - type = 'i32 (%state*)*'; - else - type = 'i32 (%state*, i8*, i64)*'; - - out += ` ${type}${isLast ? '' : ','}`; - out += ` ; ${index} => ${prop.key}`; - if (prop.args.length !== 0) - out += ` ${JSON.stringify(prop.args)}`; - out += '\n'; - - settings.set(prop.key, { index, type: prop.type, args: prop.args }); - }); - - out += '}\n'; - - return out; + const compiler = new Compiler(this.prefix, this.source); + return compiler.compile(); } } module.exports = DSL; diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js new file mode 100644 index 0000000..2c8d873 --- /dev/null +++ b/lib/dsl/compiler.js @@ -0,0 +1,183 @@ +'use strict'; + +const assert = require('assert'); +const esprima = require('esprima'); + +const types = require('./types'); + +const STATE_TYPES = types.STATE_TYPES; +const SETTINGS_TYPES = types.SETTINGS_TYPES; + +class Compiler { + constructor(prefix, source) { + this.prefix = prefix; + this.source = source; + + this.ast = null; + this.globals = null; + this.state = null; + } + + compile() { + this.ast = esprima.parse(this.source, { loc: true }); + assert.equal(this.ast.type, 'Program'); + + this.globals = this.parseGlobals(this.ast.body); + + let out = ''; + + out += ';;; State Start\n\n'; + out += this.compileState(); + out += '\n;;; State End\n\n'; + + out += ';;; Settings Start\n\n'; + out += this.compileSettings(); + out += '\n;;; Settings End \n\n'; + + return out; + } + + error(node, string) { + if (!node || !node.loc || !node.loc.start) + throw new Error(string + ' at unknown position'); + + const start = node.loc.start; + const err = new Error(string + ` at ${start.line}:${start.column}`); + err.node = node; + throw err; + } + + parseGlobals(body) { + const globals = new Map(); + + const declaration = (decl) => { + if (globals.has(decl.id.name)) + this.error(decl.id, `Duplicate definition of ${decl.id.name}`); + + if (!decl.init) + this.error(decl, `Missing init value for ${decl.id.name}`); + + globals.set(decl.id.name, decl.init); + }; + + body.forEach((node) => { + if (node.type !== 'VariableDeclaration') + return; + + node.declarations.forEach(declaration); + }); + + return globals; + } + + parseConfigObject(node, types) { + if (node.type !== 'ObjectExpression') { + this.error(node, + 'Invalid configuration value type, must be ObjectExpression'); + } + + const result = []; + node.properties.forEach((prop) => { + if (prop.computed) + this.error(prop, 'Configuration properties can\'t be computed'); + + if (prop.key.type !== 'Identifier') { + this.error(prop.key, + 'Configuration property keys must be Identifiers'); + } + + if (prop.value.type !== 'CallExpression') { + this.error(prop.key, + 'Configuration property values must be CallExpressions'); + } + + const key = prop.key.name; + + if (prop.value.callee.type !== 'Identifier') + this.error(value.callee, 'Unknown configuration property type'); + + const type = prop.value.callee.name; + const args = prop.value.arguments.map((arg) => { + if (arg.type !== 'Literal') + this.error(args, 'Invalid argument type, must be Literal'); + + return arg.value; + }); + + if (!types.hasOwnProperty(type)) + this.error(value.callee, 'Unknown `state` property type'); + + const validation = types[type](args); + if (validation !== true) { + this.error(value.callee, + `Invalid arguments for "${type}": ${validation}`); + } + + result.push({ key, type, args }); + }); + return result; + } + + compileState() { + assert(this.globals.has('state'), 'Missing `state` global object'); + + const props = this.parseConfigObject( + this.globals.get('state'), STATE_TYPES); + + const state = new Map(); + this.state = state; + + let out = '%state = type {\n'; + + props.forEach((prop, index) => { + const isLast = index === props.length - 1; + + out += ` ${prop.type}${isLast ? '' : ','}`; + out += ` ; ${index} => ${prop.key}: ${prop.type}`; + if (prop.args.length !== 0) + out += ` ${JSON.stringify(prop.args)}`; + out += '\n'; + + state.set(prop.key, { index, type: prop.type, args: prop.args }); + }); + + out += '}\n'; + + return out; + } + + compileSettings() { + assert(this.globals.has('settings'), 'Missing `settings` global object'); + + const props = this.parseConfigObject( + this.globals.get('settings'), SETTINGS_TYPES); + + const settings = new Map(); + this.settings = settings; + + let out = '%settings = type {\n'; + + props.forEach((prop, index) => { + const isLast = index === props.length - 1; + + let type; + if (prop.type === 'notify') + type = 'i32 (%state*)*'; + else + type = 'i32 (%state*, i8*, i64)*'; + + out += ` ${type}${isLast ? '' : ','}`; + out += ` ; ${index} => ${prop.key}: ${prop.type}`; + if (prop.args.length !== 0) + out += ` ${JSON.stringify(prop.args)}`; + out += '\n'; + + settings.set(prop.key, { index, type: prop.type, args: prop.args }); + }); + + out += '}\n'; + + return out; + } +} +module.exports = Compiler; diff --git a/test/dsl-test.js b/test/dsl-test.js index 3126552..cd585e3 100644 --- a/test/dsl-test.js +++ b/test/dsl-test.js @@ -6,7 +6,7 @@ const fixtures = require('./fixtures'); describe('LLParse', () => { it('should compile `example.js`', () => { - const dsl = new DSL(fixtures.source.example); + const dsl = new DSL('llparse', fixtures.source.example); const out = dsl.compile(); console.log(out); From 78cb363c6ab930f25f43454f5fdead108dc44f81 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 17:45:40 -0500 Subject: [PATCH 005/281] save --- lib/dsl/compiler.js | 75 +++++++++++++--------------------------- test/fixtures/example.js | 19 ++++++++-- 2 files changed, 40 insertions(+), 54 deletions(-) diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index 2c8d873..fc5ab9e 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -4,6 +4,8 @@ const assert = require('assert'); const esprima = require('esprima'); const types = require('./types'); +const State = require('./state'); +const Settings = require('./settings'); const STATE_TYPES = types.STATE_TYPES; const SETTINGS_TYPES = types.SETTINGS_TYPES; @@ -15,7 +17,8 @@ class Compiler { this.ast = null; this.globals = null; - this.state = null; + this.state = new State(); + this.settings = new Settings(); } compile() { @@ -23,16 +26,17 @@ class Compiler { assert.equal(this.ast.type, 'Program'); this.globals = this.parseGlobals(this.ast.body); + this.parseState(); + this.parseSettings(); - let out = ''; + this.compileStates(); - out += ';;; State Start\n\n'; - out += this.compileState(); - out += '\n;;; State End\n\n'; + let out = ''; - out += ';;; Settings Start\n\n'; - out += this.compileSettings(); - out += '\n;;; Settings End \n\n'; + out += this.state.serialize(); + out += '\n'; + out += this.settings.serialize(); + out += '\n'; return out; } @@ -118,66 +122,35 @@ class Compiler { return result; } - compileState() { + parseState() { assert(this.globals.has('state'), 'Missing `state` global object'); const props = this.parseConfigObject( this.globals.get('state'), STATE_TYPES); - const state = new Map(); - this.state = state; - - let out = '%state = type {\n'; - - props.forEach((prop, index) => { - const isLast = index === props.length - 1; + props.forEach((prop) => { + if (this.state.has(prop.key)) + this.error(prop.key, 'Duplicate key in `state` object'); - out += ` ${prop.type}${isLast ? '' : ','}`; - out += ` ; ${index} => ${prop.key}: ${prop.type}`; - if (prop.args.length !== 0) - out += ` ${JSON.stringify(prop.args)}`; - out += '\n'; - - state.set(prop.key, { index, type: prop.type, args: prop.args }); + this.state.define(prop.key, prop.type, prop.args); }); - - out += '}\n'; - - return out; } - compileSettings() { + parseSettings() { assert(this.globals.has('settings'), 'Missing `settings` global object'); const props = this.parseConfigObject( this.globals.get('settings'), SETTINGS_TYPES); - const settings = new Map(); - this.settings = settings; + props.forEach((prop) => { + if (this.settings.has(prop.key)) + this.error(prop.key, 'Duplicate key in `settings` object'); - let out = '%settings = type {\n'; - - props.forEach((prop, index) => { - const isLast = index === props.length - 1; - - let type; - if (prop.type === 'notify') - type = 'i32 (%state*)*'; - else - type = 'i32 (%state*, i8*, i64)*'; - - out += ` ${type}${isLast ? '' : ','}`; - out += ` ; ${index} => ${prop.key}: ${prop.type}`; - if (prop.args.length !== 0) - out += ` ${JSON.stringify(prop.args)}`; - out += '\n'; - - settings.set(prop.key, { index, type: prop.type, args: prop.args }); + this.settings.define(prop.key, prop.type, prop.args); }); + } - out += '}\n'; - - return out; + compileStates() { } } module.exports = Compiler; diff --git a/test/fixtures/example.js b/test/fixtures/example.js index 1bd4152..5c09057 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -23,7 +23,15 @@ const HTTP_RESPONSE = 2; const INVALID_METHOD = 1; const INVALID_URL_CHARACTER = 2; -start_req_or_res = () => { +'@default'; +const init = () => { + switch (_) { + default: + rerun(start_req_or_res); + } +}; + +const start_req_or_res = () => { switch (_) { case [ 0x0a, 0x0d ]: skip(); @@ -45,7 +53,7 @@ start_req_or_res = () => { } }; -request_after_method = () => { +const request_after_method = () => { switch (_) { case ' ': skip(); @@ -60,7 +68,7 @@ request_after_method = () => { } }; -url = () => { +const url = () => { switch (_) { case ' ': next(req_http_start); @@ -78,3 +86,8 @@ url = () => { 'URL can\'t have "\\t" or "\\f" chars in it'); } }; + +const response_slash = () => { + switch (_) { + } +}; From b2d069f3240ecfc18dd41c64bcad40c7ec040cdf Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 17:46:41 -0500 Subject: [PATCH 006/281] save --- lib/dsl/settings.js | 27 ++++++++++++++++++++++++++ lib/dsl/state.js | 26 +++++++++++++++++++++++++ lib/dsl/struct.js | 46 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 lib/dsl/settings.js create mode 100644 lib/dsl/state.js create mode 100644 lib/dsl/struct.js diff --git a/lib/dsl/settings.js b/lib/dsl/settings.js new file mode 100644 index 0000000..ce1c267 --- /dev/null +++ b/lib/dsl/settings.js @@ -0,0 +1,27 @@ +'use strict'; + +const assert = require('assert'); +const Struct = require('./struct'); + +class Settings extends Struct { + constructor() { + super('settings'); + } + + define(key, type, args) { + let nativeType; + if (type === 'notify') { + nativeType = 'i32 (%state*)*'; + } else { + assert.equal(type, 'data'); + nativeType = 'i32 (%state*, i8*, i64)*'; + } + + let comment = `${key}: ${type}`; + if (args.length !== 0) + comment += ` ${JSON.stringify(args)}`; + + super.define(key, nativeType, comment); + } +} +module.exports = Settings; diff --git a/lib/dsl/state.js b/lib/dsl/state.js new file mode 100644 index 0000000..8a8c8ef --- /dev/null +++ b/lib/dsl/state.js @@ -0,0 +1,26 @@ +'use strict'; + +const kState = Symbol('state'); +const kError = Symbol('error'); + +const Struct = require('./struct'); + +class State extends Struct { + constructor() { + super('state'); + + this.define(kState, 'i8 (%state*, i8*)', []); + this.define(kError, 'i32', []); + } + + define(key, type, args) { + const commentKey = key === kError ? '[error]' : + key === kState ? '[state]' : key; + let comment = `${commentKey}`; + if (args.length !== 0) + out += ` ${JSON.stringify(args)}`; + + super.define(key, type, comment); + } +} +module.exports = State; diff --git a/lib/dsl/struct.js b/lib/dsl/struct.js new file mode 100644 index 0000000..4278379 --- /dev/null +++ b/lib/dsl/struct.js @@ -0,0 +1,46 @@ +'use strict'; + +const assert = require('assert'); + +class Struct { + constructor(name) { + this.name = name; + + this.map = new Map(); + this.props = []; + } + + define(name, type, comment) { + assert(!this.has(name), `Duplicate entry in ${this.name}`); + + const index = this.props.length; + const entry = { name, type, comment }; + + this.props.push(entry); + this.map.set(name, index); + } + + has(name) { + return this.map.has(name); + } + + lookup(name) { + return this.map.get(name); + } + + serialize() { + let out = ''; + + out += `%${this.name} = type {\n`; + this.props.forEach((prop, index) => { + const isLast = index === this.props.length - 1; + + out += ` ${prop.type}${isLast ? '' : ','}`; + out += ` ; ${index} => ${prop.comment}\n`; + }); + out += '}\n'; + + return out; + } +} +module.exports = Struct; From d499fe33e4925d38e5d4b664044e28fd9e334231 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 17:46:55 -0500 Subject: [PATCH 007/281] save --- design.md | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 design.md diff --git a/design.md b/design.md deleted file mode 100644 index 01a97bf..0000000 --- a/design.md +++ /dev/null @@ -1,37 +0,0 @@ -# Design - -```js -const state = { - type: i32() -}; - -const settings = { - message_begin: notify() -}; - -const HTTP_REQUEST = 1; -const HTTP_RESPONSE = 2; - -start_req_or_res = (c) => { - switch (c) { - (0x0a, 0x0d): - skip(); - - 'H': - next = res_or_resp_H; - settings.message_begin(); - - default: - state.type = HTTP_REQUEST; - goto start_req; - } -} - -res_or_resp_H = (c) => { - switch (c) { - 'T': - state.type = HTTP_RESPONSE; - next = s_res_HT; - } -}; -``` From 82d653461b80482f3744da0c29c7faa22da4ffc5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 18:39:32 -0500 Subject: [PATCH 008/281] save --- lib/dsl/compiler.js | 79 +++++++++++++++++++++++++++++++++++++++- test/fixtures/example.js | 3 +- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index fc5ab9e..8af0b12 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -6,10 +6,13 @@ const esprima = require('esprima'); const types = require('./types'); const State = require('./state'); const Settings = require('./settings'); +const Code = require('./code'); const STATE_TYPES = types.STATE_TYPES; const SETTINGS_TYPES = types.SETTINGS_TYPES; +const DIRECTIVE_DEFAULT = '@default'; + class Compiler { constructor(prefix, source) { this.prefix = prefix; @@ -17,8 +20,11 @@ class Compiler { this.ast = null; this.globals = null; - this.state = new State(); - this.settings = new Settings(); + this.state = null; + this.settings = null; + + this.defaultState = null; + this.code = null; } compile() { @@ -127,6 +133,9 @@ class Compiler { const props = this.parseConfigObject( this.globals.get('state'), STATE_TYPES); + this.globals.delete('state'); + + this.state = new State(); props.forEach((prop) => { if (this.state.has(prop.key)) @@ -141,6 +150,9 @@ class Compiler { const props = this.parseConfigObject( this.globals.get('settings'), SETTINGS_TYPES); + this.globals.delete('settings'); + + this.settings = new Settings(); props.forEach((prop) => { if (this.settings.has(prop.key)) @@ -151,6 +163,69 @@ class Compiler { } compileStates() { + const blocks = new Map(); + + this.globals.forEach((value, key) => { + if (value.type !== 'ArrowFunctionExpression') + return; + if (value.body.type !== 'BlockStatement') + return; + + // TODO(indutny): validate function parameters + // TODO(indutny): ensure no iterators, etc + blocks.set(key, value.body); + }); + + const start = this.findDefault(blocks); + this.defaultState = start; + + this.code = new Map(); + this.buildGraph(blocks, start, this.ast); + } + + findDefault(blocks) { + let found = false; + + blocks.forEach((block, key) => { + const hasDirective = block.body.some((stmt) => { + return stmt.type === 'ExpressionStatement' && + stmt.directive === DIRECTIVE_DEFAULT; + }); + + if (!hasDirective) + return; + + if (found) + this.error(block, 'Two functions with `@default` directive'); + + found = key; + }); + + if (!found) + this.error(this.ast, 'No function with `@default` directive found'); + + return found; + } + + buildGraph(blocks, name, from) { + // Already-built + if (this.code.has(name)) + return this.code.get(name); + + if (!blocks.has(name)) + this.error(from, `Unknown state name ${name}`); + + const block = blocks.get(name); + const code = new Code(this, name); + + code.compile(block); + + this.code.set(name, code); + code.deps.forEach((dep) => { + code.fillDep(dep, this.buildGraph(blocks, dep, block)); + }); + + return code; } } module.exports = Compiler; diff --git a/test/fixtures/example.js b/test/fixtures/example.js index 5c09057..0cf7c8b 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -23,8 +23,9 @@ const HTTP_RESPONSE = 2; const INVALID_METHOD = 1; const INVALID_URL_CHARACTER = 2; -'@default'; const init = () => { + '@default'; + switch (_) { default: rerun(start_req_or_res); From 14c2d2b304b21b2afb38d94d9c40edac5bc69eb7 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 18:39:34 -0500 Subject: [PATCH 009/281] save --- lib/dsl/code.js | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 lib/dsl/code.js diff --git a/lib/dsl/code.js b/lib/dsl/code.js new file mode 100644 index 0000000..2ddbbfe --- /dev/null +++ b/lib/dsl/code.js @@ -0,0 +1,62 @@ +'use strict'; + +const assert = require('assert'); + +class Code { + constructor(compiler, name) { + this.compiler = compiler; + + this.name = name; + this.deps = new Map(); + } + + error(ast, message) { + this.compiler.error(ast, message); + } + + compile(block) { + const s = this.getSwitch(block); + // TODO(indutny): validate discriminant + + const cases = s.cases.map((node) => { + if (node.consequent.length === 0) { + this.error(node, + 'All `case`s in switch must have consequent statements'); + } + + return { + test: node.test, + body: this.compileBody(node, node.consequent) + }; + }) + } + + compileBody(context, node) { + console.log(node); + } + + getSwitch(block) { + let res = false; + + block.body.forEach((stmt) => { + if (stmt.type !== 'SwitchStatement') + return; + + if (res) + this.error(stmt, 'Duplicate switch statement'); + + res = stmt; + }); + + if (!res) + this.error(block, 'No switch statement found in a state function'); + + return res; + } + + fillDep(dep, code) { + assert.strictEqual(this.deps.get(dep), null); + this.deps.set(dep, code); + } +} +module.exports = Code; From 4f5e93c80d67f2a8c9be240a9cb4150bdd30bdd5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 19:04:18 -0500 Subject: [PATCH 010/281] save --- lib/dsl.js | 2 +- lib/dsl/block.js | 15 +++++++++++++++ lib/dsl/code.js | 21 ++++++++++++++++----- lib/dsl/compiler.js | 25 +++++++++++-------------- lib/dsl/index.js | 11 +++++++++++ lib/dsl/reporter.js | 14 ++++++++++++++ lib/dsl/settings.js | 4 +++- lib/dsl/state.js | 3 ++- test/fixtures/example.js | 11 +++++++++++ 9 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 lib/dsl/block.js create mode 100644 lib/dsl/index.js create mode 100644 lib/dsl/reporter.js diff --git a/lib/dsl.js b/lib/dsl.js index 89db6bf..74ee9fa 100644 --- a/lib/dsl.js +++ b/lib/dsl.js @@ -1,6 +1,6 @@ 'use strict'; -const Compiler = require('./dsl/compiler'); +const Compiler = require('./dsl/').Compiler; // API, really diff --git a/lib/dsl/block.js b/lib/dsl/block.js new file mode 100644 index 0000000..b379a65 --- /dev/null +++ b/lib/dsl/block.js @@ -0,0 +1,15 @@ +'use strict'; + +class Block { + constructor(reporter) { + this.reporter = reporter; + } + + error(node, string) { + return this.reporter.error(node, string); + } + + compile(stmts) { + } +} +module.exports = Block; diff --git a/lib/dsl/code.js b/lib/dsl/code.js index 2ddbbfe..ad6125c 100644 --- a/lib/dsl/code.js +++ b/lib/dsl/code.js @@ -2,16 +2,19 @@ const assert = require('assert'); +const dsl = require('./'); +const Block = dsl.Block; + class Code { - constructor(compiler, name) { - this.compiler = compiler; + constructor(reporter, name) { + this.reporter = reporter; this.name = name; this.deps = new Map(); } error(ast, message) { - this.compiler.error(ast, message); + this.reporter.error(ast, message); } compile(block) { @@ -31,8 +34,16 @@ class Code { }) } - compileBody(context, node) { - console.log(node); + compileBody(context, body) { + if (body[body.length - 1].type !== 'BreakStatement') { + this.error(body[body.length - 1], + 'Last statement in `case` body must be `break`'); + } + + const stmts = body.slice(0, -1); + const block = new Block(this.reporter); + block.compile(stmts); + return block; } getSwitch(block) { diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index 8af0b12..8fee8dc 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -3,13 +3,15 @@ const assert = require('assert'); const esprima = require('esprima'); -const types = require('./types'); -const State = require('./state'); -const Settings = require('./settings'); -const Code = require('./code'); +const dsl = require('./'); -const STATE_TYPES = types.STATE_TYPES; -const SETTINGS_TYPES = types.SETTINGS_TYPES; +const State = dsl.State; +const Settings = dsl.Settings; +const Code = dsl.Code; +const Reporter = dsl.Reporter; + +const STATE_TYPES = dsl.types.STATE_TYPES; +const SETTINGS_TYPES = dsl.types.SETTINGS_TYPES; const DIRECTIVE_DEFAULT = '@default'; @@ -22,6 +24,7 @@ class Compiler { this.globals = null; this.state = null; this.settings = null; + this.reporter = new Reporter(); this.defaultState = null; this.code = null; @@ -48,13 +51,7 @@ class Compiler { } error(node, string) { - if (!node || !node.loc || !node.loc.start) - throw new Error(string + ' at unknown position'); - - const start = node.loc.start; - const err = new Error(string + ` at ${start.line}:${start.column}`); - err.node = node; - throw err; + return this.reporter.error(node, string); } parseGlobals(body) { @@ -216,7 +213,7 @@ class Compiler { this.error(from, `Unknown state name ${name}`); const block = blocks.get(name); - const code = new Code(this, name); + const code = new Code(this.reporter, name); code.compile(block); diff --git a/lib/dsl/index.js b/lib/dsl/index.js new file mode 100644 index 0000000..154cb56 --- /dev/null +++ b/lib/dsl/index.js @@ -0,0 +1,11 @@ +'use strict'; + +exports.types = require('./types'); +exports.Struct = require('./struct'); +exports.Reporter = require('./reporter'); + +exports.State = require('./state'); +exports.Settings = require('./settings'); +exports.Block = require('./block'); +exports.Code = require('./code'); +exports.Compiler = require('./compiler'); diff --git a/lib/dsl/reporter.js b/lib/dsl/reporter.js new file mode 100644 index 0000000..227782d --- /dev/null +++ b/lib/dsl/reporter.js @@ -0,0 +1,14 @@ +'use strict'; + +class Reporter { + error(node, message) { + if (!node || !node.loc || !node.loc.start) + throw new Error(string + ' at unknown position'); + + const start = node.loc.start; + const err = new Error(string + ` at ${start.line}:${start.column}`); + err.node = node; + throw err; + } +} +module.exports = Reporter; diff --git a/lib/dsl/settings.js b/lib/dsl/settings.js index ce1c267..e9f4db0 100644 --- a/lib/dsl/settings.js +++ b/lib/dsl/settings.js @@ -1,7 +1,9 @@ 'use strict'; const assert = require('assert'); -const Struct = require('./struct'); + +const dsl = require('./'); +const Struct = dsl.Struct; class Settings extends Struct { constructor() { diff --git a/lib/dsl/state.js b/lib/dsl/state.js index 8a8c8ef..6fa4735 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -3,7 +3,8 @@ const kState = Symbol('state'); const kError = Symbol('error'); -const Struct = require('./struct'); +const dsl = require('./'); +const Struct = dsl.Struct; class State extends Struct { constructor() { diff --git a/test/fixtures/example.js b/test/fixtures/example.js index 0cf7c8b..d35bbca 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -29,6 +29,7 @@ const init = () => { switch (_) { default: rerun(start_req_or_res); + break; } }; @@ -36,21 +37,25 @@ const start_req_or_res = () => { switch (_) { case [ 0x0a, 0x0d ]: skip(); + break; '@notify-on-start(on_message_begin)'; case METHODS: state.type = HTTP_REQUEST; state.method = match(); next(request_after_method); + break; '@notify-on-start(on_message_begin)'; case 'HTTP': state.type = HTTP_RESPONSE; next(response_slash); + break; '@unlikely'; default: error(INVALID_METHOD, 'Unknown method'); + break; } }; @@ -58,14 +63,17 @@ const request_after_method = () => { switch (_) { case ' ': skip(); + break; '@unlikely'; case 0x0: error(INVALID_METHOD, '`\0` after method'); + break; default: settings.on_url.start(); rerun(url); + break; } }; @@ -74,17 +82,20 @@ const url = () => { case ' ': next(req_http_start); settings.on_url.end(); + break; '@unlikely'; case [ '\r', '\n' ]: error(INVALID_URL_CHARACTER, 'URL can\'t have newline chars in it'); + break; '@unlikely'; '@mode(strict)'; case [ '\t', '\f' ]: error(INVALID_URL_CHARACTER, 'URL can\'t have "\\t" or "\\f" chars in it'); + break; } }; From 2c011a8609586a1452bc5bd3287f4fa8a4367897 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 19:50:06 -0500 Subject: [PATCH 011/281] save --- lib/dsl/block.js | 15 ---- lib/dsl/body.js | 109 +++++++++++++++++++++++++++++ lib/dsl/code.js | 9 ++- lib/dsl/compiler.js | 2 +- lib/dsl/index.js | 4 +- lib/dsl/ir/index.js | 4 ++ lib/dsl/ir/instruction/base.js | 8 +++ lib/dsl/ir/instruction/error.js | 22 ++++++ lib/dsl/ir/instruction/index.js | 7 ++ lib/dsl/ir/instruction/next.js | 18 +++++ lib/dsl/ir/instruction/redirect.js | 18 +++++ lib/dsl/{ => ir}/struct.js | 0 lib/dsl/reporter.js | 4 +- lib/dsl/settings.js | 3 +- lib/dsl/state.js | 3 +- test/fixtures/example.js | 26 ++++--- 16 files changed, 215 insertions(+), 37 deletions(-) delete mode 100644 lib/dsl/block.js create mode 100644 lib/dsl/body.js create mode 100644 lib/dsl/ir/index.js create mode 100644 lib/dsl/ir/instruction/base.js create mode 100644 lib/dsl/ir/instruction/error.js create mode 100644 lib/dsl/ir/instruction/index.js create mode 100644 lib/dsl/ir/instruction/next.js create mode 100644 lib/dsl/ir/instruction/redirect.js rename lib/dsl/{ => ir}/struct.js (100%) diff --git a/lib/dsl/block.js b/lib/dsl/block.js deleted file mode 100644 index b379a65..0000000 --- a/lib/dsl/block.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -class Block { - constructor(reporter) { - this.reporter = reporter; - } - - error(node, string) { - return this.reporter.error(node, string); - } - - compile(stmts) { - } -} -module.exports = Block; diff --git a/lib/dsl/body.js b/lib/dsl/body.js new file mode 100644 index 0000000..4af364d --- /dev/null +++ b/lib/dsl/body.js @@ -0,0 +1,109 @@ +'use strict'; + +const dsl = require('./'); + +const INTRINSICS = { + next: dsl.ir.instruction.Next, + redirect: dsl.ir.instruction.Redirect, + error: dsl.ir.instruction.Error +}; + +class Body { + constructor(reporter, index, parent) { + this.reporter = reporter; + this.index = index; + this.parent = parent || null; + + this.used = new Set(); + this.local = new Map(); + this.ir = []; + } + + error(node, string) { + return this.reporter.error(node, string); + } + + compile(stmts) { + stmts.forEach(stmt => this.compileStatement(stmt)); + } + + compileStatement(stmt) { + if (stmt.type === 'ExpressionStatement') + return this.compileExpression(stmt.expression); + + this.error(stmt, `Unsupported statement type: ${stmt.type}`); + } + + compileExpression(expr) { + if (expr.type === 'CallExpression') + return this.compileCall(expr); + else if (expr.type === 'Literal') + return { type: 'Literal', value: expr.value }; + + this.error(expr, `Unsupported expression type: ${expr.type}`); + } + + compileCall(call) { + if (call.callee.type !== 'Identifier') + this.error(call.callee, 'Invalid intrinsic name, must be Identifier'); + + const name = call.callee.name; + if (!INTRINSICS.hasOwnProperty(name)) + this.error(call.callee, 'Unknown intrinsic'); + + const Instruction = INTRINSICS[name]; + + if (!Instruction.validateArgs(call.arguments)) + this.error(call, 'Invalid arguments for intrinsic'); + + this.push(new Instruction(call.arguments, this)); + + return null; + } + + getDeps() { + return this.ir + .filter(instr => instr.type === 'redirect' || instr.type === 'next') + .map(instr => instr.target); + } + + // Mostly helpers + + push(instr) { + this.ir.push(instr); + } + + lookup(id, source) { + if (this.local.has(id.name)) + return this.local.get(id.name); + + source = source || this; + if (this.parent) + return this.parent.lookup(id, source); + + source.error(id, 'Unknown local variable'); + } + + define(id, value) { + if (this.local.has(id.name)) + return this.error(id, 'Re-definition of const variable'); + + let sanitized = `b_${this.index}_v` + + name.toLowerCase().replace(/[^a-z0-9]+/g, '_'); + + if (this.used.has(sanitized)) { + for (let i = 1; ; i++) + if (!this.used.has(sanitized + `_${i}`)) + break; + + sanitized += `_${i}`; + } + + this.local.set(id.name, { name: sanitized, value }); + } + + label(name) { + return `b${this.index}_l_${name}`; + } +} +module.exports = Body; diff --git a/lib/dsl/code.js b/lib/dsl/code.js index ad6125c..2fd0c66 100644 --- a/lib/dsl/code.js +++ b/lib/dsl/code.js @@ -3,11 +3,12 @@ const assert = require('assert'); const dsl = require('./'); -const Block = dsl.Block; +const Body = dsl.Body; class Code { constructor(reporter, name) { this.reporter = reporter; + this.bodies = []; this.name = name; this.deps = new Map(); @@ -41,8 +42,12 @@ class Code { } const stmts = body.slice(0, -1); - const block = new Block(this.reporter); + const block = new Body(this.reporter, this.bodies.length); block.compile(stmts); + this.bodies.push(block); + + block.getDeps().forEach(dep => this.deps.set(dep, null)); + return block; } diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index 8fee8dc..9fcaa2c 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -218,7 +218,7 @@ class Compiler { code.compile(block); this.code.set(name, code); - code.deps.forEach((dep) => { + code.deps.forEach((_, dep) => { code.fillDep(dep, this.buildGraph(blocks, dep, block)); }); diff --git a/lib/dsl/index.js b/lib/dsl/index.js index 154cb56..088b597 100644 --- a/lib/dsl/index.js +++ b/lib/dsl/index.js @@ -1,11 +1,11 @@ 'use strict'; exports.types = require('./types'); -exports.Struct = require('./struct'); +exports.ir = require('./ir'); exports.Reporter = require('./reporter'); exports.State = require('./state'); exports.Settings = require('./settings'); -exports.Block = require('./block'); +exports.Body = require('./body'); exports.Code = require('./code'); exports.Compiler = require('./compiler'); diff --git a/lib/dsl/ir/index.js b/lib/dsl/ir/index.js new file mode 100644 index 0000000..6961658 --- /dev/null +++ b/lib/dsl/ir/index.js @@ -0,0 +1,4 @@ +'use strict'; + +exports.Struct = require('./struct'); +exports.instruction = require('./instruction'); diff --git a/lib/dsl/ir/instruction/base.js b/lib/dsl/ir/instruction/base.js new file mode 100644 index 0000000..fe2c81f --- /dev/null +++ b/lib/dsl/ir/instruction/base.js @@ -0,0 +1,8 @@ +'use strict'; + +class Instruction { + constructor(type) { + this.type = type; + } +} +module.exports = Instruction; diff --git a/lib/dsl/ir/instruction/error.js b/lib/dsl/ir/instruction/error.js new file mode 100644 index 0000000..3fafef8 --- /dev/null +++ b/lib/dsl/ir/instruction/error.js @@ -0,0 +1,22 @@ +'use strict'; + +const Base = require('./').Base; + +class Error extends Base { + static validateArgs(args) { + if (args.length !== 2) + return false; + + return args[0].type === 'Identifier' && + args[1].type === 'Literal' && + typeof args[1].value === 'string'; + } + + constructor(args, body) { + super('error'); + + this.code = body.lookup(args[0]); + this.description = args[1].value; + } +} +module.exports = Error; diff --git a/lib/dsl/ir/instruction/index.js b/lib/dsl/ir/instruction/index.js new file mode 100644 index 0000000..981bbd9 --- /dev/null +++ b/lib/dsl/ir/instruction/index.js @@ -0,0 +1,7 @@ +'use strict'; + +exports.Base = require('./base'); + +exports.Error = require('./error'); +exports.Next = require('./next'); +exports.Redirect = require('./redirect'); diff --git a/lib/dsl/ir/instruction/next.js b/lib/dsl/ir/instruction/next.js new file mode 100644 index 0000000..1b9756d --- /dev/null +++ b/lib/dsl/ir/instruction/next.js @@ -0,0 +1,18 @@ +'use strict'; + +const Base = require('./').Base; + +class Next extends Base { + constructor() { + super('next'); + this.target = args[0].name; + } + + static validateArgs(args) { + if (args.length !== 1 || !args[0]) + return false; + + return args[0].type === 'Identifier'; + } +} +module.exports = Next; diff --git a/lib/dsl/ir/instruction/redirect.js b/lib/dsl/ir/instruction/redirect.js new file mode 100644 index 0000000..a1053d0 --- /dev/null +++ b/lib/dsl/ir/instruction/redirect.js @@ -0,0 +1,18 @@ +'use strict'; + +const Base = require('./').Base; + +class Redirect extends Base { + constructor(args) { + super('redirect'); + this.target = args[0].name; + } + + static validateArgs(args) { + if (args.length !== 1 || !args[0]) + return false; + + return args[0].type === 'Identifier'; + } +} +module.exports = Redirect; diff --git a/lib/dsl/struct.js b/lib/dsl/ir/struct.js similarity index 100% rename from lib/dsl/struct.js rename to lib/dsl/ir/struct.js diff --git a/lib/dsl/reporter.js b/lib/dsl/reporter.js index 227782d..8c1978c 100644 --- a/lib/dsl/reporter.js +++ b/lib/dsl/reporter.js @@ -3,10 +3,10 @@ class Reporter { error(node, message) { if (!node || !node.loc || !node.loc.start) - throw new Error(string + ' at unknown position'); + throw new Error(message + ' at unknown position'); const start = node.loc.start; - const err = new Error(string + ` at ${start.line}:${start.column}`); + const err = new Error(message + ` at ${start.line}:${start.column}`); err.node = node; throw err; } diff --git a/lib/dsl/settings.js b/lib/dsl/settings.js index e9f4db0..bc8922c 100644 --- a/lib/dsl/settings.js +++ b/lib/dsl/settings.js @@ -3,9 +3,8 @@ const assert = require('assert'); const dsl = require('./'); -const Struct = dsl.Struct; -class Settings extends Struct { +class Settings extends dsl.ir.Struct { constructor() { super('settings'); } diff --git a/lib/dsl/state.js b/lib/dsl/state.js index 6fa4735..4c51e16 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -4,9 +4,8 @@ const kState = Symbol('state'); const kError = Symbol('error'); const dsl = require('./'); -const Struct = dsl.Struct; -class State extends Struct { +class State extends dsl.ir.Struct { constructor() { super('state'); diff --git a/test/fixtures/example.js b/test/fixtures/example.js index d35bbca..47faf2f 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -28,7 +28,7 @@ const init = () => { switch (_) { default: - rerun(start_req_or_res); + redirect(start_req_or_res); break; } }; @@ -36,24 +36,26 @@ const init = () => { const start_req_or_res = () => { switch (_) { case [ 0x0a, 0x0d ]: - skip(); break; - '@notify-on-start(on_message_begin)'; case METHODS: + '@notify-on-start(on_message_begin)'; + state.type = HTTP_REQUEST; state.method = match(); next(request_after_method); break; - '@notify-on-start(on_message_begin)'; case 'HTTP': + '@notify-on-start(on_message_begin)'; + state.type = HTTP_RESPONSE; next(response_slash); break; - '@unlikely'; default: + '@unlikely'; + error(INVALID_METHOD, 'Unknown method'); break; } @@ -62,17 +64,17 @@ const start_req_or_res = () => { const request_after_method = () => { switch (_) { case ' ': - skip(); break; - '@unlikely'; case 0x0: + '@unlikely'; + error(INVALID_METHOD, '`\0` after method'); break; default: settings.on_url.start(); - rerun(url); + redirect(url); break; } }; @@ -84,15 +86,17 @@ const url = () => { settings.on_url.end(); break; - '@unlikely'; case [ '\r', '\n' ]: + '@unlikely'; + error(INVALID_URL_CHARACTER, 'URL can\'t have newline chars in it'); break; - '@unlikely'; - '@mode(strict)'; case [ '\t', '\f' ]: + '@ifdef(strict)'; + '@unlikely'; + error(INVALID_URL_CHARACTER, 'URL can\'t have "\\t" or "\\f" chars in it'); break; From 124d7c91ffe4f7e92f81497302e8b9698b524286 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 21:33:47 -0500 Subject: [PATCH 012/281] save --- lib/dsl/body.js | 109 -------------------------- lib/dsl/code.js | 78 ------------------ lib/dsl/compiler.js | 122 +++++++---------------------- lib/dsl/index.js | 3 +- lib/dsl/ir/index.js | 1 - lib/dsl/ir/instruction/base.js | 8 -- lib/dsl/ir/instruction/error.js | 22 ------ lib/dsl/ir/instruction/index.js | 7 -- lib/dsl/ir/instruction/next.js | 18 ----- lib/dsl/ir/instruction/redirect.js | 18 ----- lib/dsl/scope.js | 32 ++++++++ test/fixtures/example.js | 19 +++-- 12 files changed, 73 insertions(+), 364 deletions(-) delete mode 100644 lib/dsl/body.js delete mode 100644 lib/dsl/code.js delete mode 100644 lib/dsl/ir/instruction/base.js delete mode 100644 lib/dsl/ir/instruction/error.js delete mode 100644 lib/dsl/ir/instruction/index.js delete mode 100644 lib/dsl/ir/instruction/next.js delete mode 100644 lib/dsl/ir/instruction/redirect.js create mode 100644 lib/dsl/scope.js diff --git a/lib/dsl/body.js b/lib/dsl/body.js deleted file mode 100644 index 4af364d..0000000 --- a/lib/dsl/body.js +++ /dev/null @@ -1,109 +0,0 @@ -'use strict'; - -const dsl = require('./'); - -const INTRINSICS = { - next: dsl.ir.instruction.Next, - redirect: dsl.ir.instruction.Redirect, - error: dsl.ir.instruction.Error -}; - -class Body { - constructor(reporter, index, parent) { - this.reporter = reporter; - this.index = index; - this.parent = parent || null; - - this.used = new Set(); - this.local = new Map(); - this.ir = []; - } - - error(node, string) { - return this.reporter.error(node, string); - } - - compile(stmts) { - stmts.forEach(stmt => this.compileStatement(stmt)); - } - - compileStatement(stmt) { - if (stmt.type === 'ExpressionStatement') - return this.compileExpression(stmt.expression); - - this.error(stmt, `Unsupported statement type: ${stmt.type}`); - } - - compileExpression(expr) { - if (expr.type === 'CallExpression') - return this.compileCall(expr); - else if (expr.type === 'Literal') - return { type: 'Literal', value: expr.value }; - - this.error(expr, `Unsupported expression type: ${expr.type}`); - } - - compileCall(call) { - if (call.callee.type !== 'Identifier') - this.error(call.callee, 'Invalid intrinsic name, must be Identifier'); - - const name = call.callee.name; - if (!INTRINSICS.hasOwnProperty(name)) - this.error(call.callee, 'Unknown intrinsic'); - - const Instruction = INTRINSICS[name]; - - if (!Instruction.validateArgs(call.arguments)) - this.error(call, 'Invalid arguments for intrinsic'); - - this.push(new Instruction(call.arguments, this)); - - return null; - } - - getDeps() { - return this.ir - .filter(instr => instr.type === 'redirect' || instr.type === 'next') - .map(instr => instr.target); - } - - // Mostly helpers - - push(instr) { - this.ir.push(instr); - } - - lookup(id, source) { - if (this.local.has(id.name)) - return this.local.get(id.name); - - source = source || this; - if (this.parent) - return this.parent.lookup(id, source); - - source.error(id, 'Unknown local variable'); - } - - define(id, value) { - if (this.local.has(id.name)) - return this.error(id, 'Re-definition of const variable'); - - let sanitized = `b_${this.index}_v` + - name.toLowerCase().replace(/[^a-z0-9]+/g, '_'); - - if (this.used.has(sanitized)) { - for (let i = 1; ; i++) - if (!this.used.has(sanitized + `_${i}`)) - break; - - sanitized += `_${i}`; - } - - this.local.set(id.name, { name: sanitized, value }); - } - - label(name) { - return `b${this.index}_l_${name}`; - } -} -module.exports = Body; diff --git a/lib/dsl/code.js b/lib/dsl/code.js deleted file mode 100644 index 2fd0c66..0000000 --- a/lib/dsl/code.js +++ /dev/null @@ -1,78 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const dsl = require('./'); -const Body = dsl.Body; - -class Code { - constructor(reporter, name) { - this.reporter = reporter; - this.bodies = []; - - this.name = name; - this.deps = new Map(); - } - - error(ast, message) { - this.reporter.error(ast, message); - } - - compile(block) { - const s = this.getSwitch(block); - // TODO(indutny): validate discriminant - - const cases = s.cases.map((node) => { - if (node.consequent.length === 0) { - this.error(node, - 'All `case`s in switch must have consequent statements'); - } - - return { - test: node.test, - body: this.compileBody(node, node.consequent) - }; - }) - } - - compileBody(context, body) { - if (body[body.length - 1].type !== 'BreakStatement') { - this.error(body[body.length - 1], - 'Last statement in `case` body must be `break`'); - } - - const stmts = body.slice(0, -1); - const block = new Body(this.reporter, this.bodies.length); - block.compile(stmts); - this.bodies.push(block); - - block.getDeps().forEach(dep => this.deps.set(dep, null)); - - return block; - } - - getSwitch(block) { - let res = false; - - block.body.forEach((stmt) => { - if (stmt.type !== 'SwitchStatement') - return; - - if (res) - this.error(stmt, 'Duplicate switch statement'); - - res = stmt; - }); - - if (!res) - this.error(block, 'No switch statement found in a state function'); - - return res; - } - - fillDep(dep, code) { - assert.strictEqual(this.deps.get(dep), null); - this.deps.set(dep, code); - } -} -module.exports = Code; diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index 9fcaa2c..40cb971 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -5,9 +5,9 @@ const esprima = require('esprima'); const dsl = require('./'); +const Scope = dsl.Scope; const State = dsl.State; const Settings = dsl.Settings; -const Code = dsl.Code; const Reporter = dsl.Reporter; const STATE_TYPES = dsl.types.STATE_TYPES; @@ -27,18 +27,20 @@ class Compiler { this.reporter = new Reporter(); this.defaultState = null; - this.code = null; } compile() { this.ast = esprima.parse(this.source, { loc: true }); assert.equal(this.ast.type, 'Program'); - this.globals = this.parseGlobals(this.ast.body); - this.parseState(); - this.parseSettings(); + this.globals = new Scope(); + this.parseTopLevel(this.ast.body); - this.compileStates(); + if (this.state === null) + this.error(this.ast, 'Missing global `state` declaration'); + + if (this.settings === null) + this.error(this.ast, 'Missing global `settings` declaration'); let out = ''; @@ -54,27 +56,35 @@ class Compiler { return this.reporter.error(node, string); } - parseGlobals(body) { + parseTopLevel(body) { const globals = new Map(); const declaration = (decl) => { - if (globals.has(decl.id.name)) - this.error(decl.id, `Duplicate definition of ${decl.id.name}`); - if (!decl.init) this.error(decl, `Missing init value for ${decl.id.name}`); - globals.set(decl.id.name, decl.init); + // Re-use of previously defined value + let init = decl.init; + while (init.type === 'Identifier') + init = this.globals.lookup(init); + + if (decl.id.name === 'state') + this.parseState(init); + else if (decl.id.name === 'settings') + this.parseSettings(init); + else + this.globals.define(decl.id, init, this); }; body.forEach((node) => { - if (node.type !== 'VariableDeclaration') + if (node.type === 'VariableDeclaration') + return node.declarations.forEach(declaration); + + if (node.type === 'ExpressionStatement' && node.directive) return; - node.declarations.forEach(declaration); + this.error(node, 'Unsupported top-level node'); }); - - return globals; } parseConfigObject(node, types) { @@ -125,12 +135,8 @@ class Compiler { return result; } - parseState() { - assert(this.globals.has('state'), 'Missing `state` global object'); - - const props = this.parseConfigObject( - this.globals.get('state'), STATE_TYPES); - this.globals.delete('state'); + parseState(node) { + const props = this.parseConfigObject(node, STATE_TYPES); this.state = new State(); @@ -142,12 +148,8 @@ class Compiler { }); } - parseSettings() { - assert(this.globals.has('settings'), 'Missing `settings` global object'); - - const props = this.parseConfigObject( - this.globals.get('settings'), SETTINGS_TYPES); - this.globals.delete('settings'); + parseSettings(node) { + const props = this.parseConfigObject(node, SETTINGS_TYPES); this.settings = new Settings(); @@ -158,71 +160,5 @@ class Compiler { this.settings.define(prop.key, prop.type, prop.args); }); } - - compileStates() { - const blocks = new Map(); - - this.globals.forEach((value, key) => { - if (value.type !== 'ArrowFunctionExpression') - return; - if (value.body.type !== 'BlockStatement') - return; - - // TODO(indutny): validate function parameters - // TODO(indutny): ensure no iterators, etc - blocks.set(key, value.body); - }); - - const start = this.findDefault(blocks); - this.defaultState = start; - - this.code = new Map(); - this.buildGraph(blocks, start, this.ast); - } - - findDefault(blocks) { - let found = false; - - blocks.forEach((block, key) => { - const hasDirective = block.body.some((stmt) => { - return stmt.type === 'ExpressionStatement' && - stmt.directive === DIRECTIVE_DEFAULT; - }); - - if (!hasDirective) - return; - - if (found) - this.error(block, 'Two functions with `@default` directive'); - - found = key; - }); - - if (!found) - this.error(this.ast, 'No function with `@default` directive found'); - - return found; - } - - buildGraph(blocks, name, from) { - // Already-built - if (this.code.has(name)) - return this.code.get(name); - - if (!blocks.has(name)) - this.error(from, `Unknown state name ${name}`); - - const block = blocks.get(name); - const code = new Code(this.reporter, name); - - code.compile(block); - - this.code.set(name, code); - code.deps.forEach((_, dep) => { - code.fillDep(dep, this.buildGraph(blocks, dep, block)); - }); - - return code; - } } module.exports = Compiler; diff --git a/lib/dsl/index.js b/lib/dsl/index.js index 088b597..a1f1010 100644 --- a/lib/dsl/index.js +++ b/lib/dsl/index.js @@ -3,9 +3,8 @@ exports.types = require('./types'); exports.ir = require('./ir'); exports.Reporter = require('./reporter'); +exports.Scope = require('./scope'); exports.State = require('./state'); exports.Settings = require('./settings'); -exports.Body = require('./body'); -exports.Code = require('./code'); exports.Compiler = require('./compiler'); diff --git a/lib/dsl/ir/index.js b/lib/dsl/ir/index.js index 6961658..5c6c495 100644 --- a/lib/dsl/ir/index.js +++ b/lib/dsl/ir/index.js @@ -1,4 +1,3 @@ 'use strict'; exports.Struct = require('./struct'); -exports.instruction = require('./instruction'); diff --git a/lib/dsl/ir/instruction/base.js b/lib/dsl/ir/instruction/base.js deleted file mode 100644 index fe2c81f..0000000 --- a/lib/dsl/ir/instruction/base.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict'; - -class Instruction { - constructor(type) { - this.type = type; - } -} -module.exports = Instruction; diff --git a/lib/dsl/ir/instruction/error.js b/lib/dsl/ir/instruction/error.js deleted file mode 100644 index 3fafef8..0000000 --- a/lib/dsl/ir/instruction/error.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -const Base = require('./').Base; - -class Error extends Base { - static validateArgs(args) { - if (args.length !== 2) - return false; - - return args[0].type === 'Identifier' && - args[1].type === 'Literal' && - typeof args[1].value === 'string'; - } - - constructor(args, body) { - super('error'); - - this.code = body.lookup(args[0]); - this.description = args[1].value; - } -} -module.exports = Error; diff --git a/lib/dsl/ir/instruction/index.js b/lib/dsl/ir/instruction/index.js deleted file mode 100644 index 981bbd9..0000000 --- a/lib/dsl/ir/instruction/index.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -exports.Base = require('./base'); - -exports.Error = require('./error'); -exports.Next = require('./next'); -exports.Redirect = require('./redirect'); diff --git a/lib/dsl/ir/instruction/next.js b/lib/dsl/ir/instruction/next.js deleted file mode 100644 index 1b9756d..0000000 --- a/lib/dsl/ir/instruction/next.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -const Base = require('./').Base; - -class Next extends Base { - constructor() { - super('next'); - this.target = args[0].name; - } - - static validateArgs(args) { - if (args.length !== 1 || !args[0]) - return false; - - return args[0].type === 'Identifier'; - } -} -module.exports = Next; diff --git a/lib/dsl/ir/instruction/redirect.js b/lib/dsl/ir/instruction/redirect.js deleted file mode 100644 index a1053d0..0000000 --- a/lib/dsl/ir/instruction/redirect.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -const Base = require('./').Base; - -class Redirect extends Base { - constructor(args) { - super('redirect'); - this.target = args[0].name; - } - - static validateArgs(args) { - if (args.length !== 1 || !args[0]) - return false; - - return args[0].type === 'Identifier'; - } -} -module.exports = Redirect; diff --git a/lib/dsl/scope.js b/lib/dsl/scope.js new file mode 100644 index 0000000..f25c3a0 --- /dev/null +++ b/lib/dsl/scope.js @@ -0,0 +1,32 @@ +'use strict'; + +class Scope { + constructor(parent) { + this.parent = parent || null; + + this.local = new Map(); + } + + forEach(cb) { + this.local.forEach(cb); + } + + lookup(id, source) { + if (this.locals.has(id.name)) + return this.locals.get(id.name); + + if (this.parent) + return this.parent.lookup(id, source); + + source.error(id, `Unknown variable name ${id.name}`); + } + + define(id, value, source) { + if (this.local.has(id.name)) + return source.error(id, 'Re-definition of const variable'); + + this.local.set(id.name, value); + } + +} +module.exports = Scope; diff --git a/test/fixtures/example.js b/test/fixtures/example.js index 47faf2f..e65a268 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -10,13 +10,6 @@ const settings = { on_url: data('url') }; -const METHODS = { - 'GET': 1, - 'HEAD': 2, - 'POST': 3, - 'PUT': 4 -}; - const HTTP_REQUEST = 1; const HTTP_RESPONSE = 2; @@ -38,7 +31,12 @@ const start_req_or_res = () => { case [ 0x0a, 0x0d ]: break; - case METHODS: + case { + 'GET': 1, + 'HEAD': 2, + 'POST': 3, + 'PUT': 4 + }: '@notify-on-start(on_message_begin)'; state.type = HTTP_REQUEST; @@ -103,6 +101,11 @@ const url = () => { } }; +const req_http_start = () => { + switch (_) { + } +}; + const response_slash = () => { switch (_) { } From 956a3cb03e6113c986de466f93cdbd6430d01bd0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 22:35:12 -0500 Subject: [PATCH 013/281] save --- lib/dsl/builder.js | 200 ++++++++++++++++++++++++++++++++++++++ lib/dsl/compiler.js | 54 ++++++---- lib/dsl/index.js | 2 + lib/dsl/ir/base.js | 11 +++ lib/dsl/ir/code.js | 18 ++++ lib/dsl/ir/constant.js | 13 +++ lib/dsl/ir/index.js | 6 ++ lib/dsl/ir/intrinsic.js | 20 ++++ lib/dsl/ir/literal.js | 12 +++ lib/dsl/ir/state-store.js | 14 +++ lib/dsl/scope.js | 4 +- test/fixtures/example.js | 6 +- 12 files changed, 336 insertions(+), 24 deletions(-) create mode 100644 lib/dsl/builder.js create mode 100644 lib/dsl/ir/base.js create mode 100644 lib/dsl/ir/code.js create mode 100644 lib/dsl/ir/constant.js create mode 100644 lib/dsl/ir/intrinsic.js create mode 100644 lib/dsl/ir/literal.js create mode 100644 lib/dsl/ir/state-store.js diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js new file mode 100644 index 0000000..fea7067 --- /dev/null +++ b/lib/dsl/builder.js @@ -0,0 +1,200 @@ +'use strict'; + +const dsl = require('./'); + +const DIRECTIVE_DEFAULT = '@default'; + +const INTRINSICS = new Set([ 'match', 'next', 'redirect', 'error' ]); + +class Block { + constructor(test, scope) { + this.test = test; + this.instructions = []; + + this.scope = new dsl.Scope(scope); + } + + push(instr) { + this.instructions.push(instr); + return instr; + } +} + +class Builder { + constructor(reporter, state, settings, globals) { + this.reporter = reporter; + this.state = state; + this.settings = settings; + this.globals = globals; + + this.id = 1; + this.blocks = []; + this.block = null; + + this.directives = { + isDefault: false + }; + } + + error(node, message) { + this.reporter.error(node, message); + } + + static build(reporter, node) { + if (node.type === 'ArrowFunctionExpression') + return new dsl.ir.Code(node); + + if (node.type === 'Literal') { + const res = new dsl.ir.Constant(node.value); + res.ast = node; + return res; + } + + reporter.error(node, + 'Unexpected global value. Only arrow functions and literal are allowed'); + } + + build(node) { + if (node.type !== 'ArrowFunctionExpression') + return this.error(node, 'Can\'t build this'); + + this.blocks = []; + + if (node.body.type !== 'BlockStatement') { + this.error(node.body, 'Global arrow functions must have block body'); + return; + } + + // TODO(indutny): validate: no async, no iterator, arguments, etc + let s = false; + node.body.body.forEach((stmt) => { + if (stmt.type === 'ExpressionStatement' && stmt.directive) { + return this.processDirective(stmt); + } else if (stmt.type === 'SwitchStatement') { + if (s) + return this.error(stmt, 'Duplicate switch statement'); + + s = stmt; + } + }); + if (!s) + return this.error(node, 'No switch statement in an arrow function'); + + this.buildSwitch(s); + + return this.blocks; + } + + processDirective(stmt) { + if (stmt.directive === DIRECTIVE_DEFAULT) + this.directives.isDefault = true; + + // TODO(indutny): warn on unknown directives + } + + buildSwitch(stmt) { + // TODO(indutny): validate discriminant + stmt.cases.forEach((c) => { + if (c.consequent.length < 1) + return this.error(c, 'Missing required `break` statement'); + + const last = c.consequent[c.consequent.length - 1]; + if (last.type !== 'BreakStatement') + return this.error(last, 'Missing required `break` statement'); + + // TODO(indutny): parse `test` + const block = new Block(c.test, this.globals); + this.blocks.push(block); + + this.block = block; + for (let i = 0; i < c.consequent.length - 1; i++) + this.buildStatement(c.consequent[i]); + this.block = null; + }); + } + + buildStatement(stmt) { + if (stmt.type === 'ExpressionStatement' && + stmt.expression.type === 'Literal') { + return this.processBlockDirective(stmt); + } else if (stmt.type === 'ExpressionStatement') { + return this.buildExpression(stmt.expression); + } + + return this.error(stmt, `Unknown statement type: ${stmt.type}`); + } + + buildExpression(expr) { + if (expr.type === 'CallExpression') + return this.buildCall(expr); + else if (expr.type === 'Identifier') + return this.buildIdentifier(expr); + else if (expr.type === 'Literal') + return this.buildLiteral(expr); + else if (expr.type === 'AssignmentExpression') + return this.buildAssignment(expr); + + return this.error(expr, `Unknown expression type: ${expr.type}`); + } + + buildCall(call) { + const callee = call.callee; + const args = call.arguments.map(arg => this.buildExpression(arg)); + + if (callee.type === 'Identifier' && INTRINSICS.has(callee.name)) + return this.buildIntrinsic(call, callee.name, args); + + return this.error(call, `Unsupported callee: ${callee.type}`); + } + + buildIntrinsic(node, name, args) { + return this.push(node, new dsl.ir.Intrinsic(name, args)); + } + + buildIdentifier(id) { + return this.block.scope.lookup(id, this); + } + + buildLiteral(literal) { + return this.push(literal, new dsl.ir.Constant(literal.value)); + } + + buildAssignment(expr) { + const left = expr.left; + + if (left.type !== 'MemberExpression' || + left.object.type !== 'Identifier' || + left.object.name !== 'state') { + return this.error(left, 'Expected a property of `state` object'); + } + + if (left.computed) + return this.error(left, 'Property of `state` object can\'t be computed'); + + const prop = left.property.name; + if (!this.state.has(prop)) { + return this.error(left.property, + `Unknown \`state\` property name ${prop}`); + } + + const index = this.state.lookup(prop); + const right = this.buildExpression(expr.right); + this.push(expr, new dsl.ir.StateStore(index, right)); + return right; + } + + processBlockDirective(stmt) { + const directive = stmt.expression.value; + + // TODO(indutny): warn on unknown directives + } + + push(ast, instr) { + if (instr.out === null) + instr.out = this.id++; + instr.ast = ast; + + return this.block.push(instr); + } +} +module.exports = Builder; diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index 40cb971..ad81cca 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -9,12 +9,11 @@ const Scope = dsl.Scope; const State = dsl.State; const Settings = dsl.Settings; const Reporter = dsl.Reporter; +const Builder = dsl.Builder; const STATE_TYPES = dsl.types.STATE_TYPES; const SETTINGS_TYPES = dsl.types.SETTINGS_TYPES; -const DIRECTIVE_DEFAULT = '@default'; - class Compiler { constructor(prefix, source) { this.prefix = prefix; @@ -24,8 +23,11 @@ class Compiler { this.globals = null; this.state = null; this.settings = null; + this.builder = null; this.reporter = new Reporter(); + this.code = new Map(); + this.defaultState = null; } @@ -34,13 +36,14 @@ class Compiler { assert.equal(this.ast.type, 'Program'); this.globals = new Scope(); - this.parseTopLevel(this.ast.body); + this.state = new State(); + this.settings = new Settings(); + this.builder = new Builder(this.reporter, this.state, this.settings, + this.globals); - if (this.state === null) - this.error(this.ast, 'Missing global `state` declaration'); + this.parseTopLevel(this.ast.body); - if (this.settings === null) - this.error(this.ast, 'Missing global `settings` declaration'); + this.buildCode(); let out = ''; @@ -61,7 +64,7 @@ class Compiler { const declaration = (decl) => { if (!decl.init) - this.error(decl, `Missing init value for ${decl.id.name}`); + return this.error(decl, `Missing init value for ${decl.id.name}`); // Re-use of previously defined value let init = decl.init; @@ -73,7 +76,7 @@ class Compiler { else if (decl.id.name === 'settings') this.parseSettings(init); else - this.globals.define(decl.id, init, this); + this.globals.define(decl.id, Builder.build(this.reporter, init), this); }; body.forEach((node) => { @@ -91,43 +94,47 @@ class Compiler { if (node.type !== 'ObjectExpression') { this.error(node, 'Invalid configuration value type, must be ObjectExpression'); + return; } const result = []; node.properties.forEach((prop) => { if (prop.computed) - this.error(prop, 'Configuration properties can\'t be computed'); + return this.error(prop, 'Configuration properties can\'t be computed'); if (prop.key.type !== 'Identifier') { this.error(prop.key, 'Configuration property keys must be Identifiers'); + return; } if (prop.value.type !== 'CallExpression') { this.error(prop.key, 'Configuration property values must be CallExpressions'); + return; } const key = prop.key.name; if (prop.value.callee.type !== 'Identifier') - this.error(value.callee, 'Unknown configuration property type'); + return this.error(value.callee, 'Unknown configuration property type'); const type = prop.value.callee.name; const args = prop.value.arguments.map((arg) => { if (arg.type !== 'Literal') - this.error(args, 'Invalid argument type, must be Literal'); + return this.error(args, 'Invalid argument type, must be Literal'); return arg.value; }); if (!types.hasOwnProperty(type)) - this.error(value.callee, 'Unknown `state` property type'); + return this.error(value.callee, 'Unknown `state` property type'); const validation = types[type](args); if (validation !== true) { this.error(value.callee, `Invalid arguments for "${type}": ${validation}`); + return; } result.push({ key, type, args }); @@ -138,27 +145,36 @@ class Compiler { parseState(node) { const props = this.parseConfigObject(node, STATE_TYPES); - this.state = new State(); - props.forEach((prop) => { if (this.state.has(prop.key)) - this.error(prop.key, 'Duplicate key in `state` object'); + return this.error(prop.key, 'Duplicate key in `state` object'); this.state.define(prop.key, prop.type, prop.args); }); + + // TODO(indutny): error on duplicate `state` definition } parseSettings(node) { const props = this.parseConfigObject(node, SETTINGS_TYPES); - this.settings = new Settings(); - props.forEach((prop) => { if (this.settings.has(prop.key)) - this.error(prop.key, 'Duplicate key in `settings` object'); + return this.error(prop.key, 'Duplicate key in `settings` object'); this.settings.define(prop.key, prop.type, prop.args); }); + + // TODO(indutny): error on duplicate `settings` definition + } + + buildCode() { + this.globals.forEach((value, key) => { + if (!(value instanceof dsl.ir.Code)) + return; + + this.code.set(key, value.build(this.builder)); + }); } } module.exports = Compiler; diff --git a/lib/dsl/index.js b/lib/dsl/index.js index a1f1010..f3c3f86 100644 --- a/lib/dsl/index.js +++ b/lib/dsl/index.js @@ -7,4 +7,6 @@ exports.Scope = require('./scope'); exports.State = require('./state'); exports.Settings = require('./settings'); + +exports.Builder = require('./builder'); exports.Compiler = require('./compiler'); diff --git a/lib/dsl/ir/base.js b/lib/dsl/ir/base.js new file mode 100644 index 0000000..3695cb1 --- /dev/null +++ b/lib/dsl/ir/base.js @@ -0,0 +1,11 @@ +'use strict'; + +class Base { + constructor(type) { + this.type = type; + this.ast = null; + + this.out = null; + } +} +module.exports = Base; diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js new file mode 100644 index 0000000..2264b4d --- /dev/null +++ b/lib/dsl/ir/code.js @@ -0,0 +1,18 @@ +'use strict'; + +const ir = require('./'); + +class Code extends ir.Base { + constructor(ast) { + super('code'); + + this.out = false; + + this.ast = ast; + } + + build(builder) { + return builder.build(this.ast); + } +} +module.exports = Code; diff --git a/lib/dsl/ir/constant.js b/lib/dsl/ir/constant.js new file mode 100644 index 0000000..8f6b2f5 --- /dev/null +++ b/lib/dsl/ir/constant.js @@ -0,0 +1,13 @@ +'use strict'; + +const ir = require('./'); + +class Constant extends ir.Base { + constructor(ast) { + super('constant'); + + this.out = false; + this.ast = ast; + } +} +module.exports = Constant; diff --git a/lib/dsl/ir/index.js b/lib/dsl/ir/index.js index 5c6c495..452c307 100644 --- a/lib/dsl/ir/index.js +++ b/lib/dsl/ir/index.js @@ -1,3 +1,9 @@ 'use strict'; +exports.Base = require('./base'); +exports.Code = require('./code'); +exports.Constant = require('./constant'); +exports.Intrinsic = require('./intrinsic'); +exports.StateStore = require('./state-store'); + exports.Struct = require('./struct'); diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js new file mode 100644 index 0000000..6b8eea9 --- /dev/null +++ b/lib/dsl/ir/intrinsic.js @@ -0,0 +1,20 @@ +'use strict'; + +const ir = require('./'); + +class Intrinsic extends ir.Base { + constructor(name, args) { + super('intrinsic'); + + this.name = name; + this.args = args; + + if (name !== 'match') + this.out = false; + } + + build(builder) { + return builder.build(this.ast); + } +} +module.exports = Intrinsic; diff --git a/lib/dsl/ir/literal.js b/lib/dsl/ir/literal.js new file mode 100644 index 0000000..4110a3c --- /dev/null +++ b/lib/dsl/ir/literal.js @@ -0,0 +1,12 @@ +'use strict'; + +const ir = require('./'); + +class Literal extends ir.Base { + constructor(ast) { + super('literal'); + + this.ast = ast; + } +} +module.exports = Literal; diff --git a/lib/dsl/ir/state-store.js b/lib/dsl/ir/state-store.js new file mode 100644 index 0000000..206a80a --- /dev/null +++ b/lib/dsl/ir/state-store.js @@ -0,0 +1,14 @@ +'use strict'; + +const ir = require('./'); + +class Constant extends ir.Base { + constructor(index, value) { + super('state:store'); + + this.out = false; + this.index = index; + this.value = value; + } +} +module.exports = Constant; diff --git a/lib/dsl/scope.js b/lib/dsl/scope.js index f25c3a0..dec886a 100644 --- a/lib/dsl/scope.js +++ b/lib/dsl/scope.js @@ -12,8 +12,8 @@ class Scope { } lookup(id, source) { - if (this.locals.has(id.name)) - return this.locals.get(id.name); + if (this.local.has(id.name)) + return this.local.get(id.name); if (this.parent) return this.parent.lookup(id, source); diff --git a/test/fixtures/example.js b/test/fixtures/example.js index e65a268..1d4d825 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -67,11 +67,11 @@ const request_after_method = () => { case 0x0: '@unlikely'; - error(INVALID_METHOD, '`\0` after method'); + error(INVALID_METHOD, '`\\0` after method'); break; default: - settings.on_url.start(); +// settings.on_url.start(); redirect(url); break; } @@ -81,7 +81,7 @@ const url = () => { switch (_) { case ' ': next(req_http_start); - settings.on_url.end(); +// settings.on_url.end(); break; case [ '\r', '\n' ]: From e6b01481ac9b3c3bc71e04afaada650915e94faf Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 22:50:21 -0500 Subject: [PATCH 014/281] save --- lib/dsl/builder.js | 15 ++++++++++++--- lib/dsl/compiler.js | 37 ++++++++++++++++++++++++------------- lib/dsl/ir/code.js | 24 ++++++++++++++++++++++-- lib/dsl/ir/constant.js | 7 +++++++ lib/dsl/ir/intrinsic.js | 31 +++++++++++++++++++++++++++++++ lib/dsl/ir/literal.js | 12 ------------ lib/dsl/ir/state-store.js | 4 ++++ test/fixtures/example.js | 4 ++++ 8 files changed, 104 insertions(+), 30 deletions(-) delete mode 100644 lib/dsl/ir/literal.js diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index fea7067..81d516b 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -18,6 +18,13 @@ class Block { this.instructions.push(instr); return instr; } + + serialize(reporter) { + let out = ''; + for (let i = 0; i < this.instructions.length; i++) + out += ' ' + this.instructions[i].serialize(reporter) + '\n'; + return out; + } } class Builder { @@ -40,9 +47,9 @@ class Builder { this.reporter.error(node, message); } - static build(reporter, node) { + static build(reporter, name, node) { if (node.type === 'ArrowFunctionExpression') - return new dsl.ir.Code(node); + return new dsl.ir.Code(name, node); if (node.type === 'Literal') { const res = new dsl.ir.Constant(node.value); @@ -156,7 +163,9 @@ class Builder { } buildLiteral(literal) { - return this.push(literal, new dsl.ir.Constant(literal.value)); + const res = new dsl.ir.Constant(literal.value); + res.ast = literal; + return res; } buildAssignment(expr) { diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index ad81cca..6ebe236 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -45,14 +45,7 @@ class Compiler { this.buildCode(); - let out = ''; - - out += this.state.serialize(); - out += '\n'; - out += this.settings.serialize(); - out += '\n'; - - return out; + return this.serialize(); } error(node, string) { @@ -72,11 +65,13 @@ class Compiler { init = this.globals.lookup(init); if (decl.id.name === 'state') - this.parseState(init); + return this.parseState(init); else if (decl.id.name === 'settings') - this.parseSettings(init); - else - this.globals.define(decl.id, Builder.build(this.reporter, init), this); + return this.parseSettings(init); + + this.globals.define(decl.id, + Builder.build(this.reporter, decl.id.name, init), + this); }; body.forEach((node) => { @@ -173,8 +168,24 @@ class Compiler { if (!(value instanceof dsl.ir.Code)) return; - this.code.set(key, value.build(this.builder)); + value.build(this.builder); + this.code.set(key, value); }); } + + serialize() { + let out = ''; + + out += this.state.serialize(); + out += '\n'; + out += this.settings.serialize(); + out += '\n'; + + this.code.forEach((code) => { + out += code.serialize(this.reporter) + '\n'; + }); + + return out; + } } module.exports = Compiler; diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js index 2264b4d..03b3fd3 100644 --- a/lib/dsl/ir/code.js +++ b/lib/dsl/ir/code.js @@ -3,16 +3,36 @@ const ir = require('./'); class Code extends ir.Base { - constructor(ast) { + constructor(name, ast) { super('code'); this.out = false; + this.name = name; this.ast = ast; + this.blocks = null; } build(builder) { - return builder.build(this.ast); + this.blocks = builder.build(this.ast); + } + + serialize(reporter) { + if (!this.blocks) + return reporter.error(this.ast, 'Not built!'); + if (this.blocks.length === 0) + return reporter.error(this.ast, 'No blocks to serialize'); + + let out = `define i32 @${this.name}(%state* %s) {\n`; + + // TODO(indutny): switch! + + this.blocks.forEach((block) => { + out += block.serialize(reporter); + }); + + out += '}\n'; + return out; } } module.exports = Code; diff --git a/lib/dsl/ir/constant.js b/lib/dsl/ir/constant.js index 8f6b2f5..a6dc04b 100644 --- a/lib/dsl/ir/constant.js +++ b/lib/dsl/ir/constant.js @@ -9,5 +9,12 @@ class Constant extends ir.Base { this.out = false; this.ast = ast; } + + serialize(reporter) { + if (typeof this.ast.value === 'number') + return `i64 ${this.ast.value | 0}`; + + return `c${JSON.stringify(this.ast.value)}`; + } } module.exports = Constant; diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index 6b8eea9..6139bf0 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -16,5 +16,36 @@ class Intrinsic extends ir.Base { build(builder) { return builder.build(this.ast); } + + serialize(reporter) { + const name = this.name; + + if (name === 'match') + return this.serializeMatch(reporter); + else if (name === 'next') + return this.serializeNext(reporter); + else if (name === 'redirect') + return this.serializeRedirect(reporter); + else if (name === 'error') + return this.serializeError(reporter); + else + return reporter.error(this.ast, `Unknown intrinsic name=${name}`); + } + + serializeMatch(reporter) { + return 'match'; + } + + serializeNext(reporter) { + return 'next'; + } + + serializeRedirect(reporter) { + return 'redirect'; + } + + serializeError(reporter) { + return 'error'; + } } module.exports = Intrinsic; diff --git a/lib/dsl/ir/literal.js b/lib/dsl/ir/literal.js deleted file mode 100644 index 4110a3c..0000000 --- a/lib/dsl/ir/literal.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -const ir = require('./'); - -class Literal extends ir.Base { - constructor(ast) { - super('literal'); - - this.ast = ast; - } -} -module.exports = Literal; diff --git a/lib/dsl/ir/state-store.js b/lib/dsl/ir/state-store.js index 206a80a..76a383b 100644 --- a/lib/dsl/ir/state-store.js +++ b/lib/dsl/ir/state-store.js @@ -10,5 +10,9 @@ class Constant extends ir.Base { this.index = index; this.value = value; } + + serialize(reporter) { + return 'store'; + } } module.exports = Constant; diff --git a/test/fixtures/example.js b/test/fixtures/example.js index 1d4d825..a5345eb 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -103,10 +103,14 @@ const url = () => { const req_http_start = () => { switch (_) { + default: + break; } }; const response_slash = () => { switch (_) { + default: + break; } }; From ae0d8d66fc7f980968ee37f31cacc7daa9d2032f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 23:06:56 -0500 Subject: [PATCH 015/281] save --- lib/dsl/builder.js | 15 +++++++++++++-- lib/dsl/ir/base.js | 1 + lib/dsl/ir/code.js | 4 +++- lib/dsl/ir/intrinsic.js | 21 +++++++++++++++++---- lib/dsl/ir/state-store.js | 2 +- lib/dsl/state.js | 2 +- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index 81d516b..bec05cc 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -21,8 +21,17 @@ class Block { serialize(reporter) { let out = ''; - for (let i = 0; i < this.instructions.length; i++) - out += ' ' + this.instructions[i].serialize(reporter) + '\n'; + for (let i = 0; i < this.instructions.length; i++) { + const instr = this.instructions[i].serialize(reporter); + + if (!Array.isArray(instr)) { + out += ' ' + instr + '\n'; + continue; + } + + for (let j = 0; j < instr.length; j++) + out += ' ' + instr[j] + '\n'; + } return out; } } @@ -201,6 +210,8 @@ class Builder { push(ast, instr) { if (instr.out === null) instr.out = this.id++; + if (instr.tmp === null) + instr.tmp = this.id++; instr.ast = ast; return this.block.push(instr); diff --git a/lib/dsl/ir/base.js b/lib/dsl/ir/base.js index 3695cb1..53e3d96 100644 --- a/lib/dsl/ir/base.js +++ b/lib/dsl/ir/base.js @@ -6,6 +6,7 @@ class Base { this.ast = null; this.out = null; + this.tmp = false; } } module.exports = Base; diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js index 03b3fd3..5110b48 100644 --- a/lib/dsl/ir/code.js +++ b/lib/dsl/ir/code.js @@ -23,7 +23,7 @@ class Code extends ir.Base { if (this.blocks.length === 0) return reporter.error(this.ast, 'No blocks to serialize'); - let out = `define i32 @${this.name}(%state* %s) {\n`; + let out = `define i8* @${this.name}(%state* %s, i8* %p, i64 %len) {\n`; // TODO(indutny): switch! @@ -31,6 +31,8 @@ class Code extends ir.Base { out += block.serialize(reporter); }); + out += ' ret i8* null\n'; + out += '}\n'; return out; } diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index 6139bf0..5e7d96f 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -6,6 +6,8 @@ class Intrinsic extends ir.Base { constructor(name, args) { super('intrinsic'); + this.tmp = null; + this.name = name; this.args = args; @@ -33,19 +35,30 @@ class Intrinsic extends ir.Base { } serializeMatch(reporter) { - return 'match'; + return '; match'; } serializeNext(reporter) { - return 'next'; + return '; next'; } serializeRedirect(reporter) { - return 'redirect'; + if (this.args.length !== 1) + return reporter.error('`redirect` must have one argument'); + + const target = this.args[0]; + if (target.type !== 'code') + return reporter.error('`redirect` takes state name as an argument'); + + return [ + `%t${this.tmp} = tail call i8* @${target.name}` + + `(%state* %s, i8* %p, i64 %len)`, + `ret i8* %t${this.tmp}` + ]; } serializeError(reporter) { - return 'error'; + return '; error'; } } module.exports = Intrinsic; diff --git a/lib/dsl/ir/state-store.js b/lib/dsl/ir/state-store.js index 76a383b..d7be97a 100644 --- a/lib/dsl/ir/state-store.js +++ b/lib/dsl/ir/state-store.js @@ -12,7 +12,7 @@ class Constant extends ir.Base { } serialize(reporter) { - return 'store'; + return '; state-store'; } } module.exports = Constant; diff --git a/lib/dsl/state.js b/lib/dsl/state.js index 4c51e16..25c3fca 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -9,7 +9,7 @@ class State extends dsl.ir.Struct { constructor() { super('state'); - this.define(kState, 'i8 (%state*, i8*)', []); + this.define(kState, 'i8* (%state*, i8*, i64)*', []); this.define(kError, 'i32', []); } From f983676e800f90c6285d9c9141a35deee2b8ca4b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 23:14:52 -0500 Subject: [PATCH 016/281] save --- lib/dsl/ir/code.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js index 5110b48..cecaea6 100644 --- a/lib/dsl/ir/code.js +++ b/lib/dsl/ir/code.js @@ -23,7 +23,8 @@ class Code extends ir.Base { if (this.blocks.length === 0) return reporter.error(this.ast, 'No blocks to serialize'); - let out = `define i8* @${this.name}(%state* %s, i8* %p, i64 %len) {\n`; + let out = `define internal i8* @${this.name}` + + `(%state* %s, i8* %p, i64 %len) {\n`; // TODO(indutny): switch! @@ -31,7 +32,10 @@ class Code extends ir.Base { out += block.serialize(reporter); }); - out += ' ret i8* null\n'; + out += '\n'; + out += ' ; return self\n'; + out += ` %self = bitcast i8* (%state*, i8*, i64)* @${this.name} to i8*\n`; + out += ' ret i8* %self\n'; out += '}\n'; return out; From 600f964942d96680ff5c9c5bc2246bd85acde780 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 23:22:12 -0500 Subject: [PATCH 017/281] save --- lib/dsl/ir/intrinsic.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index 5e7d96f..b8cd801 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -6,7 +6,9 @@ class Intrinsic extends ir.Base { constructor(name, args) { super('intrinsic'); - this.tmp = null; + // Request tmp for redirect + if (name === 'redirect') + this.tmp = null; this.name = name; this.args = args; From 719902ce9b614821ba1a006f164702d0a6e9a5e0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Feb 2018 23:44:52 -0500 Subject: [PATCH 018/281] save --- lib/dsl/builder.js | 10 +++++++-- lib/dsl/ir/intrinsic.js | 50 ++++++++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index bec05cc..0cb3c64 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -210,8 +210,14 @@ class Builder { push(ast, instr) { if (instr.out === null) instr.out = this.id++; - if (instr.tmp === null) - instr.tmp = this.id++; + + if (typeof instr.tmp === 'number') { + const tmp = []; + for (let i = 0; i < instr.tmp; i++) + tmp.push(this.id++); + instr.tmp = tmp; + } + instr.ast = ast; return this.block.push(instr); diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index b8cd801..8fdbc7f 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -8,7 +8,9 @@ class Intrinsic extends ir.Base { // Request tmp for redirect if (name === 'redirect') - this.tmp = null; + this.tmp = 1; + else if (name === 'next') + this.tmp = 4; this.name = name; this.args = args; @@ -41,26 +43,52 @@ class Intrinsic extends ir.Base { } serializeNext(reporter) { - return '; next'; + const t0 = `%t${this.tmp[0]}`; + const t1 = `%t${this.tmp[1]}`; + const t2 = `%t${this.tmp[2]}`; + const t3 = `%t${this.tmp[3]}`; + + return [ + '; next', + `${t0} = getelementptr i8, i8* %p, i32 1`, + `${t1} = add i64 -1, %len`, + `${t2} = trunc i64 ${t1} to i1`, + `br i1 ${t2}, label %non_empty${this.tmp[0]}, ` + + `label %empty${this.tmp[0]}`, + ``, + `non_empty${this.tmp[0]}:`, + ``, + `${t3} = tail call i8* @${this.getTarget(reporter)}` + + `(%state* %s, i8* ${t0}, i64 ${t1})`, + `ret i8* ${t3}`, + ``, + `empty${this.tmp[0]}:` + ]; } serializeRedirect(reporter) { - if (this.args.length !== 1) - return reporter.error('`redirect` must have one argument'); - - const target = this.args[0]; - if (target.type !== 'code') - return reporter.error('`redirect` takes state name as an argument'); - + const t0 = `%t${this.tmp[0]}`; return [ - `%t${this.tmp} = tail call i8* @${target.name}` + + `; redirect`, + `${t0} = tail call i8* @${this.getTarget(reporter)}` + `(%state* %s, i8* %p, i64 %len)`, - `ret i8* %t${this.tmp}` + `ret i8* ${t0}` ]; } serializeError(reporter) { return '; error'; } + + getTarget(reporter) { + if (this.args.length !== 1) + return reporter.error('`redirect` must have one argument'); + + const target = this.args[0]; + if (target.type !== 'code') + return reporter.error('`redirect` takes state name as an argument'); + + return target.name; + } } module.exports = Intrinsic; From 4d798d65228b9339abeebad9b84b384f3eb694bd Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 00:01:31 -0500 Subject: [PATCH 019/281] save --- .gitignore | 3 +++ lib/dsl/ir/intrinsic.js | 8 ++++---- test/fixtures/example.js | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index bc4c847..a7b58f8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ node_modules/ npm-debug.log *.ll *.o +*.c +main +1 diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index 8fdbc7f..ec84560 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -51,10 +51,10 @@ class Intrinsic extends ir.Base { return [ '; next', `${t0} = getelementptr i8, i8* %p, i32 1`, - `${t1} = add i64 -1, %len`, - `${t2} = trunc i64 ${t1} to i1`, - `br i1 ${t2}, label %non_empty${this.tmp[0]}, ` + - `label %empty${this.tmp[0]}`, + `${t1} = sub i64 %len, 1`, + `${t2} = icmp eq i64 ${t1}, 0`, + `br i1 ${t2}, label %empty${this.tmp[0]}, ` + + `label %non_empty${this.tmp[0]}`, ``, `non_empty${this.tmp[0]}:`, ``, diff --git a/test/fixtures/example.js b/test/fixtures/example.js index a5345eb..c2e1885 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -114,3 +114,19 @@ const response_slash = () => { break; } }; + +const rec1 = () => { + switch (_) { + default: + next(rec2); + break; + } +}; + +const rec2 = () => { + switch (_) { + default: + next(rec1); + break; + } +}; From 8f5143769d85e5eb788b6791dcbd655acf7ba9b0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 00:06:38 -0500 Subject: [PATCH 020/281] save --- lib/dsl/state.js | 22 +++++++++++++++++++--- test/fixtures/example.js | 16 ---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/dsl/state.js b/lib/dsl/state.js index 25c3fca..c021b0e 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -1,7 +1,8 @@ 'use strict'; -const kState = Symbol('state'); +const kCurrent = Symbol('current'); const kError = Symbol('error'); +const kIndex = Symbol('index'); const dsl = require('./'); @@ -9,18 +10,33 @@ class State extends dsl.ir.Struct { constructor() { super('state'); - this.define(kState, 'i8* (%state*, i8*, i64)*', []); + this.define(kCurrent, 'i8* (%state*, i8*, i64)*', []); + this.define(kIndex, 'i64', []); this.define(kError, 'i32', []); } define(key, type, args) { const commentKey = key === kError ? '[error]' : - key === kState ? '[state]' : key; + key === kCurrent ? '[current]' : + key === kIndex ? '[index]' : + key; let comment = `${commentKey}`; if (args.length !== 0) out += ` ${JSON.stringify(args)}`; super.define(key, type, comment); } + + current() { + return this.lookup(kCurrent); + } + + error() { + return this.lookup(kError); + } + + index() { + return this.lookup(kIndex); + } } module.exports = State; diff --git a/test/fixtures/example.js b/test/fixtures/example.js index c2e1885..a5345eb 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -114,19 +114,3 @@ const response_slash = () => { break; } }; - -const rec1 = () => { - switch (_) { - default: - next(rec2); - break; - } -}; - -const rec2 = () => { - switch (_) { - default: - next(rec1); - break; - } -}; From 6fa8f1da7c5828699538b548141398e5d19e6a13 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 00:07:36 -0500 Subject: [PATCH 021/281] save --- lib/dsl/ir/intrinsic.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index ec84560..15a25fd 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -52,9 +52,9 @@ class Intrinsic extends ir.Base { '; next', `${t0} = getelementptr i8, i8* %p, i32 1`, `${t1} = sub i64 %len, 1`, - `${t2} = icmp eq i64 ${t1}, 0`, - `br i1 ${t2}, label %empty${this.tmp[0]}, ` + - `label %non_empty${this.tmp[0]}`, + `${t2} = icmp ne i64 ${t1}, 0`, + `br i1 ${t2}, label %non_empty${this.tmp[0]}, ` + + `label %empty${this.tmp[0]}`, ``, `non_empty${this.tmp[0]}:`, ``, From fc08380c2a9a7c0c376ac8ca63a3852e4ca0cb66 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 00:56:52 -0500 Subject: [PATCH 022/281] save --- lib/dsl/builder.js | 4 +--- lib/dsl/compiler.js | 29 +++++++++++++++++++++++------ lib/dsl/index.js | 2 ++ lib/dsl/ir/struct.js | 20 +++++++++++++++++--- lib/dsl/settings.js | 8 +++++++- lib/dsl/state.js | 20 +++++++++++++++++++- 6 files changed, 69 insertions(+), 14 deletions(-) diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index 0cb3c64..960055c 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -2,8 +2,6 @@ const dsl = require('./'); -const DIRECTIVE_DEFAULT = '@default'; - const INTRINSICS = new Set([ 'match', 'next', 'redirect', 'error' ]); class Block { @@ -102,7 +100,7 @@ class Builder { } processDirective(stmt) { - if (stmt.directive === DIRECTIVE_DEFAULT) + if (stmt.directive === dsl.constants.directives.DEFAULT) this.directives.isDefault = true; // TODO(indutny): warn on unknown directives diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js index 6ebe236..4103a72 100644 --- a/lib/dsl/compiler.js +++ b/lib/dsl/compiler.js @@ -5,6 +5,7 @@ const esprima = require('esprima'); const dsl = require('./'); +const directives = dsl.constants.directives; const Scope = dsl.Scope; const State = dsl.State; const Settings = dsl.Settings; @@ -23,7 +24,6 @@ class Compiler { this.globals = null; this.state = null; this.settings = null; - this.builder = null; this.reporter = new Reporter(); this.code = new Map(); @@ -38,8 +38,6 @@ class Compiler { this.globals = new Scope(); this.state = new State(); this.settings = new Settings(); - this.builder = new Builder(this.reporter, this.state, this.settings, - this.globals); this.parseTopLevel(this.ast.body); @@ -164,21 +162,40 @@ class Compiler { } buildCode() { + this.defaultState = null; + this.globals.forEach((value, key) => { if (!(value instanceof dsl.ir.Code)) return; - value.build(this.builder); + const builder = new Builder(this.reporter, this.state, this.settings, + this.globals); + value.build(builder); + this.code.set(key, value); + + if (!builder.directives.isDefault) + return; + + if (this.defaultState !== null) { + return this.error(value.ast, + `Two states with \`${directives.DEFAULT}\` directive`); + } + + this.defaultState = key; }); + + if (this.defaultState !== null) + return; + this.error(this.ast, `No state with \`${directives.DEFAULT}\` directive`); } serialize() { let out = ''; - out += this.state.serialize(); + out += this.state.serialize(this.prefix, this.defaultState); out += '\n'; - out += this.settings.serialize(); + out += this.settings.serialize(this.prefix); out += '\n'; this.code.forEach((code) => { diff --git a/lib/dsl/index.js b/lib/dsl/index.js index f3c3f86..930bdf9 100644 --- a/lib/dsl/index.js +++ b/lib/dsl/index.js @@ -1,6 +1,8 @@ 'use strict'; +exports.constants = require('./constants'); exports.types = require('./types'); + exports.ir = require('./ir'); exports.Reporter = require('./reporter'); exports.Scope = require('./scope'); diff --git a/lib/dsl/ir/struct.js b/lib/dsl/ir/struct.js index 4278379..2025c66 100644 --- a/lib/dsl/ir/struct.js +++ b/lib/dsl/ir/struct.js @@ -10,11 +10,11 @@ class Struct { this.props = []; } - define(name, type, comment) { + define(name, type, info) { assert(!this.has(name), `Duplicate entry in ${this.name}`); const index = this.props.length; - const entry = { name, type, comment }; + const entry = { name, type, info }; this.props.push(entry); this.map.set(name, index); @@ -28,6 +28,10 @@ class Struct { return this.map.get(name); } + forEach(callback) { + return this.map.forEach(callback); + } + serialize() { let out = ''; @@ -36,11 +40,21 @@ class Struct { const isLast = index === this.props.length - 1; out += ` ${prop.type}${isLast ? '' : ','}`; - out += ` ; ${index} => ${prop.comment}\n`; + out += ` ; ${index} => ${prop.info.comment}\n`; }); out += '}\n'; return out; } + + serializeC(prefix) { + let out = ''; + const name = this.name; + + out += `typedef struct ${prefix}_${name}_s ${prefix}_${name}_t;\n`; + out += `void ${prefix}_${name}_init(${prefix}_${name}_t* s);\n`; + + return out; + } } module.exports = Struct; diff --git a/lib/dsl/settings.js b/lib/dsl/settings.js index bc8922c..599cb92 100644 --- a/lib/dsl/settings.js +++ b/lib/dsl/settings.js @@ -11,6 +11,7 @@ class Settings extends dsl.ir.Struct { define(key, type, args) { let nativeType; + let cType; if (type === 'notify') { nativeType = 'i32 (%state*)*'; } else { @@ -22,7 +23,12 @@ class Settings extends dsl.ir.Struct { if (args.length !== 0) comment += ` ${JSON.stringify(args)}`; - super.define(key, nativeType, comment); + super.define(key, nativeType, { type, comment }); + } + + serializeC(prefix) { + let out = ''; + return out + super.serializeC(); } } module.exports = Settings; diff --git a/lib/dsl/state.js b/lib/dsl/state.js index c021b0e..30ab869 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -24,7 +24,7 @@ class State extends dsl.ir.Struct { if (args.length !== 0) out += ` ${JSON.stringify(args)}`; - super.define(key, type, comment); + super.define(key, type, { comment }); } current() { @@ -38,5 +38,23 @@ class State extends dsl.ir.Struct { index() { return this.lookup(kIndex); } + + serialize(prefix, current) { + let out = ''; + + out += super.serialize(); + out += '\n'; + + // TODO(indutny): set current style + out += `define void @${prefix}_init(%state* %s) {\n`; + out += `}\n`; + + return out; + } + + serializeC(prefix) { + let out = ''; + return out + super.serializeC(); + } } module.exports = State; From 853c32715135b04a0b8a6e7036b05c7db5b79800 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 00:56:57 -0500 Subject: [PATCH 023/281] save --- lib/dsl/constants.js | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 lib/dsl/constants.js diff --git a/lib/dsl/constants.js b/lib/dsl/constants.js new file mode 100644 index 0000000..a4ac4d0 --- /dev/null +++ b/lib/dsl/constants.js @@ -0,0 +1,5 @@ +'use strict'; + +exports.directives = { + DEFAULT: '@default' +}; From f4da43e7c98bc750466c118198a93e376996e723 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 01:23:11 -0500 Subject: [PATCH 024/281] save --- lib/dsl/builder.js | 15 +++++---------- lib/dsl/ir/constant.js | 8 +++++--- lib/dsl/ir/state-store.js | 22 +++++++++++++++++++--- lib/dsl/ir/struct.js | 8 +++++--- lib/dsl/settings.js | 24 ++++++++++++++++++++++++ lib/dsl/state.js | 19 ++++++++++++++++++- test/fixtures/example.js | 2 +- 7 files changed, 77 insertions(+), 21 deletions(-) diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index 960055c..d4dd39e 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -58,11 +58,8 @@ class Builder { if (node.type === 'ArrowFunctionExpression') return new dsl.ir.Code(name, node); - if (node.type === 'Literal') { - const res = new dsl.ir.Constant(node.value); - res.ast = node; - return res; - } + if (node.type === 'Literal') + return new dsl.ir.Constant(node); reporter.error(node, 'Unexpected global value. Only arrow functions and literal are allowed'); @@ -170,9 +167,7 @@ class Builder { } buildLiteral(literal) { - const res = new dsl.ir.Constant(literal.value); - res.ast = literal; - return res; + return new dsl.ir.Constant(literal); } buildAssignment(expr) { @@ -193,9 +188,9 @@ class Builder { `Unknown \`state\` property name ${prop}`); } - const index = this.state.lookup(prop); + const lookup = this.state.lookup(prop); const right = this.buildExpression(expr.right); - this.push(expr, new dsl.ir.StateStore(index, right)); + this.push(expr, new dsl.ir.StateStore(lookup, right)); return right; } diff --git a/lib/dsl/ir/constant.js b/lib/dsl/ir/constant.js index a6dc04b..74663a9 100644 --- a/lib/dsl/ir/constant.js +++ b/lib/dsl/ir/constant.js @@ -8,13 +8,15 @@ class Constant extends ir.Base { this.out = false; this.ast = ast; + + this.value = this.ast.value; } serialize(reporter) { - if (typeof this.ast.value === 'number') - return `i64 ${this.ast.value | 0}`; + if (typeof this.value === 'number') + return `i64 ${this.value | 0}`; - return `c${JSON.stringify(this.ast.value)}`; + return `c${JSON.stringify(this.value)}`; } } module.exports = Constant; diff --git a/lib/dsl/ir/state-store.js b/lib/dsl/ir/state-store.js index d7be97a..f3aa415 100644 --- a/lib/dsl/ir/state-store.js +++ b/lib/dsl/ir/state-store.js @@ -3,16 +3,32 @@ const ir = require('./'); class Constant extends ir.Base { - constructor(index, value) { + constructor(lookup, value) { super('state:store'); this.out = false; - this.index = index; + this.tmp = 1; + + this.lookup = lookup; this.value = value; } serialize(reporter) { - return '; state-store'; + const t0 = `%t${this.tmp[0]}`; + const index = this.lookup.index; + const type = this.lookup.type; + + let value; + if (this.value.type === 'constant') + value = this.value.value; + else + value = this.value.out; + + return [ + `; state-store ${this.lookup.name}`, + `${t0} = getelementptr %state, %state* %s, i32 0, i32 ${index}`, + `store ${type} ${value}, ${type}* ${t0}`, + ]; } } module.exports = Constant; diff --git a/lib/dsl/ir/struct.js b/lib/dsl/ir/struct.js index 2025c66..6713794 100644 --- a/lib/dsl/ir/struct.js +++ b/lib/dsl/ir/struct.js @@ -13,11 +13,11 @@ class Struct { define(name, type, info) { assert(!this.has(name), `Duplicate entry in ${this.name}`); - const index = this.props.length; - const entry = { name, type, info }; + const index = this.map.size; + const entry = { name, type, info, index }; this.props.push(entry); - this.map.set(name, index); + this.map.set(name, entry); } has(name) { @@ -37,6 +37,8 @@ class Struct { out += `%${this.name} = type {\n`; this.props.forEach((prop, index) => { + assert.strictEqual(prop.index, index); + const isLast = index === this.props.length - 1; out += ` ${prop.type}${isLast ? '' : ','}`; diff --git a/lib/dsl/settings.js b/lib/dsl/settings.js index 599cb92..929c1e5 100644 --- a/lib/dsl/settings.js +++ b/lib/dsl/settings.js @@ -26,6 +26,30 @@ class Settings extends dsl.ir.Struct { super.define(key, nativeType, { type, comment }); } + serialize(prefix) { + let out = ''; + + out += super.serialize(); + out += '\n'; + + // TODO(indutny): set current style + out += `define void @${prefix}_settings_init(%settings* %s) {\n`; + let i = 0; + this.forEach((prop, key) => { + const index = prop.index; + + const t0 = `%t${i++}`; + out += ` ${t0} = getelementptr %settings, %settings* %s, i32 0, ` + + `i32 ${index}\n`; + + out += ` store ${prop.type} null, ${prop.type}* ${t0}\n`; + }); + out += ' ret void\n'; + out += '}\n'; + + return out; + } + serializeC(prefix) { let out = ''; return out + super.serializeC(); diff --git a/lib/dsl/state.js b/lib/dsl/state.js index 30ab869..7973979 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -47,7 +47,24 @@ class State extends dsl.ir.Struct { // TODO(indutny): set current style out += `define void @${prefix}_init(%state* %s) {\n`; - out += `}\n`; + let i = 0; + this.forEach((prop, key) => { + const index = prop.index; + + const t0 = `%t${i++}`; + out += + ` ${t0} = getelementptr %state, %state* %s, i32 0, i32 ${index}\n`; + + let value; + if (key === kCurrent) + value = `@${current}`; + else + value = '0'; + + out += ` store ${prop.type} ${value}, ${prop.type}* ${t0}\n`; + }); + out += ' ret void\n'; + out += '}\n'; return out; } diff --git a/test/fixtures/example.js b/test/fixtures/example.js index a5345eb..fd22882 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -16,7 +16,7 @@ const HTTP_RESPONSE = 2; const INVALID_METHOD = 1; const INVALID_URL_CHARACTER = 2; -const init = () => { +const initial = () => { '@default'; switch (_) { From 8367e7602b5ca3fce5cdfdf922d4fc7cf7f14f38 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 02:40:06 -0500 Subject: [PATCH 025/281] save --- lib/dsl/builder.js | 22 ++++++++++--- lib/dsl/constants.js | 7 +++++ lib/dsl/ir/base.js | 2 ++ lib/dsl/ir/code.js | 11 +++---- lib/dsl/ir/intrinsic.js | 68 +++++++++++++++++++++++++++++++--------- lib/dsl/ir/struct.js | 2 ++ lib/dsl/state.js | 18 +++++++++-- test/fixtures/example.js | 5 +-- 8 files changed, 105 insertions(+), 30 deletions(-) diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index d4dd39e..1225a04 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -32,6 +32,11 @@ class Block { } return out; } + + isFinished() { + return this.instructions.length !== 0 && + this.instructions[this.instructions.length - 1].terminal; + } } class Builder { @@ -106,10 +111,19 @@ class Builder { buildSwitch(stmt) { // TODO(indutny): validate discriminant stmt.cases.forEach((c) => { - if (c.consequent.length < 1) + let stmts; + + if (c.consequent.length === 1 && + c.consequent[0].type === 'BlockStatement') { + stmts = c.consequent[0].body; + } else { + stmts = c.consequent; + } + + if (stmts.length < 1) return this.error(c, 'Missing required `break` statement'); - const last = c.consequent[c.consequent.length - 1]; + const last = stmts[stmts.length - 1]; if (last.type !== 'BreakStatement') return this.error(last, 'Missing required `break` statement'); @@ -118,8 +132,8 @@ class Builder { this.blocks.push(block); this.block = block; - for (let i = 0; i < c.consequent.length - 1; i++) - this.buildStatement(c.consequent[i]); + for (let i = 0; i < stmts.length - 1; i++) + this.buildStatement(stmts[i]); this.block = null; }); } diff --git a/lib/dsl/constants.js b/lib/dsl/constants.js index a4ac4d0..53f8a1c 100644 --- a/lib/dsl/constants.js +++ b/lib/dsl/constants.js @@ -1,5 +1,12 @@ 'use strict'; +exports.state = { + ERROR_INDEX: 2, + ERROR_TYPE: 'i32', + REASON_INDEX: 3, + REASON_TYPE: 'i8*' +}; + exports.directives = { DEFAULT: '@default' }; diff --git a/lib/dsl/ir/base.js b/lib/dsl/ir/base.js index 53e3d96..96779cf 100644 --- a/lib/dsl/ir/base.js +++ b/lib/dsl/ir/base.js @@ -7,6 +7,8 @@ class Base { this.out = null; this.tmp = false; + + this.terminal = false; } } module.exports = Base; diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js index cecaea6..1dd983c 100644 --- a/lib/dsl/ir/code.js +++ b/lib/dsl/ir/code.js @@ -24,19 +24,16 @@ class Code extends ir.Base { return reporter.error(this.ast, 'No blocks to serialize'); let out = `define internal i8* @${this.name}` + - `(%state* %s, i8* %p, i64 %len) {\n`; + `(%state* %s, i8* %p, i8* %end) {\n`; +; out += ` %self = bitcast i8* (%state*, i8*, i8*)* @${this.name} to i8*\n`; // TODO(indutny): switch! - this.blocks.forEach((block) => { + this.blocks.forEach((block, index) => { + out += `clause_${index}:\n`; out += block.serialize(reporter); }); - out += '\n'; - out += ' ; return self\n'; - out += ` %self = bitcast i8* (%state*, i8*, i64)* @${this.name} to i8*\n`; - out += ' ret i8* %self\n'; - out += '}\n'; return out; } diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index 15a25fd..dc858f3 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -2,6 +2,9 @@ const ir = require('./'); +const dsl = require('../'); +const constants = dsl.constants; + class Intrinsic extends ir.Base { constructor(name, args) { super('intrinsic'); @@ -10,13 +13,18 @@ class Intrinsic extends ir.Base { if (name === 'redirect') this.tmp = 1; else if (name === 'next') - this.tmp = 4; + this.tmp = 3; + else if (name === 'error') + this.tmp = 2; this.name = name; this.args = args; if (name !== 'match') this.out = false; + + if (name === 'redirect' || name === 'error') + this.terminal = true; } build(builder) { @@ -46,23 +54,24 @@ class Intrinsic extends ir.Base { const t0 = `%t${this.tmp[0]}`; const t1 = `%t${this.tmp[1]}`; const t2 = `%t${this.tmp[2]}`; - const t3 = `%t${this.tmp[3]}`; + + const target = this.getTarget(reporter); return [ '; next', `${t0} = getelementptr i8, i8* %p, i32 1`, - `${t1} = sub i64 %len, 1`, - `${t2} = icmp ne i64 ${t1}, 0`, - `br i1 ${t2}, label %non_empty${this.tmp[0]}, ` + + `${t1} = icmp ne i64 ${t0}, %end`, + `br i1 ${t1}, label %non_empty${this.tmp[0]}, ` + `label %empty${this.tmp[0]}`, - ``, + '', `non_empty${this.tmp[0]}:`, - ``, - `${t3} = tail call i8* @${this.getTarget(reporter)}` + - `(%state* %s, i8* ${t0}, i64 ${t1})`, - `ret i8* ${t3}`, - ``, - `empty${this.tmp[0]}:` + '', + ` ${t2} = tail call i8* @${target}` + + `(%state* %s, i8* ${t0}, i8* %end)`, + ` ret i8* ${t2}`, + '', + `empty${this.tmp[0]}:`, + ` ret i8* ${target}` ]; } @@ -71,13 +80,44 @@ class Intrinsic extends ir.Base { return [ `; redirect`, `${t0} = tail call i8* @${this.getTarget(reporter)}` + - `(%state* %s, i8* %p, i64 %len)`, + `(%state* %s, i8* %p, i8* %end)`, `ret i8* ${t0}` ]; } serializeError(reporter) { - return '; error'; + if (this.args.length !== 2) + return reporter.error('`error` must have two arguments'); + + if (this.args[0].type !== 'constant' || this.args[1].type !== 'constant') + return reporter.error('`error` arguments must be constants'); + + const code = this.args[0].value; + const reason = this.args[1].value; + + if (typeof code !== 'number') + return reporter.error('`error`\'s first argument must be a number'); + + if (typeof reason !== 'string') + return reporter.error('`error`\'s second argument must be a string'); + + const t0 = `%t${this.tmp[0]}`; + const t1 = `%t${this.tmp[1]}`; + + const errIndex = constants.state.ERROR_INDEX; + const errType = constants.state.ERROR_TYPE; + const reasonIndex = constants.state.REASON_INDEX; + const reasonType = constants.state.REASON_TYPE; + + // TODO(indutny): set reason + return [ + '; error', + `${t0} = getelementptr %state, %state* %s, i32 0, i32 ${errIndex}`, + `store ${errType} code, ${errType}* ${t0}`, + `${t1} = getelementptr %state, %state* %s, i32 0, i32 ${reasonIndex}`, + `store ${reasonType} c${JSON.stringify(reason)}, ${reasonType}* ${t1}`, + 'ret i8* null' + ]; } getTarget(reporter) { diff --git a/lib/dsl/ir/struct.js b/lib/dsl/ir/struct.js index 6713794..2752113 100644 --- a/lib/dsl/ir/struct.js +++ b/lib/dsl/ir/struct.js @@ -18,6 +18,8 @@ class Struct { this.props.push(entry); this.map.set(name, entry); + + return index; } has(name) { diff --git a/lib/dsl/state.js b/lib/dsl/state.js index 7973979..2b590c4 100644 --- a/lib/dsl/state.js +++ b/lib/dsl/state.js @@ -1,30 +1,40 @@ 'use strict'; +const assert = require('assert'); + const kCurrent = Symbol('current'); const kError = Symbol('error'); +const kReason = Symbol('reason'); const kIndex = Symbol('index'); const dsl = require('./'); +const constants = dsl.constants; class State extends dsl.ir.Struct { constructor() { super('state'); - this.define(kCurrent, 'i8* (%state*, i8*, i64)*', []); + this.define(kCurrent, 'i8* (%state*, i8*, i8*)*', []); this.define(kIndex, 'i64', []); - this.define(kError, 'i32', []); + assert.strictEqual( + this.define(kError, constants.state.ERROR_TYPE, []), + constants.state.ERROR_INDEX); + assert.strictEqual( + this.define(kReason, constants.state.REASON_TYPE, []), + constants.state.REASON_INDEX); } define(key, type, args) { const commentKey = key === kError ? '[error]' : key === kCurrent ? '[current]' : key === kIndex ? '[index]' : + key === kReason ? '[reason]' : key; let comment = `${commentKey}`; if (args.length !== 0) out += ` ${JSON.stringify(args)}`; - super.define(key, type, { comment }); + return super.define(key, type, { comment }); } current() { @@ -58,6 +68,8 @@ class State extends dsl.ir.Struct { let value; if (key === kCurrent) value = `@${current}`; + else if (key === kReason) + value = 'null'; else value = '0'; diff --git a/test/fixtures/example.js b/test/fixtures/example.js index fd22882..1edd249 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -36,13 +36,14 @@ const start_req_or_res = () => { 'HEAD': 2, 'POST': 3, 'PUT': 4 - }: + }: { '@notify-on-start(on_message_begin)'; state.type = HTTP_REQUEST; state.method = match(); next(request_after_method); break; + } case 'HTTP': '@notify-on-start(on_message_begin)'; @@ -80,8 +81,8 @@ const request_after_method = () => { const url = () => { switch (_) { case ' ': - next(req_http_start); // settings.on_url.end(); + next(req_http_start); break; case [ '\r', '\n' ]: From 20391f0b89a16321506f18d2f9daf90547e6dc3a Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 02:43:05 -0500 Subject: [PATCH 026/281] save --- lib/dsl/ir/code.js | 3 +++ lib/dsl/ir/intrinsic.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js index 1dd983c..5cca4cb 100644 --- a/lib/dsl/ir/code.js +++ b/lib/dsl/ir/code.js @@ -32,6 +32,9 @@ class Code extends ir.Base { this.blocks.forEach((block, index) => { out += `clause_${index}:\n`; out += block.serialize(reporter); + + if (!block.isFinished()) + out += ` ret i8* $self\n`; }); out += '}\n'; diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index dc858f3..b6175bf 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -23,7 +23,7 @@ class Intrinsic extends ir.Base { if (name !== 'match') this.out = false; - if (name === 'redirect' || name === 'error') + if (name === 'redirect' || name === 'next' || name === 'error') this.terminal = true; } From 6af284cd080def1b9dbfa23bbfa333b2de4b1e1a Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 02:58:13 -0500 Subject: [PATCH 027/281] save --- lib/dsl/builder.js | 10 ++++++++-- lib/dsl/ir/code.js | 5 +++-- lib/dsl/ir/intrinsic.js | 15 ++++++++------- test/fixtures/example.js | 4 ++-- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js index 1225a04..055d1ad 100644 --- a/lib/dsl/builder.js +++ b/lib/dsl/builder.js @@ -27,8 +27,14 @@ class Block { continue; } - for (let j = 0; j < instr.length; j++) - out += ' ' + instr[j] + '\n'; + for (let j = 0; j < instr.length; j++) { + // TODO(indutny): technical debt? + // labels + if (/^[a-z0-9_]+:/.test(instr[j])) + out += instr[j] + '\n'; + else + out += ' ' + instr[j] + '\n'; + } } return out; } diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js index 5cca4cb..2007860 100644 --- a/lib/dsl/ir/code.js +++ b/lib/dsl/ir/code.js @@ -25,16 +25,17 @@ class Code extends ir.Base { let out = `define internal i8* @${this.name}` + `(%state* %s, i8* %p, i8* %end) {\n`; -; out += ` %self = bitcast i8* (%state*, i8*, i8*)* @${this.name} to i8*\n`; + out += ` %self = bitcast i8* (%state*, i8*, i8*)* @${this.name} to i8*\n`; // TODO(indutny): switch! + out += ' br label %clause_0\n'; this.blocks.forEach((block, index) => { out += `clause_${index}:\n`; out += block.serialize(reporter); if (!block.isFinished()) - out += ` ret i8* $self\n`; + out += ` ret i8* %self\n`; }); out += '}\n'; diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index b6175bf..568ea75 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -13,7 +13,7 @@ class Intrinsic extends ir.Base { if (name === 'redirect') this.tmp = 1; else if (name === 'next') - this.tmp = 3; + this.tmp = 4; else if (name === 'error') this.tmp = 2; @@ -54,24 +54,25 @@ class Intrinsic extends ir.Base { const t0 = `%t${this.tmp[0]}`; const t1 = `%t${this.tmp[1]}`; const t2 = `%t${this.tmp[2]}`; + const t3 = `%t${this.tmp[3]}`; const target = this.getTarget(reporter); return [ '; next', `${t0} = getelementptr i8, i8* %p, i32 1`, - `${t1} = icmp ne i64 ${t0}, %end`, + `${t1} = icmp ne i8* ${t0}, %end`, `br i1 ${t1}, label %non_empty${this.tmp[0]}, ` + `label %empty${this.tmp[0]}`, '', `non_empty${this.tmp[0]}:`, - '', - ` ${t2} = tail call i8* @${target}` + + `${t2} = tail call i8* @${target}` + `(%state* %s, i8* ${t0}, i8* %end)`, - ` ret i8* ${t2}`, + `ret i8* ${t2}`, '', `empty${this.tmp[0]}:`, - ` ret i8* ${target}` + `${t3} = bitcast i8* (%state*, i8*, i8*)* @${target} to i8*`, + `ret i8* ${t3}` ]; } @@ -113,7 +114,7 @@ class Intrinsic extends ir.Base { return [ '; error', `${t0} = getelementptr %state, %state* %s, i32 0, i32 ${errIndex}`, - `store ${errType} code, ${errType}* ${t0}`, + `store ${errType} ${code}, ${errType}* ${t0}`, `${t1} = getelementptr %state, %state* %s, i32 0, i32 ${reasonIndex}`, `store ${reasonType} c${JSON.stringify(reason)}, ${reasonType}* ${t1}`, 'ret i8* null' diff --git a/test/fixtures/example.js b/test/fixtures/example.js index 1edd249..ebc52ae 100644 --- a/test/fixtures/example.js +++ b/test/fixtures/example.js @@ -37,7 +37,7 @@ const start_req_or_res = () => { 'POST': 3, 'PUT': 4 }: { - '@notify-on-start(on_message_begin)'; + '@notify:start(on_message_begin)'; state.type = HTTP_REQUEST; state.method = match(); @@ -46,7 +46,7 @@ const start_req_or_res = () => { } case 'HTTP': - '@notify-on-start(on_message_begin)'; + '@notify:start(on_message_begin)'; state.type = HTTP_RESPONSE; next(response_slash); From a1f268c901f6c3d4e97c4f61ceb593b54ecb1286 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Feb 2018 03:04:49 -0500 Subject: [PATCH 028/281] save --- lib/dsl/ir/intrinsic.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js index 568ea75..8e01bc6 100644 --- a/lib/dsl/ir/intrinsic.js +++ b/lib/dsl/ir/intrinsic.js @@ -103,20 +103,15 @@ class Intrinsic extends ir.Base { return reporter.error('`error`\'s second argument must be a string'); const t0 = `%t${this.tmp[0]}`; - const t1 = `%t${this.tmp[1]}`; const errIndex = constants.state.ERROR_INDEX; const errType = constants.state.ERROR_TYPE; - const reasonIndex = constants.state.REASON_INDEX; - const reasonType = constants.state.REASON_TYPE; // TODO(indutny): set reason return [ '; error', `${t0} = getelementptr %state, %state* %s, i32 0, i32 ${errIndex}`, `store ${errType} ${code}, ${errType}* ${t0}`, - `${t1} = getelementptr %state, %state* %s, i32 0, i32 ${reasonIndex}`, - `store ${reasonType} c${JSON.stringify(reason)}, ${reasonType}* ${t1}`, 'ret i8* null' ]; } From b0f09aa386c303ad3372882aabeb5c6acc189365 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Feb 2018 18:25:37 -0500 Subject: [PATCH 029/281] restart --- lib/dsl.js | 18 --- lib/dsl/builder.js | 239 ------------------------------------- lib/dsl/compiler.js | 208 -------------------------------- lib/dsl/constants.js | 12 -- lib/dsl/index.js | 14 --- lib/dsl/ir/base.js | 14 --- lib/dsl/ir/code.js | 45 ------- lib/dsl/ir/constant.js | 22 ---- lib/dsl/ir/index.js | 9 -- lib/dsl/ir/intrinsic.js | 130 -------------------- lib/dsl/ir/state-store.js | 34 ------ lib/dsl/ir/struct.js | 64 ---------- lib/dsl/reporter.js | 14 --- lib/dsl/scope.js | 32 ----- lib/dsl/settings.js | 58 --------- lib/dsl/state.js | 89 -------------- lib/dsl/types.js | 20 ---- lib/llparse.js | 35 ++++++ lib/llparse/index.js | 3 + lib/llparse/node/base.js | 17 +++ lib/llparse/node/error.js | 13 ++ lib/llparse/node/index.js | 5 + lib/llparse/node/invoke.js | 13 ++ package-lock.json | 7 +- package.json | 19 ++- test/api-test.js | 23 ++++ test/dsl-test.js | 14 --- test/fixtures/example.js | 117 ------------------ test/fixtures/index.js | 12 -- 29 files changed, 129 insertions(+), 1171 deletions(-) delete mode 100644 lib/dsl.js delete mode 100644 lib/dsl/builder.js delete mode 100644 lib/dsl/compiler.js delete mode 100644 lib/dsl/constants.js delete mode 100644 lib/dsl/index.js delete mode 100644 lib/dsl/ir/base.js delete mode 100644 lib/dsl/ir/code.js delete mode 100644 lib/dsl/ir/constant.js delete mode 100644 lib/dsl/ir/index.js delete mode 100644 lib/dsl/ir/intrinsic.js delete mode 100644 lib/dsl/ir/state-store.js delete mode 100644 lib/dsl/ir/struct.js delete mode 100644 lib/dsl/reporter.js delete mode 100644 lib/dsl/scope.js delete mode 100644 lib/dsl/settings.js delete mode 100644 lib/dsl/state.js delete mode 100644 lib/dsl/types.js create mode 100644 lib/llparse.js create mode 100644 lib/llparse/index.js create mode 100644 lib/llparse/node/base.js create mode 100644 lib/llparse/node/error.js create mode 100644 lib/llparse/node/index.js create mode 100644 lib/llparse/node/invoke.js create mode 100644 test/api-test.js delete mode 100644 test/dsl-test.js delete mode 100644 test/fixtures/example.js delete mode 100644 test/fixtures/index.js diff --git a/lib/dsl.js b/lib/dsl.js deleted file mode 100644 index 74ee9fa..0000000 --- a/lib/dsl.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -const Compiler = require('./dsl/').Compiler; - -// API, really - -class DSL { - constructor(prefix, source) { - this.prefix = prefix; - this.source = source; - } - - compile() { - const compiler = new Compiler(this.prefix, this.source); - return compiler.compile(); - } -} -module.exports = DSL; diff --git a/lib/dsl/builder.js b/lib/dsl/builder.js deleted file mode 100644 index 055d1ad..0000000 --- a/lib/dsl/builder.js +++ /dev/null @@ -1,239 +0,0 @@ -'use strict'; - -const dsl = require('./'); - -const INTRINSICS = new Set([ 'match', 'next', 'redirect', 'error' ]); - -class Block { - constructor(test, scope) { - this.test = test; - this.instructions = []; - - this.scope = new dsl.Scope(scope); - } - - push(instr) { - this.instructions.push(instr); - return instr; - } - - serialize(reporter) { - let out = ''; - for (let i = 0; i < this.instructions.length; i++) { - const instr = this.instructions[i].serialize(reporter); - - if (!Array.isArray(instr)) { - out += ' ' + instr + '\n'; - continue; - } - - for (let j = 0; j < instr.length; j++) { - // TODO(indutny): technical debt? - // labels - if (/^[a-z0-9_]+:/.test(instr[j])) - out += instr[j] + '\n'; - else - out += ' ' + instr[j] + '\n'; - } - } - return out; - } - - isFinished() { - return this.instructions.length !== 0 && - this.instructions[this.instructions.length - 1].terminal; - } -} - -class Builder { - constructor(reporter, state, settings, globals) { - this.reporter = reporter; - this.state = state; - this.settings = settings; - this.globals = globals; - - this.id = 1; - this.blocks = []; - this.block = null; - - this.directives = { - isDefault: false - }; - } - - error(node, message) { - this.reporter.error(node, message); - } - - static build(reporter, name, node) { - if (node.type === 'ArrowFunctionExpression') - return new dsl.ir.Code(name, node); - - if (node.type === 'Literal') - return new dsl.ir.Constant(node); - - reporter.error(node, - 'Unexpected global value. Only arrow functions and literal are allowed'); - } - - build(node) { - if (node.type !== 'ArrowFunctionExpression') - return this.error(node, 'Can\'t build this'); - - this.blocks = []; - - if (node.body.type !== 'BlockStatement') { - this.error(node.body, 'Global arrow functions must have block body'); - return; - } - - // TODO(indutny): validate: no async, no iterator, arguments, etc - let s = false; - node.body.body.forEach((stmt) => { - if (stmt.type === 'ExpressionStatement' && stmt.directive) { - return this.processDirective(stmt); - } else if (stmt.type === 'SwitchStatement') { - if (s) - return this.error(stmt, 'Duplicate switch statement'); - - s = stmt; - } - }); - if (!s) - return this.error(node, 'No switch statement in an arrow function'); - - this.buildSwitch(s); - - return this.blocks; - } - - processDirective(stmt) { - if (stmt.directive === dsl.constants.directives.DEFAULT) - this.directives.isDefault = true; - - // TODO(indutny): warn on unknown directives - } - - buildSwitch(stmt) { - // TODO(indutny): validate discriminant - stmt.cases.forEach((c) => { - let stmts; - - if (c.consequent.length === 1 && - c.consequent[0].type === 'BlockStatement') { - stmts = c.consequent[0].body; - } else { - stmts = c.consequent; - } - - if (stmts.length < 1) - return this.error(c, 'Missing required `break` statement'); - - const last = stmts[stmts.length - 1]; - if (last.type !== 'BreakStatement') - return this.error(last, 'Missing required `break` statement'); - - // TODO(indutny): parse `test` - const block = new Block(c.test, this.globals); - this.blocks.push(block); - - this.block = block; - for (let i = 0; i < stmts.length - 1; i++) - this.buildStatement(stmts[i]); - this.block = null; - }); - } - - buildStatement(stmt) { - if (stmt.type === 'ExpressionStatement' && - stmt.expression.type === 'Literal') { - return this.processBlockDirective(stmt); - } else if (stmt.type === 'ExpressionStatement') { - return this.buildExpression(stmt.expression); - } - - return this.error(stmt, `Unknown statement type: ${stmt.type}`); - } - - buildExpression(expr) { - if (expr.type === 'CallExpression') - return this.buildCall(expr); - else if (expr.type === 'Identifier') - return this.buildIdentifier(expr); - else if (expr.type === 'Literal') - return this.buildLiteral(expr); - else if (expr.type === 'AssignmentExpression') - return this.buildAssignment(expr); - - return this.error(expr, `Unknown expression type: ${expr.type}`); - } - - buildCall(call) { - const callee = call.callee; - const args = call.arguments.map(arg => this.buildExpression(arg)); - - if (callee.type === 'Identifier' && INTRINSICS.has(callee.name)) - return this.buildIntrinsic(call, callee.name, args); - - return this.error(call, `Unsupported callee: ${callee.type}`); - } - - buildIntrinsic(node, name, args) { - return this.push(node, new dsl.ir.Intrinsic(name, args)); - } - - buildIdentifier(id) { - return this.block.scope.lookup(id, this); - } - - buildLiteral(literal) { - return new dsl.ir.Constant(literal); - } - - buildAssignment(expr) { - const left = expr.left; - - if (left.type !== 'MemberExpression' || - left.object.type !== 'Identifier' || - left.object.name !== 'state') { - return this.error(left, 'Expected a property of `state` object'); - } - - if (left.computed) - return this.error(left, 'Property of `state` object can\'t be computed'); - - const prop = left.property.name; - if (!this.state.has(prop)) { - return this.error(left.property, - `Unknown \`state\` property name ${prop}`); - } - - const lookup = this.state.lookup(prop); - const right = this.buildExpression(expr.right); - this.push(expr, new dsl.ir.StateStore(lookup, right)); - return right; - } - - processBlockDirective(stmt) { - const directive = stmt.expression.value; - - // TODO(indutny): warn on unknown directives - } - - push(ast, instr) { - if (instr.out === null) - instr.out = this.id++; - - if (typeof instr.tmp === 'number') { - const tmp = []; - for (let i = 0; i < instr.tmp; i++) - tmp.push(this.id++); - instr.tmp = tmp; - } - - instr.ast = ast; - - return this.block.push(instr); - } -} -module.exports = Builder; diff --git a/lib/dsl/compiler.js b/lib/dsl/compiler.js deleted file mode 100644 index 4103a72..0000000 --- a/lib/dsl/compiler.js +++ /dev/null @@ -1,208 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const esprima = require('esprima'); - -const dsl = require('./'); - -const directives = dsl.constants.directives; -const Scope = dsl.Scope; -const State = dsl.State; -const Settings = dsl.Settings; -const Reporter = dsl.Reporter; -const Builder = dsl.Builder; - -const STATE_TYPES = dsl.types.STATE_TYPES; -const SETTINGS_TYPES = dsl.types.SETTINGS_TYPES; - -class Compiler { - constructor(prefix, source) { - this.prefix = prefix; - this.source = source; - - this.ast = null; - this.globals = null; - this.state = null; - this.settings = null; - this.reporter = new Reporter(); - - this.code = new Map(); - - this.defaultState = null; - } - - compile() { - this.ast = esprima.parse(this.source, { loc: true }); - assert.equal(this.ast.type, 'Program'); - - this.globals = new Scope(); - this.state = new State(); - this.settings = new Settings(); - - this.parseTopLevel(this.ast.body); - - this.buildCode(); - - return this.serialize(); - } - - error(node, string) { - return this.reporter.error(node, string); - } - - parseTopLevel(body) { - const globals = new Map(); - - const declaration = (decl) => { - if (!decl.init) - return this.error(decl, `Missing init value for ${decl.id.name}`); - - // Re-use of previously defined value - let init = decl.init; - while (init.type === 'Identifier') - init = this.globals.lookup(init); - - if (decl.id.name === 'state') - return this.parseState(init); - else if (decl.id.name === 'settings') - return this.parseSettings(init); - - this.globals.define(decl.id, - Builder.build(this.reporter, decl.id.name, init), - this); - }; - - body.forEach((node) => { - if (node.type === 'VariableDeclaration') - return node.declarations.forEach(declaration); - - if (node.type === 'ExpressionStatement' && node.directive) - return; - - this.error(node, 'Unsupported top-level node'); - }); - } - - parseConfigObject(node, types) { - if (node.type !== 'ObjectExpression') { - this.error(node, - 'Invalid configuration value type, must be ObjectExpression'); - return; - } - - const result = []; - node.properties.forEach((prop) => { - if (prop.computed) - return this.error(prop, 'Configuration properties can\'t be computed'); - - if (prop.key.type !== 'Identifier') { - this.error(prop.key, - 'Configuration property keys must be Identifiers'); - return; - } - - if (prop.value.type !== 'CallExpression') { - this.error(prop.key, - 'Configuration property values must be CallExpressions'); - return; - } - - const key = prop.key.name; - - if (prop.value.callee.type !== 'Identifier') - return this.error(value.callee, 'Unknown configuration property type'); - - const type = prop.value.callee.name; - const args = prop.value.arguments.map((arg) => { - if (arg.type !== 'Literal') - return this.error(args, 'Invalid argument type, must be Literal'); - - return arg.value; - }); - - if (!types.hasOwnProperty(type)) - return this.error(value.callee, 'Unknown `state` property type'); - - const validation = types[type](args); - if (validation !== true) { - this.error(value.callee, - `Invalid arguments for "${type}": ${validation}`); - return; - } - - result.push({ key, type, args }); - }); - return result; - } - - parseState(node) { - const props = this.parseConfigObject(node, STATE_TYPES); - - props.forEach((prop) => { - if (this.state.has(prop.key)) - return this.error(prop.key, 'Duplicate key in `state` object'); - - this.state.define(prop.key, prop.type, prop.args); - }); - - // TODO(indutny): error on duplicate `state` definition - } - - parseSettings(node) { - const props = this.parseConfigObject(node, SETTINGS_TYPES); - - props.forEach((prop) => { - if (this.settings.has(prop.key)) - return this.error(prop.key, 'Duplicate key in `settings` object'); - - this.settings.define(prop.key, prop.type, prop.args); - }); - - // TODO(indutny): error on duplicate `settings` definition - } - - buildCode() { - this.defaultState = null; - - this.globals.forEach((value, key) => { - if (!(value instanceof dsl.ir.Code)) - return; - - const builder = new Builder(this.reporter, this.state, this.settings, - this.globals); - value.build(builder); - - this.code.set(key, value); - - if (!builder.directives.isDefault) - return; - - if (this.defaultState !== null) { - return this.error(value.ast, - `Two states with \`${directives.DEFAULT}\` directive`); - } - - this.defaultState = key; - }); - - if (this.defaultState !== null) - return; - this.error(this.ast, `No state with \`${directives.DEFAULT}\` directive`); - } - - serialize() { - let out = ''; - - out += this.state.serialize(this.prefix, this.defaultState); - out += '\n'; - out += this.settings.serialize(this.prefix); - out += '\n'; - - this.code.forEach((code) => { - out += code.serialize(this.reporter) + '\n'; - }); - - return out; - } -} -module.exports = Compiler; diff --git a/lib/dsl/constants.js b/lib/dsl/constants.js deleted file mode 100644 index 53f8a1c..0000000 --- a/lib/dsl/constants.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -exports.state = { - ERROR_INDEX: 2, - ERROR_TYPE: 'i32', - REASON_INDEX: 3, - REASON_TYPE: 'i8*' -}; - -exports.directives = { - DEFAULT: '@default' -}; diff --git a/lib/dsl/index.js b/lib/dsl/index.js deleted file mode 100644 index 930bdf9..0000000 --- a/lib/dsl/index.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -exports.constants = require('./constants'); -exports.types = require('./types'); - -exports.ir = require('./ir'); -exports.Reporter = require('./reporter'); -exports.Scope = require('./scope'); - -exports.State = require('./state'); -exports.Settings = require('./settings'); - -exports.Builder = require('./builder'); -exports.Compiler = require('./compiler'); diff --git a/lib/dsl/ir/base.js b/lib/dsl/ir/base.js deleted file mode 100644 index 96779cf..0000000 --- a/lib/dsl/ir/base.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -class Base { - constructor(type) { - this.type = type; - this.ast = null; - - this.out = null; - this.tmp = false; - - this.terminal = false; - } -} -module.exports = Base; diff --git a/lib/dsl/ir/code.js b/lib/dsl/ir/code.js deleted file mode 100644 index 2007860..0000000 --- a/lib/dsl/ir/code.js +++ /dev/null @@ -1,45 +0,0 @@ -'use strict'; - -const ir = require('./'); - -class Code extends ir.Base { - constructor(name, ast) { - super('code'); - - this.out = false; - - this.name = name; - this.ast = ast; - this.blocks = null; - } - - build(builder) { - this.blocks = builder.build(this.ast); - } - - serialize(reporter) { - if (!this.blocks) - return reporter.error(this.ast, 'Not built!'); - if (this.blocks.length === 0) - return reporter.error(this.ast, 'No blocks to serialize'); - - let out = `define internal i8* @${this.name}` + - `(%state* %s, i8* %p, i8* %end) {\n`; - out += ` %self = bitcast i8* (%state*, i8*, i8*)* @${this.name} to i8*\n`; - - // TODO(indutny): switch! - out += ' br label %clause_0\n'; - - this.blocks.forEach((block, index) => { - out += `clause_${index}:\n`; - out += block.serialize(reporter); - - if (!block.isFinished()) - out += ` ret i8* %self\n`; - }); - - out += '}\n'; - return out; - } -} -module.exports = Code; diff --git a/lib/dsl/ir/constant.js b/lib/dsl/ir/constant.js deleted file mode 100644 index 74663a9..0000000 --- a/lib/dsl/ir/constant.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -const ir = require('./'); - -class Constant extends ir.Base { - constructor(ast) { - super('constant'); - - this.out = false; - this.ast = ast; - - this.value = this.ast.value; - } - - serialize(reporter) { - if (typeof this.value === 'number') - return `i64 ${this.value | 0}`; - - return `c${JSON.stringify(this.value)}`; - } -} -module.exports = Constant; diff --git a/lib/dsl/ir/index.js b/lib/dsl/ir/index.js deleted file mode 100644 index 452c307..0000000 --- a/lib/dsl/ir/index.js +++ /dev/null @@ -1,9 +0,0 @@ -'use strict'; - -exports.Base = require('./base'); -exports.Code = require('./code'); -exports.Constant = require('./constant'); -exports.Intrinsic = require('./intrinsic'); -exports.StateStore = require('./state-store'); - -exports.Struct = require('./struct'); diff --git a/lib/dsl/ir/intrinsic.js b/lib/dsl/ir/intrinsic.js deleted file mode 100644 index 8e01bc6..0000000 --- a/lib/dsl/ir/intrinsic.js +++ /dev/null @@ -1,130 +0,0 @@ -'use strict'; - -const ir = require('./'); - -const dsl = require('../'); -const constants = dsl.constants; - -class Intrinsic extends ir.Base { - constructor(name, args) { - super('intrinsic'); - - // Request tmp for redirect - if (name === 'redirect') - this.tmp = 1; - else if (name === 'next') - this.tmp = 4; - else if (name === 'error') - this.tmp = 2; - - this.name = name; - this.args = args; - - if (name !== 'match') - this.out = false; - - if (name === 'redirect' || name === 'next' || name === 'error') - this.terminal = true; - } - - build(builder) { - return builder.build(this.ast); - } - - serialize(reporter) { - const name = this.name; - - if (name === 'match') - return this.serializeMatch(reporter); - else if (name === 'next') - return this.serializeNext(reporter); - else if (name === 'redirect') - return this.serializeRedirect(reporter); - else if (name === 'error') - return this.serializeError(reporter); - else - return reporter.error(this.ast, `Unknown intrinsic name=${name}`); - } - - serializeMatch(reporter) { - return '; match'; - } - - serializeNext(reporter) { - const t0 = `%t${this.tmp[0]}`; - const t1 = `%t${this.tmp[1]}`; - const t2 = `%t${this.tmp[2]}`; - const t3 = `%t${this.tmp[3]}`; - - const target = this.getTarget(reporter); - - return [ - '; next', - `${t0} = getelementptr i8, i8* %p, i32 1`, - `${t1} = icmp ne i8* ${t0}, %end`, - `br i1 ${t1}, label %non_empty${this.tmp[0]}, ` + - `label %empty${this.tmp[0]}`, - '', - `non_empty${this.tmp[0]}:`, - `${t2} = tail call i8* @${target}` + - `(%state* %s, i8* ${t0}, i8* %end)`, - `ret i8* ${t2}`, - '', - `empty${this.tmp[0]}:`, - `${t3} = bitcast i8* (%state*, i8*, i8*)* @${target} to i8*`, - `ret i8* ${t3}` - ]; - } - - serializeRedirect(reporter) { - const t0 = `%t${this.tmp[0]}`; - return [ - `; redirect`, - `${t0} = tail call i8* @${this.getTarget(reporter)}` + - `(%state* %s, i8* %p, i8* %end)`, - `ret i8* ${t0}` - ]; - } - - serializeError(reporter) { - if (this.args.length !== 2) - return reporter.error('`error` must have two arguments'); - - if (this.args[0].type !== 'constant' || this.args[1].type !== 'constant') - return reporter.error('`error` arguments must be constants'); - - const code = this.args[0].value; - const reason = this.args[1].value; - - if (typeof code !== 'number') - return reporter.error('`error`\'s first argument must be a number'); - - if (typeof reason !== 'string') - return reporter.error('`error`\'s second argument must be a string'); - - const t0 = `%t${this.tmp[0]}`; - - const errIndex = constants.state.ERROR_INDEX; - const errType = constants.state.ERROR_TYPE; - - // TODO(indutny): set reason - return [ - '; error', - `${t0} = getelementptr %state, %state* %s, i32 0, i32 ${errIndex}`, - `store ${errType} ${code}, ${errType}* ${t0}`, - 'ret i8* null' - ]; - } - - getTarget(reporter) { - if (this.args.length !== 1) - return reporter.error('`redirect` must have one argument'); - - const target = this.args[0]; - if (target.type !== 'code') - return reporter.error('`redirect` takes state name as an argument'); - - return target.name; - } -} -module.exports = Intrinsic; diff --git a/lib/dsl/ir/state-store.js b/lib/dsl/ir/state-store.js deleted file mode 100644 index f3aa415..0000000 --- a/lib/dsl/ir/state-store.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict'; - -const ir = require('./'); - -class Constant extends ir.Base { - constructor(lookup, value) { - super('state:store'); - - this.out = false; - this.tmp = 1; - - this.lookup = lookup; - this.value = value; - } - - serialize(reporter) { - const t0 = `%t${this.tmp[0]}`; - const index = this.lookup.index; - const type = this.lookup.type; - - let value; - if (this.value.type === 'constant') - value = this.value.value; - else - value = this.value.out; - - return [ - `; state-store ${this.lookup.name}`, - `${t0} = getelementptr %state, %state* %s, i32 0, i32 ${index}`, - `store ${type} ${value}, ${type}* ${t0}`, - ]; - } -} -module.exports = Constant; diff --git a/lib/dsl/ir/struct.js b/lib/dsl/ir/struct.js deleted file mode 100644 index 2752113..0000000 --- a/lib/dsl/ir/struct.js +++ /dev/null @@ -1,64 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -class Struct { - constructor(name) { - this.name = name; - - this.map = new Map(); - this.props = []; - } - - define(name, type, info) { - assert(!this.has(name), `Duplicate entry in ${this.name}`); - - const index = this.map.size; - const entry = { name, type, info, index }; - - this.props.push(entry); - this.map.set(name, entry); - - return index; - } - - has(name) { - return this.map.has(name); - } - - lookup(name) { - return this.map.get(name); - } - - forEach(callback) { - return this.map.forEach(callback); - } - - serialize() { - let out = ''; - - out += `%${this.name} = type {\n`; - this.props.forEach((prop, index) => { - assert.strictEqual(prop.index, index); - - const isLast = index === this.props.length - 1; - - out += ` ${prop.type}${isLast ? '' : ','}`; - out += ` ; ${index} => ${prop.info.comment}\n`; - }); - out += '}\n'; - - return out; - } - - serializeC(prefix) { - let out = ''; - const name = this.name; - - out += `typedef struct ${prefix}_${name}_s ${prefix}_${name}_t;\n`; - out += `void ${prefix}_${name}_init(${prefix}_${name}_t* s);\n`; - - return out; - } -} -module.exports = Struct; diff --git a/lib/dsl/reporter.js b/lib/dsl/reporter.js deleted file mode 100644 index 8c1978c..0000000 --- a/lib/dsl/reporter.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -class Reporter { - error(node, message) { - if (!node || !node.loc || !node.loc.start) - throw new Error(message + ' at unknown position'); - - const start = node.loc.start; - const err = new Error(message + ` at ${start.line}:${start.column}`); - err.node = node; - throw err; - } -} -module.exports = Reporter; diff --git a/lib/dsl/scope.js b/lib/dsl/scope.js deleted file mode 100644 index dec886a..0000000 --- a/lib/dsl/scope.js +++ /dev/null @@ -1,32 +0,0 @@ -'use strict'; - -class Scope { - constructor(parent) { - this.parent = parent || null; - - this.local = new Map(); - } - - forEach(cb) { - this.local.forEach(cb); - } - - lookup(id, source) { - if (this.local.has(id.name)) - return this.local.get(id.name); - - if (this.parent) - return this.parent.lookup(id, source); - - source.error(id, `Unknown variable name ${id.name}`); - } - - define(id, value, source) { - if (this.local.has(id.name)) - return source.error(id, 'Re-definition of const variable'); - - this.local.set(id.name, value); - } - -} -module.exports = Scope; diff --git a/lib/dsl/settings.js b/lib/dsl/settings.js deleted file mode 100644 index 929c1e5..0000000 --- a/lib/dsl/settings.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const dsl = require('./'); - -class Settings extends dsl.ir.Struct { - constructor() { - super('settings'); - } - - define(key, type, args) { - let nativeType; - let cType; - if (type === 'notify') { - nativeType = 'i32 (%state*)*'; - } else { - assert.equal(type, 'data'); - nativeType = 'i32 (%state*, i8*, i64)*'; - } - - let comment = `${key}: ${type}`; - if (args.length !== 0) - comment += ` ${JSON.stringify(args)}`; - - super.define(key, nativeType, { type, comment }); - } - - serialize(prefix) { - let out = ''; - - out += super.serialize(); - out += '\n'; - - // TODO(indutny): set current style - out += `define void @${prefix}_settings_init(%settings* %s) {\n`; - let i = 0; - this.forEach((prop, key) => { - const index = prop.index; - - const t0 = `%t${i++}`; - out += ` ${t0} = getelementptr %settings, %settings* %s, i32 0, ` + - `i32 ${index}\n`; - - out += ` store ${prop.type} null, ${prop.type}* ${t0}\n`; - }); - out += ' ret void\n'; - out += '}\n'; - - return out; - } - - serializeC(prefix) { - let out = ''; - return out + super.serializeC(); - } -} -module.exports = Settings; diff --git a/lib/dsl/state.js b/lib/dsl/state.js deleted file mode 100644 index 2b590c4..0000000 --- a/lib/dsl/state.js +++ /dev/null @@ -1,89 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const kCurrent = Symbol('current'); -const kError = Symbol('error'); -const kReason = Symbol('reason'); -const kIndex = Symbol('index'); - -const dsl = require('./'); -const constants = dsl.constants; - -class State extends dsl.ir.Struct { - constructor() { - super('state'); - - this.define(kCurrent, 'i8* (%state*, i8*, i8*)*', []); - this.define(kIndex, 'i64', []); - assert.strictEqual( - this.define(kError, constants.state.ERROR_TYPE, []), - constants.state.ERROR_INDEX); - assert.strictEqual( - this.define(kReason, constants.state.REASON_TYPE, []), - constants.state.REASON_INDEX); - } - - define(key, type, args) { - const commentKey = key === kError ? '[error]' : - key === kCurrent ? '[current]' : - key === kIndex ? '[index]' : - key === kReason ? '[reason]' : - key; - let comment = `${commentKey}`; - if (args.length !== 0) - out += ` ${JSON.stringify(args)}`; - - return super.define(key, type, { comment }); - } - - current() { - return this.lookup(kCurrent); - } - - error() { - return this.lookup(kError); - } - - index() { - return this.lookup(kIndex); - } - - serialize(prefix, current) { - let out = ''; - - out += super.serialize(); - out += '\n'; - - // TODO(indutny): set current style - out += `define void @${prefix}_init(%state* %s) {\n`; - let i = 0; - this.forEach((prop, key) => { - const index = prop.index; - - const t0 = `%t${i++}`; - out += - ` ${t0} = getelementptr %state, %state* %s, i32 0, i32 ${index}\n`; - - let value; - if (key === kCurrent) - value = `@${current}`; - else if (key === kReason) - value = 'null'; - else - value = '0'; - - out += ` store ${prop.type} ${value}, ${prop.type}* ${t0}\n`; - }); - out += ' ret void\n'; - out += '}\n'; - - return out; - } - - serializeC(prefix) { - let out = ''; - return out + super.serializeC(); - } -} -module.exports = State; diff --git a/lib/dsl/types.js b/lib/dsl/types.js deleted file mode 100644 index 4fd1678..0000000 --- a/lib/dsl/types.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -const disallowArgs = args => args.length === 0; - -exports.STATE_TYPES = { - i8: disallowArgs, - i16: disallowArgs, - i32: disallowArgs, - i64: disallowArgs -}; - -exports.SETTINGS_TYPES = { - notify: disallowArgs, - data: (args) => { - if (args.length !== 1) - return false; - - return typeof args[0] === 'string'; - } -}; diff --git a/lib/llparse.js b/lib/llparse.js new file mode 100644 index 0000000..bd9d687 --- /dev/null +++ b/lib/llparse.js @@ -0,0 +1,35 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const internal = require('./llparse/'); + +// API, really + +class LLParse { + constructor(prefix) { + this.prefix = prefix; + this.ir = new IR(); + } + + static create(prefix) { + return new LLParse(prefix); + } + + node(name) { + return new internal.node.Node(name); + } + + error(code, reason) { + return new internal.node.Error(code, reason); + } + + invoke(name, next) { + return new internal.node.Invoke(name, next); + } + + build() { + return this.ir.build(); + } +} +module.exports = LLParse; diff --git a/lib/llparse/index.js b/lib/llparse/index.js new file mode 100644 index 0000000..ae2f1a9 --- /dev/null +++ b/lib/llparse/index.js @@ -0,0 +1,3 @@ +'use strict'; + +exports.node = require('./node'); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js new file mode 100644 index 0000000..4b5b8c7 --- /dev/null +++ b/lib/llparse/node/base.js @@ -0,0 +1,17 @@ +'use strict'; + +class Node { + constructor(name) { + this.name = name; + } + + match(value, state) { + } + + select(object, state) { + } + + otherwise() { + } +} +module.exports = Node; diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js new file mode 100644 index 0000000..c94614a --- /dev/null +++ b/lib/llparse/node/error.js @@ -0,0 +1,13 @@ +'use strict'; + +const llparse = require('./'); + +class Error extends llparse.Node { + constructor(code, reason) { + super('error'); + + this.code = code; + this.reason = reason; + } +} +module.exports = Error; diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js new file mode 100644 index 0000000..d82dcdc --- /dev/null +++ b/lib/llparse/node/index.js @@ -0,0 +1,5 @@ +'use strict'; + +exports.Node = require('./base'); +exports.Error = require('./error'); +exports.Invoke = require('./invoke'); diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js new file mode 100644 index 0000000..0830341 --- /dev/null +++ b/lib/llparse/node/invoke.js @@ -0,0 +1,13 @@ +'use strict'; + +const llparse = require('./'); + +class Invoke extends llparse.Node { + constructor(callback, next) { + super('invoke_' + callback); + + this.callback = callback; + this.next = next; + } +} +module.exports = Invoke; diff --git a/package-lock.json b/package-lock.json index 163812c..bd9426d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "parser-dsl", + "name": "llparse", "version": "1.0.0", "lockfileVersion": 1, "requires": true, @@ -118,6 +118,11 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", "dev": true }, + "llvm-ir": { + "version": "1.0.0-beta1", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta1.tgz", + "integrity": "sha512-91MRTS6o5x6GY4+7becoFQvdo1hbHyamZFLFnzMhGgYBftZv48No7DeMh8AOfiX5MJZDUqQHysJVI4kD0RVIqA==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", diff --git a/package.json b/package.json index 5f59bcb..43bd978 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,7 @@ { - "name": "parser-dsl", + "name": "llparse", "version": "1.0.0", - "description": "", - "main": "lib/dsl.js", + "main": "lib/llparse.js", "scripts": { "test": "mocha --reporter=spec test/*-test.js" }, @@ -10,9 +9,19 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "esprima": "^4.0.0" + "esprima": "^4.0.0", + "llvm-ir": "^1.0.0-beta1" }, "devDependencies": { "mocha": "^5.0.0" - } + }, + "directories": { + "lib": "lib", + "test": "test" + }, + "repository": { + "type": "git", + "url": "gh:indutny/llparse" + }, + "description": "" } diff --git a/test/api-test.js b/test/api-test.js new file mode 100644 index 0000000..567fab4 --- /dev/null +++ b/test/api-test.js @@ -0,0 +1,23 @@ +'use strict'; + +const llparse = require('../'); + +describe('LLParse', () => { + it('should compile simple parser', () => { + const parse = llparse.create('llparse'); + + const start = parse.node('start'); + const request = parse.node('req'); + const response = parse.node('res'); + const error = parse.error(1, 'Invalid word'); + + start.match('HTTP', response); + start.select({ 'GET': 0, 'POST': 1, 'PUT': 2 }, request); + start.otherwise(error); + + request.otherwise(parse.invoke('on_request', start)); + + const out = parse.build(start); + console.log(out); + }); +}); diff --git a/test/dsl-test.js b/test/dsl-test.js deleted file mode 100644 index cd585e3..0000000 --- a/test/dsl-test.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -const DSL = require('../'); - -const fixtures = require('./fixtures'); - -describe('LLParse', () => { - it('should compile `example.js`', () => { - const dsl = new DSL('llparse', fixtures.source.example); - - const out = dsl.compile(); - console.log(out); - }); -}); diff --git a/test/fixtures/example.js b/test/fixtures/example.js deleted file mode 100644 index ebc52ae..0000000 --- a/test/fixtures/example.js +++ /dev/null @@ -1,117 +0,0 @@ -'use strict'; - -const state = { - type: i32(), - method: i32() -}; - -const settings = { - on_message_begin: notify(), - on_url: data('url') -}; - -const HTTP_REQUEST = 1; -const HTTP_RESPONSE = 2; - -const INVALID_METHOD = 1; -const INVALID_URL_CHARACTER = 2; - -const initial = () => { - '@default'; - - switch (_) { - default: - redirect(start_req_or_res); - break; - } -}; - -const start_req_or_res = () => { - switch (_) { - case [ 0x0a, 0x0d ]: - break; - - case { - 'GET': 1, - 'HEAD': 2, - 'POST': 3, - 'PUT': 4 - }: { - '@notify:start(on_message_begin)'; - - state.type = HTTP_REQUEST; - state.method = match(); - next(request_after_method); - break; - } - - case 'HTTP': - '@notify:start(on_message_begin)'; - - state.type = HTTP_RESPONSE; - next(response_slash); - break; - - default: - '@unlikely'; - - error(INVALID_METHOD, 'Unknown method'); - break; - } -}; - -const request_after_method = () => { - switch (_) { - case ' ': - break; - - case 0x0: - '@unlikely'; - - error(INVALID_METHOD, '`\\0` after method'); - break; - - default: -// settings.on_url.start(); - redirect(url); - break; - } -}; - -const url = () => { - switch (_) { - case ' ': -// settings.on_url.end(); - next(req_http_start); - break; - - case [ '\r', '\n' ]: - '@unlikely'; - - error(INVALID_URL_CHARACTER, - 'URL can\'t have newline chars in it'); - break; - - case [ '\t', '\f' ]: - '@ifdef(strict)'; - '@unlikely'; - - error(INVALID_URL_CHARACTER, - 'URL can\'t have "\\t" or "\\f" chars in it'); - break; - } -}; - -const req_http_start = () => { - switch (_) { - default: - break; - } -}; - -const response_slash = () => { - switch (_) { - default: - break; - } -}; diff --git a/test/fixtures/index.js b/test/fixtures/index.js deleted file mode 100644 index fafc287..0000000 --- a/test/fixtures/index.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -const fs = require('fs'); -const path = require('path'); - -const file = (name) => { - return fs.readFileSync(path.join(__dirname, name)).toString(); -}; - -exports.source = { - example: file('example.js') -}; From eeaf5f6c4386eb2d9db4df97e46b073ca0c9b27c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Feb 2018 19:16:34 -0500 Subject: [PATCH 030/281] save --- lib/llparse.js | 10 ++++----- lib/llparse/case/base.js | 9 ++++++++ lib/llparse/case/index.js | 6 ++++++ lib/llparse/case/match.js | 12 +++++++++++ lib/llparse/case/otherwise.js | 10 +++++++++ lib/llparse/case/select.js | 12 +++++++++++ lib/llparse/compiler.js | 40 +++++++++++++++++++++++++++++++++++ lib/llparse/index.js | 2 ++ lib/llparse/node/base.js | 17 ++++++++++++--- lib/llparse/node/invoke.js | 4 ++++ 10 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 lib/llparse/case/base.js create mode 100644 lib/llparse/case/index.js create mode 100644 lib/llparse/case/match.js create mode 100644 lib/llparse/case/otherwise.js create mode 100644 lib/llparse/case/select.js create mode 100644 lib/llparse/compiler.js diff --git a/lib/llparse.js b/lib/llparse.js index bd9d687..fba877d 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -1,15 +1,12 @@ 'use strict'; -const IR = require('llvm-ir'); - const internal = require('./llparse/'); // API, really class LLParse { constructor(prefix) { - this.prefix = prefix; - this.ir = new IR(); + this.prefix = prefix || 'llparse'; } static create(prefix) { @@ -28,8 +25,9 @@ class LLParse { return new internal.node.Invoke(name, next); } - build() { - return this.ir.build(); + build(root) { + const c = new internal.Compiler(this.prefix); + return c.build(root); } } module.exports = LLParse; diff --git a/lib/llparse/case/base.js b/lib/llparse/case/base.js new file mode 100644 index 0000000..37476a8 --- /dev/null +++ b/lib/llparse/case/base.js @@ -0,0 +1,9 @@ +'use strict'; + +class Case { + constructor(type, next) { + this.type = type; + this.next = next; + } +} +module.exports = Case; diff --git a/lib/llparse/case/index.js b/lib/llparse/case/index.js new file mode 100644 index 0000000..e29d9ee --- /dev/null +++ b/lib/llparse/case/index.js @@ -0,0 +1,6 @@ +'use strict'; + +exports.Case = require('./base'); +exports.Match = require('./match'); +exports.Select = require('./select'); +exports.Otherwise = require('./otherwise'); diff --git a/lib/llparse/case/match.js b/lib/llparse/case/match.js new file mode 100644 index 0000000..46c0308 --- /dev/null +++ b/lib/llparse/case/match.js @@ -0,0 +1,12 @@ +'use strict'; + +const Case = require('./').Case; + +class Match extends Case { + constructor(value, next) { + super('match', next); + + this.value = value; + } +} +module.exports = Match; diff --git a/lib/llparse/case/otherwise.js b/lib/llparse/case/otherwise.js new file mode 100644 index 0000000..9f266d4 --- /dev/null +++ b/lib/llparse/case/otherwise.js @@ -0,0 +1,10 @@ +'use strict'; + +const Case = require('./').Case; + +class Otherwise extends Case { + constructor(next) { + super('otherwise', next); + } +} +module.exports = Otherwise; diff --git a/lib/llparse/case/select.js b/lib/llparse/case/select.js new file mode 100644 index 0000000..98b9cf0 --- /dev/null +++ b/lib/llparse/case/select.js @@ -0,0 +1,12 @@ +'use strict'; + +const Case = require('./').Case; + +class Select extends Case { + constructor(map, next) { + super('select', next); + + this.map = map; + } +} +module.exports = Select; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js new file mode 100644 index 0000000..52507ee --- /dev/null +++ b/lib/llparse/compiler.js @@ -0,0 +1,40 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const TYPE_INPUT = IR.i(8).ptr(); +const TYPE_OUTPUT = IR.i(8).ptr(); + +class Compiler { + constructor(prefix) { + this.prefix = prefix; + this.ir = new IR(); + + this.state = this.ir.struct(`${this.prefix}_state`); + this.sig = this.ir.signature(TYPE_OUTPUT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT + ]); + + this.state.field(TYPE_OUTPUT, 'current'); + } + + build(root) { + // TODO(indutny): init function + // TODO(indutny): parse function + + this.buildNode(root); + + return this.ir.build(); + } + + buildNode(node) { + const fn = this.ir.fn(this.sig, + `${this.prefix}_${node.name}`, [ 's', 'p', 'endp' ]); + fn.visibility = 'internal fastcc'; + + const body = fn.body; + + body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + } +} +module.exports = Compiler; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index ae2f1a9..e7ca624 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -1,3 +1,5 @@ 'use strict'; +exports.case = require('./case'); exports.node = require('./node'); +exports.Compiler = require('./compiler'); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 4b5b8c7..a63bde9 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -1,17 +1,28 @@ 'use strict'; +const assert = require('assert'); + +const llparse = require('../'); + class Node { constructor(name) { this.name = name; + this.cases = []; } - match(value, state) { + match(value, next) { + assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); + this.cases.push(new llparse.case.Match(value, next)); } - select(object, state) { + select(map, next) { + assert(next instanceof Node, 'Invalid `next` argument of `.select()`'); + this.cases.push(new llparse.case.Select(map, next)); } - otherwise() { + otherwise(next) { + assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); + this.cases.push(new llparse.case.Otherwise(next)); } } module.exports = Node; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 0830341..c12b21c 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -1,9 +1,13 @@ 'use strict'; +const assert = require('assert'); + const llparse = require('./'); class Invoke extends llparse.Node { constructor(callback, next) { + assert(next instanceof llparse.Node, + 'Invalid `next` argument of `.invoke()`'); super('invoke_' + callback); this.callback = callback; From 54f0a11827c1703594bd6165e76c1c725a6a7088 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 15 Feb 2018 15:12:00 -0500 Subject: [PATCH 031/281] save --- lib/llparse/case/base.js | 4 ++++ lib/llparse/case/match.js | 10 ++++++++-- lib/llparse/case/otherwise.js | 4 ++++ lib/llparse/case/select.js | 6 ++++++ lib/llparse/compiler.js | 11 +++++++++++ lib/llparse/index.js | 1 + lib/llparse/trie.js | 36 +++++++++++++++++++++++++++++++++++ test/api-test.js | 2 +- 8 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 lib/llparse/trie.js diff --git a/lib/llparse/case/base.js b/lib/llparse/case/base.js index 37476a8..36ed7ca 100644 --- a/lib/llparse/case/base.js +++ b/lib/llparse/case/base.js @@ -5,5 +5,9 @@ class Case { this.type = type; this.next = next; } + + linearize() { + throw new Error('Not implemented'); + } } module.exports = Case; diff --git a/lib/llparse/case/match.js b/lib/llparse/case/match.js index 46c0308..7cb368b 100644 --- a/lib/llparse/case/match.js +++ b/lib/llparse/case/match.js @@ -1,12 +1,18 @@ 'use strict'; +const Buffer = require('buffer').Buffer; + const Case = require('./').Case; class Match extends Case { - constructor(value, next) { + constructor(key, next) { super('match', next); - this.value = value; + this.key = key; + } + + linearize() { + return [ { key: Buffer.from(this.key), next: this.next, value: null } ]; } } module.exports = Match; diff --git a/lib/llparse/case/otherwise.js b/lib/llparse/case/otherwise.js index 9f266d4..9064a58 100644 --- a/lib/llparse/case/otherwise.js +++ b/lib/llparse/case/otherwise.js @@ -6,5 +6,9 @@ class Otherwise extends Case { constructor(next) { super('otherwise', next); } + + linearize() { + return []; + } } module.exports = Otherwise; diff --git a/lib/llparse/case/select.js b/lib/llparse/case/select.js index 98b9cf0..10a2949 100644 --- a/lib/llparse/case/select.js +++ b/lib/llparse/case/select.js @@ -8,5 +8,11 @@ class Select extends Case { this.map = map; } + + linearize() { + return Object.keys(this.map).map((key) => { + return { key: Buffer.from(key), next: this.next, value: this.map[key] }; + }); + } } module.exports = Select; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 52507ee..cad6e13 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -1,5 +1,8 @@ 'use strict'; +const llparse = require('./'); + +const assert = require('assert'); const IR = require('llvm-ir'); const TYPE_INPUT = IR.i(8).ptr(); @@ -9,6 +12,7 @@ class Compiler { constructor(prefix) { this.prefix = prefix; this.ir = new IR(); + this.trie = new llparse.Trie(); this.state = this.ir.struct(`${this.prefix}_state`); this.sig = this.ir.signature(TYPE_OUTPUT, [ @@ -34,6 +38,13 @@ class Compiler { const body = fn.body; + const trie = this.trie.combine(node.cases); + + const otherwise = + node.cases.filter(c => c instanceof llparse.case.Otherwise); + assert.strictEqual(otherwise.length, 1, + 'Node must have only 1 `.otherwise()`'); + body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); } } diff --git a/lib/llparse/index.js b/lib/llparse/index.js index e7ca624..c8f40a3 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -2,4 +2,5 @@ exports.case = require('./case'); exports.node = require('./node'); +exports.Trie = require('./trie'); exports.Compiler = require('./compiler'); diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js new file mode 100644 index 0000000..b58a139 --- /dev/null +++ b/lib/llparse/trie.js @@ -0,0 +1,36 @@ +'use strict'; + +class Single { + constructor(value) { + this.type = 'single'; + this.value = value; + } +} + +class Multiple { + constructor(value) { + this.type = 'multiple'; + this.value = value; + } +} + +class Trie { + constructor() { + } + + combine(cases) { + const list = []; + cases.forEach((one) => { + one.linearize().forEach(item => list.push(item)); + }); + + list.sort((a, b) => { + return a.key.compare(b.key); + }); + + const tree = new Map(); + + return tree; + } +} +module.exports = Trie; diff --git a/test/api-test.js b/test/api-test.js index 567fab4..5bc90e8 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -12,7 +12,7 @@ describe('LLParse', () => { const error = parse.error(1, 'Invalid word'); start.match('HTTP', response); - start.select({ 'GET': 0, 'POST': 1, 'PUT': 2 }, request); + start.select({ 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3 }, request); start.otherwise(error); request.otherwise(parse.invoke('on_request', start)); From def5efa80ffba107c9e50bff70ab4bb304018639 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 16 Feb 2018 15:16:28 -0500 Subject: [PATCH 032/281] trie: save progress --- lib/llparse/compiler.js | 12 +++-- lib/llparse/trie.js | 105 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 13 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index cad6e13..59467f9 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -31,11 +31,15 @@ class Compiler { return this.ir.build(); } - buildNode(node) { - const fn = this.ir.fn(this.sig, - `${this.prefix}_${node.name}`, [ 's', 'p', 'endp' ]); + createFn(node, index) { + const name = `${this.prefix}_${node.name}_${index}`; + const fn = this.ir.fn(this.sig, name, [ 's', 'p', 'endp' ]); fn.visibility = 'internal fastcc'; + return fn; + } + buildNode(node) { + const fn = this.createFn(node, 0); const body = fn.body; const trie = this.trie.combine(node.cases); @@ -46,6 +50,8 @@ class Compiler { 'Node must have only 1 `.otherwise()`'); body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + + return fn; } } module.exports = Compiler; diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index b58a139..042c2c5 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -1,16 +1,33 @@ 'use strict'; -class Single { - constructor(value) { - this.type = 'single'; - this.value = value; +const assert = require('assert'); + +class Node { + constructor(type) { + this.type = type; + } +} + +class Single extends Node { + constructor() { + super('single'); + this.children = []; + } +} + +class Sequence extends Node { + constructor(select) { + super('sequence'); + this.select = select; + this.child = null; } } -class Multiple { - constructor(value) { - this.type = 'multiple'; - this.value = value; +class Next extends Node { + constructor(value, next) { + super('next'); + this.value = value || null; + this.next = next; } } @@ -24,13 +41,81 @@ class Trie { one.linearize().forEach(item => list.push(item)); }); + return this.level(list); + } + + level(list) { list.sort((a, b) => { return a.key.compare(b.key); }); - const tree = new Map(); + // TODO(indutny): validate non-empty keys + const first = list[0].key; + const last = list[list.length - 1].key; + + let common = 0; + const min = Math.min(first.length, last.length); + + // Leaf + if (min === 0) { + // TODO(indutny): report key, or validate elsewhere + assert.strictEqual(list.length, 1, 'Duplicate entries'); + return new Next(list[0].value, list[0].next); + } + + // Find the longest common sub-string + for (; common < min; common++) + if (first[common] !== last[common]) + break; + + // Sequence + if (common !== 0) + return this.sequence(list, first.slice(0, common)); + + // Single + return this.single(list); + } + + slice(list, off) { + return list.map((item) => { + return { + key: item.key.slice(off), + next: item.next, + value: item.value + }; + }); + } + + sequence(list, prefix) { + const res = new Sequence(prefix); + + const sliced = this.slice(list, prefix.length); + res.children = this.level(sliced); + + return res; + } + + single(list) { + const keys = new Map(); + let last = null; + let lastKey = null; + for (let i = 0; i < list.length; i++) { + const item = list[i]; + const key = item.key[0]; + + if (keys.has(key)) + keys.get(key).push(item); + else + keys.set(key, [ item ]); + } + + const res = new Single(); + keys.forEach((sublist, key) => { + const sliced = this.slice(sublist, 1); + res.children.push({ key, child: this.level(sliced) }); + }); - return tree; + return res; } } module.exports = Trie; From f3499b59f82b68cfa5f808b4145a38d8b8936e82 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 16 Feb 2018 16:28:02 -0500 Subject: [PATCH 033/281] compiler: error state --- lib/llparse/compiler.js | 73 +++++++++++++++++++++++++++++++++++++---- test/api-test.js | 1 + 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 59467f9..603d49e 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -5,8 +5,15 @@ const llparse = require('./'); const assert = require('assert'); const IR = require('llvm-ir'); +const I32 = IR.i(32); const TYPE_INPUT = IR.i(8).ptr(); const TYPE_OUTPUT = IR.i(8).ptr(); +const TYPE_ERROR = I32; +const TYPE_REASON = IR.i(8).ptr(); + +const ARG_STATE = 's'; +const ARG_POS = 'p'; +const ARG_ENDPOS = 'endp'; class Compiler { constructor(prefix) { @@ -15,11 +22,16 @@ class Compiler { this.trie = new llparse.Trie(); this.state = this.ir.struct(`${this.prefix}_state`); + this.sig = this.ir.signature(TYPE_OUTPUT, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); + this.state.field(TYPE_ERROR, 'error'); + this.state.field(TYPE_REASON, 'reason'); this.state.field(TYPE_OUTPUT, 'current'); + + this.nodeMap = new Map(); } build(root) { @@ -32,26 +44,73 @@ class Compiler { } createFn(node, index) { - const name = `${this.prefix}_${node.name}_${index}`; - const fn = this.ir.fn(this.sig, name, [ 's', 'p', 'endp' ]); + const name = `${this.prefix}_${node.name}` + + `${index === null ? '' : '_' + index}`; + const fn = this.ir.fn(this.sig, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); fn.visibility = 'internal fastcc'; return fn; } buildNode(node) { - const fn = this.createFn(node, 0); - const body = fn.body; + if (this.nodeMap.has(node)) + return this.nodeMap.get(node); - const trie = this.trie.combine(node.cases); + const fn = this.createFn(node, null); + this.nodeMap.set(node, fn); + + const body = fn.body; const otherwise = node.cases.filter(c => c instanceof llparse.case.Otherwise); assert.strictEqual(otherwise.length, 1, - 'Node must have only 1 `.otherwise()`'); + `Node "${node.name}" must have only 1 \`.otherwise()\``); + + const trie = this.trie.combine(node.cases); - body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + this.buildRedirect(fn, body, otherwise[0].next); return fn; } + + buildRedirect(fn, body, to) { + if (to instanceof llparse.node.Error) + return this.buildError(fn, body, to); + + const target = this.buildNode(to).ref(); + const bitcast = IR._('bitcast', [ target.type, target, 'to', TYPE_OUTPUT ]); + body.push(bitcast); + return body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); + } + + buildError(fn, body, error) { + const code = error.code; + const reason = this.ir.cstr(error.reason); + + const stateArg = fn.arg(ARG_STATE); + + const codeField = IR._('getelementptr', this.state, + [ stateArg.type, stateArg ], + [ I32, I32.v(0) ], + [ I32, I32.v(this.state.lookup('error')) ]); + body.push(codeField); + + const reasonField = IR._('getelementptr', this.state, + [ stateArg.type, stateArg ], + [ I32, I32.v(0) ], + [ I32, I32.v(this.state.lookup('reason')) ]); + body.push(reasonField); + + const castReason = IR._('bitcast', [ + reason.type, reason, 'to', TYPE_REASON + ]); + body.push(castReason); + + body.push(IR._('store', [ TYPE_ERROR, TYPE_ERROR.v(code) ], + [ TYPE_ERROR.ptr(), codeField ]).void()); + body.push(IR._('store', [ TYPE_REASON, castReason ], + [ TYPE_REASON.ptr(), reasonField ]).void()); + + return body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + } } module.exports = Compiler; diff --git a/test/api-test.js b/test/api-test.js index 5bc90e8..b6deabe 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -16,6 +16,7 @@ describe('LLParse', () => { start.otherwise(error); request.otherwise(parse.invoke('on_request', start)); + response.otherwise(parse.invoke('on_response', start)); const out = parse.build(start); console.log(out); From fd7224cc0ff53a77c5fe95c4662874a43984f608 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 16 Feb 2018 22:15:51 -0500 Subject: [PATCH 034/281] lib: save progress --- lib/llparse/compiler.js | 152 ++++++++++++++++++++++++++++++++----- lib/llparse/node/invoke.js | 2 +- lib/llparse/trie.js | 6 +- test/api-test.js | 2 +- 4 files changed, 141 insertions(+), 21 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 603d49e..5ca224f 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -8,6 +8,7 @@ const IR = require('llvm-ir'); const I32 = IR.i(32); const TYPE_INPUT = IR.i(8).ptr(); const TYPE_OUTPUT = IR.i(8).ptr(); +const TYPE_MATCH = I32; const TYPE_ERROR = I32; const TYPE_REASON = IR.i(8).ptr(); @@ -29,9 +30,11 @@ class Compiler { this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); + this.state.field(TYPE_MATCH, 'match'); this.state.field(TYPE_OUTPUT, 'current'); this.nodeMap = new Map(); + this.counter = new Map(); } build(root) { @@ -43,9 +46,17 @@ class Compiler { return this.ir.build(); } - createFn(node, index) { - const name = `${this.prefix}_${node.name}` + - `${index === null ? '' : '_' + index}`; + createFn(node) { + let index; + if (this.counter.has(node.name)) + index = this.counter.get(node.name); + else + index = 0; + this.counter.set(node.name, index + 1); + + const name = `${this.prefix}__${node.name}` + + `${index === 0 ? '' : '_' + index}`; + const fn = this.ir.fn(this.sig, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); fn.visibility = 'internal fastcc'; return fn; @@ -55,10 +66,14 @@ class Compiler { if (this.nodeMap.has(node)) return this.nodeMap.get(node); - const fn = this.createFn(node, null); + const fn = this.createFn(node); this.nodeMap.set(node, fn); - const body = fn.body; + if (node instanceof llparse.node.Error) { + const info = { node, fn, otherwise: null }; + this.buildError(info, fn.body); + return fn; + } const otherwise = node.cases.filter(c => c instanceof llparse.case.Otherwise); @@ -67,26 +82,129 @@ class Compiler { const trie = this.trie.combine(node.cases); - this.buildRedirect(fn, body, otherwise[0].next); + const info = { node, fn, otherwise: otherwise[0].next }; + this.buildTrie(info, fn.body, trie); return fn; } - buildRedirect(fn, body, to) { - if (to instanceof llparse.node.Error) - return this.buildError(fn, body, to); + buildSubTrie(info, body, inc, trie) { + const subFn = this.createFn(info.node); + const subInfo = { fn: subFn, node: info.node, otherwise: info.otherwise }; + this.buildTrie(subInfo, subFn.body, trie); + + this.buildRedirect(info, body, inc, subFn); + return subFn; + } + + buildTrie(info, body, trie) { + const fn = info.fn; + + // Check that we have enough to do the read + const cmp = IR._('icmp', [ 'ne', TYPE_INPUT, fn.arg(ARG_POS) ], + fn.arg(ARG_ENDPOS)); + body.push(cmp); + + const branch = body.branch('br', [ IR.i(1), cmp ]); + + // Return self when `pos === endpos` + const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); + branch.right.push(bitcast); + branch.right.terminate('ret', [ TYPE_OUTPUT, bitcast ]); + + // Load the character + body = branch.left; + const cur = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ]); + body.push(cur); + + // Increment `pos` + const inc = IR._('getelementptr', TYPE_INPUT.to, + [ TYPE_INPUT, fn.arg(ARG_POS) ], + [ I32, I32.v(1) ]); + body.push(inc); + + // Traverse the `trie` + if (trie === null) { + // no-op + } else if (trie.type === 'single') { + body = this.buildSingle(info, body, cur, inc, trie.children); + } else if (trie.type === 'sequence') { + body = this.buildSequence(info, body, cur, inc, trie); + } else if (trie.type === 'next') { + // `next` is a final trie node + return this.buildNext(info, body, inc, trie); + } else { + throw new Error('Unknown trie node type: ' + trie.type); + } + + this.buildRedirect(info, body, inc, this.buildNode(info.otherwise)); + + return body; + } + + buildSingle(info, body, cur, pos, children) { + const cases = []; + cases.push(IR.label('otherwise')); + cases.push('['); + children.forEach((child, i) => { + const isLast = i === children.length - 1; + cases.push(TYPE_INPUT.to, TYPE_INPUT.to.v(child.key)); + cases.push(',', IR.label(`case_${i}`)); + }); + cases.push(']'); + + const blocks = body.terminate('switch', [ TYPE_INPUT.to, cur ], cases); + + const otherwise = blocks[0]; + const targets = blocks.slice(1); - const target = this.buildNode(to).ref(); - const bitcast = IR._('bitcast', [ target.type, target, 'to', TYPE_OUTPUT ]); - body.push(bitcast); - return body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); + targets.forEach((target, i) => { + this.buildSubTrie(info, target, pos, children[i].child); + }); + + return otherwise; + } + + buildSequence(info, body, cur, pos, trie) { + console.log(trie); + return body; + } + + buildNext(info, body, pos, trie) { + if (trie.value !== null) { + const stateArg = info.fn.arg(ARG_STATE); + + const matchField = IR._('getelementptr', this.state, + [ stateArg.type, stateArg ], + [ I32, I32.v(0) ], + [ I32, I32.v(this.state.lookup('match')) ]); + body.push(matchField); + body.push(IR._('store', [ TYPE_MATCH, TYPE_MATCH.v(trie.value) ], + [ TYPE_MATCH.ptr(), matchField ]).void()); + } + + this.buildRedirect(info, body, pos, this.buildNode(trie.next)); + return body; + } + + buildRedirect(info, body, pos, target) { + const call = IR._('tail call fastcc', [ + TYPE_OUTPUT, target, '(', + this.state.ptr(), info.fn.arg(ARG_STATE), ',', + TYPE_INPUT, pos, ',', + TYPE_INPUT, info.fn.arg(ARG_ENDPOS), + ')' + ]); + body.push(call); + body.terminate('ret', [ TYPE_OUTPUT, call ]); + return body; } - buildError(fn, body, error) { - const code = error.code; - const reason = this.ir.cstr(error.reason); + buildError(info, body) { + const code = info.node.code; + const reason = this.ir.cstr(info.node.reason); - const stateArg = fn.arg(ARG_STATE); + const stateArg = info.fn.arg(ARG_STATE); const codeField = IR._('getelementptr', this.state, [ stateArg.type, stateArg ], diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index c12b21c..a3ecf9f 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -11,7 +11,7 @@ class Invoke extends llparse.Node { super('invoke_' + callback); this.callback = callback; - this.next = next; + this.otherwise(next); } } module.exports = Invoke; diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index 042c2c5..7ee2b0f 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -19,7 +19,6 @@ class Sequence extends Node { constructor(select) { super('sequence'); this.select = select; - this.child = null; } } @@ -41,6 +40,9 @@ class Trie { one.linearize().forEach(item => list.push(item)); }); + if (list.length === 0) + return null; + return this.level(list); } @@ -69,7 +71,7 @@ class Trie { break; // Sequence - if (common !== 0) + if (common > 1) return this.sequence(list, first.slice(0, common)); // Single diff --git a/test/api-test.js b/test/api-test.js index b6deabe..048f74c 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -19,6 +19,6 @@ describe('LLParse', () => { response.otherwise(parse.invoke('on_response', start)); const out = parse.build(start); - console.log(out); + require('fs').writeFileSync('./2.ll', out); }); }); From 373c4234a6b41d5b97c7b79b546a1c239178a09e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 00:04:21 -0500 Subject: [PATCH 035/281] lib: save progress --- lib/llparse.js | 4 +- lib/llparse/compiler.js | 144 +++++++++++++++++++++++++------------ lib/llparse/node/base.js | 1 + lib/llparse/node/error.js | 2 + lib/llparse/node/invoke.js | 16 +++-- lib/llparse/trie.js | 25 +------ test/api-test.js | 9 ++- 7 files changed, 121 insertions(+), 80 deletions(-) diff --git a/lib/llparse.js b/lib/llparse.js index fba877d..97bd5fc 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -21,8 +21,8 @@ class LLParse { return new internal.node.Error(code, reason); } - invoke(name, next) { - return new internal.node.Invoke(name, next); + invoke(name, map, otherwise) { + return new internal.node.Invoke(name, map, otherwise); } build(root) { diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 5ca224f..53ce1a2 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -24,14 +24,18 @@ class Compiler { this.state = this.ir.struct(`${this.prefix}_state`); - this.sig = this.ir.signature(TYPE_OUTPUT, [ - this.state.ptr(), TYPE_INPUT, TYPE_INPUT - ]); + this.signature = { + node: this.ir.signature(TYPE_OUTPUT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT + ]), + callback: this.ir.signature(I32, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT + ]) + }; this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); this.state.field(TYPE_MATCH, 'match'); - this.state.field(TYPE_OUTPUT, 'current'); this.nodeMap = new Map(); this.counter = new Map(); @@ -57,7 +61,8 @@ class Compiler { const name = `${this.prefix}__${node.name}` + `${index === 0 ? '' : '_' + index}`; - const fn = this.ir.fn(this.sig, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + const fn = this.ir.fn(this.signature.node, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); fn.visibility = 'internal fastcc'; return fn; } @@ -69,9 +74,11 @@ class Compiler { const fn = this.createFn(node); this.nodeMap.set(node, fn); + let body = this.buildPrologue(node, fn); + if (node instanceof llparse.node.Error) { const info = { node, fn, otherwise: null }; - this.buildError(info, fn.body); + this.buildError(info, body); return fn; } @@ -79,97 +86,144 @@ class Compiler { node.cases.filter(c => c instanceof llparse.case.Otherwise); assert.strictEqual(otherwise.length, 1, `Node "${node.name}" must have only 1 \`.otherwise()\``); + const info = { node, fn, otherwise: otherwise[0].next }; const trie = this.trie.combine(node.cases); - const info = { node, fn, otherwise: otherwise[0].next }; - this.buildTrie(info, fn.body, trie); + this.buildTrie(info, body, trie); return fn; } - buildSubTrie(info, body, inc, trie) { + buildInvoke(info, body, pos, callback) { + const external = this.ir.declare(this.signature.callback, callback); + external.attributes = 'alwaysinline'; + + const returnType = this.signature.callback.ret; + + const call = IR._('call', [ + returnType, external, '(', + this.state.ptr(), info.fn.arg(ARG_STATE), ',', + TYPE_INPUT, info.fn.arg(ARG_POS), ',', + TYPE_INPUT, info.fn.arg(ARG_ENDPOS), + ')' + ]); + body.push(call); + + const keys = Object.keys(info.node.map).map(key => key | 0); + const s = this.buildSwitch(body, returnType, call, keys); + + s.cases.forEach((body, i) => { + const subNode = info.node.map[keys[i]]; + this.buildRedirect(info, body, pos, this.buildNode(subNode)); + }); + + return s.otherwise; + } + + buildSubTrie(info, body, pos, trie) { + if (trie.type === 'next') + return this.buildNext(info, body, pos, trie); + const subFn = this.createFn(info.node); const subInfo = { fn: subFn, node: info.node, otherwise: info.otherwise }; - this.buildTrie(subInfo, subFn.body, trie); - this.buildRedirect(info, body, inc, subFn); + const subBody = this.buildPrologue(info.node, subFn); + this.buildTrie(subInfo, subBody, trie); + + this.buildRedirect(info, body, pos, subFn); return subFn; } - buildTrie(info, body, trie) { - const fn = info.fn; + buildPrologue(node, fn) { + if (node.noAdvance) + return fn.body; - // Check that we have enough to do the read + // Check that we have enough chars to do the read const cmp = IR._('icmp', [ 'ne', TYPE_INPUT, fn.arg(ARG_POS) ], fn.arg(ARG_ENDPOS)); - body.push(cmp); + fn.body.push(cmp); - const branch = body.branch('br', [ IR.i(1), cmp ]); + const branch = fn.body.branch('br', [ IR.i(1), cmp ]); // Return self when `pos === endpos` const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); branch.right.push(bitcast); branch.right.terminate('ret', [ TYPE_OUTPUT, bitcast ]); + return branch.left; + } + + buildTrie(info, body, trie) { + const fn = info.fn; + + // Increment `pos` if not invoking external callback + let pos = { current: fn.arg(ARG_POS), next: null }; + + if (info.node.noAdvance) { + pos.next = pos.current; + } else { + pos.next = IR._('getelementptr', TYPE_INPUT.to, + [ TYPE_INPUT, fn.arg(ARG_POS) ], + [ I32, I32.v(1) ]); + body.push(pos.next); + } + + if (info.node instanceof llparse.node.Invoke) + body = this.buildInvoke(info, body, pos, info.node.callback); + // Load the character - body = branch.left; const cur = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ]); body.push(cur); - // Increment `pos` - const inc = IR._('getelementptr', TYPE_INPUT.to, - [ TYPE_INPUT, fn.arg(ARG_POS) ], - [ I32, I32.v(1) ]); - body.push(inc); - // Traverse the `trie` if (trie === null) { // no-op } else if (trie.type === 'single') { - body = this.buildSingle(info, body, cur, inc, trie.children); - } else if (trie.type === 'sequence') { - body = this.buildSequence(info, body, cur, inc, trie); - } else if (trie.type === 'next') { - // `next` is a final trie node - return this.buildNext(info, body, inc, trie); + body = this.buildSingle(info, body, cur, pos, trie.children); } else { - throw new Error('Unknown trie node type: ' + trie.type); + // NOTE: `next` type must be parsed in `buildSubTrie` + throw new Error('Unexpected trie node type: ' + trie.type); } - this.buildRedirect(info, body, inc, this.buildNode(info.otherwise)); + this.buildRedirect(info, body, pos, this.buildNode(info.otherwise)); return body; } - buildSingle(info, body, cur, pos, children) { + buildSwitch(body, type, what, values) { const cases = []; cases.push(IR.label('otherwise')); cases.push('['); - children.forEach((child, i) => { - const isLast = i === children.length - 1; - cases.push(TYPE_INPUT.to, TYPE_INPUT.to.v(child.key)); + values.forEach((value, i) => { + const isLast = i === values.length - 1; + cases.push(type, type.v(value)); cases.push(',', IR.label(`case_${i}`)); }); cases.push(']'); - const blocks = body.terminate('switch', [ TYPE_INPUT.to, cur ], cases); + const blocks = body.terminate('switch', [ type, what ], cases); + + return { + otherwise: blocks[0], + cases: blocks.slice(1) + }; + } + + buildSingle(info, body, cur, pos, children) { + const keys = children.map(child => child.key); + const s = this.buildSwitch(body, TYPE_INPUT.to, cur, keys); - const otherwise = blocks[0]; - const targets = blocks.slice(1); + const otherwise = s.otherwise; + const cases = s.cases; - targets.forEach((target, i) => { + cases.forEach((target, i) => { this.buildSubTrie(info, target, pos, children[i].child); }); return otherwise; } - buildSequence(info, body, cur, pos, trie) { - console.log(trie); - return body; - } - buildNext(info, body, pos, trie) { if (trie.value !== null) { const stateArg = info.fn.arg(ARG_STATE); @@ -191,7 +245,7 @@ class Compiler { const call = IR._('tail call fastcc', [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', - TYPE_INPUT, pos, ',', + TYPE_INPUT, pos.next, ',', TYPE_INPUT, info.fn.arg(ARG_ENDPOS), ')' ]); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index a63bde9..85e6ece 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -8,6 +8,7 @@ class Node { constructor(name) { this.name = name; this.cases = []; + this.noAdvance = false; } match(value, next) { diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index c94614a..2d339d2 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -8,6 +8,8 @@ class Error extends llparse.Node { this.code = code; this.reason = reason; + + this.noAdvance = true; } } module.exports = Error; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index a3ecf9f..8f7d7f8 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -5,13 +5,21 @@ const assert = require('assert'); const llparse = require('./'); class Invoke extends llparse.Node { - constructor(callback, next) { - assert(next instanceof llparse.Node, - 'Invalid `next` argument of `.invoke()`'); + constructor(callback, map, otherwise) { + assert.strictEqual(typeof map, 'object', + 'Invalid `map` argument of `.invoke()`'); + assert(otherwise instanceof llparse.Node, + 'Invalid `otherwise` argument of `.invoke()`'); super('invoke_' + callback); this.callback = callback; - this.otherwise(next); + + // TODO(indutny): validate that keys are integers + this.map = map; + + this.otherwise(otherwise); + + this.noAdvance = true; } } module.exports = Invoke; diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index 7ee2b0f..fe792f0 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -15,13 +15,6 @@ class Single extends Node { } } -class Sequence extends Node { - constructor(select) { - super('sequence'); - this.select = select; - } -} - class Next extends Node { constructor(value, next) { super('next'); @@ -65,14 +58,7 @@ class Trie { return new Next(list[0].value, list[0].next); } - // Find the longest common sub-string - for (; common < min; common++) - if (first[common] !== last[common]) - break; - - // Sequence - if (common > 1) - return this.sequence(list, first.slice(0, common)); + // TODO(indutny): support sequences // Single return this.single(list); @@ -88,15 +74,6 @@ class Trie { }); } - sequence(list, prefix) { - const res = new Sequence(prefix); - - const sliced = this.slice(list, prefix.length); - res.children = this.level(sliced); - - return res; - } - single(list) { const keys = new Map(); let last = null; diff --git a/test/api-test.js b/test/api-test.js index 048f74c..5f991a1 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -11,12 +11,11 @@ describe('LLParse', () => { const response = parse.node('res'); const error = parse.error(1, 'Invalid word'); - start.match('HTTP', response); - start.select({ 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3 }, request); - start.otherwise(error); + start.match('HTTP', parse.invoke('on_response', { + 0: start + }, parse.error(2, '`on_response` error'))); - request.otherwise(parse.invoke('on_request', start)); - response.otherwise(parse.invoke('on_response', start)); + start.otherwise(error); const out = parse.build(start); require('fs').writeFileSync('./2.ll', out); From 9355f3f5a9dd9b1673bbff72097bf8d9c9790c30 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 00:16:16 -0500 Subject: [PATCH 036/281] test: return of request --- lib/llparse/compiler.js | 1 + test/api-test.js | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 53ce1a2..16758cc 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -225,6 +225,7 @@ class Compiler { } buildNext(info, body, pos, trie) { + // Set `state.match` if needed if (trie.value !== null) { const stateArg = info.fn.arg(ARG_STATE); diff --git a/test/api-test.js b/test/api-test.js index 5f991a1..d34a56b 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -15,6 +15,12 @@ describe('LLParse', () => { 0: start }, parse.error(2, '`on_response` error'))); + start.select({ + 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3 + }, parse.invoke('on_request', { + 0: start + }, parse.error(3, '`on_request` error'))); + start.otherwise(error); const out = parse.build(start); From 9acedc8e5598c88bf5ba546e88b07d6a174df6e4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 00:36:00 -0500 Subject: [PATCH 037/281] compiler: remove `fastcc` for now --- lib/llparse/compiler.js | 4 ++-- test/api-test.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 16758cc..60d2b28 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -63,7 +63,7 @@ class Compiler { const fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - fn.visibility = 'internal fastcc'; + fn.visibility = 'internal'; return fn; } @@ -243,7 +243,7 @@ class Compiler { } buildRedirect(info, body, pos, target) { - const call = IR._('tail call fastcc', [ + const call = IR._('tail call', [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', diff --git a/test/api-test.js b/test/api-test.js index d34a56b..3b64998 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -16,7 +16,9 @@ describe('LLParse', () => { }, parse.error(2, '`on_response` error'))); start.select({ - 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3 + 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, + 'DELETE': 4, 'OPTIONS': 5, 'CONNECT': 6, + 'TRACE': 7, 'PATCH': 8 }, parse.invoke('on_request', { 0: start }, parse.error(3, '`on_request` error'))); From 8aa66dfb138f903401fb6300b72ac7f7bc34c8d2 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 01:21:25 -0500 Subject: [PATCH 038/281] compiler: preliminary `sequence` support --- lib/llparse/compiler.js | 102 +++++++++++++++++++++++++++++++--------- lib/llparse/trie.js | 25 +++++++++- 2 files changed, 105 insertions(+), 22 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 60d2b28..a2c2e4a 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -9,6 +9,7 @@ const I32 = IR.i(32); const TYPE_INPUT = IR.i(8).ptr(); const TYPE_OUTPUT = IR.i(8).ptr(); const TYPE_MATCH = I32; +const TYPE_INDEX = I32; const TYPE_ERROR = I32; const TYPE_REASON = IR.i(8).ptr(); @@ -33,8 +34,10 @@ class Compiler { ]) }; + this.state.field(this.signature.node.ptr(), 'current'); this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); + this.state.field(TYPE_INDEX, 'index'); this.state.field(TYPE_MATCH, 'match'); this.nodeMap = new Map(); @@ -173,14 +176,16 @@ class Compiler { body = this.buildInvoke(info, body, pos, info.node.callback); // Load the character - const cur = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ]); - body.push(cur); + const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ]); + body.push(current); // Traverse the `trie` if (trie === null) { // no-op } else if (trie.type === 'single') { - body = this.buildSingle(info, body, cur, pos, trie.children); + body = this.buildSingle(info, body, current, pos, trie.children); + } else if (trie.type === 'sequence') { + body = this.buildSequence(info, body, current, pos, trie); } else { // NOTE: `next` type must be parsed in `buildSubTrie` throw new Error('Unexpected trie node type: ' + trie.type); @@ -210,9 +215,9 @@ class Compiler { }; } - buildSingle(info, body, cur, pos, children) { + buildSingle(info, body, current, pos, children) { const keys = children.map(child => child.key); - const s = this.buildSwitch(body, TYPE_INPUT.to, cur, keys); + const s = this.buildSwitch(body, TYPE_INPUT.to, current, keys); const otherwise = s.otherwise; const cases = s.cases; @@ -224,15 +229,69 @@ class Compiler { return otherwise; } + buildSequence(info, body, current, pos, trie) { + const seq = this.ir.data(trie.select); + + // Load `state.index` + const indexField = this.field(info.fn, 'index'); + body.push(indexField); + const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexField ]); + body.push(index); + + // Get `expected = seq[state.index]` + const expectedPtr = pos.next = IR._('getelementptr inbounds', seq.type.to, + [ seq.type, seq ], + [ I32, I32.v(0) ], + [ I32, index ]); + body.push(expectedPtr); + const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); + body.push(expected); + + // Check `current === expected` + const cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); + body.push(cmp); + const branch = body.branch('br', [ IR.i(1), cmp ]); + + const reset = () => { + return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], + [ TYPE_INDEX.ptr(), indexField ]).void(); + }; + + // Equal! + body = branch.left; + + // Increment `state.index` + const index1 = IR._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); + body.push(index1); + + // Check `state.index === seq.length` + const complete = IR._('icmp', [ 'eq', TYPE_INDEX, index ], + TYPE_INDEX.v(trie.select.length)); + body.push(complete); + const completeBranch = body.branch('br', [ IR.i(1), complete ]); + + // It is complete! + this.buildSubTrie(info, completeBranch.left, pos, trie.children); + + // Not yet complete + body = completeBranch.right; + body.push(IR._('store', [ TYPE_INDEX, index ], + [ TYPE_INDEX.ptr(), indexField ]).void()); + + // TODO(indutny): just do a loop + // Restart! + this.buildRedirect(info, body, pos, info.fn); + + // Not equal + // Reset `state.index` on mismatch + branch.right.push(reset()); + return branch.right; + } + buildNext(info, body, pos, trie) { // Set `state.match` if needed if (trie.value !== null) { - const stateArg = info.fn.arg(ARG_STATE); - - const matchField = IR._('getelementptr', this.state, - [ stateArg.type, stateArg ], - [ I32, I32.v(0) ], - [ I32, I32.v(this.state.lookup('match')) ]); + const matchField = this.field(info.fn, 'match'); body.push(matchField); body.push(IR._('store', [ TYPE_MATCH, TYPE_MATCH.v(trie.value) ], [ TYPE_MATCH.ptr(), matchField ]).void()); @@ -259,18 +318,10 @@ class Compiler { const code = info.node.code; const reason = this.ir.cstr(info.node.reason); - const stateArg = info.fn.arg(ARG_STATE); - - const codeField = IR._('getelementptr', this.state, - [ stateArg.type, stateArg ], - [ I32, I32.v(0) ], - [ I32, I32.v(this.state.lookup('error')) ]); + const codeField = this.field(info.fn, 'error'); body.push(codeField); - const reasonField = IR._('getelementptr', this.state, - [ stateArg.type, stateArg ], - [ I32, I32.v(0) ], - [ I32, I32.v(this.state.lookup('reason')) ]); + const reasonField = this.field(info.fn, 'reason'); body.push(reasonField); const castReason = IR._('bitcast', [ @@ -285,5 +336,14 @@ class Compiler { return body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); } + + field(fn, name) { + const stateArg = fn.arg(ARG_STATE); + + return IR._('getelementptr', this.state, + [ stateArg.type, stateArg ], + [ I32, I32.v(0) ], + [ I32, I32.v(this.state.lookup(name)) ]); + } } module.exports = Compiler; diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index fe792f0..7ee2b0f 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -15,6 +15,13 @@ class Single extends Node { } } +class Sequence extends Node { + constructor(select) { + super('sequence'); + this.select = select; + } +} + class Next extends Node { constructor(value, next) { super('next'); @@ -58,7 +65,14 @@ class Trie { return new Next(list[0].value, list[0].next); } - // TODO(indutny): support sequences + // Find the longest common sub-string + for (; common < min; common++) + if (first[common] !== last[common]) + break; + + // Sequence + if (common > 1) + return this.sequence(list, first.slice(0, common)); // Single return this.single(list); @@ -74,6 +88,15 @@ class Trie { }); } + sequence(list, prefix) { + const res = new Sequence(prefix); + + const sliced = this.slice(list, prefix.length); + res.children = this.level(sliced); + + return res; + } + single(list) { const keys = new Map(); let last = null; From 5760c5d1760c1aa58c050d3d88d3563fa451d62f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 01:52:00 -0500 Subject: [PATCH 039/281] compiler: working sequence matching --- lib/llparse/compiler.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index a2c2e4a..1d28dd0 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -239,7 +239,7 @@ class Compiler { body.push(index); // Get `expected = seq[state.index]` - const expectedPtr = pos.next = IR._('getelementptr inbounds', seq.type.to, + const expectedPtr = IR._('getelementptr inbounds', seq.type.to, [ seq.type, seq ], [ I32, I32.v(0) ], [ I32, index ]); @@ -265,17 +265,18 @@ class Compiler { body.push(index1); // Check `state.index === seq.length` - const complete = IR._('icmp', [ 'eq', TYPE_INDEX, index ], + const complete = IR._('icmp', [ 'eq', TYPE_INDEX, index1 ], TYPE_INDEX.v(trie.select.length)); body.push(complete); const completeBranch = body.branch('br', [ IR.i(1), complete ]); // It is complete! + completeBranch.right.push(reset()); this.buildSubTrie(info, completeBranch.left, pos, trie.children); // Not yet complete body = completeBranch.right; - body.push(IR._('store', [ TYPE_INDEX, index ], + body.push(IR._('store', [ TYPE_INDEX, index1 ], [ TYPE_INDEX.ptr(), indexField ]).void()); // TODO(indutny): just do a loop From 546abe44ccf132775b3a9046eb46ff42fa9c5070 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 03:12:44 -0500 Subject: [PATCH 040/281] compiler: loop through `sequence` --- lib/llparse/compiler.js | 86 ++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 1d28dd0..6a8a19b 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -150,13 +150,17 @@ class Compiler { const branch = fn.body.branch('br', [ IR.i(1), cmp ]); // Return self when `pos === endpos` - const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); - branch.right.push(bitcast); - branch.right.terminate('ret', [ TYPE_OUTPUT, bitcast ]); + this.buildSelfReturn(fn, branch.right); return branch.left; } + buildSelfReturn(fn, body) { + const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); + body.push(bitcast); + body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); + } + buildTrie(info, body, trie) { const fn = info.fn; @@ -175,17 +179,17 @@ class Compiler { if (info.node instanceof llparse.node.Invoke) body = this.buildInvoke(info, body, pos, info.node.callback); - // Load the character - const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ]); - body.push(current); - // Traverse the `trie` if (trie === null) { // no-op } else if (trie.type === 'single') { - body = this.buildSingle(info, body, current, pos, trie.children); + body = this.buildSingle(info, body, pos, trie.children); } else if (trie.type === 'sequence') { - body = this.buildSequence(info, body, current, pos, trie); + const seq = this.buildSequence(info, body, trie); + + // NOTE: `sequence` implementation loops if there's enough data + body = seq.body; + pos = seq.pos; } else { // NOTE: `next` type must be parsed in `buildSubTrie` throw new Error('Unexpected trie node type: ' + trie.type); @@ -215,7 +219,12 @@ class Compiler { }; } - buildSingle(info, body, current, pos, children) { + buildSingle(info, body, pos, children) { + // Load the character + const current = + IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, info.fn.arg(ARG_POS) ]); + body.push(current); + const keys = children.map(child => child.key); const s = this.buildSwitch(body, TYPE_INPUT.to, current, keys); @@ -229,7 +238,7 @@ class Compiler { return otherwise; } - buildSequence(info, body, current, pos, trie) { + buildSequence(info, body, trie) { const seq = this.ir.data(trie.select); // Load `state.index` @@ -238,19 +247,32 @@ class Compiler { const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexField ]); body.push(index); + const loop = body.jump('br'); + + // Loop start + const posPhi = IR._('phi', + [ TYPE_INPUT, '[', info.fn.arg(ARG_POS), ',', body.ref(), ']' ]); + loop.push(posPhi); + const indexPhi = IR._('phi', + [ TYPE_INDEX, '[', index, ',', body.ref(), ']' ]); + loop.push(indexPhi); + + const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, posPhi ]); + loop.push(current); + // Get `expected = seq[state.index]` const expectedPtr = IR._('getelementptr inbounds', seq.type.to, [ seq.type, seq ], [ I32, I32.v(0) ], - [ I32, index ]); - body.push(expectedPtr); + [ I32, indexPhi ]); + loop.push(expectedPtr); const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); - body.push(expected); + loop.push(expected); // Check `current === expected` const cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); - body.push(cmp); - const branch = body.branch('br', [ IR.i(1), cmp ]); + loop.push(cmp); + const branch = loop.branch('br', [ IR.i(1), cmp ]); const reset = () => { return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], @@ -260,8 +282,13 @@ class Compiler { // Equal! body = branch.left; + const next = IR._('getelementptr', TYPE_INPUT.to, + [ TYPE_INPUT, posPhi ], + [ I32, I32.v(1) ]); + body.push(next); + // Increment `state.index` - const index1 = IR._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); + const index1 = IR._('add', [ TYPE_INDEX, indexPhi ], TYPE_INDEX.v(1)); body.push(index1); // Check `state.index === seq.length` @@ -271,22 +298,37 @@ class Compiler { const completeBranch = body.branch('br', [ IR.i(1), complete ]); // It is complete! - completeBranch.right.push(reset()); + completeBranch.left.push(reset()); + const pos = { current, next }; this.buildSubTrie(info, completeBranch.left, pos, trie.children); // Not yet complete body = completeBranch.right; + + // Loop if there is more data + const hasMore = IR._('icmp', [ 'ne', TYPE_INPUT, posPhi ], + info.fn.arg(ARG_ENDPOS)); + body.push(hasMore); + const hasMoreBranch = body.branch('br', [ IR.i(1), hasMore ]); + + // We have more data! + body = hasMoreBranch.left; + + body.loop('br', loop); + indexPhi.append([ '[', index1, ',', body.ref(), ']' ]); + posPhi.append([ '[', next, ',', body.ref(), ']' ]); + + // No more data + body = hasMoreBranch.right; body.push(IR._('store', [ TYPE_INDEX, index1 ], [ TYPE_INDEX.ptr(), indexField ]).void()); - // TODO(indutny): just do a loop - // Restart! - this.buildRedirect(info, body, pos, info.fn); + this.buildSelfReturn(info.fn, body); // Not equal // Reset `state.index` on mismatch branch.right.push(reset()); - return branch.right; + return { pos, body: branch.right }; } buildNext(info, body, pos, trie) { From a6923d527f671a641c498467744427289844457b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 03:34:07 -0500 Subject: [PATCH 041/281] compiler: public methods --- lib/llparse/compiler.js | 75 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 6a8a19b..6069e8d 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -45,14 +45,82 @@ class Compiler { } build(root) { - // TODO(indutny): init function - // TODO(indutny): parse function + const rootFn = this.buildNode(root); - this.buildNode(root); + this.buildInit(rootFn.ref()); + this.buildParse(); return this.ir.build(); } + buildInit(fn) { + const sig = IR.signature(IR.void(), [ this.state.ptr() ]); + const init = this.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); + + const fields = { + current: this.field(init, 'current'), + error: this.field(init, 'error'), + reason: this.field(init, 'reason'), + index: this.field(init, 'index'), + match: this.field(init, 'match') + }; + + Object.keys(fields).forEach(key => init.body.push(fields[key])); + + const store = (field, type, value) => { + init.body.push(IR._('store', [ type, value ], + [ type.ptr(), field ]).void()); + } + + store(fields.current, fn.type, fn); + store(fields.error, TYPE_ERROR, TYPE_ERROR.v(0)); + store(fields.reason, TYPE_REASON, TYPE_REASON.v(null)); + store(fields.index, TYPE_INDEX, TYPE_INDEX.v(0)); + store(fields.match, TYPE_MATCH, TYPE_MATCH.v(0)); + + init.body.terminate('ret', IR.void()); + + return init; + } + + buildParse() { + const sig = IR.signature(TYPE_ERROR, + [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); + const parse = this.ir.fn(sig, this.prefix + '_parse', + [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + + const body = parse.body; + + const nodeSig = this.signature.node; + + const currentPtr = this.field(parse, 'current'); + body.push(currentPtr); + const current = IR._('load', nodeSig.ptr(), + [ nodeSig.ptr().ptr(), currentPtr ]); + body.push(current); + + const call = IR._('call', [ + TYPE_OUTPUT, current, '(', + this.state.ptr(), parse.arg(ARG_STATE), ',', + TYPE_INPUT, parse.arg(ARG_POS), ',', + TYPE_INPUT, parse.arg(ARG_ENDPOS), + ')' + ]); + body.push(call); + + const errorPtr = this.field(parse, 'error'); + body.push(errorPtr); + const error = IR._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); + body.push(error); + + const bitcast = IR._('bitcast', [ TYPE_OUTPUT, call, 'to', nodeSig.ptr() ]); + body.push(bitcast); + body.push(IR._('store', [ nodeSig.ptr(), bitcast ], + [ nodeSig.ptr().ptr(), currentPtr ]).void()); + + body.terminate('ret', [ TYPE_ERROR, error ]); + } + createFn(node) { let index; if (this.counter.has(node.name)) @@ -238,6 +306,7 @@ class Compiler { return otherwise; } + // TODO(indutny): rework this into something readable buildSequence(info, body, trie) { const seq = this.ir.data(trie.select); From 2c8bb003748567fa7a66523bb4bc3ea9ea1d5689 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 03:38:45 -0500 Subject: [PATCH 042/281] compiler: return of `fastcc` --- lib/llparse/compiler.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 6069e8d..6d96136 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -99,7 +99,7 @@ class Compiler { [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); - const call = IR._('call', [ + const call = IR._('call fastcc', [ TYPE_OUTPUT, current, '(', this.state.ptr(), parse.arg(ARG_STATE), ',', TYPE_INPUT, parse.arg(ARG_POS), ',', @@ -134,7 +134,7 @@ class Compiler { const fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - fn.visibility = 'internal'; + fn.visibility = 'internal fastcc'; return fn; } @@ -414,7 +414,7 @@ class Compiler { } buildRedirect(info, body, pos, target) { - const call = IR._('tail call', [ + const call = IR._('tail call fastcc', [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', From 33a79d3c942275f55319d0954b3a4ab762fee523 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 04:20:02 -0500 Subject: [PATCH 043/281] compiler: fix `sequence` --- lib/llparse/compiler.js | 6 +++--- test/api-test.js | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 6d96136..2d495b6 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -86,7 +86,7 @@ class Compiler { buildParse() { const sig = IR.signature(TYPE_ERROR, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); - const parse = this.ir.fn(sig, this.prefix + '_parse', + const parse = this.ir.fn(sig, this.prefix + '_execute', [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); const body = parse.body; @@ -387,9 +387,9 @@ class Compiler { indexPhi.append([ '[', index1, ',', body.ref(), ']' ]); posPhi.append([ '[', next, ',', body.ref(), ']' ]); - // No more data + // No more data (store `index`, not `index + 1`) body = hasMoreBranch.right; - body.push(IR._('store', [ TYPE_INDEX, index1 ], + body.push(IR._('store', [ TYPE_INDEX, indexPhi ], [ TYPE_INDEX.ptr(), indexField ]).void()); this.buildSelfReturn(info.fn, body); diff --git a/test/api-test.js b/test/api-test.js index 3b64998..c078fe4 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -9,11 +9,10 @@ describe('LLParse', () => { const start = parse.node('start'); const request = parse.node('req'); const response = parse.node('res'); - const error = parse.error(1, 'Invalid word'); start.match('HTTP', parse.invoke('on_response', { 0: start - }, parse.error(2, '`on_response` error'))); + }, parse.error(1, '`on_response` error'))); start.select({ 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, @@ -21,9 +20,9 @@ describe('LLParse', () => { 'TRACE': 7, 'PATCH': 8 }, parse.invoke('on_request', { 0: start - }, parse.error(3, '`on_request` error'))); + }, parse.error(2, '`on_request` error'))); - start.otherwise(error); + start.otherwise(parse.error(3, 'Invalid word')); const out = parse.build(start); require('fs').writeFileSync('./2.ll', out); From a859cb4306670663455079567f6ec44832fa52aa Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 15:50:54 -0500 Subject: [PATCH 044/281] compiler: comments, refactor --- lib/llparse/compiler.js | 166 ++++++++++++++++++++++++---------------- 1 file changed, 102 insertions(+), 64 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 2d495b6..3fe9dac 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -211,6 +211,8 @@ class Compiler { return fn.body; // Check that we have enough chars to do the read + fn.body.comment('--- Prologue ---'); + fn.body.comment('if (pos != endpos)'); const cmp = IR._('icmp', [ 'ne', TYPE_INPUT, fn.arg(ARG_POS) ], fn.arg(ARG_ENDPOS)); fn.body.push(cmp); @@ -218,8 +220,10 @@ class Compiler { const branch = fn.body.branch('br', [ IR.i(1), cmp ]); // Return self when `pos === endpos` + branch.right.name = 'prologue_end'; this.buildSelfReturn(fn, branch.right); + branch.left.name = 'prologue_normal'; return branch.left; } @@ -238,6 +242,7 @@ class Compiler { if (info.node.noAdvance) { pos.next = pos.current; } else { + body.comment('next = pos + 1'); pos.next = IR._('getelementptr', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ], [ I32, I32.v(1) ]); @@ -281,6 +286,10 @@ class Compiler { const blocks = body.terminate('switch', [ type, what ], cases); + blocks[0].name = 'switch_otherwise'; + for (let i = 0; i< values.length; i++) + blocks[i + 1].name = 'case_' + values[i]; + return { otherwise: blocks[0], cases: blocks.slice(1) @@ -306,17 +315,99 @@ class Compiler { return otherwise; } + buildSequenceIteration(fn, body, seq, indexField, pos, index) { + // Just a helper to reset `state.index` + const reset = () => { + return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], + [ TYPE_INDEX.ptr(), indexField ]).void(); + }; + + body.comment('current = *pos'); + const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); + body.push(current); + + body.comment('expected = seq[state.index]'); + const expectedPtr = IR._('getelementptr inbounds', seq.type.to, + [ seq.type, seq ], + [ I32, I32.v(0) ], + [ I32, index ]); + body.push(expectedPtr); + const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); + body.push(expected); + + body.comment('if (current == expected)'); + let cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); + body.push(cmp); + const { left: isMatch, right: isMismatch } = + body.branch('br', [ IR.i(1), cmp ]); + + // Mismatch + isMismatch.name = 'mismatch'; + isMismatch.comment('Sequence string does not match input'); + isMismatch.comment('state.index = 0'); + isMismatch.push(reset()); + + // Character matches + isMatch.name = 'match'; + isMatch.comment('Got a char match'); + isMatch.comment('next = pos + 1'); + const next = IR._('getelementptr', TYPE_INPUT.to, + [ TYPE_INPUT, pos ], + [ I32, I32.v(1) ]); + isMatch.push(next); + + isMatch.comment('index1 = index + 1'); + const index1 = IR._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); + isMatch.push(index1); + + isMatch.comment('if (index1 == seq.length)'); + cmp = IR._('icmp', [ 'eq', TYPE_INDEX, index1 ], + TYPE_INDEX.v(seq.type.to.length)); + isMatch.push(cmp); + const { left: isComplete, right: isIncomplete } = + isMatch.branch('br', [ IR.i(1), cmp ]); + + isComplete.name = 'is_complete'; + isComplete.comment('state.index = 0'); + isComplete.push(reset()); + + isIncomplete.name = 'is_incomplete'; + isIncomplete.comment('if (next != endpos)'); + cmp = IR._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); + isIncomplete.push(cmp); + const { left: moreData, right: noMoreData } = + isIncomplete.branch('br', [ IR.i(1), cmp ]); + + moreData.name = 'more_data'; + + noMoreData.name = 'no_more_data'; + noMoreData.comment('state.index = index1'); + noMoreData.push(IR._('store', [ TYPE_INDEX, index1 ], + [ TYPE_INDEX.ptr(), indexField ]).void()); + + return { + index: index1, + pos: { current: pos, next }, + complete: isComplete, + otherwise: isMismatch, + loop: moreData, + pause: noMoreData + }; + } + // TODO(indutny): rework this into something readable buildSequence(info, body, trie) { const seq = this.ir.data(trie.select); // Load `state.index` + body.comment('index = state.index'); const indexField = this.field(info.fn, 'index'); body.push(indexField); const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexField ]); body.push(index); const loop = body.jump('br'); + loop.name = 'loop'; // Loop start const posPhi = IR._('phi', @@ -326,84 +417,30 @@ class Compiler { [ TYPE_INDEX, '[', index, ',', body.ref(), ']' ]); loop.push(indexPhi); - const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, posPhi ]); - loop.push(current); - - // Get `expected = seq[state.index]` - const expectedPtr = IR._('getelementptr inbounds', seq.type.to, - [ seq.type, seq ], - [ I32, I32.v(0) ], - [ I32, indexPhi ]); - loop.push(expectedPtr); - const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); - loop.push(expected); - - // Check `current === expected` - const cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); - loop.push(cmp); - const branch = loop.branch('br', [ IR.i(1), cmp ]); - - const reset = () => { - return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], - [ TYPE_INDEX.ptr(), indexField ]).void(); - }; - - // Equal! - body = branch.left; - - const next = IR._('getelementptr', TYPE_INPUT.to, - [ TYPE_INPUT, posPhi ], - [ I32, I32.v(1) ]); - body.push(next); - - // Increment `state.index` - const index1 = IR._('add', [ TYPE_INDEX, indexPhi ], TYPE_INDEX.v(1)); - body.push(index1); - - // Check `state.index === seq.length` - const complete = IR._('icmp', [ 'eq', TYPE_INDEX, index1 ], - TYPE_INDEX.v(trie.select.length)); - body.push(complete); - const completeBranch = body.branch('br', [ IR.i(1), complete ]); + const iteration = this.buildSequenceIteration(info.fn, loop, seq, + indexField, posPhi, indexPhi); // It is complete! - completeBranch.left.push(reset()); - const pos = { current, next }; - this.buildSubTrie(info, completeBranch.left, pos, trie.children); - - // Not yet complete - body = completeBranch.right; - - // Loop if there is more data - const hasMore = IR._('icmp', [ 'ne', TYPE_INPUT, posPhi ], - info.fn.arg(ARG_ENDPOS)); - body.push(hasMore); - const hasMoreBranch = body.branch('br', [ IR.i(1), hasMore ]); + this.buildSubTrie(info, iteration.complete, iteration.pos, trie.children); // We have more data! - body = hasMoreBranch.left; - - body.loop('br', loop); - indexPhi.append([ '[', index1, ',', body.ref(), ']' ]); - posPhi.append([ '[', next, ',', body.ref(), ']' ]); - - // No more data (store `index`, not `index + 1`) - body = hasMoreBranch.right; - body.push(IR._('store', [ TYPE_INDEX, indexPhi ], - [ TYPE_INDEX.ptr(), indexField ]).void()); + iteration.loop.loop('br', loop); + indexPhi.append([ '[', iteration.index, ',', iteration.loop.ref(), ']' ]); + posPhi.append([ '[', iteration.pos.next, ',', iteration.loop.ref(), ']' ]); - this.buildSelfReturn(info.fn, body); + // Have to pause - return self + this.buildSelfReturn(info.fn, iteration.pause); // Not equal // Reset `state.index` on mismatch - branch.right.push(reset()); - return { pos, body: branch.right }; + return { pos: iteration.pos, body: iteration.otherwise }; } buildNext(info, body, pos, trie) { // Set `state.match` if needed if (trie.value !== null) { const matchField = this.field(info.fn, 'match'); + body.comment('state.match = ' + trie.value); body.push(matchField); body.push(IR._('store', [ TYPE_MATCH, TYPE_MATCH.v(trie.value) ], [ TYPE_MATCH.ptr(), matchField ]).void()); @@ -414,6 +451,7 @@ class Compiler { } buildRedirect(info, body, pos, target) { + body.comment('redirect'); const call = IR._('tail call fastcc', [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', From d23f44f594f1eb50c75393146a18cfd1b4a7b3d5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 16:06:46 -0500 Subject: [PATCH 045/281] trie: fix for falsey value --- lib/llparse/trie.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index 7ee2b0f..bcf0585 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -25,7 +25,7 @@ class Sequence extends Node { class Next extends Node { constructor(value, next) { super('next'); - this.value = value || null; + this.value = value === undefined ? null : value; this.next = next; } } From abacce51e929d802582699bb8c1b58c3594f0bb1 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 16:13:23 -0500 Subject: [PATCH 046/281] compiler: add assert --- lib/llparse/compiler.js | 2 ++ test/api-test.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 3fe9dac..1551544 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -397,6 +397,8 @@ class Compiler { // TODO(indutny): rework this into something readable buildSequence(info, body, trie) { + assert(!info.node.noAdvance); + const seq = this.ir.data(trie.select); // Load `state.index` diff --git a/test/api-test.js b/test/api-test.js index c078fe4..636c752 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -10,6 +10,8 @@ describe('LLParse', () => { const request = parse.node('req'); const response = parse.node('res'); + start.match(' ', start); + start.match('HTTP', parse.invoke('on_response', { 0: start }, parse.error(1, '`on_response` error'))); From 9c9d5f9898184f994b9bc3e13d8c39b0aed100d4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 17:22:09 -0500 Subject: [PATCH 047/281] test: tests! --- .gitignore | 2 +- lib/llparse/compiler.js | 12 +++- package-lock.json | 21 ++++++- package.json | 3 +- test/api-test.js | 15 +++-- test/fixtures/index.js | 61 +++++++++++++++++++ test/fixtures/main.c | 131 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 232 insertions(+), 13 deletions(-) create mode 100644 test/fixtures/index.js create mode 100644 test/fixtures/main.c diff --git a/.gitignore b/.gitignore index a7b58f8..fcb0295 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ node_modules/ npm-debug.log +test/tmp *.ll *.o -*.c main 1 diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 1551544..61d33de 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -41,6 +41,7 @@ class Compiler { this.state.field(TYPE_MATCH, 'match'); this.nodeMap = new Map(); + this.externalMap = new Map(); this.counter = new Map(); } @@ -167,8 +168,15 @@ class Compiler { } buildInvoke(info, body, pos, callback) { - const external = this.ir.declare(this.signature.callback, callback); - external.attributes = 'alwaysinline'; + let external; + if (this.externalMap.has(callback)) { + external = this.externalMap.get(callback); + } else { + external = this.ir.declare(this.signature.callback, callback); + external.attributes = 'alwaysinline'; + + this.externalMap.set(callback, external); + } const returnType = this.signature.callback.ret; diff --git a/package-lock.json b/package-lock.json index bd9426d..b13eed6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,15 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "async": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", + "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", + "dev": true, + "requires": { + "lodash": "4.17.5" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -119,9 +128,15 @@ "dev": true }, "llvm-ir": { - "version": "1.0.0-beta1", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta1.tgz", - "integrity": "sha512-91MRTS6o5x6GY4+7becoFQvdo1hbHyamZFLFnzMhGgYBftZv48No7DeMh8AOfiX5MJZDUqQHysJVI4kD0RVIqA==" + "version": "1.0.0-beta5", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta5.tgz", + "integrity": "sha512-C5eCCgtUtJSJ7tsLirdw9DgUF49suwlpUStW1PDw5YdH4+QhAb9yr3d/Ld3+/K7o3RtFmVhfytIZSOdprNghBQ==" + }, + "lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", + "dev": true }, "minimatch": { "version": "3.0.4", diff --git a/package.json b/package.json index 43bd978..5408655 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ "license": "MIT", "dependencies": { "esprima": "^4.0.0", - "llvm-ir": "^1.0.0-beta1" + "llvm-ir": "^1.0.0-beta5" }, "devDependencies": { + "async": "^2.6.0", "mocha": "^5.0.0" }, "directories": { diff --git a/test/api-test.js b/test/api-test.js index 636c752..4de5e9d 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -2,8 +2,10 @@ const llparse = require('../'); +const fixtures = require('./fixtures'); + describe('LLParse', () => { - it('should compile simple parser', () => { + it('should compile simple parser', (callback) => { const parse = llparse.create('llparse'); const start = parse.node('start'); @@ -12,7 +14,7 @@ describe('LLParse', () => { start.match(' ', start); - start.match('HTTP', parse.invoke('on_response', { + start.match('HTTP', parse.invoke('print_match', { 0: start }, parse.error(1, '`on_response` error'))); @@ -20,13 +22,14 @@ describe('LLParse', () => { 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, 'DELETE': 4, 'OPTIONS': 5, 'CONNECT': 6, 'TRACE': 7, 'PATCH': 8 - }, parse.invoke('on_request', { + }, parse.invoke('print_match', { 0: start - }, parse.error(2, '`on_request` error'))); + }, parse.error(2, '`print_match` error'))); start.otherwise(parse.error(3, 'Invalid word')); - const out = parse.build(start); - require('fs').writeFileSync('./2.ll', out); + const binary = fixtures.build('simple', parse.build(start)); + + binary('GET', 'off=3 match=1\n', callback); }); }); diff --git a/test/fixtures/index.js b/test/fixtures/index.js new file mode 100644 index 0000000..0187489 --- /dev/null +++ b/test/fixtures/index.js @@ -0,0 +1,61 @@ +'use strict'; + +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); +const spawn = require('child_process').spawn; +const spawnSync = require('child_process').spawnSync; +const Buffer = require('buffer').Buffer; + +const async = require('async'); + +const CLANG = process.env.CLANG || 'clang'; + +const MAIN = path.join(__dirname, 'main.c'); +const TMP_DIR = path.join(__dirname, '..', 'tmp'); + +const MAX_PARALLEL = 8; + +exports.build = (name, source) => { + try { + fs.mkdirSync(TMP_DIR); + } catch (e) { + } + + const file = path.join(TMP_DIR, name + '.ll'); + const out = path.join(TMP_DIR, name); + fs.writeFileSync(file, source); + + const res = spawnSync(CLANG, + [ '-flto', '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); + + return (input, expected, callback) => { + const buf = Buffer.from(input); + async.timesLimit(buf.length, MAX_PARALLEL, (i, callback) => { + const proc = spawn(out, [ i + 1, buf ], { + stdio: [ null, 'pipe', 'inherit' ] + }); + + let stdout = ''; + proc.stdout.on('data', chunk => stdout += chunk); + + async.parallel({ + exit: cb => proc.once('exit', (code) => cb(null, code)), + end: cb => proc.stdout.once('end', () => cb(null)) + }, (err, data) => { + if (data.exit !== 0) + return callback(new Error('Exit code: ' + data.exit)); + + callback(null, stdout); + }); + }, (err, results) => { + if (err) + return callback(err); + + for (let i = 0; i < results.length; i++) + assert.strictEqual(results[i], expected, 'Scan value: ' + (i + 1)) + + return callback(null); + }); + }; +}; diff --git a/test/fixtures/main.c b/test/fixtures/main.c new file mode 100644 index 0000000..77382a7 --- /dev/null +++ b/test/fixtures/main.c @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include + +struct state { + void* current; + int error; + const char* reason; + int index; + int match; +}; + +/* 2 gb */ +static const int64_t kBytes = 2147483648LL; + +static int bench = 0; +static const char* start; + +void llparse_init(struct state* s); +int llparse_execute(struct state* s, const char* p, const char* endp); + +int print_match(struct state* s, const char* p, const char* endp) { + if (bench) + return 0; + + fprintf(stdout, "off=%d match=%d\n", (int) (p - start), s->match); + return 0; +} + +int return_match(struct state* s, const char* p, const char* endp) { + if (bench) + return s->match; + + fprintf(stdout, "off=%d return match=%d\n", (int) (p - start), s->match); + return s->match; +} + +static int run_bench(const char* input, int len) { + struct state s; + int64_t i; + struct timeval start; + struct timeval end; + double bw; + double time; + int64_t iterations; + + llparse_init(&s); + + iterations = kBytes / (int64_t) len; + + gettimeofday(&start, NULL); + for (i = 0; i < iterations; i++) { + int code; + + code = llparse_execute(&s, input, input + len); + if (code != 0) + return code; + } + gettimeofday(&end, NULL); + + time = (end.tv_sec - start.tv_sec); + time += (end.tv_usec - start.tv_usec) * 1e-6; + bw = (double) kBytes / time; + + fprintf(stdout, "%.2f mb | %.2f mb/s | %.2f s\n", + (double) kBytes / (1024 * 1024), + bw / (1024 * 1024), + time); + + return 0; +} + + +static int run_scan(int scan, const char* input, int len) { + struct state s; + llparse_init(&s); + + if (scan <= 0) { + fprintf(stderr, "Invalid scan value\n"); + return -1; + } + + while (len > 0) { + int max; + int code; + + max = len > scan ? scan : len; + + code = llparse_execute(&s, input, input + max); + if (code != 0) { + fprintf(stderr, "code=%d error=%d reason=%s\n", code, s.error, s.reason); + return -1; + } + + input += max; + len -= max; + } + + return 0; +} + + +int main(int argc, char** argv) { + const char* input; + int len; + + if (argc < 3) { + fprintf(stderr, "%s [bench or scan-value] [input]\n", argv[0]); + return -1; + } + + if (strcmp(argv[1], "bench") == 0) + bench = 1; + + input = argv[2]; + len = strlen(input); + + if (bench && len == 0) { + fprintf(stderr, "Input can\'t be empty for benchmark"); + return -1; + } + + start = input; + + if (bench) + return run_bench(input, len); + else + return run_scan(atoi(argv[1]), input, len); +} From 2391c2a80d9532bf692fd986da5cef4836ba0f8b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 17:23:05 -0500 Subject: [PATCH 048/281] package: remove esprima --- package-lock.json | 5 ----- package.json | 1 - 2 files changed, 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index b13eed6..50f9566 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,11 +68,6 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, - "esprima": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", - "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==" - }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", diff --git a/package.json b/package.json index 5408655..f9b5659 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "esprima": "^4.0.0", "llvm-ir": "^1.0.0-beta5" }, "devDependencies": { From 15e6ba8479582cd8aa244391d6e55986d79a0455 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 17:51:41 -0500 Subject: [PATCH 049/281] compiler: skip, no-advance on otherwise --- README.md | 82 +++++++++++++++++++++++++++++++++++++++ lib/llparse.js | 4 ++ lib/llparse/compiler.js | 22 ++++++++--- lib/llparse/node/base.js | 6 +++ lib/llparse/node/index.js | 1 + lib/llparse/node/skip.js | 10 +++++ test/api-test.js | 67 +++++++++++++++++++++++++------- 7 files changed, 174 insertions(+), 18 deletions(-) create mode 100644 README.md create mode 100644 lib/llparse/node/skip.js diff --git a/README.md b/README.md new file mode 100644 index 0000000..85fb7b9 --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +# llparse +[![Build Status](https://secure.travis-ci.org/indutny/llparse.svg)](http://travis-ci.org/indutny/llparse) +[![NPM version](https://badge.fury.io/js/llparse.svg)](https://badge.fury.io/js/llparse) + +An API for generating parser in LLVM IR. + +## Usage + +```js +const p = require('llparse').create('http_parser'); + +const method = p.node('method'); +const beforeUrl = p.node('before_url'); +const url = p.node('url'); +const http = p.node('http'); + +// Invoke external C function +const onMethod = p.invoke('on_method', { + // If that function returns zero + 0: beforeUrl +}, p.error(1, '`on_method` error')); + +const urlStart = p.invoke('on_url_start', { + 0: url +}, p.error(2, '`on_url_start` error')); + +const urlEnd = p.invoke('on_url_end', { + 0: http +}, p.error(3, '`on_url_end` error')); + +const complete = p.invoke('on_complete', { + // Restart + 0: method +}, p.error(4, '`on_complete` error')); + +method + .select({ + 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, + 'DELETE': 4, 'OPTIONS': 5, 'CONNECT': 6, + 'TRACE': 7, 'PATCH': 8 + }, onMethod) + .otherwise(p.error(5, 'Expected method')); + +beforeUrl + .match(' ', beforeUrl); + .otherwise(urlStart); + +url + .match(' ', urlEnd) + .otherwise(p.skip()); + +http + .match('HTTP/1.1\r\n\r\n', complete) + .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); + +console.log(ir.build(method)); +``` + +#### LICENSE + +This software is licensed under the MIT License. + +Copyright Fedor Indutny, 2018. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/lib/llparse.js b/lib/llparse.js index 97bd5fc..19c1d50 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -25,6 +25,10 @@ class LLParse { return new internal.node.Invoke(name, map, otherwise); } + skip() { + return new internal.node.Skip(); + } + build(root) { const c = new internal.Compiler(this.prefix); return c.build(root); diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 61d33de..7cfa59d 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -46,7 +46,7 @@ class Compiler { } build(root) { - const rootFn = this.buildNode(root); + const rootFn = this.buildNode(root, null); this.buildInit(rootFn.ref()); this.buildParse(); @@ -139,7 +139,14 @@ class Compiler { return fn; } - buildNode(node) { + buildNode(node, from) { + if (node instanceof llparse.node.Skip) { + if (!from) + throw new Error('Skip node can\'t be a root state'); + + return this.buildNode(from); + } + if (this.nodeMap.has(node)) return this.nodeMap.get(node); @@ -194,7 +201,7 @@ class Compiler { s.cases.forEach((body, i) => { const subNode = info.node.map[keys[i]]; - this.buildRedirect(info, body, pos, this.buildNode(subNode)); + this.buildRedirect(info, body, pos, this.buildNode(subNode, info.node)); }); return s.otherwise; @@ -276,7 +283,12 @@ class Compiler { throw new Error('Unexpected trie node type: ' + trie.type); } - this.buildRedirect(info, body, pos, this.buildNode(info.otherwise)); + // Do not increment `pos` when falling through, unless we're skipping + const otherwisePos = { current: pos.current, next: pos.current }; + if (info.otherwise instanceof llparse.node.Skip) + otherwisePos.next = pos.next; + this.buildRedirect(info, body, otherwisePos, + this.buildNode(info.otherwise, info.node)); return body; } @@ -456,7 +468,7 @@ class Compiler { [ TYPE_MATCH.ptr(), matchField ]).void()); } - this.buildRedirect(info, body, pos, this.buildNode(trie.next)); + this.buildRedirect(info, body, pos, this.buildNode(trie.next, info.node)); return body; } diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 85e6ece..2348c06 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -14,16 +14,22 @@ class Node { match(value, next) { assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); this.cases.push(new llparse.case.Match(value, next)); + + return this; } select(map, next) { assert(next instanceof Node, 'Invalid `next` argument of `.select()`'); this.cases.push(new llparse.case.Select(map, next)); + + return this; } otherwise(next) { assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); this.cases.push(new llparse.case.Otherwise(next)); + + return this; } } module.exports = Node; diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js index d82dcdc..16f7751 100644 --- a/lib/llparse/node/index.js +++ b/lib/llparse/node/index.js @@ -3,3 +3,4 @@ exports.Node = require('./base'); exports.Error = require('./error'); exports.Invoke = require('./invoke'); +exports.Skip = require('./skip'); diff --git a/lib/llparse/node/skip.js b/lib/llparse/node/skip.js new file mode 100644 index 0000000..e82758e --- /dev/null +++ b/lib/llparse/node/skip.js @@ -0,0 +1,10 @@ +'use strict'; + +const llparse = require('./'); + +class Skip extends llparse.Node { + constructor() { + super('skip'); + } +} +module.exports = Skip; diff --git a/test/api-test.js b/test/api-test.js index 4de5e9d..2608e81 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -5,31 +5,72 @@ const llparse = require('../'); const fixtures = require('./fixtures'); describe('LLParse', () => { - it('should compile simple parser', (callback) => { - const parse = llparse.create('llparse'); + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); - const start = parse.node('start'); - const request = parse.node('req'); - const response = parse.node('res'); + const printMatch = (next) => { + return p.invoke('print_match', { + 0: next + }, p.error(1, '`print_match` error')); + }; + + it('should compile simple parser', (callback) => { + const start = p.node('start'); + const request = p.node('req'); + const response = p.node('res'); start.match(' ', start); - start.match('HTTP', parse.invoke('print_match', { - 0: start - }, parse.error(1, '`on_response` error'))); + start.match('HTTP', printMatch(start)); start.select({ 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, 'DELETE': 4, 'OPTIONS': 5, 'CONNECT': 6, 'TRACE': 7, 'PATCH': 8 - }, parse.invoke('print_match', { - 0: start - }, parse.error(2, '`print_match` error'))); + }, printMatch(start)); - start.otherwise(parse.error(3, 'Invalid word')); + start.otherwise(p.error(3, 'Invalid word')); - const binary = fixtures.build('simple', parse.build(start)); + const binary = fixtures.build('simple', p.build(start)); binary('GET', 'off=3 match=1\n', callback); }); + + describe('`.otherwise()`', () => { + it('should not advance position', (callback) => { + const p = llparse.create('llparse'); + + const a = p.node('a'); + const b = p.node('b'); + + a + .match('A', a) + .otherwise(b); + + b + .match('B', printMatch(b)) + .otherwise(a); + + + const binary = fixtures.build('otherwise-noadvance', p.build(a)); + + binary('AABAB', 'off=3 match=0\noff=5 match=0\n', callback); + }); + + it('should advance when it is `.skip()`', (callback) => { + const p = llparse.create('llparse'); + + const start = p.node('start'); + + start + .match(' ', printMatch(start)) + .otherwise(p.skip()); + + const binary = fixtures.build('otherwise-skip', p.build(start)); + + binary('HELLO WORLD', 'off=6 match=0\n', callback); + }); + }); }); From e2365dbfdfa4287ee5bedcf4c134a46e033e8603 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 18:06:15 -0500 Subject: [PATCH 050/281] lib: lint --- .eslintrc.js | 31 ++ lib/llparse/compiler.js | 11 +- lib/llparse/trie.js | 2 - package-lock.json | 1005 ++++++++++++++++++++++++++++++++++++++- package.json | 6 +- test/api-test.js | 3 +- test/fixtures/index.js | 5 +- 7 files changed, 1046 insertions(+), 17 deletions(-) create mode 100644 .eslintrc.js diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..595cf53 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,31 @@ +module.exports = { + 'env': { + 'browser': false, + 'commonjs': true, + 'es6': true, + 'node': true + }, + 'extends': 'eslint:recommended', + 'rules': { + 'max-len': [ 2, { + 'code': 80, + 'ignoreComments': true + } ], + 'indent': [ + 'error', + 2 + ], + 'linebreak-style': [ + 'error', + 'unix' + ], + 'quotes': [ + 'error', + 'single' + ], + 'semi': [ + 'error', + 'always' + ] + } +}; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 7cfa59d..043f42f 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -71,7 +71,7 @@ class Compiler { const store = (field, type, value) => { init.body.push(IR._('store', [ type, value ], [ type.ptr(), field ]).void()); - } + }; store(fields.current, fn.type, fn); store(fields.error, TYPE_ERROR, TYPE_ERROR.v(0)); @@ -298,7 +298,6 @@ class Compiler { cases.push(IR.label('otherwise')); cases.push('['); values.forEach((value, i) => { - const isLast = i === values.length - 1; cases.push(type, type.v(value)); cases.push(',', IR.label(`case_${i}`)); }); @@ -339,7 +338,7 @@ class Compiler { // Just a helper to reset `state.index` const reset = () => { return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], - [ TYPE_INDEX.ptr(), indexField ]).void(); + [ TYPE_INDEX.ptr(), indexField ]).void(); }; body.comment('current = *pos'); @@ -348,9 +347,9 @@ class Compiler { body.comment('expected = seq[state.index]'); const expectedPtr = IR._('getelementptr inbounds', seq.type.to, - [ seq.type, seq ], - [ I32, I32.v(0) ], - [ I32, index ]); + [ seq.type, seq ], + [ I32, I32.v(0) ], + [ I32, index ]); body.push(expectedPtr); const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); body.push(expected); diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index bcf0585..558fad8 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -99,8 +99,6 @@ class Trie { single(list) { const keys = new Map(); - let last = null; - let lastKey = null; for (let i = 0; i < list.length; i++) { const item = list[i]; const key = item.key[0]; diff --git a/package-lock.json b/package-lock.json index 50f9566..5875529 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,95 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "acorn": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.4.1.tgz", + "integrity": "sha512-XLmq3H/BVvW6/GbxKryGxWORz1ebilSsUDlyC27bXhWGWAZWkGwS6FLHjOlwFXNFoWFQEO/Df4u0YYd0K3BQgQ==", + "dev": true + }, + "acorn-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", + "integrity": "sha1-r9+UiPsezvyDSPb7IvRk4ypYs2s=", + "dev": true, + "requires": { + "acorn": "3.3.0" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=", + "dev": true + } + } + }, + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "dev": true, + "requires": { + "co": "4.6.0", + "fast-deep-equal": "1.0.0", + "fast-json-stable-stringify": "2.0.0", + "json-schema-traverse": "0.3.1" + } + }, + "ajv-keywords": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-2.1.1.tgz", + "integrity": "sha1-YXmX/F9gV2iUxDX5QNgZ4TW4B2I=", + "dev": true + }, + "ansi-escapes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.0.0.tgz", + "integrity": "sha512-O/klc27mWNUigtv0F8NJWbLF00OcegQalkqKURWdosW08YZKi4m6CnSUSvIZG1otNJbTWhN01Hhz389DW7mvDQ==", + "dev": true + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "1.0.3" + } + }, + "array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", + "dev": true, + "requires": { + "array-uniq": "1.0.3" + } + }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=", + "dev": true + }, + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true + }, "async": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", @@ -13,6 +102,47 @@ "lodash": "4.17.5" } }, + "babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", + "dev": true, + "requires": { + "chalk": "1.1.3", + "esutils": "2.0.2", + "js-tokens": "3.0.2" + }, + "dependencies": { + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "2.2.1", + "escape-string-regexp": "1.0.5", + "has-ansi": "2.0.0", + "strip-ansi": "3.0.1", + "supports-color": "2.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "2.1.1" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true + } + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -35,6 +165,106 @@ "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", "dev": true }, + "caller-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz", + "integrity": "sha1-lAhe9jWB7NPaqSREqP6U6CV3dR8=", + "dev": true, + "requires": { + "callsites": "0.2.0" + } + }, + "callsites": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz", + "integrity": "sha1-r6uWJikQp/M8GaV3WCXGnzTjUMo=", + "dev": true + }, + "chalk": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.1.tgz", + "integrity": "sha512-QUU4ofkDoMIVO7hcx1iPTISs88wsO8jA92RQIm4JAwZvFGGAV2hSAA1NX7oVj2Ej2Q6NDTcRDjPTFrMCRZoJ6g==", + "dev": true, + "requires": { + "ansi-styles": "3.2.0", + "escape-string-regexp": "1.0.5", + "supports-color": "5.2.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.0.tgz", + "integrity": "sha512-NnSOmMEYtVR2JVMIGTzynRkkaxtiq1xnFBcdQD/DnNCYPoEPsVJhM98BDyaoNOQIi7p4okdi3E27eN7GQbsUug==", + "dev": true, + "requires": { + "color-convert": "1.9.1" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "supports-color": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.2.0.tgz", + "integrity": "sha512-F39vS48la4YvTZUPVeTqsjsFNrvcMwrV3RLZINsmHo+7djCvuUzSIeXOnZ5hmjef4bajL1dNccN+tg5XAliO5Q==", + "dev": true, + "requires": { + "has-flag": "3.0.0" + } + } + } + }, + "chardet": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz", + "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=", + "dev": true + }, + "circular-json": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", + "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "requires": { + "restore-cursor": "2.0.0" + } + }, + "cli-width": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", + "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", + "dev": true + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", + "dev": true + }, + "color-convert": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz", + "integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, "commander": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", @@ -47,6 +277,34 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "concat-stream": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz", + "integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=", + "dev": true, + "requires": { + "inherits": "2.0.3", + "readable-stream": "2.3.4", + "typedarray": "0.0.6" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "4.1.1", + "shebang-command": "1.2.0", + "which": "1.3.0" + } + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -56,24 +314,228 @@ "ms": "2.0.0" } }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "del": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/del/-/del-2.2.2.tgz", + "integrity": "sha1-wSyYHQZ4RshLyvhiz/kw2Qf/0ag=", + "dev": true, + "requires": { + "globby": "5.0.0", + "is-path-cwd": "1.0.0", + "is-path-in-cwd": "1.0.0", + "object-assign": "4.1.1", + "pify": "2.3.0", + "pinkie-promise": "2.0.1", + "rimraf": "2.6.2" + } + }, "diff": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", "dev": true }, + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "requires": { + "esutils": "2.0.2" + } + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, + "eslint": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.18.0.tgz", + "integrity": "sha512-Ep2lUbztzXLg0gNUl48I1xvbQFy1QuWyh1C9PSympmln33jwOr8B3QfuEcXpPPE4uSwEzDaWhUxBN0sNQkzrBg==", + "dev": true, + "requires": { + "ajv": "5.5.2", + "babel-code-frame": "6.26.0", + "chalk": "2.3.1", + "concat-stream": "1.6.0", + "cross-spawn": "5.1.0", + "debug": "3.1.0", + "doctrine": "2.1.0", + "eslint-scope": "3.7.1", + "eslint-visitor-keys": "1.0.0", + "espree": "3.5.3", + "esquery": "1.0.0", + "esutils": "2.0.2", + "file-entry-cache": "2.0.0", + "functional-red-black-tree": "1.0.1", + "glob": "7.1.2", + "globals": "11.3.0", + "ignore": "3.3.7", + "imurmurhash": "0.1.4", + "inquirer": "3.3.0", + "is-resolvable": "1.1.0", + "js-yaml": "3.10.0", + "json-stable-stringify-without-jsonify": "1.0.1", + "levn": "0.3.0", + "lodash": "4.17.5", + "minimatch": "3.0.4", + "mkdirp": "0.5.1", + "natural-compare": "1.4.0", + "optionator": "0.8.2", + "path-is-inside": "1.0.2", + "pluralize": "7.0.0", + "progress": "2.0.0", + "require-uncached": "1.0.3", + "semver": "5.5.0", + "strip-ansi": "4.0.0", + "strip-json-comments": "2.0.1", + "table": "4.0.2", + "text-table": "0.2.0" + } + }, + "eslint-scope": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.1.tgz", + "integrity": "sha1-PWPD7f2gLgbgGkUq2IyqzHzctug=", + "dev": true, + "requires": { + "esrecurse": "4.2.0", + "estraverse": "4.2.0" + } + }, + "eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==", + "dev": true + }, + "espree": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.3.tgz", + "integrity": "sha512-Zy3tAJDORxQZLl2baguiRU1syPERAIg0L+JB2MWorORgTu/CplzvxS9WWA7Xh4+Q+eOQihNs/1o1Xep8cvCxWQ==", + "dev": true, + "requires": { + "acorn": "5.4.1", + "acorn-jsx": "3.0.1" + } + }, + "esprima": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", + "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==", + "dev": true + }, + "esquery": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.0.tgz", + "integrity": "sha1-z7qLV9f7qT8XKYqKAGoEzaE9gPo=", + "dev": true, + "requires": { + "estraverse": "4.2.0" + } + }, + "esrecurse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.0.tgz", + "integrity": "sha1-+pVo2Y04I/mkHZHpAtyrnqblsWM=", + "dev": true, + "requires": { + "estraverse": "4.2.0", + "object-assign": "4.1.1" + } + }, + "estraverse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=", + "dev": true + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "dev": true + }, + "external-editor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.1.0.tgz", + "integrity": "sha512-E44iT5QVOUJBKij4IIV3uvxuNlbKS38Tw1HiupxEIHPv9qtC2PrDYohbXV5U+1jnfIXttny8gUhj+oZvflFlzA==", + "dev": true, + "requires": { + "chardet": "0.4.2", + "iconv-lite": "0.4.19", + "tmp": "0.0.33" + } + }, + "fast-deep-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz", + "integrity": "sha1-liVqO8l1WV6zbYLpkp0GDYk0Of8=", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "requires": { + "escape-string-regexp": "1.0.5" + } + }, + "file-entry-cache": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz", + "integrity": "sha1-w5KZDD5oR4PYOLjISkXYoEhFg2E=", + "dev": true, + "requires": { + "flat-cache": "1.3.0", + "object-assign": "4.1.1" + } + }, + "flat-cache": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.0.tgz", + "integrity": "sha1-0wMLMrOBVPTjt+nHCfSQ9++XxIE=", + "dev": true, + "requires": { + "circular-json": "0.3.3", + "del": "2.2.2", + "graceful-fs": "4.1.11", + "write": "0.2.1" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", @@ -88,12 +550,47 @@ "path-is-absolute": "1.0.1" } }, + "globals": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.3.0.tgz", + "integrity": "sha512-kkpcKNlmQan9Z5ZmgqKH/SMbSmjxQ7QjyNqfXVc8VJcoBV2UEg+sxQD15GQofGRh2hfpwUb70VC31DR7Rq5Hdw==", + "dev": true + }, + "globby": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz", + "integrity": "sha1-69hGZ8oNuzMLmbz8aOrCvFQ3Dg0=", + "dev": true, + "requires": { + "array-union": "1.0.2", + "arrify": "1.0.1", + "glob": "7.1.2", + "object-assign": "4.1.1", + "pify": "2.3.0", + "pinkie-promise": "2.0.1" + } + }, + "graceful-fs": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", + "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", + "dev": true + }, "growl": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==", "dev": true }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, + "requires": { + "ansi-regex": "2.1.1" + } + }, "has-flag": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", @@ -106,6 +603,24 @@ "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", "dev": true }, + "iconv-lite": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==", + "dev": true + }, + "ignore": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.7.tgz", + "integrity": "sha512-YGG3ejvBNHRqu0559EOxxNFihD0AjpvHlC/pdGKd3X3ofe+CoJkYazwNJYTNebqpPKN+VVQbh4ZFn1DivMNuHA==", + "dev": true + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -122,10 +637,124 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", "dev": true }, + "inquirer": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz", + "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==", + "dev": true, + "requires": { + "ansi-escapes": "3.0.0", + "chalk": "2.3.1", + "cli-cursor": "2.1.0", + "cli-width": "2.2.0", + "external-editor": "2.1.0", + "figures": "2.0.0", + "lodash": "4.17.5", + "mute-stream": "0.0.7", + "run-async": "2.3.0", + "rx-lite": "4.0.8", + "rx-lite-aggregates": "4.0.8", + "string-width": "2.1.1", + "strip-ansi": "4.0.0", + "through": "2.3.8" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "is-path-cwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz", + "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=", + "dev": true + }, + "is-path-in-cwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.0.tgz", + "integrity": "sha1-ZHdYK4IU1gI0YJRWcAO+ip6sBNw=", + "dev": true, + "requires": { + "is-path-inside": "1.0.1" + } + }, + "is-path-inside": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", + "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=", + "dev": true, + "requires": { + "path-is-inside": "1.0.2" + } + }, + "is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", + "dev": true + }, + "is-resolvable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.1.0.tgz", + "integrity": "sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=", + "dev": true + }, + "js-yaml": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz", + "integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==", + "dev": true, + "requires": { + "argparse": "1.0.10", + "esprima": "4.0.0" + } + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=", + "dev": true + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dev": true, + "requires": { + "prelude-ls": "1.1.2", + "type-check": "0.3.2" + } + }, "llvm-ir": { - "version": "1.0.0-beta5", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta5.tgz", - "integrity": "sha512-C5eCCgtUtJSJ7tsLirdw9DgUF49suwlpUStW1PDw5YdH4+QhAb9yr3d/Ld3+/K7o3RtFmVhfytIZSOdprNghBQ==" + "version": "1.0.0-beta7", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta7.tgz", + "integrity": "sha512-kSRR3EitAYpeinavyJQuKWsvseezNySTdfpj2/g3qmgWh4ewyFa7x19FX/fAD804mwspA4mb0kYHDgpFnYbUzQ==" }, "lodash": { "version": "4.17.5", @@ -133,6 +762,22 @@ "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", "dev": true }, + "lru-cache": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.1.tgz", + "integrity": "sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==", + "dev": true, + "requires": { + "pseudomap": "1.0.2", + "yallist": "2.1.2" + } + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -181,6 +826,24 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "dev": true + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -190,12 +853,262 @@ "wrappy": "1.0.2" } }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "requires": { + "mimic-fn": "1.2.0" + } + }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "dev": true, + "requires": { + "deep-is": "0.1.3", + "fast-levenshtein": "2.0.6", + "levn": "0.3.0", + "prelude-ls": "1.1.2", + "type-check": "0.3.2", + "wordwrap": "1.0.0" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "dev": true + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, + "requires": { + "pinkie": "2.0.4" + } + }, + "pluralize": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-7.0.0.tgz", + "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==", + "dev": true + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "dev": true + }, + "progress": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", + "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=", + "dev": true + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, + "readable-stream": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.4.tgz", + "integrity": "sha512-vuYxeWYM+fde14+rajzqgeohAI7YoJcHE7kXDAc4Nk0EbuKnJfqtY9YtRkLo/tqkuF7MsBQRhPnPeyjYITp3ZQ==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.0.3", + "util-deprecate": "1.0.2" + } + }, + "require-uncached": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", + "dev": true, + "requires": { + "caller-path": "0.1.0", + "resolve-from": "1.0.1" + } + }, + "resolve-from": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz", + "integrity": "sha1-Jsv+k10a7uq7Kbw/5a6wHpPUQiY=", + "dev": true + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "requires": { + "onetime": "2.0.1", + "signal-exit": "3.0.2" + } + }, + "rimraf": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", + "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", + "dev": true, + "requires": { + "glob": "7.1.2" + } + }, + "run-async": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", + "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", + "dev": true, + "requires": { + "is-promise": "2.1.0" + } + }, + "rx-lite": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz", + "integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ=", + "dev": true + }, + "rx-lite-aggregates": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz", + "integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=", + "dev": true, + "requires": { + "rx-lite": "4.0.8" + } + }, + "safe-buffer": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==", + "dev": true + }, + "semver": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", + "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", + "dev": true + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "requires": { + "shebang-regex": "1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "slice-ansi": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz", + "integrity": "sha512-POqxBK6Lb3q6s047D/XsDVNPnF9Dl8JSaqe9h9lURl0OdNqy/ujDrOiIHtsqXMGbWWTIomRzAMaTyawAU//Reg==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "2.0.0" + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "2.0.0", + "strip-ansi": "4.0.0" + } + }, + "string_decoder": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "3.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + } + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + }, "supports-color": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", @@ -205,11 +1118,97 @@ "has-flag": "2.0.0" } }, + "table": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/table/-/table-4.0.2.tgz", + "integrity": "sha512-UUkEAPdSGxtRpiV9ozJ5cMTtYiqz7Ni1OGqLXRCynrvzdtR1p+cfOWe2RJLwvUG8hNanaSRjecIqwOjqeatDsA==", + "dev": true, + "requires": { + "ajv": "5.5.2", + "ajv-keywords": "2.1.1", + "chalk": "2.3.1", + "lodash": "4.17.5", + "slice-ansi": "1.0.0", + "string-width": "2.1.1" + } + }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true + }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "requires": { + "os-tmpdir": "1.0.2" + } + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dev": true, + "requires": { + "prelude-ls": "1.1.2" + } + }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", + "dev": true + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true + }, + "which": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz", + "integrity": "sha512-xcJpopdamTuY5duC/KnTTNBraPK54YwpenP4lzxU8H91GudWpFv38u0CKjclE1Wi2EH2EDz5LRcHcKbCIzqGyg==", + "dev": true, + "requires": { + "isexe": "2.0.0" + } + }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true + }, + "write": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz", + "integrity": "sha1-X8A4KOJkzqP+kUVUdvejxWbLB1c=", + "dev": true, + "requires": { + "mkdirp": "0.5.1" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true } } } diff --git a/package.json b/package.json index f9b5659..0d65b94 100644 --- a/package.json +++ b/package.json @@ -3,16 +3,18 @@ "version": "1.0.0", "main": "lib/llparse.js", "scripts": { - "test": "mocha --reporter=spec test/*-test.js" + "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", + "test": "mocha --reporter=spec test/*-test.js && npm run lint" }, "keywords": [], "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.0.0-beta5" + "llvm-ir": "^1.0.0-beta7" }, "devDependencies": { "async": "^2.6.0", + "eslint": "^4.18.0", "mocha": "^5.0.0" }, "directories": { diff --git a/test/api-test.js b/test/api-test.js index 2608e81..39207ed 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -1,4 +1,5 @@ 'use strict'; +/* global describe it beforeEach */ const llparse = require('../'); @@ -18,8 +19,6 @@ describe('LLParse', () => { it('should compile simple parser', (callback) => { const start = p.node('start'); - const request = p.node('req'); - const response = p.node('res'); start.match(' ', start); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 0187489..24f245f 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -20,13 +20,14 @@ exports.build = (name, source) => { try { fs.mkdirSync(TMP_DIR); } catch (e) { + // no-op } const file = path.join(TMP_DIR, name + '.ll'); const out = path.join(TMP_DIR, name); fs.writeFileSync(file, source); - const res = spawnSync(CLANG, + spawnSync(CLANG, [ '-flto', '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); return (input, expected, callback) => { @@ -53,7 +54,7 @@ exports.build = (name, source) => { return callback(err); for (let i = 0; i < results.length; i++) - assert.strictEqual(results[i], expected, 'Scan value: ' + (i + 1)) + assert.strictEqual(results[i], expected, 'Scan value: ' + (i + 1)); return callback(null); }); From 2cd67f1d0ca857801b19994717ff6d3f60168218 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 18:18:52 -0500 Subject: [PATCH 051/281] examples: add `http` --- .gitignore | 2 ++ README.md | 6 ++-- examples/http/Makefile | 9 ++++++ examples/http/index.js | 51 ++++++++++++++++++++++++++++++++ examples/http/main.c | 67 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 examples/http/Makefile create mode 100644 examples/http/index.js create mode 100644 examples/http/main.c diff --git a/.gitignore b/.gitignore index fcb0295..aac9824 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ node_modules/ npm-debug.log test/tmp +examples/http/http +*.dSYM *.ll *.o main diff --git a/README.md b/README.md index 85fb7b9..cf97b0e 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ An API for generating parser in LLVM IR. ## Usage ```js +'use strict'; + const p = require('llparse').create('http_parser'); const method = p.node('method'); @@ -42,7 +44,7 @@ method .otherwise(p.error(5, 'Expected method')); beforeUrl - .match(' ', beforeUrl); + .match(' ', beforeUrl) .otherwise(urlStart); url @@ -53,7 +55,7 @@ http .match('HTTP/1.1\r\n\r\n', complete) .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); -console.log(ir.build(method)); +console.log(p.build(method)); ``` #### LICENSE diff --git a/examples/http/Makefile b/examples/http/Makefile new file mode 100644 index 0000000..d4ecfa4 --- /dev/null +++ b/examples/http/Makefile @@ -0,0 +1,9 @@ +all: http + +http: main.c http.ll + clang -g3 -Os -flto -fvisibility=hidden -Wall http.ll main.c -o $@ + +http.ll: index.js + node $< > $@ + +.PHONY = all diff --git a/examples/http/index.js b/examples/http/index.js new file mode 100644 index 0000000..e50be11 --- /dev/null +++ b/examples/http/index.js @@ -0,0 +1,51 @@ +'use strict'; + +const p = require('../../').create('http_parser'); + +const method = p.node('method'); +const beforeUrl = p.node('before_url'); +const url = p.node('url'); +const http = p.node('http'); + +// Invoke external C function +const onMethod = p.invoke('on_method', { + // If that function returns zero + 0: beforeUrl +}, p.error(1, '`on_method` error')); + +const urlStart = p.invoke('on_url_start', { + 0: url +}, p.error(2, '`on_url_start` error')); + +const urlEnd = p.invoke('on_url_end', { + 0: http +}, p.error(3, '`on_url_end` error')); + +const complete = p.invoke('on_complete', { + // Restart + 0: method +}, p.error(4, '`on_complete` error')); + +method + .select({ + 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, + 'DELETE': 4, 'OPTIONS': 5, 'CONNECT': 6, + 'TRACE': 7, 'PATCH': 8 + }, onMethod) + .otherwise(p.error(5, 'Expected method')); + +beforeUrl + .match(' ', beforeUrl) + .otherwise(urlStart); + +url + .match(' ', urlEnd) + .otherwise(p.skip()); + +http + .match('HTTP/1.1\r\n\r\n', complete) + // Just for console testing + .match('HTTP/1.1\n\n', complete) + .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); + +console.log(p.build(method)); diff --git a/examples/http/main.c b/examples/http/main.c new file mode 100644 index 0000000..26c7eab --- /dev/null +++ b/examples/http/main.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include + +typedef struct http_parser_state_s http_parser_state_t; + +struct http_parser_state_s { + void* current; + int error; + const char* reason; + int index; + int match; +}; + +void http_parser_init(http_parser_state_t* s); +int http_parser_execute(http_parser_state_t* s, const char* p, + const char* endp); + +int on_method(http_parser_state_t* s, const char* p, const char* endp) { + fprintf(stdout, "on_method=%d\n", s->match); + return 0; +} + + +int on_url_start(http_parser_state_t* s, const char* p, const char* endp) { + fprintf(stdout, "on_url_start p=%s\n", p); + return 0; +} + + +int on_url_end(http_parser_state_t* s, const char* p, const char* endp) { + fprintf(stdout, "on_url_end p=%s\n", p); + return 0; +} + + +int on_complete(http_parser_state_t* s, const char* p, const char* endp) { + fprintf(stdout, "on_url_complete\n"); + return 0; +} + + +int main(int argc, char** argv) { + http_parser_state_t s; + + http_parser_init(&s); + + for (;;) { + char buf[16384]; + char* input; + int code; + + input = fgets(buf, sizeof(buf), stdin); + if (input == NULL) + break; + + code = http_parser_execute(&s, input, input + strlen(input)); + if (code != 0) { + fprintf(stderr, "code=%d error=%d reason=%s\n", code, s.error, s.reason); + return -1; + } + } + + return 0; +} From 0ee7f7ec0219adf4a168d9cf4ea70e7a1b6d0918 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 18:19:05 -0500 Subject: [PATCH 052/281] 1.0.0-beta0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5875529..f928645 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0", + "version": "1.0.0-beta0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 0d65b94..b172abf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0", + "version": "1.0.0-beta0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From a5ba9a92395ffab79c515af67730733ddb63c4e5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 18:29:56 -0500 Subject: [PATCH 053/281] compiler: add TODO --- lib/llparse/compiler.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 043f42f..3e5274e 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -334,6 +334,7 @@ class Compiler { return otherwise; } + // TODO(indutny): DRY this buildSequenceIteration(fn, body, seq, indexField, pos, index) { // Just a helper to reset `state.index` const reset = () => { From 7da282253bd3e107df73219e61d202da19c50d7b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 18:30:25 -0500 Subject: [PATCH 054/281] compiler: remove another todo --- lib/llparse/compiler.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 3e5274e..d3c1d0c 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -415,7 +415,6 @@ class Compiler { }; } - // TODO(indutny): rework this into something readable buildSequence(info, body, trie) { assert(!info.node.noAdvance); From a9fd1ee3b7b546fbf2dc148658a08dcc7388f3bc Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 18:56:08 -0500 Subject: [PATCH 055/281] api: `.skip()` instead of `p.skip()` --- examples/http/index.js | 2 +- lib/llparse/case/otherwise.js | 3 ++- lib/llparse/compiler.js | 26 ++++++++++++-------------- lib/llparse/node/base.js | 7 +++++++ lib/llparse/node/index.js | 1 - lib/llparse/node/skip.js | 10 ---------- test/api-test.js | 5 ++--- 7 files changed, 24 insertions(+), 30 deletions(-) delete mode 100644 lib/llparse/node/skip.js diff --git a/examples/http/index.js b/examples/http/index.js index e50be11..395b725 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -40,7 +40,7 @@ beforeUrl url .match(' ', urlEnd) - .otherwise(p.skip()); + .skip(url); http .match('HTTP/1.1\r\n\r\n', complete) diff --git a/lib/llparse/case/otherwise.js b/lib/llparse/case/otherwise.js index 9064a58..081ac6a 100644 --- a/lib/llparse/case/otherwise.js +++ b/lib/llparse/case/otherwise.js @@ -3,8 +3,9 @@ const Case = require('./').Case; class Otherwise extends Case { - constructor(next) { + constructor(next, skip = false) { super('otherwise', next); + this.skip = skip; } linearize() { diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index d3c1d0c..92ed466 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -46,7 +46,7 @@ class Compiler { } build(root) { - const rootFn = this.buildNode(root, null); + const rootFn = this.buildNode(root); this.buildInit(rootFn.ref()); this.buildParse(); @@ -139,14 +139,7 @@ class Compiler { return fn; } - buildNode(node, from) { - if (node instanceof llparse.node.Skip) { - if (!from) - throw new Error('Skip node can\'t be a root state'); - - return this.buildNode(from); - } - + buildNode(node) { if (this.nodeMap.has(node)) return this.nodeMap.get(node); @@ -165,7 +158,12 @@ class Compiler { node.cases.filter(c => c instanceof llparse.case.Otherwise); assert.strictEqual(otherwise.length, 1, `Node "${node.name}" must have only 1 \`.otherwise()\``); - const info = { node, fn, otherwise: otherwise[0].next }; + const info = { + node, + fn, + otherwise: otherwise[0].next, + skip: otherwise[0].skip + }; const trie = this.trie.combine(node.cases); @@ -201,7 +199,7 @@ class Compiler { s.cases.forEach((body, i) => { const subNode = info.node.map[keys[i]]; - this.buildRedirect(info, body, pos, this.buildNode(subNode, info.node)); + this.buildRedirect(info, body, pos, this.buildNode(subNode)); }); return s.otherwise; @@ -285,10 +283,10 @@ class Compiler { // Do not increment `pos` when falling through, unless we're skipping const otherwisePos = { current: pos.current, next: pos.current }; - if (info.otherwise instanceof llparse.node.Skip) + if (info.skip) otherwisePos.next = pos.next; this.buildRedirect(info, body, otherwisePos, - this.buildNode(info.otherwise, info.node)); + this.buildNode(info.otherwise)); return body; } @@ -467,7 +465,7 @@ class Compiler { [ TYPE_MATCH.ptr(), matchField ]).void()); } - this.buildRedirect(info, body, pos, this.buildNode(trie.next, info.node)); + this.buildRedirect(info, body, pos, this.buildNode(trie.next)); return body; } diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 2348c06..1fac686 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -31,5 +31,12 @@ class Node { return this; } + + skip(next) { + assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); + this.cases.push(new llparse.case.Otherwise(next, true)); + + return this; + } } module.exports = Node; diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js index 16f7751..d82dcdc 100644 --- a/lib/llparse/node/index.js +++ b/lib/llparse/node/index.js @@ -3,4 +3,3 @@ exports.Node = require('./base'); exports.Error = require('./error'); exports.Invoke = require('./invoke'); -exports.Skip = require('./skip'); diff --git a/lib/llparse/node/skip.js b/lib/llparse/node/skip.js deleted file mode 100644 index e82758e..0000000 --- a/lib/llparse/node/skip.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const llparse = require('./'); - -class Skip extends llparse.Node { - constructor() { - super('skip'); - } -} -module.exports = Skip; diff --git a/test/api-test.js b/test/api-test.js index 39207ed..935addf 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -38,7 +38,7 @@ describe('LLParse', () => { }); describe('`.otherwise()`', () => { - it('should not advance position', (callback) => { + it('should not advance position by default', (callback) => { const p = llparse.create('llparse'); const a = p.node('a'); @@ -52,7 +52,6 @@ describe('LLParse', () => { .match('B', printMatch(b)) .otherwise(a); - const binary = fixtures.build('otherwise-noadvance', p.build(a)); binary('AABAB', 'off=3 match=0\noff=5 match=0\n', callback); @@ -65,7 +64,7 @@ describe('LLParse', () => { start .match(' ', printMatch(start)) - .otherwise(p.skip()); + .skip(start); const binary = fixtures.build('otherwise-skip', p.build(start)); From c7e9fd6e65dddc86a015fb1738f675eed794229e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 19:00:48 -0500 Subject: [PATCH 056/281] api: `skip` => `skipTo` --- README.md | 2 +- examples/http/index.js | 2 +- lib/llparse/node/base.js | 2 +- test/api-test.js | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cf97b0e..734d44d 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ beforeUrl url .match(' ', urlEnd) - .otherwise(p.skip()); + .skipTo(url); http .match('HTTP/1.1\r\n\r\n', complete) diff --git a/examples/http/index.js b/examples/http/index.js index 395b725..1366f3e 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -40,7 +40,7 @@ beforeUrl url .match(' ', urlEnd) - .skip(url); + .skipTo(url); http .match('HTTP/1.1\r\n\r\n', complete) diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 1fac686..2135601 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -32,7 +32,7 @@ class Node { return this; } - skip(next) { + skipTo(next) { assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); this.cases.push(new llparse.case.Otherwise(next, true)); diff --git a/test/api-test.js b/test/api-test.js index 935addf..ab74397 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -57,14 +57,14 @@ describe('LLParse', () => { binary('AABAB', 'off=3 match=0\noff=5 match=0\n', callback); }); - it('should advance when it is `.skip()`', (callback) => { + it('should advance when it is `.skipTo()`', (callback) => { const p = llparse.create('llparse'); const start = p.node('start'); start .match(' ', printMatch(start)) - .skip(start); + .skipTo(start); const binary = fixtures.build('otherwise-skip', p.build(start)); From 33d24cb1fa14cdb82ff1f58a8af1b65f49d9f0f5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 17 Feb 2018 21:47:20 -0500 Subject: [PATCH 057/281] example/http: use pointers to emit URL --- examples/http/main.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/examples/http/main.c b/examples/http/main.c index 26c7eab..bf7d9d0 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -12,6 +12,8 @@ struct http_parser_state_s { const char* reason; int index; int match; + + const char* url_start; }; void http_parser_init(http_parser_state_t* s); @@ -24,20 +26,30 @@ int on_method(http_parser_state_t* s, const char* p, const char* endp) { } +static void on_url_part(const char* p, const char* endp) { + if (p == endp) + return; + + fprintf(stdout, "url_part=\"%.*s\"\n", (int) (endp - p), p); +} + + int on_url_start(http_parser_state_t* s, const char* p, const char* endp) { - fprintf(stdout, "on_url_start p=%s\n", p); + s->url_start = p; return 0; } int on_url_end(http_parser_state_t* s, const char* p, const char* endp) { - fprintf(stdout, "on_url_end p=%s\n", p); + on_url_part(s->url_start, p - 1); + fprintf(stdout, "url end\n"); + s->url_start = NULL; return 0; } int on_complete(http_parser_state_t* s, const char* p, const char* endp) { - fprintf(stdout, "on_url_complete\n"); + fprintf(stdout, "on_complete\n"); return 0; } @@ -47,20 +59,30 @@ int main(int argc, char** argv) { http_parser_init(&s); + s.url_start = NULL; + for (;;) { char buf[16384]; - char* input; + const char* input; + const char* endp; int code; input = fgets(buf, sizeof(buf), stdin); if (input == NULL) break; - code = http_parser_execute(&s, input, input + strlen(input)); + if (s.url_start != NULL) + s.url_start = input; + + endp = input + strlen(input); + code = http_parser_execute(&s, input, endp); if (code != 0) { fprintf(stderr, "code=%d error=%d reason=%s\n", code, s.error, s.reason); return -1; } + + if (s.url_start != NULL) + on_url_part(s.url_start, endp); } return 0; From 1e6703b5d458ff41c3678211600f104e43412850 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 12:50:38 -0500 Subject: [PATCH 058/281] compiler: move calling convention to a const --- lib/llparse/compiler.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 92ed466..0252439 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -5,6 +5,8 @@ const llparse = require('./'); const assert = require('assert'); const IR = require('llvm-ir'); +const CCONV = 'fastcc'; + const I32 = IR.i(32); const TYPE_INPUT = IR.i(8).ptr(); const TYPE_OUTPUT = IR.i(8).ptr(); @@ -100,7 +102,7 @@ class Compiler { [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); - const call = IR._('call fastcc', [ + const call = IR._(`call ${CCONV}`, [ TYPE_OUTPUT, current, '(', this.state.ptr(), parse.arg(ARG_STATE), ',', TYPE_INPUT, parse.arg(ARG_POS), ',', @@ -135,7 +137,7 @@ class Compiler { const fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - fn.visibility = 'internal fastcc'; + fn.visibility = `internal ${CCONV}`; return fn; } @@ -471,7 +473,7 @@ class Compiler { buildRedirect(info, body, pos, target) { body.comment('redirect'); - const call = IR._('tail call fastcc', [ + const call = IR._(`tail call ${CCONV}`, [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', From b3f01f310d662a3aadf99b412189eae8706da978 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 13:44:19 -0500 Subject: [PATCH 059/281] compiler: add `optsize` for 22% speed improvement --- lib/llparse/compiler.js | 7 ++++++- test/fixtures/index.js | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 0252439..9c7e78b 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -138,6 +138,7 @@ class Compiler { const fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); fn.visibility = `internal ${CCONV}`; + fn.attributes = 'nounwind optsize'; return fn; } @@ -254,7 +255,10 @@ class Compiler { // Increment `pos` if not invoking external callback let pos = { current: fn.arg(ARG_POS), next: null }; - if (info.node.noAdvance) { + // NOTE: `sequence` has loop inside it - so it isn't going to use + // `pos.next` anyway (as a matter of fact it doesn't get `pos` as an + // argument at all) + if (info.node.noAdvance || trie.type === 'sequence') { pos.next = pos.current; } else { body.comment('next = pos + 1'); @@ -273,6 +277,7 @@ class Compiler { } else if (trie.type === 'single') { body = this.buildSingle(info, body, pos, trie.children); } else if (trie.type === 'sequence') { + // NOTE: do not send `pos` here! (see comment above) const seq = this.buildSequence(info, body, trie); // NOTE: `sequence` implementation loops if there's enough data diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 24f245f..f36ea9b 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -27,8 +27,13 @@ exports.build = (name, source) => { const out = path.join(TMP_DIR, name); fs.writeFileSync(file, source); - spawnSync(CLANG, + const ret = spawnSync(CLANG, [ '-flto', '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); + if (ret.status !== 0) { + process.stderr.write(ret.stdout); + process.stderr.write(ret.stderr); + throw new Error('clang exit code=' + ret.status); + } return (input, expected, callback) => { const buf = Buffer.from(input); From 5a373d346778707881e281ded4eb51c1ff0ae4f4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 13:45:22 -0500 Subject: [PATCH 060/281] package: bump `llvm-ir` to beta9 --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index f928645..7ad96a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -752,9 +752,9 @@ } }, "llvm-ir": { - "version": "1.0.0-beta7", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta7.tgz", - "integrity": "sha512-kSRR3EitAYpeinavyJQuKWsvseezNySTdfpj2/g3qmgWh4ewyFa7x19FX/fAD804mwspA4mb0kYHDgpFnYbUzQ==" + "version": "1.0.0-beta9", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta9.tgz", + "integrity": "sha512-/eiFLpKJchP6Frh9ZvBC9roOUZ+VqoBG9rgZMGXOd3Fg1g0yFLDtgLPjx9gbE2MBJqevz12Hd0YSUdtZjeIN2g==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index b172abf..bffc02d 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.0.0-beta7" + "llvm-ir": "^1.0.0-beta9" }, "devDependencies": { "async": "^2.6.0", From 52b0aebde23afd98b41fe4c2d9ae86ed7f8e6cbd Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 13:56:54 -0500 Subject: [PATCH 061/281] compiler: generate less insane labels in switch --- lib/llparse/compiler.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 9c7e78b..526aef5 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -311,8 +311,10 @@ class Compiler { const blocks = body.terminate('switch', [ type, what ], cases); blocks[0].name = 'switch_otherwise'; - for (let i = 0; i< values.length; i++) - blocks[i + 1].name = 'case_' + values[i]; + for (let i = 0; i< values.length; i++) { + const v = values[i] < 0 ? 'm' + (-values[i]) : values[i]; + blocks[i + 1].name = 'case_' + v; + } return { otherwise: blocks[0], From 4b83815f12e8a6e06bc2fdee2524868d9c64c606 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 14:39:26 -0500 Subject: [PATCH 062/281] compiler: `musttail`, cache self return --- lib/llparse/compiler.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 526aef5..815c622 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -45,6 +45,8 @@ class Compiler { this.nodeMap = new Map(); this.externalMap = new Map(); this.counter = new Map(); + + this.selfReturnCache = new Map(); } build(root) { @@ -237,13 +239,22 @@ class Compiler { // Return self when `pos === endpos` branch.right.name = 'prologue_end'; - this.buildSelfReturn(fn, branch.right); + this.buildSelfReturn(fn, branch.right, true); branch.left.name = 'prologue_normal'; return branch.left; } - buildSelfReturn(fn, body) { + buildSelfReturn(fn, body, cache) { + if (this.selfReturnCache.has(fn)) { + const target = this.selfReturnCache.get(fn); + body.terminate('br', target); + return; + } + + if (cache) + this.selfReturnCache.set(fn, body); + const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); body.push(bitcast); body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); @@ -480,7 +491,7 @@ class Compiler { buildRedirect(info, body, pos, target) { body.comment('redirect'); - const call = IR._(`tail call ${CCONV}`, [ + const call = IR._(`musttail call ${CCONV}`, [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', From f6d7124d2e49b5b052dec39c6ecbb9410cf7e26d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 15:06:49 -0500 Subject: [PATCH 063/281] compiler: add performance todo --- lib/llparse/compiler.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 815c622..8a2ad82 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -491,6 +491,9 @@ class Compiler { buildRedirect(info, body, pos, target) { body.comment('redirect'); + + // TODO(indutny): looks like `musttail` gives worse performance when calling + // Invoke nodes (possibly others too). const call = IR._(`musttail call ${CCONV}`, [ TYPE_OUTPUT, target, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', From 4134f61fad1d4c667b1d3e4242f4c43066fff883 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 15:09:13 -0500 Subject: [PATCH 064/281] test/main: increase data volume for bench --- test/fixtures/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 77382a7..5c0af5b 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -12,8 +12,8 @@ struct state { int match; }; -/* 2 gb */ -static const int64_t kBytes = 2147483648LL; +/* 8 gb */ +static const int64_t kBytes = 8589934592LL; static int bench = 0; static const char* start; From 8fa2678f18b15bd83bfffdccdcef7f3a3d9c314b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 15:12:14 -0500 Subject: [PATCH 065/281] 1.0.0-beta1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7ad96a9..71f1a52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0-beta0", + "version": "1.0.0-beta1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index bffc02d..e239136 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0-beta0", + "version": "1.0.0-beta1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 80e7a7844ff33595da2217966e5cecbf8be01baa Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 15:16:52 -0500 Subject: [PATCH 066/281] compiler: fix over-reliance on `trie` presence --- lib/llparse/compiler.js | 2 +- test/api-test.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 8a2ad82..427c576 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -269,7 +269,7 @@ class Compiler { // NOTE: `sequence` has loop inside it - so it isn't going to use // `pos.next` anyway (as a matter of fact it doesn't get `pos` as an // argument at all) - if (info.node.noAdvance || trie.type === 'sequence') { + if (info.node.noAdvance || trie && trie.type === 'sequence') { pos.next = pos.current; } else { body.comment('next = pos + 1'); diff --git a/test/api-test.js b/test/api-test.js index ab74397..6b1c564 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -70,5 +70,18 @@ describe('LLParse', () => { binary('HELLO WORLD', 'off=6 match=0\n', callback); }); + + it('should skip everything with `.skipTo()`', (callback) => { + const p = llparse.create('llparse'); + + const start = p.node('start'); + + start + .skipTo(start); + + const binary = fixtures.build('all-skip', p.build(start)); + + binary('HELLO WORLD', '', callback); + }); }); }); From dc31e9c5cf8bace91856139591990131e78e9e6f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 15:17:17 -0500 Subject: [PATCH 067/281] 1.0.0-beta2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 71f1a52..6112b57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0-beta1", + "version": "1.0.0-beta2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e239136..ada27c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0-beta1", + "version": "1.0.0-beta2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 8d36c2e372462683f68075e0e35b578fafa430a7 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 16:03:55 -0500 Subject: [PATCH 068/281] compiler: specify parameter attributes --- lib/llparse/compiler.js | 4 +++- package-lock.json | 6 +++--- package.json | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 427c576..49d9971 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -29,7 +29,9 @@ class Compiler { this.signature = { node: this.ir.signature(TYPE_OUTPUT, [ - this.state.ptr(), TYPE_INPUT, TYPE_INPUT + [ this.state.ptr(), 'noalias nocapture nonnull' ], + [ TYPE_INPUT, 'noalias nocapture nonnull readonly' ], + [ TYPE_INPUT, 'noalias nocapture nonnull readnone' ] ]), callback: this.ir.signature(I32, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT diff --git a/package-lock.json b/package-lock.json index 6112b57..36d4dd6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -752,9 +752,9 @@ } }, "llvm-ir": { - "version": "1.0.0-beta9", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.0-beta9.tgz", - "integrity": "sha512-/eiFLpKJchP6Frh9ZvBC9roOUZ+VqoBG9rgZMGXOd3Fg1g0yFLDtgLPjx9gbE2MBJqevz12Hd0YSUdtZjeIN2g==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.1.tgz", + "integrity": "sha512-8YRbJlkx63gSOTwLECRssK/SlJ5loihCb9/X/BTWWB2KMP5KQ/ZoGKXHv5G+nEEeDBg+wZSff9/7SvFHC/dPbA==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index ada27c7..ff7f84b 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.0.0-beta9" + "llvm-ir": "^1.0.1" }, "devDependencies": { "async": "^2.6.0", From 7a680a4af27ace585b250904b277928853d36030 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 16:09:38 -0500 Subject: [PATCH 069/281] 1.0.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 36d4dd6..1df138d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0-beta2", + "version": "1.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index ff7f84b..e27b0d5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0-beta2", + "version": "1.0.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From f085fef07c70c8af8b88bdaabd81697437ce2a02 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 16:18:05 -0500 Subject: [PATCH 070/281] compiler: regain performance after `musttail` --- lib/llparse/compiler.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 49d9971..b0e5899 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -142,7 +142,14 @@ class Compiler { const fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); fn.visibility = `internal ${CCONV}`; - fn.attributes = 'nounwind optsize'; + + // TODO(indutny): reassess `minsize`. Looks like it gives best performance + // results right now, though. + fn.attributes = 'nounwind minsize'; + + // Errors are assumed to be rarely called + if (node instanceof llparse.node.Error) + fn.attributes += ' cold'; return fn; } From fd75447e8f90a7cfafd61a591f4968ddcfd342a0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 16:21:22 -0500 Subject: [PATCH 071/281] compiler: add `writeonly` attribute to errors --- lib/llparse/compiler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index b0e5899..d22a57c 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -149,7 +149,7 @@ class Compiler { // Errors are assumed to be rarely called if (node instanceof llparse.node.Error) - fn.attributes += ' cold'; + fn.attributes += ' cold writeonly'; return fn; } From fcedfda7c6c091d055d181290f796ae0f5414ba2 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 16:22:16 -0500 Subject: [PATCH 072/281] 1.0.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1df138d..50d92df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e27b0d5..c5f2c81 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.0", + "version": "1.0.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 2433e2a979824ae8a8f8fafe60b2a8fdaef911f8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 16:48:57 -0500 Subject: [PATCH 073/281] compiler: move constants to a file --- lib/llparse/compiler.js | 131 ++++++++++++++++++++------------------- lib/llparse/constants.js | 18 ++++++ lib/llparse/index.js | 2 + 3 files changed, 87 insertions(+), 64 deletions(-) create mode 100644 lib/llparse/constants.js diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index d22a57c..9403896 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -1,23 +1,25 @@ 'use strict'; const llparse = require('./'); +const constants = llparse.constants; const assert = require('assert'); const IR = require('llvm-ir'); -const CCONV = 'fastcc'; +const CCONV = constants.CCONV; -const I32 = IR.i(32); -const TYPE_INPUT = IR.i(8).ptr(); -const TYPE_OUTPUT = IR.i(8).ptr(); -const TYPE_MATCH = I32; -const TYPE_INDEX = I32; -const TYPE_ERROR = I32; -const TYPE_REASON = IR.i(8).ptr(); +const BOOL = constants.BOOL; +const INT = constants.INT; +const TYPE_INPUT = constants.TYPE_INPUT; +const TYPE_OUTPUT = constants.TYPE_OUTPUT; +const TYPE_MATCH = constants.TYPE_MATCH; +const TYPE_INDEX = constants.TYPE_INDEX; +const TYPE_ERROR = constants.TYPE_ERROR; +const TYPE_REASON = constants.TYPE_REASON; -const ARG_STATE = 's'; -const ARG_POS = 'p'; -const ARG_ENDPOS = 'endp'; +const ARG_STATE = constants.ARG_STATE; +const ARG_POS = constants.ARG_POS; +const ARG_ENDPOS = constants.ARG_ENDPOS; class Compiler { constructor(prefix) { @@ -33,7 +35,7 @@ class Compiler { [ TYPE_INPUT, 'noalias nocapture nonnull readonly' ], [ TYPE_INPUT, 'noalias nocapture nonnull readnone' ] ]), - callback: this.ir.signature(I32, [ + callback: this.ir.signature(INT, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]) }; @@ -244,7 +246,7 @@ class Compiler { fn.arg(ARG_ENDPOS)); fn.body.push(cmp); - const branch = fn.body.branch('br', [ IR.i(1), cmp ]); + const branch = fn.body.branch('br', [ BOOL, cmp ]); // Return self when `pos === endpos` branch.right.name = 'prologue_end'; @@ -284,7 +286,7 @@ class Compiler { body.comment('next = pos + 1'); pos.next = IR._('getelementptr', TYPE_INPUT.to, [ TYPE_INPUT, fn.arg(ARG_POS) ], - [ I32, I32.v(1) ]); + [ INT, INT.v(1) ]); body.push(pos.next); } @@ -361,6 +363,49 @@ class Compiler { return otherwise; } + buildSequence(info, body, trie) { + assert(!info.node.noAdvance); + + const seq = this.ir.data(trie.select); + + // Load `state.index` + body.comment('index = state.index'); + const indexField = this.field(info.fn, 'index'); + body.push(indexField); + const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexField ]); + body.push(index); + + const loop = body.jump('br'); + loop.name = 'loop'; + + // Loop start + const posPhi = IR._('phi', + [ TYPE_INPUT, '[', info.fn.arg(ARG_POS), ',', body.ref(), ']' ]); + loop.push(posPhi); + const indexPhi = IR._('phi', + [ TYPE_INDEX, '[', index, ',', body.ref(), ']' ]); + loop.push(indexPhi); + + const iteration = this.buildSequenceIteration(info.fn, loop, seq, + indexField, posPhi, indexPhi); + + // It is complete! + this.buildSubTrie(info, iteration.complete, iteration.pos, trie.children); + + // We have more data! + iteration.loop.loop('br', loop); + indexPhi.append([ '[', iteration.index, ',', iteration.loop.ref(), ']' ]); + posPhi.append([ '[', iteration.pos.next, ',', iteration.loop.ref(), ']' ]); + + // Have to pause - return self + this.buildSelfReturn(info.fn, iteration.pause); + + // Not equal + // Reset `state.index` on mismatch + return { pos: iteration.pos, body: iteration.otherwise }; + } + + // TODO(indutny): DRY this buildSequenceIteration(fn, body, seq, indexField, pos, index) { // Just a helper to reset `state.index` @@ -376,8 +421,8 @@ class Compiler { body.comment('expected = seq[state.index]'); const expectedPtr = IR._('getelementptr inbounds', seq.type.to, [ seq.type, seq ], - [ I32, I32.v(0) ], - [ I32, index ]); + [ INT, INT.v(0) ], + [ INT, index ]); body.push(expectedPtr); const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); body.push(expected); @@ -386,7 +431,7 @@ class Compiler { let cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); body.push(cmp); const { left: isMatch, right: isMismatch } = - body.branch('br', [ IR.i(1), cmp ]); + body.branch('br', [ BOOL, cmp ]); // Mismatch isMismatch.name = 'mismatch'; @@ -400,7 +445,7 @@ class Compiler { isMatch.comment('next = pos + 1'); const next = IR._('getelementptr', TYPE_INPUT.to, [ TYPE_INPUT, pos ], - [ I32, I32.v(1) ]); + [ INT, INT.v(1) ]); isMatch.push(next); isMatch.comment('index1 = index + 1'); @@ -412,7 +457,7 @@ class Compiler { TYPE_INDEX.v(seq.type.to.length)); isMatch.push(cmp); const { left: isComplete, right: isIncomplete } = - isMatch.branch('br', [ IR.i(1), cmp ]); + isMatch.branch('br', [ BOOL, cmp ]); isComplete.name = 'is_complete'; isComplete.comment('state.index = 0'); @@ -423,7 +468,7 @@ class Compiler { cmp = IR._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); isIncomplete.push(cmp); const { left: moreData, right: noMoreData } = - isIncomplete.branch('br', [ IR.i(1), cmp ]); + isIncomplete.branch('br', [ BOOL, cmp ]); moreData.name = 'more_data'; @@ -442,48 +487,6 @@ class Compiler { }; } - buildSequence(info, body, trie) { - assert(!info.node.noAdvance); - - const seq = this.ir.data(trie.select); - - // Load `state.index` - body.comment('index = state.index'); - const indexField = this.field(info.fn, 'index'); - body.push(indexField); - const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexField ]); - body.push(index); - - const loop = body.jump('br'); - loop.name = 'loop'; - - // Loop start - const posPhi = IR._('phi', - [ TYPE_INPUT, '[', info.fn.arg(ARG_POS), ',', body.ref(), ']' ]); - loop.push(posPhi); - const indexPhi = IR._('phi', - [ TYPE_INDEX, '[', index, ',', body.ref(), ']' ]); - loop.push(indexPhi); - - const iteration = this.buildSequenceIteration(info.fn, loop, seq, - indexField, posPhi, indexPhi); - - // It is complete! - this.buildSubTrie(info, iteration.complete, iteration.pos, trie.children); - - // We have more data! - iteration.loop.loop('br', loop); - indexPhi.append([ '[', iteration.index, ',', iteration.loop.ref(), ']' ]); - posPhi.append([ '[', iteration.pos.next, ',', iteration.loop.ref(), ']' ]); - - // Have to pause - return self - this.buildSelfReturn(info.fn, iteration.pause); - - // Not equal - // Reset `state.index` on mismatch - return { pos: iteration.pos, body: iteration.otherwise }; - } - buildNext(info, body, pos, trie) { // Set `state.match` if needed if (trie.value !== null) { @@ -543,8 +546,8 @@ class Compiler { return IR._('getelementptr', this.state, [ stateArg.type, stateArg ], - [ I32, I32.v(0) ], - [ I32, I32.v(this.state.lookup(name)) ]); + [ INT, INT.v(0) ], + [ INT, INT.v(this.state.lookup(name)) ]); } } module.exports = Compiler; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js new file mode 100644 index 0000000..295e99b --- /dev/null +++ b/lib/llparse/constants.js @@ -0,0 +1,18 @@ +'use strict'; + +const IR = require('llvm-ir'); + +exports.CCONV = 'fastcc'; + +exports.BOOL = IR.i(1); +exports.INT = IR.i(32); +exports.TYPE_INPUT = IR.i(8).ptr(); +exports.TYPE_OUTPUT = IR.i(8).ptr(); +exports.TYPE_MATCH = exports.INT; +exports.TYPE_INDEX = exports.INT; +exports.TYPE_ERROR = exports.INT; +exports.TYPE_REASON = IR.i(8).ptr(); + +exports.ARG_STATE = 's'; +exports.ARG_POS = 'p'; +exports.ARG_ENDPOS = 'endp'; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index c8f40a3..f08d18e 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -1,5 +1,7 @@ 'use strict'; +exports.constants = require('./constants'); + exports.case = require('./case'); exports.node = require('./node'); exports.Trie = require('./trie'); From b557bb7eeca41a0519c3461c357b76cabe3c75fe Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 18:40:18 -0500 Subject: [PATCH 074/281] compiler: dry sequence matching code --- lib/llparse/compiler.js | 172 ++++++++++----------------- lib/llparse/constants.js | 11 ++ lib/llparse/index.js | 1 + lib/llparse/match-sequence.js | 211 ++++++++++++++++++++++++++++++++++ package-lock.json | 6 +- package.json | 2 +- test/fixtures/index.js | 2 +- 7 files changed, 290 insertions(+), 115 deletions(-) create mode 100644 lib/llparse/match-sequence.js diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 9403896..c7d0dae 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -1,11 +1,12 @@ 'use strict'; -const llparse = require('./'); -const constants = llparse.constants; - const assert = require('assert'); const IR = require('llvm-ir'); +const llparse = require('./'); +const constants = llparse.constants; +const MatchSequence = llparse.MatchSequence; + const CCONV = constants.CCONV; const BOOL = constants.BOOL; @@ -17,10 +18,18 @@ const TYPE_INDEX = constants.TYPE_INDEX; const TYPE_ERROR = constants.TYPE_ERROR; const TYPE_REASON = constants.TYPE_REASON; +const ATTR_STATE = constants.ATTR_STATE; +const ATTR_POS = constants.ATTR_POS; +const ATTR_ENDPOS = constants.ATTR_ENDPOS; + const ARG_STATE = constants.ARG_STATE; const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; +const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; +const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; +const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; + class Compiler { constructor(prefix) { this.prefix = prefix; @@ -31,9 +40,9 @@ class Compiler { this.signature = { node: this.ir.signature(TYPE_OUTPUT, [ - [ this.state.ptr(), 'noalias nocapture nonnull' ], - [ TYPE_INPUT, 'noalias nocapture nonnull readonly' ], - [ TYPE_INPUT, 'noalias nocapture nonnull readnone' ] + [ this.state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ] ]), callback: this.ir.signature(INT, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT @@ -51,6 +60,9 @@ class Compiler { this.counter = new Map(); this.selfReturnCache = new Map(); + + const matchSequence = new MatchSequence(this.prefix, this.ir, this.state); + this.matchSequence = matchSequence.build(); } build(root) { @@ -143,7 +155,8 @@ class Compiler { const fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - fn.visibility = `internal ${CCONV}`; + fn.visibility = 'internal'; + fn.cconv = CCONV; // TODO(indutny): reassess `minsize`. Looks like it gives best performance // results right now, though. @@ -368,123 +381,62 @@ class Compiler { const seq = this.ir.data(trie.select); - // Load `state.index` - body.comment('index = state.index'); - const indexField = this.field(info.fn, 'index'); - body.push(indexField); - const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexField ]); - body.push(index); - - const loop = body.jump('br'); - loop.name = 'loop'; - - // Loop start - const posPhi = IR._('phi', - [ TYPE_INPUT, '[', info.fn.arg(ARG_POS), ',', body.ref(), ']' ]); - loop.push(posPhi); - const indexPhi = IR._('phi', - [ TYPE_INDEX, '[', index, ',', body.ref(), ']' ]); - loop.push(indexPhi); - - const iteration = this.buildSequenceIteration(info.fn, loop, seq, - indexField, posPhi, indexPhi); - - // It is complete! - this.buildSubTrie(info, iteration.complete, iteration.pos, trie.children); - - // We have more data! - iteration.loop.loop('br', loop); - indexPhi.append([ '[', iteration.index, ',', iteration.loop.ref(), ']' ]); - posPhi.append([ '[', iteration.pos.next, ',', iteration.loop.ref(), ']' ]); + const cast = IR._('getelementptr inbounds', seq.type.to, + [ seq.type, seq ], + [ INT, INT.v(0) ], + [ INT, INT.v(0) ]); + body.push(cast); - // Have to pause - return self - this.buildSelfReturn(info.fn, iteration.pause); - - // Not equal - // Reset `state.index` on mismatch - return { pos: iteration.pos, body: iteration.otherwise }; - } + const returnType = this.matchSequence.type.ret; + const call = IR._(`call ${CCONV}`, [ + returnType, this.matchSequence, '(', + this.state.ptr(), info.fn.arg(ARG_STATE), ',', + TYPE_INPUT, info.fn.arg(ARG_POS), ',', + TYPE_INPUT, info.fn.arg(ARG_ENDPOS), ',', + TYPE_INPUT, cast, ',', + INT, INT.v(seq.type.to.length), + ')' + ]); + body.push(call); - // TODO(indutny): DRY this - buildSequenceIteration(fn, body, seq, indexField, pos, index) { - // Just a helper to reset `state.index` - const reset = () => { - return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], - [ TYPE_INDEX.ptr(), indexField ]).void(); - }; + const status = IR._('extractvalue', [ returnType, call ], + INT.v(returnType.lookup('status'))); + body.push(status); - body.comment('current = *pos'); - const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); + const current = IR._('extractvalue', [ returnType, call ], + INT.v(returnType.lookup('current'))); body.push(current); - body.comment('expected = seq[state.index]'); - const expectedPtr = IR._('getelementptr inbounds', seq.type.to, - [ seq.type, seq ], - [ INT, INT.v(0) ], - [ INT, index ]); - body.push(expectedPtr); - const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); - body.push(expected); - - body.comment('if (current == expected)'); - let cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); - body.push(cmp); - const { left: isMatch, right: isMismatch } = - body.branch('br', [ BOOL, cmp ]); - - // Mismatch - isMismatch.name = 'mismatch'; - isMismatch.comment('Sequence string does not match input'); - isMismatch.comment('state.index = 0'); - isMismatch.push(reset()); - - // Character matches - isMatch.name = 'match'; - isMatch.comment('Got a char match'); - isMatch.comment('next = pos + 1'); + // This is lame, but it is easier to do it this way + // (Optimizer will remove it, if it isn't needed) + body.comment('next = pos + 1'); const next = IR._('getelementptr', TYPE_INPUT.to, - [ TYPE_INPUT, pos ], + [ TYPE_INPUT, current ], [ INT, INT.v(1) ]); - isMatch.push(next); + body.push(next); - isMatch.comment('index1 = index + 1'); - const index1 = IR._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); - isMatch.push(index1); + const pos = { current, next }; - isMatch.comment('if (index1 == seq.length)'); - cmp = IR._('icmp', [ 'eq', TYPE_INDEX, index1 ], - TYPE_INDEX.v(seq.type.to.length)); - isMatch.push(cmp); - const { left: isComplete, right: isIncomplete } = - isMatch.branch('br', [ BOOL, cmp ]); - - isComplete.name = 'is_complete'; - isComplete.comment('state.index = 0'); - isComplete.push(reset()); + const s = this.buildSwitch(body, INT, status, [ + SEQUENCE_COMPLETE, + SEQUENCE_PAUSE, + SEQUENCE_MISMATCH + ]); - isIncomplete.name = 'is_incomplete'; - isIncomplete.comment('if (next != endpos)'); - cmp = IR._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); - isIncomplete.push(cmp); - const { left: moreData, right: noMoreData } = - isIncomplete.branch('br', [ BOOL, cmp ]); + // No other values are allowed + s.otherwise.terminate('unreachable'); - moreData.name = 'more_data'; + const complete = s.cases[0]; + const pause = s.cases[1]; + const mismatch = s.cases[2]; - noMoreData.name = 'no_more_data'; - noMoreData.comment('state.index = index1'); - noMoreData.push(IR._('store', [ TYPE_INDEX, index1 ], - [ TYPE_INDEX.ptr(), indexField ]).void()); + this.buildSubTrie(info, complete, pos, trie.children); + this.buildSelfReturn(info.fn, pause); - return { - index: index1, - pos: { current: pos, next }, - complete: isComplete, - otherwise: isMismatch, - loop: moreData, - pause: noMoreData - }; + // Not equal + // Reset `state.index` on mismatch + return { pos, body: mismatch }; } buildNext(info, body, pos, trie) { diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 295e99b..2889561 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -16,3 +16,14 @@ exports.TYPE_REASON = IR.i(8).ptr(); exports.ARG_STATE = 's'; exports.ARG_POS = 'p'; exports.ARG_ENDPOS = 'endp'; +exports.ARG_SEQUENCE = 'seq'; +exports.ARG_SEQUENCE_LEN = 'slen'; + +exports.ATTR_STATE = 'noalias nocapture nonnull'; +exports.ATTR_POS = 'noalias nocapture nonnull readonly'; +exports.ATTR_ENDPOS = 'noalias nocapture nonnull readnone'; +exports.ATTR_SEQUENCE = exports.ATTR_POS; + +exports.SEQUENCE_COMPLETE = 0; +exports.SEQUENCE_PAUSE = 1; +exports.SEQUENCE_MISMATCH = 2; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index f08d18e..451b79f 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -5,4 +5,5 @@ exports.constants = require('./constants'); exports.case = require('./case'); exports.node = require('./node'); exports.Trie = require('./trie'); +exports.MatchSequence = require('./match-sequence'); exports.Compiler = require('./compiler'); diff --git a/lib/llparse/match-sequence.js b/lib/llparse/match-sequence.js new file mode 100644 index 0000000..1c9f111 --- /dev/null +++ b/lib/llparse/match-sequence.js @@ -0,0 +1,211 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const llparse = require('./'); +const constants = llparse.constants; + +const CCONV = constants.CCONV; + +const BOOL = constants.BOOL; +const INT = constants.INT; +const TYPE_INPUT = constants.TYPE_INPUT; +const TYPE_INDEX = constants.TYPE_INDEX; + +const ATTR_STATE = constants.ATTR_STATE; +const ATTR_POS = constants.ATTR_POS; +const ATTR_ENDPOS = constants.ATTR_ENDPOS; +const ATTR_SEQUENCE = constants.ATTR_SEQUENCE; + +const ARG_STATE = constants.ARG_STATE; +const ARG_POS = constants.ARG_POS; +const ARG_ENDPOS = constants.ARG_ENDPOS; +const ARG_SEQUENCE = constants.ARG_SEQUENCE; +const ARG_SEQUENCE_LEN = constants.ARG_SEQUENCE_LEN; + +const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; +const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; +const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; + +class MatchSequence { + constructor(prefix, ir, state) { + this.state = state; + this.prefix = prefix; + this.ir = ir; + + this.returnType = this.ir.struct('match_sequence_ret'); + + this.returnType.field(TYPE_INPUT, 'current'); + this.returnType.field(INT, 'status'); + + this.signature = IR.signature(this.returnType, [ + [ state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ], + [ TYPE_INPUT, ATTR_SEQUENCE ], + INT + ]); + } + + build() { + const fn = this.ir.fn(this.signature, `${this.prefix}__match_sequence`, + [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); + + this.buildBody(fn); + + fn.visibility = 'internal'; + fn.cconv = CCONV; + fn.attributes = 'nounwind norecurse'; + + return fn; + } + + buildBody(fn) { + const body = fn.body; + + // Just to have a label + const start = body.jump('br'); + start.name = 'start'; + + // Load `state.index` + start.comment('index = state.index'); + const indexPtr = this.index(fn); + start.push(indexPtr); + const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ]); + start.push(index); + + const loop = start.jump('br'); + loop.name = 'loop'; + + // Loop start + const posPhi = IR._('phi', + [ TYPE_INPUT, '[', fn.arg(ARG_POS), ',', start.ref(), ']' ]); + loop.push(posPhi); + const indexPhi = IR._('phi', + [ TYPE_INDEX, '[', index, ',', start.ref(), ']' ]); + loop.push(indexPhi); + + const iteration = this.buildIteration(fn, loop, indexPtr, posPhi, indexPhi); + + // It is complete! + this.ret(iteration.complete, iteration.pos, SEQUENCE_COMPLETE); + + // We have more data! + iteration.loop.loop('br', loop); + indexPhi.append([ '[', iteration.index, ',', iteration.loop.ref(), ']' ]); + posPhi.append([ '[', iteration.pos.next, ',', iteration.loop.ref(), ']' ]); + + // Have to pause - return self + this.ret(iteration.pause, iteration.pos, SEQUENCE_PAUSE); + + // Not equal + this.ret(iteration.mismatch, iteration.pos, SEQUENCE_MISMATCH); + } + + + buildIteration(fn, body, indexField, pos, index) { + const seq = fn.arg(ARG_SEQUENCE); + const seqLen = fn.arg(ARG_SEQUENCE_LEN); + + body.comment('current = *pos'); + const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); + body.push(current); + + body.comment('expected = seq[state.index]'); + const expectedPtr = IR._('getelementptr', seq.type.to, + [ seq.type, seq ], + [ INT, index ]); + body.push(expectedPtr); + const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); + body.push(expected); + + // NOTE: fetch this early so it would dominate all returns + body.comment('next = pos + 1'); + const next = IR._('getelementptr', TYPE_INPUT.to, + [ TYPE_INPUT, pos ], + [ INT, INT.v(1) ]); + body.push(next); + + body.comment('if (current == expected)'); + let cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); + body.push(cmp); + const { left: isMatch, right: isMismatch } = + body.branch('br', [ BOOL, cmp ]); + + // Mismatch + isMismatch.name = 'mismatch'; + isMismatch.comment('Sequence string does not match input'); + isMismatch.comment('state.index = 0'); + isMismatch.push(this.reset(indexField)); + + // Character matches + isMatch.name = 'match'; + isMatch.comment('Got a char match'); + + isMatch.comment('index1 = index + 1'); + const index1 = IR._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); + isMatch.push(index1); + + isMatch.comment('if (index1 == seq.length)'); + cmp = IR._('icmp', [ 'eq', TYPE_INDEX, index1 ], seqLen); + isMatch.push(cmp); + const { left: isComplete, right: isIncomplete } = + isMatch.branch('br', [ BOOL, cmp ]); + + isComplete.name = 'is_complete'; + isComplete.comment('state.index = 0'); + isComplete.push(this.reset(indexField)); + + isIncomplete.name = 'is_incomplete'; + isIncomplete.comment('if (next != endpos)'); + cmp = IR._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); + isIncomplete.push(cmp); + const { left: moreData, right: noMoreData } = + isIncomplete.branch('br', [ BOOL, cmp ]); + + moreData.name = 'more_data'; + + noMoreData.name = 'no_more_data'; + noMoreData.comment('state.index = index1'); + noMoreData.push(IR._('store', [ TYPE_INDEX, index1 ], + [ TYPE_INDEX.ptr(), indexField ]).void()); + + return { + index: index1, + pos: { current: pos, next }, + complete: isComplete, + mismatch: isMismatch, + loop: moreData, + pause: noMoreData + }; + } + + ret(body, pos, status) { + const create = IR._('insertvalue', [ this.returnType, 'undef' ], + [ TYPE_INPUT, pos.current ], + INT.v(this.returnType.lookup('current'))); + body.push(create); + + const amend = IR._('insertvalue', [ this.returnType, create ], + [ INT, INT.v(status) ], + INT.v(this.returnType.lookup('status'))); + body.push(amend); + + body.terminate('ret', [ this.returnType, amend ]); + } + + index(fn) { + const stateArg = fn.arg(ARG_STATE); + + return IR._('getelementptr', this.state, + [ stateArg.type, stateArg ], + [ INT, INT.v(0) ], + [ INT, INT.v(this.state.lookup('index')) ]); + } + + reset(field) { + return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], + [ TYPE_INDEX.ptr(), field ]).void(); + } +} +module.exports = MatchSequence; diff --git a/package-lock.json b/package-lock.json index 50d92df..4031599 100644 --- a/package-lock.json +++ b/package-lock.json @@ -752,9 +752,9 @@ } }, "llvm-ir": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.0.1.tgz", - "integrity": "sha512-8YRbJlkx63gSOTwLECRssK/SlJ5loihCb9/X/BTWWB2KMP5KQ/ZoGKXHv5G+nEEeDBg+wZSff9/7SvFHC/dPbA==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.3.0.tgz", + "integrity": "sha512-9FfnZCS2kVS/OoSVdInryRY6lkR35lfuQEg+m56F2DhNJ6SWDK73/di1LW+qxJPdFBjIc/Jy1NBDDcgFKUXaJw==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index c5f2c81..71f6acf 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.0.1" + "llvm-ir": "^1.3.0" }, "devDependencies": { "async": "^2.6.0", diff --git a/test/fixtures/index.js b/test/fixtures/index.js index f36ea9b..67ed94c 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -32,7 +32,7 @@ exports.build = (name, source) => { if (ret.status !== 0) { process.stderr.write(ret.stdout); process.stderr.write(ret.stderr); - throw new Error('clang exit code=' + ret.status); + throw new Error('clang exit code: ' + ret.status); } return (input, expected, callback) => { From eb270dcad6f99a1a8a1fcdc7b68fbfaa56115f00 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 18:46:39 -0500 Subject: [PATCH 075/281] match-sequence: alwaysinline for speed --- lib/llparse/match-sequence.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/match-sequence.js b/lib/llparse/match-sequence.js index 1c9f111..2d1132a 100644 --- a/lib/llparse/match-sequence.js +++ b/lib/llparse/match-sequence.js @@ -55,7 +55,7 @@ class MatchSequence { fn.visibility = 'internal'; fn.cconv = CCONV; - fn.attributes = 'nounwind norecurse'; + fn.attributes = 'nounwind norecurse alwaysinline'; return fn; } From ff3e76944b74d836267a7289f633a6c9a6888e14 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 18:46:52 -0500 Subject: [PATCH 076/281] 1.0.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4031599..a931c1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.1", + "version": "1.0.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 71f6acf..2be5e99 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.1", + "version": "1.0.2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 3bf286907b392b7c8f5e5b73f5e61d1100af563e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 19:18:05 -0500 Subject: [PATCH 077/281] api: multi-match --- lib/llparse/node/base.js | 6 ++++++ test/api-test.js | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 2135601..27c8531 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -12,6 +12,12 @@ class Node { } match(value, next) { + // .match([ ... ], next) + if (Array.isArray(value)) { + value.forEach(value => this.match(value, next)); + return this; + } + assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); this.cases.push(new llparse.case.Match(value, next)); diff --git a/test/api-test.js b/test/api-test.js index 6b1c564..b7f02e6 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -37,6 +37,25 @@ describe('LLParse', () => { binary('GET', 'off=3 match=1\n', callback); }); + it('should support multi-match', (callback) => { + const start = p.node('start'); + + start.match([ ' ', '\t', '\r', '\n' ], start); + + start.select({ + 'A': 0, + 'B': 1 + }, printMatch(start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('multi-match', p.build(start)); + binary( + 'A B\t\tA\r\nA', + 'off=1 match=0\noff=3 match=1\noff=6 match=0\noff=9 match=0\n', + callback); + }); + describe('`.otherwise()`', () => { it('should not advance position by default', (callback) => { const p = llparse.create('llparse'); From 94b04ecf629dc431d45fe044584e88ca73985264 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 19:18:10 -0500 Subject: [PATCH 078/281] 1.1.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index a931c1f..826532e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.2", + "version": "1.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 2be5e99..e7177c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.0.2", + "version": "1.1.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From ccdaa53ad772ed85ce002d54814f96a64b554c3a Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 19:49:36 -0500 Subject: [PATCH 079/281] compiler: better error reporting --- lib/llparse/compiler.js | 6 +++--- lib/llparse/trie.js | 23 ++++++++++++----------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index c7d0dae..db4b8d9 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -34,7 +34,6 @@ class Compiler { constructor(prefix) { this.prefix = prefix; this.ir = new IR(); - this.trie = new llparse.Trie(); this.state = this.ir.struct(`${this.prefix}_state`); @@ -194,9 +193,10 @@ class Compiler { skip: otherwise[0].skip }; - const trie = this.trie.combine(node.cases); + const trie = new llparse.Trie(node.name); + const combined = trie.combine(node.cases); - this.buildTrie(info, body, trie); + this.buildTrie(info, body, combined); return fn; } diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index 558fad8..fc73255 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -31,7 +31,8 @@ class Next extends Node { } class Trie { - constructor() { + constructor(name) { + this.name = name; } combine(cases) { @@ -43,10 +44,10 @@ class Trie { if (list.length === 0) return null; - return this.level(list); + return this.level(list, []); } - level(list) { + level(list, path) { list.sort((a, b) => { return a.key.compare(b.key); }); @@ -60,8 +61,8 @@ class Trie { // Leaf if (min === 0) { - // TODO(indutny): report key, or validate elsewhere - assert.strictEqual(list.length, 1, 'Duplicate entries'); + assert.strictEqual(list.length, 1, + `Duplicate entries in "${this.name}" at [ ${path.join(', ')} ]`); return new Next(list[0].value, list[0].next); } @@ -72,10 +73,10 @@ class Trie { // Sequence if (common > 1) - return this.sequence(list, first.slice(0, common)); + return this.sequence(list, first.slice(0, common), path); // Single - return this.single(list); + return this.single(list, path); } slice(list, off) { @@ -88,16 +89,16 @@ class Trie { }); } - sequence(list, prefix) { + sequence(list, prefix, path) { const res = new Sequence(prefix); const sliced = this.slice(list, prefix.length); - res.children = this.level(sliced); + res.children = this.level(sliced, path.concat(prefix)); return res; } - single(list) { + single(list, path) { const keys = new Map(); for (let i = 0; i < list.length; i++) { const item = list[i]; @@ -112,7 +113,7 @@ class Trie { const res = new Single(); keys.forEach((sublist, key) => { const sliced = this.slice(sublist, 1); - res.children.push({ key, child: this.level(sliced) }); + res.children.push({ key, child: this.level(sliced, path.concat(key)) }); }); return res; From a0c137be1fda85a2fce862908501ae17635fd403 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 19:59:36 -0500 Subject: [PATCH 080/281] match: support numeric value --- lib/llparse/case/match.js | 8 ++++++-- lib/llparse/case/select.js | 8 +++++++- lib/llparse/index.js | 1 + lib/llparse/utils.js | 10 ++++++++++ test/api-test.js | 19 +++++++++++++++++++ 5 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 lib/llparse/utils.js diff --git a/lib/llparse/case/match.js b/lib/llparse/case/match.js index 7cb368b..939f299 100644 --- a/lib/llparse/case/match.js +++ b/lib/llparse/case/match.js @@ -1,6 +1,6 @@ 'use strict'; -const Buffer = require('buffer').Buffer; +const llparse = require('../'); const Case = require('./').Case; @@ -12,7 +12,11 @@ class Match extends Case { } linearize() { - return [ { key: Buffer.from(this.key), next: this.next, value: null } ]; + return [ { + key: llparse.utils.toBuffer(this.key), + next: this.next, + value: null + } ]; } } module.exports = Match; diff --git a/lib/llparse/case/select.js b/lib/llparse/case/select.js index 10a2949..3918d69 100644 --- a/lib/llparse/case/select.js +++ b/lib/llparse/case/select.js @@ -1,5 +1,7 @@ 'use strict'; +const llparse = require('../'); + const Case = require('./').Case; class Select extends Case { @@ -11,7 +13,11 @@ class Select extends Case { linearize() { return Object.keys(this.map).map((key) => { - return { key: Buffer.from(key), next: this.next, value: this.map[key] }; + return { + key: llparse.utils.toBuffer(key), + next: this.next, + value: this.map[key] + }; }); } } diff --git a/lib/llparse/index.js b/lib/llparse/index.js index 451b79f..3615131 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -1,6 +1,7 @@ 'use strict'; exports.constants = require('./constants'); +exports.utils = require('./utils'); exports.case = require('./case'); exports.node = require('./node'); diff --git a/lib/llparse/utils.js b/lib/llparse/utils.js new file mode 100644 index 0000000..553670e --- /dev/null +++ b/lib/llparse/utils.js @@ -0,0 +1,10 @@ +'use strict'; + +const Buffer = require('buffer').Buffer; + +exports.toBuffer = (value) => { + if (typeof value === 'number') + return Buffer.from([ value ]); + else + return Buffer.from(value); +}; diff --git a/test/api-test.js b/test/api-test.js index b7f02e6..5ece4f0 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -56,6 +56,25 @@ describe('LLParse', () => { callback); }); + it('should support numeric-match', (callback) => { + const start = p.node('start'); + + start.match(32, start); + + start.select({ + 'A': 0, + 'B': 1 + }, printMatch(start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('multi-match', p.build(start)); + binary( + 'A B A A', + 'off=1 match=0\noff=3 match=1\noff=6 match=0\noff=9 match=0\n', + callback); + }); + describe('`.otherwise()`', () => { it('should not advance position by default', (callback) => { const p = llparse.create('llparse'); From 42a97bc90402837765bdbef5619a1e0299ce32e7 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 21:27:33 -0500 Subject: [PATCH 081/281] 1.2.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 826532e..b11b7bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.1.0", + "version": "1.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e7177c3..a1a4e01 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.1.0", + "version": "1.2.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From ce36bfbb7126ab07e70b6cee186a1c86258595cd Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 21:32:19 -0500 Subject: [PATCH 082/281] compiler: fix minor lint issue --- lib/llparse/compiler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index db4b8d9..bda76d0 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -346,7 +346,7 @@ class Compiler { const blocks = body.terminate('switch', [ type, what ], cases); blocks[0].name = 'switch_otherwise'; - for (let i = 0; i< values.length; i++) { + for (let i = 0; i < values.length; i++) { const v = values[i] < 0 ? 'm' + (-values[i]) : values[i]; blocks[i + 1].name = 'case_' + v; } From 65b4a5fa64708f8e3eca505b47fa8e63fc3aaebe Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 22:10:34 -0500 Subject: [PATCH 083/281] compiler: optimize joins of Next --- lib/llparse/compiler.js | 73 ++++++++++++++++++++++++++-------------- lib/llparse/node/base.js | 1 + test/api-test.js | 15 +++++++++ 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index bda76d0..a8c340a 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -58,7 +58,8 @@ class Compiler { this.externalMap = new Map(); this.counter = new Map(); - this.selfReturnCache = new Map(); + // redirect blocks by `fn` and `target` + this.redirectCache = new Map(); const matchSequence = new MatchSequence(this.prefix, this.ir, this.state); this.matchSequence = matchSequence.build(); @@ -270,15 +271,6 @@ class Compiler { } buildSelfReturn(fn, body, cache) { - if (this.selfReturnCache.has(fn)) { - const target = this.selfReturnCache.get(fn); - body.terminate('br', target); - return; - } - - if (cache) - this.selfReturnCache.set(fn, body); - const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); body.push(bitcast); body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); @@ -440,34 +432,63 @@ class Compiler { } buildNext(info, body, pos, trie) { + return this.buildRedirect(info, body, pos, this.buildNode(trie.next), + trie.value); + } + + buildRedirect(info, body, pos, target, value = null) { + const fn = info.fn; + + if (this.redirectCache.has(fn) && + this.redirectCache.get(fn).has(target)) { + const cached = this.redirectCache.get(fn).get(target); + + if (cached.phi) { + assert(value, '`.match()` and `.select()` with the same target'); + cached.phi.append([ '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); + } else { + assert(!value, '`.match()` and `.select()` with the same target'); + } + + body.terminate('br', cached.target); + return; + } + + // Split, so that others could join us from code block above + const redirect = body.jump('br'); + let phi = null; + // Set `state.match` if needed - if (trie.value !== null) { - const matchField = this.field(info.fn, 'match'); - body.comment('state.match = ' + trie.value); - body.push(matchField); - body.push(IR._('store', [ TYPE_MATCH, TYPE_MATCH.v(trie.value) ], + if (value !== null) { + redirect.comment('state.match = phi'); + phi = IR._('phi', + [ TYPE_MATCH, '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); + redirect.push(phi); + + const matchField = this.field(fn, 'match'); + redirect.push(matchField); + redirect.push(IR._('store', [ TYPE_MATCH, phi ], [ TYPE_MATCH.ptr(), matchField ]).void()); } - this.buildRedirect(info, body, pos, this.buildNode(trie.next)); - return body; - } - - buildRedirect(info, body, pos, target) { - body.comment('redirect'); + if (!this.redirectCache.has(fn)) + this.redirectCache.set(fn, new Map()); + this.redirectCache.get(fn).set(target, { + phi, + target: redirect + }); // TODO(indutny): looks like `musttail` gives worse performance when calling // Invoke nodes (possibly others too). const call = IR._(`musttail call ${CCONV}`, [ TYPE_OUTPUT, target, '(', - this.state.ptr(), info.fn.arg(ARG_STATE), ',', + this.state.ptr(), fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', - TYPE_INPUT, info.fn.arg(ARG_ENDPOS), + TYPE_INPUT, fn.arg(ARG_ENDPOS), ')' ]); - body.push(call); - body.terminate('ret', [ TYPE_OUTPUT, call ]); - return body; + redirect.push(call); + redirect.terminate('ret', [ TYPE_OUTPUT, call ]); } buildError(info, body) { diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 27c8531..5d544c8 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -25,6 +25,7 @@ class Node { } select(map, next) { + // TODO(indutny): verify that there's no `.match()` for this `next` assert(next instanceof Node, 'Invalid `next` argument of `.select()`'); this.cases.push(new llparse.case.Select(map, next)); diff --git a/test/api-test.js b/test/api-test.js index 5ece4f0..c679018 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -37,6 +37,21 @@ describe('LLParse', () => { binary('GET', 'off=3 match=1\n', callback); }); + it('should optimize shallow select', (callback) => { + const start = p.node('start'); + + start.select({ + '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, + '5': 5, '6': 6, '7': 7, '8': 8, '9': 9 + }, printMatch(start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('shallow', p.build(start)); + + binary('012', 'off=3 match=1\n', callback); + }); + it('should support multi-match', (callback) => { const start = p.node('start'); From fc6813572ea018a158cfd634c3de9dc7780f17de Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 22:14:14 -0500 Subject: [PATCH 084/281] compiler: remove unused argument --- lib/llparse/compiler.js | 2 +- test/api-test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index a8c340a..d4e5923 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -270,7 +270,7 @@ class Compiler { return branch.left; } - buildSelfReturn(fn, body, cache) { + buildSelfReturn(fn, body) { const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); body.push(bitcast); body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); diff --git a/test/api-test.js b/test/api-test.js index c679018..c11f7c0 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -49,7 +49,7 @@ describe('LLParse', () => { const binary = fixtures.build('shallow', p.build(start)); - binary('012', 'off=3 match=1\n', callback); + binary('012', 'off=1 match=0\noff=2 match=1\noff=3 match=2\n', callback); }); it('should support multi-match', (callback) => { From 45c439e042a2cd9f4034060d5d32da9c49efdfb9 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 22:15:28 -0500 Subject: [PATCH 085/281] 1.2.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b11b7bd..31d5987 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a1a4e01..4e6c64c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.2.0", + "version": "1.2.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 4576ed274b2b83f1dc61f4d270d82e8c701574b8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 22:17:27 -0500 Subject: [PATCH 086/281] compiler: readable block name --- lib/llparse/compiler.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index d4e5923..b00482f 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -456,6 +456,7 @@ class Compiler { // Split, so that others could join us from code block above const redirect = body.jump('br'); + redirect.name = body.name + '_redirect'; let phi = null; // Set `state.match` if needed From bfe186442a7280aabb93347c2cd993e45b09dd80 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 22:18:58 -0500 Subject: [PATCH 087/281] 1.2.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 31d5987..a371c52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.2.1", + "version": "1.2.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 4e6c64c..80e0a6e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.2.1", + "version": "1.2.2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From dc8a6359b9257f417909f413c261ddc93682f26e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 23:13:45 -0500 Subject: [PATCH 088/281] api: `.select(key, value, next)` --- lib/llparse/case/otherwise.js | 2 +- lib/llparse/case/select.js | 18 ++++++---- lib/llparse/compiler.js | 23 +++++++------ lib/llparse/index.js | 1 + lib/llparse/node/base.js | 62 ++++++++++++++++++++++++++++++----- lib/llparse/node/error.js | 18 +++++++--- lib/llparse/node/invoke.js | 23 +++++++++---- lib/llparse/symbols.js | 5 +++ test/api-test.js | 14 ++++++++ 9 files changed, 130 insertions(+), 36 deletions(-) create mode 100644 lib/llparse/symbols.js diff --git a/lib/llparse/case/otherwise.js b/lib/llparse/case/otherwise.js index 081ac6a..4e10d7d 100644 --- a/lib/llparse/case/otherwise.js +++ b/lib/llparse/case/otherwise.js @@ -9,7 +9,7 @@ class Otherwise extends Case { } linearize() { - return []; + throw new Error('Should not be called'); } } module.exports = Otherwise; diff --git a/lib/llparse/case/select.js b/lib/llparse/case/select.js index 3918d69..df3de74 100644 --- a/lib/llparse/case/select.js +++ b/lib/llparse/case/select.js @@ -5,20 +5,26 @@ const llparse = require('../'); const Case = require('./').Case; class Select extends Case { - constructor(map, next) { + constructor(next) { super('select', next); - this.map = map; + this.map = new Map(); + } + + add(key, value) { + this.map.set(key, value); } linearize() { - return Object.keys(this.map).map((key) => { - return { + const res = []; + this.map.forEach((value, key) => { + res.push({ key: llparse.utils.toBuffer(key), next: this.next, - value: this.map[key] - }; + value + }); }); + return res; } } module.exports = Select; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index b00482f..bd1bc2d 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -7,6 +7,10 @@ const llparse = require('./'); const constants = llparse.constants; const MatchSequence = llparse.MatchSequence; +const kOtherwise = llparse.symbols.kOtherwise; +const kNoAdvance = llparse.symbols.kNoAdvance; +const kCases = llparse.symbols.kCases; + const CCONV = constants.CCONV; const BOOL = constants.BOOL; @@ -183,19 +187,18 @@ class Compiler { return fn; } - const otherwise = - node.cases.filter(c => c instanceof llparse.case.Otherwise); - assert.strictEqual(otherwise.length, 1, - `Node "${node.name}" must have only 1 \`.otherwise()\``); + const otherwise = node[kOtherwise]; + assert.notStrictEqual(otherwise, null, + `Node "${node.name}" must have \`.otherwise()\`/\`.skipTo()\``); const info = { node, fn, - otherwise: otherwise[0].next, - skip: otherwise[0].skip + otherwise: otherwise.next, + skip: otherwise.skip }; const trie = new llparse.Trie(node.name); - const combined = trie.combine(node.cases); + const combined = trie.combine(node[kCases]); this.buildTrie(info, body, combined); @@ -250,7 +253,7 @@ class Compiler { } buildPrologue(node, fn) { - if (node.noAdvance) + if (node[kNoAdvance]) return fn.body; // Check that we have enough chars to do the read @@ -285,7 +288,7 @@ class Compiler { // NOTE: `sequence` has loop inside it - so it isn't going to use // `pos.next` anyway (as a matter of fact it doesn't get `pos` as an // argument at all) - if (info.node.noAdvance || trie && trie.type === 'sequence') { + if (info.node[kNoAdvance] || trie && trie.type === 'sequence') { pos.next = pos.current; } else { body.comment('next = pos + 1'); @@ -369,7 +372,7 @@ class Compiler { } buildSequence(info, body, trie) { - assert(!info.node.noAdvance); + assert(!info.node[kNoAdvance]); const seq = this.ir.data(trie.select); diff --git a/lib/llparse/index.js b/lib/llparse/index.js index 3615131..cd5ba46 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -1,6 +1,7 @@ 'use strict'; exports.constants = require('./constants'); +exports.symbols = require('./symbols'); exports.utils = require('./utils'); exports.case = require('./case'); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 5d544c8..ea41514 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -4,11 +4,20 @@ const assert = require('assert'); const llparse = require('../'); +const kOtherwise = llparse.symbols.kOtherwise; +const kNoAdvance = llparse.symbols.kNoAdvance; +const kCases = llparse.symbols.kCases; +const kReceives = Symbol('receives'); + class Node { constructor(name) { this.name = name; - this.cases = []; - this.noAdvance = false; + + this[kCases] = []; + this[kReceives] = null; + + this[kOtherwise] = null; + this[kNoAdvance] = false; } match(value, next) { @@ -19,29 +28,66 @@ class Node { } assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); - this.cases.push(new llparse.case.Match(value, next)); + if (next[kReceives] === null) + next[kReceives] = 'match'; + assert.strictEqual(next[kReceives], 'match', + `"${next.name}" can't be a target of ` + + 'both `.select()` and `.match()`'); + + this[kCases].push(new llparse.case.Match(value, next)); return this; } - select(map, next) { - // TODO(indutny): verify that there's no `.match()` for this `next` + select(key, value, next) { + // .select(key, value, next) + const pairs = []; + if (Buffer.isBuffer(key) || typeof key === 'number' || + typeof key === 'string') { + assert.strictEqual(typeof value, 'number', + '`.select(key, value, next)` is a signature of the method'); + + pairs.push({ key, value }); + } else { + assert.strictEqual(typeof key, 'object', + '`.select()` first argument must be either an object or a key'); + + const map = key; + next = value; + value = null; + + Object.keys(map).forEach((key) => pairs.push({ key, value: map[key] })); + } + assert(next instanceof Node, 'Invalid `next` argument of `.select()`'); - this.cases.push(new llparse.case.Select(map, next)); + if (next[kReceives] === null) + next[kReceives] = 'select'; + assert.strictEqual(next[kReceives], 'select', + `"${next.name}" can't be a target of ` + + 'both `.select()` and `.match()`'); + + const select = new llparse.case.Select(next); + pairs.forEach(pair => select.add(pair.key, pair.value)); + + this[kCases].push(select); return this; } otherwise(next) { assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); - this.cases.push(new llparse.case.Otherwise(next)); + assert.strictEqual(this[kOtherwise], null, + 'Duplicate `.otherwise()`/`.skipTo()`'); + this[kOtherwise] = new llparse.case.Otherwise(next); return this; } skipTo(next) { assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); - this.cases.push(new llparse.case.Otherwise(next, true)); + assert.strictEqual(this[kOtherwise], null, + 'Duplicate `.skipTo()`/`.otherwise()`'); + this[kOtherwise] = new llparse.case.Otherwise(next, true); return this; } diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index 2d339d2..28b6fb1 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -1,15 +1,23 @@ 'use strict'; -const llparse = require('./'); +const node = require('./'); +const llparse = require('../'); -class Error extends llparse.Node { +const kNoAdvance = llparse.symbols.kNoAdvance; +const kCode = Symbol('code'); +const kReason = Symbol('reason'); + +class Error extends node.Node { constructor(code, reason) { super('error'); - this.code = code; - this.reason = reason; + this[kCode] = code; + this[kReason] = reason; - this.noAdvance = true; + this[kNoAdvance] = true; } + + get code() { return this[kCode]; } + get reason() { return this[kReason]; } } module.exports = Error; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 8f7d7f8..4ad1c83 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -2,24 +2,35 @@ const assert = require('assert'); -const llparse = require('./'); +const node = require('./'); +const llparse = require('../'); -class Invoke extends llparse.Node { +const kNoAdvance = llparse.symbols.kNoAdvance; +const kCallback = Symbol('callback'); +const kMap = Symbol('map'); + +class Invoke extends node.Node { constructor(callback, map, otherwise) { assert.strictEqual(typeof map, 'object', 'Invalid `map` argument of `.invoke()`'); - assert(otherwise instanceof llparse.Node, + assert(otherwise instanceof node.Node, 'Invalid `otherwise` argument of `.invoke()`'); + assert.strictEqual(typeof callback, 'string', + 'Invalid `callback` argument of `.invoke()`'); + super('invoke_' + callback); - this.callback = callback; + this[kCallback] = callback; // TODO(indutny): validate that keys are integers - this.map = map; + this[kMap] = map; this.otherwise(otherwise); - this.noAdvance = true; + this[kNoAdvance] = true; } + + get callback() { return this[kCallback]; } + get map() { return this[kMap]; } } module.exports = Invoke; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js new file mode 100644 index 0000000..9d73631 --- /dev/null +++ b/lib/llparse/symbols.js @@ -0,0 +1,5 @@ +'use strict'; + +exports.kOtherwise = Symbol('otherwise'); +exports.kNoAdvance = Symbol('noAdvance'); +exports.kCases = Symbol('cases'); diff --git a/test/api-test.js b/test/api-test.js index c11f7c0..f174955 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -52,6 +52,20 @@ describe('LLParse', () => { binary('012', 'off=1 match=0\noff=2 match=1\noff=3 match=2\n', callback); }); + it('should support key-value select', (callback) => { + const start = p.node('start'); + + start.select('0', 0, printMatch(start)); + start.select('1', 1, printMatch(start)); + start.select('2', 2, printMatch(start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('kv-select', p.build(start)); + + binary('012', 'off=1 match=0\noff=2 match=1\noff=3 match=2\n', callback); + }); + it('should support multi-match', (callback) => { const start = p.node('start'); From ccc330373d5057423a5146e839eb884e09e7fc0f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 23:14:28 -0500 Subject: [PATCH 089/281] 1.3.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index a371c52..065b526 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.2.2", + "version": "1.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 80e0a6e..f9e05fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.2.2", + "version": "1.3.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 51377981b2b561dd105ab3fb9f7be8b83f30644d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 23:31:28 -0500 Subject: [PATCH 090/281] invoke: validate keys of `map` --- lib/llparse/node/invoke.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 4ad1c83..e0c5403 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -18,11 +18,14 @@ class Invoke extends node.Node { assert.strictEqual(typeof callback, 'string', 'Invalid `callback` argument of `.invoke()`'); + Object.keys(map).forEach((key) => { + assert.equal(key, key | 0, + 'Only integer keys are allowed in `.invoke()`\'s map'); + }); + super('invoke_' + callback); this[kCallback] = callback; - - // TODO(indutny): validate that keys are integers this[kMap] = map; this.otherwise(otherwise); From 1d2b49f4a27bdc9298f0fbfc710c2ea97023c52e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 18 Feb 2018 23:32:26 -0500 Subject: [PATCH 091/281] 1.3.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 065b526..0e39cc0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.3.0", + "version": "1.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f9e05fc..a4e8b3b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.3.0", + "version": "1.3.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From eb8ed7a636e17545977cced642e7750a1d3847f0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 15:52:58 -0500 Subject: [PATCH 092/281] compiler: save progress on `store` rewrite --- lib/llparse.js | 16 +++++ lib/llparse/code/base.js | 24 ++++++++ lib/llparse/code/callback.js | 10 ++++ lib/llparse/code/index.js | 5 ++ lib/llparse/code/store.js | 10 ++++ lib/llparse/compiler.js | 113 ++++++++++++++++++++++------------- lib/llparse/constants.js | 1 + lib/llparse/index.js | 1 + lib/llparse/node/base.js | 39 +++++++----- lib/llparse/node/invoke.js | 21 ++++--- lib/llparse/symbols.js | 6 +- test/api-test.js | 18 ++++-- test/fixtures/main.c | 20 +++++-- 13 files changed, 208 insertions(+), 76 deletions(-) create mode 100644 lib/llparse/code/base.js create mode 100644 lib/llparse/code/callback.js create mode 100644 lib/llparse/code/index.js create mode 100644 lib/llparse/code/store.js diff --git a/lib/llparse.js b/lib/llparse.js index 19c1d50..9fd8c72 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -2,17 +2,33 @@ const internal = require('./llparse/'); +const kCode = Symbol('code'); + // API, really +class CodeAPI { + callback(name, body) { + return new internal.code.Callback(name, body); + } + + store(name, body) { + return new internal.code.Store(name, body); + } +} + class LLParse { constructor(prefix) { this.prefix = prefix || 'llparse'; + + this[kCode] = new CodeAPI(); } static create(prefix) { return new LLParse(prefix); } + get code() { return this[kCode]; } + node(name) { return new internal.node.Node(name); } diff --git a/lib/llparse/code/base.js b/lib/llparse/code/base.js new file mode 100644 index 0000000..47210aa --- /dev/null +++ b/lib/llparse/code/base.js @@ -0,0 +1,24 @@ +'use strict'; + +const code = require('./'); +const llparse = require('../'); + +const kName = Symbol('name'); +const kType = Symbol('type'); +const kBody = llparse.symbols.kBody; + +class Code { + constructor(type, name, body = null) { + this[kType] = type; + this[kName] = name; + this[kBody] = body; + + // TODO(indutny): custom code generators + if (body) + throw new Error('Not implemented yet'); + } + + get type() { return this[kType]; } + get name() { return this[kName]; } +} +module.exports = Code; diff --git a/lib/llparse/code/callback.js b/lib/llparse/code/callback.js new file mode 100644 index 0000000..e0d6cc6 --- /dev/null +++ b/lib/llparse/code/callback.js @@ -0,0 +1,10 @@ +'use strict'; + +const code = require('./'); + +class Callback extends code.Code { + constructor(name, body) { + super('callback', name, body); + } +} +module.exports = Callback; diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js new file mode 100644 index 0000000..32c4c59 --- /dev/null +++ b/lib/llparse/code/index.js @@ -0,0 +1,5 @@ +'use strict'; + +exports.Code = require('./base'); +exports.Callback = require('./callback'); +exports.Store = require('./store'); diff --git a/lib/llparse/code/store.js b/lib/llparse/code/store.js new file mode 100644 index 0000000..a699c93 --- /dev/null +++ b/lib/llparse/code/store.js @@ -0,0 +1,10 @@ +'use strict'; + +const code = require('./'); + +class Store extends code.Code { + constructor(name, body) { + super('store', name, body); + } +} +module.exports = Store; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index bd1bc2d..ffdf236 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -7,9 +7,10 @@ const llparse = require('./'); const constants = llparse.constants; const MatchSequence = llparse.MatchSequence; -const kOtherwise = llparse.symbols.kOtherwise; -const kNoAdvance = llparse.symbols.kNoAdvance; const kCases = llparse.symbols.kCases; +const kNoAdvance = llparse.symbols.kNoAdvance; +const kOtherwise = llparse.symbols.kOtherwise; +const kSignature = llparse.symbols.kSignature; const CCONV = constants.CCONV; @@ -29,6 +30,7 @@ const ATTR_ENDPOS = constants.ATTR_ENDPOS; const ARG_STATE = constants.ARG_STATE; const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; +const ARG_MATCH = constants.ARG_MATCH; const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; @@ -42,21 +44,31 @@ class Compiler { this.state = this.ir.struct(`${this.prefix}_state`); this.signature = { - node: this.ir.signature(TYPE_OUTPUT, [ - [ this.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ] - ]), + node: { + match: this.ir.signature(TYPE_OUTPUT, [ + [ this.state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ] + ]), + value: this.ir.signature(TYPE_OUTPUT, [ + [ this.state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ], + TYPE_MATCH + ]) + }, callback: this.ir.signature(INT, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT + ]), + store: this.ir.signature(INT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_MATCH ]) }; - this.state.field(this.signature.node.ptr(), 'current'); + this.state.field(this.signature.node.match.ptr(), 'current'); this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); this.state.field(TYPE_INDEX, 'index'); - this.state.field(TYPE_MATCH, 'match'); this.nodeMap = new Map(); this.externalMap = new Map(); @@ -70,7 +82,10 @@ class Compiler { } build(root) { - const rootFn = this.buildNode(root); + // Check that we don't start with a `value` Invoke + this.checkSignatureType(root, null); + + const rootFn = this.buildNode(root).fn; this.buildInit(rootFn.ref()); this.buildParse(); @@ -86,8 +101,7 @@ class Compiler { current: this.field(init, 'current'), error: this.field(init, 'error'), reason: this.field(init, 'reason'), - index: this.field(init, 'index'), - match: this.field(init, 'match') + index: this.field(init, 'index') }; Object.keys(fields).forEach(key => init.body.push(fields[key])); @@ -101,7 +115,6 @@ class Compiler { store(fields.error, TYPE_ERROR, TYPE_ERROR.v(0)); store(fields.reason, TYPE_REASON, TYPE_REASON.v(null)); store(fields.index, TYPE_INDEX, TYPE_INDEX.v(0)); - store(fields.match, TYPE_MATCH, TYPE_MATCH.v(0)); init.body.terminate('ret', IR.void()); @@ -116,7 +129,7 @@ class Compiler { const body = parse.body; - const nodeSig = this.signature.node; + const nodeSig = this.signature.node.match; const currentPtr = this.field(parse, 'current'); body.push(currentPtr); @@ -157,8 +170,15 @@ class Compiler { const name = `${this.prefix}__${node.name}` + `${index === 0 ? '' : '_' + index}`; - const fn = this.ir.fn(this.signature.node, name, - [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + let fn; + if (node[kSignature] === 'match') { + fn = this.ir.fn(this.signature.node.match, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + } else { + assert.strictEqual(node[kSignature], 'value'); + fn = this.ir.fn(this.signature.node.value, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); + } fn.visibility = 'internal'; fn.cconv = CCONV; @@ -177,14 +197,14 @@ class Compiler { return this.nodeMap.get(node); const fn = this.createFn(node); - this.nodeMap.set(node, fn); + this.nodeMap.set(node, { fn, node }); let body = this.buildPrologue(node, fn); if (node instanceof llparse.node.Error) { const info = { node, fn, otherwise: null }; this.buildError(info, body); - return fn; + return { fn, node }; } const otherwise = node[kOtherwise]; @@ -202,7 +222,7 @@ class Compiler { this.buildTrie(info, body, combined); - return fn; + return { fn, node }; } buildInvoke(info, body, pos, callback) { @@ -248,7 +268,7 @@ class Compiler { const subBody = this.buildPrologue(info.node, subFn); this.buildTrie(subInfo, subBody, trie); - this.buildRedirect(info, body, pos, subFn); + this.buildRedirect(info, body, pos, { fn: subFn, node: info.node }); return subFn; } @@ -267,13 +287,17 @@ class Compiler { // Return self when `pos === endpos` branch.right.name = 'prologue_end'; - this.buildSelfReturn(fn, branch.right, true); + this.buildSelfReturn({ fn, node }, branch.right, true); branch.left.name = 'prologue_normal'; return branch.left; } - buildSelfReturn(fn, body) { + buildSelfReturn(info, body) { + assert.strictEqual(info.node[kSignature], 'match', + 'non-match nodes can\'t have self return'); + + const fn = info.fn; const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); body.push(bitcast); body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); @@ -427,7 +451,7 @@ class Compiler { const mismatch = s.cases[2]; this.buildSubTrie(info, complete, pos, trie.children); - this.buildSelfReturn(info.fn, pause); + this.buildSelfReturn(info, pause); // Not equal // Reset `state.index` on mismatch @@ -442,9 +466,11 @@ class Compiler { buildRedirect(info, body, pos, target, value = null) { const fn = info.fn; + this.checkSignatureType(target.node, value); + if (this.redirectCache.has(fn) && - this.redirectCache.get(fn).has(target)) { - const cached = this.redirectCache.get(fn).get(target); + this.redirectCache.get(fn).has(target.fn)) { + const cached = this.redirectCache.get(fn).get(target.fn); if (cached.phi) { assert(value, '`.match()` and `.select()` with the same target'); @@ -462,35 +488,36 @@ class Compiler { redirect.name = body.name + '_redirect'; let phi = null; - // Set `state.match` if needed + // Compute `match` if needed if (value !== null) { - redirect.comment('state.match = phi'); + redirect.comment('Select `match`'); phi = IR._('phi', [ TYPE_MATCH, '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); redirect.push(phi); - - const matchField = this.field(fn, 'match'); - redirect.push(matchField); - redirect.push(IR._('store', [ TYPE_MATCH, phi ], - [ TYPE_MATCH.ptr(), matchField ]).void()); } if (!this.redirectCache.has(fn)) this.redirectCache.set(fn, new Map()); - this.redirectCache.get(fn).set(target, { + this.redirectCache.get(fn).set(target.fn, { phi, target: redirect }); - // TODO(indutny): looks like `musttail` gives worse performance when calling - // Invoke nodes (possibly others too). - const call = IR._(`musttail call ${CCONV}`, [ - TYPE_OUTPUT, target, '(', + const args = [ + TYPE_OUTPUT, target.fn, '(', this.state.ptr(), fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', - TYPE_INPUT, fn.arg(ARG_ENDPOS), - ')' - ]); + TYPE_INPUT, fn.arg(ARG_ENDPOS) + ]; + + if (phi) + args.push(',', TYPE_MATCH, phi); + + args.push(')'); + + // TODO(indutny): looks like `musttail` gives worse performance when calling + // Invoke nodes (possibly others too). + const call = IR._(`musttail call ${CCONV}`, args); redirect.push(call); redirect.terminate('ret', [ TYPE_OUTPUT, call ]); } @@ -518,6 +545,8 @@ class Compiler { return body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); } + // Helpers + field(fn, name) { const stateArg = fn.arg(ARG_STATE); @@ -526,5 +555,9 @@ class Compiler { [ INT, INT.v(0) ], [ INT, INT.v(this.state.lookup(name)) ]); } + + checkSignatureType(node, value) { + assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); + } } module.exports = Compiler; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 2889561..dec7d0b 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -18,6 +18,7 @@ exports.ARG_POS = 'p'; exports.ARG_ENDPOS = 'endp'; exports.ARG_SEQUENCE = 'seq'; exports.ARG_SEQUENCE_LEN = 'slen'; +exports.ARG_MATCH = 'match'; exports.ATTR_STATE = 'noalias nocapture nonnull'; exports.ATTR_POS = 'noalias nocapture nonnull readonly'; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index cd5ba46..86c3b56 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -5,6 +5,7 @@ exports.symbols = require('./symbols'); exports.utils = require('./utils'); exports.case = require('./case'); +exports.code = require('./code'); exports.node = require('./node'); exports.Trie = require('./trie'); exports.MatchSequence = require('./match-sequence'); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index ea41514..89dbc14 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -2,19 +2,22 @@ const assert = require('assert'); +const node = require('./'); const llparse = require('../'); const kOtherwise = llparse.symbols.kOtherwise; const kNoAdvance = llparse.symbols.kNoAdvance; const kCases = llparse.symbols.kCases; -const kReceives = Symbol('receives'); +const kSignature = llparse.symbols.kSignature; + +const kCheckIsMatch = Symbol('checkIsMatch'); class Node { - constructor(name) { + constructor(name, signature = 'match') { this.name = name; + this[kSignature] = signature; this[kCases] = []; - this[kReceives] = null; this[kOtherwise] = null; this[kNoAdvance] = false; @@ -28,11 +31,7 @@ class Node { } assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); - if (next[kReceives] === null) - next[kReceives] = 'match'; - assert.strictEqual(next[kReceives], 'match', - `"${next.name}" can't be a target of ` + - 'both `.select()` and `.match()`'); + this[kCheckIsMatch](next, '.match()'); this[kCases].push(new llparse.case.Match(value, next)); @@ -59,12 +58,10 @@ class Node { Object.keys(map).forEach((key) => pairs.push({ key, value: map[key] })); } - assert(next instanceof Node, 'Invalid `next` argument of `.select()`'); - if (next[kReceives] === null) - next[kReceives] = 'select'; - assert.strictEqual(next[kReceives], 'select', - `"${next.name}" can't be a target of ` + - 'both `.select()` and `.match()`'); + assert(next instanceof node.Invoke, + 'Invalid `next` argument of `.select()`, must be an `.invoke()` node'); + assert.strictEqual(next[kSignature], 'value', + `Invoke of "${next.name}" can't be a target of \`.select()\``); const select = new llparse.case.Select(next); pairs.forEach(pair => select.add(pair.key, pair.value)); @@ -76,6 +73,8 @@ class Node { otherwise(next) { assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); + this[kCheckIsMatch](next, '.otherwise()'); + assert.strictEqual(this[kOtherwise], null, 'Duplicate `.otherwise()`/`.skipTo()`'); this[kOtherwise] = new llparse.case.Otherwise(next); @@ -84,12 +83,22 @@ class Node { } skipTo(next) { - assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); + assert(next instanceof Node, 'Invalid `next` argument of `.skipTo()`'); + this[kCheckIsMatch](next, '.skipTo()'); + assert.strictEqual(this[kOtherwise], null, 'Duplicate `.skipTo()`/`.otherwise()`'); this[kOtherwise] = new llparse.case.Otherwise(next, true); return this; } + + [kCheckIsMatch](next, method) { + if (!(next instanceof node.Invoke)) + return; + + assert.strictEqual(next[kSignature], 'match', + `Invoke of "${next.name}" can't be a target of \`${method}\``); + } } module.exports = Node; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index e0c5403..cb1ba73 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -6,26 +6,26 @@ const node = require('./'); const llparse = require('../'); const kNoAdvance = llparse.symbols.kNoAdvance; -const kCallback = Symbol('callback'); +const kCode = Symbol('code'); const kMap = Symbol('map'); class Invoke extends node.Node { - constructor(callback, map, otherwise) { + constructor(code, map, otherwise) { assert.strictEqual(typeof map, 'object', - 'Invalid `map` argument of `.invoke()`'); + 'Invalid `map` argument of `.invoke()`, must be an Object'); assert(otherwise instanceof node.Node, - 'Invalid `otherwise` argument of `.invoke()`'); - assert.strictEqual(typeof callback, 'string', - 'Invalid `callback` argument of `.invoke()`'); + 'Invalid `otherwise` argument of `.invoke()`, must be a Node'); + assert(code instanceof llparse.code.Code, + 'Invalid `code` argument of `.invoke()`, must be a Code instance'); Object.keys(map).forEach((key) => { assert.equal(key, key | 0, 'Only integer keys are allowed in `.invoke()`\'s map'); }); - super('invoke_' + callback); + super('invoke_' + code.name, code.type === 'store' ? 'value' : 'match'); - this[kCallback] = callback; + this[kCode] = code; this[kMap] = map; this.otherwise(otherwise); @@ -33,7 +33,10 @@ class Invoke extends node.Node { this[kNoAdvance] = true; } - get callback() { return this[kCallback]; } + get code() { return this[kCode]; } get map() { return this[kMap]; } + + match() { throw new Error('Not supported'); } + select() { throw new Error('Not supported'); } } module.exports = Invoke; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 9d73631..0851057 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -1,5 +1,7 @@ 'use strict'; -exports.kOtherwise = Symbol('otherwise'); -exports.kNoAdvance = Symbol('noAdvance'); +exports.kBody = Symbol('body'); exports.kCases = Symbol('cases'); +exports.kNoAdvance = Symbol('noAdvance'); +exports.kOtherwise = Symbol('otherwise'); +exports.kSignature = Symbol('signature'); diff --git a/test/api-test.js b/test/api-test.js index f174955..80a3fec 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -12,17 +12,27 @@ describe('LLParse', () => { }); const printMatch = (next) => { - return p.invoke('print_match', { + const code = p.code.store('print_match'); + + return p.invoke(code, { 0: next }, p.error(1, '`print_match` error')); }; + const printOff = (next) => { + const code = p.code.callback('print_off'); + + return p.invoke(code, { + 0: next + }, p.error(1, '`print_off` error')); + }; + it('should compile simple parser', (callback) => { const start = p.node('start'); start.match(' ', start); - start.match('HTTP', printMatch(start)); + start.match('HTTP', printOff(start)); start.select({ 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, @@ -116,7 +126,7 @@ describe('LLParse', () => { .otherwise(b); b - .match('B', printMatch(b)) + .match('B', printOff(b)) .otherwise(a); const binary = fixtures.build('otherwise-noadvance', p.build(a)); @@ -130,7 +140,7 @@ describe('LLParse', () => { const start = p.node('start'); start - .match(' ', printMatch(start)) + .match(' ', printOff(start)) .skipTo(start); const binary = fixtures.build('otherwise-skip', p.build(start)); diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 5c0af5b..374e971 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -21,20 +21,28 @@ static const char* start; void llparse_init(struct state* s); int llparse_execute(struct state* s, const char* p, const char* endp); -int print_match(struct state* s, const char* p, const char* endp) { +int print_off(struct state* s, const char* p, const char* endp) { if (bench) return 0; - fprintf(stdout, "off=%d match=%d\n", (int) (p - start), s->match); + fprintf(stdout, "off=%d\n", (int) (p - start)); return 0; } -int return_match(struct state* s, const char* p, const char* endp) { +int print_match(struct state* s, const char* p, const char* endp, int value) { if (bench) - return s->match; + return 0; + + fprintf(stdout, "off=%d match=%d\n", (int) (p - start), value); + return 0; +} + +int return_match(struct state* s, const char* p, const char* endp, int value) { + if (bench) + return value; - fprintf(stdout, "off=%d return match=%d\n", (int) (p - start), s->match); - return s->match; + fprintf(stdout, "off=%d return match=%d\n", (int) (p - start), value); + return value; } static int run_bench(const char* input, int len) { From ace2e84edd04ccd4d0777120fea9f6523d9294e2 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 16:10:11 -0500 Subject: [PATCH 093/281] compiler: working match/value callbacks --- lib/llparse/code/base.js | 1 - lib/llparse/code/callback.js | 2 +- lib/llparse/code/store.js | 2 +- lib/llparse/compiler.js | 99 ++++++++++++++++++++---------------- lib/llparse/constants.js | 1 + lib/llparse/node/invoke.js | 2 +- test/api-test.js | 4 +- 7 files changed, 62 insertions(+), 49 deletions(-) diff --git a/lib/llparse/code/base.js b/lib/llparse/code/base.js index 47210aa..ae81de7 100644 --- a/lib/llparse/code/base.js +++ b/lib/llparse/code/base.js @@ -1,6 +1,5 @@ 'use strict'; -const code = require('./'); const llparse = require('../'); const kName = Symbol('name'); diff --git a/lib/llparse/code/callback.js b/lib/llparse/code/callback.js index e0d6cc6..dfa0b06 100644 --- a/lib/llparse/code/callback.js +++ b/lib/llparse/code/callback.js @@ -4,7 +4,7 @@ const code = require('./'); class Callback extends code.Code { constructor(name, body) { - super('callback', name, body); + super('match', name, body); } } module.exports = Callback; diff --git a/lib/llparse/code/store.js b/lib/llparse/code/store.js index a699c93..7185a22 100644 --- a/lib/llparse/code/store.js +++ b/lib/llparse/code/store.js @@ -4,7 +4,7 @@ const code = require('./'); class Store extends code.Code { constructor(name, body) { - super('store', name, body); + super('value', name, body); } } module.exports = Store; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index ffdf236..9d5d93b 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -31,6 +31,7 @@ const ARG_STATE = constants.ARG_STATE; const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; +const ARG_UNUSED = constants.ARG_UNUSED; const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; @@ -44,28 +45,23 @@ class Compiler { this.state = this.ir.struct(`${this.prefix}_state`); this.signature = { - node: { - match: this.ir.signature(TYPE_OUTPUT, [ - [ this.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ] + node: this.ir.signature(TYPE_OUTPUT, [ + [ this.state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ], + TYPE_MATCH + ]), + callback: { + match: this.ir.signature(INT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]), - value: this.ir.signature(TYPE_OUTPUT, [ - [ this.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ], - TYPE_MATCH + value: this.ir.signature(INT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_MATCH ]) - }, - callback: this.ir.signature(INT, [ - this.state.ptr(), TYPE_INPUT, TYPE_INPUT - ]), - store: this.ir.signature(INT, [ - this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_MATCH - ]) + } }; - this.state.field(this.signature.node.match.ptr(), 'current'); + this.state.field(this.signature.node.ptr(), 'current'); this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); this.state.field(TYPE_INDEX, 'index'); @@ -129,7 +125,7 @@ class Compiler { const body = parse.body; - const nodeSig = this.signature.node.match; + const nodeSig = this.signature.node; const currentPtr = this.field(parse, 'current'); body.push(currentPtr); @@ -141,7 +137,8 @@ class Compiler { TYPE_OUTPUT, current, '(', this.state.ptr(), parse.arg(ARG_STATE), ',', TYPE_INPUT, parse.arg(ARG_POS), ',', - TYPE_INPUT, parse.arg(ARG_ENDPOS), + TYPE_INPUT, parse.arg(ARG_ENDPOS), ',', + TYPE_MATCH, TYPE_MATCH.v(0), ')' ]); body.push(call); @@ -172,11 +169,11 @@ class Compiler { let fn; if (node[kSignature] === 'match') { - fn = this.ir.fn(this.signature.node.match, name, - [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + fn = this.ir.fn(this.signature.node, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_UNUSED ]); } else { assert.strictEqual(node[kSignature], 'value'); - fn = this.ir.fn(this.signature.node.value, name, + fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); } fn.visibility = 'internal'; @@ -225,30 +222,49 @@ class Compiler { return { fn, node }; } - buildInvoke(info, body, pos, callback) { - let external; - if (this.externalMap.has(callback)) { - external = this.externalMap.get(callback); + buildCode(node, code) { + assert.strictEqual(node[kSignature], code.type); + + const signature = code.type === 'match' ? + this.signature.callback.match : + this.signature.callback.value; + + const name = code.name; + + // TODO(indutny): support code.body + if (this.externalMap.has(name)) { + return this.externalMap.get(name); } else { - external = this.ir.declare(this.signature.callback, callback); + const external = this.ir.declare(signature, name); external.attributes = 'alwaysinline'; - this.externalMap.set(callback, external); + this.externalMap.set(name, external); + return external; } + } - const returnType = this.signature.callback.ret; + buildInvoke(info, body, pos) { + const code = this.buildCode(info.node, info.node.code); - const call = IR._('call', [ - returnType, external, '(', + const args = [ + code.type.ret, code, '(', this.state.ptr(), info.fn.arg(ARG_STATE), ',', TYPE_INPUT, info.fn.arg(ARG_POS), ',', - TYPE_INPUT, info.fn.arg(ARG_ENDPOS), - ')' - ]); + TYPE_INPUT, info.fn.arg(ARG_ENDPOS) + ]; + + if (info.node[kSignature] === 'value') + args.push(',', TYPE_MATCH, info.fn.arg(ARG_MATCH)); + else + assert.strictEqual(info.node[kSignature], 'match'); + + args.push(')'); + + const call = IR._('call', args); body.push(call); const keys = Object.keys(info.node.map).map(key => key | 0); - const s = this.buildSwitch(body, returnType, call, keys); + const s = this.buildSwitch(body, code.type.ret, call, keys); s.cases.forEach((body, i) => { const subNode = info.node.map[keys[i]]; @@ -323,7 +339,7 @@ class Compiler { } if (info.node instanceof llparse.node.Invoke) - body = this.buildInvoke(info, body, pos, info.node.callback); + body = this.buildInvoke(info, body, pos); // Traverse the `trie` if (trie === null) { @@ -507,14 +523,11 @@ class Compiler { TYPE_OUTPUT, target.fn, '(', this.state.ptr(), fn.arg(ARG_STATE), ',', TYPE_INPUT, pos.next, ',', - TYPE_INPUT, fn.arg(ARG_ENDPOS) + TYPE_INPUT, fn.arg(ARG_ENDPOS), ',', + TYPE_MATCH, phi ? phi : TYPE_MATCH.v(0), + ')' ]; - if (phi) - args.push(',', TYPE_MATCH, phi); - - args.push(')'); - // TODO(indutny): looks like `musttail` gives worse performance when calling // Invoke nodes (possibly others too). const call = IR._(`musttail call ${CCONV}`, args); diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index dec7d0b..398ad9c 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -19,6 +19,7 @@ exports.ARG_ENDPOS = 'endp'; exports.ARG_SEQUENCE = 'seq'; exports.ARG_SEQUENCE_LEN = 'slen'; exports.ARG_MATCH = 'match'; +exports.ARG_UNUSED = '_unused'; exports.ATTR_STATE = 'noalias nocapture nonnull'; exports.ATTR_POS = 'noalias nocapture nonnull readonly'; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index cb1ba73..a87186f 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -23,7 +23,7 @@ class Invoke extends node.Node { 'Only integer keys are allowed in `.invoke()`\'s map'); }); - super('invoke_' + code.name, code.type === 'store' ? 'value' : 'match'); + super('invoke_' + code.name, code.type); this[kCode] = code; this[kMap] = map; diff --git a/test/api-test.js b/test/api-test.js index 80a3fec..d85695d 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -131,7 +131,7 @@ describe('LLParse', () => { const binary = fixtures.build('otherwise-noadvance', p.build(a)); - binary('AABAB', 'off=3 match=0\noff=5 match=0\n', callback); + binary('AABAB', 'off=3\noff=5\n', callback); }); it('should advance when it is `.skipTo()`', (callback) => { @@ -145,7 +145,7 @@ describe('LLParse', () => { const binary = fixtures.build('otherwise-skip', p.build(start)); - binary('HELLO WORLD', 'off=6 match=0\n', callback); + binary('HELLO WORLD', 'off=6\n', callback); }); it('should skip everything with `.skipTo()`', (callback) => { From 81578c8f41a926514a6625b2a9c9941218728408 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 16:36:55 -0500 Subject: [PATCH 094/281] compiler: custom code bodies --- lib/llparse.js | 8 +-- lib/llparse/code/base.js | 7 +-- lib/llparse/code/index.js | 4 +- lib/llparse/code/{callback.js => match.js} | 4 +- lib/llparse/code/{store.js => value.js} | 4 +- lib/llparse/compiler.js | 60 ++++++++++++++++++---- lib/llparse/node/invoke.js | 3 +- lib/llparse/symbols.js | 1 + test/api-test.js | 57 ++++++++++---------- test/fixtures/index.js | 54 +++++++++++++++++++ 10 files changed, 150 insertions(+), 52 deletions(-) rename lib/llparse/code/{callback.js => match.js} (63%) rename lib/llparse/code/{store.js => value.js} (66%) diff --git a/lib/llparse.js b/lib/llparse.js index 9fd8c72..b0dc74b 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -7,12 +7,12 @@ const kCode = Symbol('code'); // API, really class CodeAPI { - callback(name, body) { - return new internal.code.Callback(name, body); + match(name, body) { + return new internal.code.Match(name, body); } - store(name, body) { - return new internal.code.Store(name, body); + value(name, body) { + return new internal.code.Value(name, body); } } diff --git a/lib/llparse/code/base.js b/lib/llparse/code/base.js index ae81de7..3e4b7b1 100644 --- a/lib/llparse/code/base.js +++ b/lib/llparse/code/base.js @@ -3,21 +3,16 @@ const llparse = require('../'); const kName = Symbol('name'); -const kType = Symbol('type'); const kBody = llparse.symbols.kBody; +const kType = llparse.symbols.kType; class Code { constructor(type, name, body = null) { this[kType] = type; this[kName] = name; this[kBody] = body; - - // TODO(indutny): custom code generators - if (body) - throw new Error('Not implemented yet'); } - get type() { return this[kType]; } get name() { return this[kName]; } } module.exports = Code; diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index 32c4c59..424a7f2 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -1,5 +1,5 @@ 'use strict'; exports.Code = require('./base'); -exports.Callback = require('./callback'); -exports.Store = require('./store'); +exports.Match = require('./match'); +exports.Value = require('./value'); diff --git a/lib/llparse/code/callback.js b/lib/llparse/code/match.js similarity index 63% rename from lib/llparse/code/callback.js rename to lib/llparse/code/match.js index dfa0b06..973cc6b 100644 --- a/lib/llparse/code/callback.js +++ b/lib/llparse/code/match.js @@ -2,9 +2,9 @@ const code = require('./'); -class Callback extends code.Code { +class Match extends code.Code { constructor(name, body) { super('match', name, body); } } -module.exports = Callback; +module.exports = Match; diff --git a/lib/llparse/code/store.js b/lib/llparse/code/value.js similarity index 66% rename from lib/llparse/code/store.js rename to lib/llparse/code/value.js index 7185a22..1df3b40 100644 --- a/lib/llparse/code/store.js +++ b/lib/llparse/code/value.js @@ -2,9 +2,9 @@ const code = require('./'); -class Store extends code.Code { +class Value extends code.Code { constructor(name, body) { super('value', name, body); } } -module.exports = Store; +module.exports = Value; diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 9d5d93b..19db535 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -7,10 +7,12 @@ const llparse = require('./'); const constants = llparse.constants; const MatchSequence = llparse.MatchSequence; +const kBody = llparse.symbols.kBody; const kCases = llparse.symbols.kCases; const kNoAdvance = llparse.symbols.kNoAdvance; const kOtherwise = llparse.symbols.kOtherwise; const kSignature = llparse.symbols.kSignature; +const kType = llparse.symbols.kType; const CCONV = constants.CCONV; @@ -67,7 +69,7 @@ class Compiler { this.state.field(TYPE_INDEX, 'index'); this.nodeMap = new Map(); - this.externalMap = new Map(); + this.codeMap = new Map(); this.counter = new Map(); // redirect blocks by `fn` and `target` @@ -223,24 +225,64 @@ class Compiler { } buildCode(node, code) { - assert.strictEqual(node[kSignature], code.type); + assert.strictEqual(node[kSignature], code[kType]); - const signature = code.type === 'match' ? + const signature = code[kType] === 'match' ? this.signature.callback.match : this.signature.callback.value; const name = code.name; + if (this.codeMap.has(name)) { + const cached = this.codeMap.get(name); + assert.strictEqual(cached.type, signature, + `Conflicting code entries for "${name}"`); + return cached; + } - // TODO(indutny): support code.body - if (this.externalMap.has(name)) { - return this.externalMap.get(name); - } else { + let fn; + if (code[kBody] === null) { const external = this.ir.declare(signature, name); external.attributes = 'alwaysinline'; - this.externalMap.set(name, external); - return external; + fn = external; + } else { + fn = this.buildCodeWithBody(code, signature); } + + this.codeMap.set(name, fn); + return fn; + } + + buildCodeWithBody(code, signature) { + const args = [ + constants.ARG_STATE, + constants.ARG_POS, + constants.ARG_ENDPOS + ]; + + if (code[kType] === 'value') + args.push(constants.ARG_MATCH); + + const fn = this.ir.fn(signature, code.name, args); + + const context = { + fn, + + ret: fn.type.ret, + + state: fn.arg(ARG_STATE), + pos: fn.arg(ARG_POS), + endPos: fn.arg(ARG_ENDPOS), + match: null + }; + + if (code[kType] === 'value') + context.match = fn.arg(ARG_MATCH); + + fn.body.comment('custom user code'); + code[kBody].call(fn.body, this.ir, context); + + return fn; } buildInvoke(info, body, pos) { diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index a87186f..8f7c98f 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -6,6 +6,7 @@ const node = require('./'); const llparse = require('../'); const kNoAdvance = llparse.symbols.kNoAdvance; +const kType = llparse.symbols.kType; const kCode = Symbol('code'); const kMap = Symbol('map'); @@ -23,7 +24,7 @@ class Invoke extends node.Node { 'Only integer keys are allowed in `.invoke()`\'s map'); }); - super('invoke_' + code.name, code.type); + super('invoke_' + code.name, code[kType]); this[kCode] = code; this[kMap] = map; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 0851057..44e08ba 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -5,3 +5,4 @@ exports.kCases = Symbol('cases'); exports.kNoAdvance = Symbol('noAdvance'); exports.kOtherwise = Symbol('otherwise'); exports.kSignature = Symbol('signature'); +exports.kType = Symbol('type'); diff --git a/test/api-test.js b/test/api-test.js index d85695d..fc4f781 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -5,40 +5,28 @@ const llparse = require('../'); const fixtures = require('./fixtures'); +const printMatch = fixtures.printMatch; +const printOff = fixtures.printOff; +const compiledPrint = fixtures.compiledPrint; + describe('LLParse', () => { let p; beforeEach(() => { p = llparse.create('llparse'); }); - const printMatch = (next) => { - const code = p.code.store('print_match'); - - return p.invoke(code, { - 0: next - }, p.error(1, '`print_match` error')); - }; - - const printOff = (next) => { - const code = p.code.callback('print_off'); - - return p.invoke(code, { - 0: next - }, p.error(1, '`print_off` error')); - }; - it('should compile simple parser', (callback) => { const start = p.node('start'); start.match(' ', start); - start.match('HTTP', printOff(start)); + start.match('HTTP', printOff(p, start)); start.select({ 'HEAD': 0, 'GET': 1, 'POST': 2, 'PUT': 3, 'DELETE': 4, 'OPTIONS': 5, 'CONNECT': 6, 'TRACE': 7, 'PATCH': 8 - }, printMatch(start)); + }, printMatch(p, start)); start.otherwise(p.error(3, 'Invalid word')); @@ -53,7 +41,7 @@ describe('LLParse', () => { start.select({ '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9 - }, printMatch(start)); + }, printMatch(p, start)); start.otherwise(p.error(3, 'Invalid word')); @@ -65,9 +53,9 @@ describe('LLParse', () => { it('should support key-value select', (callback) => { const start = p.node('start'); - start.select('0', 0, printMatch(start)); - start.select('1', 1, printMatch(start)); - start.select('2', 2, printMatch(start)); + start.select('0', 0, printMatch(p, start)); + start.select('1', 1, printMatch(p, start)); + start.select('2', 2, printMatch(p, start)); start.otherwise(p.error(3, 'Invalid word')); @@ -84,7 +72,7 @@ describe('LLParse', () => { start.select({ 'A': 0, 'B': 1 - }, printMatch(start)); + }, printMatch(p, start)); start.otherwise(p.error(3, 'Invalid word')); @@ -103,7 +91,7 @@ describe('LLParse', () => { start.select({ 'A': 0, 'B': 1 - }, printMatch(start)); + }, printMatch(p, start)); start.otherwise(p.error(3, 'Invalid word')); @@ -114,6 +102,23 @@ describe('LLParse', () => { callback); }); + it('should support compiled code', (callback) => { + const start = p.node('start'); + + start.select({ + '0': 0, + '1': 1 + }, compiledPrint(p, start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('compiled', p.build(start)); + binary( + '0110', + 'not one\none\none\nnot one\n', + callback); + }); + describe('`.otherwise()`', () => { it('should not advance position by default', (callback) => { const p = llparse.create('llparse'); @@ -126,7 +131,7 @@ describe('LLParse', () => { .otherwise(b); b - .match('B', printOff(b)) + .match('B', printOff(p, b)) .otherwise(a); const binary = fixtures.build('otherwise-noadvance', p.build(a)); @@ -140,7 +145,7 @@ describe('LLParse', () => { const start = p.node('start'); start - .match(' ', printOff(start)) + .match(' ', printOff(p, start)) .skipTo(start); const binary = fixtures.build('otherwise-skip', p.build(start)); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 67ed94c..d0f5e15 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -65,3 +65,57 @@ exports.build = (name, source) => { }); }; }; + +exports.printMatch = (p, next) => { + const code = p.code.value('print_match'); + + return p.invoke(code, { + 0: next + }, p.error(1, '`print_match` error')); +}; + +exports.printOff = (p, next) => { + const code = p.code.match('print_off'); + + return p.invoke(code, { + 0: next + }, p.error(1, '`print_off` error')); +}; + +exports.compiledPrint = (p, next) => { + const code = p.code.value('compiled_print_off', (ir, context) => { + const INT = ir.i(32); + const CSTR = ir.i(8).ptr(); + + const body = context.fn.body; + + const puts = ir.declare(ir.signature(INT, [ CSTR ]), 'puts'); + + const one = ir.cstr('one'); + const notOne = ir.cstr('not one'); + + const castOne = ir._('getelementptr', one.type.to, [ one.type, one ], + [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + const castNotOne = ir._('getelementptr', notOne.type.to, + [ notOne.type, notOne ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + body.push(castOne); + body.push(castNotOne); + + const cmp = ir._('icmp', [ 'eq', context.match.type, context.match ], + context.match.type.v(1)); + body.push(cmp); + + const select = ir._('select', [ ir.i(1), cmp ], [ CSTR, castOne ], + [ CSTR, castNotOne ]); + body.push(select); + + body.push(ir._('call', [ puts.type.ret, puts, '(', + CSTR, select, ')' ])); + + body.terminate('ret', [ context.ret, context.ret.v(0) ]); + }); + + return p.invoke(code, { + 0: next + }, p.error(1, '`print_off` error')); +}; From 73c76525e747db7c4279faa5fc50439485dac510 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 17:22:03 -0500 Subject: [PATCH 095/281] examples: update --- .gitignore | 1 + README.md | 8 ++++---- examples/http/Makefile | 4 +++- examples/http/index.js | 9 ++++----- examples/http/main.c | 6 +++--- lib/llparse.js | 6 ++++++ lib/llparse/compiler.js | 1 + 7 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index aac9824..e66a20b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ test/tmp examples/http/http *.dSYM *.ll +*.bc *.o main 1 diff --git a/README.md b/README.md index 734d44d..2cefb64 100644 --- a/README.md +++ b/README.md @@ -17,20 +17,20 @@ const url = p.node('url'); const http = p.node('http'); // Invoke external C function -const onMethod = p.invoke('on_method', { +const onMethod = p.invoke(p.code.value('on_method'), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); -const urlStart = p.invoke('on_url_start', { +const urlStart = p.invoke(p.code.match('on_url_start'), { 0: url }, p.error(2, '`on_url_start` error')); -const urlEnd = p.invoke('on_url_end', { +const urlEnd = p.invoke(p.code.match('on_url_end'), { 0: http }, p.error(3, '`on_url_end` error')); -const complete = p.invoke('on_complete', { +const complete = p.invoke(p.code.match('on_complete'), { // Restart 0: method }, p.error(4, '`on_complete` error')); diff --git a/examples/http/Makefile b/examples/http/Makefile index d4ecfa4..8173abc 100644 --- a/examples/http/Makefile +++ b/examples/http/Makefile @@ -1,7 +1,9 @@ +CC ?= clang + all: http http: main.c http.ll - clang -g3 -Os -flto -fvisibility=hidden -Wall http.ll main.c -o $@ + $(CC) -g3 -Os -flto -fvisibility=hidden -Wall http.ll main.c -o $@ http.ll: index.js node $< > $@ diff --git a/examples/http/index.js b/examples/http/index.js index 1366f3e..480097d 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -8,20 +8,20 @@ const url = p.node('url'); const http = p.node('http'); // Invoke external C function -const onMethod = p.invoke('on_method', { +const onMethod = p.invoke(p.code.value('on_method'), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); -const urlStart = p.invoke('on_url_start', { +const urlStart = p.invoke(p.code.match('on_url_start'), { 0: url }, p.error(2, '`on_url_start` error')); -const urlEnd = p.invoke('on_url_end', { +const urlEnd = p.invoke(p.code.match('on_url_end'), { 0: http }, p.error(3, '`on_url_end` error')); -const complete = p.invoke('on_complete', { +const complete = p.invoke(p.code.match('on_complete'), { // Restart 0: method }, p.error(4, '`on_complete` error')); @@ -44,7 +44,6 @@ url http .match('HTTP/1.1\r\n\r\n', complete) - // Just for console testing .match('HTTP/1.1\n\n', complete) .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); diff --git a/examples/http/main.c b/examples/http/main.c index bf7d9d0..05772b4 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -11,7 +11,6 @@ struct http_parser_state_s { int error; const char* reason; int index; - int match; const char* url_start; }; @@ -20,8 +19,9 @@ void http_parser_init(http_parser_state_t* s); int http_parser_execute(http_parser_state_t* s, const char* p, const char* endp); -int on_method(http_parser_state_t* s, const char* p, const char* endp) { - fprintf(stdout, "on_method=%d\n", s->match); +int on_method(http_parser_state_t* s, const char* p, const char* endp, + int method) { + fprintf(stdout, "on_method=%d\n", method); return 0; } diff --git a/lib/llparse.js b/lib/llparse.js index b0dc74b..bddbabf 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -1,5 +1,7 @@ 'use strict'; +const assert = require('assert'); + const internal = require('./llparse/'); const kCode = Symbol('code'); @@ -46,6 +48,10 @@ class LLParse { } build(root) { + assert(root, 'Missing required argument for `.build(root)`'); + assert(root instanceof internal.node.Node, + 'Invalid value of `root` in `.build(root)'); + const c = new internal.Compiler(this.prefix); return c.build(root); } diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 19db535..b085413 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -120,6 +120,7 @@ class Compiler { } buildParse() { + // TODO(indutny): change signature to (state*, start*, len)? const sig = IR.signature(TYPE_ERROR, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); const parse = this.ir.fn(sig, this.prefix + '_execute', From 4e76d91c67faec1e76af2de0e99bdbc3dce8f689 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 17:30:54 -0500 Subject: [PATCH 096/281] examples: disable -flto for now --- examples/http/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/http/Makefile b/examples/http/Makefile index 8173abc..8dd1cad 100644 --- a/examples/http/Makefile +++ b/examples/http/Makefile @@ -2,8 +2,10 @@ CC ?= clang all: http +# NOTE: -flto has a bug, and can't be used right now +# See: https://bugs.llvm.org/show_bug.cgi?id=36441 http: main.c http.ll - $(CC) -g3 -Os -flto -fvisibility=hidden -Wall http.ll main.c -o $@ + $(CC) -g3 -Os -fvisibility=hidden -Wall http.ll main.c -o $@ http.ll: index.js node $< > $@ From 70ae7da341f253f3ffc24f02ba252e34882a3471 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 17:32:55 -0500 Subject: [PATCH 097/281] 2.0.0-beta0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0e39cc0..7945ef7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.3.1", + "version": "2.0.0-beta0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a4e8b3b..9afaa4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "1.3.1", + "version": "2.0.0-beta0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 85a5d3e013bee71993cebb3d661cbc613b339e06 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 18:51:57 -0500 Subject: [PATCH 098/281] compiler: custom properties! --- lib/llparse.js | 20 +++++++++-- lib/llparse/compiler.js | 28 ++++++++-------- lib/llparse/constants.js | 9 +++++ lib/llparse/context.js | 72 ++++++++++++++++++++++++++++++++++++++++ lib/llparse/index.js | 2 ++ lib/llparse/property.js | 9 +++++ test/api-test.js | 41 +++++++++++++++++++++++ test/fixtures/main.c | 16 +++++++++ 8 files changed, 180 insertions(+), 17 deletions(-) create mode 100644 lib/llparse/context.js create mode 100644 lib/llparse/property.js diff --git a/lib/llparse.js b/lib/llparse.js index bddbabf..eb1145b 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -5,6 +5,7 @@ const assert = require('assert'); const internal = require('./llparse/'); const kCode = Symbol('code'); +const kProperties = Symbol('properties'); // API, really @@ -23,6 +24,10 @@ class LLParse { this.prefix = prefix || 'llparse'; this[kCode] = new CodeAPI(); + this[kProperties] = { + set: new Set(), + list: [] + }; } static create(prefix) { @@ -43,8 +48,17 @@ class LLParse { return new internal.node.Invoke(name, map, otherwise); } - skip() { - return new internal.node.Skip(); + property(type, name) { + if (internal.constants.RESERVED_PROPERTY_NAMES.has(name)) + throw new Error(`Can't use reserved property name: "${name}"`); + + const props = this[kProperties]; + if (props.set.has(name)) + throw new Error(`Duplicate property with a name: "${name}"`); + + const prop = new internal.Property(type, name); + props.set.add(name); + props.list.push(prop); } build(root) { @@ -52,7 +66,7 @@ class LLParse { assert(root instanceof internal.node.Node, 'Invalid value of `root` in `.build(root)'); - const c = new internal.Compiler(this.prefix); + const c = new internal.Compiler(this.prefix, this[kProperties].list); return c.build(root); } } diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index b085413..23efed2 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -24,6 +24,7 @@ const TYPE_MATCH = constants.TYPE_MATCH; const TYPE_INDEX = constants.TYPE_INDEX; const TYPE_ERROR = constants.TYPE_ERROR; const TYPE_REASON = constants.TYPE_REASON; +const TYPE_DATA = constants.TYPE_DATA; const ATTR_STATE = constants.ATTR_STATE; const ATTR_POS = constants.ATTR_POS; @@ -40,7 +41,7 @@ const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; class Compiler { - constructor(prefix) { + constructor(prefix, properties) { this.prefix = prefix; this.ir = new IR(); @@ -67,6 +68,11 @@ class Compiler { this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); this.state.field(TYPE_INDEX, 'index'); + this.state.field(TYPE_DATA, 'data'); + + properties.forEach((prop) => { + this.state.field(prop.type(this.ir, this.state), prop.name); + }); this.nodeMap = new Map(); this.codeMap = new Map(); @@ -266,19 +272,11 @@ class Compiler { const fn = this.ir.fn(signature, code.name, args); - const context = { - fn, - - ret: fn.type.ret, - - state: fn.arg(ARG_STATE), - pos: fn.arg(ARG_POS), - endPos: fn.arg(ARG_ENDPOS), - match: null - }; + fn.visibility = 'internal'; + fn.cconv = CCONV; + fn.attributes = 'nounwind'; - if (code[kType] === 'value') - context.match = fn.arg(ARG_MATCH); + const context = new llparse.Context(code, fn); fn.body.comment('custom user code'); code[kBody].call(fn.body, this.ir, context); @@ -303,7 +301,9 @@ class Compiler { args.push(')'); - const call = IR._('call', args); + const cconv = code.cconv ? ' ' + code.cconv : ''; + + const call = IR._('call' + cconv, args); body.push(call); const keys = Object.keys(info.node.map).map(key => key | 0); diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 398ad9c..855c68f 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -12,6 +12,7 @@ exports.TYPE_MATCH = exports.INT; exports.TYPE_INDEX = exports.INT; exports.TYPE_ERROR = exports.INT; exports.TYPE_REASON = IR.i(8).ptr(); +exports.TYPE_DATA = IR.i(8).ptr(); exports.ARG_STATE = 's'; exports.ARG_POS = 'p'; @@ -29,3 +30,11 @@ exports.ATTR_SEQUENCE = exports.ATTR_POS; exports.SEQUENCE_COMPLETE = 0; exports.SEQUENCE_PAUSE = 1; exports.SEQUENCE_MISMATCH = 2; + +exports.RESERVED_PROPERTY_NAMES = new Set([ + 'current', + 'error', + 'reason', + 'index', + 'data' +]); diff --git a/lib/llparse/context.js b/lib/llparse/context.js new file mode 100644 index 0000000..788ff96 --- /dev/null +++ b/lib/llparse/context.js @@ -0,0 +1,72 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const llparse = require('./'); +const constants = llparse.constants; + +const kType = llparse.symbols.kType; +const kFn = Symbol('fn'); +const kLookup = Symbol('lookup'); + +const INT = constants.INT; + +const ARG_STATE = constants.ARG_STATE; +const ARG_POS = constants.ARG_POS; +const ARG_ENDPOS = constants.ARG_ENDPOS; +const ARG_MATCH = constants.ARG_MATCH; + +class Context { + constructor(code, fn) { + this[kFn] = fn; + + this.ret = this.fn.type.ret; + this.state = fn.arg(ARG_STATE); + this.pos = fn.arg(ARG_POS); + this.endPos = fn.arg(ARG_ENDPOS); + + this.match = null; + + if (code[kType] === 'value') + this.match = fn.arg(ARG_MATCH); + } + + get fn() { return this[kFn]; } + + load(body, field) { + const lookup = this[kLookup](this[kFn], field); + body.push(lookup.instr); + + const res = IR._('load', lookup.type, [ lookup.type.ptr(), lookup.instr ]); + body.push(res); + + return res; + } + + store(body, field, value) { + const lookup = this[kLookup](this[kFn], field); + body.push(lookup.instr); + + const res = IR._('store', [ lookup.type, value ], + [ lookup.type.ptr(), lookup.instr ]); + res.void(); + body.push(res); + } + + [kLookup](fn, field) { + const stateArg = fn.arg(ARG_STATE); + const stateType = stateArg.type.to; + + const index = stateType.lookup(field); + const instr = IR._('getelementptr', stateType, + [ stateArg.type, stateArg ], + [ INT, INT.v(0) ], + [ INT, INT.v(index) ]); + + return { + type: stateType.fields[index].type, + instr + }; + } +} +module.exports = Context; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index 86c3b56..065ee26 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -9,4 +9,6 @@ exports.code = require('./code'); exports.node = require('./node'); exports.Trie = require('./trie'); exports.MatchSequence = require('./match-sequence'); +exports.Property = require('./property'); +exports.Context = require('./context'); exports.Compiler = require('./compiler'); diff --git a/lib/llparse/property.js b/lib/llparse/property.js new file mode 100644 index 0000000..0fbdaa4 --- /dev/null +++ b/lib/llparse/property.js @@ -0,0 +1,9 @@ +'use strict'; + +class Property { + constructor(type, name) { + this.type = type; + this.name = name; + } +} +module.exports = Property; diff --git a/test/api-test.js b/test/api-test.js index fc4f781..81d3a7c 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -119,6 +119,47 @@ describe('LLParse', () => { callback); }); + it('should support custom state properties', (callback) => { + const start = p.node('start'); + const error = p.error(3, 'Invalid word'); + + p.property(ir => ir.i(8), 'custom'); + + const setCustom = p.code.value('set_custom', (ir, context) => { + const body = context.fn.body; + const trunc = ir._('trunc', + [ context.match.type, context.match, 'to', ir.i(8) ]); + body.push(trunc); + + context.store(body, 'custom', trunc); + body.terminate('ret', [ context.ret, context.ret.v(0) ]); + }); + + const getCustom = p.code.match('get_custom', (ir, context) => { + const body = context.fn.body; + const load = context.load(body, 'custom'); + const zext = ir._('zext', [ ir.i(8), load, 'to', context.ret.type ]); + body.push(zext); + body.terminate('ret', [ context.ret, zext ]); + }); + + const second = p.invoke(getCustom, { + 0: p.invoke(p.code.match('print_zero'), { 0: start }, error), + 1: p.invoke(p.code.match('print_one'), { 0: start }, error) + }, error); + + start + .select({ + '0': 0, + '1': 1 + }, p.invoke(setCustom, { 0: second }, error)) + .otherwise(error); + + const binary = fixtures.build('custom-prop', p.build(start)); + + binary('0110', 'zero\none\none\nzero\n', callback); + }); + describe('`.otherwise()`', () => { it('should not advance position by default', (callback) => { const p = llparse.create('llparse'); diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 374e971..10f006f 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -21,6 +21,22 @@ static const char* start; void llparse_init(struct state* s); int llparse_execute(struct state* s, const char* p, const char* endp); +int print_zero(struct state* s, const char* p, const char* endp) { + if (bench) + return 0; + + fprintf(stdout, "zero\n"); + return 0; +} + +int print_one(struct state* s, const char* p, const char* endp) { + if (bench) + return 0; + + fprintf(stdout, "one\n"); + return 0; +} + int print_off(struct state* s, const char* p, const char* endp) { if (bench) return 0; From 7a3d94b15cd05d3db6ba717f6018d780776e5cae Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 18:58:09 -0500 Subject: [PATCH 099/281] readme: update --- README.md | 16 ++++++++++++++-- examples/http/index.js | 16 ++++++++++++++-- examples/http/main.c | 19 ++++++++----------- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 2cefb64..5e51a05 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,24 @@ const beforeUrl = p.node('before_url'); const url = p.node('url'); const http = p.node('http'); -// Invoke external C function -const onMethod = p.invoke(p.code.value('on_method'), { +// Add custom uint8_t property to the state +p.property(ir => ir.i(8), 'method'); + +// Store method inside a custom property +const onMethod = p.invoke(p.code.value('on_method', (ir, context) => { + const body = context.fn.body; + const trunc = ir._('trunc', + [ context.match.type, context.match, 'to', ir.i(8) ]); + body.push(trunc); + + context.store(body, 'method', trunc); + body.terminate('ret', [ context.ret, context.ret.v(0) ]); +}), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); +// Invoke external C function const urlStart = p.invoke(p.code.match('on_url_start'), { 0: url }, p.error(2, '`on_url_start` error')); diff --git a/examples/http/index.js b/examples/http/index.js index 480097d..22861ed 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -7,12 +7,24 @@ const beforeUrl = p.node('before_url'); const url = p.node('url'); const http = p.node('http'); -// Invoke external C function -const onMethod = p.invoke(p.code.value('on_method'), { +// Add custom uint8_t property to the state +p.property(ir => ir.i(8), 'method'); + +// Store method inside a custom property +const onMethod = p.invoke(p.code.value('on_method', (ir, context) => { + const body = context.fn.body; + const trunc = ir._('trunc', + [ context.match.type, context.match, 'to', ir.i(8) ]); + body.push(trunc); + + context.store(body, 'method', trunc); + body.terminate('ret', [ context.ret, context.ret.v(0) ]); +}), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); +// Invoke external C function const urlStart = p.invoke(p.code.match('on_url_start'), { 0: url }, p.error(2, '`on_url_start` error')); diff --git a/examples/http/main.c b/examples/http/main.c index 05772b4..670df32 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -11,7 +11,9 @@ struct http_parser_state_s { int error; const char* reason; int index; + void* data; + unsigned int method : 8; const char* url_start; }; @@ -19,18 +21,13 @@ void http_parser_init(http_parser_state_t* s); int http_parser_execute(http_parser_state_t* s, const char* p, const char* endp); -int on_method(http_parser_state_t* s, const char* p, const char* endp, - int method) { - fprintf(stdout, "on_method=%d\n", method); - return 0; -} - - -static void on_url_part(const char* p, const char* endp) { +static void on_url_part(http_parser_state_t* s, const char* p, + const char* endp) { if (p == endp) return; - fprintf(stdout, "url_part=\"%.*s\"\n", (int) (endp - p), p); + fprintf(stdout, "method=%d url_part=\"%.*s\"\n", s->method, + (int) (endp - p), p); } @@ -41,7 +38,7 @@ int on_url_start(http_parser_state_t* s, const char* p, const char* endp) { int on_url_end(http_parser_state_t* s, const char* p, const char* endp) { - on_url_part(s->url_start, p - 1); + on_url_part(s, s->url_start, p - 1); fprintf(stdout, "url end\n"); s->url_start = NULL; return 0; @@ -82,7 +79,7 @@ int main(int argc, char** argv) { } if (s.url_start != NULL) - on_url_part(s.url_start, endp); + on_url_part(&s, s.url_start, endp); } return 0; From 7b5ed8ff0ee40f7e2f2526800e81fe1ee055f921 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 19:00:50 -0500 Subject: [PATCH 100/281] context: add comments to helper methods --- lib/llparse/context.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/llparse/context.js b/lib/llparse/context.js index 788ff96..37af8c3 100644 --- a/lib/llparse/context.js +++ b/lib/llparse/context.js @@ -34,6 +34,8 @@ class Context { get fn() { return this[kFn]; } load(body, field) { + body.comment(`load state[${field}]`); + const lookup = this[kLookup](this[kFn], field); body.push(lookup.instr); @@ -44,6 +46,8 @@ class Context { } store(body, field, value) { + body.comment(`store state[${field}] = ...`); + const lookup = this[kLookup](this[kFn], field); body.push(lookup.instr); From ea688af7a79a532eb210bd75ccae954650cd6f60 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 19:00:58 -0500 Subject: [PATCH 101/281] 2.0.0-beta1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7945ef7..f9d3597 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta0", + "version": "2.0.0-beta1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 9afaa4f..2143a4a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta0", + "version": "2.0.0-beta1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 060046689c93ea6b1cca1e686e497ce8f9b1b2d9 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 19:24:26 -0500 Subject: [PATCH 102/281] test: remove unused field --- test/fixtures/main.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 10f006f..a7d1076 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -9,7 +9,6 @@ struct state { int error; const char* reason; int index; - int match; }; /* 8 gb */ From a7a6ef68745fc09f93e9376d62d3732e6ce7f6c1 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 20:04:05 -0500 Subject: [PATCH 103/281] compiler: zero-initialize fields, introduce `mark` --- lib/llparse/compiler.js | 9 ++++++++- lib/llparse/constants.js | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 23efed2..b455f21 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -68,6 +68,9 @@ class Compiler { this.state.field(TYPE_ERROR, 'error'); this.state.field(TYPE_REASON, 'reason'); this.state.field(TYPE_INDEX, 'index'); + this.state.field(TYPE_INPUT, 'mark'); + + // Public fields this.state.field(TYPE_DATA, 'data'); properties.forEach((prop) => { @@ -105,7 +108,9 @@ class Compiler { current: this.field(init, 'current'), error: this.field(init, 'error'), reason: this.field(init, 'reason'), - index: this.field(init, 'index') + index: this.field(init, 'index'), + mark: this.field(init, 'mark'), + data: this.field(init, 'data') }; Object.keys(fields).forEach(key => init.body.push(fields[key])); @@ -119,6 +124,8 @@ class Compiler { store(fields.error, TYPE_ERROR, TYPE_ERROR.v(0)); store(fields.reason, TYPE_REASON, TYPE_REASON.v(null)); store(fields.index, TYPE_INDEX, TYPE_INDEX.v(0)); + store(fields.mark, TYPE_INPUT, TYPE_INPUT.v(null)); + store(fields.data, TYPE_DATA, TYPE_DATA.v(null)); init.body.terminate('ret', IR.void()); diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 855c68f..7bd1a83 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -36,5 +36,6 @@ exports.RESERVED_PROPERTY_NAMES = new Set([ 'error', 'reason', 'index', - 'data' + 'data', + 'mark' ]); From d7a980d10bbf8f9936d73887d333c4fa067e3a94 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 23:34:09 -0500 Subject: [PATCH 104/281] code: introduce store/load to disencourage manual --- README.md | 10 +-------- examples/http/index.js | 10 +-------- examples/http/main.c | 1 + lib/llparse.js | 10 +++++++++ lib/llparse/code/index.js | 2 ++ lib/llparse/code/load.js | 45 ++++++++++++++++++++++++++++++++++++++ lib/llparse/code/store.js | 46 +++++++++++++++++++++++++++++++++++++++ lib/llparse/context.js | 30 ++++++++++++++++--------- package-lock.json | 6 ++--- package.json | 2 +- test/api-test.js | 22 ++----------------- 11 files changed, 132 insertions(+), 52 deletions(-) create mode 100644 lib/llparse/code/load.js create mode 100644 lib/llparse/code/store.js diff --git a/README.md b/README.md index 5e51a05..c70488c 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,7 @@ const http = p.node('http'); p.property(ir => ir.i(8), 'method'); // Store method inside a custom property -const onMethod = p.invoke(p.code.value('on_method', (ir, context) => { - const body = context.fn.body; - const trunc = ir._('trunc', - [ context.match.type, context.match, 'to', ir.i(8) ]); - body.push(trunc); - - context.store(body, 'method', trunc); - body.terminate('ret', [ context.ret, context.ret.v(0) ]); -}), { +const onMethod = p.invoke(p.code.store('on_method', 'method'), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); diff --git a/examples/http/index.js b/examples/http/index.js index 22861ed..73edd97 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -11,15 +11,7 @@ const http = p.node('http'); p.property(ir => ir.i(8), 'method'); // Store method inside a custom property -const onMethod = p.invoke(p.code.value('on_method', (ir, context) => { - const body = context.fn.body; - const trunc = ir._('trunc', - [ context.match.type, context.match, 'to', ir.i(8) ]); - body.push(trunc); - - context.store(body, 'method', trunc); - body.terminate('ret', [ context.ret, context.ret.v(0) ]); -}), { +const onMethod = p.invoke(p.code.store('on_method', 'method'), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); diff --git a/examples/http/main.c b/examples/http/main.c index 670df32..23fa374 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -12,6 +12,7 @@ struct http_parser_state_s { const char* reason; int index; void* data; + void* mark; unsigned int method : 8; const char* url_start; diff --git a/lib/llparse.js b/lib/llparse.js index eb1145b..d7e5059 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -17,6 +17,16 @@ class CodeAPI { value(name, body) { return new internal.code.Value(name, body); } + + // Helpers + + store(name, field) { + return new internal.code.Store(name, field); + } + + load(name, field) { + return new internal.code.Load(name, field); + } } class LLParse { diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index 424a7f2..0230309 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -3,3 +3,5 @@ exports.Code = require('./base'); exports.Match = require('./match'); exports.Value = require('./value'); +exports.Store = require('./store'); +exports.Load = require('./load'); diff --git a/lib/llparse/code/load.js b/lib/llparse/code/load.js new file mode 100644 index 0000000..cb4e8cb --- /dev/null +++ b/lib/llparse/code/load.js @@ -0,0 +1,45 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +const kCompile = Symbol('compile'); + +class Load extends code.Match { + constructor(name, field) { + const store = (ir, context) => this[kCompile](ir, context, field); + + super(name, store); + } + + [kCompile](ir, context, field) { + const body = context.fn.body; + + const stateType = context.state.type.to; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(context.ret.isInt()); + + let adj = context.load(body, field); + + // Same type! + if (fieldType.type === context.ret) { + // Truncate + } else if (fieldType.width > context.ret.width) { + adj = ir._('trunc', + [ fieldType, adj, 'to', context.ret ]); + body.push(adj); + // Extend + } else { + assert(fieldType.width < context.ret.width); + adj = ir._('sext', + [ fieldType, adj, 'to', context.ret ]); + body.push(adj); + } + + body.terminate('ret', [ context.ret, adj ]); + } +} +module.exports = Load; diff --git a/lib/llparse/code/store.js b/lib/llparse/code/store.js new file mode 100644 index 0000000..50652ae --- /dev/null +++ b/lib/llparse/code/store.js @@ -0,0 +1,46 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +const kCompile = Symbol('compile'); + +class Store extends code.Value { + constructor(name, field) { + const store = (ir, context) => this[kCompile](ir, context, field); + + super(name, store); + } + + [kCompile](ir, context, field) { + const body = context.fn.body; + + const stateType = context.state.type.to; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(context.match.type.isInt()); + let adj; + + // Same type! + if (fieldType.type === context.match.type) { + adj = context.match; + // Extend + } else if (fieldType.width > context.match.type.width) { + adj = ir._('sext', + [ context.match.type, context.match, 'to', fieldType ]); + body.push(adj); + // Truncate + } else { + assert(fieldType.width < context.match.type.width); + adj = ir._('trunc', + [ context.match.type, context.match, 'to', fieldType ]); + body.push(adj); + } + + context.store(body, field, adj); + body.terminate('ret', [ context.ret, context.ret.v(0) ]); + } +} +module.exports = Store; diff --git a/lib/llparse/context.js b/lib/llparse/context.js index 37af8c3..7f9844c 100644 --- a/lib/llparse/context.js +++ b/lib/llparse/context.js @@ -7,6 +7,11 @@ const constants = llparse.constants; const kType = llparse.symbols.kType; const kFn = Symbol('fn'); +const kRet = Symbol('ret'); +const kState = Symbol('state'); +const kPos = Symbol('pos'); +const kEndPos = Symbol('endPos'); +const kMatch = Symbol('match'); const kLookup = Symbol('lookup'); const INT = constants.INT; @@ -20,23 +25,28 @@ class Context { constructor(code, fn) { this[kFn] = fn; - this.ret = this.fn.type.ret; - this.state = fn.arg(ARG_STATE); - this.pos = fn.arg(ARG_POS); - this.endPos = fn.arg(ARG_ENDPOS); + this[kRet] = this.fn.type.ret; + this[kState] = fn.arg(ARG_STATE); + this[kPos] = fn.arg(ARG_POS); + this[kEndPos] = fn.arg(ARG_ENDPOS); - this.match = null; + this[kMatch] = null; if (code[kType] === 'value') - this.match = fn.arg(ARG_MATCH); + this[kMatch] = fn.arg(ARG_MATCH); } get fn() { return this[kFn]; } + get ret() { return this[kRet]; } + get state() { return this[kState]; } + get pos() { return this[kPos]; } + get endPos() { return this[kEndPos]; } + get match() { return this[kMatch]; } load(body, field) { body.comment(`load state[${field}]`); - const lookup = this[kLookup](this[kFn], field); + const lookup = this[kLookup](field); body.push(lookup.instr); const res = IR._('load', lookup.type, [ lookup.type.ptr(), lookup.instr ]); @@ -48,7 +58,7 @@ class Context { store(body, field, value) { body.comment(`store state[${field}] = ...`); - const lookup = this[kLookup](this[kFn], field); + const lookup = this[kLookup](field); body.push(lookup.instr); const res = IR._('store', [ lookup.type, value ], @@ -57,8 +67,8 @@ class Context { body.push(res); } - [kLookup](fn, field) { - const stateArg = fn.arg(ARG_STATE); + [kLookup](field) { + const stateArg = this.state; const stateType = stateArg.type.to; const index = stateType.lookup(field); diff --git a/package-lock.json b/package-lock.json index f9d3597..419fb47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -752,9 +752,9 @@ } }, "llvm-ir": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.3.0.tgz", - "integrity": "sha512-9FfnZCS2kVS/OoSVdInryRY6lkR35lfuQEg+m56F2DhNJ6SWDK73/di1LW+qxJPdFBjIc/Jy1NBDDcgFKUXaJw==" + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.4.0.tgz", + "integrity": "sha512-41uG9yn5bZQ7BUf/d2YIrdF14wBn/hgp2NoirDQmDMW0f2kGe3ExCHEzc2wPhYHqQBPJTv/Vef3PmWLpZlXiyA==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index 2143a4a..a46307f 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.3.0" + "llvm-ir": "^1.4.0" }, "devDependencies": { "async": "^2.6.0", diff --git a/test/api-test.js b/test/api-test.js index 81d3a7c..86e3c46 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -125,25 +125,7 @@ describe('LLParse', () => { p.property(ir => ir.i(8), 'custom'); - const setCustom = p.code.value('set_custom', (ir, context) => { - const body = context.fn.body; - const trunc = ir._('trunc', - [ context.match.type, context.match, 'to', ir.i(8) ]); - body.push(trunc); - - context.store(body, 'custom', trunc); - body.terminate('ret', [ context.ret, context.ret.v(0) ]); - }); - - const getCustom = p.code.match('get_custom', (ir, context) => { - const body = context.fn.body; - const load = context.load(body, 'custom'); - const zext = ir._('zext', [ ir.i(8), load, 'to', context.ret.type ]); - body.push(zext); - body.terminate('ret', [ context.ret, zext ]); - }); - - const second = p.invoke(getCustom, { + const second = p.invoke(p.code.load('load', 'custom'), { 0: p.invoke(p.code.match('print_zero'), { 0: start }, error), 1: p.invoke(p.code.match('print_one'), { 0: start }, error) }, error); @@ -152,7 +134,7 @@ describe('LLParse', () => { .select({ '0': 0, '1': 1 - }, p.invoke(setCustom, { 0: second }, error)) + }, p.invoke(p.code.store('store', 'custom'), { 0: second }, error)) .otherwise(error); const binary = fixtures.build('custom-prop', p.build(start)); From 71033e1d56bf5ff112f562cfc2c1641ca7b8f1e6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 23:34:32 -0500 Subject: [PATCH 105/281] 2.0.0-beta2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 419fb47..7323517 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta1", + "version": "2.0.0-beta2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a46307f..b7a8d75 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta1", + "version": "2.0.0-beta2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From 57ebb862395d204f60f51c5e6aa4b5bcc2f7bbba Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 23:56:09 -0500 Subject: [PATCH 106/281] compiler: debug print option --- lib/llparse.js | 10 ++++++++-- lib/llparse/compiler.js | 44 ++++++++++++++++++++++++++++++++++++++--- package-lock.json | 6 +++--- package.json | 2 +- test/fixtures/index.js | 2 +- test/fixtures/main.c | 7 +++++++ 6 files changed, 61 insertions(+), 10 deletions(-) diff --git a/lib/llparse.js b/lib/llparse.js index d7e5059..6e9276f 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -71,12 +71,18 @@ class LLParse { props.list.push(prop); } - build(root) { + build(root, options) { assert(root, 'Missing required argument for `.build(root)`'); assert(root instanceof internal.node.Node, 'Invalid value of `root` in `.build(root)'); - const c = new internal.Compiler(this.prefix, this[kProperties].list); + options = options || {}; + + const c = new internal.Compiler({ + prefix: this.prefix, + properties: this[kProperties].list, + debug: options.debug || false + }); return c.build(root); } } diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index b455f21..29c1538 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -41,8 +41,10 @@ const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; class Compiler { - constructor(prefix, properties) { - this.prefix = prefix; + constructor(options) { + this.options = Object.assign({}, options); + + this.prefix = this.options.prefix; this.ir = new IR(); this.state = this.ir.struct(`${this.prefix}_state`); @@ -73,7 +75,7 @@ class Compiler { // Public fields this.state.field(TYPE_DATA, 'data'); - properties.forEach((prop) => { + this.options.properties.forEach((prop) => { this.state.field(prop.type(this.ir, this.state), prop.name); }); @@ -86,6 +88,8 @@ class Compiler { const matchSequence = new MatchSequence(this.prefix, this.ir, this.state); this.matchSequence = matchSequence.build(); + + this.debugMethod = null; } build(root) { @@ -205,6 +209,38 @@ class Compiler { return fn; } + debug(fn, body, string) { + if (!this.options.debug) + return body; + + const str = this.ir.cstr(string); + const cast = IR._('getelementptr', str.type.to, [ str.type, str ], + [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + body.push(cast); + + // Lazily declare debug method + if (this.debugMethod === null) { + const sig = IR.signature(IR.void(), + [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_INPUT ]); + + this.debugMethod = this.ir.declare(sig, this.options.debug); + } + + const args = [ + IR.void(), this.debugMethod, '(', + this.state.ptr(), fn.arg(ARG_STATE), ',', + TYPE_INPUT, fn.arg(ARG_POS), ',', + TYPE_INPUT, fn.arg(ARG_ENDPOS), ',', + TYPE_INPUT, cast, + ')' + ]; + + const call = IR._('call', args).void(); + body.push(call); + + return body; + } + buildNode(node) { if (this.nodeMap.has(node)) return this.nodeMap.get(node); @@ -214,6 +250,8 @@ class Compiler { let body = this.buildPrologue(node, fn); + body = this.debug(fn, body, `Entering node: "${node.name}"`); + if (node instanceof llparse.node.Error) { const info = { node, fn, otherwise: null }; this.buildError(info, body); diff --git a/package-lock.json b/package-lock.json index 7323517..1d9a29a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -752,9 +752,9 @@ } }, "llvm-ir": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.4.0.tgz", - "integrity": "sha512-41uG9yn5bZQ7BUf/d2YIrdF14wBn/hgp2NoirDQmDMW0f2kGe3ExCHEzc2wPhYHqQBPJTv/Vef3PmWLpZlXiyA==" + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.4.2.tgz", + "integrity": "sha512-wEWBCyECdHO+Mq/BWMdn9X2PbgBAT4fCe+dYTlG6KyPLfHyMn/EN0xfXgzcp6oqSsWtPaa7vyN72lL2DPVMBqg==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index b7a8d75..bd1ef5f 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.4.0" + "llvm-ir": "^1.4.2" }, "devDependencies": { "async": "^2.6.0", diff --git a/test/fixtures/index.js b/test/fixtures/index.js index d0f5e15..f4956e8 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -28,7 +28,7 @@ exports.build = (name, source) => { fs.writeFileSync(file, source); const ret = spawnSync(CLANG, - [ '-flto', '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); + [ '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); if (ret.status !== 0) { process.stderr.write(ret.stdout); process.stderr.write(ret.stderr); diff --git a/test/fixtures/main.c b/test/fixtures/main.c index a7d1076..feda281 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -20,6 +20,13 @@ static const char* start; void llparse_init(struct state* s); int llparse_execute(struct state* s, const char* p, const char* endp); +void debug(struct state* s, const char* p, const char* endp, const char* msg) { + if (bench) + return; + + fprintf(stderr, "off=%d msg=%s\n", (int) (p - start), msg); +} + int print_zero(struct state* s, const char* p, const char* endp) { if (bench) return 0; From f9080345dfdf01b94a8b58d0a84cf5132d530ba0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 19 Feb 2018 23:56:35 -0500 Subject: [PATCH 107/281] 2.0.0-beta3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1d9a29a..75aec76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta2", + "version": "2.0.0-beta3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index bd1ef5f..4332091 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta2", + "version": "2.0.0-beta3", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", From a31fc93f412981407ad0ba7a901e4816af0a25fe Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 00:04:42 -0500 Subject: [PATCH 108/281] readme: improve language --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c70488c..6842500 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ const beforeUrl = p.node('before_url'); const url = p.node('url'); const http = p.node('http'); -// Add custom uint8_t property to the state +// Add custom int8_t property to the state p.property(ir => ir.i(8), 'method'); -// Store method inside a custom property +// Store method inside the custom property const onMethod = p.invoke(p.code.store('on_method', 'method'), { // If that function returns zero 0: beforeUrl From c1c38b5ce84a4979028b7ae033d26b4096b737b3 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 19:45:54 -0500 Subject: [PATCH 109/281] code: automatic names for store/load --- README.md | 2 +- examples/http/index.js | 2 +- lib/llparse.js | 49 +++++++++++++++++++++++++++------- lib/llparse/compiler.js | 2 +- lib/llparse/node/base.js | 5 +++- lib/llparse/node/index.js | 2 ++ lib/llparse/node/invoke.js | 7 +++-- lib/llparse/node/span-end.js | 15 +++++++++++ lib/llparse/node/span-start.js | 15 +++++++++++ lib/llparse/symbols.js | 1 + test/api-test.js | 22 ++------------- test/fixtures/index.js | 38 -------------------------- 12 files changed, 84 insertions(+), 76 deletions(-) create mode 100644 lib/llparse/node/span-end.js create mode 100644 lib/llparse/node/span-start.js diff --git a/README.md b/README.md index 6842500..7aa16a6 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ const http = p.node('http'); p.property(ir => ir.i(8), 'method'); // Store method inside the custom property -const onMethod = p.invoke(p.code.store('on_method', 'method'), { +const onMethod = p.invoke(p.code.store('method'), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); diff --git a/examples/http/index.js b/examples/http/index.js index 73edd97..a51a873 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -11,7 +11,7 @@ const http = p.node('http'); p.property(ir => ir.i(8), 'method'); // Store method inside a custom property -const onMethod = p.invoke(p.code.store('on_method', 'method'), { +const onMethod = p.invoke(p.code.store('method'), { // If that function returns zero 0: beforeUrl }, p.error(1, '`on_method` error')); diff --git a/lib/llparse.js b/lib/llparse.js index 6e9276f..084749f 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -5,35 +5,59 @@ const assert = require('assert'); const internal = require('./llparse/'); const kCode = Symbol('code'); +const kName = Symbol('name'); +const kPrefix = Symbol('prefix'); const kProperties = Symbol('properties'); // API, really class CodeAPI { - match(name, body) { - return new internal.code.Match(name, body); + constructor(prefix) { + this[kPrefix] = prefix; + } + + // TODO(indutny): should we allow custom bodies here? + match(name) { + return new internal.code.Match(name, null); } - value(name, body) { - return new internal.code.Value(name, body); + // TODO(indutny): should we allow custom bodies here? + value(name) { + return new internal.code.Value(name, null); } // Helpers - store(name, field) { - return new internal.code.Store(name, field); + store(field) { + return new internal.code.Store(this[kPrefix] + '__load_' + field, field); + } + + load(field) { + return new internal.code.Load(this[kPrefix] + '__store_' + field, field); + } +} + +class Span { + constructor(name, code, prefix) { + this[kName] = name; + this[kCode] = code; + this[kPrefix] = prefix; + } + + start() { + return new internal.node.SpanStart(this[kName], this[kPrefix]); } - load(name, field) { - return new internal.code.Load(name, field); + end() { + return new internal.node.SpanEnd(this[kName], this[kPrefix], this[kCode]); } } class LLParse { constructor(prefix) { - this.prefix = prefix || 'llparse'; + this[kPrefix] = prefix || 'llparse'; - this[kCode] = new CodeAPI(); + this[kCode] = new CodeAPI(this[kPrefix]); this[kProperties] = { set: new Set(), list: [] @@ -44,6 +68,7 @@ class LLParse { return new LLParse(prefix); } + get prefix() { return this[kPrefix]; } get code() { return this[kCode]; } node(name) { @@ -71,6 +96,10 @@ class LLParse { props.list.push(prop); } + span(code) { + return new Span(code); + } + build(root, options) { assert(root, 'Missing required argument for `.build(root)`'); assert(root instanceof internal.node.Node, diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 29c1538..7ea1dfc 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -184,7 +184,7 @@ class Compiler { index = 0; this.counter.set(node.name, index + 1); - const name = `${this.prefix}__${node.name}` + + const name = `${this.prefix}__n_${node.name}` + `${index === 0 ? '' : '_' + index}`; let fn; diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 89dbc14..b63c7e9 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -10,11 +10,12 @@ const kNoAdvance = llparse.symbols.kNoAdvance; const kCases = llparse.symbols.kCases; const kSignature = llparse.symbols.kSignature; +const kName = Symbol('name'); const kCheckIsMatch = Symbol('checkIsMatch'); class Node { constructor(name, signature = 'match') { - this.name = name; + this[kName] = name; this[kSignature] = signature; this[kCases] = []; @@ -23,6 +24,8 @@ class Node { this[kNoAdvance] = false; } + get name() { return this[kName]; } + match(value, next) { // .match([ ... ], next) if (Array.isArray(value)) { diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js index d82dcdc..c17c9fc 100644 --- a/lib/llparse/node/index.js +++ b/lib/llparse/node/index.js @@ -3,3 +3,5 @@ exports.Node = require('./base'); exports.Error = require('./error'); exports.Invoke = require('./invoke'); +exports.SpanStart = require('./span-start'); +exports.SpanEnd = require('./span-end'); diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 8f7c98f..4a25714 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -11,11 +11,9 @@ const kCode = Symbol('code'); const kMap = Symbol('map'); class Invoke extends node.Node { - constructor(code, map, otherwise) { + constructor(code, map = {}, otherwise = null) { assert.strictEqual(typeof map, 'object', 'Invalid `map` argument of `.invoke()`, must be an Object'); - assert(otherwise instanceof node.Node, - 'Invalid `otherwise` argument of `.invoke()`, must be a Node'); assert(code instanceof llparse.code.Code, 'Invalid `code` argument of `.invoke()`, must be a Code instance'); @@ -29,7 +27,8 @@ class Invoke extends node.Node { this[kCode] = code; this[kMap] = map; - this.otherwise(otherwise); + if (otherwise) + this.otherwise(otherwise); this[kNoAdvance] = true; } diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js new file mode 100644 index 0000000..417a978 --- /dev/null +++ b/lib/llparse/node/span-end.js @@ -0,0 +1,15 @@ +'use strict'; + +const llparse = require('../'); +const node = require('./'); + +const kNoAdvance = llparse.symbols.kNoAdvance; + +class SpanEnd extends node.Node { + constructor() { + super(); + this[kNoAdvance] = true; + throw new Error('To be implemented'); + } +} +module.exports = SpanEnd; diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js new file mode 100644 index 0000000..e504e37 --- /dev/null +++ b/lib/llparse/node/span-start.js @@ -0,0 +1,15 @@ +'use strict'; + +const llparse = require('../'); +const node = require('./'); + +const kNoAdvance = llparse.symbols.kNoAdvance; + +class SpanStart extends node.Node { + constructor() { + super(); + this[kNoAdvance] = true; + throw new Error('To be implemented'); + } +} +module.exports = SpanStart; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 44e08ba..42af780 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -2,6 +2,7 @@ exports.kBody = Symbol('body'); exports.kCases = Symbol('cases'); +exports.kName = Symbol('name'); exports.kNoAdvance = Symbol('noAdvance'); exports.kOtherwise = Symbol('otherwise'); exports.kSignature = Symbol('signature'); diff --git a/test/api-test.js b/test/api-test.js index 86e3c46..26d85d2 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -7,7 +7,6 @@ const fixtures = require('./fixtures'); const printMatch = fixtures.printMatch; const printOff = fixtures.printOff; -const compiledPrint = fixtures.compiledPrint; describe('LLParse', () => { let p; @@ -102,30 +101,13 @@ describe('LLParse', () => { callback); }); - it('should support compiled code', (callback) => { - const start = p.node('start'); - - start.select({ - '0': 0, - '1': 1 - }, compiledPrint(p, start)); - - start.otherwise(p.error(3, 'Invalid word')); - - const binary = fixtures.build('compiled', p.build(start)); - binary( - '0110', - 'not one\none\none\nnot one\n', - callback); - }); - it('should support custom state properties', (callback) => { const start = p.node('start'); const error = p.error(3, 'Invalid word'); p.property(ir => ir.i(8), 'custom'); - const second = p.invoke(p.code.load('load', 'custom'), { + const second = p.invoke(p.code.load('custom'), { 0: p.invoke(p.code.match('print_zero'), { 0: start }, error), 1: p.invoke(p.code.match('print_one'), { 0: start }, error) }, error); @@ -134,7 +116,7 @@ describe('LLParse', () => { .select({ '0': 0, '1': 1 - }, p.invoke(p.code.store('store', 'custom'), { 0: second }, error)) + }, p.invoke(p.code.store('custom'), { 0: second }, error)) .otherwise(error); const binary = fixtures.build('custom-prop', p.build(start)); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index f4956e8..c099da1 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -81,41 +81,3 @@ exports.printOff = (p, next) => { 0: next }, p.error(1, '`print_off` error')); }; - -exports.compiledPrint = (p, next) => { - const code = p.code.value('compiled_print_off', (ir, context) => { - const INT = ir.i(32); - const CSTR = ir.i(8).ptr(); - - const body = context.fn.body; - - const puts = ir.declare(ir.signature(INT, [ CSTR ]), 'puts'); - - const one = ir.cstr('one'); - const notOne = ir.cstr('not one'); - - const castOne = ir._('getelementptr', one.type.to, [ one.type, one ], - [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - const castNotOne = ir._('getelementptr', notOne.type.to, - [ notOne.type, notOne ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - body.push(castOne); - body.push(castNotOne); - - const cmp = ir._('icmp', [ 'eq', context.match.type, context.match ], - context.match.type.v(1)); - body.push(cmp); - - const select = ir._('select', [ ir.i(1), cmp ], [ CSTR, castOne ], - [ CSTR, castNotOne ]); - body.push(select); - - body.push(ir._('call', [ puts.type.ret, puts, '(', - CSTR, select, ')' ])); - - body.terminate('ret', [ context.ret, context.ret.v(0) ]); - }); - - return p.invoke(code, { - 0: next - }, p.error(1, '`print_off` error')); -}; From 34f488f439b1befeb67c15294e9ed5e1e1ce32e1 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 20:10:35 -0500 Subject: [PATCH 110/281] invoke: make `map` optional --- README.md | 6 ++---- examples/http/index.js | 6 ++---- lib/llparse.js | 4 ++-- lib/llparse/compiler.js | 1 - lib/llparse/node/invoke.js | 10 +++++++++- test/api-test.js | 2 +- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7aa16a6..ac8ea51 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,11 @@ const http = p.node('http'); p.property(ir => ir.i(8), 'method'); // Store method inside the custom property -const onMethod = p.invoke(p.code.store('method'), { - // If that function returns zero - 0: beforeUrl -}, p.error(1, '`on_method` error')); +const onMethod = p.invoke(p.code.store('method'), beforeUrl); // Invoke external C function const urlStart = p.invoke(p.code.match('on_url_start'), { + // If that function returns zero 0: url }, p.error(2, '`on_url_start` error')); diff --git a/examples/http/index.js b/examples/http/index.js index a51a873..c9a4bcd 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -11,13 +11,11 @@ const http = p.node('http'); p.property(ir => ir.i(8), 'method'); // Store method inside a custom property -const onMethod = p.invoke(p.code.store('method'), { - // If that function returns zero - 0: beforeUrl -}, p.error(1, '`on_method` error')); +const onMethod = p.invoke(p.code.store('method'), beforeUrl); // Invoke external C function const urlStart = p.invoke(p.code.match('on_url_start'), { + // If that function returns zero 0: url }, p.error(2, '`on_url_start` error')); diff --git a/lib/llparse.js b/lib/llparse.js index 084749f..a803fa4 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -29,11 +29,11 @@ class CodeAPI { // Helpers store(field) { - return new internal.code.Store(this[kPrefix] + '__load_' + field, field); + return new internal.code.Store(this[kPrefix] + '__store_' + field, field); } load(field) { - return new internal.code.Load(this[kPrefix] + '__store_' + field, field); + return new internal.code.Load(this[kPrefix] + '__method_' + field, field); } } diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 7ea1dfc..757d82a 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -323,7 +323,6 @@ class Compiler { const context = new llparse.Context(code, fn); - fn.body.comment('custom user code'); code[kBody].call(fn.body, this.ir, context); return fn; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 4a25714..a8acc4d 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -11,7 +11,15 @@ const kCode = Symbol('code'); const kMap = Symbol('map'); class Invoke extends node.Node { - constructor(code, map = {}, otherwise = null) { + constructor(code, map, otherwise = null) { + // `.invoke(code, otherwise)` + if (map && map instanceof node.Node) { + otherwise = map; + map = null; + } + + map = map || {}; + assert.strictEqual(typeof map, 'object', 'Invalid `map` argument of `.invoke()`, must be an Object'); assert(code instanceof llparse.code.Code, diff --git a/test/api-test.js b/test/api-test.js index 26d85d2..f31d094 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -116,7 +116,7 @@ describe('LLParse', () => { .select({ '0': 0, '1': 1 - }, p.invoke(p.code.store('custom'), { 0: second }, error)) + }, p.invoke(p.code.store('custom'), second)) .otherwise(error); const binary = fixtures.build('custom-prop', p.build(start)); From fc044528935e16edb524855be82115da89305d25 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 20:12:13 -0500 Subject: [PATCH 111/281] test: remove useless maps for static invoke --- test/fixtures/index.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/fixtures/index.js b/test/fixtures/index.js index c099da1..6f430be 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -69,15 +69,11 @@ exports.build = (name, source) => { exports.printMatch = (p, next) => { const code = p.code.value('print_match'); - return p.invoke(code, { - 0: next - }, p.error(1, '`print_match` error')); + return p.invoke(code, next); }; exports.printOff = (p, next) => { const code = p.code.match('print_off'); - return p.invoke(code, { - 0: next - }, p.error(1, '`print_off` error')); + return p.invoke(code, next); }; From fe904ff80b179dda591688fe8492367f9ae028bd Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 21:44:01 -0500 Subject: [PATCH 112/281] lib: save progress on spans --- lib/llparse.js | 21 ++------- lib/llparse/compiler.js | 16 ++++++- lib/llparse/index.js | 2 + lib/llparse/node/span-end.js | 10 +++-- lib/llparse/node/span-start.js | 11 +++-- lib/llparse/span/allocator.js | 81 ++++++++++++++++++++++++++++++++++ lib/llparse/span/index.js | 4 ++ lib/llparse/span/span.js | 20 +++++++++ lib/llparse/symbols.js | 2 + test/span-test.js | 40 +++++++++++++++++ 10 files changed, 182 insertions(+), 25 deletions(-) create mode 100644 lib/llparse/span/allocator.js create mode 100644 lib/llparse/span/index.js create mode 100644 lib/llparse/span/span.js create mode 100644 test/span-test.js diff --git a/lib/llparse.js b/lib/llparse.js index a803fa4..395ebdd 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -4,6 +4,7 @@ const assert = require('assert'); const internal = require('./llparse/'); +const kCallback = Symbol('callback'); const kCode = Symbol('code'); const kName = Symbol('name'); const kPrefix = Symbol('prefix'); @@ -37,22 +38,6 @@ class CodeAPI { } } -class Span { - constructor(name, code, prefix) { - this[kName] = name; - this[kCode] = code; - this[kPrefix] = prefix; - } - - start() { - return new internal.node.SpanStart(this[kName], this[kPrefix]); - } - - end() { - return new internal.node.SpanEnd(this[kName], this[kPrefix], this[kCode]); - } -} - class LLParse { constructor(prefix) { this[kPrefix] = prefix || 'llparse'; @@ -96,8 +81,8 @@ class LLParse { props.list.push(prop); } - span(code) { - return new Span(code); + span(callback) { + return new internal.span.Span(callback); } build(root, options) { diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 757d82a..8b9b6d6 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -93,6 +93,10 @@ class Compiler { } build(root) { + const allocator = new llparse.span.Allocator(); + + allocator.execute(root); + // Check that we don't start with a `value` Invoke this.checkSignatureType(root, null); @@ -184,7 +188,17 @@ class Compiler { index = 0; this.counter.set(node.name, index + 1); - const name = `${this.prefix}__n_${node.name}` + + let name = node.name; + + if (node instanceof llparse.node.SpanStart || + node instanceof llparse.node.SpanEnd) { + // no-op + } else { + // Prefix user-supplied node names + name = 'n_' + name; + } + + name = `${this.prefix}__${name}` + `${index === 0 ? '' : '_' + index}`; let fn; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index 065ee26..5137981 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -7,6 +7,8 @@ exports.utils = require('./utils'); exports.case = require('./case'); exports.code = require('./code'); exports.node = require('./node'); +exports.span = require('./span'); + exports.Trie = require('./trie'); exports.MatchSequence = require('./match-sequence'); exports.Property = require('./property'); diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js index 417a978..3b866cb 100644 --- a/lib/llparse/node/span-end.js +++ b/lib/llparse/node/span-end.js @@ -4,12 +4,16 @@ const llparse = require('../'); const node = require('./'); const kNoAdvance = llparse.symbols.kNoAdvance; +const kSpan = llparse.symbols.kSpan; class SpanEnd extends node.Node { - constructor() { - super(); + constructor(span, callback) { + super('span_end_' + callback); this[kNoAdvance] = true; - throw new Error('To be implemented'); + this[kSpan] = span; } + + match() { throw new Error('Not supported'); } + select() { throw new Error('Not supported'); } } module.exports = SpanEnd; diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js index e504e37..106f830 100644 --- a/lib/llparse/node/span-start.js +++ b/lib/llparse/node/span-start.js @@ -4,12 +4,17 @@ const llparse = require('../'); const node = require('./'); const kNoAdvance = llparse.symbols.kNoAdvance; +const kSpan = llparse.symbols.kSpan; class SpanStart extends node.Node { - constructor() { - super(); + constructor(span, callback) { + super('span_start_' + callback); + this[kNoAdvance] = true; - throw new Error('To be implemented'); + this[kSpan] = span; } + + match() { throw new Error('Not supported'); } + select() { throw new Error('Not supported'); } } module.exports = SpanStart; diff --git a/lib/llparse/span/allocator.js b/lib/llparse/span/allocator.js new file mode 100644 index 0000000..5ba10f0 --- /dev/null +++ b/lib/llparse/span/allocator.js @@ -0,0 +1,81 @@ +'use strict'; + +const assert = require('assert'); + +const llparse = require('../'); +const span = require('./'); + +const kCallback = llparse.symbols.kCallback; +const kCases = llparse.symbols.kCases; +const kOtherwise = llparse.symbols.kOtherwise; +const kNoAdvance = llparse.symbols.kNoAdvance; +const kSpan = llparse.symbols.kSpan; + +class Allocator { + execute(root) { + const queue = new Set(); + this.getNodes(root, queue); + + const activeMap = new Map(); + queue.forEach(node => activeMap.set(node, new Set())); + + while (queue.size !== 0) { + const node = queue.values().next().value; + queue.delete(node); + + const active = activeMap.get(node); + + if (node instanceof llparse.node.SpanStart) + active.add(node[kSpan][kCallback]); + + active.forEach((span) => { + // Don't propagate span past the spanEnd + if (node instanceof llparse.node.SpanEnd && + span === node[kSpan][kCallback]) { + return; + } + + this.getChildren(node).forEach((child) => { + // Disallow loops + if (child instanceof llparse.node.SpanStart) { + assert.notStrictEqual(child[kSpan][kCallback], span, + `Detected loop in span "${span}"`); + } + + const set = activeMap.get(child); + if (set.has(span)) + return; + + set.add(span); + queue.add(child); + }); + }); + } + + console.log(activeMap); + } + + getNodes(root, set) { + const queue = [ root ]; + while (queue.length !== 0) { + const node = queue.pop(); + if (set.has(node)) + continue; + set.add(node); + + this.getChildren(node).forEach(child => queue.push(child)); + } + } + + getChildren(node) { + const res = []; + + // `error` nodes have no `otherwise` + if (node[kOtherwise] !== null) + res.push(node[kOtherwise].next); + node[kCases].forEach(c => res.push(c.next)); + + return res; + } +} +module.exports = Allocator; diff --git a/lib/llparse/span/index.js b/lib/llparse/span/index.js new file mode 100644 index 0000000..e61516a --- /dev/null +++ b/lib/llparse/span/index.js @@ -0,0 +1,4 @@ +'use strict'; + +exports.Span = require('./span'); +exports.Allocator = require('./allocator'); diff --git a/lib/llparse/span/span.js b/lib/llparse/span/span.js new file mode 100644 index 0000000..af4a796 --- /dev/null +++ b/lib/llparse/span/span.js @@ -0,0 +1,20 @@ +'use strict'; + +const llparse = require('../'); + +const kCallback = llparse.symbols.kCallback; + +class Span { + constructor(callback) { + this[kCallback] = callback; + } + + start() { + return new llparse.node.SpanStart(this, this[kCallback]); + } + + end() { + return new llparse.node.SpanEnd(this, this[kCallback]); + } +} +module.exports = Span; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 42af780..5ae94f1 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -1,9 +1,11 @@ 'use strict'; exports.kBody = Symbol('body'); +exports.kCallback = Symbol('callback'); exports.kCases = Symbol('cases'); exports.kName = Symbol('name'); exports.kNoAdvance = Symbol('noAdvance'); exports.kOtherwise = Symbol('otherwise'); exports.kSignature = Symbol('signature'); +exports.kSpan = Symbol('span'); exports.kType = Symbol('type'); diff --git a/test/span-test.js b/test/span-test.js new file mode 100644 index 0000000..fa2e5ef --- /dev/null +++ b/test/span-test.js @@ -0,0 +1,40 @@ +'use strict'; +/* global describe it beforeEach */ + +const assert = require('assert'); + +const llparse = require('../'); + +const fixtures = require('./fixtures'); + +describe('LLParse/span', () => { + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); + + it('should invoke span callback', (callback) => { + const start = p.node('start'); + const body = p.node('body'); + const span = p.span('on_data'); + + start.otherwise(span.start().otherwise(body)); + + body + .match([ '0', '1', '2' ], body) + .otherwise(span.end().otherwise(start)); + + const binary = fixtures.build('span', p.build(start)); + + binary('012', 'off=3 match=1\n', callback); + }); + + it('should throw on loops', () => { + const start = p.node('start'); + const span = p.span('on_data'); + + start.otherwise(span.start().otherwise(start)); + + assert.throws(() => p.build(start), /loop/); + }); +}); From fb1f63f9b5d6b23ceac02a575046ede5749cf068 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 21:51:00 -0500 Subject: [PATCH 113/281] allocator: minor refactor --- lib/llparse/span/allocator.js | 39 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/lib/llparse/span/allocator.js b/lib/llparse/span/allocator.js index 5ba10f0..d8c59bc 100644 --- a/lib/llparse/span/allocator.js +++ b/lib/llparse/span/allocator.js @@ -13,12 +13,31 @@ const kSpan = llparse.symbols.kSpan; class Allocator { execute(root) { - const queue = new Set(); - this.getNodes(root, queue); + const nodes = this.getNodes(root); + const activeMap = this.computeActive(nodes); + console.log(activeMap); + } + + getNodes(root, set) { + const res = new Set(); + const queue = [ root ]; + while (queue.length !== 0) { + const node = queue.pop(); + if (res.has(node)) + continue; + res.add(node); + + this.getChildren(node).forEach(child => queue.push(child)); + } + return Array.from(res); + } + + computeActive(nodes) { const activeMap = new Map(); - queue.forEach(node => activeMap.set(node, new Set())); + nodes.forEach(node => activeMap.set(node, new Set())); + const queue = new Set(nodes); while (queue.size !== 0) { const node = queue.values().next().value; queue.delete(node); @@ -52,19 +71,7 @@ class Allocator { }); } - console.log(activeMap); - } - - getNodes(root, set) { - const queue = [ root ]; - while (queue.length !== 0) { - const node = queue.pop(); - if (set.has(node)) - continue; - set.add(node); - - this.getChildren(node).forEach(child => queue.push(child)); - } + return activeMap; } getChildren(node) { From 6b950221a055c49c9fe3d623cf06bef8e886dc94 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 20 Feb 2018 23:17:38 -0500 Subject: [PATCH 114/281] span: save progress --- lib/llparse/compiler.js | 2 ++ lib/llparse/span/allocator.js | 29 +++++++++++++++++++++++------ lib/llparse/span/span.js | 14 ++++++++++---- test/span-test.js | 30 ++++++++++++++++++++++++------ 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler.js index 8b9b6d6..c691226 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler.js @@ -95,6 +95,8 @@ class Compiler { build(root) { const allocator = new llparse.span.Allocator(); + // TODO(indutny): detect and report loops + allocator.execute(root); // Check that we don't start with a `value` Invoke diff --git a/lib/llparse/span/allocator.js b/lib/llparse/span/allocator.js index d8c59bc..92f1e29 100644 --- a/lib/llparse/span/allocator.js +++ b/lib/llparse/span/allocator.js @@ -14,9 +14,8 @@ const kSpan = llparse.symbols.kSpan; class Allocator { execute(root) { const nodes = this.getNodes(root); - const activeMap = this.computeActive(nodes); - - console.log(activeMap); + const info = this.computeActive(nodes); + const graph = this.computeOverlap(info); } getNodes(root, set) { @@ -38,14 +37,18 @@ class Allocator { nodes.forEach(node => activeMap.set(node, new Set())); const queue = new Set(nodes); + const spans = new Set(); while (queue.size !== 0) { const node = queue.values().next().value; queue.delete(node); const active = activeMap.get(node); - if (node instanceof llparse.node.SpanStart) - active.add(node[kSpan][kCallback]); + if (node instanceof llparse.node.SpanStart) { + const span = node[kSpan][kCallback]; + spans.add(span); + active.add(span); + } active.forEach((span) => { // Don't propagate span past the spanEnd @@ -71,7 +74,21 @@ class Allocator { }); } - return activeMap; + const ends = nodes.filter(node => node instanceof llparse.node.SpanEnd); + ends.forEach((end) => { + const active = activeMap.get(end); + assert(active.has(end[kSpan][kCallback]), + `Unmatched span end for "${end[kSpan][kCallback]}"`); + }); + + return { active: activeMap, spans: Array.from(spans) }; + } + + computeOverlap(info) { + const active = info.active; + const overlap = new Map(); + + info.spans.forEach(span => overlap.set(span, new Set())); } getChildren(node) { diff --git a/lib/llparse/span/span.js b/lib/llparse/span/span.js index af4a796..1dae823 100644 --- a/lib/llparse/span/span.js +++ b/lib/llparse/span/span.js @@ -9,12 +9,18 @@ class Span { this[kCallback] = callback; } - start() { - return new llparse.node.SpanStart(this, this[kCallback]); + start(otherwise = null) { + const res = new llparse.node.SpanStart(this, this[kCallback]); + if (otherwise) + res.otherwise(otherwise); + return res; } - end() { - return new llparse.node.SpanEnd(this, this[kCallback]); + end(otherwise = null) { + const res = new llparse.node.SpanEnd(this, this[kCallback]); + if (otherwise) + res.otherwise(otherwise); + return res; } } module.exports = Span; diff --git a/test/span-test.js b/test/span-test.js index fa2e5ef..65949a3 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -15,14 +15,23 @@ describe('LLParse/span', () => { it('should invoke span callback', (callback) => { const start = p.node('start'); - const body = p.node('body'); - const span = p.span('on_data'); + const dot = p.node('dot'); + const dash = p.node('dash'); + const span = { + dot: p.span('on_dot'), + dash: p.span('on_dash') + }; + + start.otherwise(span.dot.start(dot)); - start.otherwise(span.start().otherwise(body)); + dot + .match('.', dot) + .match('-', span.dash.start(dash)) + .skipTo(span.dot.end(start)); - body - .match([ '0', '1', '2' ], body) - .otherwise(span.end().otherwise(start)); + dash + .match('-', dash) + .otherwise(span.dash.end(dot)); const binary = fixtures.build('span', p.build(start)); @@ -37,4 +46,13 @@ describe('LLParse/span', () => { assert.throws(() => p.build(start), /loop/); }); + + it('should throw on unmatched ends', () => { + const start = p.node('start'); + const span = p.span('on_data'); + + start.otherwise(span.end().otherwise(start)); + + assert.throws(() => p.build(start), /unmatched/i); + }); }); From aef70a67146e6f80320298ee3f552cc5d55e2438 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 00:15:22 -0500 Subject: [PATCH 115/281] allocator: color spans! --- lib/llparse/span/allocator.js | 53 ++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/llparse/span/allocator.js b/lib/llparse/span/allocator.js index 92f1e29..065e1cb 100644 --- a/lib/llparse/span/allocator.js +++ b/lib/llparse/span/allocator.js @@ -15,7 +15,10 @@ class Allocator { execute(root) { const nodes = this.getNodes(root); const info = this.computeActive(nodes); - const graph = this.computeOverlap(info); + const overlap = this.computeOverlap(info); + const color = this.color(info.spans, overlap); + + console.log(color); } getNodes(root, set) { @@ -89,6 +92,54 @@ class Allocator { const overlap = new Map(); info.spans.forEach(span => overlap.set(span, new Set())); + + const add = (one, list) => { + const set = overlap.get(one); + list.forEach((other) => { + if (other === one) + return; + set.add(other); + }); + }; + + active.forEach((spans) => { + spans.forEach(span => add(span, spans)); + }); + + return overlap; + } + + color(spans, overlapMap) { + let max = 0; + const colors = new Map(); + + const allocate = (span) => { + if (colors.has(span)) + return colors.get(span); + + const overlap = overlapMap.get(span); + const used = new Set(); + overlap.forEach((span) => { + if (colors.has(span)) + used.add(colors.get(span)); + }); + + let i; + for (i = 0; i < max + 1; i++) + if (!used.has(i)) + break; + + max = Math.max(max, i); + colors.set(span, i); + + return i; + }; + + const res = new Map(); + + spans.forEach(span => res.set(span, allocate(span))); + + return { colors: res, max }; } getChildren(node) { From df62fbbcfd8979541437f328e5323e0ea00e82f7 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 02:25:08 -0500 Subject: [PATCH 116/281] broken: save progress --- lib/llparse.js | 6 +- lib/llparse/{ => code}/context.js | 2 +- lib/llparse/code/index.js | 1 + lib/llparse/{ => compiler}/compiler.js | 108 ++++-------- lib/llparse/compiler/context.js | 163 +++++++++++++++++++ lib/llparse/compiler/index.js | 8 + lib/llparse/{ => compiler}/match-sequence.js | 28 +--- lib/llparse/compiler/node.js | 14 ++ lib/llparse/{ => compiler}/span/allocator.js | 14 +- lib/llparse/compiler/span/builder.js | 10 ++ lib/llparse/{ => compiler}/span/index.js | 2 +- lib/llparse/compiler/stage.js | 9 + lib/llparse/constants.js | 4 + lib/llparse/index.js | 6 +- lib/llparse/{span => }/span.js | 2 +- package.json | 2 +- test/span-test.js | 12 +- 17 files changed, 280 insertions(+), 111 deletions(-) rename lib/llparse/{ => code}/context.js (98%) rename lib/llparse/{ => compiler}/compiler.js (89%) create mode 100644 lib/llparse/compiler/context.js create mode 100644 lib/llparse/compiler/index.js rename lib/llparse/{ => compiler}/match-sequence.js (90%) create mode 100644 lib/llparse/compiler/node.js rename lib/llparse/{ => compiler}/span/allocator.js (92%) create mode 100644 lib/llparse/compiler/span/builder.js rename lib/llparse/{ => compiler}/span/index.js (59%) create mode 100644 lib/llparse/compiler/stage.js rename lib/llparse/{span => }/span.js (93%) diff --git a/lib/llparse.js b/lib/llparse.js index 395ebdd..76f8e73 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -71,6 +71,8 @@ class LLParse { property(type, name) { if (internal.constants.RESERVED_PROPERTY_NAMES.has(name)) throw new Error(`Can't use reserved property name: "${name}"`); + if (/^_/.test(name)) + throw new Error(`Can't use reserved property name: "${name}"`); const props = this[kProperties]; if (props.set.has(name)) @@ -82,7 +84,7 @@ class LLParse { } span(callback) { - return new internal.span.Span(callback); + return new internal.Span(callback); } build(root, options) { @@ -92,7 +94,7 @@ class LLParse { options = options || {}; - const c = new internal.Compiler({ + const c = new internal.compiler.Compiler({ prefix: this.prefix, properties: this[kProperties].list, debug: options.debug || false diff --git a/lib/llparse/context.js b/lib/llparse/code/context.js similarity index 98% rename from lib/llparse/context.js rename to lib/llparse/code/context.js index 7f9844c..18e7ade 100644 --- a/lib/llparse/context.js +++ b/lib/llparse/code/context.js @@ -2,7 +2,7 @@ const IR = require('llvm-ir'); -const llparse = require('./'); +const llparse = require('../'); const constants = llparse.constants; const kType = llparse.symbols.kType; diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index 0230309..d585f9d 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -5,3 +5,4 @@ exports.Match = require('./match'); exports.Value = require('./value'); exports.Store = require('./store'); exports.Load = require('./load'); +exports.Context = require('./context'); diff --git a/lib/llparse/compiler.js b/lib/llparse/compiler/compiler.js similarity index 89% rename from lib/llparse/compiler.js rename to lib/llparse/compiler/compiler.js index c691226..4e2d3ae 100644 --- a/lib/llparse/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -3,9 +3,8 @@ const assert = require('assert'); const IR = require('llvm-ir'); -const llparse = require('./'); +const llparse = require('../'); const constants = llparse.constants; -const MatchSequence = llparse.MatchSequence; const kBody = llparse.symbols.kBody; const kCases = llparse.symbols.kCases; @@ -40,44 +39,15 @@ const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; +const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; +const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; + class Compiler { constructor(options) { this.options = Object.assign({}, options); this.prefix = this.options.prefix; - this.ir = new IR(); - - this.state = this.ir.struct(`${this.prefix}_state`); - - this.signature = { - node: this.ir.signature(TYPE_OUTPUT, [ - [ this.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ], - TYPE_MATCH - ]), - callback: { - match: this.ir.signature(INT, [ - this.state.ptr(), TYPE_INPUT, TYPE_INPUT - ]), - value: this.ir.signature(INT, [ - this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_MATCH - ]) - } - }; - - this.state.field(this.signature.node.ptr(), 'current'); - this.state.field(TYPE_ERROR, 'error'); - this.state.field(TYPE_REASON, 'reason'); - this.state.field(TYPE_INDEX, 'index'); - this.state.field(TYPE_INPUT, 'mark'); - - // Public fields - this.state.field(TYPE_DATA, 'data'); - - this.options.properties.forEach((prop) => { - this.state.field(prop.type(this.ir, this.state), prop.name); - }); + this.ctx = new llparse.compiler.Context(this.prefix, this.options); this.nodeMap = new Map(); this.codeMap = new Map(); @@ -86,21 +56,22 @@ class Compiler { // redirect blocks by `fn` and `target` this.redirectCache = new Map(); - const matchSequence = new MatchSequence(this.prefix, this.ir, this.state); - this.matchSequence = matchSequence.build(); - this.debugMethod = null; } build(root) { - const allocator = new llparse.span.Allocator(); + // Check that we don't start with a `value` Invoke + this.checkSignatureType(root, null); // TODO(indutny): detect and report loops - allocator.execute(root); + const stages = [ + new llparse.compiler.MatchSequence(this.ctx), + new llparse.compiler.span.Builder(this.ctx), + new llparse.compiler.NodeBuilder(this.ctx) + ]; - // Check that we don't start with a `value` Invoke - this.checkSignatureType(root, null); + stages.forEach(stage => stage.build()); const rootFn = this.buildNode(root).fn; @@ -110,6 +81,24 @@ class Compiler { return this.ir.build(); } + buildSpanFields(root) { + const allocator = new llparse.span.Allocator(); + + const colors = allocator.execute(root); + + colors.concurrency.forEach((num, index) => { + this.state.field(TYPE_INPUT, SPAN_START_PREFIX + index); + + // TODO(indutny): use smaller field size if possible? + if (num === 1) + return; + + this.state.field(this.signature.callback.match.ptr(), + SPAN_CB_PREFIX + index); + }); + console.log(colors); + } + buildInit(fn) { const sig = IR.signature(IR.void(), [ this.state.ptr() ]); const init = this.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); @@ -337,7 +326,7 @@ class Compiler { fn.cconv = CCONV; fn.attributes = 'nounwind'; - const context = new llparse.Context(code, fn); + const context = new llparse.code.Context(code, fn); code[kBody].call(fn.body, this.ir, context); @@ -471,30 +460,6 @@ class Compiler { return body; } - buildSwitch(body, type, what, values) { - const cases = []; - cases.push(IR.label('otherwise')); - cases.push('['); - values.forEach((value, i) => { - cases.push(type, type.v(value)); - cases.push(',', IR.label(`case_${i}`)); - }); - cases.push(']'); - - const blocks = body.terminate('switch', [ type, what ], cases); - - blocks[0].name = 'switch_otherwise'; - for (let i = 0; i < values.length; i++) { - const v = values[i] < 0 ? 'm' + (-values[i]) : values[i]; - blocks[i + 1].name = 'case_' + v; - } - - return { - otherwise: blocks[0], - cases: blocks.slice(1) - }; - } - buildSingle(info, body, pos, children) { // Load the character const current = @@ -663,15 +628,6 @@ class Compiler { // Helpers - field(fn, name) { - const stateArg = fn.arg(ARG_STATE); - - return IR._('getelementptr', this.state, - [ stateArg.type, stateArg ], - [ INT, INT.v(0) ], - [ INT, INT.v(this.state.lookup(name)) ]); - } - checkSignatureType(node, value) { assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); } diff --git a/lib/llparse/compiler/context.js b/lib/llparse/compiler/context.js new file mode 100644 index 0000000..cc16674 --- /dev/null +++ b/lib/llparse/compiler/context.js @@ -0,0 +1,163 @@ +'use strict'; + +const assert = require('assert'); +const IR = require('llvm-ir'); + +const llparse = require('../'); +const constants = llparse.constants; + +const CCONV = constants.CCONV; + +const INT = constants.INT; +const TYPE_INPUT = constants.TYPE_INPUT; +const TYPE_OUTPUT = constants.TYPE_OUTPUT; +const TYPE_MATCH = constants.TYPE_MATCH; +const TYPE_INDEX = constants.TYPE_INDEX; +const TYPE_ERROR = constants.TYPE_ERROR; +const TYPE_REASON = constants.TYPE_REASON; +const TYPE_DATA = constants.TYPE_DATA; + +const ATTR_STATE = constants.ATTR_STATE; +const ATTR_POS = constants.ATTR_POS; +const ATTR_ENDPOS = constants.ATTR_ENDPOS; + +const ARG_STATE = constants.ARG_STATE; +const ARG_POS = constants.ARG_POS; +const ARG_ENDPOS = constants.ARG_ENDPOS; +const ARG_MATCH = constants.ARG_MATCH; + +class Context { + constructor(prefix, options) { + this.ir = new IR(); + this.prefix = prefix; + + this.state = this.ir.struct(`${this.prefix}_state`); + + this.state.field(this.signature.node.ptr(), 'current'); + this.state.field(TYPE_ERROR, 'error'); + this.state.field(TYPE_REASON, 'reason'); + this.state.field(TYPE_INDEX, 'index'); + this.state.field(TYPE_INPUT, 'mark'); + + // Public fields + this.state.field(TYPE_DATA, 'data'); + + // Custom fields + this.options.properties.forEach((prop) => { + this.state.field(prop.type(this.ir, this.state), prop.name); + }); + + this.signature = { + node: this.ir.signature(TYPE_OUTPUT, [ + [ this.state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ], + TYPE_MATCH + ]), + callback: { + match: this.ir.signature(INT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT + ]), + value: this.ir.signature(INT, [ + this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_MATCH + ]) + } + }; + } + + signature(name) { + if (name === 'node') + return this.signature.node; + else if (name === 'callback.match') + return this.signature.callback.match; + else if (name === 'callback.value') + return this.signature.callback.value; + else + throw new Error('Unknown signature: ' + signature); + } + + fn(signature, name) { + name = this.prefix + '__' + name; + + let fn; + if (signature === 'node') { + fn = this.ir.fn(this.signature.node, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); + } else if (signature === 'callback.match') { + fn = this.ir.fn(this.signature.callback.match, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + } else if (signature === 'callback.value') { + fn = this.ir.fn(this.signature.callback.match, name, + [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); + } else { + throw new Error('Unknown signature: ' + signature); + } + + fn.visibility = 'internal'; + fn.cconv = CCONV; + + return fn; + } + + call(type, signature, ref, args) { + signature = this.signature(name); + assert.strictEqual(args.length, signature.args.length); + + const irArgs = [ signature.ret, ref, '(' ]; + args.forEach((arg, i) => { + const isLast = i === args.length - 1; + irArgs.push(signature.args[i], arg); + if (!isLast) + irArgs.push(','); + }); + irArgs.push(')'); + + if (type) + type += ' '; + + let cconv = ref.cconv; + if (cconv) + cconv = ' ' + cconv; + + return IR._(`${type}call${cconv}`, args); + } + + buildSwitch(body, type, what, values) { + const cases = []; + cases.push(IR.label('otherwise')); + cases.push('['); + values.forEach((value, i) => { + cases.push(type, type.v(value)); + cases.push(',', IR.label(`case_${i}`)); + }); + cases.push(']'); + + const blocks = body.terminate('switch', [ type, what ], cases); + + blocks[0].name = 'switch_otherwise'; + for (let i = 0; i < values.length; i++) { + const v = values[i] < 0 ? 'm' + (-values[i]) : values[i]; + blocks[i + 1].name = 'case_' + v; + } + + return { + otherwise: blocks[0], + cases: blocks.slice(1) + }; + } + + field(fn, name) { + const stateArg = this.state(fn); + + return IR._('getelementptr', this.state, + [ stateArg.type, stateArg ], + [ INT, INT.v(0) ], + [ INT, INT.v(this.state.lookup(name)) ]); + } + + state(fn) { return fn.arg(ARG_STATE); } + pos(fn) { return fn.arg(ARG_POS); } + endPos(fn) { return fn.arg(ARG_ENDPOS); } + match(fn) { return fn.arg(ARG_MATCH); } +} +module.exports = Context; diff --git a/lib/llparse/compiler/index.js b/lib/llparse/compiler/index.js new file mode 100644 index 0000000..ef9560f --- /dev/null +++ b/lib/llparse/compiler/index.js @@ -0,0 +1,8 @@ +'use strict'; + +exports.Context = require('./context'); +exports.Stage = require('./stage'); +exports.MatchSequence = require('./match-sequence'); +exports.span = require('./span'); +exports.NodeBuilder = require('./node'); +exports.Compiler = require('./compiler'); diff --git a/lib/llparse/match-sequence.js b/lib/llparse/compiler/match-sequence.js similarity index 90% rename from lib/llparse/match-sequence.js rename to lib/llparse/compiler/match-sequence.js index 2d1132a..9b1e48f 100644 --- a/lib/llparse/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -2,7 +2,8 @@ const IR = require('llvm-ir'); -const llparse = require('./'); +const compiler = require('./'); +const llparse = require('../'); const constants = llparse.constants; const CCONV = constants.CCONV; @@ -27,19 +28,17 @@ const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; -class MatchSequence { - constructor(prefix, ir, state) { - this.state = state; - this.prefix = prefix; - this.ir = ir; +class MatchSequence extends compiler.Stage { + constructor(ctx) { + super(ctx, 'match-sequence'); - this.returnType = this.ir.struct('match_sequence_ret'); + this.returnType = this.ctx.ir.struct('match_sequence_ret'); this.returnType.field(TYPE_INPUT, 'current'); this.returnType.field(INT, 'status'); this.signature = IR.signature(this.returnType, [ - [ state.ptr(), ATTR_STATE ], + [ this.ctx.state.ptr(), ATTR_STATE ], [ TYPE_INPUT, ATTR_POS ], [ TYPE_INPUT, ATTR_ENDPOS ], [ TYPE_INPUT, ATTR_SEQUENCE ], @@ -48,7 +47,7 @@ class MatchSequence { } build() { - const fn = this.ir.fn(this.signature, `${this.prefix}__match_sequence`, + const fn = this.ir.fn(this.signature, `${this.ctx.prefix}__match_sequence`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); this.buildBody(fn); @@ -69,7 +68,7 @@ class MatchSequence { // Load `state.index` start.comment('index = state.index'); - const indexPtr = this.index(fn); + const indexPtr = this.ctx.field(fn, 'index'); start.push(indexPtr); const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ]); start.push(index); @@ -194,15 +193,6 @@ class MatchSequence { body.terminate('ret', [ this.returnType, amend ]); } - index(fn) { - const stateArg = fn.arg(ARG_STATE); - - return IR._('getelementptr', this.state, - [ stateArg.type, stateArg ], - [ INT, INT.v(0) ], - [ INT, INT.v(this.state.lookup('index')) ]); - } - reset(field) { return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], [ TYPE_INDEX.ptr(), field ]).void(); diff --git a/lib/llparse/compiler/node.js b/lib/llparse/compiler/node.js new file mode 100644 index 0000000..fa7c6d4 --- /dev/null +++ b/lib/llparse/compiler/node.js @@ -0,0 +1,14 @@ +'use strict'; + +const compiler = require('./'); +const llparse = require('../'); + +class NodeBuilder extends compiler.Stage { + constructor(ctx) { + super(ctx, 'node-builder'); + } + + build() { + } +} +module.exports = NodeBuilder; diff --git a/lib/llparse/span/allocator.js b/lib/llparse/compiler/span/allocator.js similarity index 92% rename from lib/llparse/span/allocator.js rename to lib/llparse/compiler/span/allocator.js index 065e1cb..0aa713e 100644 --- a/lib/llparse/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -2,8 +2,7 @@ const assert = require('assert'); -const llparse = require('../'); -const span = require('./'); +const llparse = require('../../'); const kCallback = llparse.symbols.kCallback; const kCases = llparse.symbols.kCases; @@ -18,7 +17,7 @@ class Allocator { const overlap = this.computeOverlap(info); const color = this.color(info.spans, overlap); - console.log(color); + return color; } getNodes(root, set) { @@ -118,12 +117,15 @@ class Allocator { return colors.get(span); const overlap = overlapMap.get(span); + + // See which colors are already used const used = new Set(); overlap.forEach((span) => { if (colors.has(span)) used.add(colors.get(span)); }); + // Find minimum available color let i; for (i = 0; i < max + 1; i++) if (!used.has(i)) @@ -139,7 +141,11 @@ class Allocator { spans.forEach(span => res.set(span, allocate(span))); - return { colors: res, max }; + const concurrency = new Array(max + 1).fill(0); + + spans.forEach(span => concurrency[allocate(span)]++); + + return { map: res, concurrency, max }; } getChildren(node) { diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js new file mode 100644 index 0000000..ce4dd5a --- /dev/null +++ b/lib/llparse/compiler/span/builder.js @@ -0,0 +1,10 @@ +'use strict'; + +const compiler = require('../'); + +class Builder extends compiler.Stage { + constructor(ctx) { + super(ctx, 'span-builder'); + } +} +module.exports = Builder; diff --git a/lib/llparse/span/index.js b/lib/llparse/compiler/span/index.js similarity index 59% rename from lib/llparse/span/index.js rename to lib/llparse/compiler/span/index.js index e61516a..d266f2f 100644 --- a/lib/llparse/span/index.js +++ b/lib/llparse/compiler/span/index.js @@ -1,4 +1,4 @@ 'use strict'; -exports.Span = require('./span'); exports.Allocator = require('./allocator'); +exports.Builder = require('./builder'); diff --git a/lib/llparse/compiler/stage.js b/lib/llparse/compiler/stage.js new file mode 100644 index 0000000..fc5aa8c --- /dev/null +++ b/lib/llparse/compiler/stage.js @@ -0,0 +1,9 @@ +'use strict'; + +class Stage { + constructor(ctx, name) { + this.ctx = ctx; + this.name = name; + } +} +module.exports = Stage; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 7bd1a83..7a39a56 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -31,6 +31,10 @@ exports.SEQUENCE_COMPLETE = 0; exports.SEQUENCE_PAUSE = 1; exports.SEQUENCE_MISMATCH = 2; +// NOTE: It is important to start them with `_`, see `lib/llparse.js` (property) +exports.SPAN_START_PREFIX = '_span_start'; +exports.SPAN_CB_PREFIX = '_span_cb'; + exports.RESERVED_PROPERTY_NAMES = new Set([ 'current', 'error', diff --git a/lib/llparse/index.js b/lib/llparse/index.js index 5137981..33f60b7 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -7,10 +7,8 @@ exports.utils = require('./utils'); exports.case = require('./case'); exports.code = require('./code'); exports.node = require('./node'); -exports.span = require('./span'); +exports.Span = require('./span'); exports.Trie = require('./trie'); -exports.MatchSequence = require('./match-sequence'); exports.Property = require('./property'); -exports.Context = require('./context'); -exports.Compiler = require('./compiler'); +exports.compiler = require('./compiler'); diff --git a/lib/llparse/span/span.js b/lib/llparse/span.js similarity index 93% rename from lib/llparse/span/span.js rename to lib/llparse/span.js index 1dae823..8dd6e52 100644 --- a/lib/llparse/span/span.js +++ b/lib/llparse/span.js @@ -1,6 +1,6 @@ 'use strict'; -const llparse = require('../'); +const llparse = require('./'); const kCallback = llparse.symbols.kCallback; diff --git a/package.json b/package.json index 4332091..75ff6f3 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2.0.0-beta3", "main": "lib/llparse.js", "scripts": { - "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js", + "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", "test": "mocha --reporter=spec test/*-test.js && npm run lint" }, "keywords": [], diff --git a/test/span-test.js b/test/span-test.js index 65949a3..45fda13 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -17,9 +17,12 @@ describe('LLParse/span', () => { const start = p.node('start'); const dot = p.node('dot'); const dash = p.node('dash'); + const underscore = p.node('underscore'); + const span = { dot: p.span('on_dot'), - dash: p.span('on_dash') + dash: p.span('on_dash'), + underscore: p.span('on_underscore') }; start.otherwise(span.dot.start(dot)); @@ -27,15 +30,20 @@ describe('LLParse/span', () => { dot .match('.', dot) .match('-', span.dash.start(dash)) + .match('_', span.underscore.start(underscore)) .skipTo(span.dot.end(start)); dash .match('-', dash) .otherwise(span.dash.end(dot)); + underscore + .match('_', underscore) + .otherwise(span.underscore.end(dot)); + const binary = fixtures.build('span', p.build(start)); - binary('012', 'off=3 match=1\n', callback); + binary('..--..__..', 'off=3 match=1\n', callback); }); it('should throw on loops', () => { From 323e6c4355889c6e73d688ec02cd2cc1b65c8ce6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 15:35:10 -0500 Subject: [PATCH 117/281] compiler: save progress --- lib/llparse/compiler/compiler.js | 221 ++---------------------- lib/llparse/compiler/context.js | 73 ++++---- lib/llparse/compiler/index.js | 4 +- lib/llparse/compiler/match-sequence.js | 4 +- lib/llparse/compiler/node.js | 14 -- lib/llparse/compiler/node/base.js | 155 +++++++++++++++++ lib/llparse/compiler/node/builder.js | 26 +++ lib/llparse/compiler/node/error.js | 41 +++++ lib/llparse/compiler/node/index.js | 12 ++ lib/llparse/compiler/node/invoke.js | 20 +++ lib/llparse/compiler/node/sequence.js | 25 +++ lib/llparse/compiler/node/single.js | 39 +++++ lib/llparse/compiler/node/span-end.js | 15 ++ lib/llparse/compiler/node/span-start.js | 15 ++ lib/llparse/compiler/node/translator.js | 125 ++++++++++++++ lib/llparse/compiler/span/builder.js | 29 ++++ lib/llparse/compiler/stage.js | 4 + lib/llparse/node/base.js | 2 - lib/llparse/node/error.js | 8 +- lib/llparse/node/invoke.js | 9 +- lib/llparse/node/span-end.js | 2 - lib/llparse/node/span-start.js | 2 - lib/llparse/symbols.js | 2 + lib/llparse/trie.js | 3 +- 24 files changed, 572 insertions(+), 278 deletions(-) delete mode 100644 lib/llparse/compiler/node.js create mode 100644 lib/llparse/compiler/node/base.js create mode 100644 lib/llparse/compiler/node/builder.js create mode 100644 lib/llparse/compiler/node/error.js create mode 100644 lib/llparse/compiler/node/index.js create mode 100644 lib/llparse/compiler/node/invoke.js create mode 100644 lib/llparse/compiler/node/sequence.js create mode 100644 lib/llparse/compiler/node/single.js create mode 100644 lib/llparse/compiler/node/span-end.js create mode 100644 lib/llparse/compiler/node/span-start.js create mode 100644 lib/llparse/compiler/node/translator.js diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 4e2d3ae..79da689 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -8,7 +8,6 @@ const constants = llparse.constants; const kBody = llparse.symbols.kBody; const kCases = llparse.symbols.kCases; -const kNoAdvance = llparse.symbols.kNoAdvance; const kOtherwise = llparse.symbols.kOtherwise; const kSignature = llparse.symbols.kSignature; const kType = llparse.symbols.kType; @@ -39,15 +38,11 @@ const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; -const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; -const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; - class Compiler { constructor(options) { this.options = Object.assign({}, options); this.prefix = this.options.prefix; - this.ctx = new llparse.compiler.Context(this.prefix, this.options); this.nodeMap = new Map(); this.codeMap = new Map(); @@ -60,43 +55,24 @@ class Compiler { } build(root) { - // Check that we don't start with a `value` Invoke - this.checkSignatureType(root, null); + const ctx = new llparse.compiler.Context(this.prefix, this.options, root); // TODO(indutny): detect and report loops const stages = [ - new llparse.compiler.MatchSequence(this.ctx), - new llparse.compiler.span.Builder(this.ctx), - new llparse.compiler.NodeBuilder(this.ctx) + new llparse.compiler.MatchSequence(ctx), + new llparse.compiler.span.Builder(ctx), + new llparse.compiler.node.Translator(ctx), + new llparse.compiler.node.Builder(ctx) ]; - stages.forEach(stage => stage.build()); - - const rootFn = this.buildNode(root).fn; - - this.buildInit(rootFn.ref()); - this.buildParse(); - - return this.ir.build(); - } - - buildSpanFields(root) { - const allocator = new llparse.span.Allocator(); - - const colors = allocator.execute(root); - - colors.concurrency.forEach((num, index) => { - this.state.field(TYPE_INPUT, SPAN_START_PREFIX + index); + ctx.buildStages(stages); + console.log(ctx.stageResults['node-builder']); - // TODO(indutny): use smaller field size if possible? - if (num === 1) - return; + // this.buildInit(rootFn.ref()); + // this.buildParse(); - this.state.field(this.signature.callback.match.ptr(), - SPAN_CB_PREFIX + index); - }); - console.log(colors); + return ctx.ir.build(); } buildInit(fn) { @@ -171,49 +147,6 @@ class Compiler { body.terminate('ret', [ TYPE_ERROR, error ]); } - createFn(node) { - let index; - if (this.counter.has(node.name)) - index = this.counter.get(node.name); - else - index = 0; - this.counter.set(node.name, index + 1); - - let name = node.name; - - if (node instanceof llparse.node.SpanStart || - node instanceof llparse.node.SpanEnd) { - // no-op - } else { - // Prefix user-supplied node names - name = 'n_' + name; - } - - name = `${this.prefix}__${name}` + - `${index === 0 ? '' : '_' + index}`; - - let fn; - if (node[kSignature] === 'match') { - fn = this.ir.fn(this.signature.node, name, - [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_UNUSED ]); - } else { - assert.strictEqual(node[kSignature], 'value'); - fn = this.ir.fn(this.signature.node, name, - [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); - } - fn.visibility = 'internal'; - fn.cconv = CCONV; - - // TODO(indutny): reassess `minsize`. Looks like it gives best performance - // results right now, though. - fn.attributes = 'nounwind minsize'; - - // Errors are assumed to be rarely called - if (node instanceof llparse.node.Error) - fn.attributes += ' cold writeonly'; - return fn; - } - debug(fn, body, string) { if (!this.options.debug) return body; @@ -380,37 +313,6 @@ class Compiler { return subFn; } - buildPrologue(node, fn) { - if (node[kNoAdvance]) - return fn.body; - - // Check that we have enough chars to do the read - fn.body.comment('--- Prologue ---'); - fn.body.comment('if (pos != endpos)'); - const cmp = IR._('icmp', [ 'ne', TYPE_INPUT, fn.arg(ARG_POS) ], - fn.arg(ARG_ENDPOS)); - fn.body.push(cmp); - - const branch = fn.body.branch('br', [ BOOL, cmp ]); - - // Return self when `pos === endpos` - branch.right.name = 'prologue_end'; - this.buildSelfReturn({ fn, node }, branch.right, true); - - branch.left.name = 'prologue_normal'; - return branch.left; - } - - buildSelfReturn(info, body) { - assert.strictEqual(info.node[kSignature], 'match', - 'non-match nodes can\'t have self return'); - - const fn = info.fn; - const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', TYPE_OUTPUT ]); - body.push(bitcast); - body.terminate('ret', [ TYPE_OUTPUT, bitcast ]); - } - buildTrie(info, body, trie) { const fn = info.fn; @@ -460,25 +362,6 @@ class Compiler { return body; } - buildSingle(info, body, pos, children) { - // Load the character - const current = - IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, info.fn.arg(ARG_POS) ]); - body.push(current); - - const keys = children.map(child => child.key); - const s = this.buildSwitch(body, TYPE_INPUT.to, current, keys); - - const otherwise = s.otherwise; - const cases = s.cases; - - cases.forEach((target, i) => { - this.buildSubTrie(info, target, pos, children[i].child); - }); - - return otherwise; - } - buildSequence(info, body, trie) { assert(!info.node[kNoAdvance]); @@ -542,90 +425,6 @@ class Compiler { return { pos, body: mismatch }; } - buildNext(info, body, pos, trie) { - return this.buildRedirect(info, body, pos, this.buildNode(trie.next), - trie.value); - } - - buildRedirect(info, body, pos, target, value = null) { - const fn = info.fn; - - this.checkSignatureType(target.node, value); - - if (this.redirectCache.has(fn) && - this.redirectCache.get(fn).has(target.fn)) { - const cached = this.redirectCache.get(fn).get(target.fn); - - if (cached.phi) { - assert(value, '`.match()` and `.select()` with the same target'); - cached.phi.append([ '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); - } else { - assert(!value, '`.match()` and `.select()` with the same target'); - } - - body.terminate('br', cached.target); - return; - } - - // Split, so that others could join us from code block above - const redirect = body.jump('br'); - redirect.name = body.name + '_redirect'; - let phi = null; - - // Compute `match` if needed - if (value !== null) { - redirect.comment('Select `match`'); - phi = IR._('phi', - [ TYPE_MATCH, '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); - redirect.push(phi); - } - - if (!this.redirectCache.has(fn)) - this.redirectCache.set(fn, new Map()); - this.redirectCache.get(fn).set(target.fn, { - phi, - target: redirect - }); - - const args = [ - TYPE_OUTPUT, target.fn, '(', - this.state.ptr(), fn.arg(ARG_STATE), ',', - TYPE_INPUT, pos.next, ',', - TYPE_INPUT, fn.arg(ARG_ENDPOS), ',', - TYPE_MATCH, phi ? phi : TYPE_MATCH.v(0), - ')' - ]; - - // TODO(indutny): looks like `musttail` gives worse performance when calling - // Invoke nodes (possibly others too). - const call = IR._(`musttail call ${CCONV}`, args); - redirect.push(call); - redirect.terminate('ret', [ TYPE_OUTPUT, call ]); - } - - buildError(info, body) { - const code = info.node.code; - const reason = this.ir.cstr(info.node.reason); - - const codeField = this.field(info.fn, 'error'); - body.push(codeField); - - const reasonField = this.field(info.fn, 'reason'); - body.push(reasonField); - - const castReason = IR._('bitcast', [ - reason.type, reason, 'to', TYPE_REASON - ]); - body.push(castReason); - - body.push(IR._('store', [ TYPE_ERROR, TYPE_ERROR.v(code) ], - [ TYPE_ERROR.ptr(), codeField ]).void()); - body.push(IR._('store', [ TYPE_REASON, castReason ], - [ TYPE_REASON.ptr(), reasonField ]).void()); - - return body.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); - } - // Helpers checkSignatureType(node, value) { diff --git a/lib/llparse/compiler/context.js b/lib/llparse/compiler/context.js index cc16674..8c00db8 100644 --- a/lib/llparse/compiler/context.js +++ b/lib/llparse/compiler/context.js @@ -8,6 +8,7 @@ const constants = llparse.constants; const CCONV = constants.CCONV; +const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; @@ -27,25 +28,13 @@ const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; class Context { - constructor(prefix, options) { + constructor(prefix, options, root) { this.ir = new IR(); this.prefix = prefix; - this.state = this.ir.struct(`${this.prefix}_state`); + this.root = root; - this.state.field(this.signature.node.ptr(), 'current'); - this.state.field(TYPE_ERROR, 'error'); - this.state.field(TYPE_REASON, 'reason'); - this.state.field(TYPE_INDEX, 'index'); - this.state.field(TYPE_INPUT, 'mark'); - - // Public fields - this.state.field(TYPE_DATA, 'data'); - - // Custom fields - this.options.properties.forEach((prop) => { - this.state.field(prop.type(this.ir, this.state), prop.name); - }); + this.state = this.ir.struct(`${this.prefix}_state`); this.signature = { node: this.ir.signature(TYPE_OUTPUT, [ @@ -63,30 +52,47 @@ class Context { ]) } }; + + this.state.field(this.signature.node.ptr(), 'current'); + this.state.field(TYPE_ERROR, 'error'); + this.state.field(TYPE_REASON, 'reason'); + this.state.field(TYPE_INDEX, 'index'); + this.state.field(TYPE_INPUT, 'mark'); + + // Public fields + this.state.field(TYPE_DATA, 'data'); + + // Custom fields + options.properties.forEach((prop) => { + this.state.field(prop.type(this.ir, this.state), prop.name); + }); + + this.stageResults = {}; } - signature(name) { - if (name === 'node') - return this.signature.node; - else if (name === 'callback.match') - return this.signature.callback.match; - else if (name === 'callback.value') - return this.signature.callback.value; - else - throw new Error('Unknown signature: ' + signature); + buildStages(stages) { + stages.forEach(stage => this.buildStage(stage)); + } + + buildStage(stage) { + this.stageResults[stage.name] = stage.build(this); } fn(signature, name) { name = this.prefix + '__' + name; let fn; - if (signature === 'node') { + if (signature === this.signature.node) { fn = this.ir.fn(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); - } else if (signature === 'callback.match') { + + // TODO(indutny): reassess `minsize`. Looks like it gives best performance + // results right now, though. + fn.attributes = 'nounwind minsize'; + } else if (signature === this.signature.callback.match) { fn = this.ir.fn(this.signature.callback.match, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - } else if (signature === 'callback.value') { + } else if (signature === this.signature.callback.value) { fn = this.ir.fn(this.signature.callback.match, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); } else { @@ -100,7 +106,6 @@ class Context { } call(type, signature, ref, args) { - signature = this.signature(name); assert.strictEqual(args.length, signature.args.length); const irArgs = [ signature.ret, ref, '(' ]; @@ -119,7 +124,10 @@ class Context { if (cconv) cconv = ' ' + cconv; - return IR._(`${type}call${cconv}`, args); + const res = IR._(`${type}call${cconv}`, args); + if (signature.ret.isVoid()) + res.void(); + return res; } buildSwitch(body, type, what, values) { @@ -147,6 +155,8 @@ class Context { } field(fn, name) { + const INT = this.ctx.INT; + const stateArg = this.state(fn); return IR._('getelementptr', this.state, @@ -154,10 +164,5 @@ class Context { [ INT, INT.v(0) ], [ INT, INT.v(this.state.lookup(name)) ]); } - - state(fn) { return fn.arg(ARG_STATE); } - pos(fn) { return fn.arg(ARG_POS); } - endPos(fn) { return fn.arg(ARG_ENDPOS); } - match(fn) { return fn.arg(ARG_MATCH); } } module.exports = Context; diff --git a/lib/llparse/compiler/index.js b/lib/llparse/compiler/index.js index ef9560f..3dd65e8 100644 --- a/lib/llparse/compiler/index.js +++ b/lib/llparse/compiler/index.js @@ -3,6 +3,8 @@ exports.Context = require('./context'); exports.Stage = require('./stage'); exports.MatchSequence = require('./match-sequence'); + exports.span = require('./span'); -exports.NodeBuilder = require('./node'); +exports.node = require('./node'); + exports.Compiler = require('./compiler'); diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index 9b1e48f..61859a6 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -47,7 +47,8 @@ class MatchSequence extends compiler.Stage { } build() { - const fn = this.ir.fn(this.signature, `${this.ctx.prefix}__match_sequence`, + const fn = this.ctx.ir.fn(this.signature, + `${this.ctx.prefix}__match_sequence`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); this.buildBody(fn); @@ -56,6 +57,7 @@ class MatchSequence extends compiler.Stage { fn.cconv = CCONV; fn.attributes = 'nounwind norecurse alwaysinline'; + this.ctx.matchSequence = fn; return fn; } diff --git a/lib/llparse/compiler/node.js b/lib/llparse/compiler/node.js deleted file mode 100644 index fa7c6d4..0000000 --- a/lib/llparse/compiler/node.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -const compiler = require('./'); -const llparse = require('../'); - -class NodeBuilder extends compiler.Stage { - constructor(ctx) { - super(ctx, 'node-builder'); - } - - build() { - } -} -module.exports = NodeBuilder; diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js new file mode 100644 index 0000000..1251b3b --- /dev/null +++ b/lib/llparse/compiler/node/base.js @@ -0,0 +1,155 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const node = require('./'); +const llparse = require('../../'); +const constants = llparse.constants; + +const ARG_STATE = constants.ARG_STATE; +const ARG_POS = constants.ARG_POS; +const ARG_ENDPOS = constants.ARG_ENDPOS; +const ARG_MATCH = constants.ARG_MATCH; + +class NodeContext { + constructor(ctx, fn) { + this.ctx = ctx; + this.fn = fn; + + // Re-export some properties + this.ir = ctx.ir; + this.signature = ctx.signature; + this.stateType = ctx.state; + + // Some generally useful types (to avoid unnecessary includes) + this.INT = constants.INT; + this.BOOL = constants.BOOL; + this.TYPE_MATCH = constants.TYPE_MATCH; + this.TYPE_REASON = constants.TYPE_REASON; + } + + field(name) { + return this.ctx.field(fn, name); + } + + state() { return this.fn.arg(ARG_STATE); } + pos() { return this.fn.arg(ARG_POS); } + endPos() { return this.fn.arg(ARG_ENDPOS); } + match() { return this.fn.arg(ARG_MATCH); } + + // Just proxy + call(...args) { return this.ctx.call(...args); } + buildSwitch(...args) { return this.ctx.buildSwitch(...args); } +} + +class Node { + constructor(type, name) { + this.type = type; + this.name = name; + this.otherwise = null; + this.skip = false; + + this.fn = null; + this.phis = new Map(); + } + + setOtherwise(otherwise, skip) { + this.otherwise = otherwise; + this.skip = skip; + } + + build(ctx, nodes) { + if (nodes.has(this)) + return nodes.get(this); + + const fn = ctx.fn(ctx.signature.node, this.name); + const nctx = new NodeContext(ctx, fn); + + // Errors are assumed to be rarely called + if (this instanceof node.Error) + fn.attributes += ' cold writeonly'; + + nodes.set(this, fn); + + const body = this.prologue(nctx, fn.body); + this.doBuild(nctx, body, nodes); + + return fn; + } + + prologue(ctx, body) { + const pos = ctx.pos(); + const endPos = ctx.endPos(); + + // Check that we have enough chars to do the read + body.comment('--- Prologue ---'); + body.comment('if (pos != endpos)'); + const cmp = IR._('icmp', [ 'ne', pos.type, pos ], endPos); + body.push(cmp); + + const branch = body.branch('br', [ ctx.BOOL, cmp ]); + + // Return self when `pos === endpos` + branch.right.name = 'prologue_end'; + this.pause(ctx, branch.right); + + branch.left.name = 'prologue_normal'; + return branch.left; + } + + pause(ctx, body) { + const fn = ctx.fn; + const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', fn.type.ret ]); + body.push(bitcast); + body.terminate('ret', [ fn.type.ret, bitcast ]); + } + + tailTo(ctx, body, pos, target, value = null) { + if (this.phis.has(target)) { + const cached = this.phis.get(target); + + if (cached.phi) { + assert(value, '`.match()` and `.select()` with the same target'); + cached.phi.append( + [ '[', ctx.TYPE_MATCH.v(value), ',', body.ref(), ']' ]); + } else { + assert(!value, '`.match()` and `.select()` with the same target'); + } + + body.terminate('br', cached.trampoline); + return; + } + + // Split, so that others could join us from code block above + const trampoline = body.jump('br'); + trampoline.name = body.name + '_trampoline'; + let phi = null; + + // Compute `match` if needed + if (value !== null) { + trampoline.comment('Select `match`'); + phi = IR._('phi', + [ TYPE_MATCH, '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); + trampoline.push(phi); + } + + this.phis.set(target, { phi, trampoline }); + + // TODO(indutny): looks like `musttail` gives worse performance when calling + // Invoke nodes (possibly others too). + const call = ctx.call('musttail', ctx.signature.node, target, [ + ctx.state(), + pos, + ctx.endPos(), + phi ? phi : ctx.TYPE_MATCH.v(0) + ]); + + trampoline.push(call); + trampoline.terminate('ret', [ ctx.fn.type.ret, call ]); + } + + doBuild() { + throw new Error('Not implemented'); + } +} +module.exports = Node; diff --git a/lib/llparse/compiler/node/builder.js b/lib/llparse/compiler/node/builder.js new file mode 100644 index 0000000..33922aa --- /dev/null +++ b/lib/llparse/compiler/node/builder.js @@ -0,0 +1,26 @@ +'use strict'; + +const assert = require('assert'); + +const compiler = require('../'); +const llparse = require('../../'); + +const kCases = llparse.symbols.kCases; +const kCode = llparse.symbols.kCode; +const kMap = llparse.symbols.kMap; +const kOtherwise = llparse.symbols.kOtherwise; +const kSignature = llparse.symbols.kSignature; +const kSpan = llparse.symbols.kSpan; + +class NodeBuilder extends compiler.Stage { + constructor(ctx) { + super(ctx, 'node-builder'); + + this.nodes = new Map(); + } + + build() { + return this.ctx.stageResults['node-translator'].build(this.ctx, this.nodes); + } +} +module.exports = NodeBuilder; diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js new file mode 100644 index 0000000..e18fe21 --- /dev/null +++ b/lib/llparse/compiler/node/error.js @@ -0,0 +1,41 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const node = require('./'); + +class Error extends node.Node { + constructor(name, code, reason) { + super('error', name); + + this.code = code; + this.reason = reason; + } + + prologue(ctx, fn) { + return fn.body; + } + + doBuild(ctx, fn, body) { + const reason = ctx.ir.cstr(this.reason); + + const codePtr = ctx.field(fn, 'error'); + body.push(codePtr); + + const reasonPtr = ctx.field(fn, 'reason'); + body.push(reasonPtr); + + const castReason = IR._('bitcast', [ + reason.type, reason, 'to', ctx.TYPE_REASON + ]); + body.push(castReason); + + body.push(IR._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(code) ], + [ ctx.TYPE_ERROR.ptr(), codeField ]).void()); + body.push(IR._('store', [ ctx.TYPE_REASON, castReason ], + [ ctx.TYPE_REASON.ptr(), reasonField ]).void()); + + body.terminate('ret', [ fn.ret.type, fn.ret.type.v(null) ]); + } +} +module.exports = Error; diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js new file mode 100644 index 0000000..4272f49 --- /dev/null +++ b/lib/llparse/compiler/node/index.js @@ -0,0 +1,12 @@ +'use strict'; + +exports.Node = require('./base'); +exports.Error = require('./error'); +exports.Invoke = require('./invoke'); +exports.Single = require('./single'); +exports.Sequence = require('./sequence'); +exports.SpanStart = require('./span-start'); +exports.SpanEnd = require('./span-end'); + +exports.Translator = require('./translator'); +exports.Builder = require('./builder'); diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js new file mode 100644 index 0000000..7c6a125 --- /dev/null +++ b/lib/llparse/compiler/node/invoke.js @@ -0,0 +1,20 @@ +'use strict'; + +const node = require('./'); + +class Invoke extends node.Node { + constructor(name, code, map) { + super('invoke', name); + + this.code = code; + this.map = map; + } + + prologue(ctx, fn) { + return fn.body; + } + + doBuild(ctx, fn, body) { + } +} +module.exports = Invoke; diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js new file mode 100644 index 0000000..656044c --- /dev/null +++ b/lib/llparse/compiler/node/sequence.js @@ -0,0 +1,25 @@ +'use strict'; + +const node = require('./'); + +class Sequence extends node.Node { + constructor(name, select) { + super('sequence', name); + + this.select = select; + this.next = null; + } + + setOtherwise(otherwise, skip) { + // Break loops + if (this.otherwise === otherwise) + return; + + super.setOtherwise(otherwise, skip); + this.next.setOtherwise(otherwise, skip); + } + + doBuild(ctx, fn, body, nodes) { + } +} +module.exports = Sequence; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js new file mode 100644 index 0000000..ac01f71 --- /dev/null +++ b/lib/llparse/compiler/node/single.js @@ -0,0 +1,39 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const node = require('./'); + +class Single extends node.Node { + constructor(name) { + super('single', name); + + this.children = null; + } + + setOtherwise(otherwise, skip) { + // Break loops + if (this.otherwise === otherwise) + return; + + super.setOtherwise(otherwise, skip); + this.children.forEach(child => child.next.setOtherwise(otherwise, skip)); + } + + doBuild(ctx, fn, body, nodes) { + const pos = ctx.posArg(fn); + + // Load the character + const current = IR._('load', pos.type.to, [ pos.type, pos ]); + body.push(current); + + const keys = this.children.map(child => child.key); + const s = ctx.buildSwitch(body, pos.type.to, current, keys); + + s.cases.forEach((body, i) => { + const child = this.children[i].next.build(ctx, nodes); + }); + + } +} +module.exports = Single; diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js new file mode 100644 index 0000000..2d6ff40 --- /dev/null +++ b/lib/llparse/compiler/node/span-end.js @@ -0,0 +1,15 @@ +'use strict'; + +const node = require('./'); + +class SpanEnd extends node.Node { + constructor(name, span) { + super('span-end', name); + + this.span = span; + } + + doBuild(ctx, fn, body, nodes) { + } +} +module.exports = SpanEnd; diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js new file mode 100644 index 0000000..23bf1ed --- /dev/null +++ b/lib/llparse/compiler/node/span-start.js @@ -0,0 +1,15 @@ +'use strict'; + +const node = require('./'); + +class SpanStart extends node.Node { + constructor(name, span) { + super('span-start', name); + + this.span = span; + } + + doBuild(ctx, fn, body, nodes) { + } +} +module.exports = SpanStart; diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js new file mode 100644 index 0000000..471fb99 --- /dev/null +++ b/lib/llparse/compiler/node/translator.js @@ -0,0 +1,125 @@ +'use strict'; + +const assert = require('assert'); + +const compiler = require('../'); +const llparse = require('../../'); + +const kCases = llparse.symbols.kCases; +const kCode = llparse.symbols.kCode; +const kMap = llparse.symbols.kMap; +const kOtherwise = llparse.symbols.kOtherwise; +const kSignature = llparse.symbols.kSignature; +const kSpan = llparse.symbols.kSpan; + +class NodeTranslator extends compiler.Stage { + constructor(ctx) { + super(ctx, 'node-translator'); + + this.nodes = new Map(); + this.namespace = new Set(); + } + + build() { + // TODO(indutny): detect and report loops + + return this.buildNode(this.ctx.root, null); + } + + id(node) { + let res = node.name; + if (this.namespace.has(res)) { + let i; + for (i = 1; i <= this.namespace.size; i++) + if (!this.namespace.has(res + '_' + i)) + break; + res += '_' + i; + } + + this.namespace.add(res); + return this.ctx.prefix + '__n_' + res; + } + + buildNode(node, value) { + this.checkSignature(node, value); + + if (this.nodes.has(node)) + return this.nodes.get(node); + + let res; + if (node instanceof llparse.node.Invoke) { + assert.strictEqual(node[kCases].length, 0); + res = new compiler.node.Invoke(this.id(node), node[kCode], node[kMap]); + } else if (node instanceof llparse.node.Error) { + assert.strictEqual(node[kCases].length, 0); + res = new compiler.node.Error(this.id(node), node.code, node.reason); + } else if (node instanceof llparse.node.SpanStart) { + assert.strictEqual(node[kCases].length, 0); + res = new compiler.node.SpanStart(this.id(node), node[kSpan]); + } else if (node instanceof llparse.node.SpanEnd) { + assert.strictEqual(node[kCases].length, 0); + res = new compiler.node.SpanEnd(this.id(node), node[kSpan]); + } else { + const trie = new llparse.Trie(node.name); + const combined = trie.combine(node[kCases]); + + res = this.buildTrie(node, trie.combine(node[kCases]), true); + } + + this.nodes.set(node, res); + + if (node instanceof llparse.node.Error) { + assert.strictEqual(node[kOtherwise], null); + } else { + assert.notStrictEqual(node[kOtherwise], null, + `Node "${node.name}" must have \`.otherwise()\`/\`.skipTo()\``); + const otherwise = this.buildNode(node[kOtherwise].next, null); + res.setOtherwise(otherwise, node[kOtherwise].skip); + } + + return res; + } + + buildTrie(node, trie, isRoot = false) { + if (trie.type === 'next') { + assert(!isRoot); + return this.buildNode(trie.next, trie.value); + } + + if (trie.type === 'single') + return this.buildSingle(node, trie, isRoot); + + assert.strictEqual(trie.type, 'sequence'); + return this.buildSequence(node, trie, isRoot); + } + + buildSingle(node, trie, isRoot) { + const res = new compiler.node.Single(this.id(node)); + + // Break loops + if (isRoot) + this.nodes.set(node, res); + + res.children = trie.children.map((child) => { + return { key: child.key, next: this.buildTrie(node, child.child) }; + }); + return res; + } + + buildSequence(node, trie, isRoot) { + const res = new compiler.node.Sequence(this.id(node), trie.select); + + // Break loops + if (isRoot) + this.nodes.set(node, res); + + res.next = this.buildTrie(node, trie.child); + + return res; + } + + checkSignature(node, value) { + assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); + } +} +module.exports = NodeTranslator; diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index ce4dd5a..fef1198 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -1,10 +1,39 @@ 'use strict'; +const llparse = require('../../'); const compiler = require('../'); +const constants = llparse.constants; + +const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; +const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; + + class Builder extends compiler.Stage { constructor(ctx) { super(ctx, 'span-builder'); } + + build() { + // TODO(indutny): implement me + } + + buildSpanFields() { + const allocator = new llparse.span.Allocator(); + + const colors = allocator.execute(this.ctx.root); + + colors.concurrency.forEach((num, index) => { + this.state.field(TYPE_INPUT, SPAN_START_PREFIX + index); + + // TODO(indutny): use smaller field size if possible? + if (num === 1) + return; + + this.state.field(this.signature.callback.match.ptr(), + SPAN_CB_PREFIX + index); + }); + console.log(colors); + } } module.exports = Builder; diff --git a/lib/llparse/compiler/stage.js b/lib/llparse/compiler/stage.js index fc5aa8c..113c347 100644 --- a/lib/llparse/compiler/stage.js +++ b/lib/llparse/compiler/stage.js @@ -5,5 +5,9 @@ class Stage { this.ctx = ctx; this.name = name; } + + build() { + throw new Error('Not implemented'); + } } module.exports = Stage; diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index b63c7e9..23ed8dc 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -6,7 +6,6 @@ const node = require('./'); const llparse = require('../'); const kOtherwise = llparse.symbols.kOtherwise; -const kNoAdvance = llparse.symbols.kNoAdvance; const kCases = llparse.symbols.kCases; const kSignature = llparse.symbols.kSignature; @@ -21,7 +20,6 @@ class Node { this[kCases] = []; this[kOtherwise] = null; - this[kNoAdvance] = false; } get name() { return this[kName]; } diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index 28b6fb1..c5976e2 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -3,7 +3,6 @@ const node = require('./'); const llparse = require('../'); -const kNoAdvance = llparse.symbols.kNoAdvance; const kCode = Symbol('code'); const kReason = Symbol('reason'); @@ -13,11 +12,14 @@ class Error extends node.Node { this[kCode] = code; this[kReason] = reason; - - this[kNoAdvance] = true; } get code() { return this[kCode]; } get reason() { return this[kReason]; } + + match() { throw new Error('Not supported'); } + select() { throw new Error('Not supported'); } + otherwise() { throw new Error('Not supported'); } + skipTo() { throw new Error('Not supported'); } } module.exports = Error; diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index a8acc4d..a351f5e 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -7,8 +7,8 @@ const llparse = require('../'); const kNoAdvance = llparse.symbols.kNoAdvance; const kType = llparse.symbols.kType; -const kCode = Symbol('code'); -const kMap = Symbol('map'); +const kCode = llparse.symbols.kCode; +const kMap = llparse.symbols.kMap; class Invoke extends node.Node { constructor(code, map, otherwise = null) { @@ -37,13 +37,8 @@ class Invoke extends node.Node { if (otherwise) this.otherwise(otherwise); - - this[kNoAdvance] = true; } - get code() { return this[kCode]; } - get map() { return this[kMap]; } - match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } } diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js index 3b866cb..3b7059c 100644 --- a/lib/llparse/node/span-end.js +++ b/lib/llparse/node/span-end.js @@ -3,13 +3,11 @@ const llparse = require('../'); const node = require('./'); -const kNoAdvance = llparse.symbols.kNoAdvance; const kSpan = llparse.symbols.kSpan; class SpanEnd extends node.Node { constructor(span, callback) { super('span_end_' + callback); - this[kNoAdvance] = true; this[kSpan] = span; } diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js index 106f830..78e3d22 100644 --- a/lib/llparse/node/span-start.js +++ b/lib/llparse/node/span-start.js @@ -3,14 +3,12 @@ const llparse = require('../'); const node = require('./'); -const kNoAdvance = llparse.symbols.kNoAdvance; const kSpan = llparse.symbols.kSpan; class SpanStart extends node.Node { constructor(span, callback) { super('span_start_' + callback); - this[kNoAdvance] = true; this[kSpan] = span; } diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 5ae94f1..07e0455 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -3,6 +3,8 @@ exports.kBody = Symbol('body'); exports.kCallback = Symbol('callback'); exports.kCases = Symbol('cases'); +exports.kCode = Symbol('code'); +exports.kMap = Symbol('map'); exports.kName = Symbol('name'); exports.kNoAdvance = Symbol('noAdvance'); exports.kOtherwise = Symbol('otherwise'); diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index fc73255..afda10e 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -19,6 +19,7 @@ class Sequence extends Node { constructor(select) { super('sequence'); this.select = select; + this.child = null; } } @@ -93,7 +94,7 @@ class Trie { const res = new Sequence(prefix); const sliced = this.slice(list, prefix.length); - res.children = this.level(sliced, path.concat(prefix)); + res.child = this.level(sliced, path.concat(prefix)); return res; } From 4f4ee9910ac94089b0d5a373af22f07c10f0336e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 16:29:34 -0500 Subject: [PATCH 118/281] compiler: it builds! --- lib/llparse/compiler/compiler.js | 301 +++--------------------- lib/llparse/compiler/context.js | 68 +++++- lib/llparse/compiler/node/base.js | 93 +++++--- lib/llparse/compiler/node/error.js | 32 +-- lib/llparse/compiler/node/invoke.js | 38 ++- lib/llparse/compiler/node/sequence.js | 67 +++++- lib/llparse/compiler/node/single.js | 15 +- lib/llparse/compiler/node/span-end.js | 3 +- lib/llparse/compiler/node/span-start.js | 3 +- lib/llparse/compiler/node/translator.js | 5 +- 10 files changed, 285 insertions(+), 340 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 79da689..bad8d84 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -1,7 +1,6 @@ 'use strict'; const assert = require('assert'); -const IR = require('llvm-ir'); const llparse = require('../'); const constants = llparse.constants; @@ -67,31 +66,31 @@ class Compiler { ]; ctx.buildStages(stages); - console.log(ctx.stageResults['node-builder']); - // this.buildInit(rootFn.ref()); - // this.buildParse(); + this.buildInit(ctx); + this.buildParse(ctx); return ctx.ir.build(); } - buildInit(fn) { - const sig = IR.signature(IR.void(), [ this.state.ptr() ]); - const init = this.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); + buildInit(ctx) { + const fn = ctx.stageResults['node-builder'].ref(); + const sig = ctx.ir.signature(ctx.ir.void(), [ ctx.state.ptr() ]); + const init = ctx.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); const fields = { - current: this.field(init, 'current'), - error: this.field(init, 'error'), - reason: this.field(init, 'reason'), - index: this.field(init, 'index'), - mark: this.field(init, 'mark'), - data: this.field(init, 'data') + current: ctx.field(init, 'current'), + error: ctx.field(init, 'error'), + reason: ctx.field(init, 'reason'), + index: ctx.field(init, 'index'), + mark: ctx.field(init, 'mark'), + data: ctx.field(init, 'data') }; Object.keys(fields).forEach(key => init.body.push(fields[key])); const store = (field, type, value) => { - init.body.push(IR._('store', [ type, value ], + init.body.push(ctx.ir._('store', [ type, value ], [ type.ptr(), field ]).void()); }; @@ -102,31 +101,31 @@ class Compiler { store(fields.mark, TYPE_INPUT, TYPE_INPUT.v(null)); store(fields.data, TYPE_DATA, TYPE_DATA.v(null)); - init.body.terminate('ret', IR.void()); + init.body.terminate('ret', ctx.ir.void()); return init; } - buildParse() { + buildParse(ctx) { // TODO(indutny): change signature to (state*, start*, len)? - const sig = IR.signature(TYPE_ERROR, - [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); - const parse = this.ir.fn(sig, this.prefix + '_execute', + const sig = ctx.ir.signature(TYPE_ERROR, + [ ctx.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); + const parse = ctx.ir.fn(sig, this.prefix + '_execute', [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); const body = parse.body; - const nodeSig = this.signature.node; + const nodeSig = ctx.signature.node; - const currentPtr = this.field(parse, 'current'); + const currentPtr = ctx.field(parse, 'current'); body.push(currentPtr); - const current = IR._('load', nodeSig.ptr(), + const current = ctx.ir._('load', nodeSig.ptr(), [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); - const call = IR._(`call ${CCONV}`, [ + const call = ctx.ir._(`call ${CCONV}`, [ TYPE_OUTPUT, current, '(', - this.state.ptr(), parse.arg(ARG_STATE), ',', + ctx.state.ptr(), parse.arg(ARG_STATE), ',', TYPE_INPUT, parse.arg(ARG_POS), ',', TYPE_INPUT, parse.arg(ARG_ENDPOS), ',', TYPE_MATCH, TYPE_MATCH.v(0), @@ -134,14 +133,15 @@ class Compiler { ]); body.push(call); - const errorPtr = this.field(parse, 'error'); + const errorPtr = ctx.field(parse, 'error'); body.push(errorPtr); - const error = IR._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); + const error = ctx.ir._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); body.push(error); - const bitcast = IR._('bitcast', [ TYPE_OUTPUT, call, 'to', nodeSig.ptr() ]); + const bitcast = ctx.ir._('bitcast', + [ TYPE_OUTPUT, call, 'to', nodeSig.ptr() ]); body.push(bitcast); - body.push(IR._('store', [ nodeSig.ptr(), bitcast ], + body.push(ctx.ir._('store', [ nodeSig.ptr(), bitcast ], [ nodeSig.ptr().ptr(), currentPtr ]).void()); body.terminate('ret', [ TYPE_ERROR, error ]); @@ -179,251 +179,6 @@ class Compiler { return body; } - buildNode(node) { - if (this.nodeMap.has(node)) - return this.nodeMap.get(node); - - const fn = this.createFn(node); - this.nodeMap.set(node, { fn, node }); - - let body = this.buildPrologue(node, fn); - - body = this.debug(fn, body, `Entering node: "${node.name}"`); - - if (node instanceof llparse.node.Error) { - const info = { node, fn, otherwise: null }; - this.buildError(info, body); - return { fn, node }; - } - - const otherwise = node[kOtherwise]; - assert.notStrictEqual(otherwise, null, - `Node "${node.name}" must have \`.otherwise()\`/\`.skipTo()\``); - const info = { - node, - fn, - otherwise: otherwise.next, - skip: otherwise.skip - }; - - const trie = new llparse.Trie(node.name); - const combined = trie.combine(node[kCases]); - - this.buildTrie(info, body, combined); - - return { fn, node }; - } - - buildCode(node, code) { - assert.strictEqual(node[kSignature], code[kType]); - - const signature = code[kType] === 'match' ? - this.signature.callback.match : - this.signature.callback.value; - - const name = code.name; - if (this.codeMap.has(name)) { - const cached = this.codeMap.get(name); - assert.strictEqual(cached.type, signature, - `Conflicting code entries for "${name}"`); - return cached; - } - - let fn; - if (code[kBody] === null) { - const external = this.ir.declare(signature, name); - external.attributes = 'alwaysinline'; - - fn = external; - } else { - fn = this.buildCodeWithBody(code, signature); - } - - this.codeMap.set(name, fn); - return fn; - } - - buildCodeWithBody(code, signature) { - const args = [ - constants.ARG_STATE, - constants.ARG_POS, - constants.ARG_ENDPOS - ]; - - if (code[kType] === 'value') - args.push(constants.ARG_MATCH); - - const fn = this.ir.fn(signature, code.name, args); - - fn.visibility = 'internal'; - fn.cconv = CCONV; - fn.attributes = 'nounwind'; - - const context = new llparse.code.Context(code, fn); - - code[kBody].call(fn.body, this.ir, context); - - return fn; - } - - buildInvoke(info, body, pos) { - const code = this.buildCode(info.node, info.node.code); - - const args = [ - code.type.ret, code, '(', - this.state.ptr(), info.fn.arg(ARG_STATE), ',', - TYPE_INPUT, info.fn.arg(ARG_POS), ',', - TYPE_INPUT, info.fn.arg(ARG_ENDPOS) - ]; - - if (info.node[kSignature] === 'value') - args.push(',', TYPE_MATCH, info.fn.arg(ARG_MATCH)); - else - assert.strictEqual(info.node[kSignature], 'match'); - - args.push(')'); - - const cconv = code.cconv ? ' ' + code.cconv : ''; - - const call = IR._('call' + cconv, args); - body.push(call); - - const keys = Object.keys(info.node.map).map(key => key | 0); - const s = this.buildSwitch(body, code.type.ret, call, keys); - - s.cases.forEach((body, i) => { - const subNode = info.node.map[keys[i]]; - this.buildRedirect(info, body, pos, this.buildNode(subNode)); - }); - - return s.otherwise; - } - - buildSubTrie(info, body, pos, trie) { - if (trie.type === 'next') - return this.buildNext(info, body, pos, trie); - - const subFn = this.createFn(info.node); - const subInfo = { fn: subFn, node: info.node, otherwise: info.otherwise }; - - const subBody = this.buildPrologue(info.node, subFn); - this.buildTrie(subInfo, subBody, trie); - - this.buildRedirect(info, body, pos, { fn: subFn, node: info.node }); - return subFn; - } - - buildTrie(info, body, trie) { - const fn = info.fn; - - // Increment `pos` if not invoking external callback - let pos = { current: fn.arg(ARG_POS), next: null }; - - // NOTE: `sequence` has loop inside it - so it isn't going to use - // `pos.next` anyway (as a matter of fact it doesn't get `pos` as an - // argument at all) - if (info.node[kNoAdvance] || trie && trie.type === 'sequence') { - pos.next = pos.current; - } else { - body.comment('next = pos + 1'); - pos.next = IR._('getelementptr', TYPE_INPUT.to, - [ TYPE_INPUT, fn.arg(ARG_POS) ], - [ INT, INT.v(1) ]); - body.push(pos.next); - } - - if (info.node instanceof llparse.node.Invoke) - body = this.buildInvoke(info, body, pos); - - // Traverse the `trie` - if (trie === null) { - // no-op - } else if (trie.type === 'single') { - body = this.buildSingle(info, body, pos, trie.children); - } else if (trie.type === 'sequence') { - // NOTE: do not send `pos` here! (see comment above) - const seq = this.buildSequence(info, body, trie); - - // NOTE: `sequence` implementation loops if there's enough data - body = seq.body; - pos = seq.pos; - } else { - // NOTE: `next` type must be parsed in `buildSubTrie` - throw new Error('Unexpected trie node type: ' + trie.type); - } - - // Do not increment `pos` when falling through, unless we're skipping - const otherwisePos = { current: pos.current, next: pos.current }; - if (info.skip) - otherwisePos.next = pos.next; - this.buildRedirect(info, body, otherwisePos, - this.buildNode(info.otherwise)); - - return body; - } - - buildSequence(info, body, trie) { - assert(!info.node[kNoAdvance]); - - const seq = this.ir.data(trie.select); - - const cast = IR._('getelementptr inbounds', seq.type.to, - [ seq.type, seq ], - [ INT, INT.v(0) ], - [ INT, INT.v(0) ]); - body.push(cast); - - const returnType = this.matchSequence.type.ret; - - const call = IR._(`call ${CCONV}`, [ - returnType, this.matchSequence, '(', - this.state.ptr(), info.fn.arg(ARG_STATE), ',', - TYPE_INPUT, info.fn.arg(ARG_POS), ',', - TYPE_INPUT, info.fn.arg(ARG_ENDPOS), ',', - TYPE_INPUT, cast, ',', - INT, INT.v(seq.type.to.length), - ')' - ]); - body.push(call); - - const status = IR._('extractvalue', [ returnType, call ], - INT.v(returnType.lookup('status'))); - body.push(status); - - const current = IR._('extractvalue', [ returnType, call ], - INT.v(returnType.lookup('current'))); - body.push(current); - - // This is lame, but it is easier to do it this way - // (Optimizer will remove it, if it isn't needed) - body.comment('next = pos + 1'); - const next = IR._('getelementptr', TYPE_INPUT.to, - [ TYPE_INPUT, current ], - [ INT, INT.v(1) ]); - body.push(next); - - const pos = { current, next }; - - const s = this.buildSwitch(body, INT, status, [ - SEQUENCE_COMPLETE, - SEQUENCE_PAUSE, - SEQUENCE_MISMATCH - ]); - - // No other values are allowed - s.otherwise.terminate('unreachable'); - - const complete = s.cases[0]; - const pause = s.cases[1]; - const mismatch = s.cases[2]; - - this.buildSubTrie(info, complete, pos, trie.children); - this.buildSelfReturn(info, pause); - - // Not equal - // Reset `state.index` on mismatch - return { pos, body: mismatch }; - } // Helpers diff --git a/lib/llparse/compiler/context.js b/lib/llparse/compiler/context.js index 8c00db8..bd54622 100644 --- a/lib/llparse/compiler/context.js +++ b/lib/llparse/compiler/context.js @@ -6,6 +6,9 @@ const IR = require('llvm-ir'); const llparse = require('../'); const constants = llparse.constants; +const kType = llparse.symbols.kType; +const kBody = llparse.symbols.kBody; + const CCONV = constants.CCONV; const BOOL = constants.BOOL; @@ -59,6 +62,8 @@ class Context { this.state.field(TYPE_INDEX, 'index'); this.state.field(TYPE_INPUT, 'mark'); + this.codeCache = new Map(); + // Public fields this.state.field(TYPE_DATA, 'data'); @@ -67,6 +72,7 @@ class Context { this.state.field(prop.type(this.ir, this.state), prop.name); }); + // Intermediate results from various build stages this.stageResults = {}; } @@ -78,6 +84,55 @@ class Context { this.stageResults[stage.name] = stage.build(this); } + buildCode(code) { + const signatures = this.signature.callback; + const signature = code[kType] === 'match' ? + signatures.match : signatures.value; + + const name = code.name; + if (this.codeCache.has(name)) { + const cached = this.codeCache.get(name); + assert.strictEqual(cached.type, signature, + `Conflicting code entries for "${name}"`); + return cached; + } + + let fn; + if (code[kBody] === null) { + const external = this.ir.declare(signature, name); + + // NOTE: this has no effect due to machine-specific flags + // TODO(indutny): find a way to make it inline the function + external.attributes = 'alwaysinline'; + + fn = external; + } else { + fn = this.buildCodeWithBody(code, signature); + } + + this.codeCache.set(name, fn); + return fn; + } + + buildCodeWithBody(code, signature) { + const args = [ ARG_STATE, ARG_POS, ARG_ENDPOS ]; + + if (code[kType] === 'value') + args.push(ARG_MATCH); + + const fn = this.ir.fn(signature, code.name, args); + + fn.visibility = 'internal'; + fn.cconv = CCONV; + fn.attributes = 'nounwind'; + + const context = new llparse.code.Context(code, fn); + + code[kBody].call(fn.body, this.ir, context); + + return fn; + } + fn(signature, name) { name = this.prefix + '__' + name; @@ -120,11 +175,11 @@ class Context { if (type) type += ' '; - let cconv = ref.cconv; + let cconv = ref.cconv || ''; if (cconv) cconv = ' ' + cconv; - const res = IR._(`${type}call${cconv}`, args); + const res = IR._(`${type}call${cconv}`, irArgs); if (signature.ret.isVoid()) res.void(); return res; @@ -155,14 +210,17 @@ class Context { } field(fn, name) { - const INT = this.ctx.INT; - - const stateArg = this.state(fn); + const stateArg = this.stateArg(fn); return IR._('getelementptr', this.state, [ stateArg.type, stateArg ], [ INT, INT.v(0) ], [ INT, INT.v(this.state.lookup(name)) ]); } + + stateArg(fn) { return fn.arg(ARG_STATE); } + posArg(fn) { return fn.arg(ARG_POS); } + endPosArg(fn) { return fn.arg(ARG_ENDPOS); } + matchArg(fn) { return fn.arg(ARG_MATCH); } } module.exports = Context; diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 1251b3b..785d34e 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -1,45 +1,44 @@ 'use strict'; -const IR = require('llvm-ir'); +const assert = require('assert') const node = require('./'); const llparse = require('../../'); const constants = llparse.constants; -const ARG_STATE = constants.ARG_STATE; -const ARG_POS = constants.ARG_POS; -const ARG_ENDPOS = constants.ARG_ENDPOS; -const ARG_MATCH = constants.ARG_MATCH; - class NodeContext { constructor(ctx, fn) { - this.ctx = ctx; + this.compilation = ctx; this.fn = fn; + this.state = this.compilation.stateArg(this.fn); + this.pos = { + current: this.compilation.posArg(this.fn), + next: null + }; + this.endPos = this.compilation.endPosArg(this.fn); + this.match = this.compilation.matchArg(this.fn); + // Re-export some properties - this.ir = ctx.ir; - this.signature = ctx.signature; - this.stateType = ctx.state; + this.ir = this.compilation.ir; + this.signature = this.compilation.signature; + this.stateType = this.compilation.state; // Some generally useful types (to avoid unnecessary includes) this.INT = constants.INT; this.BOOL = constants.BOOL; this.TYPE_MATCH = constants.TYPE_MATCH; this.TYPE_REASON = constants.TYPE_REASON; + this.TYPE_ERROR = constants.TYPE_ERROR; } + // Just proxy field(name) { - return this.ctx.field(fn, name); + return this.compilation.field(this.fn, name); } - state() { return this.fn.arg(ARG_STATE); } - pos() { return this.fn.arg(ARG_POS); } - endPos() { return this.fn.arg(ARG_ENDPOS); } - match() { return this.fn.arg(ARG_MATCH); } - - // Just proxy - call(...args) { return this.ctx.call(...args); } - buildSwitch(...args) { return this.ctx.buildSwitch(...args); } + call(...args) { return this.compilation.call(...args); } + buildSwitch(...args) { return this.compilation.buildSwitch(...args); } } class Node { @@ -58,48 +57,60 @@ class Node { this.skip = skip; } - build(ctx, nodes) { + build(compilation, nodes) { if (nodes.has(this)) return nodes.get(this); - const fn = ctx.fn(ctx.signature.node, this.name); - const nctx = new NodeContext(ctx, fn); + const fn = compilation.fn(compilation.signature.node, this.name); + const ctx = new NodeContext(compilation, fn); // Errors are assumed to be rarely called - if (this instanceof node.Error) + // TODO(indutny): move this to node.Error somehow? + if (this instanceof node.Error) { + fn.attributes = fn.attributes || ''; fn.attributes += ' cold writeonly'; + } nodes.set(this, fn); - const body = this.prologue(nctx, fn.body); - this.doBuild(nctx, body, nodes); + let body = fn.body; + body = this.prologue(ctx, body); + + body.comment('next = pos + 1'); + ctx.pos.next = ctx.ir._('getelementptr', ctx.pos.current.type.to, + [ ctx.pos.current.type, ctx.pos.current ], + [ ctx.INT, ctx.INT.v(1) ]); + body.push(ctx.pos.next); + + this.doBuild(ctx, body, nodes); return fn; } prologue(ctx, body) { - const pos = ctx.pos(); - const endPos = ctx.endPos(); + const pos = ctx.pos.current; + const endPos = ctx.endPos; // Check that we have enough chars to do the read body.comment('--- Prologue ---'); body.comment('if (pos != endpos)'); - const cmp = IR._('icmp', [ 'ne', pos.type, pos ], endPos); + const cmp = ctx.ir._('icmp', [ 'ne', pos.type, pos ], endPos); body.push(cmp); const branch = body.branch('br', [ ctx.BOOL, cmp ]); // Return self when `pos === endpos` - branch.right.name = 'prologue_end'; + branch.right.name = 'no_data'; this.pause(ctx, branch.right); - branch.left.name = 'prologue_normal'; + branch.left.name = 'has_data'; return branch.left; } pause(ctx, body) { const fn = ctx.fn; - const bitcast = IR._('bitcast', [ fn.type.ptr(), fn, 'to', fn.type.ret ]); + const bitcast = ctx.ir._('bitcast', + [ fn.type.ptr(), fn, 'to', fn.type.ret ]); body.push(bitcast); body.terminate('ret', [ fn.type.ret, bitcast ]); } @@ -128,8 +139,8 @@ class Node { // Compute `match` if needed if (value !== null) { trampoline.comment('Select `match`'); - phi = IR._('phi', - [ TYPE_MATCH, '[', TYPE_MATCH.v(value), ',', body.ref(), ']' ]); + phi = ctx.ir._('phi', + [ ctx.TYPE_MATCH, '[', ctx.TYPE_MATCH.v(value), ',', body.ref(), ']' ]); trampoline.push(phi); } @@ -138,9 +149,9 @@ class Node { // TODO(indutny): looks like `musttail` gives worse performance when calling // Invoke nodes (possibly others too). const call = ctx.call('musttail', ctx.signature.node, target, [ - ctx.state(), + ctx.state, pos, - ctx.endPos(), + ctx.endPos, phi ? phi : ctx.TYPE_MATCH.v(0) ]); @@ -151,5 +162,17 @@ class Node { doBuild() { throw new Error('Not implemented'); } + + doOtherwise(ctx, nodes, body, pos) { + if (!pos) + pos = ctx.pos; + + // `.skipTo()` advances by one byte + // `.otherwise()` redirects using the same byte + const next = this.skip ? pos.next : pos.current; + + assert(this.otherwise); + this.tailTo(ctx, body, next, this.otherwise.build(ctx.compilation, nodes)); + } } module.exports = Node; diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index e18fe21..11833f9 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -12,30 +12,34 @@ class Error extends node.Node { this.reason = reason; } - prologue(ctx, fn) { - return fn.body; + prologue(ctx, body) { + return body; } - doBuild(ctx, fn, body) { + doBuild(ctx, body) { + body.comment('node.Error'); + + const INT = ctx.INT; + const reason = ctx.ir.cstr(this.reason); - const codePtr = ctx.field(fn, 'error'); + const codePtr = ctx.field('error'); body.push(codePtr); - const reasonPtr = ctx.field(fn, 'reason'); + const reasonPtr = ctx.field('reason'); body.push(reasonPtr); - const castReason = IR._('bitcast', [ - reason.type, reason, 'to', ctx.TYPE_REASON - ]); - body.push(castReason); + const cast = IR._('getelementptr inbounds', reason.type.to, + [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + body.push(cast); - body.push(IR._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(code) ], - [ ctx.TYPE_ERROR.ptr(), codeField ]).void()); - body.push(IR._('store', [ ctx.TYPE_REASON, castReason ], - [ ctx.TYPE_REASON.ptr(), reasonField ]).void()); + body.push(IR._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(this.code) ], + [ ctx.TYPE_ERROR.ptr(), codePtr ]).void()); + body.push(IR._('store', [ ctx.TYPE_REASON, cast ], + [ ctx.TYPE_REASON.ptr(), reasonPtr ]).void()); - body.terminate('ret', [ fn.ret.type, fn.ret.type.v(null) ]); + const retType = ctx.fn.type.ret; + body.terminate('ret', [ retType, retType.v(null) ]); } } module.exports = Error; diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 7c6a125..1880e0c 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -1,5 +1,10 @@ 'use strict'; +const assert = require('assert'); + +const llparse = require('../../'); +const kType = llparse.symbols.kType; + const node = require('./'); class Invoke extends node.Node { @@ -10,11 +15,38 @@ class Invoke extends node.Node { this.map = map; } - prologue(ctx, fn) { - return fn.body; + prologue(ctx, body) { + return body; } - doBuild(ctx, fn, body) { + doBuild(ctx, body, nodes) { + body.comment(`node.Invoke[${this.code.name}]`); + const code = ctx.compilation.buildCode(this.code); + + const args = [ + ctx.state, + ctx.pos.current, + ctx.endPos + ]; + + if (this.code[kType] === 'value') + args.push(ctx.match); + else + assert.strictEqual(this.code[kType], 'match'); + + const call = ctx.call('', code.type, code, args); + body.push(call); + + const keys = Object.keys(this.map).map(key => key | 0); + const s = ctx.buildSwitch(body, code.type.ret, call, keys); + + s.cases.forEach((body, i) => { + const child = info.node.map[keys[i]].build(ctx.compilation, nodes); + + this.tailTo(ctx, body, ctx.pos.next, child, this.children[i].value); + }); + + this.doOtherwise(ctx, nodes, s.otherwise); } } module.exports = Invoke; diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 656044c..0c87048 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -1,13 +1,21 @@ 'use strict'; +const llparse = require('../../'); +const constants = llparse.constants; + const node = require('./'); +const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; +const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; +const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; + class Sequence extends node.Node { constructor(name, select) { super('sequence', name); this.select = select; this.next = null; + this.value = null; } setOtherwise(otherwise, skip) { @@ -19,7 +27,64 @@ class Sequence extends node.Node { this.next.setOtherwise(otherwise, skip); } - doBuild(ctx, fn, body, nodes) { + doBuild(ctx, body, nodes) { + const INT = ctx.INT; + + body.comment(`node.Sequence["${this.select.toString('hex') }"]`); + + const seq = ctx.ir.data(this.select); + + const cast = ctx.ir._('getelementptr inbounds', seq.type.to, + [ seq.type, seq ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + body.push(cast); + + const matchSequence = ctx.compilation.stageResults['match-sequence']; + + const returnType = matchSequence.type.ret; + + const call = ctx.call('', matchSequence.type, matchSequence, [ + ctx.state, + ctx.pos.current, + ctx.endPos, + cast, + INT.v(seq.type.to.length) + ]); + body.push(call); + + const status = ctx.ir._('extractvalue', [ returnType, call ], + INT.v(returnType.lookup('status'))); + body.push(status); + + const current = ctx.ir._('extractvalue', [ returnType, call ], + INT.v(returnType.lookup('current'))); + body.push(current); + + // This is lame, but it is easier to do it this way + // (Optimizer will remove it, if it isn't needed) + body.comment('next = pos + 1'); + const next = ctx.ir._('getelementptr', ctx.pos.current.type.to, + [ ctx.pos.current.type, current ], [ INT, INT.v(1) ]); + body.push(next); + + const s = ctx.buildSwitch(body, INT, status, [ + SEQUENCE_COMPLETE, + SEQUENCE_PAUSE, + SEQUENCE_MISMATCH + ]); + + // No other values are allowed + s.otherwise.terminate('unreachable'); + + const complete = s.cases[0]; + const pause = s.cases[1]; + const mismatch = s.cases[2]; + + this.tailTo(ctx, complete, next, + this.next.build(ctx.compilation, nodes), this.value); + this.pause(ctx, pause); + + // Not equal + this.doOtherwise(ctx, nodes, mismatch, { current, next }); } } module.exports = Sequence; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index ac01f71..90c92fc 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -1,7 +1,5 @@ 'use strict'; -const IR = require('llvm-ir'); - const node = require('./'); class Single extends node.Node { @@ -20,20 +18,25 @@ class Single extends node.Node { this.children.forEach(child => child.next.setOtherwise(otherwise, skip)); } - doBuild(ctx, fn, body, nodes) { - const pos = ctx.posArg(fn); + doBuild(ctx, body, nodes) { + body.comment('node.Single'); + + const pos = ctx.pos.current; // Load the character - const current = IR._('load', pos.type.to, [ pos.type, pos ]); + const current = ctx.ir._('load', pos.type.to, [ pos.type, pos ]); body.push(current); const keys = this.children.map(child => child.key); const s = ctx.buildSwitch(body, pos.type.to, current, keys); s.cases.forEach((body, i) => { - const child = this.children[i].next.build(ctx, nodes); + const child = this.children[i].next.build(ctx.compilation, nodes); + + this.tailTo(ctx, body, ctx.pos.next, child, this.children[i].value); }); + this.doOtherwise(ctx, nodes, s.otherwise); } } module.exports = Single; diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 2d6ff40..88be281 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -9,7 +9,8 @@ class SpanEnd extends node.Node { this.span = span; } - doBuild(ctx, fn, body, nodes) { + doBuild(ctx, body) { + body.terminate('unreachable'); } } module.exports = SpanEnd; diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index 23bf1ed..d0c462c 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -9,7 +9,8 @@ class SpanStart extends node.Node { this.span = span; } - doBuild(ctx, fn, body, nodes) { + doBuild(ctx, body) { + body.terminate('unreachable'); } } module.exports = SpanStart; diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 471fb99..eaeae0c 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -101,19 +101,22 @@ class NodeTranslator extends compiler.Stage { this.nodes.set(node, res); res.children = trie.children.map((child) => { - return { key: child.key, next: this.buildTrie(node, child.child) }; + const value = child.child.type === 'next' ? child.child.value : null; + return { key: child.key, next: this.buildTrie(node, child.child), value }; }); return res; } buildSequence(node, trie, isRoot) { const res = new compiler.node.Sequence(this.id(node), trie.select); + const value = trie.child.type === 'next' ? trie.child.value : null; // Break loops if (isRoot) this.nodes.set(node, res); res.next = this.buildTrie(node, trie.child); + res.value = value; return res; } From e7e4d2b22a7bc369dc2aa98712e24eb2ff91c5e6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 16:53:59 -0500 Subject: [PATCH 119/281] compiler: more progress --- lib/llparse.js | 2 +- lib/llparse/code/context.js | 2 +- lib/llparse/compiler/compiler.js | 35 -------------------- lib/llparse/compiler/context.js | 37 +++++++++++++++++++-- lib/llparse/compiler/node/base.js | 10 ++++-- lib/llparse/compiler/node/invoke.js | 8 ++--- lib/llparse/compiler/node/sequence.js | 9 ------ lib/llparse/compiler/node/single.js | 9 ------ lib/llparse/compiler/node/translator.js | 43 +++++++++++++++++++------ test/fixtures/main.c | 2 +- 10 files changed, 83 insertions(+), 74 deletions(-) diff --git a/lib/llparse.js b/lib/llparse.js index 76f8e73..89331de 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -34,7 +34,7 @@ class CodeAPI { } load(field) { - return new internal.code.Load(this[kPrefix] + '__method_' + field, field); + return new internal.code.Load(this[kPrefix] + '__load_' + field, field); } } diff --git a/lib/llparse/code/context.js b/lib/llparse/code/context.js index 18e7ade..99b08c6 100644 --- a/lib/llparse/code/context.js +++ b/lib/llparse/code/context.js @@ -72,7 +72,7 @@ class Context { const stateType = stateArg.type.to; const index = stateType.lookup(field); - const instr = IR._('getelementptr', stateType, + const instr = IR._('getelementptr inbounds', stateType, [ stateArg.type, stateArg ], [ INT, INT.v(0) ], [ INT, INT.v(index) ]); diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index bad8d84..1a63f06 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -49,8 +49,6 @@ class Compiler { // redirect blocks by `fn` and `target` this.redirectCache = new Map(); - - this.debugMethod = null; } build(root) { @@ -147,39 +145,6 @@ class Compiler { body.terminate('ret', [ TYPE_ERROR, error ]); } - debug(fn, body, string) { - if (!this.options.debug) - return body; - - const str = this.ir.cstr(string); - const cast = IR._('getelementptr', str.type.to, [ str.type, str ], - [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - body.push(cast); - - // Lazily declare debug method - if (this.debugMethod === null) { - const sig = IR.signature(IR.void(), - [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_INPUT ]); - - this.debugMethod = this.ir.declare(sig, this.options.debug); - } - - const args = [ - IR.void(), this.debugMethod, '(', - this.state.ptr(), fn.arg(ARG_STATE), ',', - TYPE_INPUT, fn.arg(ARG_POS), ',', - TYPE_INPUT, fn.arg(ARG_ENDPOS), ',', - TYPE_INPUT, cast, - ')' - ]; - - const call = IR._('call', args).void(); - body.push(call); - - return body; - } - - // Helpers checkSignatureType(node, value) { diff --git a/lib/llparse/compiler/context.js b/lib/llparse/compiler/context.js index bd54622..7b3f182 100644 --- a/lib/llparse/compiler/context.js +++ b/lib/llparse/compiler/context.js @@ -32,6 +32,8 @@ const ARG_MATCH = constants.ARG_MATCH; class Context { constructor(prefix, options, root) { + this.options = options || {}; + this.ir = new IR(); this.prefix = prefix; @@ -63,12 +65,13 @@ class Context { this.state.field(TYPE_INPUT, 'mark'); this.codeCache = new Map(); + this.debugMethod = null; // Public fields this.state.field(TYPE_DATA, 'data'); // Custom fields - options.properties.forEach((prop) => { + this.options.properties.forEach((prop) => { this.state.field(prop.type(this.ir, this.state), prop.name); }); @@ -133,6 +136,36 @@ class Context { return fn; } + debug(fn, body, string) { + if (!this.options.debug) + return body; + + const str = this.ir.cstr(string); + const cast = IR._('getelementptr inbounds', str.type.to, [ str.type, str ], + [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + body.push(cast); + + // Lazily declare debug method + if (this.debugMethod === null) { + const sig = IR.signature(IR.void(), + [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_INPUT ]); + + this.debugMethod = this.ir.declare(sig, this.options.debug); + } + + const args = [ + this.stateArg(fn), + this.posArg(fn), + this.endPosArg(fn), + cast + ]; + + const call = this.call('', this.debugMethod.type, this.debugMethod, args); + body.push(call); + + return body; + } + fn(signature, name) { name = this.prefix + '__' + name; @@ -212,7 +245,7 @@ class Context { field(fn, name) { const stateArg = this.stateArg(fn); - return IR._('getelementptr', this.state, + return IR._('getelementptr inbounds', this.state, [ stateArg.type, stateArg ], [ INT, INT.v(0) ], [ INT, INT.v(this.state.lookup(name)) ]); diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 785d34e..b6e4cd1 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -7,8 +7,9 @@ const llparse = require('../../'); const constants = llparse.constants; class NodeContext { - constructor(ctx, fn) { + constructor(ctx, name, fn) { this.compilation = ctx; + this.name = name; this.fn = fn; this.state = this.compilation.stateArg(this.fn); @@ -32,6 +33,10 @@ class NodeContext { this.TYPE_ERROR = constants.TYPE_ERROR; } + debug(body, message) { + this.compilation.debug(this.fn, body, `${this.name}: ${message}`); + } + // Just proxy field(name) { return this.compilation.field(this.fn, name); @@ -62,7 +67,7 @@ class Node { return nodes.get(this); const fn = compilation.fn(compilation.signature.node, this.name); - const ctx = new NodeContext(compilation, fn); + const ctx = new NodeContext(compilation, this.name, fn); // Errors are assumed to be rarely called // TODO(indutny): move this to node.Error somehow? @@ -74,6 +79,7 @@ class Node { nodes.set(this, fn); let body = fn.body; + ctx.debug(body, 'enter'); body = this.prologue(ctx, body); body.comment('next = pos + 1'); diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 1880e0c..467d1ba 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -8,11 +8,11 @@ const kType = llparse.symbols.kType; const node = require('./'); class Invoke extends node.Node { - constructor(name, code, map) { + constructor(name, code) { super('invoke', name); this.code = code; - this.map = map; + this.map = null; } prologue(ctx, body) { @@ -41,9 +41,9 @@ class Invoke extends node.Node { const s = ctx.buildSwitch(body, code.type.ret, call, keys); s.cases.forEach((body, i) => { - const child = info.node.map[keys[i]].build(ctx.compilation, nodes); + const child = this.map[keys[i]].build(ctx.compilation, nodes); - this.tailTo(ctx, body, ctx.pos.next, child, this.children[i].value); + this.tailTo(ctx, body, ctx.pos.next, child); }); this.doOtherwise(ctx, nodes, s.otherwise); diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 0c87048..fc8528b 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -18,15 +18,6 @@ class Sequence extends node.Node { this.value = null; } - setOtherwise(otherwise, skip) { - // Break loops - if (this.otherwise === otherwise) - return; - - super.setOtherwise(otherwise, skip); - this.next.setOtherwise(otherwise, skip); - } - doBuild(ctx, body, nodes) { const INT = ctx.INT; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 90c92fc..4140c03 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -9,15 +9,6 @@ class Single extends node.Node { this.children = null; } - setOtherwise(otherwise, skip) { - // Break loops - if (this.otherwise === otherwise) - return; - - super.setOtherwise(otherwise, skip); - this.children.forEach(child => child.next.setOtherwise(otherwise, skip)); - } - doBuild(ctx, body, nodes) { body.comment('node.Single'); diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index eaeae0c..0260262 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -47,9 +47,10 @@ class NodeTranslator extends compiler.Stage { return this.nodes.get(node); let res; + let list; if (node instanceof llparse.node.Invoke) { assert.strictEqual(node[kCases].length, 0); - res = new compiler.node.Invoke(this.id(node), node[kCode], node[kMap]); + res = new compiler.node.Invoke(this.id(node), node[kCode]); } else if (node instanceof llparse.node.Error) { assert.strictEqual(node[kCases].length, 0); res = new compiler.node.Error(this.id(node), node.code, node.reason); @@ -63,37 +64,52 @@ class NodeTranslator extends compiler.Stage { const trie = new llparse.Trie(node.name); const combined = trie.combine(node[kCases]); - res = this.buildTrie(node, trie.combine(node[kCases]), true); + list = []; + res = this.buildTrie(node, trie.combine(node[kCases]), list, true); } + // Prevent loops this.nodes.set(node, res); + // Build `invoke`'s map + if (node instanceof llparse.node.Invoke) { + res.map = {}; + Object.keys(node[kMap]).forEach((key) => { + res.map[key] = this.buildNode(node[kMap][key], null); + }); + } + if (node instanceof llparse.node.Error) { assert.strictEqual(node[kOtherwise], null); } else { assert.notStrictEqual(node[kOtherwise], null, `Node "${node.name}" must have \`.otherwise()\`/\`.skipTo()\``); + + if (!list) + list = [ res ]; const otherwise = this.buildNode(node[kOtherwise].next, null); - res.setOtherwise(otherwise, node[kOtherwise].skip); + list.forEach((entry) => { + entry.setOtherwise(otherwise, node[kOtherwise].skip); + }); } return res; } - buildTrie(node, trie, isRoot = false) { + buildTrie(node, trie, list, isRoot = false) { if (trie.type === 'next') { assert(!isRoot); return this.buildNode(trie.next, trie.value); } if (trie.type === 'single') - return this.buildSingle(node, trie, isRoot); + return this.buildSingle(node, trie, list, isRoot); assert.strictEqual(trie.type, 'sequence'); - return this.buildSequence(node, trie, isRoot); + return this.buildSequence(node, trie, list, isRoot); } - buildSingle(node, trie, isRoot) { + buildSingle(node, trie, list, isRoot) { const res = new compiler.node.Single(this.id(node)); // Break loops @@ -102,12 +118,18 @@ class NodeTranslator extends compiler.Stage { res.children = trie.children.map((child) => { const value = child.child.type === 'next' ? child.child.value : null; - return { key: child.key, next: this.buildTrie(node, child.child), value }; + return { + key: child.key, + next: this.buildTrie(node, child.child, list), + value + }; }); + + list.push(res); return res; } - buildSequence(node, trie, isRoot) { + buildSequence(node, trie, list, isRoot) { const res = new compiler.node.Sequence(this.id(node), trie.select); const value = trie.child.type === 'next' ? trie.child.value : null; @@ -115,9 +137,10 @@ class NodeTranslator extends compiler.Stage { if (isRoot) this.nodes.set(node, res); - res.next = this.buildTrie(node, trie.child); + res.next = this.buildTrie(node, trie.child, list); res.value = value; + list.push(res); return res; } diff --git a/test/fixtures/main.c b/test/fixtures/main.c index feda281..43c143a 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -24,7 +24,7 @@ void debug(struct state* s, const char* p, const char* endp, const char* msg) { if (bench) return; - fprintf(stderr, "off=%d msg=%s\n", (int) (p - start), msg); + fprintf(stderr, "off=%d > %s\n", (int) (p - start), msg); } int print_zero(struct state* s, const char* p, const char* endp) { From a53b96e059c02d9677b73376a2aee5d01c338951 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 17:02:04 -0500 Subject: [PATCH 120/281] compiler: last fixes --- lib/llparse/compiler/node/base.js | 2 +- lib/llparse/compiler/node/empty.js | 18 ++++++++++++++++++ lib/llparse/compiler/node/index.js | 1 + lib/llparse/compiler/node/invoke.js | 2 +- lib/llparse/compiler/node/translator.js | 10 +++++++--- 5 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 lib/llparse/compiler/node/empty.js diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index b6e4cd1..da329dd 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -83,7 +83,7 @@ class Node { body = this.prologue(ctx, body); body.comment('next = pos + 1'); - ctx.pos.next = ctx.ir._('getelementptr', ctx.pos.current.type.to, + ctx.pos.next = ctx.ir._('getelementptr inbounds', ctx.pos.current.type.to, [ ctx.pos.current.type, ctx.pos.current ], [ ctx.INT, ctx.INT.v(1) ]); body.push(ctx.pos.next); diff --git a/lib/llparse/compiler/node/empty.js b/lib/llparse/compiler/node/empty.js new file mode 100644 index 0000000..7625c74 --- /dev/null +++ b/lib/llparse/compiler/node/empty.js @@ -0,0 +1,18 @@ +'use strict'; + +const IR = require('llvm-ir'); + +const node = require('./'); + +class Empty extends node.Node { + constructor(name) { + super('empty', name); + } + + doBuild(ctx, body, nodes) { + body.comment('node.Empty'); + + this.doOtherwise(ctx, nodes, body); + } +} +module.exports = Empty; diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index 4272f49..6a87609 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -1,6 +1,7 @@ 'use strict'; exports.Node = require('./base'); +exports.Empty = require('./empty'); exports.Error = require('./error'); exports.Invoke = require('./invoke'); exports.Single = require('./single'); diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 467d1ba..061c88e 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -43,7 +43,7 @@ class Invoke extends node.Node { s.cases.forEach((body, i) => { const child = this.map[keys[i]].build(ctx.compilation, nodes); - this.tailTo(ctx, body, ctx.pos.next, child); + this.tailTo(ctx, body, ctx.pos.current, child); }); this.doOtherwise(ctx, nodes, s.otherwise); diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 0260262..1cd734a 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -62,10 +62,14 @@ class NodeTranslator extends compiler.Stage { res = new compiler.node.SpanEnd(this.id(node), node[kSpan]); } else { const trie = new llparse.Trie(node.name); - const combined = trie.combine(node[kCases]); - list = []; - res = this.buildTrie(node, trie.combine(node[kCases]), list, true); + const combined = trie.combine(node[kCases]); + if (combined === null) { + res = new compiler.node.Empty(this.id(node)); + } else { + list = []; + res = this.buildTrie(node, combined, list, true); + } } // Prevent loops From 3d59ce46ea60ab758ef5d0059b5d5aaf15c97f3f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 17:03:37 -0500 Subject: [PATCH 121/281] compiler: rename context => compilation --- lib/llparse/compiler/{context.js => compilation.js} | 4 ++-- lib/llparse/compiler/compiler.js | 3 ++- lib/llparse/compiler/index.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) rename lib/llparse/compiler/{context.js => compilation.js} (99%) diff --git a/lib/llparse/compiler/context.js b/lib/llparse/compiler/compilation.js similarity index 99% rename from lib/llparse/compiler/context.js rename to lib/llparse/compiler/compilation.js index 7b3f182..204d0ce 100644 --- a/lib/llparse/compiler/context.js +++ b/lib/llparse/compiler/compilation.js @@ -30,7 +30,7 @@ const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; -class Context { +class Compilation { constructor(prefix, options, root) { this.options = options || {}; @@ -256,4 +256,4 @@ class Context { endPosArg(fn) { return fn.arg(ARG_ENDPOS); } matchArg(fn) { return fn.arg(ARG_MATCH); } } -module.exports = Context; +module.exports = Compilation; diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 1a63f06..d8f4474 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -52,7 +52,8 @@ class Compiler { } build(root) { - const ctx = new llparse.compiler.Context(this.prefix, this.options, root); + const ctx = new llparse.compiler.Compilation(this.prefix, this.options, + root); // TODO(indutny): detect and report loops diff --git a/lib/llparse/compiler/index.js b/lib/llparse/compiler/index.js index 3dd65e8..c8e84a7 100644 --- a/lib/llparse/compiler/index.js +++ b/lib/llparse/compiler/index.js @@ -1,6 +1,6 @@ 'use strict'; -exports.Context = require('./context'); +exports.Compilation = require('./compilation'); exports.Stage = require('./stage'); exports.MatchSequence = require('./match-sequence'); From 9769bc3bd42bc74540dac8343ff84200603e3d1c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 17:10:54 -0500 Subject: [PATCH 122/281] span: iterate on API --- lib/llparse.js | 4 ++++ lib/llparse/code/index.js | 2 ++ lib/llparse/code/span.js | 7 +++++++ lib/llparse/node/span-end.js | 4 ++-- lib/llparse/node/span-start.js | 4 ++-- lib/llparse/span.js | 15 ++++++++++----- lib/llparse/symbols.js | 1 - test/span-test.js | 10 +++++----- 8 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 lib/llparse/code/span.js diff --git a/lib/llparse.js b/lib/llparse.js index 89331de..f3941d2 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -27,6 +27,10 @@ class CodeAPI { return new internal.code.Value(name, null); } + span(name) { + return new internal.code.Span(name, null); + } + // Helpers store(field) { diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index d585f9d..bb0b0f4 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -5,4 +5,6 @@ exports.Match = require('./match'); exports.Value = require('./value'); exports.Store = require('./store'); exports.Load = require('./load'); +exports.Span = require('./span'); + exports.Context = require('./context'); diff --git a/lib/llparse/code/span.js b/lib/llparse/code/span.js new file mode 100644 index 0000000..e9713f5 --- /dev/null +++ b/lib/llparse/code/span.js @@ -0,0 +1,7 @@ +'use strict'; + +const code = require('./'); + +class Span extends code.Match { +} +module.exports = Span; diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js index 3b7059c..ee19591 100644 --- a/lib/llparse/node/span-end.js +++ b/lib/llparse/node/span-end.js @@ -6,8 +6,8 @@ const node = require('./'); const kSpan = llparse.symbols.kSpan; class SpanEnd extends node.Node { - constructor(span, callback) { - super('span_end_' + callback); + constructor(span, code) { + super('span_end_' + code.name); this[kSpan] = span; } diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js index 78e3d22..296ad32 100644 --- a/lib/llparse/node/span-start.js +++ b/lib/llparse/node/span-start.js @@ -6,8 +6,8 @@ const node = require('./'); const kSpan = llparse.symbols.kSpan; class SpanStart extends node.Node { - constructor(span, callback) { - super('span_start_' + callback); + constructor(span, code) { + super('span_start_' + code.name); this[kSpan] = span; } diff --git a/lib/llparse/span.js b/lib/llparse/span.js index 8dd6e52..2b4ffc4 100644 --- a/lib/llparse/span.js +++ b/lib/llparse/span.js @@ -1,23 +1,28 @@ 'use strict'; +const assert = require('assert'); + const llparse = require('./'); -const kCallback = llparse.symbols.kCallback; +const kCode = llparse.symbols.kCode; class Span { - constructor(callback) { - this[kCallback] = callback; + constructor(code) { + assert(code instanceof llparse.code.Code, + 'Invalid `code` argument of `.span()`, must be a Code instance'); + + this[kCode] = code; } start(otherwise = null) { - const res = new llparse.node.SpanStart(this, this[kCallback]); + const res = new llparse.node.SpanStart(this, this[kCode]); if (otherwise) res.otherwise(otherwise); return res; } end(otherwise = null) { - const res = new llparse.node.SpanEnd(this, this[kCallback]); + const res = new llparse.node.SpanEnd(this, this[kCode]); if (otherwise) res.otherwise(otherwise); return res; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 07e0455..abe4add 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -1,7 +1,6 @@ 'use strict'; exports.kBody = Symbol('body'); -exports.kCallback = Symbol('callback'); exports.kCases = Symbol('cases'); exports.kCode = Symbol('code'); exports.kMap = Symbol('map'); diff --git a/test/span-test.js b/test/span-test.js index 45fda13..af2f398 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -20,9 +20,9 @@ describe('LLParse/span', () => { const underscore = p.node('underscore'); const span = { - dot: p.span('on_dot'), - dash: p.span('on_dash'), - underscore: p.span('on_underscore') + dot: p.span(p.code.span('on_dot')), + dash: p.span(p.code.span('on_dash')), + underscore: p.span(p.code.span('on_underscore')) }; start.otherwise(span.dot.start(dot)); @@ -48,7 +48,7 @@ describe('LLParse/span', () => { it('should throw on loops', () => { const start = p.node('start'); - const span = p.span('on_data'); + const span = p.span(p.code.span('on_data')); start.otherwise(span.start().otherwise(start)); @@ -57,7 +57,7 @@ describe('LLParse/span', () => { it('should throw on unmatched ends', () => { const start = p.node('start'); - const span = p.span('on_data'); + const span = p.span(p.code.span('on_data')); start.otherwise(span.end().otherwise(start)); From fb42566ad7a87970a354c85c6b3a4989bfe46597 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 17:19:40 -0500 Subject: [PATCH 123/281] compiler: integrate span allocator --- lib/llparse/compiler/compiler.js | 1 + lib/llparse/compiler/span/allocator.js | 27 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index d8f4474..472d81b 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -59,6 +59,7 @@ class Compiler { const stages = [ new llparse.compiler.MatchSequence(ctx), + new llparse.compiler.span.Allocator(ctx), new llparse.compiler.span.Builder(ctx), new llparse.compiler.node.Translator(ctx), new llparse.compiler.node.Builder(ctx) diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/span/allocator.js index 0aa713e..70fcb60 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -2,17 +2,26 @@ const assert = require('assert'); +const compiler = require('../'); const llparse = require('../../'); -const kCallback = llparse.symbols.kCallback; +const kCode = llparse.symbols.kCode; const kCases = llparse.symbols.kCases; const kOtherwise = llparse.symbols.kOtherwise; const kNoAdvance = llparse.symbols.kNoAdvance; const kSpan = llparse.symbols.kSpan; -class Allocator { - execute(root) { - const nodes = this.getNodes(root); +class Allocator extends compiler.Stage { + constructor(ctx) { + super(ctx, 'span-allocator'); + } + + id(node) { + return node[kSpan][kCode].name; + } + + build() { + const nodes = this.getNodes(this.ctx.root); const info = this.computeActive(nodes); const overlap = this.computeOverlap(info); const color = this.color(info.spans, overlap); @@ -47,7 +56,7 @@ class Allocator { const active = activeMap.get(node); if (node instanceof llparse.node.SpanStart) { - const span = node[kSpan][kCallback]; + const span = this.id(node); spans.add(span); active.add(span); } @@ -55,14 +64,14 @@ class Allocator { active.forEach((span) => { // Don't propagate span past the spanEnd if (node instanceof llparse.node.SpanEnd && - span === node[kSpan][kCallback]) { + span === this.id(node)) { return; } this.getChildren(node).forEach((child) => { // Disallow loops if (child instanceof llparse.node.SpanStart) { - assert.notStrictEqual(child[kSpan][kCallback], span, + assert.notStrictEqual(this.id(child), span, `Detected loop in span "${span}"`); } @@ -79,8 +88,8 @@ class Allocator { const ends = nodes.filter(node => node instanceof llparse.node.SpanEnd); ends.forEach((end) => { const active = activeMap.get(end); - assert(active.has(end[kSpan][kCallback]), - `Unmatched span end for "${end[kSpan][kCallback]}"`); + assert(active.has(this.id(end)), + `Unmatched span end for "${this.id(end)}"`); }); return { active: activeMap, spans: Array.from(spans) }; From c6c4286a66bab11b01413f99d34e59f383a085db Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 17:36:22 -0500 Subject: [PATCH 124/281] compilation: one way to initialize all the fields --- lib/llparse/compiler/compilation.js | 42 +++++++++++++++++++++------- lib/llparse/compiler/compiler.js | 24 +--------------- lib/llparse/compiler/span/builder.js | 18 ++++++------ 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 204d0ce..896b664 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -40,6 +40,7 @@ class Compilation { this.root = root; this.state = this.ir.struct(`${this.prefix}_state`); + this.initializers = []; this.signature = { node: this.ir.signature(TYPE_OUTPUT, [ @@ -54,27 +55,32 @@ class Compilation { ]), value: this.ir.signature(INT, [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_MATCH - ]) + ]), + span: null } }; + this.signature.callback.span = this.signature.callback.match; - this.state.field(this.signature.node.ptr(), 'current'); - this.state.field(TYPE_ERROR, 'error'); - this.state.field(TYPE_REASON, 'reason'); - this.state.field(TYPE_INDEX, 'index'); - this.state.field(TYPE_INPUT, 'mark'); + this.declareField(this.signature.node.ptr(), 'current', + (type, ctx) => ctx.stageResults['node-builder'].ref()); - this.codeCache = new Map(); - this.debugMethod = null; + this.declareField(TYPE_ERROR, 'error', type => type.v(0)); + this.declareField(TYPE_REASON, 'reason', type => type.v(null)); + this.declareField(TYPE_INDEX, 'index', type => type.v(0)); // Public fields - this.state.field(TYPE_DATA, 'data'); + this.declareField(TYPE_DATA, 'data', type => type.v(null)); // Custom fields this.options.properties.forEach((prop) => { - this.state.field(prop.type(this.ir, this.state), prop.name); + // TODO(indutny): support non-int types? + this.declareField(prop.type(this.ir, this.state), prop.name, + type => type.v(0)); }); + this.codeCache = new Map(); + this.debugMethod = null; + // Intermediate results from various build stages this.stageResults = {}; } @@ -242,6 +248,22 @@ class Compilation { }; } + declareField(type, name, init) { + this.state.field(type, name); + + this.initializers.push({ type, name, init }); + } + + initFields(fn, body) { + this.initializers.forEach((entry) => { + const field = this.field(fn, entry.name); + body.push(field); + + body.push(this.ir._('store', [ entry.type, entry.init(entry.type, this) ], + [ entry.type.ptr(), field ]).void()); + }); + } + field(fn, name) { const stateArg = this.stateArg(fn); diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 472d81b..3d8996c 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -74,32 +74,10 @@ class Compiler { } buildInit(ctx) { - const fn = ctx.stageResults['node-builder'].ref(); const sig = ctx.ir.signature(ctx.ir.void(), [ ctx.state.ptr() ]); const init = ctx.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); - const fields = { - current: ctx.field(init, 'current'), - error: ctx.field(init, 'error'), - reason: ctx.field(init, 'reason'), - index: ctx.field(init, 'index'), - mark: ctx.field(init, 'mark'), - data: ctx.field(init, 'data') - }; - - Object.keys(fields).forEach(key => init.body.push(fields[key])); - - const store = (field, type, value) => { - init.body.push(ctx.ir._('store', [ type, value ], - [ type.ptr(), field ]).void()); - }; - - store(fields.current, fn.type, fn); - store(fields.error, TYPE_ERROR, TYPE_ERROR.v(0)); - store(fields.reason, TYPE_REASON, TYPE_REASON.v(null)); - store(fields.index, TYPE_INDEX, TYPE_INDEX.v(0)); - store(fields.mark, TYPE_INPUT, TYPE_INPUT.v(null)); - store(fields.data, TYPE_DATA, TYPE_DATA.v(null)); + ctx.initFields(init, init.body); init.body.terminate('ret', ctx.ir.void()); diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index fef1198..d055bbb 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -5,10 +5,11 @@ const compiler = require('../'); const constants = llparse.constants; +const TYPE_INPUT = constants.TYPE_INPUT; + const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; - class Builder extends compiler.Stage { constructor(ctx) { super(ctx, 'span-builder'); @@ -16,24 +17,25 @@ class Builder extends compiler.Stage { build() { // TODO(indutny): implement me + this.buildFields(); } - buildSpanFields() { - const allocator = new llparse.span.Allocator(); + buildFields() { + const colors = this.ctx.stageResults['span-allocator']; - const colors = allocator.execute(this.ctx.root); + const callbackType = this.ctx.signature.callback.span.ptr(); colors.concurrency.forEach((num, index) => { - this.state.field(TYPE_INPUT, SPAN_START_PREFIX + index); + this.ctx.declareField(TYPE_INPUT, SPAN_START_PREFIX + index, + type => type.v(null)); // TODO(indutny): use smaller field size if possible? if (num === 1) return; - this.state.field(this.signature.callback.match.ptr(), - SPAN_CB_PREFIX + index); + this.ctx.declareField(callbackType, SPAN_CB_PREFIX + index, + type => type.v(null)); }); - console.log(colors); } } module.exports = Builder; From 236f50928bd771845d8fe86ef2003431d62011cc Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 17:43:46 -0500 Subject: [PATCH 125/281] translator: remove extraneous prefix --- lib/llparse/compiler/node/translator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 1cd734a..cf4f7f2 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -37,7 +37,7 @@ class NodeTranslator extends compiler.Stage { } this.namespace.add(res); - return this.ctx.prefix + '__n_' + res; + return 'n_' + res; } buildNode(node, value) { From 6514bf6bfcae11039356a85ebb24d2cc06aebf6d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:09:18 -0500 Subject: [PATCH 126/281] compiler: lint, remove unused code --- lib/llparse.js | 2 -- lib/llparse/compiler/compilation.js | 1 - lib/llparse/compiler/compiler.js | 43 +++----------------------- lib/llparse/compiler/match-sequence.js | 1 + lib/llparse/compiler/node/base.js | 2 +- lib/llparse/compiler/node/builder.js | 10 ------ lib/llparse/compiler/node/empty.js | 2 -- lib/llparse/compiler/node/error.js | 8 ++--- lib/llparse/compiler/span/allocator.js | 11 ++++--- lib/llparse/compiler/span/builder.js | 34 ++++++++++++++++++-- lib/llparse/node/error.js | 1 - lib/llparse/node/invoke.js | 1 - 12 files changed, 47 insertions(+), 69 deletions(-) diff --git a/lib/llparse.js b/lib/llparse.js index f3941d2..6f29082 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -4,9 +4,7 @@ const assert = require('assert'); const internal = require('./llparse/'); -const kCallback = Symbol('callback'); const kCode = Symbol('code'); -const kName = Symbol('name'); const kPrefix = Symbol('prefix'); const kProperties = Symbol('properties'); diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 896b664..4748ee4 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -11,7 +11,6 @@ const kBody = llparse.symbols.kBody; const CCONV = constants.CCONV; -const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 3d8996c..e68bb68 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -1,41 +1,16 @@ 'use strict'; -const assert = require('assert'); - const llparse = require('../'); const constants = llparse.constants; -const kBody = llparse.symbols.kBody; -const kCases = llparse.symbols.kCases; -const kOtherwise = llparse.symbols.kOtherwise; -const kSignature = llparse.symbols.kSignature; -const kType = llparse.symbols.kType; - -const CCONV = constants.CCONV; - -const BOOL = constants.BOOL; -const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_MATCH = constants.TYPE_MATCH; -const TYPE_INDEX = constants.TYPE_INDEX; const TYPE_ERROR = constants.TYPE_ERROR; -const TYPE_REASON = constants.TYPE_REASON; -const TYPE_DATA = constants.TYPE_DATA; - -const ATTR_STATE = constants.ATTR_STATE; -const ATTR_POS = constants.ATTR_POS; -const ATTR_ENDPOS = constants.ATTR_ENDPOS; const ARG_STATE = constants.ARG_STATE; const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; -const ARG_MATCH = constants.ARG_MATCH; -const ARG_UNUSED = constants.ARG_UNUSED; - -const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; -const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; -const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; class Compiler { constructor(options) { @@ -101,13 +76,11 @@ class Compiler { [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); - const call = ctx.ir._(`call ${CCONV}`, [ - TYPE_OUTPUT, current, '(', - ctx.state.ptr(), parse.arg(ARG_STATE), ',', - TYPE_INPUT, parse.arg(ARG_POS), ',', - TYPE_INPUT, parse.arg(ARG_ENDPOS), ',', - TYPE_MATCH, TYPE_MATCH.v(0), - ')' + const call = ctx.call('', nodeSig, current, [ + ctx.stateArg(parse), + ctx.posArg(parse), + ctx.endPosArg(parse), + TYPE_MATCH.v(0) ]); body.push(call); @@ -124,11 +97,5 @@ class Compiler { body.terminate('ret', [ TYPE_ERROR, error ]); } - - // Helpers - - checkSignatureType(node, value) { - assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); - } } module.exports = Compiler; diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index 61859a6..a51d7f8 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -61,6 +61,7 @@ class MatchSequence extends compiler.Stage { return fn; } + // TODO(indutny): initialize `state.index` before calling matcher? buildBody(fn) { const body = fn.body; diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index da329dd..c3a735a 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -1,6 +1,6 @@ 'use strict'; -const assert = require('assert') +const assert = require('assert'); const node = require('./'); const llparse = require('../../'); diff --git a/lib/llparse/compiler/node/builder.js b/lib/llparse/compiler/node/builder.js index 33922aa..3b4f41c 100644 --- a/lib/llparse/compiler/node/builder.js +++ b/lib/llparse/compiler/node/builder.js @@ -1,16 +1,6 @@ 'use strict'; -const assert = require('assert'); - const compiler = require('../'); -const llparse = require('../../'); - -const kCases = llparse.symbols.kCases; -const kCode = llparse.symbols.kCode; -const kMap = llparse.symbols.kMap; -const kOtherwise = llparse.symbols.kOtherwise; -const kSignature = llparse.symbols.kSignature; -const kSpan = llparse.symbols.kSpan; class NodeBuilder extends compiler.Stage { constructor(ctx) { diff --git a/lib/llparse/compiler/node/empty.js b/lib/llparse/compiler/node/empty.js index 7625c74..7b7df8b 100644 --- a/lib/llparse/compiler/node/empty.js +++ b/lib/llparse/compiler/node/empty.js @@ -1,7 +1,5 @@ 'use strict'; -const IR = require('llvm-ir'); - const node = require('./'); class Empty extends node.Node { diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index 11833f9..a6de55f 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -1,7 +1,5 @@ 'use strict'; -const IR = require('llvm-ir'); - const node = require('./'); class Error extends node.Node { @@ -29,13 +27,13 @@ class Error extends node.Node { const reasonPtr = ctx.field('reason'); body.push(reasonPtr); - const cast = IR._('getelementptr inbounds', reason.type.to, + const cast = ctx.ir._('getelementptr inbounds', reason.type.to, [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); body.push(cast); - body.push(IR._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(this.code) ], + body.push(ctx.ir._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(this.code) ], [ ctx.TYPE_ERROR.ptr(), codePtr ]).void()); - body.push(IR._('store', [ ctx.TYPE_REASON, cast ], + body.push(ctx.ir._('store', [ ctx.TYPE_REASON, cast ], [ ctx.TYPE_REASON.ptr(), reasonPtr ]).void()); const retType = ctx.fn.type.ret; diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/span/allocator.js index 70fcb60..0c53be5 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -8,7 +8,6 @@ const llparse = require('../../'); const kCode = llparse.symbols.kCode; const kCases = llparse.symbols.kCases; const kOtherwise = llparse.symbols.kOtherwise; -const kNoAdvance = llparse.symbols.kNoAdvance; const kSpan = llparse.symbols.kSpan; class Allocator extends compiler.Stage { @@ -17,7 +16,7 @@ class Allocator extends compiler.Stage { } id(node) { - return node[kSpan][kCode].name; + return node[kSpan][kCode]; } build() { @@ -29,7 +28,7 @@ class Allocator extends compiler.Stage { return color; } - getNodes(root, set) { + getNodes(root) { const res = new Set(); const queue = [ root ]; while (queue.length !== 0) { @@ -150,9 +149,11 @@ class Allocator extends compiler.Stage { spans.forEach(span => res.set(span, allocate(span))); - const concurrency = new Array(max + 1).fill(0); + const concurrency = new Array(max + 1); + for (let i = 0; i < concurrency.length; i++) + concurrency[i] = []; - spans.forEach(span => concurrency[allocate(span)]++); + spans.forEach(span => concurrency[allocate(span)].push(span)); return { map: res, concurrency, max }; } diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index d055bbb..0bd18ce 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -16,8 +16,11 @@ class Builder extends compiler.Stage { } build() { - // TODO(indutny): implement me this.buildFields(); + + return { + epilogue: (body) => this.buildEpilogue(body) + }; } buildFields() { @@ -25,11 +28,11 @@ class Builder extends compiler.Stage { const callbackType = this.ctx.signature.callback.span.ptr(); - colors.concurrency.forEach((num, index) => { + colors.concurrency.forEach((list, index) => { + const num = list.length; this.ctx.declareField(TYPE_INPUT, SPAN_START_PREFIX + index, type => type.v(null)); - // TODO(indutny): use smaller field size if possible? if (num === 1) return; @@ -37,5 +40,30 @@ class Builder extends compiler.Stage { type => type.v(null)); }); } + + buildEpilogue(body) { + const colors = this.ctx.stageResults['span-allocator']; + + const callbackType = this.ctx.signature.callback.span.ptr(); + + colors.concurrency.forEach((list, index) => { + const num = list.length; + + const startPtr = this.ctx.field(SPAN_START_PREFIX + index); + + // TODO(indutny): branch and call + + let cb; + if (num === 1) { + cb = this.buildCode(list[0]); + } else { + const cbPtr = this.ctx.field(SPAN_CB_PREFIX + index); + body.push(cbPtr); + + cb = this.ctx.ir._('load', callbackType.to, [ callbackType, cbPtr ]); + body.push(cb); + } + }); + } } module.exports = Builder; diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index c5976e2..6d76174 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -1,7 +1,6 @@ 'use strict'; const node = require('./'); -const llparse = require('../'); const kCode = Symbol('code'); const kReason = Symbol('reason'); diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index a351f5e..b93ab3e 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -5,7 +5,6 @@ const assert = require('assert'); const node = require('./'); const llparse = require('../'); -const kNoAdvance = llparse.symbols.kNoAdvance; const kType = llparse.symbols.kType; const kCode = llparse.symbols.kCode; const kMap = llparse.symbols.kMap; From 9a6b90b68aa21688d2e0a75df977836fad5011a9 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:09:42 -0500 Subject: [PATCH 127/281] compiler: `buildParse` => `buildExecute` --- lib/llparse/compiler/compiler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index e68bb68..8f63a7e 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -43,7 +43,7 @@ class Compiler { ctx.buildStages(stages); this.buildInit(ctx); - this.buildParse(ctx); + this.buildExecute(ctx); return ctx.ir.build(); } @@ -59,7 +59,7 @@ class Compiler { return init; } - buildParse(ctx) { + buildExecute(ctx) { // TODO(indutny): change signature to (state*, start*, len)? const sig = ctx.ir.signature(TYPE_ERROR, [ ctx.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); From bbf2de72185551a3c39965aa73fb5b89e2dacb11 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:21:19 -0500 Subject: [PATCH 128/281] compiler: branch on `execute` result --- lib/llparse/compiler/compiler.js | 35 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 8f63a7e..1d995b1 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -3,6 +3,8 @@ const llparse = require('../'); const constants = llparse.constants; +const BOOL = constants.BOOL; + const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_MATCH = constants.TYPE_MATCH; @@ -84,18 +86,37 @@ class Compiler { ]); body.push(call); - const errorPtr = ctx.field(parse, 'error'); - body.push(errorPtr); - const error = ctx.ir._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); - body.push(error); + const cmp = ctx.ir._('icmp', [ 'ne', nodeSig.ret, call ], + nodeSig.ret.v(null)); + body.push(cmp); + + const branch = body.branch('br', [ BOOL, cmp ]); + const success = branch.left; + const error = branch.right; + + // Success + success.name = 'success'; const bitcast = ctx.ir._('bitcast', [ TYPE_OUTPUT, call, 'to', nodeSig.ptr() ]); - body.push(bitcast); - body.push(ctx.ir._('store', [ nodeSig.ptr(), bitcast ], + success.push(bitcast); + success.push(ctx.ir._('store', [ nodeSig.ptr(), bitcast ], + [ nodeSig.ptr().ptr(), currentPtr ]).void()); + + success.terminate('ret', [ TYPE_ERROR, TYPE_ERROR.v(0) ]); + + // Error + error.name = 'error'; + + const errorPtr = ctx.field(parse, 'error'); + error.push(errorPtr); + const code = ctx.ir._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); + error.push(code); + + error.push(ctx.ir._('store', [ nodeSig.ptr(), nodeSig.ptr().v(null) ], [ nodeSig.ptr().ptr(), currentPtr ]).void()); - body.terminate('ret', [ TYPE_ERROR, error ]); + error.terminate('ret', [ TYPE_ERROR, code ]); } } module.exports = Compiler; From ff71afab569396025fd1aa73915f85bfdad92925 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:29:27 -0500 Subject: [PATCH 129/281] span: hook up epilogue into `execute` --- lib/llparse/compiler/compilation.js | 7 +++++-- lib/llparse/compiler/compiler.js | 20 +++++++++++--------- lib/llparse/compiler/span/builder.js | 16 ++++++++++------ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 4748ee4..a8efd22 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -198,7 +198,11 @@ class Compilation { return fn; } - call(type, signature, ref, args) { + call(type, signature, ref, cconv, args) { + if (Array.isArray(cconv)) { + args = cconv; + cconv = ref.cconv || ''; + } assert.strictEqual(args.length, signature.args.length); const irArgs = [ signature.ret, ref, '(' ]; @@ -213,7 +217,6 @@ class Compilation { if (type) type += ' '; - let cconv = ref.cconv || ''; if (cconv) cconv = ' ' + cconv; diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 1d995b1..67f5357 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -65,23 +65,23 @@ class Compiler { // TODO(indutny): change signature to (state*, start*, len)? const sig = ctx.ir.signature(TYPE_ERROR, [ ctx.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); - const parse = ctx.ir.fn(sig, this.prefix + '_execute', + const execute = ctx.ir.fn(sig, this.prefix + '_execute', [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - const body = parse.body; + const body = execute.body; const nodeSig = ctx.signature.node; - const currentPtr = ctx.field(parse, 'current'); + const currentPtr = ctx.field(execute, 'current'); body.push(currentPtr); const current = ctx.ir._('load', nodeSig.ptr(), [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); - const call = ctx.call('', nodeSig, current, [ - ctx.stateArg(parse), - ctx.posArg(parse), - ctx.endPosArg(parse), + const call = ctx.call('', nodeSig, current, constants.CCONV, [ + ctx.stateArg(execute), + ctx.posArg(execute), + ctx.endPosArg(execute), TYPE_MATCH.v(0) ]); body.push(call); @@ -103,12 +103,14 @@ class Compiler { success.push(ctx.ir._('store', [ nodeSig.ptr(), bitcast ], [ nodeSig.ptr().ptr(), currentPtr ]).void()); - success.terminate('ret', [ TYPE_ERROR, TYPE_ERROR.v(0) ]); + // Invoke spans and exit + ctx.stageResults['span-builder'].postExecute(execute, success) + .terminate('ret', [ TYPE_ERROR, TYPE_ERROR.v(0) ]); // Error error.name = 'error'; - const errorPtr = ctx.field(parse, 'error'); + const errorPtr = ctx.field(execute, 'error'); error.push(errorPtr); const code = ctx.ir._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); error.push(code); diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index 0bd18ce..3d93777 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -19,7 +19,7 @@ class Builder extends compiler.Stage { this.buildFields(); return { - epilogue: (body) => this.buildEpilogue(body) + postExecute: (fn, body) => this.buildEpilogue(fn, body) }; } @@ -41,29 +41,33 @@ class Builder extends compiler.Stage { }); } - buildEpilogue(body) { + buildEpilogue(fn, body) { const colors = this.ctx.stageResults['span-allocator']; const callbackType = this.ctx.signature.callback.span.ptr(); + body.comment('execute spans'); + colors.concurrency.forEach((list, index) => { const num = list.length; - const startPtr = this.ctx.field(SPAN_START_PREFIX + index); + const startPtr = this.ctx.field(fn, SPAN_START_PREFIX + index); // TODO(indutny): branch and call let cb; if (num === 1) { - cb = this.buildCode(list[0]); + cb = this.ctx.buildCode(list[0]); } else { - const cbPtr = this.ctx.field(SPAN_CB_PREFIX + index); + const cbPtr = this.ctx.field(fn, SPAN_CB_PREFIX + index); body.push(cbPtr); - cb = this.ctx.ir._('load', callbackType.to, [ callbackType, cbPtr ]); + cb = this.ctx.ir._('load', callbackType, [ callbackType.ptr(), cbPtr ]); body.push(cb); } }); + + return body; } } module.exports = Builder; From 073f017704c4c0361a401231508636ec5ec87896 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:32:04 -0500 Subject: [PATCH 130/281] span/allocator: fix spurious concurrency --- lib/llparse/compiler/span/allocator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/span/allocator.js index 0c53be5..e19bbac 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -117,7 +117,7 @@ class Allocator extends compiler.Stage { } color(spans, overlapMap) { - let max = 0; + let max = -1; const colors = new Map(); const allocate = (span) => { From 7f936fe4c272d1c09b218a6de24786ea00783aeb Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:47:09 -0500 Subject: [PATCH 131/281] span/builder: stub out epilogue --- lib/llparse/compiler/compiler.js | 5 +++- lib/llparse/compiler/span/builder.js | 36 ++++++++++++++++++++++++++-- test/fixtures/main.c | 22 +++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 67f5357..0707e0d 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -68,7 +68,10 @@ class Compiler { const execute = ctx.ir.fn(sig, this.prefix + '_execute', [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); - const body = execute.body; + let body = execute.body; + + // Set unfinished spans + body = ctx.stageResults['span-builder'].preExecute(execute, body); const nodeSig = ctx.signature.node; diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index 3d93777..187b5cd 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -5,6 +5,7 @@ const compiler = require('../'); const constants = llparse.constants; +const BOOL = constants.BOOL; const TYPE_INPUT = constants.TYPE_INPUT; const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; @@ -19,6 +20,7 @@ class Builder extends compiler.Stage { this.buildFields(); return { + preExecute: (fn, body) => this.buildPrologue(fn, body), postExecute: (fn, body) => this.buildEpilogue(fn, body) }; } @@ -41,6 +43,10 @@ class Builder extends compiler.Stage { }); } + buildPrologue(fn, body) { + return body; + } + buildEpilogue(fn, body) { const colors = this.ctx.stageResults['span-allocator']; @@ -52,6 +58,20 @@ class Builder extends compiler.Stage { const num = list.length; const startPtr = this.ctx.field(fn, SPAN_START_PREFIX + index); + body.push(startPtr); + const start = this.ctx.ir._('load', TYPE_INPUT, + [ TYPE_INPUT.ptr(), startPtr ]); + body.push(start); + + const cmp = this.ctx.ir._('icmp', [ 'ne', TYPE_INPUT, start ], + TYPE_INPUT.v(null)); + body.push(cmp); + + const branch = body.branch('br', [ BOOL, cmp ]); + const present = branch.left; + const empty = branch.right; + present.name = 'present_' + index; + empty.name = 'empty_' + index; // TODO(indutny): branch and call @@ -60,11 +80,23 @@ class Builder extends compiler.Stage { cb = this.ctx.buildCode(list[0]); } else { const cbPtr = this.ctx.field(fn, SPAN_CB_PREFIX + index); - body.push(cbPtr); + present.push(cbPtr); cb = this.ctx.ir._('load', callbackType, [ callbackType.ptr(), cbPtr ]); - body.push(cb); + present.push(cb); } + + // TODO(indutny): this won't work with internal callbacks due to + // cconv mismatch + const call = this.ctx.call('', callbackType.to, cb, [ + this.ctx.stateArg(fn), + start, + this.ctx.endPosArg(fn) + ]); + present.push(call); + + present.terminate('br', empty); + body = empty; }); return body; diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 43c143a..f2b7bf9 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -67,6 +67,28 @@ int return_match(struct state* s, const char* p, const char* endp, int value) { return value; } +int on_dot(struct state* s, const char* p, const char* endp, int value) { + if (bench) + return value; + + fprintf(stdout, "off=%d dot=%.*s\n", (int) (p - start), (int) (endp - p), p); +} + +int on_dash(struct state* s, const char* p, const char* endp, int value) { + if (bench) + return value; + + fprintf(stdout, "off=%d dash=%.*s\n", (int) (p - start), (int) (endp - p), p); +} + +int on_underscore(struct state* s, const char* p, const char* endp, int value) { + if (bench) + return value; + + fprintf(stdout, "off=%d underscore=%.*s\n", (int) (p - start), + (int) (endp - p), p); +} + static int run_bench(const char* input, int len) { struct state s; int64_t i; From bccfcdbb45b83f6b97b44bf1a07130bf4ba463a5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 18:53:26 -0500 Subject: [PATCH 132/281] span/builder: restart spans --- lib/llparse/compiler/compiler.js | 3 +++ lib/llparse/compiler/span/builder.js | 34 ++++++++++++++++++++++++++++ test/fixtures/main.c | 15 +++++++----- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 0707e0d..7681bd7 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -72,6 +72,9 @@ class Compiler { // Set unfinished spans body = ctx.stageResults['span-builder'].preExecute(execute, body); + body.name = 'execute'; + + body.comment('execute'); const nodeSig = ctx.signature.node; diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index 187b5cd..7f49279 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -44,6 +44,38 @@ class Builder extends compiler.Stage { } buildPrologue(fn, body) { + const colors = this.ctx.stageResults['span-allocator']; + + const callbackType = this.ctx.signature.callback.span.ptr(); + + body.comment('restart spans'); + + colors.concurrency.forEach((_, index) => { + const startPtr = this.ctx.field(fn, SPAN_START_PREFIX + index); + body.push(startPtr); + const start = this.ctx.ir._('load', TYPE_INPUT, + [ TYPE_INPUT.ptr(), startPtr ]); + body.push(start); + + const cmp = this.ctx.ir._('icmp', [ 'eq', TYPE_INPUT, start ], + [ TYPE_INPUT.v(null) ]); + body.push(cmp); + + const branch = body.branch('br', [ BOOL, cmp ]); + const empty = branch.left; + const restart = branch.right; + empty.name = 'empty_' + index; + restart.name = 'restart_' + index; + + const store = this.ctx.ir._('store', [ TYPE_INPUT, this.ctx.posArg(fn) ], + [ TYPE_INPUT.ptr(), startPtr ]); + store.void(); + restart.push(store); + + restart.terminate('br', empty); + body = empty; + }); + return body; } @@ -95,6 +127,8 @@ class Builder extends compiler.Stage { ]); present.push(call); + // TODO(indutny): check return value + present.terminate('br', empty); body = empty; }); diff --git a/test/fixtures/main.c b/test/fixtures/main.c index f2b7bf9..ce945c9 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -67,26 +67,29 @@ int return_match(struct state* s, const char* p, const char* endp, int value) { return value; } -int on_dot(struct state* s, const char* p, const char* endp, int value) { +int on_dot(struct state* s, const char* p, const char* endp) { if (bench) - return value; + return 0; fprintf(stdout, "off=%d dot=%.*s\n", (int) (p - start), (int) (endp - p), p); + return 0; } -int on_dash(struct state* s, const char* p, const char* endp, int value) { +int on_dash(struct state* s, const char* p, const char* endp) { if (bench) - return value; + return 0; fprintf(stdout, "off=%d dash=%.*s\n", (int) (p - start), (int) (endp - p), p); + return 0; } -int on_underscore(struct state* s, const char* p, const char* endp, int value) { +int on_underscore(struct state* s, const char* p, const char* endp) { if (bench) - return value; + return 0; fprintf(stdout, "off=%d underscore=%.*s\n", (int) (p - start), (int) (endp - p), p); + return 0; } static int run_bench(const char* input, int len) { From e70af72fbcafbe7670b93fdad6bd89f4495e8213 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 19:13:18 -0500 Subject: [PATCH 133/281] span/builder: check return value --- lib/llparse/compiler/span/builder.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index 7f49279..68dee30 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -1,5 +1,7 @@ 'use strict'; +const assert = require('assert'); + const llparse = require('../../'); const compiler = require('../'); @@ -127,9 +129,22 @@ class Builder extends compiler.Stage { ]); present.push(call); - // TODO(indutny): check return value + // Check return value + const errorCmp = this.ctx.ir._('icmp', + [ 'eq', callbackType.to.ret, call ], [ callbackType.to.ret.v(0) ]); + present.push(errorCmp); + + const errorBranch = present.branch('br', [ BOOL, errorCmp ]); + const noError = errorBranch.left; + const error = errorBranch.right; + noError.name = 'no_error_' + index; + error.name = 'error_' + index; + + // Make sure that the types match + assert.strictEqual(fn.type.ret.type, callbackType.to.ret.type); + error.terminate('ret', [ fn.type.ret, call ]); - present.terminate('br', empty); + noError.terminate('br', empty); body = empty; }); From df873fc4a199be08d32e77c84a09c1b415e596c6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 19:43:26 -0500 Subject: [PATCH 134/281] span/builder: span/start end implementation --- lib/llparse/compiler/node/span-end.js | 11 ++- lib/llparse/compiler/node/span-start.js | 11 ++- lib/llparse/compiler/node/translator.js | 4 +- lib/llparse/compiler/span/builder.js | 106 +++++++++++++++++++++++- test/fixtures/index.js | 2 +- 5 files changed, 120 insertions(+), 14 deletions(-) diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 88be281..99653a5 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -3,14 +3,17 @@ const node = require('./'); class SpanEnd extends node.Node { - constructor(name, span) { + constructor(name, code) { super('span-end', name); - this.span = span; + this.code = code; } - doBuild(ctx, body) { - body.terminate('unreachable'); + doBuild(ctx, body, nodes) { + body.comment(`node.SpanEnd[${this.code.name}]`); + body = ctx.compilation.stageResults['span-builder'].spanEnd( + ctx.fn, body, this.code); + this.doOtherwise(ctx, nodes, body); } } module.exports = SpanEnd; diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index d0c462c..61ba54e 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -3,14 +3,17 @@ const node = require('./'); class SpanStart extends node.Node { - constructor(name, span) { + constructor(name, code) { super('span-start', name); - this.span = span; + this.code = code; } - doBuild(ctx, body) { - body.terminate('unreachable'); + doBuild(ctx, body, nodes) { + body.comment(`node.SpanStart[${this.code.name}]`); + body = ctx.compilation.stageResults['span-builder'].spanStart( + ctx.fn, body, this.code); + this.doOtherwise(ctx, nodes, body); } } module.exports = SpanStart; diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index cf4f7f2..bf0d736 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -56,10 +56,10 @@ class NodeTranslator extends compiler.Stage { res = new compiler.node.Error(this.id(node), node.code, node.reason); } else if (node instanceof llparse.node.SpanStart) { assert.strictEqual(node[kCases].length, 0); - res = new compiler.node.SpanStart(this.id(node), node[kSpan]); + res = new compiler.node.SpanStart(this.id(node), node[kSpan][kCode]); } else if (node instanceof llparse.node.SpanEnd) { assert.strictEqual(node[kCases].length, 0); - res = new compiler.node.SpanEnd(this.id(node), node[kSpan]); + res = new compiler.node.SpanEnd(this.id(node), node[kSpan][kCode]); } else { const trie = new llparse.Trie(node.name); diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index 68dee30..691e869 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -9,6 +9,8 @@ const constants = llparse.constants; const BOOL = constants.BOOL; const TYPE_INPUT = constants.TYPE_INPUT; +const TYPE_OUTPUT = constants.TYPE_OUTPUT; +const TYPE_ERROR = constants.TYPE_ERROR; const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; @@ -22,6 +24,9 @@ class Builder extends compiler.Stage { this.buildFields(); return { + spanStart: (fn, body, code) => this.buildSpanStart(fn, body, code), + spanEnd: (fn, body, code) => this.buildSpanEnd(fn, body, code), + preExecute: (fn, body) => this.buildPrologue(fn, body), postExecute: (fn, body) => this.buildEpilogue(fn, body) }; @@ -45,6 +50,93 @@ class Builder extends compiler.Stage { }); } + // Nodes + + buildSpanStart(fn, body, code) { + const colors = this.ctx.stageResults['span-allocator']; + const index = colors.map.get(code); + + const startPtr = this.startField(fn, index); + body.push(startPtr); + + const store = this.ctx.ir._('store', [ TYPE_INPUT, this.ctx.posArg(fn) ], + [ TYPE_INPUT.ptr(), startPtr ]); + store.void(); + body.push(store); + + const num = colors.concurrency[index].length; + if (num !== 1) { + const callbackType = this.ctx.signature.callback.span.ptr(); + + const cbPtr = this.callbackField(fn, index); + body.push(cbPtr); + + const store = this.ctx.ir._('store', + [ callbackType, this.ctx.buildCode(code) ], + [ callbackType.ptr(), cbPtr ]); + store.void(); + body.push(store); + } + + return body; + } + + buildSpanEnd(fn, body, code) { + const colors = this.ctx.stageResults['span-allocator']; + const index = colors.map.get(code); + + const startPtr = this.startField(fn, index); + body.push(startPtr); + + // Load + const start = this.ctx.ir._('load', TYPE_INPUT, + [ TYPE_INPUT.ptr(), startPtr ]); + body.push(start); + + // ...and reset + const store = this.ctx.ir._('store', [ TYPE_INPUT, TYPE_INPUT.v(null) ], + [ TYPE_INPUT.ptr(), startPtr ]); + store.void(); + body.push(store); + + const signature = this.ctx.signature.callback.span; + const cb = this.ctx.buildCode(code); + + // TODO(indutny): this won't work with internal callbacks due to + // cconv mismatch + const call = this.ctx.call('', signature, this.ctx.buildCode(code), [ + this.ctx.stateArg(fn), + start, + this.ctx.posArg(fn) + ]); + body.push(call); + + // Check return value + const errorCmp = this.ctx.ir._('icmp', + [ 'eq', signature.ret, call ], [ signature.ret.v(0) ]); + body.push(errorCmp); + + const errorBranch = body.branch('br', [ BOOL, errorCmp ]); + const noError = errorBranch.left; + const error = errorBranch.right; + noError.name = 'no_error_' + index; + error.name = 'error_' + index; + + // Handle error + const codePtr = this.ctx.field(fn, 'error'); + error.push(codePtr); + + assert.strictEqual(TYPE_ERROR.type, signature.ret.type); + error.push(this.ctx.ir._('store', + [ TYPE_ERROR, call ], + [ TYPE_ERROR.ptr(), codePtr ]).void()); + error.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + + return noError; + } + + // ${prefix}_execute + buildPrologue(fn, body) { const colors = this.ctx.stageResults['span-allocator']; @@ -53,7 +145,7 @@ class Builder extends compiler.Stage { body.comment('restart spans'); colors.concurrency.forEach((_, index) => { - const startPtr = this.ctx.field(fn, SPAN_START_PREFIX + index); + const startPtr = this.startField(fn, index); body.push(startPtr); const start = this.ctx.ir._('load', TYPE_INPUT, [ TYPE_INPUT.ptr(), startPtr ]); @@ -91,7 +183,7 @@ class Builder extends compiler.Stage { colors.concurrency.forEach((list, index) => { const num = list.length; - const startPtr = this.ctx.field(fn, SPAN_START_PREFIX + index); + const startPtr = this.startField(fn, index); body.push(startPtr); const start = this.ctx.ir._('load', TYPE_INPUT, [ TYPE_INPUT.ptr(), startPtr ]); @@ -113,7 +205,7 @@ class Builder extends compiler.Stage { if (num === 1) { cb = this.ctx.buildCode(list[0]); } else { - const cbPtr = this.ctx.field(fn, SPAN_CB_PREFIX + index); + const cbPtr = this.callbackField(fn, index); present.push(cbPtr); cb = this.ctx.ir._('load', callbackType, [ callbackType.ptr(), cbPtr ]); @@ -150,5 +242,13 @@ class Builder extends compiler.Stage { return body; } + + startField(fn, index) { + return this.ctx.field(fn, SPAN_START_PREFIX + index); + } + + callbackField(fn, index) { + return this.ctx.field(fn, SPAN_CB_PREFIX + index); + } } module.exports = Builder; diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 6f430be..2a3cf1b 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -28,7 +28,7 @@ exports.build = (name, source) => { fs.writeFileSync(file, source); const ret = spawnSync(CLANG, - [ '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); + [ '-g3', '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); if (ret.status !== 0) { process.stderr.write(ret.stdout); process.stderr.write(ret.stderr); From ae8f16ef3d13f643e11a97044a1f813bd9183a10 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 19:51:52 -0500 Subject: [PATCH 135/281] spans: remove prologue --- lib/llparse/compiler/node/span-end.js | 4 ++++ lib/llparse/compiler/node/span-start.js | 4 ++++ test/span-test.js | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 99653a5..d9277a0 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -9,6 +9,10 @@ class SpanEnd extends node.Node { this.code = code; } + prologue(ctx, body) { + return body; + } + doBuild(ctx, body, nodes) { body.comment(`node.SpanEnd[${this.code.name}]`); body = ctx.compilation.stageResults['span-builder'].spanEnd( diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index 61ba54e..c7726bd 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -9,6 +9,10 @@ class SpanStart extends node.Node { this.code = code; } + prologue(ctx, body) { + return body; + } + doBuild(ctx, body, nodes) { body.comment(`node.SpanStart[${this.code.name}]`); body = ctx.compilation.stageResults['span-builder'].spanStart( diff --git a/test/span-test.js b/test/span-test.js index af2f398..cd9809a 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -41,7 +41,9 @@ describe('LLParse/span', () => { .match('_', underscore) .otherwise(span.underscore.end(dot)); - const binary = fixtures.build('span', p.build(start)); + const binary = fixtures.build('span', p.build(start, { + debug: 'debug' + })); binary('..--..__..', 'off=3 match=1\n', callback); }); From afe60e6bf4939a53e2b91f3b203f0a58068dfffc Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 23:13:52 -0500 Subject: [PATCH 136/281] node: `.peek()` --- lib/llparse/case/index.js | 1 + lib/llparse/case/match.js | 11 +++++++--- lib/llparse/case/peek.js | 27 +++++++++++++++++++++++++ lib/llparse/case/select.js | 3 ++- lib/llparse/compiler/node/single.js | 6 ++++-- lib/llparse/compiler/node/translator.js | 1 + lib/llparse/node/base.js | 15 ++++++++++++++ lib/llparse/node/error.js | 1 + lib/llparse/node/invoke.js | 1 + lib/llparse/node/span-end.js | 1 + lib/llparse/node/span-start.js | 1 + lib/llparse/trie.js | 20 ++++++++++++++++-- test/api-test.js | 20 ++++++++++++++++++ test/span-test.js | 4 ++-- 14 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 lib/llparse/case/peek.js diff --git a/lib/llparse/case/index.js b/lib/llparse/case/index.js index e29d9ee..cadd5c4 100644 --- a/lib/llparse/case/index.js +++ b/lib/llparse/case/index.js @@ -2,5 +2,6 @@ exports.Case = require('./base'); exports.Match = require('./match'); +exports.Peek = require('./peek'); exports.Select = require('./select'); exports.Otherwise = require('./otherwise'); diff --git a/lib/llparse/case/match.js b/lib/llparse/case/match.js index 939f299..9070849 100644 --- a/lib/llparse/case/match.js +++ b/lib/llparse/case/match.js @@ -1,5 +1,7 @@ 'use strict'; +const assert = require('assert'); + const llparse = require('../'); const Case = require('./').Case; @@ -8,14 +10,17 @@ class Match extends Case { constructor(key, next) { super('match', next); - this.key = key; + this.key = llparse.utils.toBuffer(key); + assert(this.key.length > 0, + '`.match()` must get at least 1 byte as a first argument'); } linearize() { return [ { - key: llparse.utils.toBuffer(this.key), + key: this.key, next: this.next, - value: null + value: null, + noAdvance: false } ]; } } diff --git a/lib/llparse/case/peek.js b/lib/llparse/case/peek.js new file mode 100644 index 0000000..f5933f3 --- /dev/null +++ b/lib/llparse/case/peek.js @@ -0,0 +1,27 @@ +'use strict'; + +const assert = require('assert'); + +const llparse = require('../'); + +const Case = require('./').Case; + +class Peek extends Case { + constructor(key, next) { + super('peek', next); + + this.key = llparse.utils.toBuffer(key); + assert.strictEqual(this.key.length, 1, + '`.peek()` must get exactly 1 byte as a first argument'); + } + + linearize() { + return [ { + key: this.key, + next: this.next, + value: null, + noAdvance: true + } ]; + } +} +module.exports = Peek; diff --git a/lib/llparse/case/select.js b/lib/llparse/case/select.js index df3de74..38a33a2 100644 --- a/lib/llparse/case/select.js +++ b/lib/llparse/case/select.js @@ -21,7 +21,8 @@ class Select extends Case { res.push({ key: llparse.utils.toBuffer(key), next: this.next, - value + value, + noAdvance: false }); }); return res; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 4140c03..64ecd4e 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -22,9 +22,11 @@ class Single extends node.Node { const s = ctx.buildSwitch(body, pos.type.to, current, keys); s.cases.forEach((body, i) => { - const child = this.children[i].next.build(ctx.compilation, nodes); + const child = this.children[i]; + const target = child.next.build(ctx.compilation, nodes); - this.tailTo(ctx, body, ctx.pos.next, child, this.children[i].value); + this.tailTo(ctx, body, child.noAdvance ? ctx.pos.current : ctx.pos.next, + target, child.value); }); this.doOtherwise(ctx, nodes, s.otherwise); diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index bf0d736..acc681b 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -125,6 +125,7 @@ class NodeTranslator extends compiler.Stage { return { key: child.key, next: this.buildTrie(node, child.child, list), + noAdvance: child.noAdvance, value }; }); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 23ed8dc..9f3e4b5 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -24,6 +24,21 @@ class Node { get name() { return this[kName]; } + peek(value, next) { + // .peek([ ... ], next) + if (Array.isArray(value)) { + value.forEach(value => this.peek(value, next)); + return this; + } + + assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); + this[kCheckIsMatch](next, '.peek()'); + + this[kCases].push(new llparse.case.Peek(value, next)); + + return this; + } + match(value, next) { // .match([ ... ], next) if (Array.isArray(value)) { diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index 6d76174..70bba21 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -16,6 +16,7 @@ class Error extends node.Node { get code() { return this[kCode]; } get reason() { return this[kReason]; } + peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } otherwise() { throw new Error('Not supported'); } diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index b93ab3e..28e923f 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -38,6 +38,7 @@ class Invoke extends node.Node { this.otherwise(otherwise); } + peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } } diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js index ee19591..ceb02a9 100644 --- a/lib/llparse/node/span-end.js +++ b/lib/llparse/node/span-end.js @@ -11,6 +11,7 @@ class SpanEnd extends node.Node { this[kSpan] = span; } + peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } } diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js index 296ad32..a7e6484 100644 --- a/lib/llparse/node/span-start.js +++ b/lib/llparse/node/span-start.js @@ -12,6 +12,7 @@ class SpanStart extends node.Node { this[kSpan] = span; } + peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } } diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index afda10e..fc852a5 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -85,7 +85,8 @@ class Trie { return { key: item.key.slice(off), next: item.next, - value: item.value + value: item.value, + noAdvance: item.noAdvance }; }); } @@ -94,6 +95,8 @@ class Trie { const res = new Sequence(prefix); const sliced = this.slice(list, prefix.length); + const noAdvance = sliced.some(item => item.noAdvance); + assert(!noAdvance); res.child = this.level(sliced, path.concat(prefix)); return res; @@ -114,7 +117,20 @@ class Trie { const res = new Single(); keys.forEach((sublist, key) => { const sliced = this.slice(sublist, 1); - res.children.push({ key, child: this.level(sliced, path.concat(key)) }); + const subpath = path.concat(key); + + const noAdvance = sublist.some(item => item.noAdvance); + assert( + sublist.every(item => item.noAdvance === noAdvance) || + sublist.length === 0, + 'Conflicting `.peek()` and `.match()` entries in ' + + `"${this.name}" at [ ${subpath.join(', ')} ]`); + + res.children.push({ + key, + noAdvance, + child: this.level(sliced, subpath) + }); }); return res; diff --git a/test/api-test.js b/test/api-test.js index f31d094..04efd19 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -124,6 +124,26 @@ describe('LLParse', () => { binary('0110', 'zero\none\none\nzero\n', callback); }); + describe('`.peek()`', () => { + it('should not advance position', (callback) => { + const start = p.node('start'); + const ab = p.node('ab'); + const error = p.error(3, 'Invalid word'); + + start + .peek([ 'a', 'b' ], ab) + .otherwise(error); + + ab + .match([ 'a', 'b' ], printOff(p, start)) + .otherwise(error); + + const binary = fixtures.build('peek', p.build(start)); + + binary('ab', 'off=1\noff=2\n', callback); + }); + }); + describe('`.otherwise()`', () => { it('should not advance position by default', (callback) => { const p = llparse.create('llparse'); diff --git a/test/span-test.js b/test/span-test.js index cd9809a..354dd8d 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -29,8 +29,8 @@ describe('LLParse/span', () => { dot .match('.', dot) - .match('-', span.dash.start(dash)) - .match('_', span.underscore.start(underscore)) + .peek('-', span.dash.start(dash)) + .peek('_', span.underscore.start(underscore)) .skipTo(span.dot.end(start)); dash From e4229a107bc8c6fe19de5f37e1f0173baaef0c75 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 23:54:12 -0500 Subject: [PATCH 137/281] test: spans! --- lib/llparse/compiler/span/builder.js | 3 -- test/fixtures/index.js | 53 +++++++++++++++++++++++++++- test/fixtures/main.c | 22 ++++++------ test/span-test.js | 15 +++++--- 4 files changed, 72 insertions(+), 21 deletions(-) diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/span/builder.js index 691e869..8c4ad03 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/span/builder.js @@ -100,7 +100,6 @@ class Builder extends compiler.Stage { body.push(store); const signature = this.ctx.signature.callback.span; - const cb = this.ctx.buildCode(code); // TODO(indutny): this won't work with internal callbacks due to // cconv mismatch @@ -140,8 +139,6 @@ class Builder extends compiler.Stage { buildPrologue(fn, body) { const colors = this.ctx.stageResults['span-allocator']; - const callbackType = this.ctx.signature.callback.span.ptr(); - body.comment('restart spans'); colors.concurrency.forEach((_, index) => { diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 2a3cf1b..093e1a5 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -16,7 +16,55 @@ const TMP_DIR = path.join(__dirname, '..', 'tmp'); const MAX_PARALLEL = 8; -exports.build = (name, source) => { +function normalizeSpan(source) { + const lines = source.split(/\n/g); + + const parse = (line) => { + const match = line.match( + /^off=(\d+)\s+len=(\d+)\s+span\[([^\]]+)\]="(.*)"$/); + if (!match) { + throw new Error('Failed to parse the span output: '+ + JSON.stringify(line)); + } + + return { + off: match[1] | 0, + len: match[2] | 0, + span: match[3], + value: match[4] + }; + }; + + const parsed = lines.filter(l => l).map(parse); + const lastMap = new Map(); + const res = []; + + parsed.forEach((obj) => { + if (lastMap.has(obj.span)) { + const last = lastMap.get(obj.span); + if (last.off + last.len === obj.off) { + last.len += obj.len; + last.value += obj.value; + + // Move it to the end + res.splice(res.indexOf(last), 1); + res.push(last); + return; + } + } + res.push(obj); + lastMap.set(obj.span, obj); + }); + + const stringify = (obj) => { + return `off=${obj.off} len=${obj.len} span[${obj.span}]="${obj.value}"`; + }; + return res.map(stringify).join('\n') + '\n'; +} + +exports.build = (name, source, options) => { + options = options || {}; + try { fs.mkdirSync(TMP_DIR); } catch (e) { @@ -52,6 +100,9 @@ exports.build = (name, source) => { if (data.exit !== 0) return callback(new Error('Exit code: ' + data.exit)); + if (options.normalize === 'span') + stdout = normalizeSpan(stdout); + callback(null, stdout); }); }, (err, results) => { diff --git a/test/fixtures/main.c b/test/fixtures/main.c index ce945c9..4768d44 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -67,28 +67,26 @@ int return_match(struct state* s, const char* p, const char* endp, int value) { return value; } -int on_dot(struct state* s, const char* p, const char* endp) { +static void print_span(const char* name, const char* p, const char* endp) { if (bench) - return 0; + return; - fprintf(stdout, "off=%d dot=%.*s\n", (int) (p - start), (int) (endp - p), p); + fprintf(stdout, "off=%d len=%d span[%s]=\"%.*s\"\n", (int) (p - start), + (int) (endp - p), name, (int) (endp - p), p); +} + +int on_dot(struct state* s, const char* p, const char* endp) { + print_span("dot", p, endp); return 0; } int on_dash(struct state* s, const char* p, const char* endp) { - if (bench) - return 0; - - fprintf(stdout, "off=%d dash=%.*s\n", (int) (p - start), (int) (endp - p), p); + print_span("dash", p, endp); return 0; } int on_underscore(struct state* s, const char* p, const char* endp) { - if (bench) - return 0; - - fprintf(stdout, "off=%d underscore=%.*s\n", (int) (p - start), - (int) (endp - p), p); + print_span("underscore", p, endp); return 0; } diff --git a/test/span-test.js b/test/span-test.js index 354dd8d..14af253 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -41,11 +41,16 @@ describe('LLParse/span', () => { .match('_', underscore) .otherwise(span.underscore.end(dot)); - const binary = fixtures.build('span', p.build(start, { - debug: 'debug' - })); - - binary('..--..__..', 'off=3 match=1\n', callback); + const binary = fixtures.build('span', p.build(start), { + normalize: 'span' + }); + + binary( + '..--..__..', + 'off=2 len=2 span[dash]="--"\n' + + 'off=6 len=2 span[underscore]="__"\n' + + 'off=0 len=10 span[dot]="..--..__.."\n', + callback); }); it('should throw on loops', () => { From 17a584c9500e15b947e3ee62784b0fae22e8e6a5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 23:58:18 -0500 Subject: [PATCH 138/281] readme: add `.span()` example --- README.md | 22 +++++++--------------- examples/http/index.js | 20 ++++++-------------- examples/http/main.c | 31 ++++--------------------------- 3 files changed, 17 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index ac8ea51..36d7867 100644 --- a/README.md +++ b/README.md @@ -13,25 +13,17 @@ const p = require('llparse').create('http_parser'); const method = p.node('method'); const beforeUrl = p.node('before_url'); +const urlSpan = p.span(p.code.span('on_url')); const url = p.node('url'); const http = p.node('http'); -// Add custom int8_t property to the state +// Add custom uint8_t property to the state p.property(ir => ir.i(8), 'method'); -// Store method inside the custom property +// Store method inside a custom property const onMethod = p.invoke(p.code.store('method'), beforeUrl); -// Invoke external C function -const urlStart = p.invoke(p.code.match('on_url_start'), { - // If that function returns zero - 0: url -}, p.error(2, '`on_url_start` error')); - -const urlEnd = p.invoke(p.code.match('on_url_end'), { - 0: http -}, p.error(3, '`on_url_end` error')); - +// Invoke custom C function const complete = p.invoke(p.code.match('on_complete'), { // Restart 0: method @@ -47,14 +39,14 @@ method beforeUrl .match(' ', beforeUrl) - .otherwise(urlStart); + .otherwise(urlSpan.start(url)); url - .match(' ', urlEnd) + .peek(' ', urlSpan.end(http)) .skipTo(url); http - .match('HTTP/1.1\r\n\r\n', complete) + .match(' HTTP/1.1\r\n\r\n', complete) .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); console.log(p.build(method)); diff --git a/examples/http/index.js b/examples/http/index.js index c9a4bcd..a6fb838 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -4,6 +4,7 @@ const p = require('../../').create('http_parser'); const method = p.node('method'); const beforeUrl = p.node('before_url'); +const urlSpan = p.span(p.code.span('on_url')); const url = p.node('url'); const http = p.node('http'); @@ -13,16 +14,7 @@ p.property(ir => ir.i(8), 'method'); // Store method inside a custom property const onMethod = p.invoke(p.code.store('method'), beforeUrl); -// Invoke external C function -const urlStart = p.invoke(p.code.match('on_url_start'), { - // If that function returns zero - 0: url -}, p.error(2, '`on_url_start` error')); - -const urlEnd = p.invoke(p.code.match('on_url_end'), { - 0: http -}, p.error(3, '`on_url_end` error')); - +// Invoke custom C function const complete = p.invoke(p.code.match('on_complete'), { // Restart 0: method @@ -38,15 +30,15 @@ method beforeUrl .match(' ', beforeUrl) - .otherwise(urlStart); + .otherwise(urlSpan.start(url)); url - .match(' ', urlEnd) + .peek(' ', urlSpan.end(http)) .skipTo(url); http - .match('HTTP/1.1\r\n\r\n', complete) - .match('HTTP/1.1\n\n', complete) + .match(' HTTP/1.1\r\n\r\n', complete) + .match(' HTTP/1.1\n\n', complete) .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); console.log(p.build(method)); diff --git a/examples/http/main.c b/examples/http/main.c index 23fa374..7e6fe0c 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -12,36 +12,21 @@ struct http_parser_state_s { const char* reason; int index; void* data; - void* mark; - unsigned int method : 8; - const char* url_start; + uint8_t method; + void* mark; }; void http_parser_init(http_parser_state_t* s); int http_parser_execute(http_parser_state_t* s, const char* p, const char* endp); -static void on_url_part(http_parser_state_t* s, const char* p, - const char* endp) { +int on_url(http_parser_state_t* s, const char* p, const char* endp) { if (p == endp) - return; + return 0; fprintf(stdout, "method=%d url_part=\"%.*s\"\n", s->method, (int) (endp - p), p); -} - - -int on_url_start(http_parser_state_t* s, const char* p, const char* endp) { - s->url_start = p; - return 0; -} - - -int on_url_end(http_parser_state_t* s, const char* p, const char* endp) { - on_url_part(s, s->url_start, p - 1); - fprintf(stdout, "url end\n"); - s->url_start = NULL; return 0; } @@ -57,8 +42,6 @@ int main(int argc, char** argv) { http_parser_init(&s); - s.url_start = NULL; - for (;;) { char buf[16384]; const char* input; @@ -69,18 +52,12 @@ int main(int argc, char** argv) { if (input == NULL) break; - if (s.url_start != NULL) - s.url_start = input; - endp = input + strlen(input); code = http_parser_execute(&s, input, endp); if (code != 0) { fprintf(stderr, "code=%d error=%d reason=%s\n", code, s.error, s.reason); return -1; } - - if (s.url_start != NULL) - on_url_part(&s, s.url_start, endp); } return 0; From 911073a95b7100fc9e3a2aed9dfb624c5adb8575 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 23:59:19 -0500 Subject: [PATCH 139/281] package: update description --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 75ff6f3..0cd8172 100644 --- a/package.json +++ b/package.json @@ -25,5 +25,5 @@ "type": "git", "url": "gh:indutny/llparse" }, - "description": "" + "description": "Generating parsers in LLVM IR" } From f8b0a12661dfeed2e9ddf756c418db72df5f6b66 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 21 Feb 2018 23:59:31 -0500 Subject: [PATCH 140/281] 2.0.0-beta4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 75aec76..3687a4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta3", + "version": "2.0.0-beta4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 0cd8172..52b6369 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta3", + "version": "2.0.0-beta4", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From c7e1be7624cca05c4bd30eed4ce91f49a827be6d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 01:10:00 -0500 Subject: [PATCH 141/281] lib: validate `.property()` arguments --- lib/llparse.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/llparse.js b/lib/llparse.js index 6f29082..e241172 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -71,6 +71,11 @@ class LLParse { } property(type, name) { + assert.strictEqual(typeof type, 'function', + 'The first argument of `.property()` must be a function'); + assert.strictEqual(typeof name, 'string', + 'The second argument of `.property()` must be a string'); + if (internal.constants.RESERVED_PROPERTY_NAMES.has(name)) throw new Error(`Can't use reserved property name: "${name}"`); if (/^_/.test(name)) From 15acb38c581455e987ef36584a632283b48bed66 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 01:20:09 -0500 Subject: [PATCH 142/281] span/allocator: process Invoke.map --- lib/llparse/compiler/span/allocator.js | 9 +++++++-- test/span-test.js | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/span/allocator.js index e19bbac..aab50c1 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -8,6 +8,7 @@ const llparse = require('../../'); const kCode = llparse.symbols.kCode; const kCases = llparse.symbols.kCases; const kOtherwise = llparse.symbols.kOtherwise; +const kMap = llparse.symbols.kMap; const kSpan = llparse.symbols.kSpan; class Allocator extends compiler.Stage { @@ -71,7 +72,7 @@ class Allocator extends compiler.Stage { // Disallow loops if (child instanceof llparse.node.SpanStart) { assert.notStrictEqual(this.id(child), span, - `Detected loop in span "${span}"`); + `Detected loop in span "${span.name}"`); } const set = activeMap.get(child); @@ -88,7 +89,7 @@ class Allocator extends compiler.Stage { ends.forEach((end) => { const active = activeMap.get(end); assert(active.has(this.id(end)), - `Unmatched span end for "${this.id(end)}"`); + `Unmatched span end for "${this.id(end).name}"`); }); return { active: activeMap, spans: Array.from(spans) }; @@ -166,6 +167,10 @@ class Allocator extends compiler.Stage { res.push(node[kOtherwise].next); node[kCases].forEach(c => res.push(c.next)); + if (node instanceof llparse.node.Invoke) { + Object.keys(node[kMap]).forEach(key => res.push(node[kMap][key])); + } + return res; } } diff --git a/test/span-test.js b/test/span-test.js index 14af253..ab76560 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -59,7 +59,7 @@ describe('LLParse/span', () => { start.otherwise(span.start().otherwise(start)); - assert.throws(() => p.build(start), /loop/); + assert.throws(() => p.build(start), /loop.*on_data/); }); it('should throw on unmatched ends', () => { @@ -68,6 +68,19 @@ describe('LLParse/span', () => { start.otherwise(span.end().otherwise(start)); - assert.throws(() => p.build(start), /unmatched/i); + assert.throws(() => p.build(start), /unmatched.*on_data/i); + }); + + it('should propagate through the Invoke map', () => { + const start = p.node('start'); + const span = p.span(p.code.span('on_data')); + + p.property(ir => ir.i(8), 'custom'); + + start.otherwise(p.invoke(p.code.load('custom'), { + 0: span.end().otherwise(start) + }, span.end().otherwise(start))); + + assert.doesNotThrow(() => p.build(span.start(start))); }); }); From 60907f810ba0492bf3fdebba6475d5f520b3451d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 01:24:33 -0500 Subject: [PATCH 143/281] 2.0.0-beta5 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3687a4a..707edf4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta4", + "version": "2.0.0-beta5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 52b6369..8181d03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta4", + "version": "2.0.0-beta5", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 1395c0e4599c7a4c08ac8edb31e579499e0636e9 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 01:34:44 -0500 Subject: [PATCH 144/281] test: fix benchmark --- test/fixtures/main.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 4768d44..2804aa9 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -9,6 +9,9 @@ struct state { int error; const char* reason; int index; + + /* Give huge leeway */ + void* fields[1024]; }; /* 8 gb */ @@ -114,7 +117,7 @@ static int run_bench(const char* input, int len) { gettimeofday(&end, NULL); time = (end.tv_sec - start.tv_sec); - time += (end.tv_usec - start.tv_usec) * 1e-6; + time += (double) (end.tv_usec - start.tv_usec) * 1e-6; bw = (double) kBytes / time; fprintf(stdout, "%.2f mb | %.2f mb/s | %.2f s\n", From c61e6b5269dfd3eb6b83d3a916ec21a40363bd6f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 13:12:48 -0500 Subject: [PATCH 145/281] compilation: add todo --- lib/llparse/compiler/compilation.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index a8efd22..3cb889b 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -65,6 +65,8 @@ class Compilation { this.declareField(TYPE_ERROR, 'error', type => type.v(0)); this.declareField(TYPE_REASON, 'reason', type => type.v(null)); + + // TODO(indutny): auto-size this field this.declareField(TYPE_INDEX, 'index', type => type.v(0)); // Public fields From 5361429fd926013f41b74509c2ef8c89e4112277 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:20:26 -0500 Subject: [PATCH 146/281] compilation: auto-size `index` field --- lib/llparse/compiler/compilation.js | 58 ++++++++++++++++--------- lib/llparse/compiler/compiler.js | 32 +++++++------- lib/llparse/compiler/match-sequence.js | 22 +++++++--- lib/llparse/compiler/node/sequence.js | 4 +- lib/llparse/compiler/node/translator.js | 9 +++- lib/llparse/constants.js | 2 +- 6 files changed, 82 insertions(+), 45 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 3cb889b..435dee8 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -15,7 +15,6 @@ const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_MATCH = constants.TYPE_MATCH; -const TYPE_INDEX = constants.TYPE_INDEX; const TYPE_ERROR = constants.TYPE_ERROR; const TYPE_REASON = constants.TYPE_REASON; const TYPE_DATA = constants.TYPE_DATA; @@ -30,13 +29,13 @@ const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; class Compilation { - constructor(prefix, options, root) { + constructor(options) { this.options = options || {}; this.ir = new IR(); - this.prefix = prefix; + this.prefix = this.options.prefix; - this.root = root; + this.root = this.options.root; this.state = this.ir.struct(`${this.prefix}_state`); this.initializers = []; @@ -60,14 +59,47 @@ class Compilation { }; this.signature.callback.span = this.signature.callback.match; + this.codeCache = new Map(); + this.debugMethod = null; + + // Filled by `node.Translator`, maximum sequence length + this.maxSequence = 0; + this.TYPE_INDEX = null; + + // Intermediate results from various build stages + this.stageResults = {}; + + this.buildStages(this.options.before); + + this.declareFields(); + + this.buildStages(this.options.after); + } + + buildStages(stages) { + stages.forEach(Stage => this.buildStage(Stage)); + } + + buildStage(Stage) { + const stage = new Stage(this); + this.stageResults[stage.name] = stage.build(); + } + + declareFields() { this.declareField(this.signature.node.ptr(), 'current', (type, ctx) => ctx.stageResults['node-builder'].ref()); this.declareField(TYPE_ERROR, 'error', type => type.v(0)); this.declareField(TYPE_REASON, 'reason', type => type.v(null)); - // TODO(indutny): auto-size this field - this.declareField(TYPE_INDEX, 'index', type => type.v(0)); + // Find number of bits for an index field + let width = 8; + for (; (1 << width) <= this.maxSequence && width < 32; width += 8) { + // No-op + } + assert((1 << width) > this.maxSequence, 'Sequence length OOB'); + this.TYPE_INDEX = this.ir.i(width); + this.declareField(this.TYPE_INDEX, 'index', type => type.v(0)); // Public fields this.declareField(TYPE_DATA, 'data', type => type.v(null)); @@ -78,20 +110,6 @@ class Compilation { this.declareField(prop.type(this.ir, this.state), prop.name, type => type.v(0)); }); - - this.codeCache = new Map(); - this.debugMethod = null; - - // Intermediate results from various build stages - this.stageResults = {}; - } - - buildStages(stages) { - stages.forEach(stage => this.buildStage(stage)); - } - - buildStage(stage) { - this.stageResults[stage.name] = stage.build(this); } buildCode(code) { diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 7681bd7..35d21be 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -29,21 +29,23 @@ class Compiler { } build(root) { - const ctx = new llparse.compiler.Compilation(this.prefix, this.options, - root); - - // TODO(indutny): detect and report loops - - const stages = [ - new llparse.compiler.MatchSequence(ctx), - new llparse.compiler.span.Allocator(ctx), - new llparse.compiler.span.Builder(ctx), - new llparse.compiler.node.Translator(ctx), - new llparse.compiler.node.Builder(ctx) - ]; - - ctx.buildStages(stages); - + const compOptions = Object.assign({}, this.options, { + root, + + before: [ + llparse.compiler.node.Translator, + llparse.compiler.span.Allocator + ], + after: [ + llparse.compiler.MatchSequence, + llparse.compiler.span.Builder, + llparse.compiler.node.Builder + ] + }); + + const ctx = new llparse.compiler.Compilation(compOptions); + + // TODO(indutny): make these stages? this.buildInit(ctx); this.buildExecute(ctx); diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index a51d7f8..2069a53 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -11,7 +11,7 @@ const CCONV = constants.CCONV; const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; -const TYPE_INDEX = constants.TYPE_INDEX; +const TYPE_STATUS = constants.TYPE_STATUS; const ATTR_STATE = constants.ATTR_STATE; const ATTR_POS = constants.ATTR_POS; @@ -35,18 +35,20 @@ class MatchSequence extends compiler.Stage { this.returnType = this.ctx.ir.struct('match_sequence_ret'); this.returnType.field(TYPE_INPUT, 'current'); - this.returnType.field(INT, 'status'); + this.returnType.field(TYPE_STATUS, 'status'); + this.signature = null; + } + + build() { this.signature = IR.signature(this.returnType, [ [ this.ctx.state.ptr(), ATTR_STATE ], [ TYPE_INPUT, ATTR_POS ], [ TYPE_INPUT, ATTR_ENDPOS ], [ TYPE_INPUT, ATTR_SEQUENCE ], - INT + this.ctx.TYPE_INDEX ]); - } - build() { const fn = this.ctx.ir.fn(this.signature, `${this.ctx.prefix}__match_sequence`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); @@ -63,6 +65,8 @@ class MatchSequence extends compiler.Stage { // TODO(indutny): initialize `state.index` before calling matcher? buildBody(fn) { + const TYPE_INDEX = this.ctx.TYPE_INDEX; + const body = fn.body; // Just to have a label @@ -106,6 +110,8 @@ class MatchSequence extends compiler.Stage { buildIteration(fn, body, indexField, pos, index) { + const TYPE_INDEX = this.ctx.TYPE_INDEX; + const seq = fn.arg(ARG_SEQUENCE); const seqLen = fn.arg(ARG_SEQUENCE_LEN); @@ -116,7 +122,7 @@ class MatchSequence extends compiler.Stage { body.comment('expected = seq[state.index]'); const expectedPtr = IR._('getelementptr', seq.type.to, [ seq.type, seq ], - [ INT, index ]); + [ TYPE_INDEX, index ]); body.push(expectedPtr); const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); body.push(expected); @@ -189,7 +195,7 @@ class MatchSequence extends compiler.Stage { body.push(create); const amend = IR._('insertvalue', [ this.returnType, create ], - [ INT, INT.v(status) ], + [ TYPE_STATUS, TYPE_STATUS.v(status) ], INT.v(this.returnType.lookup('status'))); body.push(amend); @@ -197,6 +203,8 @@ class MatchSequence extends compiler.Stage { } reset(field) { + const TYPE_INDEX = this.ctx.TYPE_INDEX; + return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], [ TYPE_INDEX.ptr(), field ]).void(); } diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index fc8528b..5f5f376 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -5,6 +5,8 @@ const constants = llparse.constants; const node = require('./'); +const TYPE_STATUS = constants.TYPE_STATUS; + const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; @@ -57,7 +59,7 @@ class Sequence extends node.Node { [ ctx.pos.current.type, current ], [ INT, INT.v(1) ]); body.push(next); - const s = ctx.buildSwitch(body, INT, status, [ + const s = ctx.buildSwitch(body, TYPE_STATUS, status, [ SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index acc681b..8808ed1 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -18,12 +18,17 @@ class NodeTranslator extends compiler.Stage { this.nodes = new Map(); this.namespace = new Set(); + + // Maximum sequence length + this.maxSequence = 0; } build() { // TODO(indutny): detect and report loops - return this.buildNode(this.ctx.root, null); + const root = this.buildNode(this.ctx.root, null); + this.ctx.maxSequence = this.maxSequence; + return root; } id(node) { @@ -135,6 +140,8 @@ class NodeTranslator extends compiler.Stage { } buildSequence(node, trie, list, isRoot) { + this.maxSequence = Math.max(this.maxSequence, trie.select.length); + const res = new compiler.node.Sequence(this.id(node), trie.select); const value = trie.child.type === 'next' ? trie.child.value : null; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 7a39a56..75387e5 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -9,10 +9,10 @@ exports.INT = IR.i(32); exports.TYPE_INPUT = IR.i(8).ptr(); exports.TYPE_OUTPUT = IR.i(8).ptr(); exports.TYPE_MATCH = exports.INT; -exports.TYPE_INDEX = exports.INT; exports.TYPE_ERROR = exports.INT; exports.TYPE_REASON = IR.i(8).ptr(); exports.TYPE_DATA = IR.i(8).ptr(); +exports.TYPE_STATUS = IR.i(3); exports.ARG_STATE = 's'; exports.ARG_POS = 'p'; From b97c2b76dcece031820bad1fc2f8ac3b72fdeb1c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:21:50 -0500 Subject: [PATCH 147/281] test: update struct --- examples/http/main.c | 2 +- test/fixtures/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/http/main.c b/examples/http/main.c index 7e6fe0c..7997df4 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -10,7 +10,7 @@ struct http_parser_state_s { void* current; int error; const char* reason; - int index; + uint8_t index; void* data; uint8_t method; diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 2804aa9..64a39b4 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -8,7 +8,7 @@ struct state { void* current; int error; const char* reason; - int index; + uint8_t index; /* Give huge leeway */ void* fields[1024]; From eaf885a41a223801b0770cb9836d952d94fe9518 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:44:52 -0500 Subject: [PATCH 148/281] compilation: fix auto-sizing of index field LLVM treats offsets as signed values. Allocate extra bit to account for this. --- lib/llparse/compiler/compilation.js | 7 +++++-- test/api-test.js | 32 +++++++++++++++++++++++++++++ test/fixtures/index.js | 9 +++++--- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 435dee8..00f1e35 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -94,10 +94,13 @@ class Compilation { // Find number of bits for an index field let width = 8; - for (; (1 << width) <= this.maxSequence && width < 32; width += 8) { + + // Add extra bit, because LLVM IR expects signed offsets + const max = this.maxSequence * 2; + for (; Math.pow(2, width) <= max && width < 32; width += 8) { // No-op } - assert((1 << width) > this.maxSequence, 'Sequence length OOB'); + assert(Math.pow(2, width) > this.maxSequence, 'Sequence length OOB'); this.TYPE_INDEX = this.ir.i(width); this.declareField(this.TYPE_INDEX, 'index', type => type.v(0)); diff --git a/test/api-test.js b/test/api-test.js index 04efd19..0fdab0c 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -34,6 +34,38 @@ describe('LLParse', () => { binary('GET', 'off=3 match=1\n', callback); }); + it('should compile 16-bit sequence', function(callback) { + // It takes a lot of time to scan + this.timeout(5000); + + const start = p.node('start'); + + const input = new Array(512).fill('a').join(''); + + start.match(input, printOff(p, start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('16bit', p.build(start)); + + binary(input, 'off=512\n', callback); + }); + + + it('should account for sign extension of offset', (callback) => { + const start = p.node('start'); + + const input = new Array(129).fill('a').join(''); + + start.match(input, printOff(p, start)); + + start.otherwise(p.error(3, 'Invalid word')); + + const binary = fixtures.build('sign-ext', p.build(start)); + + binary(input, 'off=129\n', callback); + }); + it('should optimize shallow select', (callback) => { const start = p.node('start'); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 093e1a5..d0159f0 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -94,11 +94,14 @@ exports.build = (name, source, options) => { proc.stdout.on('data', chunk => stdout += chunk); async.parallel({ - exit: cb => proc.once('exit', (code) => cb(null, code)), + exit: cb => proc.once('exit', (code, sig) => cb(null, { code, sig })), end: cb => proc.stdout.once('end', () => cb(null)) }, (err, data) => { - if (data.exit !== 0) - return callback(new Error('Exit code: ' + data.exit)); + if (data.exit.sig) + return callback(new Error('Received signal: ' + data.exit.sig)); + + if (data.exit.code !== 0) + return callback(new Error('Exit code: ' + data.exit.code)); if (options.normalize === 'span') stdout = normalizeSpan(stdout); From 7bd5f5a836a62013a9b85d78e6c4fea31d4fdb12 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:49:39 -0500 Subject: [PATCH 149/281] Revert "compilation: fix auto-sizing of index field" This reverts commit eaf885a41a223801b0770cb9836d952d94fe9518. --- lib/llparse/compiler/compilation.js | 7 ++----- test/api-test.js | 32 ----------------------------- test/fixtures/index.js | 9 +++----- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 00f1e35..435dee8 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -94,13 +94,10 @@ class Compilation { // Find number of bits for an index field let width = 8; - - // Add extra bit, because LLVM IR expects signed offsets - const max = this.maxSequence * 2; - for (; Math.pow(2, width) <= max && width < 32; width += 8) { + for (; (1 << width) <= this.maxSequence && width < 32; width += 8) { // No-op } - assert(Math.pow(2, width) > this.maxSequence, 'Sequence length OOB'); + assert((1 << width) > this.maxSequence, 'Sequence length OOB'); this.TYPE_INDEX = this.ir.i(width); this.declareField(this.TYPE_INDEX, 'index', type => type.v(0)); diff --git a/test/api-test.js b/test/api-test.js index 0fdab0c..04efd19 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -34,38 +34,6 @@ describe('LLParse', () => { binary('GET', 'off=3 match=1\n', callback); }); - it('should compile 16-bit sequence', function(callback) { - // It takes a lot of time to scan - this.timeout(5000); - - const start = p.node('start'); - - const input = new Array(512).fill('a').join(''); - - start.match(input, printOff(p, start)); - - start.otherwise(p.error(3, 'Invalid word')); - - const binary = fixtures.build('16bit', p.build(start)); - - binary(input, 'off=512\n', callback); - }); - - - it('should account for sign extension of offset', (callback) => { - const start = p.node('start'); - - const input = new Array(129).fill('a').join(''); - - start.match(input, printOff(p, start)); - - start.otherwise(p.error(3, 'Invalid word')); - - const binary = fixtures.build('sign-ext', p.build(start)); - - binary(input, 'off=129\n', callback); - }); - it('should optimize shallow select', (callback) => { const start = p.node('start'); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index d0159f0..093e1a5 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -94,14 +94,11 @@ exports.build = (name, source, options) => { proc.stdout.on('data', chunk => stdout += chunk); async.parallel({ - exit: cb => proc.once('exit', (code, sig) => cb(null, { code, sig })), + exit: cb => proc.once('exit', (code) => cb(null, code)), end: cb => proc.stdout.once('end', () => cb(null)) }, (err, data) => { - if (data.exit.sig) - return callback(new Error('Received signal: ' + data.exit.sig)); - - if (data.exit.code !== 0) - return callback(new Error('Exit code: ' + data.exit.code)); + if (data.exit !== 0) + return callback(new Error('Exit code: ' + data.exit)); if (options.normalize === 'span') stdout = normalizeSpan(stdout); From 90b1dd27cfb0e678c43cfebb527757d6cb60c2c4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:49:45 -0500 Subject: [PATCH 150/281] Revert "test: update struct" This reverts commit b97c2b76dcece031820bad1fc2f8ac3b72fdeb1c. --- examples/http/main.c | 2 +- test/fixtures/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/http/main.c b/examples/http/main.c index 7997df4..7e6fe0c 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -10,7 +10,7 @@ struct http_parser_state_s { void* current; int error; const char* reason; - uint8_t index; + int index; void* data; uint8_t method; diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 64a39b4..2804aa9 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -8,7 +8,7 @@ struct state { void* current; int error; const char* reason; - uint8_t index; + int index; /* Give huge leeway */ void* fields[1024]; From 1dbf8c517e9453730c5ff88a1d42f9f74382608a Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:49:54 -0500 Subject: [PATCH 151/281] Revert "compilation: auto-size `index` field" This reverts commit 5361429fd926013f41b74509c2ef8c89e4112277. --- lib/llparse/compiler/compilation.js | 58 +++++++++---------------- lib/llparse/compiler/compiler.js | 32 +++++++------- lib/llparse/compiler/match-sequence.js | 22 +++------- lib/llparse/compiler/node/sequence.js | 4 +- lib/llparse/compiler/node/translator.js | 9 +--- lib/llparse/constants.js | 2 +- 6 files changed, 45 insertions(+), 82 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 435dee8..3cb889b 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -15,6 +15,7 @@ const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_MATCH = constants.TYPE_MATCH; +const TYPE_INDEX = constants.TYPE_INDEX; const TYPE_ERROR = constants.TYPE_ERROR; const TYPE_REASON = constants.TYPE_REASON; const TYPE_DATA = constants.TYPE_DATA; @@ -29,13 +30,13 @@ const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; class Compilation { - constructor(options) { + constructor(prefix, options, root) { this.options = options || {}; this.ir = new IR(); - this.prefix = this.options.prefix; + this.prefix = prefix; - this.root = this.options.root; + this.root = root; this.state = this.ir.struct(`${this.prefix}_state`); this.initializers = []; @@ -59,47 +60,14 @@ class Compilation { }; this.signature.callback.span = this.signature.callback.match; - this.codeCache = new Map(); - this.debugMethod = null; - - // Filled by `node.Translator`, maximum sequence length - this.maxSequence = 0; - this.TYPE_INDEX = null; - - // Intermediate results from various build stages - this.stageResults = {}; - - this.buildStages(this.options.before); - - this.declareFields(); - - this.buildStages(this.options.after); - } - - buildStages(stages) { - stages.forEach(Stage => this.buildStage(Stage)); - } - - buildStage(Stage) { - const stage = new Stage(this); - this.stageResults[stage.name] = stage.build(); - } - - declareFields() { this.declareField(this.signature.node.ptr(), 'current', (type, ctx) => ctx.stageResults['node-builder'].ref()); this.declareField(TYPE_ERROR, 'error', type => type.v(0)); this.declareField(TYPE_REASON, 'reason', type => type.v(null)); - // Find number of bits for an index field - let width = 8; - for (; (1 << width) <= this.maxSequence && width < 32; width += 8) { - // No-op - } - assert((1 << width) > this.maxSequence, 'Sequence length OOB'); - this.TYPE_INDEX = this.ir.i(width); - this.declareField(this.TYPE_INDEX, 'index', type => type.v(0)); + // TODO(indutny): auto-size this field + this.declareField(TYPE_INDEX, 'index', type => type.v(0)); // Public fields this.declareField(TYPE_DATA, 'data', type => type.v(null)); @@ -110,6 +78,20 @@ class Compilation { this.declareField(prop.type(this.ir, this.state), prop.name, type => type.v(0)); }); + + this.codeCache = new Map(); + this.debugMethod = null; + + // Intermediate results from various build stages + this.stageResults = {}; + } + + buildStages(stages) { + stages.forEach(stage => this.buildStage(stage)); + } + + buildStage(stage) { + this.stageResults[stage.name] = stage.build(this); } buildCode(code) { diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 35d21be..7681bd7 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -29,23 +29,21 @@ class Compiler { } build(root) { - const compOptions = Object.assign({}, this.options, { - root, - - before: [ - llparse.compiler.node.Translator, - llparse.compiler.span.Allocator - ], - after: [ - llparse.compiler.MatchSequence, - llparse.compiler.span.Builder, - llparse.compiler.node.Builder - ] - }); - - const ctx = new llparse.compiler.Compilation(compOptions); - - // TODO(indutny): make these stages? + const ctx = new llparse.compiler.Compilation(this.prefix, this.options, + root); + + // TODO(indutny): detect and report loops + + const stages = [ + new llparse.compiler.MatchSequence(ctx), + new llparse.compiler.span.Allocator(ctx), + new llparse.compiler.span.Builder(ctx), + new llparse.compiler.node.Translator(ctx), + new llparse.compiler.node.Builder(ctx) + ]; + + ctx.buildStages(stages); + this.buildInit(ctx); this.buildExecute(ctx); diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index 2069a53..a51d7f8 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -11,7 +11,7 @@ const CCONV = constants.CCONV; const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; -const TYPE_STATUS = constants.TYPE_STATUS; +const TYPE_INDEX = constants.TYPE_INDEX; const ATTR_STATE = constants.ATTR_STATE; const ATTR_POS = constants.ATTR_POS; @@ -35,20 +35,18 @@ class MatchSequence extends compiler.Stage { this.returnType = this.ctx.ir.struct('match_sequence_ret'); this.returnType.field(TYPE_INPUT, 'current'); - this.returnType.field(TYPE_STATUS, 'status'); + this.returnType.field(INT, 'status'); - this.signature = null; - } - - build() { this.signature = IR.signature(this.returnType, [ [ this.ctx.state.ptr(), ATTR_STATE ], [ TYPE_INPUT, ATTR_POS ], [ TYPE_INPUT, ATTR_ENDPOS ], [ TYPE_INPUT, ATTR_SEQUENCE ], - this.ctx.TYPE_INDEX + INT ]); + } + build() { const fn = this.ctx.ir.fn(this.signature, `${this.ctx.prefix}__match_sequence`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); @@ -65,8 +63,6 @@ class MatchSequence extends compiler.Stage { // TODO(indutny): initialize `state.index` before calling matcher? buildBody(fn) { - const TYPE_INDEX = this.ctx.TYPE_INDEX; - const body = fn.body; // Just to have a label @@ -110,8 +106,6 @@ class MatchSequence extends compiler.Stage { buildIteration(fn, body, indexField, pos, index) { - const TYPE_INDEX = this.ctx.TYPE_INDEX; - const seq = fn.arg(ARG_SEQUENCE); const seqLen = fn.arg(ARG_SEQUENCE_LEN); @@ -122,7 +116,7 @@ class MatchSequence extends compiler.Stage { body.comment('expected = seq[state.index]'); const expectedPtr = IR._('getelementptr', seq.type.to, [ seq.type, seq ], - [ TYPE_INDEX, index ]); + [ INT, index ]); body.push(expectedPtr); const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); body.push(expected); @@ -195,7 +189,7 @@ class MatchSequence extends compiler.Stage { body.push(create); const amend = IR._('insertvalue', [ this.returnType, create ], - [ TYPE_STATUS, TYPE_STATUS.v(status) ], + [ INT, INT.v(status) ], INT.v(this.returnType.lookup('status'))); body.push(amend); @@ -203,8 +197,6 @@ class MatchSequence extends compiler.Stage { } reset(field) { - const TYPE_INDEX = this.ctx.TYPE_INDEX; - return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], [ TYPE_INDEX.ptr(), field ]).void(); } diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 5f5f376..fc8528b 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -5,8 +5,6 @@ const constants = llparse.constants; const node = require('./'); -const TYPE_STATUS = constants.TYPE_STATUS; - const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; @@ -59,7 +57,7 @@ class Sequence extends node.Node { [ ctx.pos.current.type, current ], [ INT, INT.v(1) ]); body.push(next); - const s = ctx.buildSwitch(body, TYPE_STATUS, status, [ + const s = ctx.buildSwitch(body, INT, status, [ SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 8808ed1..acc681b 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -18,17 +18,12 @@ class NodeTranslator extends compiler.Stage { this.nodes = new Map(); this.namespace = new Set(); - - // Maximum sequence length - this.maxSequence = 0; } build() { // TODO(indutny): detect and report loops - const root = this.buildNode(this.ctx.root, null); - this.ctx.maxSequence = this.maxSequence; - return root; + return this.buildNode(this.ctx.root, null); } id(node) { @@ -140,8 +135,6 @@ class NodeTranslator extends compiler.Stage { } buildSequence(node, trie, list, isRoot) { - this.maxSequence = Math.max(this.maxSequence, trie.select.length); - const res = new compiler.node.Sequence(this.id(node), trie.select); const value = trie.child.type === 'next' ? trie.child.value : null; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 75387e5..7a39a56 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -9,10 +9,10 @@ exports.INT = IR.i(32); exports.TYPE_INPUT = IR.i(8).ptr(); exports.TYPE_OUTPUT = IR.i(8).ptr(); exports.TYPE_MATCH = exports.INT; +exports.TYPE_INDEX = exports.INT; exports.TYPE_ERROR = exports.INT; exports.TYPE_REASON = IR.i(8).ptr(); exports.TYPE_DATA = IR.i(8).ptr(); -exports.TYPE_STATUS = IR.i(3); exports.ARG_STATE = 's'; exports.ARG_POS = 'p'; From d4abf0a03f9c72954f89788e8f98b7865ebe1be8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:50:01 -0500 Subject: [PATCH 152/281] Revert "compilation: add todo" This reverts commit c61e6b5269dfd3eb6b83d3a916ec21a40363bd6f. --- lib/llparse/compiler/compilation.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 3cb889b..a8efd22 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -65,8 +65,6 @@ class Compilation { this.declareField(TYPE_ERROR, 'error', type => type.v(0)); this.declareField(TYPE_REASON, 'reason', type => type.v(null)); - - // TODO(indutny): auto-size this field this.declareField(TYPE_INDEX, 'index', type => type.v(0)); // Public fields From 7374a62bfd3f3aa1ce0364572711319a31fc9a6e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 14:50:51 -0500 Subject: [PATCH 153/281] test: handle signals --- test/fixtures/index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 093e1a5..1db7a69 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -94,11 +94,13 @@ exports.build = (name, source, options) => { proc.stdout.on('data', chunk => stdout += chunk); async.parallel({ - exit: cb => proc.once('exit', (code) => cb(null, code)), + exit: cb => proc.once('exit', (code, sig) => cb(null, { code, sig })), end: cb => proc.stdout.once('end', () => cb(null)) }, (err, data) => { - if (data.exit !== 0) - return callback(new Error('Exit code: ' + data.exit)); + if (data.exit.sig) + return callback(new Error('Killed with: ' + data.exit.sig)); + if (data.exit.code !== 0) + return callback(new Error('Exit code: ' + data.exit.code)); if (options.normalize === 'span') stdout = normalizeSpan(stdout); From 84d9d095444d9d47ccd69ddb70ec4551309061eb Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 15:56:41 -0500 Subject: [PATCH 154/281] transform: `toLower` --- lib/llparse.js | 10 ++++++++ lib/llparse/compiler/compilation.js | 13 ++++++++++ lib/llparse/compiler/match-sequence.js | 34 ++++++++++++++++++++----- lib/llparse/compiler/node/base.js | 1 + lib/llparse/compiler/node/sequence.js | 3 ++- lib/llparse/compiler/node/single.js | 10 +++++++- lib/llparse/compiler/node/translator.js | 14 +++++++--- lib/llparse/index.js | 1 + lib/llparse/node/base.js | 11 ++++++++ lib/llparse/node/error.js | 1 + lib/llparse/node/invoke.js | 1 + lib/llparse/node/span-end.js | 1 + lib/llparse/node/span-start.js | 1 + lib/llparse/symbols.js | 1 + lib/llparse/transform/base.js | 12 +++++++++ lib/llparse/transform/index.js | 3 +++ test/transform-test.js | 27 ++++++++++++++++++++ 17 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 lib/llparse/transform/base.js create mode 100644 lib/llparse/transform/index.js create mode 100644 test/transform-test.js diff --git a/lib/llparse.js b/lib/llparse.js index e241172..b36ea5d 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -5,6 +5,7 @@ const assert = require('assert'); const internal = require('./llparse/'); const kCode = Symbol('code'); +const kTransform = Symbol('transform'); const kPrefix = Symbol('prefix'); const kProperties = Symbol('properties'); @@ -40,11 +41,19 @@ class CodeAPI { } } +class TransformAPI { + toLower() { + return new internal.transform.Transform('toLower'); + } +} + class LLParse { constructor(prefix) { this[kPrefix] = prefix || 'llparse'; this[kCode] = new CodeAPI(this[kPrefix]); + this[kTransform] = new TransformAPI(); + this[kProperties] = { set: new Set(), list: [] @@ -57,6 +66,7 @@ class LLParse { get prefix() { return this[kPrefix]; } get code() { return this[kCode]; } + get transform() { return this[kTransform]; } node(name) { return new internal.node.Node(name); diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index a8efd22..2ebb914 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -250,6 +250,19 @@ class Compilation { }; } + buildTransform(transform, body, current) { + body.comment(`transform[${transform.name}]`); + if (transform.name === 'toLower') { + current = this.ir._('or', [ TYPE_INPUT.to, current ], + TYPE_INPUT.to.v(0x20)); + body.push(current); + } else { + throw new Error('Unsupported transform: ' + transform.name); + } + + return { body, current }; + } + declareField(type, name, init) { this.state.field(type, name); diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index a51d7f8..bdfa612 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -44,25 +44,38 @@ class MatchSequence extends compiler.Stage { [ TYPE_INPUT, ATTR_SEQUENCE ], INT ]); + + this.cache = new Map(); } build() { + return { + get: transform => this.get(transform) + }; + } + + get(transform = null) { + const cacheKey = transform === null ? null : transform.name; + if (this.cache.has(cacheKey)) + return this.cache.get(cacheKey); + const fn = this.ctx.ir.fn(this.signature, `${this.ctx.prefix}__match_sequence`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); - this.buildBody(fn); + this.buildBody(fn, transform); fn.visibility = 'internal'; fn.cconv = CCONV; fn.attributes = 'nounwind norecurse alwaysinline'; - this.ctx.matchSequence = fn; + this.cache.set(cacheKey, fn); + return fn; } // TODO(indutny): initialize `state.index` before calling matcher? - buildBody(fn) { + buildBody(fn, transform) { const body = fn.body; // Just to have a label @@ -87,7 +100,8 @@ class MatchSequence extends compiler.Stage { [ TYPE_INDEX, '[', index, ',', start.ref(), ']' ]); loop.push(indexPhi); - const iteration = this.buildIteration(fn, loop, indexPtr, posPhi, indexPhi); + const iteration = this.buildIteration(fn, loop, indexPtr, posPhi, indexPhi, + transform); // It is complete! this.ret(iteration.complete, iteration.pos, SEQUENCE_COMPLETE); @@ -105,14 +119,22 @@ class MatchSequence extends compiler.Stage { } - buildIteration(fn, body, indexField, pos, index) { + buildIteration(fn, body, indexField, pos, index, transform) { const seq = fn.arg(ARG_SEQUENCE); const seqLen = fn.arg(ARG_SEQUENCE_LEN); body.comment('current = *pos'); - const current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); + let current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); body.push(current); + // Transform the character if needed + if (transform) { + const res = this.ctx.buildTransform(transform, + body, current); + body = res.body; + current = res.current; + } + body.comment('expected = seq[state.index]'); const expectedPtr = IR._('getelementptr', seq.type.to, [ seq.type, seq ], diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index c3a735a..c8d71b0 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -52,6 +52,7 @@ class Node { this.name = name; this.otherwise = null; this.skip = false; + this.transform = null; this.fn = null; this.phis = new Map(); diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index fc8528b..a6c7d96 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -29,7 +29,8 @@ class Sequence extends node.Node { [ seq.type, seq ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); body.push(cast); - const matchSequence = ctx.compilation.stageResults['match-sequence']; + const matchSequence = ctx.compilation.stageResults['match-sequence'] + .get(this.transform); const returnType = matchSequence.type.ret; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 64ecd4e..195cb4a 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -15,9 +15,17 @@ class Single extends node.Node { const pos = ctx.pos.current; // Load the character - const current = ctx.ir._('load', pos.type.to, [ pos.type, pos ]); + let current = ctx.ir._('load', pos.type.to, [ pos.type, pos ]); body.push(current); + // Transform the character if needed + if (this.transform) { + const res = ctx.compilation.buildTransform(this.transform, + body, current); + body = res.body; + current = res.current; + } + const keys = this.children.map(child => child.key); const s = ctx.buildSwitch(body, pos.type.to, current, keys); diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index acc681b..85b1e86 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -11,6 +11,7 @@ const kMap = llparse.symbols.kMap; const kOtherwise = llparse.symbols.kOtherwise; const kSignature = llparse.symbols.kSignature; const kSpan = llparse.symbols.kSpan; +const kTransform = llparse.symbols.kTransform; class NodeTranslator extends compiler.Stage { constructor(ctx) { @@ -48,19 +49,17 @@ class NodeTranslator extends compiler.Stage { let res; let list; + let hasTrie = false; if (node instanceof llparse.node.Invoke) { - assert.strictEqual(node[kCases].length, 0); res = new compiler.node.Invoke(this.id(node), node[kCode]); } else if (node instanceof llparse.node.Error) { - assert.strictEqual(node[kCases].length, 0); res = new compiler.node.Error(this.id(node), node.code, node.reason); } else if (node instanceof llparse.node.SpanStart) { - assert.strictEqual(node[kCases].length, 0); res = new compiler.node.SpanStart(this.id(node), node[kSpan][kCode]); } else if (node instanceof llparse.node.SpanEnd) { - assert.strictEqual(node[kCases].length, 0); res = new compiler.node.SpanEnd(this.id(node), node[kSpan][kCode]); } else { + hasTrie = true; const trie = new llparse.Trie(node.name); const combined = trie.combine(node[kCases]); @@ -72,6 +71,11 @@ class NodeTranslator extends compiler.Stage { } } + if (!hasTrie) { + assert.strictEqual(node[kTransform], null); + assert.strictEqual(node[kCases].length, 0); + } + // Prevent loops this.nodes.set(node, res); @@ -120,6 +124,7 @@ class NodeTranslator extends compiler.Stage { if (isRoot) this.nodes.set(node, res); + res.transform = node[kTransform]; res.children = trie.children.map((child) => { const value = child.child.type === 'next' ? child.child.value : null; return { @@ -142,6 +147,7 @@ class NodeTranslator extends compiler.Stage { if (isRoot) this.nodes.set(node, res); + res.transform = node[kTransform]; res.next = this.buildTrie(node, trie.child, list); res.value = value; diff --git a/lib/llparse/index.js b/lib/llparse/index.js index 33f60b7..d3caa6c 100644 --- a/lib/llparse/index.js +++ b/lib/llparse/index.js @@ -7,6 +7,7 @@ exports.utils = require('./utils'); exports.case = require('./case'); exports.code = require('./code'); exports.node = require('./node'); +exports.transform = require('./transform'); exports.Span = require('./span'); exports.Trie = require('./trie'); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 9f3e4b5..64c2631 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -8,6 +8,7 @@ const llparse = require('../'); const kOtherwise = llparse.symbols.kOtherwise; const kCases = llparse.symbols.kCases; const kSignature = llparse.symbols.kSignature; +const kTransform = llparse.symbols.kTransform; const kName = Symbol('name'); const kCheckIsMatch = Symbol('checkIsMatch'); @@ -16,6 +17,7 @@ class Node { constructor(name, signature = 'match') { this[kName] = name; this[kSignature] = signature; + this[kTransform] = null; this[kCases] = []; @@ -24,6 +26,15 @@ class Node { get name() { return this[kName]; } + transform(t) { + assert.strictEqual(this[kTransform], null, 'Can\'t apply transform twice'); + assert(t instanceof llparse.transform.Transform, + '`.transform()` argument must be a `Transform` instance'); + this[kTransform] = t; + + return this; + } + peek(value, next) { // .peek([ ... ], next) if (Array.isArray(value)) { diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index 70bba21..e57a26b 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -16,6 +16,7 @@ class Error extends node.Node { get code() { return this[kCode]; } get reason() { return this[kReason]; } + transform() { throw new Error('Not supported'); } peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 28e923f..50af3d1 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -38,6 +38,7 @@ class Invoke extends node.Node { this.otherwise(otherwise); } + transform() { throw new Error('Not supported'); } peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js index ceb02a9..05a09b5 100644 --- a/lib/llparse/node/span-end.js +++ b/lib/llparse/node/span-end.js @@ -11,6 +11,7 @@ class SpanEnd extends node.Node { this[kSpan] = span; } + transform() { throw new Error('Not supported'); } peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js index a7e6484..73c4508 100644 --- a/lib/llparse/node/span-start.js +++ b/lib/llparse/node/span-start.js @@ -12,6 +12,7 @@ class SpanStart extends node.Node { this[kSpan] = span; } + transform() { throw new Error('Not supported'); } peek() { throw new Error('Not supported'); } match() { throw new Error('Not supported'); } select() { throw new Error('Not supported'); } diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index abe4add..125d4da 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -9,4 +9,5 @@ exports.kNoAdvance = Symbol('noAdvance'); exports.kOtherwise = Symbol('otherwise'); exports.kSignature = Symbol('signature'); exports.kSpan = Symbol('span'); +exports.kTransform = Symbol('transform'); exports.kType = Symbol('type'); diff --git a/lib/llparse/transform/base.js b/lib/llparse/transform/base.js new file mode 100644 index 0000000..29c67f2 --- /dev/null +++ b/lib/llparse/transform/base.js @@ -0,0 +1,12 @@ +'use strict'; + +const kName = Symbol('name'); + +class Transform { + constructor(name) { + this[kName] = name; + } + + get name() { return this[kName]; } +} +module.exports = Transform; diff --git a/lib/llparse/transform/index.js b/lib/llparse/transform/index.js new file mode 100644 index 0000000..99514d5 --- /dev/null +++ b/lib/llparse/transform/index.js @@ -0,0 +1,3 @@ +'use strict'; + +exports.Transform = require('./base'); diff --git a/test/transform-test.js b/test/transform-test.js new file mode 100644 index 0000000..4dd2af5 --- /dev/null +++ b/test/transform-test.js @@ -0,0 +1,27 @@ +'use strict'; +/* global describe it beforeEach */ + +const llparse = require('../'); + +const fixtures = require('./fixtures'); + +describe('LLParse/transform', () => { + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); + + it('should apply transformation before the match', (callback) => { + const start = p.node('start'); + + start + .transform(p.transform.toLower()) + .match('connect', fixtures.printOff(p, start)) + .match('close', fixtures.printOff(p, start)) + .otherwise(p.error(1, 'error')); + + const binary = fixtures.build('transform-lower', p.build(start)); + + binary('connectCLOSEcOnNeCt', 'off=7\noff=12\noff=19\n', callback); + }); +}); From 91056c024db87d0642791ed9b6b81570199b0bb4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 15:56:52 -0500 Subject: [PATCH 155/281] 2.0.0-beta6 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 707edf4..4390094 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta5", + "version": "2.0.0-beta6", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 8181d03..cb6eef6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta5", + "version": "2.0.0-beta6", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 69b483b3ed3f0d385976b0a4fd4dc20bbf2e1217 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 15:58:44 -0500 Subject: [PATCH 156/281] match-sequence: fix for name collision --- lib/llparse/compiler/match-sequence.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index bdfa612..70fc4e1 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -59,8 +59,10 @@ class MatchSequence extends compiler.Stage { if (this.cache.has(cacheKey)) return this.cache.get(cacheKey); + const postfix = cacheKey ? '_' + cacheKey.toLowerCase() : ''; + const fn = this.ctx.ir.fn(this.signature, - `${this.ctx.prefix}__match_sequence`, + `${this.ctx.prefix}__match_sequence${postfix}`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); this.buildBody(fn, transform); From 5ebcf0f46f0ff5509183837c0e8b0fa4cb975d47 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 15:58:59 -0500 Subject: [PATCH 157/281] 2.0.0-beta7 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4390094..d072580 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta6", + "version": "2.0.0-beta7", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index cb6eef6..e26c860 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta6", + "version": "2.0.0-beta7", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From efc0a67f6e43edf9664d4c293c56c9cf1fdd56c6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 16:20:21 -0500 Subject: [PATCH 158/281] package: bump deps --- package-lock.json | 58 ++++++++++++++++++++++++----------------------- package.json | 4 ++-- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index d072580..45f0b6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -134,12 +134,6 @@ "requires": { "ansi-regex": "2.1.1" } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "dev": true } } }, @@ -200,12 +194,6 @@ "color-convert": "1.9.1" } }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true - }, "supports-color": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.2.0.tgz", @@ -357,9 +345,9 @@ "dev": true }, "eslint": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.18.0.tgz", - "integrity": "sha512-Ep2lUbztzXLg0gNUl48I1xvbQFy1QuWyh1C9PSympmln33jwOr8B3QfuEcXpPPE4uSwEzDaWhUxBN0sNQkzrBg==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.18.1.tgz", + "integrity": "sha512-gPSfpSRCHre1GLxGmO68tZNxOlL2y7xBd95VcLD+Eo4S2js31YoMum3CAQIOaxY24hqYOMksMvW38xuuWKQTgw==", "dev": true, "requires": { "ajv": "5.5.2", @@ -592,9 +580,9 @@ } }, "has-flag": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", - "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true }, "he": { @@ -803,9 +791,9 @@ } }, "mocha": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.0.0.tgz", - "integrity": "sha512-ukB2dF+u4aeJjc6IGtPNnJXfeby5d4ZqySlIBT0OEyva/DrMjVm5HkQxKnHDLKEfEQBsEnwTg9HHhtPHJdTd8w==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.0.1.tgz", + "integrity": "sha512-SpwyojlnE/WRBNGtvJSNfllfm5PqEDFxcWluSIgLeSBJtXG4DmoX2NNAeEA7rP5kK+79VgtVq8nG6HskaL1ykg==", "dev": true, "requires": { "browser-stdout": "1.3.0", @@ -818,6 +806,23 @@ "he": "1.1.1", "mkdirp": "0.5.1", "supports-color": "4.4.0" + }, + "dependencies": { + "has-flag": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", + "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", + "dev": true + }, + "supports-color": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", + "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "dev": true, + "requires": { + "has-flag": "2.0.0" + } + } } }, "ms": { @@ -1110,13 +1115,10 @@ "dev": true }, "supports-color": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", - "dev": true, - "requires": { - "has-flag": "2.0.0" - } + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true }, "table": { "version": "4.0.2", diff --git a/package.json b/package.json index e26c860..8742ee0 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,8 @@ }, "devDependencies": { "async": "^2.6.0", - "eslint": "^4.18.0", - "mocha": "^5.0.0" + "eslint": "^4.18.1", + "mocha": "^5.0.1" }, "directories": { "lib": "lib", From 8722aaee22322ed1a0a1c27f29a6dd292a06234f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 17:11:05 -0500 Subject: [PATCH 159/281] transform: rename `.toLower` => `.toLowerUnsafe` --- lib/llparse.js | 4 ++-- lib/llparse/compiler/compilation.js | 2 +- test/transform-test.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/llparse.js b/lib/llparse.js index b36ea5d..ef03caf 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -42,8 +42,8 @@ class CodeAPI { } class TransformAPI { - toLower() { - return new internal.transform.Transform('toLower'); + toLowerUnsafe() { + return new internal.transform.Transform('to_lower_unsafe'); } } diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 2ebb914..7717a06 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -252,7 +252,7 @@ class Compilation { buildTransform(transform, body, current) { body.comment(`transform[${transform.name}]`); - if (transform.name === 'toLower') { + if (transform.name === 'to_lower_unsafe') { current = this.ir._('or', [ TYPE_INPUT.to, current ], TYPE_INPUT.to.v(0x20)); body.push(current); diff --git a/test/transform-test.js b/test/transform-test.js index 4dd2af5..cb96e47 100644 --- a/test/transform-test.js +++ b/test/transform-test.js @@ -15,7 +15,7 @@ describe('LLParse/transform', () => { const start = p.node('start'); start - .transform(p.transform.toLower()) + .transform(p.transform.toLowerUnsafe()) .match('connect', fixtures.printOff(p, start)) .match('close', fixtures.printOff(p, start)) .otherwise(p.error(1, 'error')); From ac641f241df70471ed61247368948e006a80c9b2 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 17:11:20 -0500 Subject: [PATCH 160/281] 2.0.0-beta8 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 45f0b6a..10613cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta7", + "version": "2.0.0-beta8", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 8742ee0..9280a79 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta7", + "version": "2.0.0-beta8", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 6ec0edcc842e9195ef7d452d72a2a0fc4250d025 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 18:45:25 -0500 Subject: [PATCH 161/281] llparse: restrict `.property()` types --- README.md | 2 +- examples/http/index.js | 2 +- lib/llparse.js | 10 +++++++--- lib/llparse/compiler/compilation.js | 9 ++++++--- lib/llparse/constants.js | 8 ++++++++ test/api-test.js | 2 +- test/span-test.js | 2 +- 7 files changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 36d7867..6c0dc32 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ const url = p.node('url'); const http = p.node('http'); // Add custom uint8_t property to the state -p.property(ir => ir.i(8), 'method'); +p.property('i8', 'method'); // Store method inside a custom property const onMethod = p.invoke(p.code.store('method'), beforeUrl); diff --git a/examples/http/index.js b/examples/http/index.js index a6fb838..8cd14dc 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -9,7 +9,7 @@ const url = p.node('url'); const http = p.node('http'); // Add custom uint8_t property to the state -p.property(ir => ir.i(8), 'method'); +p.property('i8', 'method'); // Store method inside a custom property const onMethod = p.invoke(p.code.store('method'), beforeUrl); diff --git a/lib/llparse.js b/lib/llparse.js index ef03caf..3702895 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -81,10 +81,10 @@ class LLParse { } property(type, name) { - assert.strictEqual(typeof type, 'function', - 'The first argument of `.property()` must be a function'); + assert.strictEqual(typeof type, 'string', + 'The first argument of `.property()` must be a type name'); assert.strictEqual(typeof name, 'string', - 'The second argument of `.property()` must be a string'); + 'The second argument of `.property()` must be a property name'); if (internal.constants.RESERVED_PROPERTY_NAMES.has(name)) throw new Error(`Can't use reserved property name: "${name}"`); @@ -95,6 +95,10 @@ class LLParse { if (props.set.has(name)) throw new Error(`Duplicate property with a name: "${name}"`); + if (!internal.constants.USER_TYPES.hasOwnProperty(type)) + throw new Error(`Unknown property type: "${type}"`); + type = internal.constants.USER_TYPES[type]; + const prop = new internal.Property(type, name); props.set.add(name); props.list.push(prop); diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 7717a06..0b917ef 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -72,9 +72,12 @@ class Compilation { // Custom fields this.options.properties.forEach((prop) => { - // TODO(indutny): support non-int types? - this.declareField(prop.type(this.ir, this.state), prop.name, - type => type.v(0)); + this.declareField(prop.type, prop.name, (type) => { + if (type.isPointer()) + return type.v(null); + else + return type.v(0); + }); }); this.codeCache = new Map(); diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 7a39a56..c2dfe92 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -43,3 +43,11 @@ exports.RESERVED_PROPERTY_NAMES = new Set([ 'data', 'mark' ]); + +exports.USER_TYPES = { + i8: IR.i(8), + i16: IR.i(16), + i32: IR.i(32), + i64: IR.i(64), + 'ptr': IR.i(8).ptr() +}; diff --git a/test/api-test.js b/test/api-test.js index 04efd19..3838335 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -105,7 +105,7 @@ describe('LLParse', () => { const start = p.node('start'); const error = p.error(3, 'Invalid word'); - p.property(ir => ir.i(8), 'custom'); + p.property('i8', 'custom'); const second = p.invoke(p.code.load('custom'), { 0: p.invoke(p.code.match('print_zero'), { 0: start }, error), diff --git a/test/span-test.js b/test/span-test.js index ab76560..075f905 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -75,7 +75,7 @@ describe('LLParse/span', () => { const start = p.node('start'); const span = p.span(p.code.span('on_data')); - p.property(ir => ir.i(8), 'custom'); + p.property('i8', 'custom'); start.otherwise(p.invoke(p.code.load('custom'), { 0: span.end().otherwise(start) From 19a3d6968628eb5a8e8b05c423fb63a655a39fc5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 18:58:12 -0500 Subject: [PATCH 162/281] compilation: organize stages into before/after --- lib/llparse/compiler/compilation.js | 31 ++++++++++++++++++----------- lib/llparse/compiler/compiler.js | 29 ++++++++++++++------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 0b917ef..1edf62e 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -30,13 +30,13 @@ const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; class Compilation { - constructor(prefix, options, root) { - this.options = options || {}; + constructor(options) { + this.options = Object.assign({}, options); this.ir = new IR(); - this.prefix = prefix; + this.prefix = this.options.prefix; - this.root = root; + this.root = this.options.root; this.state = this.ir.struct(`${this.prefix}_state`); this.initializers = []; @@ -67,6 +67,16 @@ class Compilation { this.declareField(TYPE_REASON, 'reason', type => type.v(null)); this.declareField(TYPE_INDEX, 'index', type => type.v(0)); + this.codeCache = new Map(); + this.debugMethod = null; + + // Intermediate results from various build stages + this.stageResults = {}; + } + + build() { + this.buildStages(this.options.before); + // Public fields this.declareField(TYPE_DATA, 'data', type => type.v(null)); @@ -80,19 +90,16 @@ class Compilation { }); }); - this.codeCache = new Map(); - this.debugMethod = null; - - // Intermediate results from various build stages - this.stageResults = {}; + this.buildStages(this.options.after); } buildStages(stages) { - stages.forEach(stage => this.buildStage(stage)); + stages.forEach(Stage => this.buildStage(Stage)); } - buildStage(stage) { - this.stageResults[stage.name] = stage.build(this); + buildStage(Stage) { + const stage = new Stage(this); + this.stageResults[stage.name] = stage.build(); } buildCode(code) { diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 7681bd7..5bf0fda 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -29,20 +29,21 @@ class Compiler { } build(root) { - const ctx = new llparse.compiler.Compilation(this.prefix, this.options, - root); - - // TODO(indutny): detect and report loops - - const stages = [ - new llparse.compiler.MatchSequence(ctx), - new llparse.compiler.span.Allocator(ctx), - new llparse.compiler.span.Builder(ctx), - new llparse.compiler.node.Translator(ctx), - new llparse.compiler.node.Builder(ctx) - ]; - - ctx.buildStages(stages); + const compOpts = Object.assign({}, this.options, { + root, + before: [ + llparse.compiler.MatchSequence, + llparse.compiler.span.Allocator, + llparse.compiler.span.Builder, + llparse.compiler.node.Translator + ], + after: [ + llparse.compiler.node.Builder + ] + }); + const ctx = new llparse.compiler.Compilation(compOpts); + + ctx.build(); this.buildInit(ctx); this.buildExecute(ctx); From 67c7ad1bc23fd012d04d5c7a43681a8ec8a3d5ab Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 19:39:30 -0500 Subject: [PATCH 163/281] compilation: simplify, user fields after internal --- lib/llparse/compiler/compilation.js | 17 ++++++++--------- lib/llparse/compiler/compiler.js | 6 ++---- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 1edf62e..e29f1e3 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -60,13 +60,6 @@ class Compilation { }; this.signature.callback.span = this.signature.callback.match; - this.declareField(this.signature.node.ptr(), 'current', - (type, ctx) => ctx.stageResults['node-builder'].ref()); - - this.declareField(TYPE_ERROR, 'error', type => type.v(0)); - this.declareField(TYPE_REASON, 'reason', type => type.v(null)); - this.declareField(TYPE_INDEX, 'index', type => type.v(0)); - this.codeCache = new Map(); this.debugMethod = null; @@ -75,7 +68,13 @@ class Compilation { } build() { - this.buildStages(this.options.before); + // Private fields + this.declareField(this.signature.node.ptr(), 'current', + (type, ctx) => ctx.stageResults['node-builder'].ref()); + + this.declareField(TYPE_ERROR, 'error', type => type.v(0)); + this.declareField(TYPE_REASON, 'reason', type => type.v(null)); + this.declareField(TYPE_INDEX, 'index', type => type.v(0)); // Public fields this.declareField(TYPE_DATA, 'data', type => type.v(null)); @@ -90,7 +89,7 @@ class Compilation { }); }); - this.buildStages(this.options.after); + this.buildStages(this.options.stages); } buildStages(stages) { diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 5bf0fda..ae7759c 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -31,13 +31,11 @@ class Compiler { build(root) { const compOpts = Object.assign({}, this.options, { root, - before: [ + stages: [ llparse.compiler.MatchSequence, llparse.compiler.span.Allocator, llparse.compiler.span.Builder, - llparse.compiler.node.Translator - ], - after: [ + llparse.compiler.node.Translator, llparse.compiler.node.Builder ] }); From 12f948555f5d02146326f339a1f629ff111b9f03 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 20:04:32 -0500 Subject: [PATCH 164/281] lib: build C header --- README.md | 8 +++++- examples/http/Makefile | 8 +++--- examples/http/http_parser.h | 20 +++++++++++++ examples/http/index.js | 9 +++++- examples/http/main.c | 18 +----------- lib/llparse.js | 4 +-- lib/llparse/compiler/compilation.js | 40 +++++++++++++++++++++++--- lib/llparse/compiler/compiler.js | 37 ++++++++++++++++++++---- lib/llparse/compiler/match-sequence.js | 2 +- lib/llparse/constants.js | 9 ------ test/fixtures/index.js | 11 +++++-- test/fixtures/main.c | 39 ++++++++++--------------- 12 files changed, 132 insertions(+), 73 deletions(-) create mode 100644 examples/http/http_parser.h diff --git a/README.md b/README.md index 6c0dc32..80fd463 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,13 @@ http .match(' HTTP/1.1\r\n\r\n', complete) .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); -console.log(p.build(method)); +const artifacts = p.build(method); +console.log('----- LLVM -----'); +console.log(artifacts.llvm); +console.log('----- LLVM END -----'); +console.log('----- HEADER -----'); +console.log(artifacts.header); +console.log('----- HEADER END -----'); ``` #### LICENSE diff --git a/examples/http/Makefile b/examples/http/Makefile index 8dd1cad..f23dc94 100644 --- a/examples/http/Makefile +++ b/examples/http/Makefile @@ -4,10 +4,10 @@ all: http # NOTE: -flto has a bug, and can't be used right now # See: https://bugs.llvm.org/show_bug.cgi?id=36441 -http: main.c http.ll - $(CC) -g3 -Os -fvisibility=hidden -Wall http.ll main.c -o $@ +http: main.c http_parser.ll + $(CC) -g3 -Os -fvisibility=hidden -Wall -I. http_parser.ll main.c -o $@ -http.ll: index.js - node $< > $@ +http_parser.ll: index.js + node $< .PHONY = all diff --git a/examples/http/http_parser.h b/examples/http/http_parser.h new file mode 100644 index 0000000..89405d9 --- /dev/null +++ b/examples/http/http_parser.h @@ -0,0 +1,20 @@ +#ifndef LLPARSE_HEADER_HTTP_PARSER +#define LLPARSE_HEADER_HTTP_PARSER + +#include + +typedef struct http_parser_state_s http_parser_state_t; +struct http_parser_state_s { + void* _current; + uint32_t _index; + uint32_t error; + void* reason; + void* data; + uint8_t method; + void* _span_start0; +}; + +void http_parser_init(http_parser_state_t* s); +int http_parser_execute(http_parser_state_t* s, const char* p, const char* endp); + +#endif /* LLPARSE_HEADER_HTTP_PARSER */ diff --git a/examples/http/index.js b/examples/http/index.js index 8cd14dc..65d7839 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -41,4 +41,11 @@ http .match(' HTTP/1.1\n\n', complete) .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); -console.log(p.build(method)); +// Build + +const fs = require('fs'); +const path = require('path'); + +const artifacts = p.build(method); +fs.writeFileSync(path.join(__dirname, 'http_parser.h'), artifacts.header); +fs.writeFileSync(path.join(__dirname, 'http_parser.ll'), artifacts.llvm); diff --git a/examples/http/main.c b/examples/http/main.c index 7e6fe0c..358df99 100644 --- a/examples/http/main.c +++ b/examples/http/main.c @@ -1,25 +1,9 @@ -#include #include #include #include #include -typedef struct http_parser_state_s http_parser_state_t; - -struct http_parser_state_s { - void* current; - int error; - const char* reason; - int index; - void* data; - - uint8_t method; - void* mark; -}; - -void http_parser_init(http_parser_state_t* s); -int http_parser_execute(http_parser_state_t* s, const char* p, - const char* endp); +#include "http_parser.h" int on_url(http_parser_state_t* s, const char* p, const char* endp) { if (p == endp) diff --git a/lib/llparse.js b/lib/llparse.js index 3702895..7d7136d 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -86,10 +86,8 @@ class LLParse { assert.strictEqual(typeof name, 'string', 'The second argument of `.property()` must be a property name'); - if (internal.constants.RESERVED_PROPERTY_NAMES.has(name)) - throw new Error(`Can't use reserved property name: "${name}"`); if (/^_/.test(name)) - throw new Error(`Can't use reserved property name: "${name}"`); + throw new Error(`Can't use internal property name: "${name}"`); const props = this[kProperties]; if (props.set.has(name)) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index e29f1e3..8a1db2d 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -69,14 +69,14 @@ class Compilation { build() { // Private fields - this.declareField(this.signature.node.ptr(), 'current', + this.declareField(this.signature.node.ptr(), '_current', (type, ctx) => ctx.stageResults['node-builder'].ref()); - this.declareField(TYPE_ERROR, 'error', type => type.v(0)); - this.declareField(TYPE_REASON, 'reason', type => type.v(null)); - this.declareField(TYPE_INDEX, 'index', type => type.v(0)); + this.declareField(TYPE_INDEX, '_index', type => type.v(0)); // Public fields + this.declareField(TYPE_ERROR, 'error', type => type.v(0)); + this.declareField(TYPE_REASON, 'reason', type => type.v(null)); this.declareField(TYPE_DATA, 'data', type => type.v(null)); // Custom fields @@ -92,6 +92,38 @@ class Compilation { this.buildStages(this.options.stages); } + buildCState() { + const out = []; + + out.push(`typedef struct ${this.prefix}_state_s ${this.prefix}_state_t;`); + out.push(`struct ${this.prefix}_state_s {`); + + this.state.fields.forEach((field) => { + let type = field.type; + if (type.isPointer()) { + type = 'void*'; + } else { + assert(type.isInt(), 'Unsupported type: ' + type.type); + if (type.width === 8) + type = 'uint8_t'; + else if (type.width === 16) + type = 'uint16_t'; + else if (type.width === 32) + type = 'uint32_t'; + else if (type.width === 64) + type = 'uint64_t'; + else + throw new Error('Unsupported type width: ' + type.width); + } + + out.push(` ${type} ${field.name};`); + }); + + out.push('};'); + + return out.join('\n'); + } + buildStages(stages) { stages.forEach(Stage => this.buildStage(Stage)); } diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index ae7759c..9e59935 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -43,10 +43,32 @@ class Compiler { ctx.build(); - this.buildInit(ctx); - this.buildExecute(ctx); - - return ctx.ir.build(); + const init = this.buildInit(ctx); + const execute = this.buildExecute(ctx); + + const out = { + header: '', + llvm: '' + }; + + const def = 'LLPARSE_HEADER_' + + this.prefix.toUpperCase().replace(/[^A-Z0-9]/g, '_'); + + out.header += `#ifndef ${def}\n`; + out.header += `#define ${def}\n`; + out.header += '\n'; + out.header += '#include \n'; + out.header += '\n'; + out.header += ctx.buildCState() + '\n'; + out.header += '\n'; + out.header += init + '\n'; + out.header += execute + '\n'; + out.header += '\n'; + out.header += `#endif /* ${def} */\n`; + + out.llvm += ctx.ir.build(); + + return out; } buildInit(ctx) { @@ -57,7 +79,7 @@ class Compiler { init.body.terminate('ret', ctx.ir.void()); - return init; + return `void ${this.prefix}_init(${this.prefix}_state_t* s);`; } buildExecute(ctx) { @@ -77,7 +99,7 @@ class Compiler { const nodeSig = ctx.signature.node; - const currentPtr = ctx.field(execute, 'current'); + const currentPtr = ctx.field(execute, '_current'); body.push(currentPtr); const current = ctx.ir._('load', nodeSig.ptr(), [ nodeSig.ptr().ptr(), currentPtr ]); @@ -124,6 +146,9 @@ class Compiler { [ nodeSig.ptr().ptr(), currentPtr ]).void()); error.terminate('ret', [ TYPE_ERROR, code ]); + + return `int ${this.prefix}_execute(${this.prefix}_state_t* s, ` + + 'const char* p, const char* endp);'; } } module.exports = Compiler; diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index 70fc4e1..95eca9d 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -86,7 +86,7 @@ class MatchSequence extends compiler.Stage { // Load `state.index` start.comment('index = state.index'); - const indexPtr = this.ctx.field(fn, 'index'); + const indexPtr = this.ctx.field(fn, '_index'); start.push(indexPtr); const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ]); start.push(index); diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index c2dfe92..df0d1bf 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -35,15 +35,6 @@ exports.SEQUENCE_MISMATCH = 2; exports.SPAN_START_PREFIX = '_span_start'; exports.SPAN_CB_PREFIX = '_span_cb'; -exports.RESERVED_PROPERTY_NAMES = new Set([ - 'current', - 'error', - 'reason', - 'index', - 'data', - 'mark' -]); - exports.USER_TYPES = { i8: IR.i(8), i16: IR.i(16), diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 1db7a69..a5ad1ee 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -72,11 +72,16 @@ exports.build = (name, source, options) => { } const file = path.join(TMP_DIR, name + '.ll'); + const header = path.join(TMP_DIR, name + '.h'); const out = path.join(TMP_DIR, name); - fs.writeFileSync(file, source); - const ret = spawnSync(CLANG, - [ '-g3', '-Os', '-fvisibility=hidden', MAIN, file, '-o', out ]); + fs.writeFileSync(file, source.llvm); + fs.writeFileSync(header, source.header); + + const ret = spawnSync(CLANG, [ + '-g3', '-Os', '-fvisibility=hidden', + '-include', header, MAIN, file, '-o', out + ]); if (ret.status !== 0) { process.stderr.write(ret.stdout); process.stderr.write(ret.stderr); diff --git a/test/fixtures/main.c b/test/fixtures/main.c index 2804aa9..dcc9876 100644 --- a/test/fixtures/main.c +++ b/test/fixtures/main.c @@ -1,18 +1,9 @@ -#include #include #include #include #include -struct state { - void* current; - int error; - const char* reason; - int index; - - /* Give huge leeway */ - void* fields[1024]; -}; +/* NOTE: include is inserted through `-include` clang argument */ /* 8 gb */ static const int64_t kBytes = 8589934592LL; @@ -20,17 +11,15 @@ static const int64_t kBytes = 8589934592LL; static int bench = 0; static const char* start; -void llparse_init(struct state* s); -int llparse_execute(struct state* s, const char* p, const char* endp); - -void debug(struct state* s, const char* p, const char* endp, const char* msg) { +void debug(llparse_state_t* s, const char* p, const char* endp, + const char* msg) { if (bench) return; fprintf(stderr, "off=%d > %s\n", (int) (p - start), msg); } -int print_zero(struct state* s, const char* p, const char* endp) { +int print_zero(llparse_state_t* s, const char* p, const char* endp) { if (bench) return 0; @@ -38,7 +27,7 @@ int print_zero(struct state* s, const char* p, const char* endp) { return 0; } -int print_one(struct state* s, const char* p, const char* endp) { +int print_one(llparse_state_t* s, const char* p, const char* endp) { if (bench) return 0; @@ -46,7 +35,7 @@ int print_one(struct state* s, const char* p, const char* endp) { return 0; } -int print_off(struct state* s, const char* p, const char* endp) { +int print_off(llparse_state_t* s, const char* p, const char* endp) { if (bench) return 0; @@ -54,7 +43,8 @@ int print_off(struct state* s, const char* p, const char* endp) { return 0; } -int print_match(struct state* s, const char* p, const char* endp, int value) { +int print_match(llparse_state_t* s, const char* p, const char* endp, + int value) { if (bench) return 0; @@ -62,7 +52,8 @@ int print_match(struct state* s, const char* p, const char* endp, int value) { return 0; } -int return_match(struct state* s, const char* p, const char* endp, int value) { +int return_match(llparse_state_t* s, const char* p, const char* endp, + int value) { if (bench) return value; @@ -78,23 +69,23 @@ static void print_span(const char* name, const char* p, const char* endp) { (int) (endp - p), name, (int) (endp - p), p); } -int on_dot(struct state* s, const char* p, const char* endp) { +int on_dot(llparse_state_t* s, const char* p, const char* endp) { print_span("dot", p, endp); return 0; } -int on_dash(struct state* s, const char* p, const char* endp) { +int on_dash(llparse_state_t* s, const char* p, const char* endp) { print_span("dash", p, endp); return 0; } -int on_underscore(struct state* s, const char* p, const char* endp) { +int on_underscore(llparse_state_t* s, const char* p, const char* endp) { print_span("underscore", p, endp); return 0; } static int run_bench(const char* input, int len) { - struct state s; + llparse_state_t s; int64_t i; struct timeval start; struct timeval end; @@ -130,7 +121,7 @@ static int run_bench(const char* input, int len) { static int run_scan(int scan, const char* input, int len) { - struct state s; + llparse_state_t s; llparse_init(&s); if (scan <= 0) { From 893dc90a751d6a26195975c264bb294123f4481d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 20:04:53 -0500 Subject: [PATCH 165/281] 2.0.0-beta9 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 10613cb..d1632a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta8", + "version": "2.0.0-beta9", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 9280a79..f28364b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta8", + "version": "2.0.0-beta9", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 220ad28a30657c378c177ccce6833bcab2ee33f8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 20:22:23 -0500 Subject: [PATCH 166/281] compiler: return of stage split --- lib/llparse/compiler/compilation.js | 6 +++++- lib/llparse/compiler/compiler.js | 18 +++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 8a1db2d..a8caebe 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -74,6 +74,9 @@ class Compilation { this.declareField(TYPE_INDEX, '_index', type => type.v(0)); + // Some stages may add more private fields + this.buildStages(this.options.stages.before); + // Public fields this.declareField(TYPE_ERROR, 'error', type => type.v(0)); this.declareField(TYPE_REASON, 'reason', type => type.v(null)); @@ -89,7 +92,8 @@ class Compilation { }); }); - this.buildStages(this.options.stages); + // Some stages may add more private fields + this.buildStages(this.options.stages.after); } buildCState() { diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 9e59935..0f41205 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -31,13 +31,17 @@ class Compiler { build(root) { const compOpts = Object.assign({}, this.options, { root, - stages: [ - llparse.compiler.MatchSequence, - llparse.compiler.span.Allocator, - llparse.compiler.span.Builder, - llparse.compiler.node.Translator, - llparse.compiler.node.Builder - ] + stages: { + before: [ + llparse.compiler.MatchSequence, + llparse.compiler.span.Allocator, + llparse.compiler.span.Builder, + llparse.compiler.node.Translator + ], + after: [ + llparse.compiler.node.Builder + ] + } }); const ctx = new llparse.compiler.Compilation(compOpts); From 3c8431ef8c456a27093234dee982d99deae192ca Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 20:22:37 -0500 Subject: [PATCH 167/281] 2.0.0-beta10 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index d1632a1..f0e435f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta9", + "version": "2.0.0-beta10", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f28364b..3235f5a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta9", + "version": "2.0.0-beta10", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 1718a119094894731d42a5c95f20d4f8bdaaab4d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 20:25:11 -0500 Subject: [PATCH 168/281] gitignore: ignore headers --- .gitignore | 1 + examples/http/http_parser.h | 20 -------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 examples/http/http_parser.h diff --git a/.gitignore b/.gitignore index e66a20b..b311d4a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ examples/http/http *.ll *.bc *.o +*.h main 1 diff --git a/examples/http/http_parser.h b/examples/http/http_parser.h deleted file mode 100644 index 89405d9..0000000 --- a/examples/http/http_parser.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef LLPARSE_HEADER_HTTP_PARSER -#define LLPARSE_HEADER_HTTP_PARSER - -#include - -typedef struct http_parser_state_s http_parser_state_t; -struct http_parser_state_s { - void* _current; - uint32_t _index; - uint32_t error; - void* reason; - void* data; - uint8_t method; - void* _span_start0; -}; - -void http_parser_init(http_parser_state_t* s); -int http_parser_execute(http_parser_state_t* s, const char* p, const char* endp); - -#endif /* LLPARSE_HEADER_HTTP_PARSER */ From 94a02b1e8c9d178fa7327b13052c1106fc23d8f3 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 22:45:17 -0500 Subject: [PATCH 169/281] test: port to `llparse-test-fixture` --- package-lock.json | 9 +++ package.json | 11 ++- test/api-test.js | 30 +++---- test/fixtures/extra.c | 41 ++++++++++ test/fixtures/index.js | 129 ++--------------------------- test/fixtures/main.c | 178 ----------------------------------------- test/span-test.js | 16 ++-- test/transform-test.js | 2 +- 8 files changed, 90 insertions(+), 326 deletions(-) create mode 100644 test/fixtures/extra.c delete mode 100644 test/fixtures/main.c diff --git a/package-lock.json b/package-lock.json index f0e435f..cfc0521 100644 --- a/package-lock.json +++ b/package-lock.json @@ -739,6 +739,15 @@ "type-check": "0.3.2" } }, + "llparse-test-fixture": { + "version": "1.0.0-beta1", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta1.tgz", + "integrity": "sha512-jFXs0mzyDl8goVeASK1rHx6+bIgAiBe50Sm0IzKJH5lo+pJPj3Qq9XHJOCnfCktYrzLIH9vggWUnBCfGM7YQuQ==", + "dev": true, + "requires": { + "async": "2.6.0" + } + }, "llvm-ir": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.4.2.tgz", diff --git a/package.json b/package.json index 3235f5a..c0be42a 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,15 @@ "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", "test": "mocha --reporter=spec test/*-test.js && npm run lint" }, - "keywords": [], + "files": [ + "lib" + ], + "keywords": [ + "llparse", + "llvm", + "ir", + "dfa" + ], "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { @@ -15,6 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", + "llparse-test-fixture": "^1.0.0-beta1", "mocha": "^5.0.1" }, "directories": { diff --git a/test/api-test.js b/test/api-test.js index 3838335..624f080 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -5,8 +5,8 @@ const llparse = require('../'); const fixtures = require('./fixtures'); -const printMatch = fixtures.printMatch; const printOff = fixtures.printOff; +const printMatch = fixtures.printMatch; describe('LLParse', () => { let p; @@ -29,7 +29,7 @@ describe('LLParse', () => { start.otherwise(p.error(3, 'Invalid word')); - const binary = fixtures.build('simple', p.build(start)); + const binary = fixtures.build(p, start, 'simple'); binary('GET', 'off=3 match=1\n', callback); }); @@ -44,7 +44,7 @@ describe('LLParse', () => { start.otherwise(p.error(3, 'Invalid word')); - const binary = fixtures.build('shallow', p.build(start)); + const binary = fixtures.build(p, start, 'shallow'); binary('012', 'off=1 match=0\noff=2 match=1\noff=3 match=2\n', callback); }); @@ -58,7 +58,7 @@ describe('LLParse', () => { start.otherwise(p.error(3, 'Invalid word')); - const binary = fixtures.build('kv-select', p.build(start)); + const binary = fixtures.build(p, start, 'kv-select'); binary('012', 'off=1 match=0\noff=2 match=1\noff=3 match=2\n', callback); }); @@ -75,7 +75,7 @@ describe('LLParse', () => { start.otherwise(p.error(3, 'Invalid word')); - const binary = fixtures.build('multi-match', p.build(start)); + const binary = fixtures.build(p, start, 'multi-match'); binary( 'A B\t\tA\r\nA', 'off=1 match=0\noff=3 match=1\noff=6 match=0\noff=9 match=0\n', @@ -94,7 +94,7 @@ describe('LLParse', () => { start.otherwise(p.error(3, 'Invalid word')); - const binary = fixtures.build('multi-match', p.build(start)); + const binary = fixtures.build(p, start, 'multi-match'); binary( 'A B A A', 'off=1 match=0\noff=3 match=1\noff=6 match=0\noff=9 match=0\n', @@ -108,8 +108,8 @@ describe('LLParse', () => { p.property('i8', 'custom'); const second = p.invoke(p.code.load('custom'), { - 0: p.invoke(p.code.match('print_zero'), { 0: start }, error), - 1: p.invoke(p.code.match('print_one'), { 0: start }, error) + 0: p.invoke(p.code.match('llparse__print_zero'), { 0: start }, error), + 1: p.invoke(p.code.match('llparse__print_one'), { 0: start }, error) }, error); start @@ -119,9 +119,9 @@ describe('LLParse', () => { }, p.invoke(p.code.store('custom'), second)) .otherwise(error); - const binary = fixtures.build('custom-prop', p.build(start)); + const binary = fixtures.build(p, start, 'custom-prop'); - binary('0110', 'zero\none\none\nzero\n', callback); + binary('0110', 'off=1 0\noff=2 1\noff=3 1\noff=4 0\n', callback); }); describe('`.peek()`', () => { @@ -138,7 +138,7 @@ describe('LLParse', () => { .match([ 'a', 'b' ], printOff(p, start)) .otherwise(error); - const binary = fixtures.build('peek', p.build(start)); + const binary = fixtures.build(p, start, 'peek'); binary('ab', 'off=1\noff=2\n', callback); }); @@ -159,7 +159,7 @@ describe('LLParse', () => { .match('B', printOff(p, b)) .otherwise(a); - const binary = fixtures.build('otherwise-noadvance', p.build(a)); + const binary = fixtures.build(p, a, 'otherwise-noadvance'); binary('AABAB', 'off=3\noff=5\n', callback); }); @@ -173,7 +173,7 @@ describe('LLParse', () => { .match(' ', printOff(p, start)) .skipTo(start); - const binary = fixtures.build('otherwise-skip', p.build(start)); + const binary = fixtures.build(p, start, 'otherwise-skip'); binary('HELLO WORLD', 'off=6\n', callback); }); @@ -186,9 +186,9 @@ describe('LLParse', () => { start .skipTo(start); - const binary = fixtures.build('all-skip', p.build(start)); + const binary = fixtures.build(p, start, 'all-skip'); - binary('HELLO WORLD', '', callback); + binary('HELLO WORLD', '\n', callback); }); }); }); diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c new file mode 100644 index 0000000..d502755 --- /dev/null +++ b/test/fixtures/extra.c @@ -0,0 +1,41 @@ +#include "fixture.h" + +int llparse__print_zero(llparse_state_t* s, const char* p, const char* endp) { + llparse__print(p, endp, "0"); + return 0; +} + + +int llparse__print_one(llparse_state_t* s, const char* p, const char* endp) { + llparse__print(p, endp, "1"); + return 0; +} + + +int llparse__print_off(llparse_state_t* s, const char* p, const char* endp) { + llparse__print(p, endp, ""); + return 0; +} + + +int llparse__print_match(llparse_state_t* s, const char* p, const char* endp, + int value) { + llparse__print(p, endp, "match=%d", value); + return 0; +} + + +int llparse__on_dot(llparse_state_t* s, const char* p, const char* endp) { + return llparse__print_span("dot", p, endp); +} + + +int llparse__on_dash(llparse_state_t* s, const char* p, const char* endp) { + return llparse__print_span("dash", p, endp); +} + + +int llparse__on_underscore(llparse_state_t* s, const char* p, + const char* endp) { + return llparse__print_span("underscore", p, endp); +} diff --git a/test/fixtures/index.js b/test/fixtures/index.js index a5ad1ee..f372a41 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -1,137 +1,22 @@ 'use strict'; -const assert = require('assert'); const path = require('path'); -const fs = require('fs'); -const spawn = require('child_process').spawn; -const spawnSync = require('child_process').spawnSync; -const Buffer = require('buffer').Buffer; -const async = require('async'); +const fixtures = require('llparse-test-fixture').create({ + buildDir: path.join(__dirname, '..', 'tmp'), + extra: [ path.join(__dirname, 'extra.c') ] +}); -const CLANG = process.env.CLANG || 'clang'; - -const MAIN = path.join(__dirname, 'main.c'); -const TMP_DIR = path.join(__dirname, '..', 'tmp'); - -const MAX_PARALLEL = 8; - -function normalizeSpan(source) { - const lines = source.split(/\n/g); - - const parse = (line) => { - const match = line.match( - /^off=(\d+)\s+len=(\d+)\s+span\[([^\]]+)\]="(.*)"$/); - if (!match) { - throw new Error('Failed to parse the span output: '+ - JSON.stringify(line)); - } - - return { - off: match[1] | 0, - len: match[2] | 0, - span: match[3], - value: match[4] - }; - }; - - const parsed = lines.filter(l => l).map(parse); - const lastMap = new Map(); - const res = []; - - parsed.forEach((obj) => { - if (lastMap.has(obj.span)) { - const last = lastMap.get(obj.span); - if (last.off + last.len === obj.off) { - last.len += obj.len; - last.value += obj.value; - - // Move it to the end - res.splice(res.indexOf(last), 1); - res.push(last); - return; - } - } - res.push(obj); - lastMap.set(obj.span, obj); - }); - - const stringify = (obj) => { - return `off=${obj.off} len=${obj.len} span[${obj.span}]="${obj.value}"`; - }; - return res.map(stringify).join('\n') + '\n'; -} - -exports.build = (name, source, options) => { - options = options || {}; - - try { - fs.mkdirSync(TMP_DIR); - } catch (e) { - // no-op - } - - const file = path.join(TMP_DIR, name + '.ll'); - const header = path.join(TMP_DIR, name + '.h'); - const out = path.join(TMP_DIR, name); - - fs.writeFileSync(file, source.llvm); - fs.writeFileSync(header, source.header); - - const ret = spawnSync(CLANG, [ - '-g3', '-Os', '-fvisibility=hidden', - '-include', header, MAIN, file, '-o', out - ]); - if (ret.status !== 0) { - process.stderr.write(ret.stdout); - process.stderr.write(ret.stderr); - throw new Error('clang exit code: ' + ret.status); - } - - return (input, expected, callback) => { - const buf = Buffer.from(input); - async.timesLimit(buf.length, MAX_PARALLEL, (i, callback) => { - const proc = spawn(out, [ i + 1, buf ], { - stdio: [ null, 'pipe', 'inherit' ] - }); - - let stdout = ''; - proc.stdout.on('data', chunk => stdout += chunk); - - async.parallel({ - exit: cb => proc.once('exit', (code, sig) => cb(null, { code, sig })), - end: cb => proc.stdout.once('end', () => cb(null)) - }, (err, data) => { - if (data.exit.sig) - return callback(new Error('Killed with: ' + data.exit.sig)); - if (data.exit.code !== 0) - return callback(new Error('Exit code: ' + data.exit.code)); - - if (options.normalize === 'span') - stdout = normalizeSpan(stdout); - - callback(null, stdout); - }); - }, (err, results) => { - if (err) - return callback(err); - - for (let i = 0; i < results.length; i++) - assert.strictEqual(results[i], expected, 'Scan value: ' + (i + 1)); - - return callback(null); - }); - }; -}; +exports.build = (...args) => fixtures.build(...args); exports.printMatch = (p, next) => { - const code = p.code.value('print_match'); + const code = p.code.value('llparse__print_match'); return p.invoke(code, next); }; exports.printOff = (p, next) => { - const code = p.code.match('print_off'); + const code = p.code.match('llparse__print_off'); return p.invoke(code, next); }; diff --git a/test/fixtures/main.c b/test/fixtures/main.c deleted file mode 100644 index dcc9876..0000000 --- a/test/fixtures/main.c +++ /dev/null @@ -1,178 +0,0 @@ -#include -#include -#include -#include - -/* NOTE: include is inserted through `-include` clang argument */ - -/* 8 gb */ -static const int64_t kBytes = 8589934592LL; - -static int bench = 0; -static const char* start; - -void debug(llparse_state_t* s, const char* p, const char* endp, - const char* msg) { - if (bench) - return; - - fprintf(stderr, "off=%d > %s\n", (int) (p - start), msg); -} - -int print_zero(llparse_state_t* s, const char* p, const char* endp) { - if (bench) - return 0; - - fprintf(stdout, "zero\n"); - return 0; -} - -int print_one(llparse_state_t* s, const char* p, const char* endp) { - if (bench) - return 0; - - fprintf(stdout, "one\n"); - return 0; -} - -int print_off(llparse_state_t* s, const char* p, const char* endp) { - if (bench) - return 0; - - fprintf(stdout, "off=%d\n", (int) (p - start)); - return 0; -} - -int print_match(llparse_state_t* s, const char* p, const char* endp, - int value) { - if (bench) - return 0; - - fprintf(stdout, "off=%d match=%d\n", (int) (p - start), value); - return 0; -} - -int return_match(llparse_state_t* s, const char* p, const char* endp, - int value) { - if (bench) - return value; - - fprintf(stdout, "off=%d return match=%d\n", (int) (p - start), value); - return value; -} - -static void print_span(const char* name, const char* p, const char* endp) { - if (bench) - return; - - fprintf(stdout, "off=%d len=%d span[%s]=\"%.*s\"\n", (int) (p - start), - (int) (endp - p), name, (int) (endp - p), p); -} - -int on_dot(llparse_state_t* s, const char* p, const char* endp) { - print_span("dot", p, endp); - return 0; -} - -int on_dash(llparse_state_t* s, const char* p, const char* endp) { - print_span("dash", p, endp); - return 0; -} - -int on_underscore(llparse_state_t* s, const char* p, const char* endp) { - print_span("underscore", p, endp); - return 0; -} - -static int run_bench(const char* input, int len) { - llparse_state_t s; - int64_t i; - struct timeval start; - struct timeval end; - double bw; - double time; - int64_t iterations; - - llparse_init(&s); - - iterations = kBytes / (int64_t) len; - - gettimeofday(&start, NULL); - for (i = 0; i < iterations; i++) { - int code; - - code = llparse_execute(&s, input, input + len); - if (code != 0) - return code; - } - gettimeofday(&end, NULL); - - time = (end.tv_sec - start.tv_sec); - time += (double) (end.tv_usec - start.tv_usec) * 1e-6; - bw = (double) kBytes / time; - - fprintf(stdout, "%.2f mb | %.2f mb/s | %.2f s\n", - (double) kBytes / (1024 * 1024), - bw / (1024 * 1024), - time); - - return 0; -} - - -static int run_scan(int scan, const char* input, int len) { - llparse_state_t s; - llparse_init(&s); - - if (scan <= 0) { - fprintf(stderr, "Invalid scan value\n"); - return -1; - } - - while (len > 0) { - int max; - int code; - - max = len > scan ? scan : len; - - code = llparse_execute(&s, input, input + max); - if (code != 0) { - fprintf(stderr, "code=%d error=%d reason=%s\n", code, s.error, s.reason); - return -1; - } - - input += max; - len -= max; - } - - return 0; -} - - -int main(int argc, char** argv) { - const char* input; - int len; - - if (argc < 3) { - fprintf(stderr, "%s [bench or scan-value] [input]\n", argv[0]); - return -1; - } - - if (strcmp(argv[1], "bench") == 0) - bench = 1; - - input = argv[2]; - len = strlen(input); - - if (bench && len == 0) { - fprintf(stderr, "Input can\'t be empty for benchmark"); - return -1; - } - - start = input; - - if (bench) - return run_bench(input, len); - else - return run_scan(atoi(argv[1]), input, len); -} diff --git a/test/span-test.js b/test/span-test.js index 075f905..517e1d1 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -20,9 +20,9 @@ describe('LLParse/span', () => { const underscore = p.node('underscore'); const span = { - dot: p.span(p.code.span('on_dot')), - dash: p.span(p.code.span('on_dash')), - underscore: p.span(p.code.span('on_underscore')) + dot: p.span(p.code.span('llparse__on_dot')), + dash: p.span(p.code.span('llparse__on_dash')), + underscore: p.span(p.code.span('llparse__on_underscore')) }; start.otherwise(span.dot.start(dot)); @@ -41,9 +41,7 @@ describe('LLParse/span', () => { .match('_', underscore) .otherwise(span.underscore.end(dot)); - const binary = fixtures.build('span', p.build(start), { - normalize: 'span' - }); + const binary = fixtures.build(p, start, 'span'); binary( '..--..__..', @@ -55,7 +53,7 @@ describe('LLParse/span', () => { it('should throw on loops', () => { const start = p.node('start'); - const span = p.span(p.code.span('on_data')); + const span = p.span(p.code.span('llparse__on_data')); start.otherwise(span.start().otherwise(start)); @@ -64,7 +62,7 @@ describe('LLParse/span', () => { it('should throw on unmatched ends', () => { const start = p.node('start'); - const span = p.span(p.code.span('on_data')); + const span = p.span(p.code.span('llparse__on_data')); start.otherwise(span.end().otherwise(start)); @@ -73,7 +71,7 @@ describe('LLParse/span', () => { it('should propagate through the Invoke map', () => { const start = p.node('start'); - const span = p.span(p.code.span('on_data')); + const span = p.span(p.code.span('llparse__on_data')); p.property('i8', 'custom'); diff --git a/test/transform-test.js b/test/transform-test.js index cb96e47..7e39628 100644 --- a/test/transform-test.js +++ b/test/transform-test.js @@ -20,7 +20,7 @@ describe('LLParse/transform', () => { .match('close', fixtures.printOff(p, start)) .otherwise(p.error(1, 'error')); - const binary = fixtures.build('transform-lower', p.build(start)); + const binary = fixtures.build(p, start, 'transform-lower'); binary('connectCLOSEcOnNeCt', 'off=7\noff=12\noff=19\n', callback); }); From ba7aab39aacee2e69c2f925bb76ba3989742d203 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 22:45:29 -0500 Subject: [PATCH 170/281] 2.0.0-beta11 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index cfc0521..3b09755 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta10", + "version": "2.0.0-beta11", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c0be42a..c5eaae4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta10", + "version": "2.0.0-beta11", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From d2202b2f8505bb02a4ae99ff36ede9f62cb50e89 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 22 Feb 2018 23:21:42 -0500 Subject: [PATCH 171/281] package: bump fixture --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3b09755..4011ef9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -740,9 +740,9 @@ } }, "llparse-test-fixture": { - "version": "1.0.0-beta1", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta1.tgz", - "integrity": "sha512-jFXs0mzyDl8goVeASK1rHx6+bIgAiBe50Sm0IzKJH5lo+pJPj3Qq9XHJOCnfCktYrzLIH9vggWUnBCfGM7YQuQ==", + "version": "1.0.0-beta4", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta4.tgz", + "integrity": "sha512-37KYu+Vne5Fq0jwaGhLsmVuFXI43YWT1gajxdBruGQZJbyReJSBiaU6v6WVYYxrBues8vz0NgnB+5TzhG7gLzA==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index c5eaae4..91308fa 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.0.0-beta1", + "llparse-test-fixture": "^1.0.0-beta4", "mocha": "^5.0.1" }, "directories": { From 5bf968b51cf219c90437b9e1fd3cd9a99abd276d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 23 Feb 2018 01:08:15 -0500 Subject: [PATCH 172/281] travis: add file --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b5efd79 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +sudo: false +language: node_js +node_js: + - "stable" From a728c78f76be22204446998c149eb1f448e06e08 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 23 Feb 2018 17:21:14 -0500 Subject: [PATCH 173/281] constants: TYPE_STATUS --- lib/llparse/compiler/match-sequence.js | 5 +++-- lib/llparse/compiler/node/sequence.js | 4 +++- lib/llparse/constants.js | 1 + package-lock.json | 6 +++--- package.json | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/match-sequence.js index 95eca9d..d155cb6 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/match-sequence.js @@ -12,6 +12,7 @@ const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_INDEX = constants.TYPE_INDEX; +const TYPE_STATUS = constants.TYPE_STATUS; const ATTR_STATE = constants.ATTR_STATE; const ATTR_POS = constants.ATTR_POS; @@ -35,7 +36,7 @@ class MatchSequence extends compiler.Stage { this.returnType = this.ctx.ir.struct('match_sequence_ret'); this.returnType.field(TYPE_INPUT, 'current'); - this.returnType.field(INT, 'status'); + this.returnType.field(TYPE_STATUS, 'status'); this.signature = IR.signature(this.returnType, [ [ this.ctx.state.ptr(), ATTR_STATE ], @@ -213,7 +214,7 @@ class MatchSequence extends compiler.Stage { body.push(create); const amend = IR._('insertvalue', [ this.returnType, create ], - [ INT, INT.v(status) ], + [ TYPE_STATUS, TYPE_STATUS.v(status) ], INT.v(this.returnType.lookup('status'))); body.push(amend); diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index a6c7d96..63f956a 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -5,6 +5,8 @@ const constants = llparse.constants; const node = require('./'); +const TYPE_STATUS = constants.TYPE_STATUS; + const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; @@ -58,7 +60,7 @@ class Sequence extends node.Node { [ ctx.pos.current.type, current ], [ INT, INT.v(1) ]); body.push(next); - const s = ctx.buildSwitch(body, INT, status, [ + const s = ctx.buildSwitch(body, TYPE_STATUS, status, [ SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index df0d1bf..8e961cf 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -13,6 +13,7 @@ exports.TYPE_INDEX = exports.INT; exports.TYPE_ERROR = exports.INT; exports.TYPE_REASON = IR.i(8).ptr(); exports.TYPE_DATA = IR.i(8).ptr(); +exports.TYPE_STATUS = IR.i(32); exports.ARG_STATE = 's'; exports.ARG_POS = 'p'; diff --git a/package-lock.json b/package-lock.json index 4011ef9..675320b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -740,9 +740,9 @@ } }, "llparse-test-fixture": { - "version": "1.0.0-beta4", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta4.tgz", - "integrity": "sha512-37KYu+Vne5Fq0jwaGhLsmVuFXI43YWT1gajxdBruGQZJbyReJSBiaU6v6WVYYxrBues8vz0NgnB+5TzhG7gLzA==", + "version": "1.0.0-beta5", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta5.tgz", + "integrity": "sha512-P5rUyJsf6TrG6nZZxf5O0DrYq8IHlyC3jMB6SoioTrQ6+K49UERXv1s8pe59Q4lOdjN29akG6jtPjROT4O39dw==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index 91308fa..771db49 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.0.0-beta4", + "llparse-test-fixture": "^1.0.0-beta5", "mocha": "^5.0.1" }, "directories": { From 2df4d48a3c4c9acdf72d41fb80bdc6b22b26dffb Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 23 Feb 2018 22:08:43 -0500 Subject: [PATCH 174/281] test: optimize for benchmarks --- package-lock.json | 6 +++--- package.json | 2 +- test/fixtures/extra.c | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 675320b..05aee7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -740,9 +740,9 @@ } }, "llparse-test-fixture": { - "version": "1.0.0-beta5", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta5.tgz", - "integrity": "sha512-P5rUyJsf6TrG6nZZxf5O0DrYq8IHlyC3jMB6SoioTrQ6+K49UERXv1s8pe59Q4lOdjN29akG6jtPjROT4O39dw==", + "version": "1.0.0-beta6", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta6.tgz", + "integrity": "sha512-75vrovuQSL2spAJLW/M2MrFqILmRDBy8e1Is06elEj5+pqc4XdfVlEu2a69LAAgnpHNB3kRUlnCdwlt9aPM+Ow==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index 771db49..d6a79f7 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.0.0-beta5", + "llparse-test-fixture": "^1.0.0-beta6", "mocha": "^5.0.1" }, "directories": { diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c index d502755..f3839fd 100644 --- a/test/fixtures/extra.c +++ b/test/fixtures/extra.c @@ -1,18 +1,24 @@ #include "fixture.h" int llparse__print_zero(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; llparse__print(p, endp, "0"); return 0; } int llparse__print_one(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; llparse__print(p, endp, "1"); return 0; } int llparse__print_off(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; llparse__print(p, endp, ""); return 0; } @@ -20,22 +26,30 @@ int llparse__print_off(llparse_state_t* s, const char* p, const char* endp) { int llparse__print_match(llparse_state_t* s, const char* p, const char* endp, int value) { + if (llparse__in_bench) + return 0; llparse__print(p, endp, "match=%d", value); return 0; } int llparse__on_dot(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; return llparse__print_span("dot", p, endp); } int llparse__on_dash(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; return llparse__print_span("dash", p, endp); } int llparse__on_underscore(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; return llparse__print_span("underscore", p, endp); } From 87f0a74b7ce7f72ba80b3d9d11c9a46968d08880 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 02:11:37 -0500 Subject: [PATCH 175/281] test: increase default timeout --- test/api-test.js | 4 +++- test/fixtures/index.js | 3 +++ test/span-test.js | 4 +++- test/transform-test.js | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/api-test.js b/test/api-test.js index 624f080..76952b8 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -8,7 +8,9 @@ const fixtures = require('./fixtures'); const printOff = fixtures.printOff; const printMatch = fixtures.printMatch; -describe('LLParse', () => { +describe('LLParse', function() { + this.timeout(fixtures.TIMEOUT); + let p; beforeEach(() => { p = llparse.create('llparse'); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index f372a41..c6ecdc2 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -20,3 +20,6 @@ exports.printOff = (p, next) => { return p.invoke(code, next); }; + +// Reasonable timeout for CI +exports.TIMEOUT = 10000; diff --git a/test/span-test.js b/test/span-test.js index 517e1d1..24750ca 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -7,7 +7,9 @@ const llparse = require('../'); const fixtures = require('./fixtures'); -describe('LLParse/span', () => { +describe('LLParse/span', function() { + this.timeout(fixtures.TIMEOUT); + let p; beforeEach(() => { p = llparse.create('llparse'); diff --git a/test/transform-test.js b/test/transform-test.js index 7e39628..8cde65b 100644 --- a/test/transform-test.js +++ b/test/transform-test.js @@ -5,7 +5,9 @@ const llparse = require('../'); const fixtures = require('./fixtures'); -describe('LLParse/transform', () => { +describe('LLParse/transform', function() { + this.timeout(fixtures.TIMEOUT); + let p; beforeEach(() => { p = llparse.create('llparse'); From 49e26ebad6809d46d2178576390e5d2388979dfe Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 15:46:57 -0500 Subject: [PATCH 176/281] compilation: add extra attributes to functions --- lib/llparse/compiler/compilation.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index a8caebe..6a7b685 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -227,6 +227,10 @@ class Compilation { // TODO(indutny): reassess `minsize`. Looks like it gives best performance // results right now, though. fn.attributes = 'nounwind minsize'; + + // These are ABI dependent, but should bring us close to C-function + // inlining on x86_64 through -flto + fn.attribute += ' ssp uwtable'; } else if (signature === this.signature.callback.match) { fn = this.ir.fn(this.signature.callback.match, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); From 91b6fe6a9b6a6115057986ca22a2ad31f3d992d4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 16:08:46 -0500 Subject: [PATCH 177/281] gitignore: ignore ir files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b311d4a..5b7a295 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ examples/http/http *.bc *.o *.h +*.s main 1 From 7cacc19ec426a0421a8e1bfb81fcacf4da3d85ef Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 16:11:11 -0500 Subject: [PATCH 178/281] constants: remove `nocapture`, spans do it --- lib/llparse/constants.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 8e961cf..e4b3d6a 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -23,9 +23,9 @@ exports.ARG_SEQUENCE_LEN = 'slen'; exports.ARG_MATCH = 'match'; exports.ARG_UNUSED = '_unused'; -exports.ATTR_STATE = 'noalias nocapture nonnull'; -exports.ATTR_POS = 'noalias nocapture nonnull readonly'; -exports.ATTR_ENDPOS = 'noalias nocapture nonnull readnone'; +exports.ATTR_STATE = 'noalias nonnull'; +exports.ATTR_POS = 'noalias nonnull readonly'; +exports.ATTR_ENDPOS = 'noalias nonnull readnone'; exports.ATTR_SEQUENCE = exports.ATTR_POS; exports.SEQUENCE_COMPLETE = 0; From 1fe4783d0d3a78d024f58c5168fa55ef8382f696 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 16:26:43 -0500 Subject: [PATCH 179/281] node: minor refactor base => base, match --- lib/llparse.js | 2 +- lib/llparse/compiler/node/translator.js | 9 +-- lib/llparse/compiler/span/allocator.js | 6 +- lib/llparse/node/base.js | 81 +-------------------- lib/llparse/node/error.js | 6 +- lib/llparse/node/index.js | 2 + lib/llparse/node/invoke.js | 5 -- lib/llparse/node/match.js | 94 +++++++++++++++++++++++++ lib/llparse/node/span-end.js | 7 +- lib/llparse/node/span-start.js | 7 +- lib/llparse/symbols.js | 1 + 11 files changed, 108 insertions(+), 112 deletions(-) create mode 100644 lib/llparse/node/match.js diff --git a/lib/llparse.js b/lib/llparse.js index 7d7136d..6b9e6b7 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -69,7 +69,7 @@ class LLParse { get transform() { return this[kTransform]; } node(name) { - return new internal.node.Node(name); + return new internal.node.Match(name); } error(code, reason) { diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 85b1e86..38d25d3 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -49,7 +49,6 @@ class NodeTranslator extends compiler.Stage { let res; let list; - let hasTrie = false; if (node instanceof llparse.node.Invoke) { res = new compiler.node.Invoke(this.id(node), node[kCode]); } else if (node instanceof llparse.node.Error) { @@ -59,7 +58,8 @@ class NodeTranslator extends compiler.Stage { } else if (node instanceof llparse.node.SpanEnd) { res = new compiler.node.SpanEnd(this.id(node), node[kSpan][kCode]); } else { - hasTrie = true; + assert(node instanceof llparse.node.Match); + const trie = new llparse.Trie(node.name); const combined = trie.combine(node[kCases]); @@ -71,11 +71,6 @@ class NodeTranslator extends compiler.Stage { } } - if (!hasTrie) { - assert.strictEqual(node[kTransform], null); - assert.strictEqual(node[kCases].length, 0); - } - // Prevent loops this.nodes.set(node, res); diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/span/allocator.js index aab50c1..422a3d9 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -165,11 +165,11 @@ class Allocator extends compiler.Stage { // `error` nodes have no `otherwise` if (node[kOtherwise] !== null) res.push(node[kOtherwise].next); - node[kCases].forEach(c => res.push(c.next)); - if (node instanceof llparse.node.Invoke) { + if (node instanceof llparse.node.Match) + node[kCases].forEach(c => res.push(c.next)); + else if (node instanceof llparse.node.Invoke) Object.keys(node[kMap]).forEach(key => res.push(node[kMap][key])); - } return res; } diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js index 64c2631..4ef343c 100644 --- a/lib/llparse/node/base.js +++ b/lib/llparse/node/base.js @@ -5,99 +5,22 @@ const assert = require('assert'); const node = require('./'); const llparse = require('../'); +const kCheckIsMatch = llparse.symbols.kCheckIsMatch; const kOtherwise = llparse.symbols.kOtherwise; -const kCases = llparse.symbols.kCases; const kSignature = llparse.symbols.kSignature; -const kTransform = llparse.symbols.kTransform; const kName = Symbol('name'); -const kCheckIsMatch = Symbol('checkIsMatch'); class Node { - constructor(name, signature = 'match') { + constructor(name, signature) { this[kName] = name; this[kSignature] = signature; - this[kTransform] = null; - - this[kCases] = []; this[kOtherwise] = null; } get name() { return this[kName]; } - transform(t) { - assert.strictEqual(this[kTransform], null, 'Can\'t apply transform twice'); - assert(t instanceof llparse.transform.Transform, - '`.transform()` argument must be a `Transform` instance'); - this[kTransform] = t; - - return this; - } - - peek(value, next) { - // .peek([ ... ], next) - if (Array.isArray(value)) { - value.forEach(value => this.peek(value, next)); - return this; - } - - assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); - this[kCheckIsMatch](next, '.peek()'); - - this[kCases].push(new llparse.case.Peek(value, next)); - - return this; - } - - match(value, next) { - // .match([ ... ], next) - if (Array.isArray(value)) { - value.forEach(value => this.match(value, next)); - return this; - } - - assert(next instanceof Node, 'Invalid `next` argument of `.match()`'); - this[kCheckIsMatch](next, '.match()'); - - this[kCases].push(new llparse.case.Match(value, next)); - - return this; - } - - select(key, value, next) { - // .select(key, value, next) - const pairs = []; - if (Buffer.isBuffer(key) || typeof key === 'number' || - typeof key === 'string') { - assert.strictEqual(typeof value, 'number', - '`.select(key, value, next)` is a signature of the method'); - - pairs.push({ key, value }); - } else { - assert.strictEqual(typeof key, 'object', - '`.select()` first argument must be either an object or a key'); - - const map = key; - next = value; - value = null; - - Object.keys(map).forEach((key) => pairs.push({ key, value: map[key] })); - } - - assert(next instanceof node.Invoke, - 'Invalid `next` argument of `.select()`, must be an `.invoke()` node'); - assert.strictEqual(next[kSignature], 'value', - `Invoke of "${next.name}" can't be a target of \`.select()\``); - - const select = new llparse.case.Select(next); - pairs.forEach(pair => select.add(pair.key, pair.value)); - - this[kCases].push(select); - - return this; - } - otherwise(next) { assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); this[kCheckIsMatch](next, '.otherwise()'); diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index e57a26b..45e41e0 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -7,7 +7,7 @@ const kReason = Symbol('reason'); class Error extends node.Node { constructor(code, reason) { - super('error'); + super('error', 'match'); this[kCode] = code; this[kReason] = reason; @@ -16,10 +16,6 @@ class Error extends node.Node { get code() { return this[kCode]; } get reason() { return this[kReason]; } - transform() { throw new Error('Not supported'); } - peek() { throw new Error('Not supported'); } - match() { throw new Error('Not supported'); } - select() { throw new Error('Not supported'); } otherwise() { throw new Error('Not supported'); } skipTo() { throw new Error('Not supported'); } } diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js index c17c9fc..dfc4e90 100644 --- a/lib/llparse/node/index.js +++ b/lib/llparse/node/index.js @@ -1,6 +1,8 @@ 'use strict'; exports.Node = require('./base'); + +exports.Match = require('./match'); exports.Error = require('./error'); exports.Invoke = require('./invoke'); exports.SpanStart = require('./span-start'); diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 50af3d1..60fffb7 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -37,10 +37,5 @@ class Invoke extends node.Node { if (otherwise) this.otherwise(otherwise); } - - transform() { throw new Error('Not supported'); } - peek() { throw new Error('Not supported'); } - match() { throw new Error('Not supported'); } - select() { throw new Error('Not supported'); } } module.exports = Invoke; diff --git a/lib/llparse/node/match.js b/lib/llparse/node/match.js new file mode 100644 index 0000000..3829567 --- /dev/null +++ b/lib/llparse/node/match.js @@ -0,0 +1,94 @@ +'use strict'; + +const assert = require('assert'); + +const node = require('./'); +const llparse = require('../'); + +const kCases = llparse.symbols.kCases; +const kCheckIsMatch = llparse.symbols.kCheckIsMatch; +const kSignature = llparse.symbols.kSignature; +const kTransform = llparse.symbols.kTransform; + +class Match extends node.Node { + constructor(name) { + super(name, 'match'); + + this[kTransform] = null; + + this[kCases] = []; + } + + transform(t) { + assert.strictEqual(this[kTransform], null, 'Can\'t apply transform twice'); + assert(t instanceof llparse.transform.Transform, + '`.transform()` argument must be a `Transform` instance'); + this[kTransform] = t; + + return this; + } + + peek(value, next) { + // .peek([ ... ], next) + if (Array.isArray(value)) { + value.forEach(value => this.peek(value, next)); + return this; + } + + assert(next instanceof node.Node, 'Invalid `next` argument of `.match()`'); + this[kCheckIsMatch](next, '.peek()'); + + this[kCases].push(new llparse.case.Peek(value, next)); + + return this; + } + + match(value, next) { + // .match([ ... ], next) + if (Array.isArray(value)) { + value.forEach(value => this.match(value, next)); + return this; + } + + assert(next instanceof node.Node, 'Invalid `next` argument of `.match()`'); + this[kCheckIsMatch](next, '.match()'); + + this[kCases].push(new llparse.case.Match(value, next)); + + return this; + } + + select(key, value, next) { + // .select(key, value, next) + const pairs = []; + if (Buffer.isBuffer(key) || typeof key === 'number' || + typeof key === 'string') { + assert.strictEqual(typeof value, 'number', + '`.select(key, value, next)` is a signature of the method'); + + pairs.push({ key, value }); + } else { + assert.strictEqual(typeof key, 'object', + '`.select()` first argument must be either an object or a key'); + + const map = key; + next = value; + value = null; + + Object.keys(map).forEach((key) => pairs.push({ key, value: map[key] })); + } + + assert(next instanceof node.Invoke, + 'Invalid `next` argument of `.select()`, must be an `.invoke()` node'); + assert.strictEqual(next[kSignature], 'value', + `Invoke of "${next.name}" can't be a target of \`.select()\``); + + const select = new llparse.case.Select(next); + pairs.forEach(pair => select.add(pair.key, pair.value)); + + this[kCases].push(select); + + return this; + } +} +module.exports = Match; diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js index 05a09b5..687b920 100644 --- a/lib/llparse/node/span-end.js +++ b/lib/llparse/node/span-end.js @@ -7,13 +7,8 @@ const kSpan = llparse.symbols.kSpan; class SpanEnd extends node.Node { constructor(span, code) { - super('span_end_' + code.name); + super('span_end_' + code.name, 'match'); this[kSpan] = span; } - - transform() { throw new Error('Not supported'); } - peek() { throw new Error('Not supported'); } - match() { throw new Error('Not supported'); } - select() { throw new Error('Not supported'); } } module.exports = SpanEnd; diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js index 73c4508..dcd13dc 100644 --- a/lib/llparse/node/span-start.js +++ b/lib/llparse/node/span-start.js @@ -7,14 +7,9 @@ const kSpan = llparse.symbols.kSpan; class SpanStart extends node.Node { constructor(span, code) { - super('span_start_' + code.name); + super('span_start_' + code.name, 'match'); this[kSpan] = span; } - - transform() { throw new Error('Not supported'); } - peek() { throw new Error('Not supported'); } - match() { throw new Error('Not supported'); } - select() { throw new Error('Not supported'); } } module.exports = SpanStart; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 125d4da..16fe05b 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -2,6 +2,7 @@ exports.kBody = Symbol('body'); exports.kCases = Symbol('cases'); +exports.kCheckIsMatch = Symbol('checkIsMatch'); exports.kCode = Symbol('code'); exports.kMap = Symbol('map'); exports.kName = Symbol('name'); From 2f8757645479a20fc89d7d27d5b6e55b833a735b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 17:45:31 -0500 Subject: [PATCH 180/281] node: consume Fix: #1 --- lib/llparse.js | 4 ++ lib/llparse/compiler/node/base.js | 4 +- lib/llparse/compiler/node/consume.js | 80 +++++++++++++++++++++++++ lib/llparse/compiler/node/index.js | 2 + lib/llparse/compiler/node/set-index.js | 44 ++++++++++++++ lib/llparse/compiler/node/translator.js | 27 ++++++++- lib/llparse/constants.js | 4 ++ lib/llparse/node/consume.js | 23 +++++++ lib/llparse/node/index.js | 1 + test/consume-test.js | 40 +++++++++++++ 10 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 lib/llparse/compiler/node/consume.js create mode 100644 lib/llparse/compiler/node/set-index.js create mode 100644 lib/llparse/node/consume.js create mode 100644 test/consume-test.js diff --git a/lib/llparse.js b/lib/llparse.js index 6b9e6b7..de5ea1b 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -106,6 +106,10 @@ class LLParse { return new internal.Span(callback); } + consume(code) { + return new internal.node.Consume(code); + } + build(root, options) { assert(root, 'Missing required argument for `.build(root)`'); assert(root instanceof internal.node.Node, diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index c8d71b0..6a01e79 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -31,6 +31,8 @@ class NodeContext { this.TYPE_MATCH = constants.TYPE_MATCH; this.TYPE_REASON = constants.TYPE_REASON; this.TYPE_ERROR = constants.TYPE_ERROR; + this.TYPE_INDEX = constants.TYPE_INDEX; + this.TYPE_INTPTR = constants.TYPE_INTPTR; } debug(body, message) { @@ -140,7 +142,7 @@ class Node { // Split, so that others could join us from code block above const trampoline = body.jump('br'); - trampoline.name = body.name + '_trampoline'; + trampoline.name = body.name + '.trampoline'; let phi = null; // Compute `match` if needed diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js new file mode 100644 index 0000000..9b18cf3 --- /dev/null +++ b/lib/llparse/compiler/node/consume.js @@ -0,0 +1,80 @@ +'use strict'; + +const assert = require('assert'); + +const node = require('./'); + +class Consume extends node.Node { + constructor(name) { + super('consume', name); + } + + prologue(ctx, body) { + return body; + } + + // TODO(indutny): remove unnecessary load + doBuild(ctx, body, nodes) { + body.comment('node.Consume'); + + const pos = ctx.pos.current; + + body.comment('index = state.index'); + const indexPtr = ctx.field('_index'); + body.push(indexPtr); + const index = ctx.ir._('load', ctx.TYPE_INDEX, + [ ctx.TYPE_INDEX.ptr(), indexPtr ]); + body.push(index); + + const need = ctx.ir._('zext', + [ ctx.TYPE_INDEX, index, 'to', ctx.TYPE_INTPTR ]); + body.push(need); + + body.comment('avail = endp - p'); + const intPos = ctx.ir._('ptrtoint', + [ pos.type, pos, 'to', ctx.TYPE_INTPTR ]); + body.push(intPos); + const intEndPos = ctx.ir._('ptrtoint', + [ pos.type, ctx.endPos, 'to', ctx.TYPE_INTPTR ]); + body.push(intEndPos); + + const avail = ctx.ir._('sub', [ ctx.TYPE_INTPTR, intEndPos ], intPos); + body.push(avail); + + body.comment('if (avail >= need)'); + const cmp = ctx.ir._('icmp', [ 'uge', ctx.TYPE_INTPTR, avail ], need); + body.push(cmp); + const branch = body.branch('br', [ ctx.BOOL, cmp ]); + + const hasData = branch.left; + const noData = branch.right; + hasData.name = 'has_data'; + noData.name = 'no_data'; + + // Continue! + const next = ctx.ir._('getelementptr', pos.type.to, + [ pos.type, pos ], + [ ctx.TYPE_INDEX, index ]); + hasData.push(next); + + assert(!this.skip); + hasData.comment('state.index = 0'); + hasData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, ctx.TYPE_INDEX.v(0) ], + [ ctx.TYPE_INDEX.ptr(), indexPtr ]).void()); + this.doOtherwise(ctx, nodes, hasData, { current: next, next: null }); + + // Pause! + noData.comment('state.index = need - avail'); + const left = ctx.ir._('sub', [ ctx.TYPE_INTPTR, need ], avail); + noData.push(left); + + const leftTrunc = ctx.ir._('trunc', + [ ctx.TYPE_INTPTR, left, 'to', ctx.TYPE_INDEX ]); + noData.push(leftTrunc); + + noData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, leftTrunc ], + [ ctx.TYPE_INDEX.ptr(), indexPtr ]).void()); + this.pause(ctx, noData); + } +} +module.exports = Consume; diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index 6a87609..8071832 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -8,6 +8,8 @@ exports.Single = require('./single'); exports.Sequence = require('./sequence'); exports.SpanStart = require('./span-start'); exports.SpanEnd = require('./span-end'); +exports.SetIndex = require('./set-index'); +exports.Consume = require('./consume'); exports.Translator = require('./translator'); exports.Builder = require('./builder'); diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js new file mode 100644 index 0000000..3d8257a --- /dev/null +++ b/lib/llparse/compiler/node/set-index.js @@ -0,0 +1,44 @@ +'use strict'; + +const assert = require('assert'); + +const llparse = require('../../'); +const kType = llparse.symbols.kType; + +const node = require('./'); + +class SetIndex extends node.Node { + constructor(name, code) { + super('set-index', name); + + this.code = code; + } + + prologue(ctx, body) { + return body; + } + + doBuild(ctx, body, nodes) { + body.comment(`node.SetIndex[${this.code.name}]`); + + const code = ctx.compilation.buildCode(this.code); + + const args = [ + ctx.state, + ctx.pos.current, + ctx.endPos + ]; + + assert.strictEqual(this.code[kType], 'match'); + const call = ctx.call('', code.type, code, args); + body.push(call); + + const ptr = ctx.field('_index'); + body.push(ptr); + body.push(ctx.ir._('store', [ ctx.TYPE_INDEX, call ], + [ ctx.TYPE_INDEX.ptr(), ptr ]).void()); + + this.doOtherwise(ctx, nodes, body); + } +} +module.exports = SetIndex; diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 38d25d3..25bd1ee 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -27,8 +27,8 @@ class NodeTranslator extends compiler.Stage { return this.buildNode(this.ctx.root, null); } - id(node) { - let res = node.name; + id(node, postfix = '') { + let res = node.name + postfix; if (this.namespace.has(res)) { let i; for (i = 1; i <= this.namespace.size; i++) @@ -49,6 +49,7 @@ class NodeTranslator extends compiler.Stage { let res; let list; + let last; if (node instanceof llparse.node.Invoke) { res = new compiler.node.Invoke(this.id(node), node[kCode]); } else if (node instanceof llparse.node.Error) { @@ -57,6 +58,10 @@ class NodeTranslator extends compiler.Stage { res = new compiler.node.SpanStart(this.id(node), node[kSpan][kCode]); } else if (node instanceof llparse.node.SpanEnd) { res = new compiler.node.SpanEnd(this.id(node), node[kSpan][kCode]); + } else if (node instanceof llparse.node.Consume) { + res = this.buildConsume(node); + last = res.last; + res = res.first; } else { assert(node instanceof llparse.node.Match); @@ -71,6 +76,9 @@ class NodeTranslator extends compiler.Stage { } } + if (!last) + last = res; + // Prevent loops this.nodes.set(node, res); @@ -89,7 +97,7 @@ class NodeTranslator extends compiler.Stage { `Node "${node.name}" must have \`.otherwise()\`/\`.skipTo()\``); if (!list) - list = [ res ]; + list = [ last ]; const otherwise = this.buildNode(node[kOtherwise].next, null); list.forEach((entry) => { entry.setOtherwise(otherwise, node[kOtherwise].skip); @@ -150,6 +158,19 @@ class NodeTranslator extends compiler.Stage { return res; } + buildConsume(node) { + const setIndex = new compiler.node.SetIndex(this.id(node, '_set_index'), + node[kCode]); + + const consume = new compiler.node.Consume(this.id(node, '_consume')); + setIndex.setOtherwise(consume, false); + + return { + first: setIndex, + last: consume + }; + } + checkSignature(node, value) { assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); } diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index e4b3d6a..bf3f026 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -15,6 +15,10 @@ exports.TYPE_REASON = IR.i(8).ptr(); exports.TYPE_DATA = IR.i(8).ptr(); exports.TYPE_STATUS = IR.i(32); +// TODO(indutny): although it works through zero extension, is there any +// cross-platform way to do it? +exports.TYPE_INTPTR = IR.i(64); + exports.ARG_STATE = 's'; exports.ARG_POS = 'p'; exports.ARG_ENDPOS = 'endp'; diff --git a/lib/llparse/node/consume.js b/lib/llparse/node/consume.js new file mode 100644 index 0000000..6f70ee2 --- /dev/null +++ b/lib/llparse/node/consume.js @@ -0,0 +1,23 @@ +'use strict'; + +const assert = require('assert'); + +const node = require('./'); +const llparse = require('../'); + +const kType = llparse.symbols.kType; +const kCode = llparse.symbols.kCode; + +class Consume extends node.Node { + constructor(code) { + assert(code instanceof llparse.code.Code, + 'Invalid `code` argument of `.consume()`, must be a Code instance'); + assert.strictEqual(code[kType], 'match', + '`code` argument of `.consume()` must be have a `match` type'); + + super('consume_' + code.name, 'match'); + + this[kCode] = code; + } +} +module.exports = Consume; diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js index dfc4e90..5476ce7 100644 --- a/lib/llparse/node/index.js +++ b/lib/llparse/node/index.js @@ -7,3 +7,4 @@ exports.Error = require('./error'); exports.Invoke = require('./invoke'); exports.SpanStart = require('./span-start'); exports.SpanEnd = require('./span-end'); +exports.Consume = require('./consume'); diff --git a/test/consume-test.js b/test/consume-test.js new file mode 100644 index 0000000..0ffbe08 --- /dev/null +++ b/test/consume-test.js @@ -0,0 +1,40 @@ +'use strict'; +/* global describe it beforeEach */ + +const assert = require('assert'); + +const llparse = require('../'); + +const fixtures = require('./fixtures'); + +const printOff = fixtures.printOff; + +describe('LLParse/consume', function() { + this.timeout(fixtures.TIMEOUT); + + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); + + it('should consume bytes', (callback) => { + p.property('i8', 'to_consume'); + + const start = p.node('start'); + const consume = p.consume(p.code.load('to_consume')); + + start.select({ + '0': 0, '1': 1, '2': 2, '3': 3, '4': 4 + }, p.invoke(p.code.store('to_consume'), consume)); + + start + .otherwise(p.error(1, 'unexpected')); + + consume + .otherwise(printOff(p, start)); + + const binary = fixtures.build(p, start, 'consume'); + + binary('3aaa2bb1a01b', 'off=4\noff=7\noff=9\noff=10\noff=12\n', callback); + }); +}); From f7b913c11fb511bd8d5ce3c7766e38b6cdce4064 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 18:03:31 -0500 Subject: [PATCH 181/281] code: minor refactor --- lib/llparse/code/context.js | 40 +++++++++++++++++++++++++---- lib/llparse/code/load.js | 23 +++-------------- lib/llparse/code/store.js | 24 ++++------------- lib/llparse/compiler/compilation.js | 4 +-- test/consume-test.js | 2 -- 5 files changed, 46 insertions(+), 47 deletions(-) diff --git a/lib/llparse/code/context.js b/lib/llparse/code/context.js index 99b08c6..b0f05d6 100644 --- a/lib/llparse/code/context.js +++ b/lib/llparse/code/context.js @@ -1,12 +1,13 @@ 'use strict'; -const IR = require('llvm-ir'); +const assert = require('assert'); const llparse = require('../'); const constants = llparse.constants; const kType = llparse.symbols.kType; const kFn = Symbol('fn'); +const kIR = Symbol('IR'); const kRet = Symbol('ret'); const kState = Symbol('state'); const kPos = Symbol('pos'); @@ -22,8 +23,9 @@ const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; class Context { - constructor(code, fn) { + constructor(code, fn, ir) { this[kFn] = fn; + this[kIR] = ir; this[kRet] = this.fn.type.ret; this[kState] = fn.arg(ARG_STATE); @@ -37,6 +39,7 @@ class Context { } get fn() { return this[kFn]; } + get ir() { return this[kIR]; } get ret() { return this[kRet]; } get state() { return this[kState]; } get pos() { return this[kPos]; } @@ -49,7 +52,8 @@ class Context { const lookup = this[kLookup](field); body.push(lookup.instr); - const res = IR._('load', lookup.type, [ lookup.type.ptr(), lookup.instr ]); + const res = this.ir._('load', lookup.type, + [ lookup.type.ptr(), lookup.instr ]); body.push(res); return res; @@ -61,18 +65,44 @@ class Context { const lookup = this[kLookup](field); body.push(lookup.instr); - const res = IR._('store', [ lookup.type, value ], + const res = this.ir._('store', + [ lookup.type, value ], [ lookup.type.ptr(), lookup.instr ]); res.void(); body.push(res); } + truncate(body, fromType, from, toType, isSigned = false) { + assert(toType.isInt()); + assert(fromType.isInt()); + + let res; + + // Same type! + if (fromType.type === toType) { + return from; + // Extend + } else if (fromType.width < toType.width) { + if (isSigned) + res = this.ir._('sext', [ fromType, from, 'to', toType ]); + else + res = this.ir._('zext', [ fromType, from, 'to', toType ]); + // Truncate + } else { + assert(fromType.width > toType.width); + res = this.ir._('trunc', [ fromType, from, 'to', toType ]); + } + + body.push(res); + return res; + } + [kLookup](field) { const stateArg = this.state; const stateType = stateArg.type.to; const index = stateType.lookup(field); - const instr = IR._('getelementptr inbounds', stateType, + const instr = this.ir._('getelementptr inbounds', stateType, [ stateArg.type, stateArg ], [ INT, INT.v(0) ], [ INT, INT.v(index) ]); diff --git a/lib/llparse/code/load.js b/lib/llparse/code/load.js index cb4e8cb..6f9d613 100644 --- a/lib/llparse/code/load.js +++ b/lib/llparse/code/load.js @@ -8,12 +8,12 @@ const kCompile = Symbol('compile'); class Load extends code.Match { constructor(name, field) { - const store = (ir, context) => this[kCompile](ir, context, field); + const store = (context) => this[kCompile](context, field); super(name, store); } - [kCompile](ir, context, field) { + [kCompile](context, field) { const body = context.fn.body; const stateType = context.state.type.to; @@ -22,23 +22,8 @@ class Load extends code.Match { assert(fieldType.isInt(), `"${field}" field is not of integer type`); assert(context.ret.isInt()); - let adj = context.load(body, field); - - // Same type! - if (fieldType.type === context.ret) { - // Truncate - } else if (fieldType.width > context.ret.width) { - adj = ir._('trunc', - [ fieldType, adj, 'to', context.ret ]); - body.push(adj); - // Extend - } else { - assert(fieldType.width < context.ret.width); - adj = ir._('sext', - [ fieldType, adj, 'to', context.ret ]); - body.push(adj); - } - + const adj = context.truncate(body, fieldType, context.load(body, field), + context.ret); body.terminate('ret', [ context.ret, adj ]); } } diff --git a/lib/llparse/code/store.js b/lib/llparse/code/store.js index 50652ae..6c1c2f6 100644 --- a/lib/llparse/code/store.js +++ b/lib/llparse/code/store.js @@ -8,12 +8,12 @@ const kCompile = Symbol('compile'); class Store extends code.Value { constructor(name, field) { - const store = (ir, context) => this[kCompile](ir, context, field); + const store = context => this[kCompile](context, field); super(name, store); } - [kCompile](ir, context, field) { + [kCompile](context, field) { const body = context.fn.body; const stateType = context.state.type.to; @@ -21,23 +21,9 @@ class Store extends code.Value { assert(fieldType.isInt(), `"${field}" field is not of integer type`); assert(context.match.type.isInt()); - let adj; - - // Same type! - if (fieldType.type === context.match.type) { - adj = context.match; - // Extend - } else if (fieldType.width > context.match.type.width) { - adj = ir._('sext', - [ context.match.type, context.match, 'to', fieldType ]); - body.push(adj); - // Truncate - } else { - assert(fieldType.width < context.match.type.width); - adj = ir._('trunc', - [ context.match.type, context.match, 'to', fieldType ]); - body.push(adj); - } + + const adj = context.truncate(body, context.match.type, context.match, + fieldType); context.store(body, field, adj); body.terminate('ret', [ context.ret, context.ret.v(0) ]); diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 6a7b685..075a728 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -179,9 +179,9 @@ class Compilation { fn.cconv = CCONV; fn.attributes = 'nounwind'; - const context = new llparse.code.Context(code, fn); + const context = new llparse.code.Context(code, fn, this.ir); - code[kBody].call(fn.body, this.ir, context); + code[kBody].call(fn.body, context); return fn; } diff --git a/test/consume-test.js b/test/consume-test.js index 0ffbe08..a45c40d 100644 --- a/test/consume-test.js +++ b/test/consume-test.js @@ -1,8 +1,6 @@ 'use strict'; /* global describe it beforeEach */ -const assert = require('assert'); - const llparse = require('../'); const fixtures = require('./fixtures'); From a9fb9ddce5e3f4dafbc8ea597d89a1a8326b8b8e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 19:07:32 -0500 Subject: [PATCH 182/281] code: mul-add Fix: #2 --- lib/llparse.js | 5 + lib/llparse/code/context.js | 10 +- lib/llparse/code/index.js | 1 + lib/llparse/code/mul-add.js | 136 ++++++++++++++++++++++++++++ lib/llparse/compiler/compilation.js | 2 +- package-lock.json | 6 +- package.json | 2 +- test/code-test.js | 79 ++++++++++++++++ test/fixtures/index.js | 4 + 9 files changed, 236 insertions(+), 9 deletions(-) create mode 100644 lib/llparse/code/mul-add.js create mode 100644 test/code-test.js diff --git a/lib/llparse.js b/lib/llparse.js index de5ea1b..6775e3e 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -39,6 +39,11 @@ class CodeAPI { load(field) { return new internal.code.Load(this[kPrefix] + '__load_' + field, field); } + + mulAdd(field, options) { + return new internal.code.MulAdd(this[kPrefix] + '__muladd_' + field, field, + options); + } } class TransformAPI { diff --git a/lib/llparse/code/context.js b/lib/llparse/code/context.js index b0f05d6..ffe26e1 100644 --- a/lib/llparse/code/context.js +++ b/lib/llparse/code/context.js @@ -7,7 +7,7 @@ const constants = llparse.constants; const kType = llparse.symbols.kType; const kFn = Symbol('fn'); -const kIR = Symbol('IR'); +const kCompilation = Symbol('compilation'); const kRet = Symbol('ret'); const kState = Symbol('state'); const kPos = Symbol('pos'); @@ -23,9 +23,9 @@ const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; class Context { - constructor(code, fn, ir) { + constructor(compilation, code, fn) { + this[kCompilation] = compilation; this[kFn] = fn; - this[kIR] = ir; this[kRet] = this.fn.type.ret; this[kState] = fn.arg(ARG_STATE); @@ -39,7 +39,7 @@ class Context { } get fn() { return this[kFn]; } - get ir() { return this[kIR]; } + get ir() { return this[kCompilation].ir; } get ret() { return this[kRet]; } get state() { return this[kState]; } get pos() { return this[kPos]; } @@ -97,6 +97,8 @@ class Context { return res; } + call(...args) { return this[kCompilation].call(...args); } + [kLookup](field) { const stateArg = this.state; const stateType = stateArg.type.to; diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index bb0b0f4..267c84f 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -6,5 +6,6 @@ exports.Value = require('./value'); exports.Store = require('./store'); exports.Load = require('./load'); exports.Span = require('./span'); +exports.MulAdd = require('./mul-add'); exports.Context = require('./context'); diff --git a/lib/llparse/code/mul-add.js b/lib/llparse/code/mul-add.js new file mode 100644 index 0000000..9af216b --- /dev/null +++ b/lib/llparse/code/mul-add.js @@ -0,0 +1,136 @@ +'use strict'; + +const assert = require('assert'); + +const llparse = require('../'); +const code = require('./'); + +const BOOL = llparse.constants.BOOL; +const INT = llparse.constants.INT; + +const kOptions = Symbol('options'); +const kCompile = Symbol('compile'); + +class MulAdd extends code.Value { + constructor(name, field, options) { + const store = context => this[kCompile](context, field); + + super(name, store); + + options = Object.assign({ + max: 0, + signed: true + }, options); + assert.strictEqual(typeof options.max, 'number', + '`MulAdd.options.max` must be a number'); + assert.strictEqual(typeof options.base, 'number', + '`MulAdd.options.base` must be a number'); + assert(options.max >= 0, + '`MulAdd.options.max` must be a non-negative number'); + assert(options.base > 0, + '`MulAdd.options.base` must be a positive number'); + assert.strictEqual(options.max, options.max | 0, + '`MulAdd.options.max` must be an integer'); + assert.strictEqual(options.base, options.base | 0, + '`MulAdd.options.max` must be an integer'); + + this[kOptions] = options; + } + + [kCompile](context, field) { + const ir = context.ir; + const body = context.fn.body; + const matchType = context.match.type; + const options = this[kOptions]; + + const stateType = context.state.type.to; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + // Declare intrinsic functions + const overRet = ir.struct([ + [ fieldType, 'result' ], + [ BOOL, 'overflow' ] + ]); + + const overSig = ir.signature(overRet, [ fieldType, fieldType ]); + const mulFn = ir.declare( + overSig, + `llvm.${options.signed ? 'smul': 'umul'}.with.overflow.${fieldType.type}` + ); + const addFn = ir.declare( + overSig, + `llvm.${options.signed ? 'sadd': 'uadd'}.with.overflow.${fieldType.type}` + ); + + // Truncate match and load field + const value = context.truncate(body, matchType, context.match, fieldType, + options.signed); + const fieldValue = context.load(body, field); + + // Multiply + const mul = context.call('', overSig, mulFn, '', + [ fieldValue, matchType.v(options.base) ]); + body.push(mul); + + body.comment('extract product'); + const product = ir._('extractvalue', [ overRet, mul ], + INT.v(overRet.lookup('result'))); + body.push(product); + + body.comment('extract overflow'); + const overflowBit = ir._('extractvalue', [ overRet, mul ], + INT.v(overRet.lookup('overflow'))); + body.push(overflowBit); + + const { left: isOverflow, right: normal } = + body.branch('br', [ BOOL, overflowBit ]); + isOverflow.name = 'overflow'; + isOverflow.terminate('ret', [ context.ret, context.ret.v(1) ]); + + normal.name = 'no_overflow'; + + // Add + const add = context.call('', overSig, addFn, '', [ product, value ]); + normal.push(add); + + normal.comment('extract result'); + const result = ir._('extractvalue', [ overRet, add ], + INT.v(overRet.lookup('result'))); + normal.push(result); + + normal.comment('extract overflow'); + const addOverflowBit = ir._('extractvalue', [ overRet, add ], + INT.v(overRet.lookup('overflow'))); + normal.push(addOverflowBit); + + const { left: isAddOverflow, right: check } = + normal.branch('br', [ BOOL, addOverflowBit ]); + isAddOverflow.name = 'add_overflow'; + check.name = 'check'; + + isAddOverflow.terminate('ret', [ context.ret, context.ret.v(1) ]); + + // Check that we're within the limits + let store; + if (options.max) { + const cond = options.signed ? 'sgt' : 'ugt'; + const cmp = ir._('icmp', [ cond, fieldType, result ], + fieldType.v(options.max)); + check.push(cmp); + + const branch = check.branch('br', [ BOOL, cmp ]); + branch.left.name = 'max_overflow'; + + branch.left.terminate('ret', [ context.ret, context.ret.v(1) ]); + + store = branch.right; + } else { + store = check; + } + store.name = 'store'; + + context.store(store, field, result); + store.terminate('ret', [ context.ret, context.ret.v(0) ]); + } +} +module.exports = MulAdd; diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 075a728..1bc26d2 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -179,7 +179,7 @@ class Compilation { fn.cconv = CCONV; fn.attributes = 'nounwind'; - const context = new llparse.code.Context(code, fn, this.ir); + const context = new llparse.code.Context(this, code, fn); code[kBody].call(fn.body, context); diff --git a/package-lock.json b/package-lock.json index 05aee7a..e3fba50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -749,9 +749,9 @@ } }, "llvm-ir": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.4.2.tgz", - "integrity": "sha512-wEWBCyECdHO+Mq/BWMdn9X2PbgBAT4fCe+dYTlG6KyPLfHyMn/EN0xfXgzcp6oqSsWtPaa7vyN72lL2DPVMBqg==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.6.1.tgz", + "integrity": "sha512-WwcKUhVxsm+TEV/a/Rmf2o6abjLMyMk1U27LASwzGCRpoh8NvhV8iRXw5ORT+/hctEZG2CoAC7pZy+RVENlOGQ==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index d6a79f7..b875dd6 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.4.2" + "llvm-ir": "^1.6.1" }, "devDependencies": { "async": "^2.6.0", diff --git a/test/code-test.js b/test/code-test.js new file mode 100644 index 0000000..bd9d900 --- /dev/null +++ b/test/code-test.js @@ -0,0 +1,79 @@ +'use strict'; +/* global describe it beforeEach */ + +const llparse = require('../'); + +const fixtures = require('./fixtures'); + +const printOff = fixtures.printOff; + +describe('LLParse/Code', function() { + this.timeout(fixtures.TIMEOUT); + + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); + + describe('`.mulAdd()`', () => { + it('should operate normally', (callback) => { + const start = p.node('start'); + + p.property('i64', 'counter'); + + const is1337 = p.invoke(p.code.load('counter'), { + 1337: printOff(p, start) + }, p.error(1, 'Invalid result')); + + const count = p.invoke(p.code.mulAdd('counter', { base: 10 }), start); + + start + .select(fixtures.NUM, count) + .match('.', is1337) + .otherwise(p.error(1, 'Unexpected')); + + const binary = fixtures.build(p, start, 'mul-add'); + + binary('1337.', 'off=5\n', callback); + }); + + it('should operate fail on overflow', (callback) => { + const start = p.node('start'); + + p.property('i8', 'counter'); + + const count = p.invoke(p.code.mulAdd('counter', { base: 10 }), { + 1: printOff(p, start) + }, start); + + start + .select(fixtures.NUM, count) + .otherwise(p.error(1, 'Unexpected')); + + const binary = fixtures.build(p, start, 'mul-add-overflow'); + + binary('1111', 'off=4\n', callback); + }); + + it('should operate fail on greater than max', (callback) => { + const start = p.node('start'); + + p.property('i64', 'counter'); + + const count = p.invoke(p.code.mulAdd('counter', { + base: 10, + max: 1000 + }), { + 1: printOff(p, start) + }, start); + + start + .select(fixtures.NUM, count) + .otherwise(p.error(1, 'Unexpected')); + + const binary = fixtures.build(p, start, 'mul-add-max-overflow'); + + binary('1111', 'off=4\n', callback); + }); + }); +}); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index c6ecdc2..e8d957e 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -21,5 +21,9 @@ exports.printOff = (p, next) => { return p.invoke(code, next); }; +exports.NUM = { + '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9 +}; + // Reasonable timeout for CI exports.TIMEOUT = 10000; From f63fa91790b001b66526d0c9f85fee186635b268 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 19:11:19 -0500 Subject: [PATCH 183/281] test: add extra char for benchmarks --- test/code-test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/code-test.js b/test/code-test.js index bd9d900..017a60f 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -30,6 +30,9 @@ describe('LLParse/Code', function() { start .select(fixtures.NUM, count) .match('.', is1337) + // TODO(indutny): replace with `code.reset()` + // Just for benchmarks + .select({ 'r': 0 }, p.invoke(p.code.store('counter'), start)) .otherwise(p.error(1, 'Unexpected')); const binary = fixtures.build(p, start, 'mul-add'); From 91a0d9b9c659647403b08014eddc544a41ee3086 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 19:47:21 -0500 Subject: [PATCH 184/281] code: `.update()` Fix: #5 --- lib/llparse.js | 6 ++++++ lib/llparse/code/index.js | 1 + lib/llparse/code/mul-add.js | 4 ++-- lib/llparse/code/update.js | 33 +++++++++++++++++++++++++++++++++ test/code-test.js | 27 +++++++++++++++++++++++---- 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 lib/llparse/code/update.js diff --git a/lib/llparse.js b/lib/llparse.js index 6775e3e..7b952fa 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -32,6 +32,7 @@ class CodeAPI { // Helpers + // TODO(indutny): do something with these long names store(field) { return new internal.code.Store(this[kPrefix] + '__store_' + field, field); } @@ -44,6 +45,11 @@ class CodeAPI { return new internal.code.MulAdd(this[kPrefix] + '__muladd_' + field, field, options); } + + update(field, value) { + return new internal.code.Update(this[kPrefix] + '__update_' + field, field, + value); + } } class TransformAPI { diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index 267c84f..87c9c7d 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -7,5 +7,6 @@ exports.Store = require('./store'); exports.Load = require('./load'); exports.Span = require('./span'); exports.MulAdd = require('./mul-add'); +exports.Update = require('./update'); exports.Context = require('./context'); diff --git a/lib/llparse/code/mul-add.js b/lib/llparse/code/mul-add.js index 9af216b..8f03fe1 100644 --- a/lib/llparse/code/mul-add.js +++ b/lib/llparse/code/mul-add.js @@ -13,9 +13,9 @@ const kCompile = Symbol('compile'); class MulAdd extends code.Value { constructor(name, field, options) { - const store = context => this[kCompile](context, field); + const body = context => this[kCompile](context, field); - super(name, store); + super(name, body); options = Object.assign({ max: 0, diff --git a/lib/llparse/code/update.js b/lib/llparse/code/update.js new file mode 100644 index 0000000..1979794 --- /dev/null +++ b/lib/llparse/code/update.js @@ -0,0 +1,33 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +const kCompile = Symbol('compile'); + +class Update extends code.Match { + constructor(name, field, value) { + assert.strictEqual(typeof value, 'number', + '`.update()`\'s `value` argument must be a number'); + assert.strictEqual(value, value | 0, + '`.update()`\'s `value` argument must be an integer'); + + const update = context => this[kCompile](context, field, value); + + super(name, update); + } + + [kCompile](context, field, value) { + const body = context.fn.body; + + const stateType = context.state.type.to; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + + context.store(body, field, fieldType.v(value)); + body.terminate('ret', [ context.ret, context.ret.v(0) ]); + } +} +module.exports = Update; diff --git a/test/code-test.js b/test/code-test.js index 017a60f..893b913 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -22,7 +22,7 @@ describe('LLParse/Code', function() { p.property('i64', 'counter'); const is1337 = p.invoke(p.code.load('counter'), { - 1337: printOff(p, start) + 1337: printOff(p, p.invoke(p.code.update('counter', 0), start)) }, p.error(1, 'Invalid result')); const count = p.invoke(p.code.mulAdd('counter', { base: 10 }), start); @@ -30,9 +30,6 @@ describe('LLParse/Code', function() { start .select(fixtures.NUM, count) .match('.', is1337) - // TODO(indutny): replace with `code.reset()` - // Just for benchmarks - .select({ 'r': 0 }, p.invoke(p.code.store('counter'), start)) .otherwise(p.error(1, 'Unexpected')); const binary = fixtures.build(p, start, 'mul-add'); @@ -79,4 +76,26 @@ describe('LLParse/Code', function() { binary('1111', 'off=4\n', callback); }); }); + + describe('`.update()`', () => { + it('should operate normally', (callback) => { + const start = p.node('start'); + + p.property('i64', 'counter'); + + const update = p.invoke(p.code.update('counter', 42)); + + start + .skipTo(update); + + update + .otherwise(p.invoke(p.code.load('counter'), { + 42: printOff(p, start) + }, p.error(1, 'Unexpected'))); + + const binary = fixtures.build(p, start, 'update'); + + binary('.', 'off=1\n', callback); + }); + }); }); From 3cea8011ae9722e3b1f7c984d12b3b05d94f4541 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 19:58:22 -0500 Subject: [PATCH 185/281] code: `isEqual()` Fix: #3 --- lib/llparse.js | 5 +++++ lib/llparse/code/index.js | 6 ++++- lib/llparse/code/is-equal.js | 43 ++++++++++++++++++++++++++++++++++++ test/code-test.js | 21 ++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 lib/llparse/code/is-equal.js diff --git a/lib/llparse.js b/lib/llparse.js index 7b952fa..0145cdf 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -50,6 +50,11 @@ class CodeAPI { return new internal.code.Update(this[kPrefix] + '__update_' + field, field, value); } + + isEqual(field, value) { + return new internal.code.IsEqual(this[kPrefix] + '__is_eq_' + field, field, + value); + } } class TransformAPI { diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index 87c9c7d..62ecd4e 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -3,10 +3,14 @@ exports.Code = require('./base'); exports.Match = require('./match'); exports.Value = require('./value'); +exports.Span = require('./span'); + +// Helpers + exports.Store = require('./store'); exports.Load = require('./load'); -exports.Span = require('./span'); exports.MulAdd = require('./mul-add'); exports.Update = require('./update'); +exports.IsEqual = require('./is-equal'); exports.Context = require('./context'); diff --git a/lib/llparse/code/is-equal.js b/lib/llparse/code/is-equal.js new file mode 100644 index 0000000..23d2555 --- /dev/null +++ b/lib/llparse/code/is-equal.js @@ -0,0 +1,43 @@ +'use strict'; + +const assert = require('assert'); +const llparse = require('../'); + +const BOOL = llparse.constants.BOOL; + +const code = require('./'); + +const kCompile = Symbol('compile'); + +class IsEqual extends code.Match { + constructor(name, field, value) { + assert.strictEqual(typeof value, 'number', + '`.update()`\'s `value` argument must be a number'); + assert.strictEqual(value, value | 0, + '`.update()`\'s `value` argument must be an integer'); + + const body = (context) => this[kCompile](context, field, value); + + super(name, body); + } + + [kCompile](context, field, value) { + const body = context.fn.body; + + const stateType = context.state.type.to; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(context.ret.isInt()); + + const fieldValue = context.load(body, field); + + const cmp = context.ir._('icmp', [ 'eq', fieldType, fieldValue ], + fieldType.v(value)); + body.push(cmp); + const res = context.truncate(body, BOOL, cmp, context.ret); + + body.terminate('ret', [ context.ret, res ]); + } +} +module.exports = IsEqual; diff --git a/test/code-test.js b/test/code-test.js index 893b913..2e9f29f 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -98,4 +98,25 @@ describe('LLParse/Code', function() { binary('.', 'off=1\n', callback); }); }); + + describe('`.isEqual()`', () => { + it('should operate normally', (callback) => { + const start = p.node('start'); + + p.property('i64', 'counter'); + + const check = p.invoke(p.code.isEqual('counter', 1), { + 0: fixtures.printOff(p, start), + 1: start + }, p.error(1, 'Unexpected')); + + start + .select(fixtures.NUM, p.invoke(p.code.store('counter'), check)) + .otherwise(p.error(1, 'Unexpected')); + + const binary = fixtures.build(p, start, 'update'); + + binary('010', 'off=1\noff=3\n', callback); + }); + }); }); From 51c17f1f15c7df433ddae604ad9daa9654a5ad67 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 20:52:15 -0500 Subject: [PATCH 186/281] compiler: move `getChildren` to nodes --- lib/llparse/compiler/compiler.js | 4 +-- lib/llparse/compiler/node/base.js | 6 ++++ lib/llparse/compiler/node/error.js | 4 +++ lib/llparse/compiler/node/invoke.js | 6 ++++ lib/llparse/compiler/node/sequence.js | 4 +++ lib/llparse/compiler/node/single.js | 6 ++++ lib/llparse/compiler/node/translator.js | 2 -- lib/llparse/compiler/span/allocator.js | 46 ++++++++----------------- 8 files changed, 42 insertions(+), 36 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 0f41205..9f90d4f 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -34,9 +34,9 @@ class Compiler { stages: { before: [ llparse.compiler.MatchSequence, + llparse.compiler.node.Translator, llparse.compiler.span.Allocator, - llparse.compiler.span.Builder, - llparse.compiler.node.Translator + llparse.compiler.span.Builder ], after: [ llparse.compiler.node.Builder diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 6a01e79..6cd3f64 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -65,6 +65,12 @@ class Node { this.skip = skip; } + getChildren() { + return [ { node: this.otherwise, noAdvance: true } ]; + } + + // Building + build(compilation, nodes) { if (nodes.has(this)) return nodes.get(this); diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index a6de55f..ef446ca 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -10,6 +10,10 @@ class Error extends node.Node { this.reason = reason; } + getChildren() { + return []; + } + prologue(ctx, body) { return body; } diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 061c88e..27bd7dd 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -15,6 +15,12 @@ class Invoke extends node.Node { this.map = null; } + getChildren() { + return Object.keys(this.map).map((key) => { + return { node: this.map[key], noAdvance: true }; + }); + } + prologue(ctx, body) { return body; } diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 63f956a..c2cf1b4 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -20,6 +20,10 @@ class Sequence extends node.Node { this.value = null; } + getChildren() { + return super.getChildren().concat({ node: this.next, noAdvance: false }); + } + doBuild(ctx, body, nodes) { const INT = ctx.INT; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 195cb4a..9c0f6d6 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -9,6 +9,12 @@ class Single extends node.Node { this.children = null; } + getChildren() { + return super.getChildren().concat(this.children.map((child) => { + return { node: child.next, noAdvance: child.noAdvance }; + })); + } + doBuild(ctx, body, nodes) { body.comment('node.Single'); diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/node/translator.js index 25bd1ee..337e433 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/node/translator.js @@ -22,8 +22,6 @@ class NodeTranslator extends compiler.Stage { } build() { - // TODO(indutny): detect and report loops - return this.buildNode(this.ctx.root, null); } diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/span/allocator.js index 422a3d9..acf61e7 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/span/allocator.js @@ -5,23 +5,17 @@ const assert = require('assert'); const compiler = require('../'); const llparse = require('../../'); -const kCode = llparse.symbols.kCode; -const kCases = llparse.symbols.kCases; -const kOtherwise = llparse.symbols.kOtherwise; -const kMap = llparse.symbols.kMap; -const kSpan = llparse.symbols.kSpan; - class Allocator extends compiler.Stage { constructor(ctx) { super(ctx, 'span-allocator'); } id(node) { - return node[kSpan][kCode]; + return node.code; } build() { - const nodes = this.getNodes(this.ctx.root); + const nodes = this.getNodes(this.ctx.stageResults['node-translator']); const info = this.computeActive(nodes); const overlap = this.computeOverlap(info); const color = this.color(info.spans, overlap); @@ -38,7 +32,7 @@ class Allocator extends compiler.Stage { continue; res.add(node); - this.getChildren(node).forEach(child => queue.push(child)); + node.getChildren().forEach(child => queue.push(child.node)); } return Array.from(res); } @@ -55,7 +49,7 @@ class Allocator extends compiler.Stage { const active = activeMap.get(node); - if (node instanceof llparse.node.SpanStart) { + if (node instanceof llparse.compiler.node.SpanStart) { const span = this.id(node); spans.add(span); active.add(span); @@ -63,29 +57,32 @@ class Allocator extends compiler.Stage { active.forEach((span) => { // Don't propagate span past the spanEnd - if (node instanceof llparse.node.SpanEnd && + if (node instanceof llparse.compiler.node.SpanEnd && span === this.id(node)) { return; } - this.getChildren(node).forEach((child) => { + node.getChildren().forEach((child) => { + const node = child.node; + // Disallow loops - if (child instanceof llparse.node.SpanStart) { - assert.notStrictEqual(this.id(child), span, + if (node instanceof llparse.compiler.node.SpanStart) { + assert.notStrictEqual(this.id(node), span, `Detected loop in span "${span.name}"`); } - const set = activeMap.get(child); + const set = activeMap.get(node); if (set.has(span)) return; set.add(span); - queue.add(child); + queue.add(node); }); }); } - const ends = nodes.filter(node => node instanceof llparse.node.SpanEnd); + const ends = nodes + .filter(node => node instanceof llparse.compiler.node.SpanEnd); ends.forEach((end) => { const active = activeMap.get(end); assert(active.has(this.id(end)), @@ -158,20 +155,5 @@ class Allocator extends compiler.Stage { return { map: res, concurrency, max }; } - - getChildren(node) { - const res = []; - - // `error` nodes have no `otherwise` - if (node[kOtherwise] !== null) - res.push(node[kOtherwise].next); - - if (node instanceof llparse.node.Match) - node[kCases].forEach(c => res.push(c.next)); - else if (node instanceof llparse.node.Invoke) - Object.keys(node[kMap]).forEach(key => res.push(node[kMap][key])); - - return res; - } } module.exports = Allocator; From acecc3727f25923b3eab1625b0ba74b6e3b33147 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:01:29 -0500 Subject: [PATCH 187/281] compiler: move stages to a separate folder --- lib/llparse/compiler/compiler.js | 10 +++++----- lib/llparse/compiler/index.js | 8 +++----- lib/llparse/compiler/node/index.js | 3 --- lib/llparse/compiler/span/index.js | 4 ---- lib/llparse/compiler/{stage.js => stage/base.js} | 0 lib/llparse/compiler/stage/index.js | 9 +++++++++ lib/llparse/compiler/{ => stage}/match-sequence.js | 6 +++--- .../{node/builder.js => stage/node-builder.js} | 4 ++-- .../translator.js => stage/node-translator.js} | 3 ++- .../{span/allocator.js => stage/span-allocator.js} | 14 ++++++-------- .../{span/builder.js => stage/span-builder.js} | 4 ++-- 11 files changed, 32 insertions(+), 33 deletions(-) delete mode 100644 lib/llparse/compiler/span/index.js rename lib/llparse/compiler/{stage.js => stage/base.js} (100%) create mode 100644 lib/llparse/compiler/stage/index.js rename lib/llparse/compiler/{ => stage}/match-sequence.js (98%) rename lib/llparse/compiler/{node/builder.js => stage/node-builder.js} (75%) rename lib/llparse/compiler/{node/translator.js => stage/node-translator.js} (98%) rename lib/llparse/compiler/{span/allocator.js => stage/span-allocator.js} (89%) rename lib/llparse/compiler/{span/builder.js => stage/span-builder.js} (98%) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 9f90d4f..546ab97 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -33,13 +33,13 @@ class Compiler { root, stages: { before: [ - llparse.compiler.MatchSequence, - llparse.compiler.node.Translator, - llparse.compiler.span.Allocator, - llparse.compiler.span.Builder + llparse.compiler.stage.MatchSequence, + llparse.compiler.stage.NodeTranslator, + llparse.compiler.stage.SpanAllocator, + llparse.compiler.stage.SpanBuilder ], after: [ - llparse.compiler.node.Builder + llparse.compiler.stage.NodeBuilder ] } }); diff --git a/lib/llparse/compiler/index.js b/lib/llparse/compiler/index.js index c8e84a7..4df2983 100644 --- a/lib/llparse/compiler/index.js +++ b/lib/llparse/compiler/index.js @@ -1,10 +1,8 @@ 'use strict'; -exports.Compilation = require('./compilation'); -exports.Stage = require('./stage'); -exports.MatchSequence = require('./match-sequence'); - -exports.span = require('./span'); exports.node = require('./node'); +exports.stage = require('./stage'); + +exports.Compilation = require('./compilation'); exports.Compiler = require('./compiler'); diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index 8071832..498ef82 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -10,6 +10,3 @@ exports.SpanStart = require('./span-start'); exports.SpanEnd = require('./span-end'); exports.SetIndex = require('./set-index'); exports.Consume = require('./consume'); - -exports.Translator = require('./translator'); -exports.Builder = require('./builder'); diff --git a/lib/llparse/compiler/span/index.js b/lib/llparse/compiler/span/index.js deleted file mode 100644 index d266f2f..0000000 --- a/lib/llparse/compiler/span/index.js +++ /dev/null @@ -1,4 +0,0 @@ -'use strict'; - -exports.Allocator = require('./allocator'); -exports.Builder = require('./builder'); diff --git a/lib/llparse/compiler/stage.js b/lib/llparse/compiler/stage/base.js similarity index 100% rename from lib/llparse/compiler/stage.js rename to lib/llparse/compiler/stage/base.js diff --git a/lib/llparse/compiler/stage/index.js b/lib/llparse/compiler/stage/index.js new file mode 100644 index 0000000..5a92e94 --- /dev/null +++ b/lib/llparse/compiler/stage/index.js @@ -0,0 +1,9 @@ +'use strict'; + +exports.Stage = require('./base'); + +exports.MatchSequence = require('./match-sequence'); +exports.NodeTranslator = require('./node-translator'); +exports.NodeBuilder = require('./node-builder'); +exports.SpanAllocator = require('./span-allocator'); +exports.SpanBuilder = require('./span-builder'); diff --git a/lib/llparse/compiler/match-sequence.js b/lib/llparse/compiler/stage/match-sequence.js similarity index 98% rename from lib/llparse/compiler/match-sequence.js rename to lib/llparse/compiler/stage/match-sequence.js index d155cb6..a1b694a 100644 --- a/lib/llparse/compiler/match-sequence.js +++ b/lib/llparse/compiler/stage/match-sequence.js @@ -2,8 +2,8 @@ const IR = require('llvm-ir'); -const compiler = require('./'); -const llparse = require('../'); +const Stage = require('./').Stage; +const llparse = require('../../'); const constants = llparse.constants; const CCONV = constants.CCONV; @@ -29,7 +29,7 @@ const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; -class MatchSequence extends compiler.Stage { +class MatchSequence extends Stage { constructor(ctx) { super(ctx, 'match-sequence'); diff --git a/lib/llparse/compiler/node/builder.js b/lib/llparse/compiler/stage/node-builder.js similarity index 75% rename from lib/llparse/compiler/node/builder.js rename to lib/llparse/compiler/stage/node-builder.js index 3b4f41c..e5ddada 100644 --- a/lib/llparse/compiler/node/builder.js +++ b/lib/llparse/compiler/stage/node-builder.js @@ -1,8 +1,8 @@ 'use strict'; -const compiler = require('../'); +const Stage = require('./').Stage; -class NodeBuilder extends compiler.Stage { +class NodeBuilder extends Stage { constructor(ctx) { super(ctx, 'node-builder'); diff --git a/lib/llparse/compiler/node/translator.js b/lib/llparse/compiler/stage/node-translator.js similarity index 98% rename from lib/llparse/compiler/node/translator.js rename to lib/llparse/compiler/stage/node-translator.js index 337e433..083a159 100644 --- a/lib/llparse/compiler/node/translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -2,6 +2,7 @@ const assert = require('assert'); +const Stage = require('./').Stage; const compiler = require('../'); const llparse = require('../../'); @@ -13,7 +14,7 @@ const kSignature = llparse.symbols.kSignature; const kSpan = llparse.symbols.kSpan; const kTransform = llparse.symbols.kTransform; -class NodeTranslator extends compiler.Stage { +class NodeTranslator extends Stage { constructor(ctx) { super(ctx, 'node-translator'); diff --git a/lib/llparse/compiler/span/allocator.js b/lib/llparse/compiler/stage/span-allocator.js similarity index 89% rename from lib/llparse/compiler/span/allocator.js rename to lib/llparse/compiler/stage/span-allocator.js index acf61e7..5abe1fe 100644 --- a/lib/llparse/compiler/span/allocator.js +++ b/lib/llparse/compiler/stage/span-allocator.js @@ -2,10 +2,10 @@ const assert = require('assert'); +const Stage = require('./').Stage; const compiler = require('../'); -const llparse = require('../../'); -class Allocator extends compiler.Stage { +class Allocator extends Stage { constructor(ctx) { super(ctx, 'span-allocator'); } @@ -49,7 +49,7 @@ class Allocator extends compiler.Stage { const active = activeMap.get(node); - if (node instanceof llparse.compiler.node.SpanStart) { + if (node instanceof compiler.node.SpanStart) { const span = this.id(node); spans.add(span); active.add(span); @@ -57,16 +57,14 @@ class Allocator extends compiler.Stage { active.forEach((span) => { // Don't propagate span past the spanEnd - if (node instanceof llparse.compiler.node.SpanEnd && - span === this.id(node)) { + if (node instanceof compiler.node.SpanEnd && span === this.id(node)) return; - } node.getChildren().forEach((child) => { const node = child.node; // Disallow loops - if (node instanceof llparse.compiler.node.SpanStart) { + if (node instanceof compiler.node.SpanStart) { assert.notStrictEqual(this.id(node), span, `Detected loop in span "${span.name}"`); } @@ -82,7 +80,7 @@ class Allocator extends compiler.Stage { } const ends = nodes - .filter(node => node instanceof llparse.compiler.node.SpanEnd); + .filter(node => node instanceof compiler.node.SpanEnd); ends.forEach((end) => { const active = activeMap.get(end); assert(active.has(this.id(end)), diff --git a/lib/llparse/compiler/span/builder.js b/lib/llparse/compiler/stage/span-builder.js similarity index 98% rename from lib/llparse/compiler/span/builder.js rename to lib/llparse/compiler/stage/span-builder.js index 8c4ad03..519413d 100644 --- a/lib/llparse/compiler/span/builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -2,8 +2,8 @@ const assert = require('assert'); +const Stage = require('./').Stage; const llparse = require('../../'); -const compiler = require('../'); const constants = llparse.constants; @@ -15,7 +15,7 @@ const TYPE_ERROR = constants.TYPE_ERROR; const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; -class Builder extends compiler.Stage { +class Builder extends Stage { constructor(ctx) { super(ctx, 'span-builder'); } From 1efce5791a5944ac8613afdc2d00d29f7298d0e4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:37:22 -0500 Subject: [PATCH 188/281] compiler: detect infinite lops Fix: #6 --- lib/llparse/compiler/compiler.js | 1 + lib/llparse/compiler/node/base.js | 7 +- lib/llparse/compiler/node/consume.js | 4 +- lib/llparse/compiler/node/empty.js | 4 +- lib/llparse/compiler/node/error.js | 4 +- lib/llparse/compiler/node/invoke.js | 6 +- lib/llparse/compiler/node/sequence.js | 10 ++- lib/llparse/compiler/node/set-index.js | 4 +- lib/llparse/compiler/node/single.js | 6 +- lib/llparse/compiler/node/span-end.js | 4 +- lib/llparse/compiler/node/span-start.js | 4 +- lib/llparse/compiler/stage/index.js | 1 + .../compiler/stage/node-loop-checker.js | 78 +++++++++++++++++++ lib/llparse/compiler/stage/node-translator.js | 2 +- test/api-test.js | 2 +- test/loop-check-test.js | 53 +++++++++++++ test/span-test.js | 8 +- 17 files changed, 168 insertions(+), 30 deletions(-) create mode 100644 lib/llparse/compiler/stage/node-loop-checker.js create mode 100644 test/loop-check-test.js diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 546ab97..88c2047 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -35,6 +35,7 @@ class Compiler { before: [ llparse.compiler.stage.MatchSequence, llparse.compiler.stage.NodeTranslator, + llparse.compiler.stage.NodeLoopChecker, llparse.compiler.stage.SpanAllocator, llparse.compiler.stage.SpanBuilder ], diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 6cd3f64..31200f2 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -49,9 +49,10 @@ class NodeContext { } class Node { - constructor(type, name) { + constructor(type, id) { this.type = type; - this.name = name; + this.name = id.name; + this.sourceName = id.sourceName; this.otherwise = null; this.skip = false; this.transform = null; @@ -66,7 +67,7 @@ class Node { } getChildren() { - return [ { node: this.otherwise, noAdvance: true } ]; + return [ { node: this.otherwise, noAdvance: !this.skip, key: null } ]; } // Building diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 9b18cf3..9c95ce8 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -5,8 +5,8 @@ const assert = require('assert'); const node = require('./'); class Consume extends node.Node { - constructor(name) { - super('consume', name); + constructor(...args) { + super('consume', ...args); } prologue(ctx, body) { diff --git a/lib/llparse/compiler/node/empty.js b/lib/llparse/compiler/node/empty.js index 7b7df8b..6f089fd 100644 --- a/lib/llparse/compiler/node/empty.js +++ b/lib/llparse/compiler/node/empty.js @@ -3,8 +3,8 @@ const node = require('./'); class Empty extends node.Node { - constructor(name) { - super('empty', name); + constructor(...args) { + super('empty', ...args); } doBuild(ctx, body, nodes) { diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index ef446ca..214c7f7 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -3,8 +3,8 @@ const node = require('./'); class Error extends node.Node { - constructor(name, code, reason) { - super('error', name); + constructor(id, code, reason) { + super('error', id); this.code = code; this.reason = reason; diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 27bd7dd..78d87a6 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -8,8 +8,8 @@ const kType = llparse.symbols.kType; const node = require('./'); class Invoke extends node.Node { - constructor(name, code) { - super('invoke', name); + constructor(id, code) { + super('invoke', id); this.code = code; this.map = null; @@ -17,7 +17,7 @@ class Invoke extends node.Node { getChildren() { return Object.keys(this.map).map((key) => { - return { node: this.map[key], noAdvance: true }; + return { node: this.map[key], noAdvance: true, key: null }; }); } diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index c2cf1b4..b8fac8e 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -12,8 +12,8 @@ const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; class Sequence extends node.Node { - constructor(name, select) { - super('sequence', name); + constructor(id, select) { + super('sequence', id); this.select = select; this.next = null; @@ -21,7 +21,11 @@ class Sequence extends node.Node { } getChildren() { - return super.getChildren().concat({ node: this.next, noAdvance: false }); + return super.getChildren().concat({ + node: this.next, + noAdvance: false, + key: this.select + }); } doBuild(ctx, body, nodes) { diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js index 3d8257a..a2c324c 100644 --- a/lib/llparse/compiler/node/set-index.js +++ b/lib/llparse/compiler/node/set-index.js @@ -8,8 +8,8 @@ const kType = llparse.symbols.kType; const node = require('./'); class SetIndex extends node.Node { - constructor(name, code) { - super('set-index', name); + constructor(id, code) { + super('set-index', id); this.code = code; } diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 9c0f6d6..7cfd1c3 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -3,15 +3,15 @@ const node = require('./'); class Single extends node.Node { - constructor(name) { - super('single', name); + constructor(...args) { + super('single', ...args); this.children = null; } getChildren() { return super.getChildren().concat(this.children.map((child) => { - return { node: child.next, noAdvance: child.noAdvance }; + return { node: child.next, noAdvance: child.noAdvance, key: child.key }; })); } diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index d9277a0..270c761 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -3,8 +3,8 @@ const node = require('./'); class SpanEnd extends node.Node { - constructor(name, code) { - super('span-end', name); + constructor(id, code) { + super('span-end', id); this.code = code; } diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index c7726bd..e9210b1 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -3,8 +3,8 @@ const node = require('./'); class SpanStart extends node.Node { - constructor(name, code) { - super('span-start', name); + constructor(id, code) { + super('span-start', id); this.code = code; } diff --git a/lib/llparse/compiler/stage/index.js b/lib/llparse/compiler/stage/index.js index 5a92e94..de2bf0e 100644 --- a/lib/llparse/compiler/stage/index.js +++ b/lib/llparse/compiler/stage/index.js @@ -4,6 +4,7 @@ exports.Stage = require('./base'); exports.MatchSequence = require('./match-sequence'); exports.NodeTranslator = require('./node-translator'); +exports.NodeLoopChecker = require('./node-loop-checker'); exports.NodeBuilder = require('./node-builder'); exports.SpanAllocator = require('./span-allocator'); exports.SpanBuilder = require('./span-builder'); diff --git a/lib/llparse/compiler/stage/node-loop-checker.js b/lib/llparse/compiler/stage/node-loop-checker.js new file mode 100644 index 0000000..5f0a27c --- /dev/null +++ b/lib/llparse/compiler/stage/node-loop-checker.js @@ -0,0 +1,78 @@ +'use strict'; + +const Stage = require('./').Stage; + +class NodeLoopChecker extends Stage { + constructor(ctx) { + super(ctx, 'node-loop-checker'); + + this.reachableMap = new Map(); + } + + reachable(from) { + if (this.reachableMap.has(from)) + return this.reachableMap.get(from); + + const res = new Set([ from ]); + this.reachableMap.set(from, res); + return res; + } + + addEdge(from, to) { + const target = this.reachable(to); + + let changed = false; + this.reachable(from).forEach((node) => { + if (to === node) { + throw new Error(`Loop detected in "${to.sourceName}", ` + + `through the backedge from "${from.sourceName}"`); + } + + if (target.has(node)) + return; + + changed = true; + target.add(node); + }); + + return changed; + } + + build() { + const queue = [ { + node: this.ctx.stageResults['node-translator'], + key: null + } ]; + + while (queue.length !== 0) { + const item = queue.pop(); + const lastKey = item.key; + const node = item.node; + + let children = node.getChildren(); + + // Loops like: + // + // `nodeA: peek(A)` => `nodeB: match(A), otherwise -> nodeA` + // + // are valid. + if (lastKey !== null && typeof lastKey === 'number') { + const sameKey = children.some((child) => { + return child.key === lastKey && !child.noAdvance; + }); + if (sameKey) + return; + } + + children = children.filter(child => child.noAdvance); + + children.forEach((child) => { + if (this.addEdge(node, child.node)) + queue.push({ node: child.node, key: child.key || lastKey }); + }); + } + + return true; + } +} +module.exports = NodeLoopChecker; diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 083a159..bc256a0 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -37,7 +37,7 @@ class NodeTranslator extends Stage { } this.namespace.add(res); - return 'n_' + res; + return { name: 'n_' + res, sourceName: node.name }; } buildNode(node, value) { diff --git a/test/api-test.js b/test/api-test.js index 76952b8..72b7d67 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -159,7 +159,7 @@ describe('LLParse', function() { b .match('B', printOff(p, b)) - .otherwise(a); + .skipTo(a); const binary = fixtures.build(p, a, 'otherwise-noadvance'); diff --git a/test/loop-check-test.js b/test/loop-check-test.js new file mode 100644 index 0000000..d050f3c --- /dev/null +++ b/test/loop-check-test.js @@ -0,0 +1,53 @@ +'use strict'; +/* global describe it beforeEach */ + +const assert = require('assert'); + +const llparse = require('../'); + +const fixtures = require('./fixtures'); + +describe('LLParse/node-loop-checker', function() { + this.timeout(fixtures.TIMEOUT); + + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); + + it('should detect loops', () => { + const start = p.node('start'); + const a = p.node('a'); + const invoke = p.invoke(p.code.match('nop'), { + 0: start + }, p.error(1, 'error')); + + start + .peek('a', a) + .otherwise(p.error(1, 'error')); + + a.otherwise(invoke); + + assert.throws(() => { + p.build(start); + }, /detected in "start".*"invoke_nop"/); + }); + + it('should ignore loops through `peek` to `match`', () => { + const start = p.node('start'); + const a = p.node('a'); + const invoke = p.invoke(p.code.match('nop'), { + 0: start + }, p.error(1, 'error')); + + start + .peek('a', a) + .otherwise(p.error(1, 'error')); + + a + .match('a', invoke) + .otherwise(start); + + assert.doesNotThrow(() => p.build(start)); + }); +}); diff --git a/test/span-test.js b/test/span-test.js index 24750ca..88c5d96 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -57,7 +57,7 @@ describe('LLParse/span', function() { const start = p.node('start'); const span = p.span(p.code.span('llparse__on_data')); - start.otherwise(span.start().otherwise(start)); + start.otherwise(span.start().skipTo(start)); assert.throws(() => p.build(start), /loop.*on_data/); }); @@ -66,7 +66,7 @@ describe('LLParse/span', function() { const start = p.node('start'); const span = p.span(p.code.span('llparse__on_data')); - start.otherwise(span.end().otherwise(start)); + start.otherwise(span.end().skipTo(start)); assert.throws(() => p.build(start), /unmatched.*on_data/i); }); @@ -78,8 +78,8 @@ describe('LLParse/span', function() { p.property('i8', 'custom'); start.otherwise(p.invoke(p.code.load('custom'), { - 0: span.end().otherwise(start) - }, span.end().otherwise(start))); + 0: span.end().skipTo(start) + }, span.end().skipTo(start))); assert.doesNotThrow(() => p.build(span.start(start))); }); From 76502a125f6a4da025c0830678e3bac4f20aabd1 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:40:20 -0500 Subject: [PATCH 189/281] node/invoke: fix `.getChildren()` --- lib/llparse/compiler/node/invoke.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 78d87a6..b517e24 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -16,9 +16,9 @@ class Invoke extends node.Node { } getChildren() { - return Object.keys(this.map).map((key) => { + return super.getChildren().concat(Object.keys(this.map).map((key) => { return { node: this.map[key], noAdvance: true, key: null }; - }); + })); } prologue(ctx, body) { From 91482f287db3c29caa2f5dce8a106b754cf2cd90 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:41:32 -0500 Subject: [PATCH 190/281] 2.0.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index e3fba50..24ef878 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta11", + "version": "2.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index b875dd6..e275976 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0-beta11", + "version": "2.0.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From e61759d4dc59d970eaade7aa0e0368fb05083539 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:43:04 -0500 Subject: [PATCH 191/281] package: bump `llparse-test-fixture` to stable --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 24ef878..cea15e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -740,9 +740,9 @@ } }, "llparse-test-fixture": { - "version": "1.0.0-beta6", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0-beta6.tgz", - "integrity": "sha512-75vrovuQSL2spAJLW/M2MrFqILmRDBy8e1Is06elEj5+pqc4XdfVlEu2a69LAAgnpHNB3kRUlnCdwlt9aPM+Ow==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0.tgz", + "integrity": "sha512-HxTURBDxijBtbCUCwRtvyAsCvmPM+kfhIXpRc87yGH+Ca84w0jy8gApdS8JUFCtSG0os6zNvPRx4dv5mnQbxQw==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index e275976..6049286 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.0.0-beta6", + "llparse-test-fixture": "^1.0.0", "mocha": "^5.0.1" }, "directories": { From ef7102006d0169982e27e805d9a2fac90102ca9a Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:43:12 -0500 Subject: [PATCH 192/281] 2.0.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index cea15e4..8d08171 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0", + "version": "2.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 6049286..7966eeb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.0", + "version": "2.0.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From ca07dcdc17b750230dd470acb20db79403727c95 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:47:35 -0500 Subject: [PATCH 193/281] readme: note about crashing `clang` --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 80fd463..d5f360d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ An API for generating parser in LLVM IR. +**NOTE: `clang` 5.0.0 and later crashes on some of the generated output. The +fixes were [submitted][0] [to][1] [upstream][2]. Please use -O0 if compilation +is crashing/failing until these fixes are landed.** + ## Usage ```js @@ -82,3 +86,7 @@ NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +[0]: https://reviews.llvm.org/D43729 +[1]: https://reviews.llvm.org/D43708 +[2]: https://reviews.llvm.org/D43695 From f88e2bd9048985360aa71f0007daf316c991da4b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:50:54 -0500 Subject: [PATCH 194/281] travis: fix attempt --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b5efd79..b381e1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,3 +2,5 @@ sudo: false language: node_js node_js: - "stable" +script: + CFLAGS="-O0" npm test From 91651ec4d1f7fe83ed772bbf8bf1fb83d738e564 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sat, 24 Feb 2018 21:54:44 -0500 Subject: [PATCH 195/281] span: remove stale todo --- lib/llparse/compiler/stage/span-builder.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 519413d..77a1288 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -196,8 +196,6 @@ class Builder extends Stage { present.name = 'present_' + index; empty.name = 'empty_' + index; - // TODO(indutny): branch and call - let cb; if (num === 1) { cb = this.ctx.buildCode(list[0]); From bd5ec54a000fda411bb013d23abb8a1eaeaf7ee6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 12:36:41 -0500 Subject: [PATCH 196/281] loop-checker: support unreachable `peek` loop --- lib/llparse/compiler/stage/node-loop-checker.js | 8 +++++++- test/loop-check-test.js | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/stage/node-loop-checker.js b/lib/llparse/compiler/stage/node-loop-checker.js index 5f0a27c..91cf5f5 100644 --- a/lib/llparse/compiler/stage/node-loop-checker.js +++ b/lib/llparse/compiler/stage/node-loop-checker.js @@ -55,8 +55,14 @@ class NodeLoopChecker extends Stage { // // `nodeA: peek(A)` => `nodeB: match(A), otherwise -> nodeA` // - // are valid. + // should pass the check if (lastKey !== null && typeof lastKey === 'number') { + // Remove all unreachable clauses + children = children.filter((child) => { + return child.key === null || child.key === lastKey; + }); + + // See if there is a matching peek clause const sameKey = children.some((child) => { return child.key === lastKey && !child.noAdvance; }); diff --git a/test/loop-check-test.js b/test/loop-check-test.js index d050f3c..18713a7 100644 --- a/test/loop-check-test.js +++ b/test/loop-check-test.js @@ -50,4 +50,19 @@ describe('LLParse/node-loop-checker', function() { assert.doesNotThrow(() => p.build(start)); }); + + it('should ignore irrelevant `peek`s', () => { + const start = p.node('start'); + const a = p.node('a'); + + start + .peek('a', a) + .otherwise(p.error(1, 'error')); + + a + .peek('b', start) + .otherwise(p.error(1, 'error')); + + assert.doesNotThrow(() => p.build(start)); + }); }); From 297495781624f08741499585765aae0480ce17d7 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 13:15:01 -0500 Subject: [PATCH 197/281] compiler: use invariant groups --- lib/llparse/compiler/compilation.js | 2 ++ lib/llparse/compiler/compiler.js | 15 ++++++++++++--- lib/llparse/compiler/node/base.js | 2 ++ lib/llparse/compiler/node/consume.js | 11 ++++++++--- lib/llparse/compiler/node/set-index.js | 5 ++++- lib/llparse/compiler/stage/match-sequence.js | 9 ++++++--- package-lock.json | 6 +++--- package.json | 2 +- test/code-test.js | 4 ++++ 9 files changed, 42 insertions(+), 14 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 1bc26d2..da73737 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -60,6 +60,8 @@ class Compilation { }; this.signature.callback.span = this.signature.callback.match; + this.INVARIANT_GROUP = this.ir.metadata('!"llparse.invariant"'); + this.codeCache = new Map(); this.debugMethod = null; diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 88c2047..93a9b5c 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -14,6 +14,10 @@ const ARG_STATE = constants.ARG_STATE; const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; +const ATTR_STATE = constants.ATTR_STATE; +const ATTR_POS = constants.ATTR_POS; +const ATTR_ENDPOS = constants.ATTR_ENDPOS; + class Compiler { constructor(options) { this.options = Object.assign({}, options); @@ -77,7 +81,9 @@ class Compiler { } buildInit(ctx) { - const sig = ctx.ir.signature(ctx.ir.void(), [ ctx.state.ptr() ]); + const sig = ctx.ir.signature(ctx.ir.void(), [ + [ ctx.state.ptr(), ATTR_STATE ] + ]); const init = ctx.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); ctx.initFields(init, init.body); @@ -89,8 +95,11 @@ class Compiler { buildExecute(ctx) { // TODO(indutny): change signature to (state*, start*, len)? - const sig = ctx.ir.signature(TYPE_ERROR, - [ ctx.state.ptr(), TYPE_INPUT, TYPE_INPUT ]); + const sig = ctx.ir.signature(TYPE_ERROR, [ + [ ctx.state.ptr(), ATTR_STATE ], + [ TYPE_INPUT, ATTR_POS ], + [ TYPE_INPUT, ATTR_ENDPOS ] + ]); const execute = ctx.ir.fn(sig, this.prefix + '_execute', [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 31200f2..cc02b95 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -33,6 +33,8 @@ class NodeContext { this.TYPE_ERROR = constants.TYPE_ERROR; this.TYPE_INDEX = constants.TYPE_INDEX; this.TYPE_INTPTR = constants.TYPE_INTPTR; + + this.INVARIANT_GROUP = ctx.INVARIANT_GROUP; } debug(body, message) { diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 9c95ce8..93f5d41 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -15,6 +15,8 @@ class Consume extends node.Node { // TODO(indutny): remove unnecessary load doBuild(ctx, body, nodes) { + const INVARIANT_GROUP = ctx.INVARIANT_GROUP; + body.comment('node.Consume'); const pos = ctx.pos.current; @@ -23,7 +25,8 @@ class Consume extends node.Node { const indexPtr = ctx.field('_index'); body.push(indexPtr); const index = ctx.ir._('load', ctx.TYPE_INDEX, - [ ctx.TYPE_INDEX.ptr(), indexPtr ]); + [ ctx.TYPE_INDEX.ptr(), indexPtr ], + [ '!invariant.group', INVARIANT_GROUP ]); body.push(index); const need = ctx.ir._('zext', @@ -60,7 +63,8 @@ class Consume extends node.Node { assert(!this.skip); hasData.comment('state.index = 0'); hasData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, ctx.TYPE_INDEX.v(0) ], - [ ctx.TYPE_INDEX.ptr(), indexPtr ]).void()); + [ ctx.TYPE_INDEX.ptr(), indexPtr ], + [ '!invariant.group', INVARIANT_GROUP ]).void()); this.doOtherwise(ctx, nodes, hasData, { current: next, next: null }); // Pause! @@ -73,7 +77,8 @@ class Consume extends node.Node { noData.push(leftTrunc); noData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, leftTrunc ], - [ ctx.TYPE_INDEX.ptr(), indexPtr ]).void()); + [ ctx.TYPE_INDEX.ptr(), indexPtr ], + [ '!invariant.group', INVARIANT_GROUP ]).void()); this.pause(ctx, noData); } } diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js index a2c324c..f0fb23f 100644 --- a/lib/llparse/compiler/node/set-index.js +++ b/lib/llparse/compiler/node/set-index.js @@ -19,6 +19,8 @@ class SetIndex extends node.Node { } doBuild(ctx, body, nodes) { + const INVARIANT_GROUP = ctx.INVARIANT_GROUP; + body.comment(`node.SetIndex[${this.code.name}]`); const code = ctx.compilation.buildCode(this.code); @@ -36,7 +38,8 @@ class SetIndex extends node.Node { const ptr = ctx.field('_index'); body.push(ptr); body.push(ctx.ir._('store', [ ctx.TYPE_INDEX, call ], - [ ctx.TYPE_INDEX.ptr(), ptr ]).void()); + [ ctx.TYPE_INDEX.ptr(), ptr ], + [ '!invariant.group', INVARIANT_GROUP ]).void()); this.doOtherwise(ctx, nodes, body); } diff --git a/lib/llparse/compiler/stage/match-sequence.js b/lib/llparse/compiler/stage/match-sequence.js index a1b694a..398ce68 100644 --- a/lib/llparse/compiler/stage/match-sequence.js +++ b/lib/llparse/compiler/stage/match-sequence.js @@ -89,7 +89,8 @@ class MatchSequence extends Stage { start.comment('index = state.index'); const indexPtr = this.ctx.field(fn, '_index'); start.push(indexPtr); - const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ]); + const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ], + [ '!invariant.group', this.ctx.INVARIANT_GROUP ]); start.push(index); const loop = start.jump('br'); @@ -195,7 +196,8 @@ class MatchSequence extends Stage { noMoreData.name = 'no_more_data'; noMoreData.comment('state.index = index1'); noMoreData.push(IR._('store', [ TYPE_INDEX, index1 ], - [ TYPE_INDEX.ptr(), indexField ]).void()); + [ TYPE_INDEX.ptr(), indexField ], + [ '!invariant.group', this.ctx.INVARIANT_GROUP ]).void()); return { index: index1, @@ -223,7 +225,8 @@ class MatchSequence extends Stage { reset(field) { return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], - [ TYPE_INDEX.ptr(), field ]).void(); + [ TYPE_INDEX.ptr(), field ], + [ '!invariant.group', this.ctx.INVARIANT_GROUP ]).void(); } } module.exports = MatchSequence; diff --git a/package-lock.json b/package-lock.json index 8d08171..674c6e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -749,9 +749,9 @@ } }, "llvm-ir": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.6.1.tgz", - "integrity": "sha512-WwcKUhVxsm+TEV/a/Rmf2o6abjLMyMk1U27LASwzGCRpoh8NvhV8iRXw5ORT+/hctEZG2CoAC7pZy+RVENlOGQ==" + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.7.0.tgz", + "integrity": "sha512-ZDkevNC7tCTRKN/xfnwlwNOsGQAw4UA+R/VMrSIyjouzWHVpf7tiXyB6W0BZIMK1ismAk+8vXgXbmJfXTZZwiA==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index 7966eeb..c5be740 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.6.1" + "llvm-ir": "^1.7.0" }, "devDependencies": { "async": "^2.6.0", diff --git a/test/code-test.js b/test/code-test.js index 2e9f29f..0e16866 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -18,6 +18,7 @@ describe('LLParse/Code', function() { describe('`.mulAdd()`', () => { it('should operate normally', (callback) => { const start = p.node('start'); + const dot = p.node('dot'); p.property('i64', 'counter'); @@ -29,6 +30,9 @@ describe('LLParse/Code', function() { start .select(fixtures.NUM, count) + .otherwise(dot); + + dot .match('.', is1337) .otherwise(p.error(1, 'Unexpected')); From 8de263cdac5e18259a3ec62d136feee8550e4647 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 13:24:21 -0500 Subject: [PATCH 198/281] 2.0.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 674c6e0..ee83206 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.1", + "version": "2.0.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c5be740..76c087c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.1", + "version": "2.0.2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 5e6e33d2c07e5a7d235796bb4a168decb5414c92 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 13:33:11 -0500 Subject: [PATCH 199/281] compiler: specify callees for dynamic call --- lib/llparse/compiler/compilation.js | 2 +- lib/llparse/compiler/compiler.js | 8 ++++++++ lib/llparse/compiler/stage/node-builder.js | 6 +++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index da73737..124de9b 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -72,7 +72,7 @@ class Compilation { build() { // Private fields this.declareField(this.signature.node.ptr(), '_current', - (type, ctx) => ctx.stageResults['node-builder'].ref()); + (type, ctx) => ctx.stageResults['node-builder'].entry.ref()); this.declareField(TYPE_INDEX, '_index', type => type.v(0)); diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 93a9b5c..81b9791 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -119,12 +119,20 @@ class Compiler { [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); + // TODO(indutny): remove `Invoke`, `Error`, and other nodes without a + // prologue out of this list. + const nodes = ctx.stageResults['node-builder'].list; + const callees = ctx.ir.metadata(nodes.map((node) => { + return node.type.ptr().type + ' @' + node.name; + }).join(', ')); + const call = ctx.call('', nodeSig, current, constants.CCONV, [ ctx.stateArg(execute), ctx.posArg(execute), ctx.endPosArg(execute), TYPE_MATCH.v(0) ]); + call.append([ '!callees', callees ]); body.push(call); const cmp = ctx.ir._('icmp', [ 'ne', nodeSig.ret, call ], diff --git a/lib/llparse/compiler/stage/node-builder.js b/lib/llparse/compiler/stage/node-builder.js index e5ddada..dc6a9df 100644 --- a/lib/llparse/compiler/stage/node-builder.js +++ b/lib/llparse/compiler/stage/node-builder.js @@ -10,7 +10,11 @@ class NodeBuilder extends Stage { } build() { - return this.ctx.stageResults['node-translator'].build(this.ctx, this.nodes); + return { + entry: + this.ctx.stageResults['node-translator'].build(this.ctx, this.nodes), + list: Array.from(this.nodes.values()) + }; } } module.exports = NodeBuilder; From 6f81b5f69b63d674bf6dcb8bf70f4557d33b0a9b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 13:40:17 -0500 Subject: [PATCH 200/281] span-builder: supply `!callees` metadata --- lib/llparse/compiler/stage/span-builder.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 77a1288..d994dde 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -178,7 +178,8 @@ class Builder extends Stage { body.comment('execute spans'); colors.concurrency.forEach((list, index) => { - const num = list.length; + const codes = list.map(code => this.ctx.buildCode(code)); + const num = codes.length; const startPtr = this.startField(fn, index); body.push(startPtr); @@ -198,7 +199,7 @@ class Builder extends Stage { let cb; if (num === 1) { - cb = this.ctx.buildCode(list[0]); + cb = codes[0]; } else { const cbPtr = this.callbackField(fn, index); present.push(cbPtr); @@ -214,6 +215,12 @@ class Builder extends Stage { start, this.ctx.endPosArg(fn) ]); + if (num > 1) { + const callees = codes.map((code) => { + return code.type.ptr().type + ' @' + code.name; + }).join(', '); + call.append([ '!callees', this.ctx.ir.metadata(callees) ]); + } present.push(call); // Check return value From cf7484101ef458806d200c3f3819b0bc5f70587d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 13:40:50 -0500 Subject: [PATCH 201/281] 2.0.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index ee83206..e6680a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.2", + "version": "2.0.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 76c087c..a495a2e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.2", + "version": "2.0.3", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From c8534fa410f212ebf19ebc6546de30f89d810794 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 14:16:10 -0500 Subject: [PATCH 202/281] compiler: put only "pausable" nodes into `callees` --- lib/llparse/compiler/compiler.js | 15 ++++++++------- lib/llparse/compiler/node/base.js | 5 +++++ lib/llparse/compiler/stage/node-builder.js | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 81b9791..ca0b812 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -119,12 +119,13 @@ class Compiler { [ nodeSig.ptr().ptr(), currentPtr ]); body.push(current); - // TODO(indutny): remove `Invoke`, `Error`, and other nodes without a - // prologue out of this list. - const nodes = ctx.stageResults['node-builder'].list; - const callees = ctx.ir.metadata(nodes.map((node) => { - return node.type.ptr().type + ' @' + node.name; - }).join(', ')); + const nodes = ctx.stageResults['node-builder'].map; + const callees = []; + nodes.forEach((fn, node) => { + // Only nodes that can pause can be present in `_current` + if (node.hasPause) + callees.push(fn.type.ptr().type + ' @' + fn.name); + }); const call = ctx.call('', nodeSig, current, constants.CCONV, [ ctx.stateArg(execute), @@ -132,7 +133,7 @@ class Compiler { ctx.endPosArg(execute), TYPE_MATCH.v(0) ]); - call.append([ '!callees', callees ]); + call.append([ '!callees', ctx.ir.metadata(callees.join(', ')) ]); body.push(call); const cmp = ctx.ir._('icmp', [ 'ne', nodeSig.ret, call ], diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index cc02b95..1e7dd4e 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -35,6 +35,8 @@ class NodeContext { this.TYPE_INTPTR = constants.TYPE_INTPTR; this.INVARIANT_GROUP = ctx.INVARIANT_GROUP; + + this.hasPause = false; } debug(body, message) { @@ -131,6 +133,9 @@ class Node { [ fn.type.ptr(), fn, 'to', fn.type.ret ]); body.push(bitcast); body.terminate('ret', [ fn.type.ret, bitcast ]); + + // To be used in `compiler.js` + this.hasPause = true; } tailTo(ctx, body, pos, target, value = null) { diff --git a/lib/llparse/compiler/stage/node-builder.js b/lib/llparse/compiler/stage/node-builder.js index dc6a9df..3d7507f 100644 --- a/lib/llparse/compiler/stage/node-builder.js +++ b/lib/llparse/compiler/stage/node-builder.js @@ -13,7 +13,7 @@ class NodeBuilder extends Stage { return { entry: this.ctx.stageResults['node-translator'].build(this.ctx, this.nodes), - list: Array.from(this.nodes.values()) + map: this.nodes }; } } From 5acf94c5a8b63c79543eb0f31074dd66a2e7a364 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 25 Feb 2018 14:19:27 -0500 Subject: [PATCH 203/281] 2.0.4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index e6680a0..430ec29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.3", + "version": "2.0.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a495a2e..fc0a1dc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.3", + "version": "2.0.4", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 32305f5cfd9ea7528d4aafc6fb2f7df51487bdd4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 15:49:36 -0500 Subject: [PATCH 204/281] node/error: store `error_pos` Fix: #9 --- lib/llparse/compiler/compilation.js | 1 + lib/llparse/compiler/node/error.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 124de9b..3de3ac9 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -82,6 +82,7 @@ class Compilation { // Public fields this.declareField(TYPE_ERROR, 'error', type => type.v(0)); this.declareField(TYPE_REASON, 'reason', type => type.v(null)); + this.declareField(TYPE_INPUT, 'error_pos', type => type.v(null)); this.declareField(TYPE_DATA, 'data', type => type.v(null)); // Custom fields diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index 214c7f7..a87d35b 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -31,6 +31,9 @@ class Error extends node.Node { const reasonPtr = ctx.field('reason'); body.push(reasonPtr); + const posPtr = ctx.field('error_pos'); + body.push(posPtr); + const cast = ctx.ir._('getelementptr inbounds', reason.type.to, [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); body.push(cast); @@ -39,6 +42,8 @@ class Error extends node.Node { [ ctx.TYPE_ERROR.ptr(), codePtr ]).void()); body.push(ctx.ir._('store', [ ctx.TYPE_REASON, cast ], [ ctx.TYPE_REASON.ptr(), reasonPtr ]).void()); + body.push(ctx.ir._('store', [ ctx.pos.current.type, ctx.pos.current ], + [ ctx.pos.current.type.ptr(), posPtr ]).void()); const retType = ctx.fn.type.ret; body.terminate('ret', [ retType, retType.v(null) ]); From 61da1f8a6911ba86a7159e7f3bd48108b9895526 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 15:52:17 -0500 Subject: [PATCH 205/281] compilation: qualify C struct field types `error_pos` and `reason` are string types. --- lib/llparse/compiler/compilation.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 3de3ac9..4c10d85 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -108,7 +108,10 @@ class Compilation { this.state.fields.forEach((field) => { let type = field.type; if (type.isPointer()) { - type = 'void*'; + if (field.name === 'error_pos' || field.name === 'reason') + type = 'const char*'; + else + type = 'void*'; } else { assert(type.isInt(), 'Unsupported type: ' + type.type); if (type.width === 8) From 2146ea7ddcf8ee08734df82941d6da2d2ef6fdd7 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 18:35:09 -0500 Subject: [PATCH 206/281] span-builder: always store error code and position --- lib/llparse/compiler/stage/span-builder.js | 30 ++++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index d994dde..7511006 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -100,13 +100,14 @@ class Builder extends Stage { body.push(store); const signature = this.ctx.signature.callback.span; + const pos = this.ctx.posArg(fn); // TODO(indutny): this won't work with internal callbacks due to // cconv mismatch const call = this.ctx.call('', signature, this.ctx.buildCode(code), [ this.ctx.stateArg(fn), start, - this.ctx.posArg(fn) + pos ]); body.push(call); @@ -122,18 +123,28 @@ class Builder extends Stage { error.name = 'error_' + index; // Handle error - const codePtr = this.ctx.field(fn, 'error'); - error.push(codePtr); - assert.strictEqual(TYPE_ERROR.type, signature.ret.type); - error.push(this.ctx.ir._('store', - [ TYPE_ERROR, call ], - [ TYPE_ERROR.ptr(), codePtr ]).void()); + this.buildError(fn, error, pos, call); error.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); return noError; } + buildError(fn, body, pos, code) { + const codePtr = this.ctx.field(fn, 'error'); + body.push(codePtr); + + const errorPosPtr = this.ctx.field(fn, 'error_pos'); + body.push(errorPosPtr); + + body.push(this.ctx.ir._('store', + [ TYPE_ERROR, code ], + [ TYPE_ERROR.ptr(), codePtr ]).void()); + body.push(this.ctx.ir._('store', + [ pos.type, pos ], + [ pos.type.ptr(), errorPosPtr ]).void()); + } + // ${prefix}_execute buildPrologue(fn, body) { @@ -208,12 +219,14 @@ class Builder extends Stage { present.push(cb); } + const endPos = this.ctx.endPosArg(fn); + // TODO(indutny): this won't work with internal callbacks due to // cconv mismatch const call = this.ctx.call('', callbackType.to, cb, [ this.ctx.stateArg(fn), start, - this.ctx.endPosArg(fn) + endPos ]); if (num > 1) { const callees = codes.map((code) => { @@ -236,6 +249,7 @@ class Builder extends Stage { // Make sure that the types match assert.strictEqual(fn.type.ret.type, callbackType.to.ret.type); + this.buildError(fn, error, endPos, call); error.terminate('ret', [ fn.type.ret, call ]); noError.terminate('br', empty); From e774b1b1198b2cacff04d216198bf96f2384beb3 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 18:38:16 -0500 Subject: [PATCH 207/281] span-builder: store error `reason` too --- lib/llparse/compiler/stage/span-builder.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 7511006..8537691 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -8,9 +8,11 @@ const llparse = require('../../'); const constants = llparse.constants; const BOOL = constants.BOOL; +const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_ERROR = constants.TYPE_ERROR; +const TYPE_REASON = constants.TYPE_REASON; const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; @@ -131,15 +133,27 @@ class Builder extends Stage { } buildError(fn, body, pos, code) { + const reason = this.ctx.ir.cstr('Span callback error'); + + const cast = this.ctx.ir._('getelementptr inbounds', reason.type.to, + [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); + body.push(cast); + const codePtr = this.ctx.field(fn, 'error'); body.push(codePtr); + const reasonPtr = this.ctx.field(fn, 'reason'); + body.push(reasonPtr); + const errorPosPtr = this.ctx.field(fn, 'error_pos'); body.push(errorPosPtr); body.push(this.ctx.ir._('store', [ TYPE_ERROR, code ], [ TYPE_ERROR.ptr(), codePtr ]).void()); + body.push(this.ctx.ir._('store', + [ TYPE_REASON, cast ], + [ TYPE_REASON.ptr(), reasonPtr ]).void()); body.push(this.ctx.ir._('store', [ pos.type, pos ], [ pos.type.ptr(), errorPosPtr ]).void()); From 3eb1473626f475206a58588ec4311929bc6409e9 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 18:38:53 -0500 Subject: [PATCH 208/281] 2.1.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 430ec29..c0344a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.4", + "version": "2.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index fc0a1dc..8f25b8f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.0.4", + "version": "2.1.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From ee86a482d53819c5d3714219e90541534d63bb62 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 21:35:04 -0500 Subject: [PATCH 209/281] node: stop passing `nodes` everywhere --- lib/llparse/compiler/node/base.js | 13 ++++++++----- lib/llparse/compiler/node/consume.js | 4 ++-- lib/llparse/compiler/node/empty.js | 4 ++-- lib/llparse/compiler/node/invoke.js | 8 +++----- lib/llparse/compiler/node/sequence.js | 7 +++---- lib/llparse/compiler/node/set-index.js | 4 ++-- lib/llparse/compiler/node/single.js | 7 +++---- lib/llparse/compiler/node/span-end.js | 4 ++-- lib/llparse/compiler/node/span-start.js | 4 ++-- 9 files changed, 27 insertions(+), 28 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 1e7dd4e..29c54a1 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -7,10 +7,11 @@ const llparse = require('../../'); const constants = llparse.constants; class NodeContext { - constructor(ctx, name, fn) { + constructor(ctx, name, fn, nodes) { this.compilation = ctx; this.name = name; this.fn = fn; + this.nodes = nodes; this.state = this.compilation.stateArg(this.fn); this.pos = { @@ -81,7 +82,7 @@ class Node { return nodes.get(this); const fn = compilation.fn(compilation.signature.node, this.name); - const ctx = new NodeContext(compilation, this.name, fn); + const ctx = new NodeContext(compilation, this.name, fn, nodes); // Errors are assumed to be rarely called // TODO(indutny): move this to node.Error somehow? @@ -102,7 +103,7 @@ class Node { [ ctx.INT, ctx.INT.v(1) ]); body.push(ctx.pos.next); - this.doBuild(ctx, body, nodes); + this.doBuild(ctx, body); return fn; } @@ -139,6 +140,8 @@ class Node { } tailTo(ctx, body, pos, target, value = null) { + target = target.build(ctx.compilation, ctx.nodes); + if (this.phis.has(target)) { const cached = this.phis.get(target); @@ -186,7 +189,7 @@ class Node { throw new Error('Not implemented'); } - doOtherwise(ctx, nodes, body, pos) { + doOtherwise(ctx, body, pos) { if (!pos) pos = ctx.pos; @@ -195,7 +198,7 @@ class Node { const next = this.skip ? pos.next : pos.current; assert(this.otherwise); - this.tailTo(ctx, body, next, this.otherwise.build(ctx.compilation, nodes)); + this.tailTo(ctx, body, next, this.otherwise); } } module.exports = Node; diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 93f5d41..feede91 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -14,7 +14,7 @@ class Consume extends node.Node { } // TODO(indutny): remove unnecessary load - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { const INVARIANT_GROUP = ctx.INVARIANT_GROUP; body.comment('node.Consume'); @@ -65,7 +65,7 @@ class Consume extends node.Node { hasData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, ctx.TYPE_INDEX.v(0) ], [ ctx.TYPE_INDEX.ptr(), indexPtr ], [ '!invariant.group', INVARIANT_GROUP ]).void()); - this.doOtherwise(ctx, nodes, hasData, { current: next, next: null }); + this.doOtherwise(ctx, hasData, { current: next, next: null }); // Pause! noData.comment('state.index = need - avail'); diff --git a/lib/llparse/compiler/node/empty.js b/lib/llparse/compiler/node/empty.js index 6f089fd..6d9625f 100644 --- a/lib/llparse/compiler/node/empty.js +++ b/lib/llparse/compiler/node/empty.js @@ -7,10 +7,10 @@ class Empty extends node.Node { super('empty', ...args); } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { body.comment('node.Empty'); - this.doOtherwise(ctx, nodes, body); + this.doOtherwise(ctx, body); } } module.exports = Empty; diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index b517e24..27f64b5 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -25,7 +25,7 @@ class Invoke extends node.Node { return body; } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { body.comment(`node.Invoke[${this.code.name}]`); const code = ctx.compilation.buildCode(this.code); @@ -47,12 +47,10 @@ class Invoke extends node.Node { const s = ctx.buildSwitch(body, code.type.ret, call, keys); s.cases.forEach((body, i) => { - const child = this.map[keys[i]].build(ctx.compilation, nodes); - - this.tailTo(ctx, body, ctx.pos.current, child); + this.tailTo(ctx, body, ctx.pos.current, this.map[keys[i]]); }); - this.doOtherwise(ctx, nodes, s.otherwise); + this.doOtherwise(ctx, s.otherwise); } } module.exports = Invoke; diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index b8fac8e..9131ee0 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -28,7 +28,7 @@ class Sequence extends node.Node { }); } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { const INT = ctx.INT; body.comment(`node.Sequence["${this.select.toString('hex') }"]`); @@ -81,12 +81,11 @@ class Sequence extends node.Node { const pause = s.cases[1]; const mismatch = s.cases[2]; - this.tailTo(ctx, complete, next, - this.next.build(ctx.compilation, nodes), this.value); + this.tailTo(ctx, complete, next, this.next, this.value); this.pause(ctx, pause); // Not equal - this.doOtherwise(ctx, nodes, mismatch, { current, next }); + this.doOtherwise(ctx, mismatch, { current, next }); } } module.exports = Sequence; diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js index f0fb23f..8d043cb 100644 --- a/lib/llparse/compiler/node/set-index.js +++ b/lib/llparse/compiler/node/set-index.js @@ -18,7 +18,7 @@ class SetIndex extends node.Node { return body; } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { const INVARIANT_GROUP = ctx.INVARIANT_GROUP; body.comment(`node.SetIndex[${this.code.name}]`); @@ -41,7 +41,7 @@ class SetIndex extends node.Node { [ ctx.TYPE_INDEX.ptr(), ptr ], [ '!invariant.group', INVARIANT_GROUP ]).void()); - this.doOtherwise(ctx, nodes, body); + this.doOtherwise(ctx, body); } } module.exports = SetIndex; diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 7cfd1c3..ec76de4 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -15,7 +15,7 @@ class Single extends node.Node { })); } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { body.comment('node.Single'); const pos = ctx.pos.current; @@ -37,13 +37,12 @@ class Single extends node.Node { s.cases.forEach((body, i) => { const child = this.children[i]; - const target = child.next.build(ctx.compilation, nodes); this.tailTo(ctx, body, child.noAdvance ? ctx.pos.current : ctx.pos.next, - target, child.value); + child.next, child.value); }); - this.doOtherwise(ctx, nodes, s.otherwise); + this.doOtherwise(ctx, s.otherwise); } } module.exports = Single; diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 270c761..4d35bc4 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -13,11 +13,11 @@ class SpanEnd extends node.Node { return body; } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { body.comment(`node.SpanEnd[${this.code.name}]`); body = ctx.compilation.stageResults['span-builder'].spanEnd( ctx.fn, body, this.code); - this.doOtherwise(ctx, nodes, body); + this.doOtherwise(ctx, body); } } module.exports = SpanEnd; diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index e9210b1..e6042df 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -13,11 +13,11 @@ class SpanStart extends node.Node { return body; } - doBuild(ctx, body, nodes) { + doBuild(ctx, body) { body.comment(`node.SpanStart[${this.code.name}]`); body = ctx.compilation.stageResults['span-builder'].spanStart( ctx.fn, body, this.code); - this.doOtherwise(ctx, nodes, body); + this.doOtherwise(ctx, body); } } module.exports = SpanStart; From 7ba597dab91e6dbf3ea746dc3a4d2a5818a0e1d3 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 21:45:52 -0500 Subject: [PATCH 210/281] node/base: store current fn before error --- lib/llparse/compiler/node/base.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 29c54a1..82b9d70 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -139,8 +139,8 @@ class Node { this.hasPause = true; } - tailTo(ctx, body, pos, target, value = null) { - target = target.build(ctx.compilation, ctx.nodes); + tailTo(ctx, body, pos, node, value = null) { + const target = node.build(ctx.compilation, ctx.nodes); if (this.phis.has(target)) { const cached = this.phis.get(target); @@ -172,6 +172,18 @@ class Node { this.phis.set(target, { phi, trampoline }); + // Save state when tailing to Error node for future resumption + if (node.type === 'error') { + trampoline.comment('save current'); + + const currentPtr = ctx.field('_current'); + trampoline.push(currentPtr); + + trampoline.push(ctx.ir._('store', + [ ctx.fn.type.ptr(), ctx.fn ], + [ ctx.fn.type.ptr().ptr(), currentPtr ]).void()); + } + // TODO(indutny): looks like `musttail` gives worse performance when calling // Invoke nodes (possibly others too). const call = ctx.call('musttail', ctx.signature.node, target, [ From d6d26863b395b0169cbd0a3933b75d2c28e721f8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 22:02:16 -0500 Subject: [PATCH 211/281] span-end: set resumption target --- lib/llparse/compiler/node/base.js | 6 ++++-- lib/llparse/compiler/node/span-end.js | 5 +++-- lib/llparse/compiler/stage/span-builder.js | 18 ++++++++++++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 82b9d70..a63b2c2 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -154,7 +154,7 @@ class Node { } body.terminate('br', cached.trampoline); - return; + return target; } // Split, so that others could join us from code block above @@ -195,6 +195,8 @@ class Node { trampoline.push(call); trampoline.terminate('ret', [ ctx.fn.type.ret, call ]); + + return target; } doBuild() { @@ -210,7 +212,7 @@ class Node { const next = this.skip ? pos.next : pos.current; assert(this.otherwise); - this.tailTo(ctx, body, next, this.otherwise); + return this.tailTo(ctx, body, next, this.otherwise); } } module.exports = Node; diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 4d35bc4..c4f3ddc 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -15,9 +15,10 @@ class SpanEnd extends node.Node { doBuild(ctx, body) { body.comment(`node.SpanEnd[${this.code.name}]`); - body = ctx.compilation.stageResults['span-builder'].spanEnd( + const result = ctx.compilation.stageResults['span-builder'].spanEnd( ctx.fn, body, this.code); - this.doOtherwise(ctx, body); + + result.updateResumptionTarget(this.doOtherwise(ctx, result.body)); } } module.exports = SpanEnd; diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 8537691..26c073d 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -127,12 +127,26 @@ class Builder extends Stage { // Handle error assert.strictEqual(TYPE_ERROR.type, signature.ret.type); this.buildError(fn, error, pos, call); - error.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); - return noError; + return { + body: noError, + updateResumptionTarget: (target) => { + error.comment('store current state'); + + const currentPtr = this.ctx.field(fn, '_current'); + error.push(currentPtr); + + error.push(this.ctx.ir._('store', + [ target.type.ptr(), target.ref() ], + [ target.type.ptr().ptr(), currentPtr ]).void()); + + error.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + } + }; } buildError(fn, body, pos, code) { + body.comment('span error'); const reason = this.ctx.ir.cstr('Span callback error'); const cast = this.ctx.ir._('getelementptr inbounds', reason.type.to, From 1ac3392a194c63e574e968b13a744e898cd3e1bc Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 22:05:26 -0500 Subject: [PATCH 212/281] compiler: update resumption comments See: #7 --- lib/llparse/compiler/node/base.js | 2 +- lib/llparse/compiler/stage/span-builder.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index a63b2c2..8c338cd 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -174,7 +174,7 @@ class Node { // Save state when tailing to Error node for future resumption if (node.type === 'error') { - trampoline.comment('save current'); + trampoline.comment('store resumption target (error)'); const currentPtr = ctx.field('_current'); trampoline.push(currentPtr); diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 26c073d..0d14ab0 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -131,7 +131,7 @@ class Builder extends Stage { return { body: noError, updateResumptionTarget: (target) => { - error.comment('store current state'); + error.comment('store resumption target'); const currentPtr = this.ctx.field(fn, '_current'); error.push(currentPtr); From 157daa6039fc12574d5b5195ae2b6b11ebff1606 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 22:59:22 -0500 Subject: [PATCH 213/281] test: test errors --- package-lock.json | 6 +++--- package.json | 2 +- test/api-test.js | 11 +++++++++++ test/fixtures/extra.c | 8 ++++++++ test/span-test.js | 22 ++++++++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index c0344a4..f27d8f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -740,9 +740,9 @@ } }, "llparse-test-fixture": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.0.0.tgz", - "integrity": "sha512-HxTURBDxijBtbCUCwRtvyAsCvmPM+kfhIXpRc87yGH+Ca84w0jy8gApdS8JUFCtSG0os6zNvPRx4dv5mnQbxQw==", + "version": "1.1.0-beta1", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.1.0-beta1.tgz", + "integrity": "sha512-azTgN8DqmfRJw/c/zBK/5FE2KTKTjm5E8WwNFIrm3I88/5teOv2TS6RaH71iSojh7WgW2xNTtqRbqMDMaRvrJw==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index 8f25b8f..fb196e8 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.0.0", + "llparse-test-fixture": "^1.1.0-beta1", "mocha": "^5.0.1" }, "directories": { diff --git a/test/api-test.js b/test/api-test.js index 72b7d67..c3c8309 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -126,6 +126,17 @@ describe('LLParse', function() { binary('0110', 'off=1 0\noff=2 1\noff=3 1\noff=4 0\n', callback); }); + it('should return error code/reason', (callback) => { + const start = p.node('start'); + + start.match('a', start); + start.otherwise(p.error(42, 'some reason')); + + const binary = fixtures.build(p, start, 'error'); + + binary('aab', 'off=2 error code=42 reason="some reason"\n', callback); + }); + describe('`.peek()`', () => { it('should not advance position', (callback) => { const start = p.node('start'); diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c index f3839fd..6ab260e 100644 --- a/test/fixtures/extra.c +++ b/test/fixtures/extra.c @@ -53,3 +53,11 @@ int llparse__on_underscore(llparse_state_t* s, const char* p, return 0; return llparse__print_span("underscore", p, endp); } + + +/* A span callback, really */ +int llparse__please_fail(llparse_state_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 1; + return 1; +} diff --git a/test/span-test.js b/test/span-test.js index 88c5d96..929f84f 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -83,4 +83,26 @@ describe('LLParse/span', function() { assert.doesNotThrow(() => p.build(span.start(start))); }); + + it('should return error', (callback) => { + const start = p.node('start'); + const dot = p.node('dot'); + + const span = { + pleaseFail: p.span(p.code.span('llparse__please_fail')) + }; + + start.otherwise(span.pleaseFail.start(dot)); + + dot + .match('.', dot) + .skipTo(span.pleaseFail.end(start)); + + const binary = fixtures.build(p, start, 'span'); + + binary( + '....a', + /off=\d+ error code=1 reason="Span callback error"\n/, + callback); + }); }); From c98690583f078028f0aa22db5bfb973bbd2bd259 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 23:44:30 -0500 Subject: [PATCH 214/281] test: test resumption, errors --- lib/llparse/compiler/compiler.js | 3 -- lib/llparse/node/error.js | 9 ++++++ test/fixtures/extra.c | 15 +++++++++ test/fixtures/index.js | 6 +++- test/resumption-test.js | 53 ++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 test/resumption-test.js diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index ca0b812..8917f35 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -165,9 +165,6 @@ class Compiler { const code = ctx.ir._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); error.push(code); - error.push(ctx.ir._('store', [ nodeSig.ptr(), nodeSig.ptr().v(null) ], - [ nodeSig.ptr().ptr(), currentPtr ]).void()); - error.terminate('ret', [ TYPE_ERROR, code ]); return `int ${this.prefix}_execute(${this.prefix}_state_t* s, ` + diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js index 45e41e0..a5e9c56 100644 --- a/lib/llparse/node/error.js +++ b/lib/llparse/node/error.js @@ -1,5 +1,7 @@ 'use strict'; +const assert = require('assert'); + const node = require('./'); const kCode = Symbol('code'); @@ -9,6 +11,13 @@ class Error extends node.Node { constructor(code, reason) { super('error', 'match'); + assert.strictEqual(typeof code, 'number', + 'The first argument of `.error()` must be a numeric error code'); + assert.strictEqual(code, code | 0, + 'The first argument of `.error()` must be an integer error code'); + assert.strictEqual(typeof reason, 'string', + 'The second argument of `.error()` must be a string error description'); + this[kCode] = code; this[kReason] = reason; } diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c index 6ab260e..b180ea9 100644 --- a/test/fixtures/extra.c +++ b/test/fixtures/extra.c @@ -61,3 +61,18 @@ int llparse__please_fail(llparse_state_t* s, const char* p, const char* endp) { return 1; return 1; } + + +/* A span callback, really */ +static llparse__pause_once_counter; + +int llparse__pause_once(llparse_state_t* s, const char* p, const char* endp) { + if (!llparse__in_bench) + llparse__print_span("pause", p, endp); + + if (llparse__pause_once_counter != 0) + return 0; + llparse__pause_once_counter = 1; + + return LLPARSE__ERROR_PAUSE; +} diff --git a/test/fixtures/index.js b/test/fixtures/index.js index e8d957e..bdf04cc 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -2,7 +2,9 @@ const path = require('path'); -const fixtures = require('llparse-test-fixture').create({ +const Fixture = require('llparse-test-fixture'); + +const fixtures = Fixture.create({ buildDir: path.join(__dirname, '..', 'tmp'), extra: [ path.join(__dirname, 'extra.c') ] }); @@ -25,5 +27,7 @@ exports.NUM = { '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9 }; +exports.ERROR_PAUSE = Fixture.ERROR_PAUSE; + // Reasonable timeout for CI exports.TIMEOUT = 10000; diff --git a/test/resumption-test.js b/test/resumption-test.js new file mode 100644 index 0000000..62a0dee --- /dev/null +++ b/test/resumption-test.js @@ -0,0 +1,53 @@ +'use strict'; +/* global describe it beforeEach */ + +const llparse = require('../'); + +const fixtures = require('./fixtures'); + +describe('LLParse/resumption', function() { + this.timeout(fixtures.TIMEOUT); + + let p; + beforeEach(() => { + p = llparse.create('llparse'); + }); + + it('should resume after error', (callback) => { + const start = p.node('start'); + + start + .match('a', start) + .skipTo(p.error(fixtures.ERROR_PAUSE, 'pause')); + + const binary = fixtures.build(p, start, 'resume-error'); + + binary('abab', 'off=2 pause\noff=4 pause\n', callback); + }); + + it('should resume after span end pause', (callback) => { + const start = p.node('start'); + const a = p.node('a'); + const span = p.span(p.code.span('llparse__pause_once')); + + start + .peek('a', span.start(a)) + .skipTo(start); + + a + .match('a', a) + .otherwise(span.end(start)); + + const binary = fixtures.build(p, start, 'resume-span'); + + binary('baaab', + new RegExp( + '^('+ + 'off=\\d+ pause\\noff=1 len=3 span\\[pause\\]="aaa"' + + '|' + + 'off=1 len=3 span\\[pause\\]="aaa"\noff=4 pause' + + ')\\n$' + , 'g'), + callback); + }); +}); From 278537d880dcbf7cf8e5e7395fc330fd3506a202 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 26 Feb 2018 23:47:50 -0500 Subject: [PATCH 215/281] package: set fixture to stable --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index f27d8f3..1d65f93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -740,9 +740,9 @@ } }, "llparse-test-fixture": { - "version": "1.1.0-beta1", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.1.0-beta1.tgz", - "integrity": "sha512-azTgN8DqmfRJw/c/zBK/5FE2KTKTjm5E8WwNFIrm3I88/5teOv2TS6RaH71iSojh7WgW2xNTtqRbqMDMaRvrJw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.1.0.tgz", + "integrity": "sha512-LSlFr7w2zA3215nBvPEuEF2iw6D6wnS9GkTG7KgswLgPr6As0c7uCVR+AbAVO2JDVnDtyvtTKFUo+tpt2XNS4g==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index fb196e8..b7b2a92 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.1.0-beta1", + "llparse-test-fixture": "^1.1.0", "mocha": "^5.0.1" }, "directories": { From dc831ef1d840c095d042220d4144ec341f2ba5d6 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 01:12:27 -0500 Subject: [PATCH 216/281] 2.2.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1d65f93..8e8a805 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.1.0", + "version": "2.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index b7b2a92..d745efe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.1.0", + "version": "2.2.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From f4dbbbef3bb8291d2084cf821f2ef30cc22a3e30 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 02:19:52 -0500 Subject: [PATCH 217/281] compiler: errors are not recoverable! --- lib/llparse/compiler/node/base.js | 12 ------------ lib/llparse/compiler/node/error.js | 8 ++++++++ test/resumption-test.js | 12 ------------ 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 8c338cd..ee90466 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -172,18 +172,6 @@ class Node { this.phis.set(target, { phi, trampoline }); - // Save state when tailing to Error node for future resumption - if (node.type === 'error') { - trampoline.comment('store resumption target (error)'); - - const currentPtr = ctx.field('_current'); - trampoline.push(currentPtr); - - trampoline.push(ctx.ir._('store', - [ ctx.fn.type.ptr(), ctx.fn ], - [ ctx.fn.type.ptr().ptr(), currentPtr ]).void()); - } - // TODO(indutny): looks like `musttail` gives worse performance when calling // Invoke nodes (possibly others too). const call = ctx.call('musttail', ctx.signature.node, target, [ diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index a87d35b..72086c0 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -25,6 +25,9 @@ class Error extends node.Node { const reason = ctx.ir.cstr(this.reason); + const currentPtr = ctx.field('_current'); + body.push(currentPtr); + const codePtr = ctx.field('error'); body.push(codePtr); @@ -38,6 +41,11 @@ class Error extends node.Node { [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); body.push(cast); + // Non-recoverable state + const nodeSig = ctx.compilation.signature.node; + body.push(ctx.ir._('store', [ nodeSig.ptr(), nodeSig.ptr().v(null) ], + [ nodeSig.ptr().ptr(), currentPtr ]).void()); + body.push(ctx.ir._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(this.code) ], [ ctx.TYPE_ERROR.ptr(), codePtr ]).void()); body.push(ctx.ir._('store', [ ctx.TYPE_REASON, cast ], diff --git a/test/resumption-test.js b/test/resumption-test.js index 62a0dee..47ac078 100644 --- a/test/resumption-test.js +++ b/test/resumption-test.js @@ -13,18 +13,6 @@ describe('LLParse/resumption', function() { p = llparse.create('llparse'); }); - it('should resume after error', (callback) => { - const start = p.node('start'); - - start - .match('a', start) - .skipTo(p.error(fixtures.ERROR_PAUSE, 'pause')); - - const binary = fixtures.build(p, start, 'resume-error'); - - binary('abab', 'off=2 pause\noff=4 pause\n', callback); - }); - it('should resume after span end pause', (callback) => { const start = p.node('start'); const a = p.node('a'); From bcd35a472caa8b670597bbd06fa4a177f226186b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 14:07:58 -0500 Subject: [PATCH 218/281] test: fix warnings --- test/fixtures/extra.c | 2 +- test/span-test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c index b180ea9..3a3f9d6 100644 --- a/test/fixtures/extra.c +++ b/test/fixtures/extra.c @@ -64,7 +64,7 @@ int llparse__please_fail(llparse_state_t* s, const char* p, const char* endp) { /* A span callback, really */ -static llparse__pause_once_counter; +static int llparse__pause_once_counter; int llparse__pause_once(llparse_state_t* s, const char* p, const char* endp) { if (!llparse__in_bench) diff --git a/test/span-test.js b/test/span-test.js index 929f84f..4e76f2b 100644 --- a/test/span-test.js +++ b/test/span-test.js @@ -98,7 +98,7 @@ describe('LLParse/span', function() { .match('.', dot) .skipTo(span.pleaseFail.end(start)); - const binary = fixtures.build(p, start, 'span'); + const binary = fixtures.build(p, start, 'span-error'); binary( '....a', From 367d9112c1ba342c43e0cab2b0fc5e7704b01a18 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 16:51:25 -0500 Subject: [PATCH 219/281] node: `.pause()` Fix: #7 --- lib/llparse.js | 4 +++ lib/llparse/compiler/node/base.js | 6 +++- lib/llparse/compiler/node/error.js | 29 +++++++++++------- lib/llparse/compiler/node/index.js | 1 + lib/llparse/compiler/node/pause.js | 29 ++++++++++++++++++ lib/llparse/compiler/stage/node-translator.js | 2 ++ lib/llparse/node/index.js | 1 + lib/llparse/node/pause.js | 30 +++++++++++++++++++ test/resumption-test.js | 18 +++++++++++ 9 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 lib/llparse/compiler/node/pause.js create mode 100644 lib/llparse/node/pause.js diff --git a/lib/llparse.js b/lib/llparse.js index 0145cdf..8126c50 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -126,6 +126,10 @@ class LLParse { return new internal.node.Consume(code); } + pause(code, reason) { + return new internal.node.Pause(code, reason); + } + build(root, options) { assert(root, 'Missing required argument for `.build(root)`'); assert(root instanceof internal.node.Node, diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index ee90466..64948d5 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -139,8 +139,12 @@ class Node { this.hasPause = true; } + buildNode(ctx, node) { + return node.build(ctx.compilation, ctx.nodes); + } + tailTo(ctx, body, pos, node, value = null) { - const target = node.build(ctx.compilation, ctx.nodes); + const target = this.buildNode(ctx, node); if (this.phis.has(target)) { const cached = this.phis.get(target); diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index 72086c0..1a1adc2 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -18,16 +18,11 @@ class Error extends node.Node { return body; } - doBuild(ctx, body) { - body.comment('node.Error'); - + buildStoreError(ctx, body) { const INT = ctx.INT; const reason = ctx.ir.cstr(this.reason); - const currentPtr = ctx.field('_current'); - body.push(currentPtr); - const codePtr = ctx.field('error'); body.push(codePtr); @@ -41,11 +36,6 @@ class Error extends node.Node { [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); body.push(cast); - // Non-recoverable state - const nodeSig = ctx.compilation.signature.node; - body.push(ctx.ir._('store', [ nodeSig.ptr(), nodeSig.ptr().v(null) ], - [ nodeSig.ptr().ptr(), currentPtr ]).void()); - body.push(ctx.ir._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(this.code) ], [ ctx.TYPE_ERROR.ptr(), codePtr ]).void()); body.push(ctx.ir._('store', [ ctx.TYPE_REASON, cast ], @@ -53,6 +43,23 @@ class Error extends node.Node { body.push(ctx.ir._('store', [ ctx.pos.current.type, ctx.pos.current ], [ ctx.pos.current.type.ptr(), posPtr ]).void()); + return body; + } + + doBuild(ctx, body) { + body.comment('node.Error'); + + body = this.buildStoreError(ctx, body); + + body.comment('state.current = null'); + const currentPtr = ctx.field('_current'); + body.push(currentPtr); + + // Non-recoverable state + const nodeSig = ctx.compilation.signature.node; + body.push(ctx.ir._('store', [ nodeSig.ptr(), nodeSig.ptr().v(null) ], + [ nodeSig.ptr().ptr(), currentPtr ]).void()); + const retType = ctx.fn.type.ret; body.terminate('ret', [ retType, retType.v(null) ]); } diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index 498ef82..6fda8cf 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -10,3 +10,4 @@ exports.SpanStart = require('./span-start'); exports.SpanEnd = require('./span-end'); exports.SetIndex = require('./set-index'); exports.Consume = require('./consume'); +exports.Pause = require('./pause'); diff --git a/lib/llparse/compiler/node/pause.js b/lib/llparse/compiler/node/pause.js new file mode 100644 index 0000000..c8d28aa --- /dev/null +++ b/lib/llparse/compiler/node/pause.js @@ -0,0 +1,29 @@ +'use strict'; + +const node = require('./'); + +class Pause extends node.Error { + constructor(id, code, reason) { + super(id, code, reason); + this.type = 'pause'; + } + + doBuild(ctx, body) { + body.comment('node.Pause'); + + body = this.buildStoreError(ctx, body); + + body.comment('state.current = next'); + const currentPtr = ctx.field('_current'); + body.push(currentPtr); + + // Recoverable state + const target = this.buildNode(ctx, this.otherwise); + body.push(ctx.ir._('store', [ target.type.ptr(), target.ref() ], + [ target.type.ptr().ptr(), currentPtr ]).void()); + + const retType = ctx.fn.type.ret; + body.terminate('ret', [ retType, retType.v(null) ]); + } +} +module.exports = Pause; diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index bc256a0..ea3cd9c 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -57,6 +57,8 @@ class NodeTranslator extends Stage { res = new compiler.node.SpanStart(this.id(node), node[kSpan][kCode]); } else if (node instanceof llparse.node.SpanEnd) { res = new compiler.node.SpanEnd(this.id(node), node[kSpan][kCode]); + } else if (node instanceof llparse.node.Pause) { + res = new compiler.node.Pause(this.id(node), node.code, node.reason); } else if (node instanceof llparse.node.Consume) { res = this.buildConsume(node); last = res.last; diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js index 5476ce7..9cc500d 100644 --- a/lib/llparse/node/index.js +++ b/lib/llparse/node/index.js @@ -8,3 +8,4 @@ exports.Invoke = require('./invoke'); exports.SpanStart = require('./span-start'); exports.SpanEnd = require('./span-end'); exports.Consume = require('./consume'); +exports.Pause = require('./pause'); diff --git a/lib/llparse/node/pause.js b/lib/llparse/node/pause.js new file mode 100644 index 0000000..16b525b --- /dev/null +++ b/lib/llparse/node/pause.js @@ -0,0 +1,30 @@ +'use strict'; + +const assert = require('assert'); + +const node = require('./'); + +const kCode = Symbol('code'); +const kReason = Symbol('reason'); + +class Pause extends node.Node { + constructor(code, reason) { + super('pause', 'match'); + + assert.strictEqual(typeof code, 'number', + 'The first argument of `.error()` must be a numeric error code'); + assert.strictEqual(code, code | 0, + 'The first argument of `.error()` must be an integer error code'); + assert.strictEqual(typeof reason, 'string', + 'The second argument of `.error()` must be a string error description'); + + this[kCode] = code; + this[kReason] = reason; + } + + get code() { return this[kCode]; } + get reason() { return this[kReason]; } + + skipTo() { throw new Error('Not supported, please use `pause.otherwise()`'); } +} +module.exports = Pause; diff --git a/test/resumption-test.js b/test/resumption-test.js index 47ac078..d71c3b1 100644 --- a/test/resumption-test.js +++ b/test/resumption-test.js @@ -38,4 +38,22 @@ describe('LLParse/resumption', function() { , 'g'), callback); }); + + it('should resume after `pause` node', (callback) => { + const start = p.node('start'); + const pause = p.pause(fixtures.ERROR_PAUSE, 'paused'); + + start + .match('p', pause) + .skipTo(start); + + pause + .otherwise(fixtures.printOff(p, start)); + + const binary = fixtures.build(p, start, 'resume-pause'); + + binary('..p....p..', + 'off=3 pause\noff=3\noff=8 pause\noff=8\n', + callback); + }); }); From 37aa02f7b8db3e301c75b10e47f61a72c7c3a61d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 16:52:44 -0500 Subject: [PATCH 220/281] 2.3.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8e8a805..2b35a3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.2.0", + "version": "2.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index d745efe..617df84 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.2.0", + "version": "2.3.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 1cf40a87c9b5688318cc1dab72604acb89a962e2 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 22:15:47 -0500 Subject: [PATCH 221/281] compilation: fix function attributes --- lib/llparse/compiler/compilation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 4c10d85..48eab38 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -236,7 +236,7 @@ class Compilation { // These are ABI dependent, but should bring us close to C-function // inlining on x86_64 through -flto - fn.attribute += ' ssp uwtable'; + fn.attributes += ' ssp uwtable'; } else if (signature === this.signature.callback.match) { fn = this.ir.fn(this.signature.callback.match, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); From 88d7fa3065bea96edd24452f872d5ddb41a14b43 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 22:33:06 -0500 Subject: [PATCH 222/281] match-sequence: add !range metadata to the load Fix: #12 --- lib/llparse/compiler/compiler.js | 2 +- lib/llparse/compiler/stage/match-sequence.js | 45 +++++++++++-------- lib/llparse/compiler/stage/node-builder.js | 4 +- .../compiler/stage/node-loop-checker.js | 2 +- lib/llparse/compiler/stage/node-translator.js | 8 +++- lib/llparse/compiler/stage/span-allocator.js | 3 +- 6 files changed, 39 insertions(+), 25 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 8917f35..26ac9a5 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -37,8 +37,8 @@ class Compiler { root, stages: { before: [ - llparse.compiler.stage.MatchSequence, llparse.compiler.stage.NodeTranslator, + llparse.compiler.stage.MatchSequence, llparse.compiler.stage.NodeLoopChecker, llparse.compiler.stage.SpanAllocator, llparse.compiler.stage.SpanBuilder diff --git a/lib/llparse/compiler/stage/match-sequence.js b/lib/llparse/compiler/stage/match-sequence.js index 398ce68..e2e7bb7 100644 --- a/lib/llparse/compiler/stage/match-sequence.js +++ b/lib/llparse/compiler/stage/match-sequence.js @@ -1,7 +1,5 @@ 'use strict'; -const IR = require('llvm-ir'); - const Stage = require('./').Stage; const llparse = require('../../'); const constants = llparse.constants; @@ -38,7 +36,7 @@ class MatchSequence extends Stage { this.returnType.field(TYPE_INPUT, 'current'); this.returnType.field(TYPE_STATUS, 'status'); - this.signature = IR.signature(this.returnType, [ + this.signature = this.ctx.ir.signature(this.returnType, [ [ this.ctx.state.ptr(), ATTR_STATE ], [ TYPE_INPUT, ATTR_POS ], [ TYPE_INPUT, ATTR_ENDPOS ], @@ -80,6 +78,10 @@ class MatchSequence extends Stage { // TODO(indutny): initialize `state.index` before calling matcher? buildBody(fn, transform) { const body = fn.body; + const ir = this.ctx.ir; + + const maxSeqLen = this.ctx.stageResults['node-translator'].maxSequenceLen; + const range = `${TYPE_INDEX.type} 0, ${TYPE_INDEX.type} ${maxSeqLen}`; // Just to have a label const start = body.jump('br'); @@ -89,18 +91,19 @@ class MatchSequence extends Stage { start.comment('index = state.index'); const indexPtr = this.ctx.field(fn, '_index'); start.push(indexPtr); - const index = IR._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ], - [ '!invariant.group', this.ctx.INVARIANT_GROUP ]); + const index = ir._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ], + [ '!invariant.group', this.ctx.INVARIANT_GROUP ], + [ '!range', ir.metadata(range) ]); start.push(index); const loop = start.jump('br'); loop.name = 'loop'; // Loop start - const posPhi = IR._('phi', + const posPhi = ir._('phi', [ TYPE_INPUT, '[', fn.arg(ARG_POS), ',', start.ref(), ']' ]); loop.push(posPhi); - const indexPhi = IR._('phi', + const indexPhi = ir._('phi', [ TYPE_INDEX, '[', index, ',', start.ref(), ']' ]); loop.push(indexPhi); @@ -124,11 +127,13 @@ class MatchSequence extends Stage { buildIteration(fn, body, indexField, pos, index, transform) { + const ir = this.ctx.ir; + const seq = fn.arg(ARG_SEQUENCE); const seqLen = fn.arg(ARG_SEQUENCE_LEN); body.comment('current = *pos'); - let current = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); + let current = ir._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); body.push(current); // Transform the character if needed @@ -140,22 +145,22 @@ class MatchSequence extends Stage { } body.comment('expected = seq[state.index]'); - const expectedPtr = IR._('getelementptr', seq.type.to, + const expectedPtr = ir._('getelementptr', seq.type.to, [ seq.type, seq ], [ INT, index ]); body.push(expectedPtr); - const expected = IR._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); + const expected = ir._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); body.push(expected); // NOTE: fetch this early so it would dominate all returns body.comment('next = pos + 1'); - const next = IR._('getelementptr', TYPE_INPUT.to, + const next = ir._('getelementptr', TYPE_INPUT.to, [ TYPE_INPUT, pos ], [ INT, INT.v(1) ]); body.push(next); body.comment('if (current == expected)'); - let cmp = IR._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); + let cmp = ir._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); body.push(cmp); const { left: isMatch, right: isMismatch } = body.branch('br', [ BOOL, cmp ]); @@ -171,11 +176,11 @@ class MatchSequence extends Stage { isMatch.comment('Got a char match'); isMatch.comment('index1 = index + 1'); - const index1 = IR._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); + const index1 = ir._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); isMatch.push(index1); isMatch.comment('if (index1 == seq.length)'); - cmp = IR._('icmp', [ 'eq', TYPE_INDEX, index1 ], seqLen); + cmp = ir._('icmp', [ 'eq', TYPE_INDEX, index1 ], seqLen); isMatch.push(cmp); const { left: isComplete, right: isIncomplete } = isMatch.branch('br', [ BOOL, cmp ]); @@ -186,7 +191,7 @@ class MatchSequence extends Stage { isIncomplete.name = 'is_incomplete'; isIncomplete.comment('if (next != endpos)'); - cmp = IR._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); + cmp = ir._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); isIncomplete.push(cmp); const { left: moreData, right: noMoreData } = isIncomplete.branch('br', [ BOOL, cmp ]); @@ -195,7 +200,7 @@ class MatchSequence extends Stage { noMoreData.name = 'no_more_data'; noMoreData.comment('state.index = index1'); - noMoreData.push(IR._('store', [ TYPE_INDEX, index1 ], + noMoreData.push(ir._('store', [ TYPE_INDEX, index1 ], [ TYPE_INDEX.ptr(), indexField ], [ '!invariant.group', this.ctx.INVARIANT_GROUP ]).void()); @@ -210,12 +215,14 @@ class MatchSequence extends Stage { } ret(body, pos, status) { - const create = IR._('insertvalue', [ this.returnType, 'undef' ], + const ir = this.ctx.ir; + + const create = ir._('insertvalue', [ this.returnType, 'undef' ], [ TYPE_INPUT, pos.current ], INT.v(this.returnType.lookup('current'))); body.push(create); - const amend = IR._('insertvalue', [ this.returnType, create ], + const amend = ir._('insertvalue', [ this.returnType, create ], [ TYPE_STATUS, TYPE_STATUS.v(status) ], INT.v(this.returnType.lookup('status'))); body.push(amend); @@ -224,7 +231,7 @@ class MatchSequence extends Stage { } reset(field) { - return IR._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], + return this.ctx.ir._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], [ TYPE_INDEX.ptr(), field ], [ '!invariant.group', this.ctx.INVARIANT_GROUP ]).void(); } diff --git a/lib/llparse/compiler/stage/node-builder.js b/lib/llparse/compiler/stage/node-builder.js index 3d7507f..84024b0 100644 --- a/lib/llparse/compiler/stage/node-builder.js +++ b/lib/llparse/compiler/stage/node-builder.js @@ -10,9 +10,9 @@ class NodeBuilder extends Stage { } build() { + const root = this.ctx.stageResults['node-translator'].root; return { - entry: - this.ctx.stageResults['node-translator'].build(this.ctx, this.nodes), + entry: root.build(this.ctx, this.nodes), map: this.nodes }; } diff --git a/lib/llparse/compiler/stage/node-loop-checker.js b/lib/llparse/compiler/stage/node-loop-checker.js index 91cf5f5..a5b1aff 100644 --- a/lib/llparse/compiler/stage/node-loop-checker.js +++ b/lib/llparse/compiler/stage/node-loop-checker.js @@ -40,7 +40,7 @@ class NodeLoopChecker extends Stage { build() { const queue = [ { - node: this.ctx.stageResults['node-translator'], + node: this.ctx.stageResults['node-translator'].root, key: null } ]; diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index ea3cd9c..5415af6 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -20,10 +20,14 @@ class NodeTranslator extends Stage { this.nodes = new Map(); this.namespace = new Set(); + this.maxSequenceLen = 0; } build() { - return this.buildNode(this.ctx.root, null); + return { + root: this.buildNode(this.ctx.root, null), + maxSequenceLen: this.maxSequenceLen + }; } id(node, postfix = '') { @@ -147,6 +151,8 @@ class NodeTranslator extends Stage { const res = new compiler.node.Sequence(this.id(node), trie.select); const value = trie.child.type === 'next' ? trie.child.value : null; + this.maxSequenceLen = Math.max(this.maxSequenceLen, trie.select.length); + // Break loops if (isRoot) this.nodes.set(node, res); diff --git a/lib/llparse/compiler/stage/span-allocator.js b/lib/llparse/compiler/stage/span-allocator.js index 5abe1fe..709cd78 100644 --- a/lib/llparse/compiler/stage/span-allocator.js +++ b/lib/llparse/compiler/stage/span-allocator.js @@ -15,7 +15,8 @@ class Allocator extends Stage { } build() { - const nodes = this.getNodes(this.ctx.stageResults['node-translator']); + const root = this.ctx.stageResults['node-translator'].root; + const nodes = this.getNodes(root); const info = this.computeActive(nodes); const overlap = this.computeOverlap(info); const color = this.color(info.spans, overlap); From 8db96e9448513b8a1756f913897b1df11c3b6459 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 23:02:33 -0500 Subject: [PATCH 223/281] compiler: add weights to branches See: #13 --- lib/llparse/compiler/compilation.js | 32 ++++++++++++++++++++ lib/llparse/compiler/compiler.js | 4 +-- lib/llparse/compiler/node/base.js | 3 +- lib/llparse/compiler/stage/match-sequence.js | 2 +- lib/llparse/compiler/stage/span-builder.js | 3 +- 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 48eab38..a26bb0c 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -11,6 +11,7 @@ const kBody = llparse.symbols.kBody; const CCONV = constants.CCONV; +const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; @@ -281,6 +282,37 @@ class Compilation { return res; } + branch(body, cmp, weights) { + const extra = []; + if (weights) { + assert(Array.isArray(weights)); + assert.strictEqual(weights.length, 2); + + weights = weights.map((w) => { + if (w === 'likely') + return 0x10000; + else if (w === 'unlikely') + return 0x0; + + assert.strictEqual(w, w | 0); + return w; + }); + + const meta = this.ir.metadata( + `!"branch_weights", i32 ${weights[0]}, i32 ${weights[1]}`); + extra.push([ '!prof', meta ]); + } + const branches = body.terminate('br', [ BOOL, cmp ], + this.ir.label('true'), + this.ir.label('false'), + ...extra); + + return { + left: branches[0], + right: branches[1] + }; + } + buildSwitch(body, type, what, values) { const cases = []; cases.push(IR.label('otherwise')); diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 26ac9a5..207fcdb 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -3,8 +3,6 @@ const llparse = require('../'); const constants = llparse.constants; -const BOOL = constants.BOOL; - const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_MATCH = constants.TYPE_MATCH; @@ -140,7 +138,7 @@ class Compiler { nodeSig.ret.v(null)); body.push(cmp); - const branch = body.branch('br', [ BOOL, cmp ]); + const branch = ctx.branch(body, cmp, [ 'likely', 'unlikely' ]); const success = branch.left; const error = branch.right; diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 64948d5..97c82b1 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -51,6 +51,7 @@ class NodeContext { call(...args) { return this.compilation.call(...args); } buildSwitch(...args) { return this.compilation.buildSwitch(...args); } + branch(...args) { return this.compilation.branch(...args); } } class Node { @@ -118,7 +119,7 @@ class Node { const cmp = ctx.ir._('icmp', [ 'ne', pos.type, pos ], endPos); body.push(cmp); - const branch = body.branch('br', [ ctx.BOOL, cmp ]); + const branch = ctx.branch(body, cmp); // Return self when `pos === endpos` branch.right.name = 'no_data'; diff --git a/lib/llparse/compiler/stage/match-sequence.js b/lib/llparse/compiler/stage/match-sequence.js index e2e7bb7..e85ae21 100644 --- a/lib/llparse/compiler/stage/match-sequence.js +++ b/lib/llparse/compiler/stage/match-sequence.js @@ -194,7 +194,7 @@ class MatchSequence extends Stage { cmp = ir._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); isIncomplete.push(cmp); const { left: moreData, right: noMoreData } = - isIncomplete.branch('br', [ BOOL, cmp ]); + this.ctx.branch(isIncomplete, cmp, [ 'likely', 'unlikely' ]); moreData.name = 'more_data'; diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 0d14ab0..c8723c3 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -118,7 +118,8 @@ class Builder extends Stage { [ 'eq', signature.ret, call ], [ signature.ret.v(0) ]); body.push(errorCmp); - const errorBranch = body.branch('br', [ BOOL, errorCmp ]); + const errorBranch = this.ctx.branch(body, errorCmp, + [ 'likely', 'unlikely' ]); const noError = errorBranch.left; const error = errorBranch.right; noError.name = 'no_error_' + index; From a5bf3c6fe41b5d3dc5eb4796dfd3c63991358c37 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 23:34:15 -0500 Subject: [PATCH 224/281] compiler: add weights to most branches See: #13 --- lib/llparse/code/context.js | 1 + lib/llparse/code/mul-add.js | 6 +++--- lib/llparse/compiler/node/consume.js | 2 +- lib/llparse/compiler/stage/span-builder.js | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/llparse/code/context.js b/lib/llparse/code/context.js index ffe26e1..ffdc3ea 100644 --- a/lib/llparse/code/context.js +++ b/lib/llparse/code/context.js @@ -98,6 +98,7 @@ class Context { } call(...args) { return this[kCompilation].call(...args); } + branch(...args) { return this[kCompilation].branch(...args); } [kLookup](field) { const stateArg = this.state; diff --git a/lib/llparse/code/mul-add.js b/lib/llparse/code/mul-add.js index 8f03fe1..02b6c8c 100644 --- a/lib/llparse/code/mul-add.js +++ b/lib/llparse/code/mul-add.js @@ -83,7 +83,7 @@ class MulAdd extends code.Value { body.push(overflowBit); const { left: isOverflow, right: normal } = - body.branch('br', [ BOOL, overflowBit ]); + context.branch(body, overflowBit, [ 'unlikely', 'likely' ]); isOverflow.name = 'overflow'; isOverflow.terminate('ret', [ context.ret, context.ret.v(1) ]); @@ -104,7 +104,7 @@ class MulAdd extends code.Value { normal.push(addOverflowBit); const { left: isAddOverflow, right: check } = - normal.branch('br', [ BOOL, addOverflowBit ]); + context.branch(normal, addOverflowBit, [ 'unlikely', 'likely' ]); isAddOverflow.name = 'add_overflow'; check.name = 'check'; @@ -118,7 +118,7 @@ class MulAdd extends code.Value { fieldType.v(options.max)); check.push(cmp); - const branch = check.branch('br', [ BOOL, cmp ]); + const branch = context.branch(check, cmp, [ 'unlikely', 'likely' ]); branch.left.name = 'max_overflow'; branch.left.terminate('ret', [ context.ret, context.ret.v(1) ]); diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index feede91..77772fc 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -47,7 +47,7 @@ class Consume extends node.Node { body.comment('if (avail >= need)'); const cmp = ctx.ir._('icmp', [ 'uge', ctx.TYPE_INTPTR, avail ], need); body.push(cmp); - const branch = body.branch('br', [ ctx.BOOL, cmp ]); + const branch = ctx.branch(body, cmp, [ 'likely', 'unlikely' ]); const hasData = branch.left; const noData = branch.right; diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index c8723c3..26e6b82 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -270,7 +270,8 @@ class Builder extends Stage { [ 'eq', callbackType.to.ret, call ], [ callbackType.to.ret.v(0) ]); present.push(errorCmp); - const errorBranch = present.branch('br', [ BOOL, errorCmp ]); + const errorBranch = this.ctx.branch(present, errorCmp, + [ 'likely', 'unlikely' ]); const noError = errorBranch.left; const error = errorBranch.right; noError.name = 'no_error_' + index; From 4324a5a1ee6c629a8317ab7c91136023d99b33b4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 27 Feb 2018 23:41:56 -0500 Subject: [PATCH 225/281] base: error nodes can't recurse --- lib/llparse/compiler/node/base.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 97c82b1..a6174f4 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -89,7 +89,7 @@ class Node { // TODO(indutny): move this to node.Error somehow? if (this instanceof node.Error) { fn.attributes = fn.attributes || ''; - fn.attributes += ' cold writeonly'; + fn.attributes += ' norecurse cold writeonly'; } nodes.set(this, fn); From 20e890049be4f32fb268ee8048c6ac9d604341db Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 28 Feb 2018 00:23:30 -0500 Subject: [PATCH 226/281] 2.3.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2b35a3d..2b4c577 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.0", + "version": "2.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 617df84..3296bb5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.0", + "version": "2.3.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From adcaf0ad704a8c3a3cc90184e0acd7d3ef682f80 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 28 Feb 2018 00:42:33 -0500 Subject: [PATCH 227/281] base: pass `undef` instead of 0 match value --- lib/llparse/compiler/compilation.js | 4 ++-- lib/llparse/compiler/node/base.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index a26bb0c..e651756 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -290,9 +290,9 @@ class Compilation { weights = weights.map((w) => { if (w === 'likely') - return 0x10000; + return 0xffffffff; else if (w === 'unlikely') - return 0x0; + return 0x1; assert.strictEqual(w, w | 0); return w; diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index a6174f4..e65d363 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -183,7 +183,7 @@ class Node { ctx.state, pos, ctx.endPos, - phi ? phi : ctx.TYPE_MATCH.v(0) + phi ? phi : 'undef' ]); trampoline.push(call); From 90bfd7ede3dd32c7b2b366d2b7681cdc6256a147 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 28 Feb 2018 02:37:35 -0500 Subject: [PATCH 228/281] 2.3.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2b4c577..9737196 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.1", + "version": "2.3.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 3296bb5..6756fa7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.1", + "version": "2.3.2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 6ea73ae67d11ad2020258ed506594615ac77a876 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 28 Feb 2018 23:31:12 -0500 Subject: [PATCH 229/281] node: mark `Error`s unlikely Fix: #13 --- lib/llparse/compiler/compilation.js | 37 ++++++++++++++++++--------- lib/llparse/compiler/node/base.js | 2 +- lib/llparse/compiler/node/sequence.js | 9 +++++++ lib/llparse/compiler/node/single.js | 13 +++++++++- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index e651756..5e209d1 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -282,22 +282,23 @@ class Compilation { return res; } + toBranchWeight(value) { + if (value === 'likely') + return 0x10000; + else if (value === 'unlikely') + return 0x1; + + assert.strictEqual(value, value | 0); + return value; + } + branch(body, cmp, weights) { const extra = []; if (weights) { assert(Array.isArray(weights)); assert.strictEqual(weights.length, 2); - weights = weights.map((w) => { - if (w === 'likely') - return 0xffffffff; - else if (w === 'unlikely') - return 0x1; - - assert.strictEqual(w, w | 0); - return w; - }); - + weights = weights.map(w => this.toBranchWeight(w)); const meta = this.ir.metadata( `!"branch_weights", i32 ${weights[0]}, i32 ${weights[1]}`); extra.push([ '!prof', meta ]); @@ -313,7 +314,7 @@ class Compilation { }; } - buildSwitch(body, type, what, values) { + buildSwitch(body, type, what, values, weights) { const cases = []; cases.push(IR.label('otherwise')); cases.push('['); @@ -323,7 +324,19 @@ class Compilation { }); cases.push(']'); - const blocks = body.terminate('switch', [ type, what ], cases); + const extra = []; + + if (weights) { + assert(Array.isArray(weights)); + assert.strictEqual(weights.length, 1 + values.length); + + weights = weights.map(w => 'i32 ' + this.toBranchWeight(w)); + const meta = this.ir.metadata( + `!"branch_weights", ${weights.join(', ')}`); + extra.push([ '!prof', meta ]); + } + + const blocks = body.terminate('switch', [ type, what ], cases, ...extra); blocks[0].name = 'switch_otherwise'; for (let i = 0; i < values.length; i++) { diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index e65d363..6514130 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -89,7 +89,7 @@ class Node { // TODO(indutny): move this to node.Error somehow? if (this instanceof node.Error) { fn.attributes = fn.attributes || ''; - fn.attributes += ' norecurse cold writeonly'; + fn.attributes += ' norecurse cold writeonly noinline'; } nodes.set(this, fn); diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 9131ee0..5267bb4 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -72,6 +72,12 @@ class Sequence extends node.Node { SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH + ], [ + 'unlikely', // default + + 'likely', + 'unlikely', // pause + this.otherwise instanceof node.Error ? 'unlikely' : 'likely' ]); // No other values are allowed @@ -80,6 +86,9 @@ class Sequence extends node.Node { const complete = s.cases[0]; const pause = s.cases[1]; const mismatch = s.cases[2]; + complete.name = 'complete'; + pause.name = 'pause'; + mismatch.name = 'mismatch'; this.tailTo(ctx, complete, next, this.next, this.value); this.pause(ctx, pause); diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index ec76de4..2d75401 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -32,8 +32,19 @@ class Single extends node.Node { current = res.current; } + const weights = new Array(1 + this.children.length).fill('likely'); + + // Mark error branches as unlikely + this.children.forEach((child, i) => { + if (child.next instanceof node.Error) + weights[i + 1] = 'unlikely'; + }); + + if (this.otherwise instanceof node.Error) + weights[0] = 'unlikely'; + const keys = this.children.map(child => child.key); - const s = ctx.buildSwitch(body, pos.type.to, current, keys); + const s = ctx.buildSwitch(body, pos.type.to, current, keys, weights); s.cases.forEach((body, i) => { const child = this.children[i]; From 4117741a1ef3bfaecde01e96d8be3d0d2897a08e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 28 Feb 2018 23:56:18 -0500 Subject: [PATCH 230/281] 2.3.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9737196..3d01f60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.2", + "version": "2.3.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 6756fa7..dc170e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.2", + "version": "2.3.3", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 2e8700fa9d048b623f89c25fcf1b4e9621c6490d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 00:11:08 -0500 Subject: [PATCH 231/281] utils: more checks --- lib/llparse/utils.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/llparse/utils.js b/lib/llparse/utils.js index 553670e..d1f6a45 100644 --- a/lib/llparse/utils.js +++ b/lib/llparse/utils.js @@ -1,10 +1,18 @@ 'use strict'; +const assert = require('assert'); const Buffer = require('buffer').Buffer; exports.toBuffer = (value) => { - if (typeof value === 'number') + if (typeof value === 'number') { + assert.strictEqual(value, value >>> 0, + 'Invalid char value, must be integer'); + assert(0 <= value <= 255, 'Invalid char value, must be between 0 and 255'); + return Buffer.from([ value ]); - else + } else { + assert.strictEqual(typeof value, 'string', + 'Invalid value for a Buffer'); return Buffer.from(value); + } }; From 80763b4f4cb150da1a9ad477783908351bbcb117 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 17:21:50 -0500 Subject: [PATCH 232/281] bench: compare bitmap vs generated switch See: #10 --- bench/.gitignore | 1 + bench/bitmap-vs-switch.c | 113 ++++++++++++++++++++++++++++++++++++++ bench/bitmap-vs-switch.ll | 25 +++++++++ 3 files changed, 139 insertions(+) create mode 100644 bench/.gitignore create mode 100644 bench/bitmap-vs-switch.c create mode 100644 bench/bitmap-vs-switch.ll diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 0000000..2e27c28 --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1 @@ +!*.ll diff --git a/bench/bitmap-vs-switch.c b/bench/bitmap-vs-switch.c new file mode 100644 index 0000000..7591b8b --- /dev/null +++ b/bench/bitmap-vs-switch.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include + +uint8_t llparse__bench_switch(const char* p, const char* endp); + +static const uint8_t normal_url_char[32] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128, +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, }; + +#ifndef BIT_AT +# define BIT_AT(a, i) \ + (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \ + (1 << ((unsigned int) (i) & 7)))) +#endif + + +static uint8_t llparse__bench_bitmap(const char* p, const char* endp) { + for (; p != endp; p++) { + unsigned char c; + + c = (unsigned char) *p; + if (!BIT_AT(normal_url_char, c)) + return 0; + } + return 1; +} + + +/* 8 gb */ +static const int64_t kBytes = 8589934592LL; + + +static void bench(int llvm, const char* input) { + const char* endp; + int len; + + struct timeval start; + struct timeval end; + double bw; + double time; + int64_t i; + int64_t iterations; + uint8_t res; + + len = strlen(input); + endp = input + len; + iterations = kBytes / (int64_t) len; + + res = 0; + gettimeofday(&start, NULL); + if (llvm) { + for (i = 0; i < iterations; i++) + res |= llparse__bench_switch(input, endp); + } else { + for (i = 0; i < iterations; i++) + res |= llparse__bench_bitmap(input, endp); + } + + gettimeofday(&end, NULL); + + time = (end.tv_sec - start.tv_sec); + time += (double) (end.tv_usec - start.tv_usec) * 1e-6; + bw = (double) kBytes / time; + + fprintf(stdout, "%s[%d]: %.2f mb | %.2f mb/s | %.2f s\n", + llvm ? "switch" : "bitmap", + res, + (double) kBytes / (1024 * 1024), + bw / (1024 * 1024), + time); +} + + +int main(int argc, const char** argv) { + if (argc < 2) + return -1; + + bench(0, argv[1]); + bench(1, argv[1]); + return 0; +} diff --git a/bench/bitmap-vs-switch.ll b/bench/bitmap-vs-switch.ll new file mode 100644 index 0000000..8428a40 --- /dev/null +++ b/bench/bitmap-vs-switch.ll @@ -0,0 +1,25 @@ +define i8 @llparse__bench_switch(i8* %p, i8* %endp) nounwind minsize ssp uwtable { +start: + br label %loop + +loop: + %current = phi i8* [ %p, %start ], [ %next, %match ] + ; --- Prologue --- + ; if (pos != endpos) + %i0 = icmp ne i8* %current, %endp + br i1 %i0, label %has_data, label %no_data + +has_data: + ; node.Single + %c = load i8, i8* %current + switch i8 %c, label %no_match [ i8 9 , label %match i8 12 , label %match i8 33 , label %match i8 34 , label %match i8 36 , label %match i8 37 , label %match i8 38 , label %match i8 39 , label %match i8 40 , label %match i8 41 , label %match i8 42 , label %match i8 43 , label %match i8 44 , label %match i8 45 , label %match i8 46 , label %match i8 47 , label %match i8 48 , label %match i8 49 , label %match i8 50 , label %match i8 51 , label %match i8 52 , label %match i8 53 , label %match i8 54 , label %match i8 55 , label %match i8 56 , label %match i8 57 , label %match i8 58 , label %match i8 59 , label %match i8 60 , label %match i8 61 , label %match i8 62 , label %match i8 64 , label %match i8 65 , label %match i8 66 , label %match i8 67 , label %match i8 68 , label %match i8 69 , label %match i8 70 , label %match i8 71 , label %match i8 72 , label %match i8 73 , label %match i8 74 , label %match i8 75 , label %match i8 76 , label %match i8 77 , label %match i8 78 , label %match i8 79 , label %match i8 80 , label %match i8 81 , label %match i8 82 , label %match i8 83 , label %match i8 84 , label %match i8 85 , label %match i8 86 , label %match i8 87 , label %match i8 88 , label %match i8 89 , label %match i8 90 , label %match i8 91 , label %match i8 92 , label %match i8 93 , label %match i8 94 , label %match i8 95 , label %match i8 96 , label %match i8 97 , label %match i8 98 , label %match i8 99 , label %match i8 100 , label %match i8 101 , label %match i8 102 , label %match i8 103 , label %match i8 104 , label %match i8 105 , label %match i8 106 , label %match i8 107 , label %match i8 108 , label %match i8 109 , label %match i8 110 , label %match i8 111 , label %match i8 112 , label %match i8 113 , label %match i8 114 , label %match i8 115 , label %match i8 116 , label %match i8 117 , label %match i8 118 , label %match i8 119 , label %match i8 120 , label %match i8 121 , label %match i8 122 , label %match i8 123 , label %match i8 125 , label %match i8 126 , label %match ] +no_match: + ret i8 0 +match: + ; next = pos + 1 + %next = getelementptr inbounds i8, i8* %current, i32 1 + br label %loop + +no_data: + ret i8 1 +} From b3e64c2746ca565cd92626dfa000f8db930a3d57 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 17:32:13 -0500 Subject: [PATCH 233/281] bench: move to a separate dir See: #10 --- .../bitmap.c} | 62 +------------------ bench/bitmap_vs_switch/main.c | 59 ++++++++++++++++++ .../switch.ll} | 0 3 files changed, 60 insertions(+), 61 deletions(-) rename bench/{bitmap-vs-switch.c => bitmap_vs_switch/bitmap.c} (69%) create mode 100644 bench/bitmap_vs_switch/main.c rename bench/{bitmap-vs-switch.ll => bitmap_vs_switch/switch.ll} (100%) diff --git a/bench/bitmap-vs-switch.c b/bench/bitmap_vs_switch/bitmap.c similarity index 69% rename from bench/bitmap-vs-switch.c rename to bench/bitmap_vs_switch/bitmap.c index 7591b8b..166a606 100644 --- a/bench/bitmap-vs-switch.c +++ b/bench/bitmap_vs_switch/bitmap.c @@ -1,9 +1,4 @@ #include -#include -#include -#include - -uint8_t llparse__bench_switch(const char* p, const char* endp); static const uint8_t normal_url_char[32] = { /* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ @@ -46,7 +41,7 @@ static const uint8_t normal_url_char[32] = { #endif -static uint8_t llparse__bench_bitmap(const char* p, const char* endp) { +uint8_t llparse__bench_bitmap(const char* p, const char* endp) { for (; p != endp; p++) { unsigned char c; @@ -56,58 +51,3 @@ static uint8_t llparse__bench_bitmap(const char* p, const char* endp) { } return 1; } - - -/* 8 gb */ -static const int64_t kBytes = 8589934592LL; - - -static void bench(int llvm, const char* input) { - const char* endp; - int len; - - struct timeval start; - struct timeval end; - double bw; - double time; - int64_t i; - int64_t iterations; - uint8_t res; - - len = strlen(input); - endp = input + len; - iterations = kBytes / (int64_t) len; - - res = 0; - gettimeofday(&start, NULL); - if (llvm) { - for (i = 0; i < iterations; i++) - res |= llparse__bench_switch(input, endp); - } else { - for (i = 0; i < iterations; i++) - res |= llparse__bench_bitmap(input, endp); - } - - gettimeofday(&end, NULL); - - time = (end.tv_sec - start.tv_sec); - time += (double) (end.tv_usec - start.tv_usec) * 1e-6; - bw = (double) kBytes / time; - - fprintf(stdout, "%s[%d]: %.2f mb | %.2f mb/s | %.2f s\n", - llvm ? "switch" : "bitmap", - res, - (double) kBytes / (1024 * 1024), - bw / (1024 * 1024), - time); -} - - -int main(int argc, const char** argv) { - if (argc < 2) - return -1; - - bench(0, argv[1]); - bench(1, argv[1]); - return 0; -} diff --git a/bench/bitmap_vs_switch/main.c b/bench/bitmap_vs_switch/main.c new file mode 100644 index 0000000..29272c5 --- /dev/null +++ b/bench/bitmap_vs_switch/main.c @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +/* Please don't use -flto, it breaks my heart... and benchmarks */ +uint8_t llparse__bench_switch(const char* p, const char* endp); +uint8_t llparse__bench_bitmap(const char* p, const char* endp); + +/* 8 gb */ +static const int64_t kBytes = 8589934592LL; + + +static void bench(int llvm, const char* input) { + const char* endp; + int len; + + struct timeval start; + struct timeval end; + double bw; + double time; + int64_t i; + int64_t iterations; + + len = strlen(input); + endp = input + len; + iterations = kBytes / (int64_t) len; + + gettimeofday(&start, NULL); + if (llvm) { + for (i = 0; i < iterations; i++) + llparse__bench_switch(input, endp); + } else { + for (i = 0; i < iterations; i++) + llparse__bench_bitmap(input, endp); + } + + gettimeofday(&end, NULL); + + time = (end.tv_sec - start.tv_sec); + time += (double) (end.tv_usec - start.tv_usec) * 1e-6; + bw = (double) kBytes / time; + + fprintf(stdout, "%s: %.2f mb | %.2f mb/s | %.2f s\n", + llvm ? "switch" : "bitmap", + (double) kBytes / (1024 * 1024), + bw / (1024 * 1024), + time); +} + + +int main(int argc, const char** argv) { + if (argc < 2) + return -1; + + bench(0, argv[1]); + bench(1, argv[1]); + return 0; +} diff --git a/bench/bitmap-vs-switch.ll b/bench/bitmap_vs_switch/switch.ll similarity index 100% rename from bench/bitmap-vs-switch.ll rename to bench/bitmap_vs_switch/switch.ll From 29c56ea5a9b0e5db15c75baf97784bf9c29883e0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 19:43:32 -0500 Subject: [PATCH 234/281] lib: `check` optimization Fix: #10 --- lib/llparse/compiler/node/check.js | 102 ++++++++++++++++++ lib/llparse/compiler/node/index.js | 1 + lib/llparse/compiler/stage/node-translator.js | 53 ++++++++- lib/llparse/constants.js | 2 + lib/llparse/trie.js | 1 + package-lock.json | 6 +- package.json | 2 +- test/api-test.js | 15 +++ test/fixtures/index.js | 7 ++ 9 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 lib/llparse/compiler/node/check.js diff --git a/lib/llparse/compiler/node/check.js b/lib/llparse/compiler/node/check.js new file mode 100644 index 0000000..407ce17 --- /dev/null +++ b/lib/llparse/compiler/node/check.js @@ -0,0 +1,102 @@ +'use strict'; + +const node = require('./'); + +const CELL_WIDTH = 32; +const INDEX_SHIFT = 5; +const OFF_MASK = 31; + +class Check extends node.Node { + constructor(id, keys) { + super('check', id); + + this.keys = keys; + this.next = null; + } + + getChildren() { + return super.getChildren().concat(this.keys.map((key) => { + return { node: this.next.node, noAdvance: this.next.noAdvance, key }; + })); + } + + buildTable(ctx) { + // TODO(indutny): 64-bit table through BN support? + // 8 x i32 + const table = new Array(256 / CELL_WIDTH).fill(0); + + this.keys.forEach((key) => { + const bit = 1 << (key & OFF_MASK); + const index = key >> INDEX_SHIFT; + table[index] |= bit; + }); + + return ctx.ir.array(ctx.ir.i(CELL_WIDTH).array(table.length), + table.map(t => t >>> 0)); + } + + doBuild(ctx, body) { + body.comment('node.Check'); + + const pos = ctx.pos.current; + + // Load the character + let current = ctx.ir._('load', pos.type.to, [ pos.type, pos ]); + body.push(current); + + // Transform the character if needed + if (this.transform) { + const res = ctx.compilation.buildTransform(this.transform, + body, current); + body = res.body; + current = res.current; + } + + // Build global lookup table + const table = this.buildTable(ctx); + + const cellType = table.type.to.of; + + body.comment('compute index'); + const index = ctx.ir._('lshr', [ pos.type.to, current ], + pos.type.to.v(INDEX_SHIFT)); + body.push(index); + + body.comment('compute off'); + let off = ctx.ir._('and', [ pos.type.to, current ], + pos.type.to.v(OFF_MASK)); + body.push(off); + off = ctx.ir._('zext', [ pos.type.to, off, 'to', cellType ]); + body.push(off); + off = ctx.ir._('shl', [ cellType, cellType.v(1) ], off); + body.push(off); + + body.comment('l = table[index]'); + const ptr = ctx.ir._('getelementptr inbounds', table.type.to, + [ table.type, table ], + [ pos.type.to, pos.type.to.v(0) ], + [ pos.type.to, index ]); + body.push(ptr); + const load = ctx.ir._('load', cellType, [ cellType.ptr(), ptr ]); + body.push(load); + + body.comment('if (l & off)'); + const and = ctx.ir._('and', [ cellType, load ], off); + body.push(and); + const cmp = ctx.ir._('icmp', [ 'ne', cellType, and ], cellType.v(0)); + body.push(cmp); + + const weights = [ 'likely', 'likely' ]; + if (this.otherwise instanceof node.Error) + weights[1] = 'unlikely'; + + const branch = ctx.branch(body, cmp, weights); + + this.tailTo(ctx, branch.left, + this.next.noAdvance ? ctx.pos.current : ctx.pos.next, + this.next.node, null); + + this.doOtherwise(ctx, branch.right); + } +} +module.exports = Check; diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index 6fda8cf..ab0601a 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -11,3 +11,4 @@ exports.SpanEnd = require('./span-end'); exports.SetIndex = require('./set-index'); exports.Consume = require('./consume'); exports.Pause = require('./pause'); +exports.Check = require('./check'); diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 5415af6..9dec4a4 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -18,6 +18,12 @@ class NodeTranslator extends Stage { constructor(ctx) { super(ctx, 'node-translator'); + this.options = Object.assign({ + // Minimum number of cases of `single` node to make it eligable for + // `check` optimization + minCheckSize: llparse.constants.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE + }, this.ctx.options.translator); + this.nodes = new Map(); this.namespace = new Set(); this.maxSequenceLen = 0; @@ -126,6 +132,30 @@ class NodeTranslator extends Stage { } buildSingle(node, trie, list, isRoot) { + assert.notStrictEqual(trie.children.length, 0); + + const first = trie.children[0]; + const isCheck = + first.child.type === 'next' && + trie.children.length >= this.options.minCheckSize && + trie.children.every((child) => { + if (child.child.value !== null) + return false; + if (child.noAdvance !== first.noAdvance) + return false; + + if (child.child.type !== 'next') + return false; + + return child.child.next === first.child.next; + }); + + // TODO(indutny): we can split a Single node if only small amount of + // children leaves elsewhere + // Fast case, every child leads to the same result and no values are passed + if (isCheck) + return this.buildCheck(node, trie, list, isRoot); + const res = new compiler.node.Single(this.id(node)); // Break loops @@ -134,12 +164,11 @@ class NodeTranslator extends Stage { res.transform = node[kTransform]; res.children = trie.children.map((child) => { - const value = child.child.type === 'next' ? child.child.value : null; return { key: child.key, next: this.buildTrie(node, child.child, list), noAdvance: child.noAdvance, - value + value: child.child.value }; }); @@ -147,6 +176,26 @@ class NodeTranslator extends Stage { return res; } + buildCheck(node, trie, list, isRoot) { + const keys = trie.children.map(child => child.key); + const res = new compiler.node.Check(this.id(node), keys); + + // Break loops + if (isRoot) + this.nodes.set(node, res); + + res.transform = node[kTransform]; + const next = trie.children[0]; + + res.next = { + node: this.buildTrie(node, next.child, list), + noAdvance: next.noAdvance + }; + + list.push(res); + return res; + } + buildSequence(node, trie, list, isRoot) { const res = new compiler.node.Sequence(this.id(node), trie.select); const value = trie.child.type === 'next' ? trie.child.value : null; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index bf3f026..371e5db 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -40,6 +40,8 @@ exports.SEQUENCE_MISMATCH = 2; exports.SPAN_START_PREFIX = '_span_start'; exports.SPAN_CB_PREFIX = '_span_cb'; +exports.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE = 32; + exports.USER_TYPES = { i8: IR.i(8), i16: IR.i(16), diff --git a/lib/llparse/trie.js b/lib/llparse/trie.js index fc852a5..2f07b69 100644 --- a/lib/llparse/trie.js +++ b/lib/llparse/trie.js @@ -5,6 +5,7 @@ const assert = require('assert'); class Node { constructor(type) { this.type = type; + this.value = null; } } diff --git a/package-lock.json b/package-lock.json index 3d01f60..5128ec5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -749,9 +749,9 @@ } }, "llvm-ir": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.7.0.tgz", - "integrity": "sha512-ZDkevNC7tCTRKN/xfnwlwNOsGQAw4UA+R/VMrSIyjouzWHVpf7tiXyB6W0BZIMK1ismAk+8vXgXbmJfXTZZwiA==" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.8.0.tgz", + "integrity": "sha512-34IWteY5iEmoynLHJgwAOBGAHFfQPkSMW6ifjXAPv1F1hfVBJJgof9jP9Os8LxMcz8cYqEU4CzKFlL2yO1FBrg==" }, "lodash": { "version": "4.17.5", diff --git a/package.json b/package.json index dc170e7..f4ffda2 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "llvm-ir": "^1.7.0" + "llvm-ir": "^1.8.0" }, "devDependencies": { "async": "^2.6.0", diff --git a/test/api-test.js b/test/api-test.js index c3c8309..6c03d17 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -137,6 +137,21 @@ describe('LLParse', function() { binary('aab', 'off=2 error code=42 reason="some reason"\n', callback); }); + describe('`.match()`', () => { + it('should compile to a check node', (callback) => { + const start = p.node('start'); + + start + .match(fixtures.ALPHA, start) + .skipTo(printOff(p, start)); + + // TODO(indutny): validate compilation result? + const binary = fixtures.build(p, start, 'match-check'); + + binary('pecan.is.dead.', 'off=6\noff=9\noff=14\n', callback); + }); + }); + describe('`.peek()`', () => { it('should not advance position', (callback) => { const start = p.node('start'); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index bdf04cc..00493c1 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -27,6 +27,13 @@ exports.NUM = { '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9 }; +exports.ALPHA = [ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' +]; + exports.ERROR_PAUSE = Fixture.ERROR_PAUSE; // Reasonable timeout for CI From 64b070f538eb90e23128640f38d1798393740048 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 19:49:59 -0500 Subject: [PATCH 235/281] check: less moving parts --- lib/llparse/compiler/node/check.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/node/check.js b/lib/llparse/compiler/node/check.js index 407ce17..10d5bde 100644 --- a/lib/llparse/compiler/node/check.js +++ b/lib/llparse/compiler/node/check.js @@ -2,9 +2,9 @@ const node = require('./'); -const CELL_WIDTH = 32; const INDEX_SHIFT = 5; -const OFF_MASK = 31; +const CELL_WIDTH = 1 << INDEX_SHIFT; +const OFF_MASK = CELL_WIDTH - 1; class Check extends node.Node { constructor(id, keys) { From 2122b66c1d867ac5ba97e9b52819cdb109fb6720 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 20:28:47 -0500 Subject: [PATCH 236/281] compiler: proper `callees` for root/pause/span-end --- lib/llparse/compiler/compiler.js | 10 +++++++--- lib/llparse/compiler/node/base.js | 7 +++++++ lib/llparse/compiler/node/pause.js | 4 ++++ lib/llparse/compiler/node/span-end.js | 4 ++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 207fcdb..0358ded 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -118,11 +118,15 @@ class Compiler { body.push(current); const nodes = ctx.stageResults['node-builder'].map; - const callees = []; + let callees = new Set([ ctx.stageResults['node-builder'].entry ]); nodes.forEach((fn, node) => { // Only nodes that can pause can be present in `_current` - if (node.hasPause) - callees.push(fn.type.ptr().type + ' @' + fn.name); + node.getResumptionTargets().forEach((target) => { + callees.add(nodes.get(target)); + }); + }); + callees = Array.from(callees).map((fn) => { + return fn.type.ptr().type + ' @' + fn.name; }); const call = ctx.call('', nodeSig, current, constants.CCONV, [ diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 6514130..5c55451 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -76,6 +76,13 @@ class Node { return [ { node: this.otherwise, noAdvance: !this.skip, key: null } ]; } + getResumptionTargets() { + if (this.hasPause) + return [ this ]; + else + return []; + } + // Building build(compilation, nodes) { diff --git a/lib/llparse/compiler/node/pause.js b/lib/llparse/compiler/node/pause.js index c8d28aa..06d11ab 100644 --- a/lib/llparse/compiler/node/pause.js +++ b/lib/llparse/compiler/node/pause.js @@ -8,6 +8,10 @@ class Pause extends node.Error { this.type = 'pause'; } + getResumptionTargets() { + return super.getResumptionTargets().concat(this.otherwise); + } + doBuild(ctx, body) { body.comment('node.Pause'); diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index c4f3ddc..69f33bd 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -13,6 +13,10 @@ class SpanEnd extends node.Node { return body; } + getResumptionTargets() { + return super.getResumptionTargets().concat(this.otherwise); + } + doBuild(ctx, body) { body.comment(`node.SpanEnd[${this.code.name}]`); const result = ctx.compilation.stageResults['span-builder'].spanEnd( From cca723684ad797d55d463ea26fbe680e2b0d195f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 21:06:30 -0500 Subject: [PATCH 237/281] 2.4.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5128ec5..57ec883 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.3", + "version": "2.4.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f4ffda2..35eefde 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.3.3", + "version": "2.4.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 2040fbf60574219e39546ae31133d3e37f30692d Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 1 Mar 2018 21:19:06 -0500 Subject: [PATCH 238/281] node: rename Check into BitCheck See: #15 --- lib/llparse/compiler/node/{check.js => bit-check.js} | 8 ++++---- lib/llparse/compiler/node/index.js | 2 +- lib/llparse/compiler/stage/node-translator.js | 10 +++++----- test/api-test.js | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) rename lib/llparse/compiler/node/{check.js => bit-check.js} (95%) diff --git a/lib/llparse/compiler/node/check.js b/lib/llparse/compiler/node/bit-check.js similarity index 95% rename from lib/llparse/compiler/node/check.js rename to lib/llparse/compiler/node/bit-check.js index 10d5bde..8ad42d2 100644 --- a/lib/llparse/compiler/node/check.js +++ b/lib/llparse/compiler/node/bit-check.js @@ -6,9 +6,9 @@ const INDEX_SHIFT = 5; const CELL_WIDTH = 1 << INDEX_SHIFT; const OFF_MASK = CELL_WIDTH - 1; -class Check extends node.Node { +class BitCheck extends node.Node { constructor(id, keys) { - super('check', id); + super('bit-check', id); this.keys = keys; this.next = null; @@ -36,7 +36,7 @@ class Check extends node.Node { } doBuild(ctx, body) { - body.comment('node.Check'); + body.comment('node.BitCheck'); const pos = ctx.pos.current; @@ -99,4 +99,4 @@ class Check extends node.Node { this.doOtherwise(ctx, branch.right); } } -module.exports = Check; +module.exports = BitCheck; diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index ab0601a..27da776 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -11,4 +11,4 @@ exports.SpanEnd = require('./span-end'); exports.SetIndex = require('./set-index'); exports.Consume = require('./consume'); exports.Pause = require('./pause'); -exports.Check = require('./check'); +exports.BitCheck = require('./bit-check'); diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 9dec4a4..f5d29d6 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -135,7 +135,7 @@ class NodeTranslator extends Stage { assert.notStrictEqual(trie.children.length, 0); const first = trie.children[0]; - const isCheck = + const isBitCheck = first.child.type === 'next' && trie.children.length >= this.options.minCheckSize && trie.children.every((child) => { @@ -153,8 +153,8 @@ class NodeTranslator extends Stage { // TODO(indutny): we can split a Single node if only small amount of // children leaves elsewhere // Fast case, every child leads to the same result and no values are passed - if (isCheck) - return this.buildCheck(node, trie, list, isRoot); + if (isBitCheck) + return this.buildBitCheck(node, trie, list, isRoot); const res = new compiler.node.Single(this.id(node)); @@ -176,9 +176,9 @@ class NodeTranslator extends Stage { return res; } - buildCheck(node, trie, list, isRoot) { + buildBitCheck(node, trie, list, isRoot) { const keys = trie.children.map(child => child.key); - const res = new compiler.node.Check(this.id(node), keys); + const res = new compiler.node.BitCheck(this.id(node), keys); // Break loops if (isRoot) diff --git a/test/api-test.js b/test/api-test.js index 6c03d17..f38eb4c 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -138,7 +138,7 @@ describe('LLParse', function() { }); describe('`.match()`', () => { - it('should compile to a check node', (callback) => { + it('should compile to a bit-check node', (callback) => { const start = p.node('start'); start @@ -146,7 +146,7 @@ describe('LLParse', function() { .skipTo(printOff(p, start)); // TODO(indutny): validate compilation result? - const binary = fixtures.build(p, start, 'match-check'); + const binary = fixtures.build(p, start, 'match-bit-check'); binary('pecan.is.dead.', 'off=6\noff=9\noff=14\n', callback); }); From 6f7e5dddb3724ff868abf4cdcf11719510a248fe Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 00:53:50 -0500 Subject: [PATCH 239/281] bit-check: rework to support wide bitfield Fix: #15 --- lib/llparse/compiler/node/bit-check.js | 109 +++++++++++------- lib/llparse/compiler/stage/node-translator.js | 95 ++++++++++----- lib/llparse/constants.js | 1 + lib/llparse/utils.js | 42 +++++++ package-lock.json | 4 +- package.json | 1 + test/api-test.js | 21 +++- test/code-test.js | 8 +- test/fixtures/index.js | 4 +- 9 files changed, 203 insertions(+), 82 deletions(-) diff --git a/lib/llparse/compiler/node/bit-check.js b/lib/llparse/compiler/node/bit-check.js index 8ad42d2..f27b864 100644 --- a/lib/llparse/compiler/node/bit-check.js +++ b/lib/llparse/compiler/node/bit-check.js @@ -1,38 +1,42 @@ 'use strict'; const node = require('./'); +const llparse = require('../../'); -const INDEX_SHIFT = 5; -const CELL_WIDTH = 1 << INDEX_SHIFT; -const OFF_MASK = CELL_WIDTH - 1; +const CHAR_WIDTH = 8; +const WORD_WIDTH = 5; class BitCheck extends node.Node { - constructor(id, keys) { + constructor(id) { super('bit-check', id); - this.keys = keys; - this.next = null; + this.map = null; } getChildren() { - return super.getChildren().concat(this.keys.map((key) => { - return { node: this.next.node, noAdvance: this.next.noAdvance, key }; - })); + const res = super.getChildren(); + this.map.forEach((entry) => { + entry.keys.forEach((key) => { + res.push({ node: entry.node, noAdvance: entry.noAdvance, key }); + }); + }); + return res; } buildTable(ctx) { - // TODO(indutny): 64-bit table through BN support? - // 8 x i32 - const table = new Array(256 / CELL_WIDTH).fill(0); - - this.keys.forEach((key) => { - const bit = 1 << (key & OFF_MASK); - const index = key >> INDEX_SHIFT; - table[index] |= bit; - }); + const table = llparse.utils.buildLookupTable(WORD_WIDTH, CHAR_WIDTH, + this.map.map(entry => entry.keys)); + + const arrayTy = ctx.ir.i(1 << WORD_WIDTH).array(table.table.length); - return ctx.ir.array(ctx.ir.i(CELL_WIDTH).array(table.length), - table.map(t => t >>> 0)); + return { + global: ctx.ir.array(arrayTy, table.table), + + indexShift: table.indexShift, + shiftMask: table.shiftMask, + shiftMul: table.shiftMul, + valueMask: table.valueMask + }; } doBuild(ctx, body) { @@ -55,48 +59,65 @@ class BitCheck extends node.Node { // Build global lookup table const table = this.buildTable(ctx); - const cellType = table.type.to.of; + const cellType = table.global.type.to.of; body.comment('compute index'); const index = ctx.ir._('lshr', [ pos.type.to, current ], - pos.type.to.v(INDEX_SHIFT)); + pos.type.to.v(table.indexShift)); body.push(index); - body.comment('compute off'); - let off = ctx.ir._('and', [ pos.type.to, current ], - pos.type.to.v(OFF_MASK)); - body.push(off); - off = ctx.ir._('zext', [ pos.type.to, off, 'to', cellType ]); - body.push(off); - off = ctx.ir._('shl', [ cellType, cellType.v(1) ], off); - body.push(off); + body.comment('compute shift'); + let shift = ctx.ir._('and', [ pos.type.to, current ], + pos.type.to.v(table.shiftMask)); + body.push(shift); + shift = ctx.ir._('zext', [ pos.type.to, shift, 'to', cellType ]); + body.push(shift); + + // Compiler can handle it, but why generate more code? + if (table.shiftMul !== 1) { + shift = ctx.ir._('mul', [ cellType, shift ], cellType.v(table.shiftMul)); + body.push(shift); + } body.comment('l = table[index]'); - const ptr = ctx.ir._('getelementptr inbounds', table.type.to, - [ table.type, table ], + const ptr = ctx.ir._('getelementptr inbounds', table.global.type.to, + [ table.global.type, table.global ], [ pos.type.to, pos.type.to.v(0) ], [ pos.type.to, index ]); body.push(ptr); const load = ctx.ir._('load', cellType, [ cellType.ptr(), ptr ]); body.push(load); - body.comment('if (l & off)'); - const and = ctx.ir._('and', [ cellType, load ], off); - body.push(and); - const cmp = ctx.ir._('icmp', [ 'ne', cellType, and ], cellType.v(0)); - body.push(cmp); + body.comment('l >>= shift'); + const shr = ctx.ir._('lshr', [ cellType, load ], shift); + body.push(shr); + + body.comment('l &= valueMask'); + const masked = ctx.ir._('and', [ cellType, shr ], + cellType.v(table.valueMask)); + body.push(masked); + + const weights = new Array(this.map.length + 1).fill('likely'); + + this.map.forEach((entry, i) => { + if (entry.node instanceof node.Error) + weights[i + 1] = 'unlikely'; + }); - const weights = [ 'likely', 'likely' ]; if (this.otherwise instanceof node.Error) - weights[1] = 'unlikely'; + weights[0] = 'unlikely'; - const branch = ctx.branch(body, cmp, weights); + const keys = this.map.map((entry, i) => i + 1); + const s = ctx.buildSwitch(body, cellType, masked, keys, weights); - this.tailTo(ctx, branch.left, - this.next.noAdvance ? ctx.pos.current : ctx.pos.next, - this.next.node, null); + s.cases.forEach((body, i) => { + const child = this.map[i]; + + this.tailTo(ctx, body, child.noAdvance ? ctx.pos.current : ctx.pos.next, + child.node, null); + }); - this.doOtherwise(ctx, branch.right); + this.doOtherwise(ctx, s.otherwise); } } module.exports = BitCheck; diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index f5d29d6..093cca0 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -1,6 +1,7 @@ 'use strict'; const assert = require('assert'); +const debugOpt = require('debug')('llparse:opt'); const Stage = require('./').Stage; const compiler = require('../'); @@ -20,8 +21,11 @@ class NodeTranslator extends Stage { this.options = Object.assign({ // Minimum number of cases of `single` node to make it eligable for - // `check` optimization - minCheckSize: llparse.constants.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE + // `BitCheck` optimization + minCheckSize: llparse.constants.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE, + + // Maximum width of entry in a bitfield for a `BitCheck` optimization + maxCheckWidth: llparse.constants.DEFAULT_TRANSLATOR_MAX_CHECK_WIDTH }, this.ctx.options.translator); this.nodes = new Map(); @@ -134,27 +138,10 @@ class NodeTranslator extends Stage { buildSingle(node, trie, list, isRoot) { assert.notStrictEqual(trie.children.length, 0); - const first = trie.children[0]; - const isBitCheck = - first.child.type === 'next' && - trie.children.length >= this.options.minCheckSize && - trie.children.every((child) => { - if (child.child.value !== null) - return false; - if (child.noAdvance !== first.noAdvance) - return false; - - if (child.child.type !== 'next') - return false; - - return child.child.next === first.child.next; - }); - - // TODO(indutny): we can split a Single node if only small amount of - // children leaves elsewhere // Fast case, every child leads to the same result and no values are passed - if (isBitCheck) - return this.buildBitCheck(node, trie, list, isRoot); + const bitCheck = this.buildBitCheck(node, trie, list, isRoot); + if (bitCheck) + return bitCheck; const res = new compiler.node.Single(this.id(node)); @@ -177,20 +164,70 @@ class NodeTranslator extends Stage { } buildBitCheck(node, trie, list, isRoot) { - const keys = trie.children.map(child => child.key); - const res = new compiler.node.BitCheck(this.id(node), keys); + if (trie.children.length < this.options.minCheckSize) + return false; + + const targets = new Map(); + + const bailout = !trie.children.every((child) => { + if (child.child.value !== null) + return false; + + if (child.child.type !== 'next') + return false; + + const target = child.child.next; + if (!targets.has(target)) { + targets.set(target, { + noAdvance: child.noAdvance, + keys: [ child.key ], + trie: child.child + }); + return true; + } + + const existing = targets.get(target); + + // TODO(indutny): just use it as a sub-key? + if (existing.noAdvance !== child.noAdvance) { + debugOpt('Can\'t transform single to bitcheck due to ' + + '`.peek()`/`.match()` conflict'); + return false; + } + + existing.keys.push(child.key); + return true; + }); + + if (bailout) + return false; + + // We've width limit for this optimization + if (targets.size >= (1 << this.options.maxCheckWidth)) { + debugOpt('Can\'t transform single to bitcheck due to ' + + 'large distinct target count=%d', targets.size); + return false; + } + + const res = new compiler.node.BitCheck(this.id(node)); // Break loops if (isRoot) this.nodes.set(node, res); res.transform = node[kTransform]; - const next = trie.children[0]; - res.next = { - node: this.buildTrie(node, next.child, list), - noAdvance: next.noAdvance - }; + const map = []; + let totalKeys = 0; + targets.forEach((info) => { + const next = this.buildTrie(node, info.trie, list); + totalKeys += info.keys.length; + map.push({ node: next, noAdvance: info.noAdvance, keys: info.keys }); + }); + res.map = map; + + debugOpt('Transformed Single into BitCheck targets.len=%d total_keys=%d', + res.map.length, totalKeys); list.push(res); return res; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 371e5db..6d97ecf 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -41,6 +41,7 @@ exports.SPAN_START_PREFIX = '_span_start'; exports.SPAN_CB_PREFIX = '_span_cb'; exports.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE = 32; +exports.DEFAULT_TRANSLATOR_MAX_CHECK_WIDTH = 4; exports.USER_TYPES = { i8: IR.i(8), diff --git a/lib/llparse/utils.js b/lib/llparse/utils.js index d1f6a45..9b0e8b2 100644 --- a/lib/llparse/utils.js +++ b/lib/llparse/utils.js @@ -16,3 +16,45 @@ exports.toBuffer = (value) => { return Buffer.from(value); } }; + +exports.powerOfTwo = num => Math.pow(2, Math.ceil(Math.log2(num))); + +exports.buildLookupTable = (wordWidth, charWidth, list) => { + // Entry values: + // 0 - no hit + // 1 - map[0] + // ... + // n - map[n] + const width = exports.powerOfTwo(Math.ceil(Math.log2(list.length + 1))); + assert(width >= 1); + + assert(charWidth + width - wordWidth > 0); + + // TODO(indutny): 64-bit table through BN support? + const indexShift = wordWidth - Math.log2(width); + const table = new Array(1 << (charWidth - indexShift)).fill(0); + + assert(indexShift > 0); + + const shiftMask = (1 << indexShift) - 1; + const shiftMul = width; + + list.forEach((entry, i) => { + const val = i + 1; + + entry.forEach((key) => { + const index = key >> indexShift; + const shift = (key & shiftMask) * shiftMul; + + table[index] |= val << shift; + }); + }); + + return { + table, + indexShift, + shiftMask, + shiftMul, + valueMask: (1 << width) - 1 + }; +}; diff --git a/package-lock.json b/package-lock.json index 57ec883..e09fd50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -297,7 +297,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, "requires": { "ms": "2.0.0" } @@ -837,8 +836,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "mute-stream": { "version": "0.0.7", diff --git a/package.json b/package.json index 35eefde..529f75a 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { + "debug": "^3.1.0", "llvm-ir": "^1.8.0" }, "devDependencies": { diff --git a/test/api-test.js b/test/api-test.js index f38eb4c..092d611 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -138,7 +138,7 @@ describe('LLParse', function() { }); describe('`.match()`', () => { - it('should compile to a bit-check node', (callback) => { + it('should compile to a single-bit bit-check node', (callback) => { const start = p.node('start'); start @@ -150,6 +150,25 @@ describe('LLParse', function() { binary('pecan.is.dead.', 'off=6\noff=9\noff=14\n', callback); }); + + it('should compile to a multi-bit bit-check node', (callback) => { + const start = p.node('start'); + const another = p.node('another'); + + start + .match(fixtures.ALPHA, start) + .peek(fixtures.NUM, another) + .skipTo(printOff(p, start)); + + another + .match(fixtures.NUM, another) + .otherwise(start); + + // TODO(indutny): validate compilation result? + const binary = fixtures.build(p, start, 'match-multi-bit-check'); + + binary('pecan.135.is.dead.', 'off=6\noff=10\noff=13\noff=18\n', callback); + }); }); describe('`.peek()`', () => { diff --git a/test/code-test.js b/test/code-test.js index 0e16866..372eef2 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -29,7 +29,7 @@ describe('LLParse/Code', function() { const count = p.invoke(p.code.mulAdd('counter', { base: 10 }), start); start - .select(fixtures.NUM, count) + .select(fixtures.NUM_SELECT, count) .otherwise(dot); dot @@ -51,7 +51,7 @@ describe('LLParse/Code', function() { }, start); start - .select(fixtures.NUM, count) + .select(fixtures.NUM_SELECT, count) .otherwise(p.error(1, 'Unexpected')); const binary = fixtures.build(p, start, 'mul-add-overflow'); @@ -72,7 +72,7 @@ describe('LLParse/Code', function() { }, start); start - .select(fixtures.NUM, count) + .select(fixtures.NUM_SELECT, count) .otherwise(p.error(1, 'Unexpected')); const binary = fixtures.build(p, start, 'mul-add-max-overflow'); @@ -115,7 +115,7 @@ describe('LLParse/Code', function() { }, p.error(1, 'Unexpected')); start - .select(fixtures.NUM, p.invoke(p.code.store('counter'), check)) + .select(fixtures.NUM_SELECT, p.invoke(p.code.store('counter'), check)) .otherwise(p.error(1, 'Unexpected')); const binary = fixtures.build(p, start, 'update'); diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 00493c1..011c37f 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -23,10 +23,12 @@ exports.printOff = (p, next) => { return p.invoke(code, next); }; -exports.NUM = { +exports.NUM_SELECT = { '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9 }; +exports.NUM = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ]; + exports.ALPHA = [ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', From c53f9dff28e205166d2b71e8f723fb0b206f8d89 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 00:56:23 -0500 Subject: [PATCH 240/281] 2.4.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index e09fd50..c23bf95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.4.0", + "version": "2.4.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 529f75a..35805ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.4.0", + "version": "2.4.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 5d89426065796bceff01f559abbff4a8ed7e2466 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 14:01:27 -0500 Subject: [PATCH 241/281] test: add missing file --- test/utils-test.js | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 test/utils-test.js diff --git a/test/utils-test.js b/test/utils-test.js new file mode 100644 index 0000000..435ae39 --- /dev/null +++ b/test/utils-test.js @@ -0,0 +1,73 @@ +'use strict'; +/* global describe it */ + +const assert = require('assert'); + +const llparse = require('../lib/llparse/'); + +describe('LLParse/utils', function() { + describe('lookup table', () => { + const enumerate = (table) => { + const res = new Map(); + + for (let c = 0; c < 256; c++) { + const index = c >>> table.indexShift; + const shift = (c & table.shiftMask) * table.shiftMul; + + const v = (table.table[index] >>> shift) & table.valueMask; + if (v === 0) + continue; + + if (res.has(v - 1)) + res.get(v - 1).push(c); + else + res.set(v - 1, [ c ]); + } + + const sorted = []; + res.forEach((values, key) => sorted.push({ key, values })); + return sorted.sort((a, b) => a.key - b.key); + }; + + it('should build correct lookup table width=1', () => { + const table = llparse.utils.buildLookupTable(5, 8, [ + [ 1, 3, 5, 101, 255 ] + ]); + + const res = enumerate(table); + assert.deepStrictEqual(res, [ + { key: 0, values: [ 1, 3, 5, 101, 255 ] } + ]); + }); + + it('should build correct lookup table width=2', () => { + const table = llparse.utils.buildLookupTable(5, 8, [ + [ 1, 3, 5, 101, 254 ], + [ 2, 4, 6, 102, 255 ] + ]); + + const res = enumerate(table); + assert.deepStrictEqual(res, [ + { key: 0, values: [ 1, 3, 5, 101, 254 ] }, + { key: 1, values: [ 2, 4, 6, 102, 255 ] } + ]); + }); + + it('should build correct lookup table width=3', () => { + const table = llparse.utils.buildLookupTable(5, 8, [ + [ 1, 5, 9, 13, 17, 117 ], + [ 2, 6, 10, 14, 18, 84 ], + [ 3, 7, 11, 15, 19, 90 ], + [ 4, 8, 12, 16, 20, 255 ] + ]); + + const res = enumerate(table); + assert.deepStrictEqual(res, [ + { key: 0, values: [ 1, 5, 9, 13, 17, 117 ] }, + { key: 1, values: [ 2, 6, 10, 14, 18, 84 ] }, + { key: 2, values: [ 3, 7, 11, 15, 19, 90 ] }, + { key: 3, values: [ 4, 8, 12, 16, 20, 255 ] } + ]); + }); + }); +}); From e1b1c897bae590a5d97ec080c5a996cc64680592 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 16:28:50 -0500 Subject: [PATCH 242/281] code: refactor, test/or --- lib/llparse.js | 22 +-- lib/llparse/code/base.js | 7 +- lib/llparse/code/context.js | 119 -------------- lib/llparse/code/field-value.js | 22 +++ lib/llparse/code/field.js | 16 ++ lib/llparse/code/index.js | 7 +- lib/llparse/code/is-equal.js | 39 +---- lib/llparse/code/load.js | 26 +--- lib/llparse/code/match.js | 4 +- lib/llparse/code/mul-add.js | 109 +------------ lib/llparse/code/or.js | 10 ++ lib/llparse/code/span.js | 1 + lib/llparse/code/store.js | 28 +--- lib/llparse/code/test.js | 10 ++ lib/llparse/code/update.js | 29 +--- lib/llparse/code/value.js | 4 +- lib/llparse/compiler/code/base.js | 25 +++ lib/llparse/compiler/code/index.js | 19 +++ lib/llparse/compiler/code/is-equal.js | 41 +++++ lib/llparse/compiler/code/load.js | 31 ++++ lib/llparse/compiler/code/match.js | 17 ++ lib/llparse/compiler/code/mul-add.js | 118 ++++++++++++++ lib/llparse/compiler/code/or.js | 35 +++++ lib/llparse/compiler/code/span.js | 17 ++ lib/llparse/compiler/code/store.js | 34 ++++ lib/llparse/compiler/code/test.js | 44 ++++++ lib/llparse/compiler/code/update.js | 31 ++++ lib/llparse/compiler/code/value.js | 17 ++ lib/llparse/compiler/compilation.js | 146 ++++++++++++++++-- lib/llparse/compiler/index.js | 1 + lib/llparse/compiler/stage/node-translator.js | 13 +- lib/llparse/symbols.js | 1 - test/code-test.js | 45 ++++++ 33 files changed, 705 insertions(+), 383 deletions(-) delete mode 100644 lib/llparse/code/context.js create mode 100644 lib/llparse/code/field-value.js create mode 100644 lib/llparse/code/field.js create mode 100644 lib/llparse/code/or.js create mode 100644 lib/llparse/code/test.js create mode 100644 lib/llparse/compiler/code/base.js create mode 100644 lib/llparse/compiler/code/index.js create mode 100644 lib/llparse/compiler/code/is-equal.js create mode 100644 lib/llparse/compiler/code/load.js create mode 100644 lib/llparse/compiler/code/match.js create mode 100644 lib/llparse/compiler/code/mul-add.js create mode 100644 lib/llparse/compiler/code/or.js create mode 100644 lib/llparse/compiler/code/span.js create mode 100644 lib/llparse/compiler/code/store.js create mode 100644 lib/llparse/compiler/code/test.js create mode 100644 lib/llparse/compiler/code/update.js create mode 100644 lib/llparse/compiler/code/value.js diff --git a/lib/llparse.js b/lib/llparse.js index 8126c50..d363474 100644 --- a/lib/llparse.js +++ b/lib/llparse.js @@ -32,28 +32,32 @@ class CodeAPI { // Helpers - // TODO(indutny): do something with these long names store(field) { - return new internal.code.Store(this[kPrefix] + '__store_' + field, field); + return new internal.code.Store(field); } load(field) { - return new internal.code.Load(this[kPrefix] + '__load_' + field, field); + return new internal.code.Load(field); } mulAdd(field, options) { - return new internal.code.MulAdd(this[kPrefix] + '__muladd_' + field, field, - options); + return new internal.code.MulAdd(field, options); } update(field, value) { - return new internal.code.Update(this[kPrefix] + '__update_' + field, field, - value); + return new internal.code.Update(field, value); } isEqual(field, value) { - return new internal.code.IsEqual(this[kPrefix] + '__is_eq_' + field, field, - value); + return new internal.code.IsEqual(field, value); + } + + or(field, value) { + return new internal.code.Or(field, value); + } + + test(field, value) { + return new internal.code.Test(field, value); } } diff --git a/lib/llparse/code/base.js b/lib/llparse/code/base.js index 3e4b7b1..9144ae0 100644 --- a/lib/llparse/code/base.js +++ b/lib/llparse/code/base.js @@ -3,14 +3,13 @@ const llparse = require('../'); const kName = Symbol('name'); -const kBody = llparse.symbols.kBody; const kType = llparse.symbols.kType; class Code { - constructor(type, name, body = null) { - this[kType] = type; + constructor(signature, name) { + // TODO(indutny): rename to kSignature everywhere + this[kType] = signature; this[kName] = name; - this[kBody] = body; } get name() { return this[kName]; } diff --git a/lib/llparse/code/context.js b/lib/llparse/code/context.js deleted file mode 100644 index ffdc3ea..0000000 --- a/lib/llparse/code/context.js +++ /dev/null @@ -1,119 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const llparse = require('../'); -const constants = llparse.constants; - -const kType = llparse.symbols.kType; -const kFn = Symbol('fn'); -const kCompilation = Symbol('compilation'); -const kRet = Symbol('ret'); -const kState = Symbol('state'); -const kPos = Symbol('pos'); -const kEndPos = Symbol('endPos'); -const kMatch = Symbol('match'); -const kLookup = Symbol('lookup'); - -const INT = constants.INT; - -const ARG_STATE = constants.ARG_STATE; -const ARG_POS = constants.ARG_POS; -const ARG_ENDPOS = constants.ARG_ENDPOS; -const ARG_MATCH = constants.ARG_MATCH; - -class Context { - constructor(compilation, code, fn) { - this[kCompilation] = compilation; - this[kFn] = fn; - - this[kRet] = this.fn.type.ret; - this[kState] = fn.arg(ARG_STATE); - this[kPos] = fn.arg(ARG_POS); - this[kEndPos] = fn.arg(ARG_ENDPOS); - - this[kMatch] = null; - - if (code[kType] === 'value') - this[kMatch] = fn.arg(ARG_MATCH); - } - - get fn() { return this[kFn]; } - get ir() { return this[kCompilation].ir; } - get ret() { return this[kRet]; } - get state() { return this[kState]; } - get pos() { return this[kPos]; } - get endPos() { return this[kEndPos]; } - get match() { return this[kMatch]; } - - load(body, field) { - body.comment(`load state[${field}]`); - - const lookup = this[kLookup](field); - body.push(lookup.instr); - - const res = this.ir._('load', lookup.type, - [ lookup.type.ptr(), lookup.instr ]); - body.push(res); - - return res; - } - - store(body, field, value) { - body.comment(`store state[${field}] = ...`); - - const lookup = this[kLookup](field); - body.push(lookup.instr); - - const res = this.ir._('store', - [ lookup.type, value ], - [ lookup.type.ptr(), lookup.instr ]); - res.void(); - body.push(res); - } - - truncate(body, fromType, from, toType, isSigned = false) { - assert(toType.isInt()); - assert(fromType.isInt()); - - let res; - - // Same type! - if (fromType.type === toType) { - return from; - // Extend - } else if (fromType.width < toType.width) { - if (isSigned) - res = this.ir._('sext', [ fromType, from, 'to', toType ]); - else - res = this.ir._('zext', [ fromType, from, 'to', toType ]); - // Truncate - } else { - assert(fromType.width > toType.width); - res = this.ir._('trunc', [ fromType, from, 'to', toType ]); - } - - body.push(res); - return res; - } - - call(...args) { return this[kCompilation].call(...args); } - branch(...args) { return this[kCompilation].branch(...args); } - - [kLookup](field) { - const stateArg = this.state; - const stateType = stateArg.type.to; - - const index = stateType.lookup(field); - const instr = this.ir._('getelementptr inbounds', stateType, - [ stateArg.type, stateArg ], - [ INT, INT.v(0) ], - [ INT, INT.v(index) ]); - - return { - type: stateType.fields[index].type, - instr - }; - } -} -module.exports = Context; diff --git a/lib/llparse/code/field-value.js b/lib/llparse/code/field-value.js new file mode 100644 index 0000000..d3203de --- /dev/null +++ b/lib/llparse/code/field-value.js @@ -0,0 +1,22 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +const kValue = Symbol('value'); + +class FieldValue extends code.Field { + constructor(signature, name, field, value) { + super(signature, name, field); + + assert.strictEqual(typeof value, 'number', + '`value` argument must be a number'); + assert.strictEqual(value, value | 0, '`value` argument must be an integer'); + + this[kValue] = value; + } + + get value() { return this[kValue]; } +} +module.exports = FieldValue; diff --git a/lib/llparse/code/field.js b/lib/llparse/code/field.js new file mode 100644 index 0000000..8e3d218 --- /dev/null +++ b/lib/llparse/code/field.js @@ -0,0 +1,16 @@ +'use strict'; + +const code = require('./'); + +const kField = Symbol('field'); + +class Field extends code.Code { + constructor(signature, name, field) { + super(signature, name); + + this[kField] = field; + } + + get field() { return this[kField]; } +} +module.exports = Field; diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js index 62ecd4e..ca3eecb 100644 --- a/lib/llparse/code/index.js +++ b/lib/llparse/code/index.js @@ -7,10 +7,13 @@ exports.Span = require('./span'); // Helpers +exports.Field = require('./field'); +exports.FieldValue = require('./field-value'); + exports.Store = require('./store'); exports.Load = require('./load'); exports.MulAdd = require('./mul-add'); exports.Update = require('./update'); exports.IsEqual = require('./is-equal'); - -exports.Context = require('./context'); +exports.Or = require('./or'); +exports.Test = require('./test'); diff --git a/lib/llparse/code/is-equal.js b/lib/llparse/code/is-equal.js index 23d2555..16a254e 100644 --- a/lib/llparse/code/is-equal.js +++ b/lib/llparse/code/is-equal.js @@ -1,43 +1,10 @@ 'use strict'; -const assert = require('assert'); -const llparse = require('../'); - -const BOOL = llparse.constants.BOOL; - const code = require('./'); -const kCompile = Symbol('compile'); - -class IsEqual extends code.Match { - constructor(name, field, value) { - assert.strictEqual(typeof value, 'number', - '`.update()`\'s `value` argument must be a number'); - assert.strictEqual(value, value | 0, - '`.update()`\'s `value` argument must be an integer'); - - const body = (context) => this[kCompile](context, field, value); - - super(name, body); - } - - [kCompile](context, field, value) { - const body = context.fn.body; - - const stateType = context.state.type.to; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(context.ret.isInt()); - - const fieldValue = context.load(body, field); - - const cmp = context.ir._('icmp', [ 'eq', fieldType, fieldValue ], - fieldType.v(value)); - body.push(cmp); - const res = context.truncate(body, BOOL, cmp, context.ret); - - body.terminate('ret', [ context.ret, res ]); +class IsEqual extends code.FieldValue { + constructor(field, value) { + super('match', 'is_equal', field, value); } } module.exports = IsEqual; diff --git a/lib/llparse/code/load.js b/lib/llparse/code/load.js index 6f9d613..a0af7f5 100644 --- a/lib/llparse/code/load.js +++ b/lib/llparse/code/load.js @@ -1,30 +1,10 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); -const kCompile = Symbol('compile'); - -class Load extends code.Match { - constructor(name, field) { - const store = (context) => this[kCompile](context, field); - - super(name, store); - } - - [kCompile](context, field) { - const body = context.fn.body; - - const stateType = context.state.type.to; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(context.ret.isInt()); - - const adj = context.truncate(body, fieldType, context.load(body, field), - context.ret); - body.terminate('ret', [ context.ret, adj ]); +class Load extends code.Field { + constructor(field) { + super('match', 'load', field); } } module.exports = Load; diff --git a/lib/llparse/code/match.js b/lib/llparse/code/match.js index 973cc6b..f998a85 100644 --- a/lib/llparse/code/match.js +++ b/lib/llparse/code/match.js @@ -3,8 +3,8 @@ const code = require('./'); class Match extends code.Code { - constructor(name, body) { - super('match', name, body); + constructor(name) { + super('match', name); } } module.exports = Match; diff --git a/lib/llparse/code/mul-add.js b/lib/llparse/code/mul-add.js index 02b6c8c..eb1cf88 100644 --- a/lib/llparse/code/mul-add.js +++ b/lib/llparse/code/mul-add.js @@ -2,20 +2,13 @@ const assert = require('assert'); -const llparse = require('../'); const code = require('./'); -const BOOL = llparse.constants.BOOL; -const INT = llparse.constants.INT; - const kOptions = Symbol('options'); -const kCompile = Symbol('compile'); - -class MulAdd extends code.Value { - constructor(name, field, options) { - const body = context => this[kCompile](context, field); - super(name, body); +class MulAdd extends code.Field { + constructor(field, options) { + super('value', 'mul_add', field); options = Object.assign({ max: 0, @@ -37,100 +30,6 @@ class MulAdd extends code.Value { this[kOptions] = options; } - [kCompile](context, field) { - const ir = context.ir; - const body = context.fn.body; - const matchType = context.match.type; - const options = this[kOptions]; - - const stateType = context.state.type.to; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - // Declare intrinsic functions - const overRet = ir.struct([ - [ fieldType, 'result' ], - [ BOOL, 'overflow' ] - ]); - - const overSig = ir.signature(overRet, [ fieldType, fieldType ]); - const mulFn = ir.declare( - overSig, - `llvm.${options.signed ? 'smul': 'umul'}.with.overflow.${fieldType.type}` - ); - const addFn = ir.declare( - overSig, - `llvm.${options.signed ? 'sadd': 'uadd'}.with.overflow.${fieldType.type}` - ); - - // Truncate match and load field - const value = context.truncate(body, matchType, context.match, fieldType, - options.signed); - const fieldValue = context.load(body, field); - - // Multiply - const mul = context.call('', overSig, mulFn, '', - [ fieldValue, matchType.v(options.base) ]); - body.push(mul); - - body.comment('extract product'); - const product = ir._('extractvalue', [ overRet, mul ], - INT.v(overRet.lookup('result'))); - body.push(product); - - body.comment('extract overflow'); - const overflowBit = ir._('extractvalue', [ overRet, mul ], - INT.v(overRet.lookup('overflow'))); - body.push(overflowBit); - - const { left: isOverflow, right: normal } = - context.branch(body, overflowBit, [ 'unlikely', 'likely' ]); - isOverflow.name = 'overflow'; - isOverflow.terminate('ret', [ context.ret, context.ret.v(1) ]); - - normal.name = 'no_overflow'; - - // Add - const add = context.call('', overSig, addFn, '', [ product, value ]); - normal.push(add); - - normal.comment('extract result'); - const result = ir._('extractvalue', [ overRet, add ], - INT.v(overRet.lookup('result'))); - normal.push(result); - - normal.comment('extract overflow'); - const addOverflowBit = ir._('extractvalue', [ overRet, add ], - INT.v(overRet.lookup('overflow'))); - normal.push(addOverflowBit); - - const { left: isAddOverflow, right: check } = - context.branch(normal, addOverflowBit, [ 'unlikely', 'likely' ]); - isAddOverflow.name = 'add_overflow'; - check.name = 'check'; - - isAddOverflow.terminate('ret', [ context.ret, context.ret.v(1) ]); - - // Check that we're within the limits - let store; - if (options.max) { - const cond = options.signed ? 'sgt' : 'ugt'; - const cmp = ir._('icmp', [ cond, fieldType, result ], - fieldType.v(options.max)); - check.push(cmp); - - const branch = context.branch(check, cmp, [ 'unlikely', 'likely' ]); - branch.left.name = 'max_overflow'; - - branch.left.terminate('ret', [ context.ret, context.ret.v(1) ]); - - store = branch.right; - } else { - store = check; - } - store.name = 'store'; - - context.store(store, field, result); - store.terminate('ret', [ context.ret, context.ret.v(0) ]); - } + get options() { return this[kOptions]; } } module.exports = MulAdd; diff --git a/lib/llparse/code/or.js b/lib/llparse/code/or.js new file mode 100644 index 0000000..ea134ca --- /dev/null +++ b/lib/llparse/code/or.js @@ -0,0 +1,10 @@ +'use strict'; + +const code = require('./'); + +class Or extends code.FieldValue { + constructor(field, value) { + super('match', 'or', field, value); + } +} +module.exports = Or; diff --git a/lib/llparse/code/span.js b/lib/llparse/code/span.js index e9713f5..d11fa85 100644 --- a/lib/llparse/code/span.js +++ b/lib/llparse/code/span.js @@ -3,5 +3,6 @@ const code = require('./'); class Span extends code.Match { + // no-op } module.exports = Span; diff --git a/lib/llparse/code/store.js b/lib/llparse/code/store.js index 6c1c2f6..61380c3 100644 --- a/lib/llparse/code/store.js +++ b/lib/llparse/code/store.js @@ -1,32 +1,10 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); -const kCompile = Symbol('compile'); - -class Store extends code.Value { - constructor(name, field) { - const store = context => this[kCompile](context, field); - - super(name, store); - } - - [kCompile](context, field) { - const body = context.fn.body; - - const stateType = context.state.type.to; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(context.match.type.isInt()); - - const adj = context.truncate(body, context.match.type, context.match, - fieldType); - - context.store(body, field, adj); - body.terminate('ret', [ context.ret, context.ret.v(0) ]); +class Store extends code.Field { + constructor(field) { + super('value', 'store', field); } } module.exports = Store; diff --git a/lib/llparse/code/test.js b/lib/llparse/code/test.js new file mode 100644 index 0000000..e9669e3 --- /dev/null +++ b/lib/llparse/code/test.js @@ -0,0 +1,10 @@ +'use strict'; + +const code = require('./'); + +class Test extends code.FieldValue { + constructor(field, mask) { + super('match', 'test', field, mask); + } +} +module.exports = Test; diff --git a/lib/llparse/code/update.js b/lib/llparse/code/update.js index 1979794..5d30200 100644 --- a/lib/llparse/code/update.js +++ b/lib/llparse/code/update.js @@ -1,33 +1,10 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); -const kCompile = Symbol('compile'); - -class Update extends code.Match { - constructor(name, field, value) { - assert.strictEqual(typeof value, 'number', - '`.update()`\'s `value` argument must be a number'); - assert.strictEqual(value, value | 0, - '`.update()`\'s `value` argument must be an integer'); - - const update = context => this[kCompile](context, field, value); - - super(name, update); - } - - [kCompile](context, field, value) { - const body = context.fn.body; - - const stateType = context.state.type.to; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - - context.store(body, field, fieldType.v(value)); - body.terminate('ret', [ context.ret, context.ret.v(0) ]); +class Update extends code.FieldValue { + constructor(field, value) { + super('match', 'update', field, value); } } module.exports = Update; diff --git a/lib/llparse/code/value.js b/lib/llparse/code/value.js index 1df3b40..eac064b 100644 --- a/lib/llparse/code/value.js +++ b/lib/llparse/code/value.js @@ -3,8 +3,8 @@ const code = require('./'); class Value extends code.Code { - constructor(name, body) { - super('value', name, body); + constructor(name) { + super('value', name); } } module.exports = Value; diff --git a/lib/llparse/compiler/code/base.js b/lib/llparse/compiler/code/base.js new file mode 100644 index 0000000..24c74e5 --- /dev/null +++ b/lib/llparse/compiler/code/base.js @@ -0,0 +1,25 @@ +'use strict'; + +class Code { + constructor(type, signature, name) { + this.type = type; + this.signature = signature; + this.name = name; + + this.isExternal = false; + this.cacheKey = this; + } + + // Just for cache key generation + numKey(num) { + if (num < 0) + return 'm' + (-num); + else + return num.toString(); + } + + build() { + throw new Error('Not implemented'); + } +} +module.exports = Code; diff --git a/lib/llparse/compiler/code/index.js b/lib/llparse/compiler/code/index.js new file mode 100644 index 0000000..60ec343 --- /dev/null +++ b/lib/llparse/compiler/code/index.js @@ -0,0 +1,19 @@ +'use strict'; + +exports.Code = require('./base'); + +// External + +exports.Match = require('./match'); +exports.Value = require('./value'); +exports.Span = require('./span'); + +// Internal + +exports.IsEqual = require('./is-equal'); +exports.Load = require('./load'); +exports.MulAdd = require('./mul-add'); +exports.Or = require('./or'); +exports.Store = require('./store'); +exports.Test = require('./test'); +exports.Update = require('./update'); diff --git a/lib/llparse/compiler/code/is-equal.js b/lib/llparse/compiler/code/is-equal.js new file mode 100644 index 0000000..c153963 --- /dev/null +++ b/lib/llparse/compiler/code/is-equal.js @@ -0,0 +1,41 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); +const llparse = require('../../'); + +const BOOL = llparse.constants.BOOL; + +class IsEqual extends code.Code { + constructor(name, field, value) { + super('is-equal', 'match', name); + + this.field = field; + this.value = value; + this.cacheKey = `is_equal_${this.field}_${this.numKey(this.value)}`; + } + + build(ctx, fn) { + const body = fn.body; + const field = this.field; + const value = this.value; + + // TODO(indutny): de-duplicate here and everywhere + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(fn.type.ret.isInt()); + + const fieldValue = ctx.load(fn, body, field); + + const cmp = ctx.ir._('icmp', [ 'eq', fieldType, fieldValue ], + fieldType.v(value)); + body.push(cmp); + const res = ctx.truncate(body, BOOL, cmp, fn.type.ret); + + body.terminate('ret', [ fn.type.ret, res ]); + } +} +module.exports = IsEqual; diff --git a/lib/llparse/compiler/code/load.js b/lib/llparse/compiler/code/load.js new file mode 100644 index 0000000..076c958 --- /dev/null +++ b/lib/llparse/compiler/code/load.js @@ -0,0 +1,31 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +class Load extends code.Code { + constructor(name, field) { + super('load', 'match', name); + + this.field = field; + this.cacheKey = `load_${this.field}`; + } + + build(ctx, fn) { + const body = fn.body; + const field = this.field; + + // TODO(indutny): de-duplicate here and everywhere + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(fn.type.ret.isInt()); + + const adj = ctx.truncate(body, fieldType, ctx.load(fn, body, field), + fn.type.ret); + body.terminate('ret', [ fn.type.ret, adj ]); + } +} +module.exports = Load; diff --git a/lib/llparse/compiler/code/match.js b/lib/llparse/compiler/code/match.js new file mode 100644 index 0000000..75de9b9 --- /dev/null +++ b/lib/llparse/compiler/code/match.js @@ -0,0 +1,17 @@ +'use strict'; + +const code = require('./'); + +class Match extends code.Code { + constructor(name) { + super('match', 'match', name); + + this.isExternal = true; + this.cacheKey = 'external_' + name; + } + + build() { + throw new Error('External code can\'t be built'); + } +} +module.exports = Match; diff --git a/lib/llparse/compiler/code/mul-add.js b/lib/llparse/compiler/code/mul-add.js new file mode 100644 index 0000000..43a865f --- /dev/null +++ b/lib/llparse/compiler/code/mul-add.js @@ -0,0 +1,118 @@ +'use strict'; + +const code = require('./'); +const llparse = require('../../'); + +const BOOL = llparse.constants.BOOL; +const INT = llparse.constants.INT; + +class MulAdd extends code.Code { + constructor(name, field, options) { + super('mulAdd', 'value', name); + + this.field = field; + this.options = options; + this.cacheKey = `mul_add_${this.field}_${JSON.stringify(this.options)}`; + } + + build(ctx, fn) { + const ir = ctx.ir; + const body = fn.body; + const match = ctx.matchArg(fn); + const matchType = match.type; + const ret = fn.type.ret; + const field = this.field; + const options = this.options; + + // TODO(indutny): de-duplicate this + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + // Declare intrinsic functions + const overRet = ir.struct([ + [ fieldType, 'result' ], + [ BOOL, 'overflow' ] + ]); + + const overSig = ir.signature(overRet, [ fieldType, fieldType ]); + const mulFn = ir.declare( + overSig, + `llvm.${options.signed ? 'smul': 'umul'}.with.overflow.${fieldType.type}` + ); + const addFn = ir.declare( + overSig, + `llvm.${options.signed ? 'sadd': 'uadd'}.with.overflow.${fieldType.type}` + ); + + // Truncate match and load field + const value = ctx.truncate(body, matchType, match, fieldType, + options.signed); + const fieldValue = ctx.load(fn, body, field); + + // Multiply + const mul = ctx.call('', overSig, mulFn, '', + [ fieldValue, matchType.v(options.base) ]); + body.push(mul); + + body.comment('extract product'); + const product = ir._('extractvalue', [ overRet, mul ], + INT.v(overRet.lookup('result'))); + body.push(product); + + body.comment('extract overflow'); + const overflowBit = ir._('extractvalue', [ overRet, mul ], + INT.v(overRet.lookup('overflow'))); + body.push(overflowBit); + + const { left: isOverflow, right: normal } = + ctx.branch(body, overflowBit, [ 'unlikely', 'likely' ]); + isOverflow.name = 'overflow'; + isOverflow.terminate('ret', [ ret, ret.v(1) ]); + + normal.name = 'no_overflow'; + + // Add + const add = ctx.call('', overSig, addFn, '', [ product, value ]); + normal.push(add); + + normal.comment('extract result'); + const result = ir._('extractvalue', [ overRet, add ], + INT.v(overRet.lookup('result'))); + normal.push(result); + + normal.comment('extract overflow'); + const addOverflowBit = ir._('extractvalue', [ overRet, add ], + INT.v(overRet.lookup('overflow'))); + normal.push(addOverflowBit); + + const { left: isAddOverflow, right: check } = + ctx.branch(normal, addOverflowBit, [ 'unlikely', 'likely' ]); + isAddOverflow.name = 'add_overflow'; + check.name = 'check'; + + isAddOverflow.terminate('ret', [ ret, ret.v(1) ]); + + // Check that we're within the limits + let store; + if (options.max) { + const cond = options.signed ? 'sgt' : 'ugt'; + const cmp = ir._('icmp', [ cond, fieldType, result ], + fieldType.v(options.max)); + check.push(cmp); + + const branch = ctx.branch(check, cmp, [ 'unlikely', 'likely' ]); + branch.left.name = 'max_overflow'; + + branch.left.terminate('ret', [ ret, ret.v(1) ]); + + store = branch.right; + } else { + store = check; + } + store.name = 'store'; + + ctx.store(fn, store, field, result); + store.terminate('ret', [ ret, ret.v(0) ]); + } +} +module.exports = MulAdd; diff --git a/lib/llparse/compiler/code/or.js b/lib/llparse/compiler/code/or.js new file mode 100644 index 0000000..4339680 --- /dev/null +++ b/lib/llparse/compiler/code/or.js @@ -0,0 +1,35 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +class Or extends code.Code { + constructor(name, field, value) { + super('or', 'match', name); + + this.field = field; + this.value = value; + this.cacheKey = `or_${this.field}_${this.numKey(this.value)}`; + } + + build(ctx, fn) { + const body = fn.body; + const field = this.field; + const value = this.value; + + // TODO(indutny): de-duplicate + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + const current = ctx.load(fn, body, field); + const result = ctx.ir._('or', [ fieldType, current ], + fieldType.v(value)); + body.push(result); + + ctx.store(fn, body, field, result); + body.terminate('ret', [ fn.type.ret, fn.type.ret.v(0) ]); + } +} +module.exports = Or; diff --git a/lib/llparse/compiler/code/span.js b/lib/llparse/compiler/code/span.js new file mode 100644 index 0000000..788f034 --- /dev/null +++ b/lib/llparse/compiler/code/span.js @@ -0,0 +1,17 @@ +'use strict'; + +const code = require('./'); + +class Span extends code.Code { + constructor(name) { + super('span', 'match', name); + + this.isExternal = true; + this.cacheKey = 'external_' + name; + } + + build() { + throw new Error('External code can\'t be built'); + } +} +module.exports = Span; diff --git a/lib/llparse/compiler/code/store.js b/lib/llparse/compiler/code/store.js new file mode 100644 index 0000000..5ec501e --- /dev/null +++ b/lib/llparse/compiler/code/store.js @@ -0,0 +1,34 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +class Store extends code.Code { + constructor(name, field) { + super('store', 'value', name); + + this.field = field; + this.cacheKey = `store_${this.field}`; + } + + build(ctx, fn) { + const body = fn.body; + const field = this.field; + + const match = ctx.matchArg(fn); + + // TODO(indutny): de-duplicate + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(match.type.isInt()); + + const adj = ctx.truncate(body, match.type, match, fieldType); + + ctx.store(fn, body, field, adj); + body.terminate('ret', [ fn.type.ret, fn.type.ret.v(0) ]); + } +} +module.exports = Store; diff --git a/lib/llparse/compiler/code/test.js b/lib/llparse/compiler/code/test.js new file mode 100644 index 0000000..6ba8963 --- /dev/null +++ b/lib/llparse/compiler/code/test.js @@ -0,0 +1,44 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); +const llparse = require('../../'); +const constants = llparse.constants; + +const BOOL = constants.BOOL; + +class Test extends code.Code { + constructor(name, field, value) { + super('test', 'match', name); + + this.field = field; + this.value = value; + this.cacheKey = `test_${this.field}_${this.numKey(this.value)}`; + } + + build(ctx, fn) { + const body = fn.body; + const field = this.field; + const value = this.value; + + // TODO(indutny): de-duplicate + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + const current = ctx.load(fn, body, field); + + const masked = ctx.ir._('and', [ fieldType, current ], + fieldType.v(value)); + body.push(masked); + + const bool = ctx.ir._('icmp', [ 'ne', fieldType, masked ], + fieldType.v(0)); + body.push(bool); + + const adj = ctx.truncate(body, BOOL, bool, fn.type.ret); + body.terminate('ret', [ fn.type.ret, adj ]); + } +} +module.exports = Test; diff --git a/lib/llparse/compiler/code/update.js b/lib/llparse/compiler/code/update.js new file mode 100644 index 0000000..2403f39 --- /dev/null +++ b/lib/llparse/compiler/code/update.js @@ -0,0 +1,31 @@ +'use strict'; + +const assert = require('assert'); + +const code = require('./'); + +class Update extends code.Code { + constructor(name, field, value) { + super('update', 'match', name); + + this.field = field; + this.value = value; + this.cacheKey = `update_${this.field}_${this.numKey(this.value)}`; + } + + build(ctx, fn) { + const body = fn.body; + const field = this.field; + const value = this.value; + + // TODO(indutny): de-duplicate + const stateType = ctx.state; + const fieldType = stateType.fields[stateType.lookup(field)].type; + + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + + ctx.store(fn, body, field, fieldType.v(value)); + body.terminate('ret', [ fn.type.ret, fn.type.ret.v(0) ]); + } +} +module.exports = Update; diff --git a/lib/llparse/compiler/code/value.js b/lib/llparse/compiler/code/value.js new file mode 100644 index 0000000..bcf63ce --- /dev/null +++ b/lib/llparse/compiler/code/value.js @@ -0,0 +1,17 @@ +'use strict'; + +const code = require('./'); + +class Value extends code.Code { + constructor(name) { + super('value', 'value', name); + + this.isExternal = true; + this.cacheKey = 'external_' + name; + } + + build() { + throw new Error('External code can\'t be built'); + } +} +module.exports = Value; diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 5e209d1..6e1d291 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -4,11 +4,9 @@ const assert = require('assert'); const IR = require('llvm-ir'); const llparse = require('../'); +const compiler = require('./'); const constants = llparse.constants; -const kType = llparse.symbols.kType; -const kBody = llparse.symbols.kBody; - const CCONV = constants.CCONV; const BOOL = constants.BOOL; @@ -66,10 +64,26 @@ class Compilation { this.codeCache = new Map(); this.debugMethod = null; + this.namespace = new Set(); + // Intermediate results from various build stages this.stageResults = {}; } + id(name, postfix = '') { + let res = name + postfix; + if (this.namespace.has(res)) { + let i; + for (i = 1; i <= this.namespace.size; i++) + if (!this.namespace.has(res + '_' + i)) + break; + res += '_' + i; + } + + this.namespace.add(res); + return { name: 'n_' + res, sourceName: name }; + } + build() { // Private fields this.declareField(this.signature.node.ptr(), '_current', @@ -144,22 +158,58 @@ class Compilation { this.stageResults[stage.name] = stage.build(); } + // TODO(indutny): find better place for it? + translateCode(code) { + // User callbacks + if (code instanceof llparse.code.Match) + return new compiler.code.Match(code.name); + else if (code instanceof llparse.code.Value) + return new compiler.code.Value(code.name); + else if (code instanceof llparse.code.Span) + return new compiler.code.Span(code.name); + + // Internal helpers + let name = 'c_' + code.name; + if (code.field) + name += '_' + code.field; + + const id = this.id(name).name; + if (code instanceof llparse.code.IsEqual) + return new compiler.code.IsEqual(id, code.field, code.value); + else if (code instanceof llparse.code.Load) + return new compiler.code.Load(id, code.field); + else if (code instanceof llparse.code.MulAdd) + return new compiler.code.MulAdd(id, code.field, code.options); + else if (code instanceof llparse.code.Or) + return new compiler.code.Or(id, code.field, code.value); + else if (code instanceof llparse.code.Store) + return new compiler.code.Store(id, code.field); + else if (code instanceof llparse.code.Test) + return new compiler.code.Test(id, code.field, code.value); + else if (code instanceof llparse.code.Update) + return new compiler.code.Update(id, code.field, code.value); + else + throw new Error('Unexpected code type of: ' + code.name); + } + buildCode(code) { + const native = this.translateCode(code); + const signatures = this.signature.callback; - const signature = code[kType] === 'match' ? + const signature = native.signature === 'match' ? signatures.match : signatures.value; - const name = code.name; - if (this.codeCache.has(name)) { - const cached = this.codeCache.get(name); + const cacheKey = native.cacheKey; + if (this.codeCache.has(cacheKey)) { + const cached = this.codeCache.get(cacheKey); assert.strictEqual(cached.type, signature, - `Conflicting code entries for "${name}"`); + `Conflicting code entries for "${native.name}"`); return cached; } let fn; - if (code[kBody] === null) { - const external = this.ir.declare(signature, name); + if (native.isExternal) { + const external = this.ir.declare(signature, native.name); // NOTE: this has no effect due to machine-specific flags // TODO(indutny): find a way to make it inline the function @@ -167,28 +217,26 @@ class Compilation { fn = external; } else { - fn = this.buildCodeWithBody(code, signature); + fn = this.buildCodeWithBody(native, signature); } - this.codeCache.set(name, fn); + this.codeCache.set(cacheKey, fn); return fn; } buildCodeWithBody(code, signature) { const args = [ ARG_STATE, ARG_POS, ARG_ENDPOS ]; - if (code[kType] === 'value') + if (code.signature === 'value') args.push(ARG_MATCH); const fn = this.ir.fn(signature, code.name, args); fn.visibility = 'internal'; fn.cconv = CCONV; - fn.attributes = 'nounwind'; - - const context = new llparse.code.Context(this, code, fn); + fn.attributes = 'nounwind norecurse ssp uwtable'; - code[kBody].call(fn.body, context); + code.build(this, fn); return fn; } @@ -363,6 +411,70 @@ class Compilation { return { body, current }; } + truncate(body, fromType, from, toType, isSigned = false) { + assert(toType.isInt()); + assert(fromType.isInt()); + + let res; + + // Same type! + if (fromType.type === toType) { + return from; + // Extend + } else if (fromType.width < toType.width) { + if (isSigned) + res = this.ir._('sext', [ fromType, from, 'to', toType ]); + else + res = this.ir._('zext', [ fromType, from, 'to', toType ]); + // Truncate + } else { + assert(fromType.width > toType.width); + res = this.ir._('trunc', [ fromType, from, 'to', toType ]); + } + + body.push(res); + return res; + } + + load(fn, body, field) { + body.comment(`load state[${field}]`); + + const lookup = this.lookup(fn, field); + body.push(lookup.instr); + + const res = this.ir._('load', lookup.type, + [ lookup.type.ptr(), lookup.instr ]); + body.push(res); + + return res; + } + + store(fn, body, field, value) { + body.comment(`store state[${field}] = ...`); + + const lookup = this.lookup(fn, field); + body.push(lookup.instr); + + const res = this.ir._('store', + [ lookup.type, value ], + [ lookup.type.ptr(), lookup.instr ]); + res.void(); + body.push(res); + } + + lookup(fn, field) { + const index = this.state.lookup(field); + const instr = this.ir._('getelementptr inbounds', this.state, + [ this.state.ptr(), this.stateArg(fn) ], + [ INT, INT.v(0) ], + [ INT, INT.v(index) ]); + + return { + type: this.state.fields[index].type, + instr + }; + } + declareField(type, name, init) { this.state.field(type, name); diff --git a/lib/llparse/compiler/index.js b/lib/llparse/compiler/index.js index 4df2983..3a70484 100644 --- a/lib/llparse/compiler/index.js +++ b/lib/llparse/compiler/index.js @@ -2,6 +2,7 @@ exports.node = require('./node'); exports.stage = require('./stage'); +exports.code = require('./code'); exports.Compilation = require('./compilation'); diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 093cca0..fe099ff 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -29,7 +29,6 @@ class NodeTranslator extends Stage { }, this.ctx.options.translator); this.nodes = new Map(); - this.namespace = new Set(); this.maxSequenceLen = 0; } @@ -41,17 +40,7 @@ class NodeTranslator extends Stage { } id(node, postfix = '') { - let res = node.name + postfix; - if (this.namespace.has(res)) { - let i; - for (i = 1; i <= this.namespace.size; i++) - if (!this.namespace.has(res + '_' + i)) - break; - res += '_' + i; - } - - this.namespace.add(res); - return { name: 'n_' + res, sourceName: node.name }; + return this.ctx.id(node.name, postfix); } buildNode(node, value) { diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index 16fe05b..e7c9a76 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -1,6 +1,5 @@ 'use strict'; -exports.kBody = Symbol('body'); exports.kCases = Symbol('cases'); exports.kCheckIsMatch = Symbol('checkIsMatch'); exports.kCode = Symbol('code'); diff --git a/test/code-test.js b/test/code-test.js index 372eef2..e71e78a 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -123,4 +123,49 @@ describe('LLParse/Code', function() { binary('010', 'off=1\noff=3\n', callback); }); }); + + describe('`.or()`/`.test()`', () => { + it('should set and retrieve bits', (callback) => { + const start = p.node('start'); + const test = p.node('test'); + + p.property('i64', 'flag'); + + start + .match('1', p.invoke(p.code.or('flag', 1), start)) + .match('2', p.invoke(p.code.or('flag', 2), start)) + .match('4', p.invoke(p.code.or('flag', 4), start)) + // Reset + .match('r', p.invoke(p.code.update('flag', 0), start)) + // Test + .match('-', test) + .otherwise(p.error(1, 'start')); + + test + .match('1', p.invoke(p.code.test('flag', 1), { + 0: test, + 1: printOff(p, test) + }, p.error(2, 'test-1'))) + .match('2', p.invoke(p.code.test('flag', 2), { + 0: test, + 1: printOff(p, test) + }, p.error(3, 'test-2'))) + .match('4', p.invoke(p.code.test('flag', 4), { + 0: test, + 1: printOff(p, test) + }, p.error(4, 'test-3'))) + // Restart + .match('.', start) + .otherwise(p.error(5, 'test')); + + const binary = fixtures.build(p, start, 'or-test'); + + binary('1-124.2-124.4-124.r4-124.', + 'off=3\n' + + 'off=9\noff=10\n' + + 'off=15\noff=16\noff=17\n' + + 'off=24\n', + callback); + }); + }); }); From 9f808f1216ae50281ebafa2a820a77f4c6b02597 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 16:30:58 -0500 Subject: [PATCH 243/281] code: rename kType => kSignature --- lib/llparse/code/base.js | 5 ++--- lib/llparse/compiler/node/invoke.js | 6 +++--- lib/llparse/compiler/node/set-index.js | 4 ++-- lib/llparse/node/consume.js | 4 ++-- lib/llparse/node/invoke.js | 4 ++-- lib/llparse/symbols.js | 1 - 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/lib/llparse/code/base.js b/lib/llparse/code/base.js index 9144ae0..2ba080d 100644 --- a/lib/llparse/code/base.js +++ b/lib/llparse/code/base.js @@ -3,12 +3,11 @@ const llparse = require('../'); const kName = Symbol('name'); -const kType = llparse.symbols.kType; +const kSignature = llparse.symbols.kSignature; class Code { constructor(signature, name) { - // TODO(indutny): rename to kSignature everywhere - this[kType] = signature; + this[kSignature] = signature; this[kName] = name; } diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 27f64b5..7b4dc5d 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -3,7 +3,7 @@ const assert = require('assert'); const llparse = require('../../'); -const kType = llparse.symbols.kType; +const kSignature = llparse.symbols.kSignature; const node = require('./'); @@ -35,10 +35,10 @@ class Invoke extends node.Node { ctx.endPos ]; - if (this.code[kType] === 'value') + if (this.code[kSignature] === 'value') args.push(ctx.match); else - assert.strictEqual(this.code[kType], 'match'); + assert.strictEqual(this.code[kSignature], 'match'); const call = ctx.call('', code.type, code, args); body.push(call); diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js index 8d043cb..52c35b7 100644 --- a/lib/llparse/compiler/node/set-index.js +++ b/lib/llparse/compiler/node/set-index.js @@ -3,7 +3,7 @@ const assert = require('assert'); const llparse = require('../../'); -const kType = llparse.symbols.kType; +const kSignature = llparse.symbols.kSignature; const node = require('./'); @@ -31,7 +31,7 @@ class SetIndex extends node.Node { ctx.endPos ]; - assert.strictEqual(this.code[kType], 'match'); + assert.strictEqual(this.code[kSignature], 'match'); const call = ctx.call('', code.type, code, args); body.push(call); diff --git a/lib/llparse/node/consume.js b/lib/llparse/node/consume.js index 6f70ee2..9bf3c27 100644 --- a/lib/llparse/node/consume.js +++ b/lib/llparse/node/consume.js @@ -5,14 +5,14 @@ const assert = require('assert'); const node = require('./'); const llparse = require('../'); -const kType = llparse.symbols.kType; +const kSignature = llparse.symbols.kSignature; const kCode = llparse.symbols.kCode; class Consume extends node.Node { constructor(code) { assert(code instanceof llparse.code.Code, 'Invalid `code` argument of `.consume()`, must be a Code instance'); - assert.strictEqual(code[kType], 'match', + assert.strictEqual(code[kSignature], 'match', '`code` argument of `.consume()` must be have a `match` type'); super('consume_' + code.name, 'match'); diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 60fffb7..2debae9 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -5,7 +5,7 @@ const assert = require('assert'); const node = require('./'); const llparse = require('../'); -const kType = llparse.symbols.kType; +const kSignature = llparse.symbols.kSignature; const kCode = llparse.symbols.kCode; const kMap = llparse.symbols.kMap; @@ -29,7 +29,7 @@ class Invoke extends node.Node { 'Only integer keys are allowed in `.invoke()`\'s map'); }); - super('invoke_' + code.name, code[kType]); + super('invoke_' + code.name, code[kSignature]); this[kCode] = code; this[kMap] = map; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js index e7c9a76..a8065ac 100644 --- a/lib/llparse/symbols.js +++ b/lib/llparse/symbols.js @@ -10,4 +10,3 @@ exports.kOtherwise = Symbol('otherwise'); exports.kSignature = Symbol('signature'); exports.kSpan = Symbol('span'); exports.kTransform = Symbol('transform'); -exports.kType = Symbol('type'); From 5ef44b35608f0380f343d8e1c337c5bca953809c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 16:34:38 -0500 Subject: [PATCH 244/281] compiler: vanity changes --- lib/llparse/code/field.js | 2 +- lib/llparse/compiler/compilation.js | 12 ++++++------ lib/llparse/compiler/stage/node-translator.js | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/llparse/code/field.js b/lib/llparse/code/field.js index 8e3d218..eae1529 100644 --- a/lib/llparse/code/field.js +++ b/lib/llparse/code/field.js @@ -6,7 +6,7 @@ const kField = Symbol('field'); class Field extends code.Code { constructor(signature, name, field) { - super(signature, name); + super(signature, name + '_' + field); this[kField] = field; } diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 6e1d291..6bed7ce 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -70,8 +70,8 @@ class Compilation { this.stageResults = {}; } - id(name, postfix = '') { - let res = name + postfix; + id(name, prefix = '', postfix = '') { + let res = prefix + name + postfix; if (this.namespace.has(res)) { let i; for (i = 1; i <= this.namespace.size; i++) @@ -81,7 +81,7 @@ class Compilation { } this.namespace.add(res); - return { name: 'n_' + res, sourceName: name }; + return { name: res, sourceName: name }; } build() { @@ -169,11 +169,11 @@ class Compilation { return new compiler.code.Span(code.name); // Internal helpers - let name = 'c_' + code.name; + let name = code.name; if (code.field) name += '_' + code.field; - const id = this.id(name).name; + const id = this.id(name, 'c_').name; if (code instanceof llparse.code.IsEqual) return new compiler.code.IsEqual(id, code.field, code.value); else if (code instanceof llparse.code.Load) @@ -230,7 +230,7 @@ class Compilation { if (code.signature === 'value') args.push(ARG_MATCH); - const fn = this.ir.fn(signature, code.name, args); + const fn = this.ir.fn(signature, this.prefix + '_' + code.name, args); fn.visibility = 'internal'; fn.cconv = CCONV; diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index fe099ff..2970236 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -40,7 +40,7 @@ class NodeTranslator extends Stage { } id(node, postfix = '') { - return this.ctx.id(node.name, postfix); + return this.ctx.id(node.name, 'n_', postfix); } buildNode(node, value) { From 022712e523cd8e0fac9d8158dd74e179e65edc49 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 16:38:35 -0500 Subject: [PATCH 245/281] 2.5.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index c23bf95..b00b7d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.4.1", + "version": "2.5.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 35805ce..7764a1f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.4.1", + "version": "2.5.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From d21b5f9cb12bcbee683a3bf8236e535b10e98505 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 17:03:14 -0500 Subject: [PATCH 246/281] invoke: add extra assert --- lib/llparse/node/invoke.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js index 2debae9..99574d5 100644 --- a/lib/llparse/node/invoke.js +++ b/lib/llparse/node/invoke.js @@ -27,6 +27,8 @@ class Invoke extends node.Node { Object.keys(map).forEach((key) => { assert.equal(key, key | 0, 'Only integer keys are allowed in `.invoke()`\'s map'); + assert(map[key] instanceof node.Node, + 'Only Node values are allowed in `.invoke()`\'s map'); }); super('invoke_' + code.name, code[kSignature]); From 6244425b409c178fddad999de19e1de090fb52f8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 17:10:15 -0500 Subject: [PATCH 247/281] span: cache `.start()`/`.end()` by `otherwise` --- lib/llparse/span.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/llparse/span.js b/lib/llparse/span.js index 2b4ffc4..44d0b37 100644 --- a/lib/llparse/span.js +++ b/lib/llparse/span.js @@ -5,6 +5,8 @@ const assert = require('assert'); const llparse = require('./'); const kCode = llparse.symbols.kCode; +const kStartCache = Symbol('startCache'); +const kEndCache = Symbol('endCache'); class Span { constructor(code) { @@ -12,19 +14,33 @@ class Span { 'Invalid `code` argument of `.span()`, must be a Code instance'); this[kCode] = code; + this[kStartCache] = new Map(); + this[kEndCache] = new Map(); } start(otherwise = null) { + const cache = this[kStartCache]; + if (otherwise && cache.has(otherwise)) + return cache.get(otherwise); + const res = new llparse.node.SpanStart(this, this[kCode]); - if (otherwise) + if (otherwise) { res.otherwise(otherwise); + cache.set(otherwise, res); + } return res; } end(otherwise = null) { + const cache = this[kEndCache]; + if (otherwise && cache.has(otherwise)) + return cache.get(otherwise); + const res = new llparse.node.SpanEnd(this, this[kCode]); - if (otherwise) + if (otherwise) { res.otherwise(otherwise); + cache.set(otherwise, res); + } return res; } } From e70e7e475b77a8059b033090000e488a8c6516a8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 17:11:09 -0500 Subject: [PATCH 248/281] 2.5.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b00b7d2..954fd07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.0", + "version": "2.5.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 7764a1f..a9c6785 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.0", + "version": "2.5.1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 81056f99beac0912a4e8332820ca77b68099cdfc Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 20:49:30 -0500 Subject: [PATCH 249/281] code: fix `test` --- lib/llparse/compiler/code/test.js | 4 ++-- package-lock.json | 6 +++--- package.json | 2 +- test/code-test.js | 18 +++++++++++------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/llparse/compiler/code/test.js b/lib/llparse/compiler/code/test.js index 6ba8963..064ccb9 100644 --- a/lib/llparse/compiler/code/test.js +++ b/lib/llparse/compiler/code/test.js @@ -33,8 +33,8 @@ class Test extends code.Code { fieldType.v(value)); body.push(masked); - const bool = ctx.ir._('icmp', [ 'ne', fieldType, masked ], - fieldType.v(0)); + const bool = ctx.ir._('icmp', [ 'eq', fieldType, masked ], + fieldType.v(value)); body.push(bool); const adj = ctx.truncate(body, BOOL, bool, fn.type.ret); diff --git a/package-lock.json b/package-lock.json index 954fd07..117b339 100644 --- a/package-lock.json +++ b/package-lock.json @@ -739,9 +739,9 @@ } }, "llparse-test-fixture": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.1.0.tgz", - "integrity": "sha512-LSlFr7w2zA3215nBvPEuEF2iw6D6wnS9GkTG7KgswLgPr6As0c7uCVR+AbAVO2JDVnDtyvtTKFUo+tpt2XNS4g==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.2.1.tgz", + "integrity": "sha512-zIe5Eyz2sOmA3Lg0HFOy4Z3qOevLup4l24nYpgtYS1wixIr8dlZaQ7teBlIVMM90nafR6dIH8xB1/UF7XDX00w==", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index a9c6785..89d987d 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "devDependencies": { "async": "^2.6.0", "eslint": "^4.18.1", - "llparse-test-fixture": "^1.1.0", + "llparse-test-fixture": "^1.2.1", "mocha": "^5.0.1" }, "directories": { diff --git a/test/code-test.js b/test/code-test.js index e71e78a..74211bc 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -154,18 +154,22 @@ describe('LLParse/Code', function() { 0: test, 1: printOff(p, test) }, p.error(4, 'test-3'))) + .match('7', p.invoke(p.code.test('flag', 7), { + 0: test, + 1: printOff(p, test) + }, p.error(5, 'test-7'))) // Restart .match('.', start) - .otherwise(p.error(5, 'test')); + .otherwise(p.error(6, 'test')); const binary = fixtures.build(p, start, 'or-test'); - binary('1-124.2-124.4-124.r4-124.', - 'off=3\n' + - 'off=9\noff=10\n' + - 'off=15\noff=16\noff=17\n' + - 'off=24\n', - callback); + binary('1-124.2-1247.4-1247.r4-124.', [ + 'off=3', + 'off=9', 'off=10', + 'off=16', 'off=17', 'off=18', 'off=19', + 'off=26' + ], callback); }); }); }); From 9e977e2c75b1e8da4a1318fe431c4047cc119972 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 20:49:36 -0500 Subject: [PATCH 250/281] 2.5.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 117b339..b4c4b56 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.1", + "version": "2.5.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 89d987d..cee13e9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.1", + "version": "2.5.2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 77e74a91d39d4b1109bbac5c9eb6a04dac7e6a73 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 22:08:21 -0500 Subject: [PATCH 251/281] node-translator: cache errors by code/reason --- lib/llparse/compiler/stage/node-translator.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 2970236..4df8e8d 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -29,6 +29,7 @@ class NodeTranslator extends Stage { }, this.ctx.options.translator); this.nodes = new Map(); + this.errorCache = new Map(); this.maxSequenceLen = 0; } @@ -55,7 +56,7 @@ class NodeTranslator extends Stage { if (node instanceof llparse.node.Invoke) { res = new compiler.node.Invoke(this.id(node), node[kCode]); } else if (node instanceof llparse.node.Error) { - res = new compiler.node.Error(this.id(node), node.code, node.reason); + res = this.buildError(node); } else if (node instanceof llparse.node.SpanStart) { res = new compiler.node.SpanStart(this.id(node), node[kSpan][kCode]); } else if (node instanceof llparse.node.SpanEnd) { @@ -111,6 +112,16 @@ class NodeTranslator extends Stage { return res; } + buildError(node) { + const cacheKey = (node.code >>> 0) + ':' + node.reason; + if (this.errorCache.has(cacheKey)) + return this.errorCache.get(cacheKey); + + const res = new compiler.node.Error(this.id(node), node.code, node.reason); + this.errorCache.set(cacheKey, res); + return res; + } + buildTrie(node, trie, list, isRoot = false) { if (trie.type === 'next') { assert(!isRoot); From 1c487e1e2e7d4d13c4f0efeca7b303e68bf2b4d1 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 22:41:09 -0500 Subject: [PATCH 252/281] node: `noPrologueCheck` --- lib/llparse/compiler/node/base.js | 4 ++++ lib/llparse/compiler/node/consume.js | 5 +---- lib/llparse/compiler/node/error.js | 5 +---- lib/llparse/compiler/node/invoke.js | 5 +---- lib/llparse/compiler/node/set-index.js | 5 +---- lib/llparse/compiler/node/span-end.js | 5 +---- lib/llparse/compiler/node/span-start.js | 5 +---- 7 files changed, 10 insertions(+), 24 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 5c55451..8c1d7f7 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -62,6 +62,7 @@ class Node { this.otherwise = null; this.skip = false; this.transform = null; + this.noPrologueCheck = false; this.fn = null; this.phis = new Map(); @@ -117,6 +118,9 @@ class Node { } prologue(ctx, body) { + if (this.noPrologueCheck) + return body; + const pos = ctx.pos.current; const endPos = ctx.endPos; diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 77772fc..8dd7c18 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -7,10 +7,7 @@ const node = require('./'); class Consume extends node.Node { constructor(...args) { super('consume', ...args); - } - - prologue(ctx, body) { - return body; + this.noPrologueCheck = true; } // TODO(indutny): remove unnecessary load diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index 1a1adc2..c3afe3e 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -8,16 +8,13 @@ class Error extends node.Node { this.code = code; this.reason = reason; + this.noPrologueCheck = true; } getChildren() { return []; } - prologue(ctx, body) { - return body; - } - buildStoreError(ctx, body) { const INT = ctx.INT; diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index 7b4dc5d..a30ad91 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -13,6 +13,7 @@ class Invoke extends node.Node { this.code = code; this.map = null; + this.noPrologueCheck = true; } getChildren() { @@ -21,10 +22,6 @@ class Invoke extends node.Node { })); } - prologue(ctx, body) { - return body; - } - doBuild(ctx, body) { body.comment(`node.Invoke[${this.code.name}]`); const code = ctx.compilation.buildCode(this.code); diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js index 52c35b7..31eb5e8 100644 --- a/lib/llparse/compiler/node/set-index.js +++ b/lib/llparse/compiler/node/set-index.js @@ -12,10 +12,7 @@ class SetIndex extends node.Node { super('set-index', id); this.code = code; - } - - prologue(ctx, body) { - return body; + this.noPrologueCheck = true; } doBuild(ctx, body) { diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 69f33bd..95ba5d1 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -7,10 +7,7 @@ class SpanEnd extends node.Node { super('span-end', id); this.code = code; - } - - prologue(ctx, body) { - return body; + this.noPrologueCheck = true; } getResumptionTargets() { diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index e6042df..7662dab 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -7,10 +7,7 @@ class SpanStart extends node.Node { super('span-start', id); this.code = code; - } - - prologue(ctx, body) { - return body; + this.noPrologueCheck = true; } doBuild(ctx, body) { From e441acc183c4740e730faecba97789eb729e2397 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 22:47:04 -0500 Subject: [PATCH 253/281] node-translator: "peephole" optimization --- lib/llparse/compiler/stage/node-translator.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 4df8e8d..499d856 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -109,6 +109,24 @@ class NodeTranslator extends Stage { }); } + // Do a "peephole"-like optimization. If there are empty nodes that do + // not skip over the input, and lead to nodes with prologue check + // (p != endp) - stitch the node straight to its `otherwise` recursively. + for (;;) { + if (!(res instanceof compiler.node.Empty)) + break; + if (res.skip) + break; + + const otherwise = res.otherwise; + // Since we're running in de-looped mode, sometimes `otherwise` may not + // be set yet. + if (!otherwise || otherwise.noPrologueCheck) + break; + + res = otherwise; + } + return res; } From 4a9c880f901c549031fa86217f9dacac53b87bf4 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Fri, 2 Mar 2018 22:48:47 -0500 Subject: [PATCH 254/281] 2.5.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b4c4b56..1fb8d92 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.2", + "version": "2.5.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index cee13e9..0a9d62c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.2", + "version": "2.5.3", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From d03139b9d20e165d30da02ef0473c776efc719f5 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Sun, 11 Mar 2018 19:22:56 -0400 Subject: [PATCH 255/281] lib: port to `bitcode` and `bitcode-builder` --- lib/llparse/compiler/code/base.js | 11 + lib/llparse/compiler/code/is-equal.js | 21 +- lib/llparse/compiler/code/load.js | 14 +- lib/llparse/compiler/code/mul-add.js | 71 ++--- lib/llparse/compiler/code/or.js | 14 +- lib/llparse/compiler/code/store.js | 13 +- lib/llparse/compiler/code/test.js | 23 +- lib/llparse/compiler/code/update.js | 12 +- lib/llparse/compiler/compilation.js | 287 +++++++++---------- lib/llparse/compiler/compiler.js | 67 ++--- lib/llparse/compiler/node/base.js | 58 ++-- lib/llparse/compiler/node/bit-check.js | 57 ++-- lib/llparse/compiler/node/consume.js | 57 +--- lib/llparse/compiler/node/empty.js | 2 - lib/llparse/compiler/node/error.js | 39 +-- lib/llparse/compiler/node/invoke.js | 6 +- lib/llparse/compiler/node/pause.js | 12 +- lib/llparse/compiler/node/sequence.js | 37 +-- lib/llparse/compiler/node/set-index.js | 17 +- lib/llparse/compiler/node/single.js | 7 +- lib/llparse/compiler/node/span-end.js | 1 - lib/llparse/compiler/node/span-start.js | 1 - lib/llparse/compiler/stage/match-sequence.js | 170 +++++------ lib/llparse/compiler/stage/span-builder.js | 169 ++++------- lib/llparse/constants.js | 8 +- package-lock.json | 145 +++++----- package.json | 10 +- test/code-test.js | 2 +- 28 files changed, 514 insertions(+), 817 deletions(-) diff --git a/lib/llparse/compiler/code/base.js b/lib/llparse/compiler/code/base.js index 24c74e5..0e05fc6 100644 --- a/lib/llparse/compiler/code/base.js +++ b/lib/llparse/compiler/code/base.js @@ -1,5 +1,7 @@ 'use strict'; +const assert = require('assert'); + class Code { constructor(type, signature, name) { this.type = type; @@ -18,6 +20,15 @@ class Code { return num.toString(); } + getTypes(ctx, fn, field) { + const fieldType = ctx.state.lookupField(field).ty; + const returnType = fn.ty.toSignature().returnType; + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(returnType.isInt()); + + return { fieldType, returnType }; + } + build() { throw new Error('Not implemented'); } diff --git a/lib/llparse/compiler/code/is-equal.js b/lib/llparse/compiler/code/is-equal.js index c153963..ce1b417 100644 --- a/lib/llparse/compiler/code/is-equal.js +++ b/lib/llparse/compiler/code/is-equal.js @@ -1,11 +1,6 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); -const llparse = require('../../'); - -const BOOL = llparse.constants.BOOL; class IsEqual extends code.Code { constructor(name, field, value) { @@ -21,21 +16,11 @@ class IsEqual extends code.Code { const field = this.field; const value = this.value; - // TODO(indutny): de-duplicate here and everywhere - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(fn.type.ret.isInt()); - + const { fieldType, returnType } = this.getTypes(ctx, fn, field); const fieldValue = ctx.load(fn, body, field); - const cmp = ctx.ir._('icmp', [ 'eq', fieldType, fieldValue ], - fieldType.v(value)); - body.push(cmp); - const res = ctx.truncate(body, BOOL, cmp, fn.type.ret); - - body.terminate('ret', [ fn.type.ret, res ]); + const cmp = body.icmp('eq', fieldValue, fieldType.val(value)); + body.ret(ctx.truncate(body, cmp, returnType)); } } module.exports = IsEqual; diff --git a/lib/llparse/compiler/code/load.js b/lib/llparse/compiler/code/load.js index 076c958..ca310dc 100644 --- a/lib/llparse/compiler/code/load.js +++ b/lib/llparse/compiler/code/load.js @@ -1,7 +1,5 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); class Load extends code.Code { @@ -16,16 +14,10 @@ class Load extends code.Code { const body = fn.body; const field = this.field; - // TODO(indutny): de-duplicate here and everywhere - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(fn.type.ret.isInt()); + const { returnType } = this.getTypes(ctx, fn, field); - const adj = ctx.truncate(body, fieldType, ctx.load(fn, body, field), - fn.type.ret); - body.terminate('ret', [ fn.type.ret, adj ]); + const adj = ctx.truncate(body, ctx.load(fn, body, field), returnType); + body.ret(adj); } } module.exports = Load; diff --git a/lib/llparse/compiler/code/mul-add.js b/lib/llparse/compiler/code/mul-add.js index 43a865f..a4a99f5 100644 --- a/lib/llparse/compiler/code/mul-add.js +++ b/lib/llparse/compiler/code/mul-add.js @@ -4,7 +4,6 @@ const code = require('./'); const llparse = require('../../'); const BOOL = llparse.constants.BOOL; -const INT = llparse.constants.INT; class MulAdd extends code.Code { constructor(name, field, options) { @@ -19,91 +18,71 @@ class MulAdd extends code.Code { const ir = ctx.ir; const body = fn.body; const match = ctx.matchArg(fn); - const matchType = match.type; - const ret = fn.type.ret; const field = this.field; const options = this.options; - // TODO(indutny): de-duplicate this - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; + const { fieldType, returnType } = this.getTypes(ctx, fn, field); // Declare intrinsic functions - const overRet = ir.struct([ - [ fieldType, 'result' ], - [ BOOL, 'overflow' ] - ]); + const overRet = ir.struct(); + overRet.addField(fieldType, 'result'); + overRet.addField(BOOL, 'overflow'); + overRet.finalize(); const overSig = ir.signature(overRet, [ fieldType, fieldType ]); - const mulFn = ir.declare( + const postfix = `with.overflow.${fieldType.typeString}`; + const mulFn = ctx.declareFunction( overSig, - `llvm.${options.signed ? 'smul': 'umul'}.with.overflow.${fieldType.type}` + `llvm.${options.signed ? 'smul': 'umul'}.${postfix}` ); - const addFn = ir.declare( + const addFn = ctx.declareFunction( overSig, - `llvm.${options.signed ? 'sadd': 'uadd'}.with.overflow.${fieldType.type}` + `llvm.${options.signed ? 'sadd': 'uadd'}.${postfix}` ); // Truncate match and load field - const value = ctx.truncate(body, matchType, match, fieldType, + const value = ctx.truncate(body, match, fieldType, options.signed); const fieldValue = ctx.load(fn, body, field); // Multiply - const mul = ctx.call('', overSig, mulFn, '', - [ fieldValue, matchType.v(options.base) ]); - body.push(mul); + const mul = body.call(mulFn, [ fieldValue, fieldType.val(options.base) ]); - body.comment('extract product'); - const product = ir._('extractvalue', [ overRet, mul ], - INT.v(overRet.lookup('result'))); - body.push(product); - - body.comment('extract overflow'); - const overflowBit = ir._('extractvalue', [ overRet, mul ], - INT.v(overRet.lookup('overflow'))); - body.push(overflowBit); + const product = body.extractvalue(mul, overRet.lookupField('result').index); + const overflowBit = body.extractvalue(mul, + overRet.lookupField('overflow').index); const { left: isOverflow, right: normal } = ctx.branch(body, overflowBit, [ 'unlikely', 'likely' ]); isOverflow.name = 'overflow'; - isOverflow.terminate('ret', [ ret, ret.v(1) ]); + isOverflow.ret(returnType.val(1)); normal.name = 'no_overflow'; // Add - const add = ctx.call('', overSig, addFn, '', [ product, value ]); - normal.push(add); - - normal.comment('extract result'); - const result = ir._('extractvalue', [ overRet, add ], - INT.v(overRet.lookup('result'))); - normal.push(result); + const add = normal.call(addFn, [ product, value ]); - normal.comment('extract overflow'); - const addOverflowBit = ir._('extractvalue', [ overRet, add ], - INT.v(overRet.lookup('overflow'))); - normal.push(addOverflowBit); + const result = normal.extractvalue(add, + overRet.lookupField('result').index); + const addOverflowBit = normal.extractvalue(add, + overRet.lookupField('overflow').index); const { left: isAddOverflow, right: check } = ctx.branch(normal, addOverflowBit, [ 'unlikely', 'likely' ]); isAddOverflow.name = 'add_overflow'; check.name = 'check'; - isAddOverflow.terminate('ret', [ ret, ret.v(1) ]); + isAddOverflow.ret(returnType.val(1)); // Check that we're within the limits let store; if (options.max) { const cond = options.signed ? 'sgt' : 'ugt'; - const cmp = ir._('icmp', [ cond, fieldType, result ], - fieldType.v(options.max)); - check.push(cmp); - + const cmp = check.icmp(cond, result, fieldType.val(options.max)); const branch = ctx.branch(check, cmp, [ 'unlikely', 'likely' ]); branch.left.name = 'max_overflow'; - branch.left.terminate('ret', [ ret, ret.v(1) ]); + branch.left.ret(returnType.val(1)); store = branch.right; } else { @@ -112,7 +91,7 @@ class MulAdd extends code.Code { store.name = 'store'; ctx.store(fn, store, field, result); - store.terminate('ret', [ ret, ret.v(0) ]); + store.ret(returnType.val(0)); } } module.exports = MulAdd; diff --git a/lib/llparse/compiler/code/or.js b/lib/llparse/compiler/code/or.js index 4339680..b13a65b 100644 --- a/lib/llparse/compiler/code/or.js +++ b/lib/llparse/compiler/code/or.js @@ -1,7 +1,5 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); class Or extends code.Code { @@ -18,18 +16,12 @@ class Or extends code.Code { const field = this.field; const value = this.value; - // TODO(indutny): de-duplicate - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; + const { fieldType, returnType } = this.getTypes(ctx, fn, field); - assert(fieldType.isInt(), `"${field}" field is not of integer type`); const current = ctx.load(fn, body, field); - const result = ctx.ir._('or', [ fieldType, current ], - fieldType.v(value)); - body.push(result); - + const result = body.binop('or', current, fieldType.val(value)); ctx.store(fn, body, field, result); - body.terminate('ret', [ fn.type.ret, fn.type.ret.v(0) ]); + body.ret(returnType.val(0)); } } module.exports = Or; diff --git a/lib/llparse/compiler/code/store.js b/lib/llparse/compiler/code/store.js index 5ec501e..c7f3146 100644 --- a/lib/llparse/compiler/code/store.js +++ b/lib/llparse/compiler/code/store.js @@ -1,7 +1,5 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); class Store extends code.Code { @@ -18,17 +16,12 @@ class Store extends code.Code { const match = ctx.matchArg(fn); - // TODO(indutny): de-duplicate - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(match.type.isInt()); + const { fieldType, returnType } = this.getTypes(ctx, fn, field); - const adj = ctx.truncate(body, match.type, match, fieldType); + const adj = ctx.truncate(body, match, fieldType); ctx.store(fn, body, field, adj); - body.terminate('ret', [ fn.type.ret, fn.type.ret.v(0) ]); + body.ret(returnType.val(0)); } } module.exports = Store; diff --git a/lib/llparse/compiler/code/test.js b/lib/llparse/compiler/code/test.js index 064ccb9..0e07d41 100644 --- a/lib/llparse/compiler/code/test.js +++ b/lib/llparse/compiler/code/test.js @@ -1,12 +1,6 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); -const llparse = require('../../'); -const constants = llparse.constants; - -const BOOL = constants.BOOL; class Test extends code.Code { constructor(name, field, value) { @@ -22,23 +16,14 @@ class Test extends code.Code { const field = this.field; const value = this.value; - // TODO(indutny): de-duplicate - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; + const { fieldType, returnType } = this.getTypes(ctx, fn, field); - assert(fieldType.isInt(), `"${field}" field is not of integer type`); const current = ctx.load(fn, body, field); - const masked = ctx.ir._('and', [ fieldType, current ], - fieldType.v(value)); - body.push(masked); - - const bool = ctx.ir._('icmp', [ 'eq', fieldType, masked ], - fieldType.v(value)); - body.push(bool); + const masked = body.binop('and', current, fieldType.val(value)); + const bool = body.icmp('eq', masked, fieldType.val(value)); - const adj = ctx.truncate(body, BOOL, bool, fn.type.ret); - body.terminate('ret', [ fn.type.ret, adj ]); + body.ret(ctx.truncate(body, bool, returnType)); } } module.exports = Test; diff --git a/lib/llparse/compiler/code/update.js b/lib/llparse/compiler/code/update.js index 2403f39..0488cf2 100644 --- a/lib/llparse/compiler/code/update.js +++ b/lib/llparse/compiler/code/update.js @@ -1,7 +1,5 @@ 'use strict'; -const assert = require('assert'); - const code = require('./'); class Update extends code.Code { @@ -18,14 +16,10 @@ class Update extends code.Code { const field = this.field; const value = this.value; - // TODO(indutny): de-duplicate - const stateType = ctx.state; - const fieldType = stateType.fields[stateType.lookup(field)].type; - - assert(fieldType.isInt(), `"${field}" field is not of integer type`); + const { fieldType, returnType } = this.getTypes(ctx, fn, field); - ctx.store(fn, body, field, fieldType.v(value)); - body.terminate('ret', [ fn.type.ret, fn.type.ret.v(0) ]); + ctx.store(fn, body, field, fieldType.val(value)); + body.ret(returnType.val(0)); } } module.exports = Update; diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index 6bed7ce..b74d05c 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -1,7 +1,7 @@ 'use strict'; const assert = require('assert'); -const IR = require('llvm-ir'); +const bitcode = require('bitcode'); const llparse = require('../'); const compiler = require('./'); @@ -9,7 +9,6 @@ const constants = llparse.constants; const CCONV = constants.CCONV; -const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; @@ -32,7 +31,8 @@ class Compilation { constructor(options) { this.options = Object.assign({}, options); - this.ir = new IR(); + this.bitcode = new bitcode.Module(); + this.ir = this.bitcode.createBuilder(); this.prefix = this.options.prefix; this.root = this.options.root; @@ -42,9 +42,9 @@ class Compilation { this.signature = { node: this.ir.signature(TYPE_OUTPUT, [ - [ this.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ], + this.state.ptr(), + TYPE_INPUT, + TYPE_INPUT, TYPE_MATCH ]), callback: { @@ -59,7 +59,9 @@ class Compilation { }; this.signature.callback.span = this.signature.callback.match; - this.INVARIANT_GROUP = this.ir.metadata('!"llparse.invariant"'); + this.INVARIANT_GROUP = this.ir.metadata([ + this.ir.metadata('llparse.invariant') + ]); this.codeCache = new Map(); this.debugMethod = null; @@ -87,33 +89,40 @@ class Compilation { build() { // Private fields this.declareField(this.signature.node.ptr(), '_current', - (type, ctx) => ctx.stageResults['node-builder'].entry.ref()); + (type, ctx) => ctx.stageResults['node-builder'].entry); - this.declareField(TYPE_INDEX, '_index', type => type.v(0)); + this.declareField(TYPE_INDEX, '_index', type => type.val(0)); // Some stages may add more private fields this.buildStages(this.options.stages.before); // Public fields - this.declareField(TYPE_ERROR, 'error', type => type.v(0)); - this.declareField(TYPE_REASON, 'reason', type => type.v(null)); - this.declareField(TYPE_INPUT, 'error_pos', type => type.v(null)); - this.declareField(TYPE_DATA, 'data', type => type.v(null)); + this.declareField(TYPE_ERROR, 'error', type => type.val(0)); + this.declareField(TYPE_REASON, 'reason', type => type.val(null)); + this.declareField(TYPE_INPUT, 'error_pos', type => type.val(null)); + this.declareField(TYPE_DATA, 'data', type => type.val(null)); // Custom fields this.options.properties.forEach((prop) => { this.declareField(prop.type, prop.name, (type) => { if (type.isPointer()) - return type.v(null); + return type.val(null); else - return type.v(0); + return type.val(0); }); }); + // Lock up the struct + this.state.finalize(); + // Some stages may add more private fields this.buildStages(this.options.stages.after); } + end() { + return this.bitcode.build(this.ir); + } + buildCState() { const out = []; @@ -121,7 +130,7 @@ class Compilation { out.push(`struct ${this.prefix}_state_s {`); this.state.fields.forEach((field) => { - let type = field.type; + let type = field.ty; if (type.isPointer()) { if (field.name === 'error_pos' || field.name === 'reason') type = 'const char*'; @@ -158,6 +167,18 @@ class Compilation { this.stageResults[stage.name] = stage.build(); } + declareFunction(signature, name) { + const res = signature.declareFunction(name); + this.bitcode.add(res); + return res; + } + + defineFunction(signature, name, paramNames) { + const res = signature.defineFunction(name, paramNames); + this.bitcode.add(res); + return res; + } + // TODO(indutny): find better place for it? translateCode(code) { // User callbacks @@ -202,18 +223,18 @@ class Compilation { const cacheKey = native.cacheKey; if (this.codeCache.has(cacheKey)) { const cached = this.codeCache.get(cacheKey); - assert.strictEqual(cached.type, signature, + assert(cached.ty.isEqual(signature), `Conflicting code entries for "${native.name}"`); return cached; } let fn; if (native.isExternal) { - const external = this.ir.declare(signature, native.name); + const external = this.declareFunction(signature, native.name); // NOTE: this has no effect due to machine-specific flags // TODO(indutny): find a way to make it inline the function - external.attributes = 'alwaysinline'; + external.attrs.add('alwaysinline'); fn = external; } else { @@ -230,32 +251,48 @@ class Compilation { if (code.signature === 'value') args.push(ARG_MATCH); - const fn = this.ir.fn(signature, this.prefix + '_' + code.name, args); + const fn = this.defineFunction(signature, this.prefix + '_' + code.name, + args); - fn.visibility = 'internal'; + fn.linkage = 'internal'; fn.cconv = CCONV; - fn.attributes = 'nounwind norecurse ssp uwtable'; + fn.attrs.add([ 'nounwind', 'norecurse', 'ssp', 'uwtable' ]); code.build(this, fn); return fn; } + cstring(string) { + return this.addGlobalConst('cstr', this.ir.cstring(string)); + } + + blob(data) { + return this.addGlobalConst('blob', this.ir.blob(data)); + } + + addGlobalConst(name, value) { + const id = this.id(name, 'g_').name; + const glob = this.ir.global(value.ty.ptr(), id, value); + glob.linkage = 'internal'; + glob.markConstant(); + this.bitcode.add(glob); + return glob; + } + debug(fn, body, string) { if (!this.options.debug) return body; - const str = this.ir.cstr(string); - const cast = IR._('getelementptr inbounds', str.type.to, [ str.type, str ], - [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - body.push(cast); + const str = this.cstring(string); + const cast = body.getelementptr(str, INT.val(0), INT.val(0)); // Lazily declare debug method if (this.debugMethod === null) { - const sig = IR.signature(IR.void(), + const sig = this.ir.signature(this.ir.void(), [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_INPUT ]); - this.debugMethod = this.ir.declare(sig, this.options.debug); + this.debugMethod = this.declareFunction(sig, this.options.debug); } const args = [ @@ -265,8 +302,7 @@ class Compilation { cast ]; - const call = this.call('', this.debugMethod.type, this.debugMethod, args); - body.push(call); + body.call(this.debugMethod, args); return body; } @@ -276,60 +312,36 @@ class Compilation { let fn; if (signature === this.signature.node) { - fn = this.ir.fn(this.signature.node, name, + fn = this.defineFunction(this.signature.node, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); // TODO(indutny): reassess `minsize`. Looks like it gives best performance // results right now, though. - fn.attributes = 'nounwind minsize'; + fn.attrs.add([ 'nounwind', 'minsize' ]); // These are ABI dependent, but should bring us close to C-function // inlining on x86_64 through -flto - fn.attributes += ' ssp uwtable'; + fn.attrs.add([ 'ssp', 'uwtable' ]); + + fn.paramAttrs[0].add(ATTR_STATE); + fn.paramAttrs[1].add(ATTR_POS); + fn.paramAttrs[2].add(ATTR_ENDPOS); } else if (signature === this.signature.callback.match) { - fn = this.ir.fn(this.signature.callback.match, name, + fn = this.defineFunction(this.signature.callback.match, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); } else if (signature === this.signature.callback.value) { - fn = this.ir.fn(this.signature.callback.match, name, + fn = this.defineFunction(this.signature.callback.match, name, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH ]); } else { throw new Error('Unknown signature: ' + signature); } - fn.visibility = 'internal'; + fn.linkage = 'internal'; fn.cconv = CCONV; return fn; } - call(type, signature, ref, cconv, args) { - if (Array.isArray(cconv)) { - args = cconv; - cconv = ref.cconv || ''; - } - assert.strictEqual(args.length, signature.args.length); - - const irArgs = [ signature.ret, ref, '(' ]; - args.forEach((arg, i) => { - const isLast = i === args.length - 1; - irArgs.push(signature.args[i], arg); - if (!isLast) - irArgs.push(','); - }); - irArgs.push(')'); - - if (type) - type += ' '; - - if (cconv) - cconv = ' ' + cconv; - - const res = IR._(`${type}call${cconv}`, irArgs); - if (signature.ret.isVoid()) - res.void(); - return res; - } - toBranchWeight(value) { if (value === 'likely') return 0x10000; @@ -341,69 +353,68 @@ class Compilation { } branch(body, cmp, weights) { - const extra = []; + const onTrue = body.parent.createBlock('true'); + const onFalse = body.parent.createBlock('false'); + const branch = body.branch(cmp, onTrue, onFalse); + if (weights) { assert(Array.isArray(weights)); assert.strictEqual(weights.length, 2); weights = weights.map(w => this.toBranchWeight(w)); - const meta = this.ir.metadata( - `!"branch_weights", i32 ${weights[0]}, i32 ${weights[1]}`); - extra.push([ '!prof', meta ]); + const meta = this.ir.metadata([ + this.ir.metadata('branch_weights'), + this.ir.metadata(INT.val(weights[0])), + this.ir.metadata(INT.val(weights[1])) + ]); + branch.metadata.set('prof', meta); } - const branches = body.terminate('br', [ BOOL, cmp ], - this.ir.label('true'), - this.ir.label('false'), - ...extra); + // TODO(indutny): rename `left`/`right` to `onTrue`/`onFalse` return { - left: branches[0], - right: branches[1] + left: onTrue, + right: onFalse }; } - buildSwitch(body, type, what, values, weights) { + buildSwitch(body, what, values, weights) { const cases = []; - cases.push(IR.label('otherwise')); - cases.push('['); + const blocks = []; values.forEach((value, i) => { - cases.push(type, type.v(value)); - cases.push(',', IR.label(`case_${i}`)); + const block = body.parent.createBlock(`case_${i}`); + blocks.push(block); + cases.push({ + value: what.ty.val(value), + block, + }); }); - cases.push(']'); - const extra = []; + const otherwise = body.parent.createBlock('otherwise'); + const sw = body.switch(what, otherwise, cases); if (weights) { assert(Array.isArray(weights)); assert.strictEqual(weights.length, 1 + values.length); - weights = weights.map(w => 'i32 ' + this.toBranchWeight(w)); - const meta = this.ir.metadata( - `!"branch_weights", ${weights.join(', ')}`); - extra.push([ '!prof', meta ]); - } - - const blocks = body.terminate('switch', [ type, what ], cases, ...extra); + weights = weights.map((weight) => { + return this.ir.metadata(INT.val(this.toBranchWeight(weight))); + }); - blocks[0].name = 'switch_otherwise'; - for (let i = 0; i < values.length; i++) { - const v = values[i] < 0 ? 'm' + (-values[i]) : values[i]; - blocks[i + 1].name = 'case_' + v; + const meta = this.ir.metadata([ + this.ir.metadata('branch_weights') + ].concat(weights)); + sw.metadata.set('prof', meta); } return { - otherwise: blocks[0], - cases: blocks.slice(1) + otherwise, + cases: blocks }; } buildTransform(transform, body, current) { - body.comment(`transform[${transform.name}]`); if (transform.name === 'to_lower_unsafe') { - current = this.ir._('or', [ TYPE_INPUT.to, current ], - TYPE_INPUT.to.v(0x20)); - body.push(current); + current = body.binop('or', current, TYPE_INPUT.to.val(0x20)); } else { throw new Error('Unsupported transform: ' + transform.name); } @@ -411,98 +422,64 @@ class Compilation { return { body, current }; } - truncate(body, fromType, from, toType, isSigned = false) { + truncate(body, from, toType, isSigned = false) { + const fromTy = from.ty; assert(toType.isInt()); - assert(fromType.isInt()); + assert(fromTy.isInt()); let res; // Same type! - if (fromType.type === toType) { + if (fromTy.isEqual(toType)) { return from; // Extend - } else if (fromType.width < toType.width) { + } else if (fromTy.width < toType.width) { if (isSigned) - res = this.ir._('sext', [ fromType, from, 'to', toType ]); + res = body.cast('sext', from, toType); else - res = this.ir._('zext', [ fromType, from, 'to', toType ]); + res = body.cast('zext', from, toType); // Truncate } else { - assert(fromType.width > toType.width); - res = this.ir._('trunc', [ fromType, from, 'to', toType ]); + assert(fromTy.width > toType.width); + res = body.cast('trunc', from, toType); } - body.push(res); return res; } load(fn, body, field) { - body.comment(`load state[${field}]`); - - const lookup = this.lookup(fn, field); - body.push(lookup.instr); - - const res = this.ir._('load', lookup.type, - [ lookup.type.ptr(), lookup.instr ]); - body.push(res); - - return res; + const lookup = this.stateField(fn, body, field); + return body.load(lookup); } store(fn, body, field, value) { - body.comment(`store state[${field}] = ...`); - - const lookup = this.lookup(fn, field); - body.push(lookup.instr); - - const res = this.ir._('store', - [ lookup.type, value ], - [ lookup.type.ptr(), lookup.instr ]); - res.void(); - body.push(res); - } - - lookup(fn, field) { - const index = this.state.lookup(field); - const instr = this.ir._('getelementptr inbounds', this.state, - [ this.state.ptr(), this.stateArg(fn) ], - [ INT, INT.v(0) ], - [ INT, INT.v(index) ]); - - return { - type: this.state.fields[index].type, - instr - }; + const lookup = this.stateField(fn, body, field); + body.store(value, lookup); } declareField(type, name, init) { - this.state.field(type, name); + this.state.addField(type, name); this.initializers.push({ type, name, init }); } initFields(fn, body) { this.initializers.forEach((entry) => { - const field = this.field(fn, entry.name); - body.push(field); - - body.push(this.ir._('store', [ entry.type, entry.init(entry.type, this) ], - [ entry.type.ptr(), field ]).void()); + const field = this.stateField(fn, body, entry.name); + body.store(entry.init(entry.type, this), field); }); } - field(fn, name) { + stateField(fn, body, name) { const stateArg = this.stateArg(fn); - return IR._('getelementptr inbounds', this.state, - [ stateArg.type, stateArg ], - [ INT, INT.v(0) ], - [ INT, INT.v(this.state.lookup(name)) ]); + return body.getelementptr(stateArg, INT.val(0), + INT.val(this.state.lookupField(name).index), true); } - stateArg(fn) { return fn.arg(ARG_STATE); } - posArg(fn) { return fn.arg(ARG_POS); } - endPosArg(fn) { return fn.arg(ARG_ENDPOS); } - matchArg(fn) { return fn.arg(ARG_MATCH); } + stateArg(fn) { return fn.getArgument(ARG_STATE); } + posArg(fn) { return fn.getArgument(ARG_POS); } + endPosArg(fn) { return fn.getArgument(ARG_ENDPOS); } + matchArg(fn) { return fn.getArgument(ARG_MATCH); } } module.exports = Compilation; diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 0358ded..33c0630 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -4,7 +4,6 @@ const llparse = require('../'); const constants = llparse.constants; const TYPE_INPUT = constants.TYPE_INPUT; -const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_MATCH = constants.TYPE_MATCH; const TYPE_ERROR = constants.TYPE_ERROR; @@ -55,7 +54,7 @@ class Compiler { const out = { header: '', - llvm: '' + bitcode: null }; const def = 'LLPARSE_HEADER_' + @@ -73,20 +72,19 @@ class Compiler { out.header += '\n'; out.header += `#endif /* ${def} */\n`; - out.llvm += ctx.ir.build(); + out.bitcode = ctx.end(); return out; } buildInit(ctx) { - const sig = ctx.ir.signature(ctx.ir.void(), [ - [ ctx.state.ptr(), ATTR_STATE ] - ]); - const init = ctx.ir.fn(sig, this.prefix + '_init', [ ARG_STATE ]); + const sig = ctx.ir.signature(ctx.ir.void(), [ ctx.state.ptr() ]); + const init = ctx.defineFunction(sig, this.prefix + '_init', [ ARG_STATE ]); + init.paramAttrs[0].add(ATTR_STATE); ctx.initFields(init, init.body); - init.body.terminate('ret', ctx.ir.void()); + init.body.ret(); return `void ${this.prefix}_init(${this.prefix}_state_t* s);`; } @@ -94,12 +92,15 @@ class Compiler { buildExecute(ctx) { // TODO(indutny): change signature to (state*, start*, len)? const sig = ctx.ir.signature(TYPE_ERROR, [ - [ ctx.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ] + ctx.state.ptr(), + TYPE_INPUT, + TYPE_INPUT ]); - const execute = ctx.ir.fn(sig, this.prefix + '_execute', + const execute = ctx.defineFunction(sig, this.prefix + '_execute', [ ARG_STATE, ARG_POS, ARG_ENDPOS ]); + execute.paramAttrs[0].add(ATTR_STATE); + execute.paramAttrs[1].add(ATTR_POS); + execute.paramAttrs[2].add(ATTR_ENDPOS); let body = execute.body; @@ -107,15 +108,10 @@ class Compiler { body = ctx.stageResults['span-builder'].preExecute(execute, body); body.name = 'execute'; - body.comment('execute'); - const nodeSig = ctx.signature.node; - const currentPtr = ctx.field(execute, '_current'); - body.push(currentPtr); - const current = ctx.ir._('load', nodeSig.ptr(), - [ nodeSig.ptr().ptr(), currentPtr ]); - body.push(current); + const currentPtr = ctx.stateField(execute, body, '_current'); + const current = body.load(currentPtr); const nodes = ctx.stageResults['node-builder'].map; let callees = new Set([ ctx.stageResults['node-builder'].entry ]); @@ -126,21 +122,18 @@ class Compiler { }); }); callees = Array.from(callees).map((fn) => { - return fn.type.ptr().type + ' @' + fn.name; + return ctx.ir.metadata(fn); }); - const call = ctx.call('', nodeSig, current, constants.CCONV, [ + const call = body.call(current, [ ctx.stateArg(execute), ctx.posArg(execute), ctx.endPosArg(execute), - TYPE_MATCH.v(0) - ]); - call.append([ '!callees', ctx.ir.metadata(callees.join(', ')) ]); - body.push(call); + TYPE_MATCH.val(0) + ], 'normal', constants.CCONV); + call.metadata.set('callees', ctx.ir.metadata(callees)); - const cmp = ctx.ir._('icmp', [ 'ne', nodeSig.ret, call ], - nodeSig.ret.v(null)); - body.push(cmp); + const cmp = body.icmp('ne', call, nodeSig.returnType.val(null)); const branch = ctx.branch(body, cmp, [ 'likely', 'unlikely' ]); const success = branch.left; @@ -149,25 +142,19 @@ class Compiler { // Success success.name = 'success'; - const bitcast = ctx.ir._('bitcast', - [ TYPE_OUTPUT, call, 'to', nodeSig.ptr() ]); - success.push(bitcast); - success.push(ctx.ir._('store', [ nodeSig.ptr(), bitcast ], - [ nodeSig.ptr().ptr(), currentPtr ]).void()); + const bitcast = success.cast('bitcast', call, nodeSig.ptr()); + success.store(bitcast, currentPtr); // Invoke spans and exit ctx.stageResults['span-builder'].postExecute(execute, success) - .terminate('ret', [ TYPE_ERROR, TYPE_ERROR.v(0) ]); + .ret(TYPE_ERROR.val(0)); // Error error.name = 'error'; - const errorPtr = ctx.field(execute, 'error'); - error.push(errorPtr); - const code = ctx.ir._('load', TYPE_ERROR, [ TYPE_ERROR.ptr(), errorPtr ]); - error.push(code); - - error.terminate('ret', [ TYPE_ERROR, code ]); + const errorPtr = ctx.stateField(execute, error, 'error'); + const code = error.load(errorPtr); + error.ret(code); return `int ${this.prefix}_execute(${this.prefix}_state_t* s, ` + 'const char* p, const char* endp);'; diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 8c1d7f7..86ac64d 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -45,10 +45,13 @@ class NodeContext { } // Just proxy - field(name) { - return this.compilation.field(this.fn, name); + stateField(body, name) { + return this.compilation.stateField(this.fn, body, name); } + cstring(...args) { return this.compilation.cstring(...args); } + blob(...args) { return this.compilation.blob(...args); } + addGlobalConst(...args) { return this.compilation.addGlobalConst(...args); } call(...args) { return this.compilation.call(...args); } buildSwitch(...args) { return this.compilation.buildSwitch(...args); } branch(...args) { return this.compilation.branch(...args); } @@ -96,8 +99,7 @@ class Node { // Errors are assumed to be rarely called // TODO(indutny): move this to node.Error somehow? if (this instanceof node.Error) { - fn.attributes = fn.attributes || ''; - fn.attributes += ' norecurse cold writeonly noinline'; + fn.attrs.add([ 'norecurse', 'cold', 'writeonly', 'noinline' ]); } nodes.set(this, fn); @@ -106,11 +108,7 @@ class Node { ctx.debug(body, 'enter'); body = this.prologue(ctx, body); - body.comment('next = pos + 1'); - ctx.pos.next = ctx.ir._('getelementptr inbounds', ctx.pos.current.type.to, - [ ctx.pos.current.type, ctx.pos.current ], - [ ctx.INT, ctx.INT.v(1) ]); - body.push(ctx.pos.next); + ctx.pos.next = body.getelementptr(ctx.pos.current, ctx.INT.val(1)); this.doBuild(ctx, body); @@ -125,10 +123,7 @@ class Node { const endPos = ctx.endPos; // Check that we have enough chars to do the read - body.comment('--- Prologue ---'); - body.comment('if (pos != endpos)'); - const cmp = ctx.ir._('icmp', [ 'ne', pos.type, pos ], endPos); - body.push(cmp); + const cmp = body.icmp('ne', pos, endPos); const branch = ctx.branch(body, cmp); @@ -142,10 +137,8 @@ class Node { pause(ctx, body) { const fn = ctx.fn; - const bitcast = ctx.ir._('bitcast', - [ fn.type.ptr(), fn, 'to', fn.type.ret ]); - body.push(bitcast); - body.terminate('ret', [ fn.type.ret, bitcast ]); + const bitcast = body.cast('bitcast', fn, fn.ty.toSignature().returnType); + body.ret(bitcast); // To be used in `compiler.js` this.hasPause = true; @@ -163,42 +156,41 @@ class Node { if (cached.phi) { assert(value, '`.match()` and `.select()` with the same target'); - cached.phi.append( - [ '[', ctx.TYPE_MATCH.v(value), ',', body.ref(), ']' ]); + cached.phi.addEdge({ + fromBlock: body, + value: ctx.TYPE_MATCH.val(value) + }); } else { assert(!value, '`.match()` and `.select()` with the same target'); } - body.terminate('br', cached.trampoline); + body.jmp(cached.trampoline); return target; } // Split, so that others could join us from code block above - const trampoline = body.jump('br'); - trampoline.name = body.name + '.trampoline'; + const trampoline = body.parent.createBlock(body.name + '.trampoline'); + body.jmp(trampoline); let phi = null; // Compute `match` if needed if (value !== null) { - trampoline.comment('Select `match`'); - phi = ctx.ir._('phi', - [ ctx.TYPE_MATCH, '[', ctx.TYPE_MATCH.v(value), ',', body.ref(), ']' ]); - trampoline.push(phi); + phi = trampoline.phi({ + fromBlock: body, + value: ctx.TYPE_MATCH.val(value) + }); } this.phis.set(target, { phi, trampoline }); - // TODO(indutny): looks like `musttail` gives worse performance when calling - // Invoke nodes (possibly others too). - const call = ctx.call('musttail', ctx.signature.node, target, [ + const call = trampoline.call(target, [ ctx.state, pos, ctx.endPos, - phi ? phi : 'undef' - ]); + phi ? phi : ctx.TYPE_MATCH.undef() + ], 'musttail'); - trampoline.push(call); - trampoline.terminate('ret', [ ctx.fn.type.ret, call ]); + trampoline.ret(call); return target; } diff --git a/lib/llparse/compiler/node/bit-check.js b/lib/llparse/compiler/node/bit-check.js index f27b864..8f7c23e 100644 --- a/lib/llparse/compiler/node/bit-check.js +++ b/lib/llparse/compiler/node/bit-check.js @@ -27,10 +27,15 @@ class BitCheck extends node.Node { const table = llparse.utils.buildLookupTable(WORD_WIDTH, CHAR_WIDTH, this.map.map(entry => entry.keys)); - const arrayTy = ctx.ir.i(1 << WORD_WIDTH).array(table.table.length); + const cellTy = ctx.ir.i(1 << WORD_WIDTH); + const arrayTy = ctx.ir.array(table.table.length, cellTy); + + const global = ctx.addGlobalConst('bit_check_table', + arrayTy.val(table.table.map((elem) => cellTy.val(elem)))); return { - global: ctx.ir.array(arrayTy, table.table), + global, + cellTy, indexShift: table.indexShift, shiftMask: table.shiftMask, @@ -40,13 +45,10 @@ class BitCheck extends node.Node { } doBuild(ctx, body) { - body.comment('node.BitCheck'); - const pos = ctx.pos.current; // Load the character - let current = ctx.ir._('load', pos.type.to, [ pos.type, pos ]); - body.push(current); + let current = body.load(pos); // Transform the character if needed if (this.transform) { @@ -59,43 +61,26 @@ class BitCheck extends node.Node { // Build global lookup table const table = this.buildTable(ctx); - const cellType = table.global.type.to.of; + const cellTy = table.cellTy; - body.comment('compute index'); - const index = ctx.ir._('lshr', [ pos.type.to, current ], - pos.type.to.v(table.indexShift)); - body.push(index); + const index = body.binop('lshr', current, + pos.ty.toPointer().to.val(table.indexShift)); - body.comment('compute shift'); - let shift = ctx.ir._('and', [ pos.type.to, current ], - pos.type.to.v(table.shiftMask)); - body.push(shift); - shift = ctx.ir._('zext', [ pos.type.to, shift, 'to', cellType ]); - body.push(shift); + let shift = body.binop('and', current, + pos.ty.toPointer().to.val(table.shiftMask)); + shift = body.cast('zext', shift, cellTy); // Compiler can handle it, but why generate more code? if (table.shiftMul !== 1) { - shift = ctx.ir._('mul', [ cellType, shift ], cellType.v(table.shiftMul)); - body.push(shift); + shift = body.binop('mul', shift, cellTy.val(table.shiftMul)); } - body.comment('l = table[index]'); - const ptr = ctx.ir._('getelementptr inbounds', table.global.type.to, - [ table.global.type, table.global ], - [ pos.type.to, pos.type.to.v(0) ], - [ pos.type.to, index ]); - body.push(ptr); - const load = ctx.ir._('load', cellType, [ cellType.ptr(), ptr ]); - body.push(load); - - body.comment('l >>= shift'); - const shr = ctx.ir._('lshr', [ cellType, load ], shift); - body.push(shr); + const ptr = body.getelementptr(table.global, ctx.INT.val(0), + index, true); + const load = body.load(ptr); - body.comment('l &= valueMask'); - const masked = ctx.ir._('and', [ cellType, shr ], - cellType.v(table.valueMask)); - body.push(masked); + const shr = body.binop('lshr', load, shift); + const masked = body.binop('and', shr, cellTy.val(table.valueMask)); const weights = new Array(this.map.length + 1).fill('likely'); @@ -108,7 +93,7 @@ class BitCheck extends node.Node { weights[0] = 'unlikely'; const keys = this.map.map((entry, i) => i + 1); - const s = ctx.buildSwitch(body, cellType, masked, keys, weights); + const s = ctx.buildSwitch(body, masked, keys, weights); s.cases.forEach((body, i) => { const child = this.map[i]; diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 8dd7c18..246b1c1 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -14,36 +14,19 @@ class Consume extends node.Node { doBuild(ctx, body) { const INVARIANT_GROUP = ctx.INVARIANT_GROUP; - body.comment('node.Consume'); - const pos = ctx.pos.current; - body.comment('index = state.index'); - const indexPtr = ctx.field('_index'); - body.push(indexPtr); - const index = ctx.ir._('load', ctx.TYPE_INDEX, - [ ctx.TYPE_INDEX.ptr(), indexPtr ], - [ '!invariant.group', INVARIANT_GROUP ]); - body.push(index); - - const need = ctx.ir._('zext', - [ ctx.TYPE_INDEX, index, 'to', ctx.TYPE_INTPTR ]); - body.push(need); + const indexPtr = ctx.stateField(body, '_index'); + const index = body.load(indexPtr); + index.metadata.set('invariant.group', INVARIANT_GROUP); - body.comment('avail = endp - p'); - const intPos = ctx.ir._('ptrtoint', - [ pos.type, pos, 'to', ctx.TYPE_INTPTR ]); - body.push(intPos); - const intEndPos = ctx.ir._('ptrtoint', - [ pos.type, ctx.endPos, 'to', ctx.TYPE_INTPTR ]); - body.push(intEndPos); + const need = body.cast('zext', index, ctx.TYPE_INTPTR); - const avail = ctx.ir._('sub', [ ctx.TYPE_INTPTR, intEndPos ], intPos); - body.push(avail); + const intPos = body.cast('ptrtoint', pos, ctx.TYPE_INTPTR); + const intEndPos = body.cast('ptrtoint', ctx.endPos, ctx.TYPE_INTPTR); - body.comment('if (avail >= need)'); - const cmp = ctx.ir._('icmp', [ 'uge', ctx.TYPE_INTPTR, avail ], need); - body.push(cmp); + const avail = body.binop('sub', intEndPos, intPos); + const cmp = body.icmp('uge', avail, need); const branch = ctx.branch(body, cmp, [ 'likely', 'unlikely' ]); const hasData = branch.left; @@ -52,30 +35,20 @@ class Consume extends node.Node { noData.name = 'no_data'; // Continue! - const next = ctx.ir._('getelementptr', pos.type.to, - [ pos.type, pos ], - [ ctx.TYPE_INDEX, index ]); - hasData.push(next); + const next = hasData.getelementptr(pos, index); assert(!this.skip); - hasData.comment('state.index = 0'); - hasData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, ctx.TYPE_INDEX.v(0) ], - [ ctx.TYPE_INDEX.ptr(), indexPtr ], - [ '!invariant.group', INVARIANT_GROUP ]).void()); + hasData.store(ctx.TYPE_INDEX.val(0), indexPtr) + .metadata.set('invariant.group', INVARIANT_GROUP); this.doOtherwise(ctx, hasData, { current: next, next: null }); // Pause! - noData.comment('state.index = need - avail'); - const left = ctx.ir._('sub', [ ctx.TYPE_INTPTR, need ], avail); - noData.push(left); + const left = noData.binop('sub', need, avail); - const leftTrunc = ctx.ir._('trunc', - [ ctx.TYPE_INTPTR, left, 'to', ctx.TYPE_INDEX ]); - noData.push(leftTrunc); + const leftTrunc = noData.cast('trunc', left, ctx.TYPE_INDEX); - noData.push(ctx.ir._('store', [ ctx.TYPE_INDEX, leftTrunc ], - [ ctx.TYPE_INDEX.ptr(), indexPtr ], - [ '!invariant.group', INVARIANT_GROUP ]).void()); + noData.store(leftTrunc, indexPtr) + .metadata.set('invariant.group', INVARIANT_GROUP); this.pause(ctx, noData); } } diff --git a/lib/llparse/compiler/node/empty.js b/lib/llparse/compiler/node/empty.js index 6d9625f..e4004fb 100644 --- a/lib/llparse/compiler/node/empty.js +++ b/lib/llparse/compiler/node/empty.js @@ -8,8 +8,6 @@ class Empty extends node.Node { } doBuild(ctx, body) { - body.comment('node.Empty'); - this.doOtherwise(ctx, body); } } diff --git a/lib/llparse/compiler/node/error.js b/lib/llparse/compiler/node/error.js index c3afe3e..7be0e2f 100644 --- a/lib/llparse/compiler/node/error.js +++ b/lib/llparse/compiler/node/error.js @@ -18,47 +18,32 @@ class Error extends node.Node { buildStoreError(ctx, body) { const INT = ctx.INT; - const reason = ctx.ir.cstr(this.reason); + const reason = ctx.cstring(this.reason); - const codePtr = ctx.field('error'); - body.push(codePtr); + const codePtr = ctx.stateField(body, 'error'); + const reasonPtr = ctx.stateField(body, 'reason'); + const posPtr = ctx.stateField(body, 'error_pos'); - const reasonPtr = ctx.field('reason'); - body.push(reasonPtr); + const cast = body.getelementptr(reason, INT.val(0), INT.val(0), true); - const posPtr = ctx.field('error_pos'); - body.push(posPtr); - - const cast = ctx.ir._('getelementptr inbounds', reason.type.to, - [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - body.push(cast); - - body.push(ctx.ir._('store', [ ctx.TYPE_ERROR, ctx.TYPE_ERROR.v(this.code) ], - [ ctx.TYPE_ERROR.ptr(), codePtr ]).void()); - body.push(ctx.ir._('store', [ ctx.TYPE_REASON, cast ], - [ ctx.TYPE_REASON.ptr(), reasonPtr ]).void()); - body.push(ctx.ir._('store', [ ctx.pos.current.type, ctx.pos.current ], - [ ctx.pos.current.type.ptr(), posPtr ]).void()); + body.store(ctx.TYPE_ERROR.val(this.code), codePtr); + body.store(cast, reasonPtr); + body.store(ctx.pos.current, posPtr); return body; } doBuild(ctx, body) { - body.comment('node.Error'); - body = this.buildStoreError(ctx, body); - body.comment('state.current = null'); - const currentPtr = ctx.field('_current'); - body.push(currentPtr); + const currentPtr = ctx.stateField(body, '_current'); // Non-recoverable state const nodeSig = ctx.compilation.signature.node; - body.push(ctx.ir._('store', [ nodeSig.ptr(), nodeSig.ptr().v(null) ], - [ nodeSig.ptr().ptr(), currentPtr ]).void()); + body.store(nodeSig.ptr().val(null), currentPtr); - const retType = ctx.fn.type.ret; - body.terminate('ret', [ retType, retType.v(null) ]); + const retType = ctx.fn.ty.toSignature().returnType; + body.ret(retType.val(null)); } } module.exports = Error; diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index a30ad91..db99c61 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -23,7 +23,6 @@ class Invoke extends node.Node { } doBuild(ctx, body) { - body.comment(`node.Invoke[${this.code.name}]`); const code = ctx.compilation.buildCode(this.code); const args = [ @@ -37,11 +36,10 @@ class Invoke extends node.Node { else assert.strictEqual(this.code[kSignature], 'match'); - const call = ctx.call('', code.type, code, args); - body.push(call); + const call = body.call(code, args); const keys = Object.keys(this.map).map(key => key | 0); - const s = ctx.buildSwitch(body, code.type.ret, call, keys); + const s = ctx.buildSwitch(body, call, keys); s.cases.forEach((body, i) => { this.tailTo(ctx, body, ctx.pos.current, this.map[keys[i]]); diff --git a/lib/llparse/compiler/node/pause.js b/lib/llparse/compiler/node/pause.js index 06d11ab..73ddb09 100644 --- a/lib/llparse/compiler/node/pause.js +++ b/lib/llparse/compiler/node/pause.js @@ -13,21 +13,15 @@ class Pause extends node.Error { } doBuild(ctx, body) { - body.comment('node.Pause'); - body = this.buildStoreError(ctx, body); - body.comment('state.current = next'); - const currentPtr = ctx.field('_current'); - body.push(currentPtr); + const currentPtr = ctx.stateField(body, '_current'); // Recoverable state const target = this.buildNode(ctx, this.otherwise); - body.push(ctx.ir._('store', [ target.type.ptr(), target.ref() ], - [ target.type.ptr().ptr(), currentPtr ]).void()); + body.store(target, currentPtr); - const retType = ctx.fn.type.ret; - body.terminate('ret', [ retType, retType.v(null) ]); + body.ret(ctx.fn.ty.toSignature().returnType.val(null)); } } module.exports = Pause; diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 5267bb4..7dda18a 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -5,8 +5,6 @@ const constants = llparse.constants; const node = require('./'); -const TYPE_STATUS = constants.TYPE_STATUS; - const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; @@ -31,44 +29,33 @@ class Sequence extends node.Node { doBuild(ctx, body) { const INT = ctx.INT; - body.comment(`node.Sequence["${this.select.toString('hex') }"]`); - - const seq = ctx.ir.data(this.select); + const seq = ctx.blob(this.select); - const cast = ctx.ir._('getelementptr inbounds', seq.type.to, - [ seq.type, seq ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - body.push(cast); + const cast = body.getelementptr(seq, INT.val(0), INT.val(0), true); const matchSequence = ctx.compilation.stageResults['match-sequence'] .get(this.transform); - const returnType = matchSequence.type.ret; + const returnType = matchSequence.ty.toSignature().returnType; - const call = ctx.call('', matchSequence.type, matchSequence, [ + const call = body.call(matchSequence, [ ctx.state, ctx.pos.current, ctx.endPos, cast, - INT.v(seq.type.to.length) + INT.val(seq.ty.toPointer().to.length) ]); - body.push(call); - - const status = ctx.ir._('extractvalue', [ returnType, call ], - INT.v(returnType.lookup('status'))); - body.push(status); - const current = ctx.ir._('extractvalue', [ returnType, call ], - INT.v(returnType.lookup('current'))); - body.push(current); + const status = body.extractvalue(call, + returnType.lookupField('status').index); + const current = body.extractvalue(call, + returnType.lookupField('current').index); // This is lame, but it is easier to do it this way // (Optimizer will remove it, if it isn't needed) - body.comment('next = pos + 1'); - const next = ctx.ir._('getelementptr', ctx.pos.current.type.to, - [ ctx.pos.current.type, current ], [ INT, INT.v(1) ]); - body.push(next); + const next = body.getelementptr(current, INT.val(1)); - const s = ctx.buildSwitch(body, TYPE_STATUS, status, [ + const s = ctx.buildSwitch(body, status, [ SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH @@ -81,7 +68,7 @@ class Sequence extends node.Node { ]); // No other values are allowed - s.otherwise.terminate('unreachable'); + s.otherwise.unreachable(); const complete = s.cases[0]; const pause = s.cases[1]; diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js index 31eb5e8..18bab54 100644 --- a/lib/llparse/compiler/node/set-index.js +++ b/lib/llparse/compiler/node/set-index.js @@ -16,10 +16,6 @@ class SetIndex extends node.Node { } doBuild(ctx, body) { - const INVARIANT_GROUP = ctx.INVARIANT_GROUP; - - body.comment(`node.SetIndex[${this.code.name}]`); - const code = ctx.compilation.buildCode(this.code); const args = [ @@ -29,14 +25,11 @@ class SetIndex extends node.Node { ]; assert.strictEqual(this.code[kSignature], 'match'); - const call = ctx.call('', code.type, code, args); - body.push(call); - - const ptr = ctx.field('_index'); - body.push(ptr); - body.push(ctx.ir._('store', [ ctx.TYPE_INDEX, call ], - [ ctx.TYPE_INDEX.ptr(), ptr ], - [ '!invariant.group', INVARIANT_GROUP ]).void()); + const call = body.call(code, args); + + const ptr = ctx.stateField(body, '_index'); + body.store(call, ptr) + .metadata.set('invariant.group', ctx.INVARIANT_GROUP); this.doOtherwise(ctx, body); } diff --git a/lib/llparse/compiler/node/single.js b/lib/llparse/compiler/node/single.js index 2d75401..85baac2 100644 --- a/lib/llparse/compiler/node/single.js +++ b/lib/llparse/compiler/node/single.js @@ -16,13 +16,10 @@ class Single extends node.Node { } doBuild(ctx, body) { - body.comment('node.Single'); - const pos = ctx.pos.current; // Load the character - let current = ctx.ir._('load', pos.type.to, [ pos.type, pos ]); - body.push(current); + let current = body.load(pos); // Transform the character if needed if (this.transform) { @@ -44,7 +41,7 @@ class Single extends node.Node { weights[0] = 'unlikely'; const keys = this.children.map(child => child.key); - const s = ctx.buildSwitch(body, pos.type.to, current, keys, weights); + const s = ctx.buildSwitch(body, current, keys, weights); s.cases.forEach((body, i) => { const child = this.children[i]; diff --git a/lib/llparse/compiler/node/span-end.js b/lib/llparse/compiler/node/span-end.js index 95ba5d1..26838ef 100644 --- a/lib/llparse/compiler/node/span-end.js +++ b/lib/llparse/compiler/node/span-end.js @@ -15,7 +15,6 @@ class SpanEnd extends node.Node { } doBuild(ctx, body) { - body.comment(`node.SpanEnd[${this.code.name}]`); const result = ctx.compilation.stageResults['span-builder'].spanEnd( ctx.fn, body, this.code); diff --git a/lib/llparse/compiler/node/span-start.js b/lib/llparse/compiler/node/span-start.js index 7662dab..e92d455 100644 --- a/lib/llparse/compiler/node/span-start.js +++ b/lib/llparse/compiler/node/span-start.js @@ -11,7 +11,6 @@ class SpanStart extends node.Node { } doBuild(ctx, body) { - body.comment(`node.SpanStart[${this.code.name}]`); body = ctx.compilation.stageResults['span-builder'].spanStart( ctx.fn, body, this.code); this.doOtherwise(ctx, body); diff --git a/lib/llparse/compiler/stage/match-sequence.js b/lib/llparse/compiler/stage/match-sequence.js index e85ae21..e50a176 100644 --- a/lib/llparse/compiler/stage/match-sequence.js +++ b/lib/llparse/compiler/stage/match-sequence.js @@ -6,7 +6,6 @@ const constants = llparse.constants; const CCONV = constants.CCONV; -const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_INDEX = constants.TYPE_INDEX; @@ -32,15 +31,15 @@ class MatchSequence extends Stage { super(ctx, 'match-sequence'); this.returnType = this.ctx.ir.struct('match_sequence_ret'); - - this.returnType.field(TYPE_INPUT, 'current'); - this.returnType.field(TYPE_STATUS, 'status'); + this.returnType.addField(TYPE_INPUT, 'current'); + this.returnType.addField(TYPE_STATUS, 'status'); + this.returnType.finalize(); this.signature = this.ctx.ir.signature(this.returnType, [ - [ this.ctx.state.ptr(), ATTR_STATE ], - [ TYPE_INPUT, ATTR_POS ], - [ TYPE_INPUT, ATTR_ENDPOS ], - [ TYPE_INPUT, ATTR_SEQUENCE ], + this.ctx.state.ptr(), + TYPE_INPUT, + TYPE_INPUT, + TYPE_INPUT, INT ]); @@ -60,15 +59,20 @@ class MatchSequence extends Stage { const postfix = cacheKey ? '_' + cacheKey.toLowerCase() : ''; - const fn = this.ctx.ir.fn(this.signature, + const fn = this.ctx.defineFunction(this.signature, `${this.ctx.prefix}__match_sequence${postfix}`, [ ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN ]); + fn.paramAttrs[0].add(ATTR_STATE); + fn.paramAttrs[1].add(ATTR_POS); + fn.paramAttrs[2].add(ATTR_ENDPOS); + fn.paramAttrs[3].add(ATTR_SEQUENCE); + this.buildBody(fn, transform); - fn.visibility = 'internal'; + fn.linkage = 'internal'; fn.cconv = CCONV; - fn.attributes = 'nounwind norecurse alwaysinline'; + fn.attrs.add([ 'nounwind', 'norecurse', 'alwaysinline' ]); this.cache.set(cacheKey, fn); @@ -78,34 +82,29 @@ class MatchSequence extends Stage { // TODO(indutny): initialize `state.index` before calling matcher? buildBody(fn, transform) { const body = fn.body; - const ir = this.ctx.ir; const maxSeqLen = this.ctx.stageResults['node-translator'].maxSequenceLen; - const range = `${TYPE_INDEX.type} 0, ${TYPE_INDEX.type} ${maxSeqLen}`; + const range = this.ctx.ir.metadata([ + this.ctx.ir.metadata(TYPE_INDEX.val(0)), + this.ctx.ir.metadata(TYPE_INDEX.val(maxSeqLen)) + ]); // Just to have a label - const start = body.jump('br'); - start.name = 'start'; + const start = body.parent.createBlock('start'); + body.jmp(start); // Load `state.index` - start.comment('index = state.index'); - const indexPtr = this.ctx.field(fn, '_index'); - start.push(indexPtr); - const index = ir._('load', TYPE_INDEX, [ TYPE_INDEX.ptr(), indexPtr ], - [ '!invariant.group', this.ctx.INVARIANT_GROUP ], - [ '!range', ir.metadata(range) ]); - start.push(index); - - const loop = start.jump('br'); - loop.name = 'loop'; + const indexPtr = this.ctx.stateField(fn, start, '_index'); + const index = start.load(indexPtr); + index.metadata.set('invariant.group', this.ctx.INVARIANT_GROUP); + index.metadata.set('range', range); // Loop start - const posPhi = ir._('phi', - [ TYPE_INPUT, '[', fn.arg(ARG_POS), ',', start.ref(), ']' ]); - loop.push(posPhi); - const indexPhi = ir._('phi', - [ TYPE_INDEX, '[', index, ',', start.ref(), ']' ]); - loop.push(indexPhi); + const loop = body.parent.createBlock('loop'); + start.jmp(loop); + + const posPhi = loop.phi({ fromBlock: start, value: this.ctx.posArg(fn) }); + const indexPhi = loop.phi({ fromBlock: start, value: index }); const iteration = this.buildIteration(fn, loop, indexPtr, posPhi, indexPhi, transform); @@ -114,9 +113,9 @@ class MatchSequence extends Stage { this.ret(iteration.complete, iteration.pos, SEQUENCE_COMPLETE); // We have more data! - iteration.loop.loop('br', loop); - indexPhi.append([ '[', iteration.index, ',', iteration.loop.ref(), ']' ]); - posPhi.append([ '[', iteration.pos.next, ',', iteration.loop.ref(), ']' ]); + iteration.loop.jmp(loop); + indexPhi.addEdge({ fromBlock: iteration.loop, value: iteration.index }); + posPhi.addEdge({ fromBlock: iteration.loop, value: iteration.pos.next }); // Have to pause - return self this.ret(iteration.pause, iteration.pos, SEQUENCE_PAUSE); @@ -125,16 +124,11 @@ class MatchSequence extends Stage { this.ret(iteration.mismatch, iteration.pos, SEQUENCE_MISMATCH); } - buildIteration(fn, body, indexField, pos, index, transform) { - const ir = this.ctx.ir; + const seq = fn.getArgument(ARG_SEQUENCE); + const seqLen = fn.getArgument(ARG_SEQUENCE_LEN); - const seq = fn.arg(ARG_SEQUENCE); - const seqLen = fn.arg(ARG_SEQUENCE_LEN); - - body.comment('current = *pos'); - let current = ir._('load', TYPE_INPUT.to, [ TYPE_INPUT, pos ]); - body.push(current); + let current = body.load(pos); // Transform the character if needed if (transform) { @@ -144,65 +138,39 @@ class MatchSequence extends Stage { current = res.current; } - body.comment('expected = seq[state.index]'); - const expectedPtr = ir._('getelementptr', seq.type.to, - [ seq.type, seq ], - [ INT, index ]); - body.push(expectedPtr); - const expected = ir._('load', TYPE_INPUT.to, [ TYPE_INPUT, expectedPtr ]); - body.push(expected); + const expectedPtr = body.getelementptr(seq, index); + const expected = body.load(expectedPtr); // NOTE: fetch this early so it would dominate all returns - body.comment('next = pos + 1'); - const next = ir._('getelementptr', TYPE_INPUT.to, - [ TYPE_INPUT, pos ], - [ INT, INT.v(1) ]); - body.push(next); - - body.comment('if (current == expected)'); - let cmp = ir._('icmp', [ 'eq', TYPE_INPUT.to, current ], expected); - body.push(cmp); - const { left: isMatch, right: isMismatch } = - body.branch('br', [ BOOL, cmp ]); + const next = body.getelementptr(pos, INT.val(1)); + + let cmp = body.icmp('eq', current, expected); + const isMatch = fn.createBlock('match'); + const isMismatch = fn.createBlock('mismatch'); + body.branch(cmp, isMatch, isMismatch); // Mismatch - isMismatch.name = 'mismatch'; - isMismatch.comment('Sequence string does not match input'); - isMismatch.comment('state.index = 0'); - isMismatch.push(this.reset(indexField)); + this.reset(isMismatch, indexField); // Character matches - isMatch.name = 'match'; - isMatch.comment('Got a char match'); - - isMatch.comment('index1 = index + 1'); - const index1 = ir._('add', [ TYPE_INDEX, index ], TYPE_INDEX.v(1)); - isMatch.push(index1); - - isMatch.comment('if (index1 == seq.length)'); - cmp = ir._('icmp', [ 'eq', TYPE_INDEX, index1 ], seqLen); - isMatch.push(cmp); - const { left: isComplete, right: isIncomplete } = - isMatch.branch('br', [ BOOL, cmp ]); - - isComplete.name = 'is_complete'; - isComplete.comment('state.index = 0'); - isComplete.push(this.reset(indexField)); - - isIncomplete.name = 'is_incomplete'; - isIncomplete.comment('if (next != endpos)'); - cmp = ir._('icmp', [ 'ne', TYPE_INPUT, next ], fn.arg(ARG_ENDPOS)); - isIncomplete.push(cmp); + const index1 = isMatch.binop('add', index, TYPE_INDEX.val(1)); + cmp = isMatch.icmp('eq', index1, seqLen); + + const isComplete = fn.createBlock('is_complete'); + const isIncomplete = fn.createBlock('is_incomplete'); + isMatch.branch(cmp, isComplete, isIncomplete); + + this.reset(isComplete, indexField); + + cmp = isIncomplete.icmp('ne', next, this.ctx.endPosArg(fn)); const { left: moreData, right: noMoreData } = this.ctx.branch(isIncomplete, cmp, [ 'likely', 'unlikely' ]); moreData.name = 'more_data'; noMoreData.name = 'no_more_data'; - noMoreData.comment('state.index = index1'); - noMoreData.push(ir._('store', [ TYPE_INDEX, index1 ], - [ TYPE_INDEX.ptr(), indexField ], - [ '!invariant.group', this.ctx.INVARIANT_GROUP ]).void()); + const store = noMoreData.store(index1, indexField); + store.metadata.set('invariant.group', this.ctx.INVARIANT_GROUP); return { index: index1, @@ -215,25 +183,19 @@ class MatchSequence extends Stage { } ret(body, pos, status) { - const ir = this.ctx.ir; - - const create = ir._('insertvalue', [ this.returnType, 'undef' ], - [ TYPE_INPUT, pos.current ], - INT.v(this.returnType.lookup('current'))); - body.push(create); + const create = body.insertvalue(this.returnType.undef(), pos.current, + this.returnType.lookupField('current').index); - const amend = ir._('insertvalue', [ this.returnType, create ], - [ TYPE_STATUS, TYPE_STATUS.v(status) ], - INT.v(this.returnType.lookup('status'))); - body.push(amend); + const amend = body.insertvalue(create, TYPE_STATUS.val(status), + this.returnType.lookupField('status').index); - body.terminate('ret', [ this.returnType, amend ]); + body.ret(amend); } - reset(field) { - return this.ctx.ir._('store', [ TYPE_INDEX, TYPE_INDEX.v(0) ], - [ TYPE_INDEX.ptr(), field ], - [ '!invariant.group', this.ctx.INVARIANT_GROUP ]).void(); + reset(body, field) { + const store = body.store(TYPE_INDEX.val(0), field); + store.metadata.set('invariant.group', this.ctx.INVARIANT_GROUP); + return store; } } module.exports = MatchSequence; diff --git a/lib/llparse/compiler/stage/span-builder.js b/lib/llparse/compiler/stage/span-builder.js index 26e6b82..eed306b 100644 --- a/lib/llparse/compiler/stage/span-builder.js +++ b/lib/llparse/compiler/stage/span-builder.js @@ -7,12 +7,10 @@ const llparse = require('../../'); const constants = llparse.constants; -const BOOL = constants.BOOL; const INT = constants.INT; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_OUTPUT = constants.TYPE_OUTPUT; const TYPE_ERROR = constants.TYPE_ERROR; -const TYPE_REASON = constants.TYPE_REASON; const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; @@ -42,13 +40,13 @@ class Builder extends Stage { colors.concurrency.forEach((list, index) => { const num = list.length; this.ctx.declareField(TYPE_INPUT, SPAN_START_PREFIX + index, - type => type.v(null)); + type => type.val(null)); if (num === 1) return; this.ctx.declareField(callbackType, SPAN_CB_PREFIX + index, - type => type.v(null)); + type => type.val(null)); }); } @@ -58,26 +56,14 @@ class Builder extends Stage { const colors = this.ctx.stageResults['span-allocator']; const index = colors.map.get(code); - const startPtr = this.startField(fn, index); - body.push(startPtr); + const startPtr = this.startField(fn, body, index); - const store = this.ctx.ir._('store', [ TYPE_INPUT, this.ctx.posArg(fn) ], - [ TYPE_INPUT.ptr(), startPtr ]); - store.void(); - body.push(store); + body.store(this.ctx.posArg(fn), startPtr); const num = colors.concurrency[index].length; if (num !== 1) { - const callbackType = this.ctx.signature.callback.span.ptr(); - - const cbPtr = this.callbackField(fn, index); - body.push(cbPtr); - - const store = this.ctx.ir._('store', - [ callbackType, this.ctx.buildCode(code) ], - [ callbackType.ptr(), cbPtr ]); - store.void(); - body.push(store); + const cbPtr = this.callbackField(fn, body, index); + body.store(this.ctx.buildCode(code), cbPtr); } return body; @@ -87,36 +73,27 @@ class Builder extends Stage { const colors = this.ctx.stageResults['span-allocator']; const index = colors.map.get(code); - const startPtr = this.startField(fn, index); - body.push(startPtr); + const startPtr = this.startField(fn, body, index); // Load - const start = this.ctx.ir._('load', TYPE_INPUT, - [ TYPE_INPUT.ptr(), startPtr ]); - body.push(start); + const start = body.load(startPtr); // ...and reset - const store = this.ctx.ir._('store', [ TYPE_INPUT, TYPE_INPUT.v(null) ], - [ TYPE_INPUT.ptr(), startPtr ]); - store.void(); - body.push(store); + body.store(TYPE_INPUT.val(null), startPtr); const signature = this.ctx.signature.callback.span; const pos = this.ctx.posArg(fn); // TODO(indutny): this won't work with internal callbacks due to // cconv mismatch - const call = this.ctx.call('', signature, this.ctx.buildCode(code), [ + const call = body.call(this.ctx.buildCode(code), [ this.ctx.stateArg(fn), start, pos ]); - body.push(call); // Check return value - const errorCmp = this.ctx.ir._('icmp', - [ 'eq', signature.ret, call ], [ signature.ret.v(0) ]); - body.push(errorCmp); + const errorCmp = body.icmp('eq', call, signature.returnType.val(0)); const errorBranch = this.ctx.branch(body, errorCmp, [ 'likely', 'unlikely' ]); @@ -126,52 +103,31 @@ class Builder extends Stage { error.name = 'error_' + index; // Handle error - assert.strictEqual(TYPE_ERROR.type, signature.ret.type); + assert(TYPE_ERROR.isEqual(signature.returnType)); this.buildError(fn, error, pos, call); return { body: noError, updateResumptionTarget: (target) => { - error.comment('store resumption target'); - - const currentPtr = this.ctx.field(fn, '_current'); - error.push(currentPtr); + const currentPtr = this.ctx.stateField(fn, error, '_current'); - error.push(this.ctx.ir._('store', - [ target.type.ptr(), target.ref() ], - [ target.type.ptr().ptr(), currentPtr ]).void()); - - error.terminate('ret', [ TYPE_OUTPUT, TYPE_OUTPUT.v(null) ]); + error.store(target, currentPtr); + error.ret(TYPE_OUTPUT.val(null)); } }; } buildError(fn, body, pos, code) { - body.comment('span error'); - const reason = this.ctx.ir.cstr('Span callback error'); - - const cast = this.ctx.ir._('getelementptr inbounds', reason.type.to, - [ reason.type, reason ], [ INT, INT.v(0) ], [ INT, INT.v(0) ]); - body.push(cast); - - const codePtr = this.ctx.field(fn, 'error'); - body.push(codePtr); - - const reasonPtr = this.ctx.field(fn, 'reason'); - body.push(reasonPtr); - - const errorPosPtr = this.ctx.field(fn, 'error_pos'); - body.push(errorPosPtr); - - body.push(this.ctx.ir._('store', - [ TYPE_ERROR, code ], - [ TYPE_ERROR.ptr(), codePtr ]).void()); - body.push(this.ctx.ir._('store', - [ TYPE_REASON, cast ], - [ TYPE_REASON.ptr(), reasonPtr ]).void()); - body.push(this.ctx.ir._('store', - [ pos.type, pos ], - [ pos.type.ptr(), errorPosPtr ]).void()); + const reason = this.ctx.cstring('Span callback error'); + + const cast = body.getelementptr(reason, INT.val(0), INT.val(0), true); + const codePtr = this.ctx.stateField(fn, body, 'error'); + const reasonPtr = this.ctx.stateField(fn, body, 'reason'); + const errorPosPtr = this.ctx.stateField(fn, body, 'error_pos'); + + body.store(code, codePtr); + body.store(cast, reasonPtr); + body.store(pos, errorPosPtr); } // ${prefix}_execute @@ -179,31 +135,21 @@ class Builder extends Stage { buildPrologue(fn, body) { const colors = this.ctx.stageResults['span-allocator']; - body.comment('restart spans'); - colors.concurrency.forEach((_, index) => { - const startPtr = this.startField(fn, index); - body.push(startPtr); - const start = this.ctx.ir._('load', TYPE_INPUT, - [ TYPE_INPUT.ptr(), startPtr ]); - body.push(start); + const startPtr = this.startField(fn, body, index); + const start = body.load(startPtr); - const cmp = this.ctx.ir._('icmp', [ 'eq', TYPE_INPUT, start ], - [ TYPE_INPUT.v(null) ]); - body.push(cmp); + const cmp = body.icmp('eq', start, TYPE_INPUT.val(null)); - const branch = body.branch('br', [ BOOL, cmp ]); + const branch = this.ctx.branch(body, cmp); const empty = branch.left; const restart = branch.right; empty.name = 'empty_' + index; restart.name = 'restart_' + index; - const store = this.ctx.ir._('store', [ TYPE_INPUT, this.ctx.posArg(fn) ], - [ TYPE_INPUT.ptr(), startPtr ]); - store.void(); - restart.push(store); + restart.store(this.ctx.posArg(fn), startPtr); - restart.terminate('br', empty); + restart.jmp(empty); body = empty; }); @@ -215,23 +161,16 @@ class Builder extends Stage { const callbackType = this.ctx.signature.callback.span.ptr(); - body.comment('execute spans'); - colors.concurrency.forEach((list, index) => { const codes = list.map(code => this.ctx.buildCode(code)); const num = codes.length; - const startPtr = this.startField(fn, index); - body.push(startPtr); - const start = this.ctx.ir._('load', TYPE_INPUT, - [ TYPE_INPUT.ptr(), startPtr ]); - body.push(start); + const startPtr = this.startField(fn, body, index); + const start = body.load(startPtr); - const cmp = this.ctx.ir._('icmp', [ 'ne', TYPE_INPUT, start ], - TYPE_INPUT.v(null)); - body.push(cmp); + const cmp = body.icmp('ne', start, TYPE_INPUT.val(null)); - const branch = body.branch('br', [ BOOL, cmp ]); + const branch = this.ctx.branch(body, cmp); const present = branch.left; const empty = branch.right; present.name = 'present_' + index; @@ -241,34 +180,29 @@ class Builder extends Stage { if (num === 1) { cb = codes[0]; } else { - const cbPtr = this.callbackField(fn, index); - present.push(cbPtr); - - cb = this.ctx.ir._('load', callbackType, [ callbackType.ptr(), cbPtr ]); - present.push(cb); + const cbPtr = this.callbackField(fn, present, index); + cb = present.load(cbPtr); } const endPos = this.ctx.endPosArg(fn); // TODO(indutny): this won't work with internal callbacks due to // cconv mismatch - const call = this.ctx.call('', callbackType.to, cb, [ + const call = present.call(cb, [ this.ctx.stateArg(fn), start, endPos - ]); + ], 'normal', 'ccc'); if (num > 1) { const callees = codes.map((code) => { - return code.type.ptr().type + ' @' + code.name; - }).join(', '); - call.append([ '!callees', this.ctx.ir.metadata(callees) ]); + return this.ctx.ir.metadata(code); + }); + call.metadata.set('callees', this.ctx.ir.metadata(callees)); } - present.push(call); // Check return value - const errorCmp = this.ctx.ir._('icmp', - [ 'eq', callbackType.to.ret, call ], [ callbackType.to.ret.v(0) ]); - present.push(errorCmp); + const errorCmp = present.icmp('eq', call, + callbackType.to.toSignature().returnType.val(0)); const errorBranch = this.ctx.branch(present, errorCmp, [ 'likely', 'unlikely' ]); @@ -278,23 +212,24 @@ class Builder extends Stage { error.name = 'error_' + index; // Make sure that the types match - assert.strictEqual(fn.type.ret.type, callbackType.to.ret.type); + assert(fn.ty.toSignature().returnType.isEqual( + callbackType.toPointer().to.toSignature().returnType)); this.buildError(fn, error, endPos, call); - error.terminate('ret', [ fn.type.ret, call ]); + error.ret(call); - noError.terminate('br', empty); + noError.jmp(empty); body = empty; }); return body; } - startField(fn, index) { - return this.ctx.field(fn, SPAN_START_PREFIX + index); + startField(fn, body, index) { + return this.ctx.stateField(fn, body, SPAN_START_PREFIX + index); } - callbackField(fn, index) { - return this.ctx.field(fn, SPAN_CB_PREFIX + index); + callbackField(fn, body, index) { + return this.ctx.stateField(fn, body, SPAN_CB_PREFIX + index); } } module.exports = Builder; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index 6d97ecf..dad323d 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -1,6 +1,6 @@ 'use strict'; -const IR = require('llvm-ir'); +const IR = require('bitcode').Builder; exports.CCONV = 'fastcc'; @@ -27,9 +27,9 @@ exports.ARG_SEQUENCE_LEN = 'slen'; exports.ARG_MATCH = 'match'; exports.ARG_UNUSED = '_unused'; -exports.ATTR_STATE = 'noalias nonnull'; -exports.ATTR_POS = 'noalias nonnull readonly'; -exports.ATTR_ENDPOS = 'noalias nonnull readnone'; +exports.ATTR_STATE = [ 'noalias', 'nonnull' ]; +exports.ATTR_POS = [ 'noalias', 'nonnull', 'readonly' ]; +exports.ATTR_ENDPOS = [ 'noalias', 'nonnull', 'readnone' ]; exports.ATTR_SEQUENCE = exports.ATTR_POS; exports.SEQUENCE_COMPLETE = 0; diff --git a/package-lock.json b/package-lock.json index 1fb8d92..9075294 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "acorn": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.4.1.tgz", - "integrity": "sha512-XLmq3H/BVvW6/GbxKryGxWORz1ebilSsUDlyC27bXhWGWAZWkGwS6FLHjOlwFXNFoWFQEO/Df4u0YYd0K3BQgQ==", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.5.3.tgz", + "integrity": "sha512-jd5MkIUlbbmb07nXH0DT3y7rDVtkzDi4XZOUVWAer8ajmF/DTSSbl5oNFyDOl/OXA33Bl79+ypHhl2pN20VeOQ==", "dev": true }, "acorn-jsx": { @@ -34,7 +34,7 @@ "dev": true, "requires": { "co": "4.6.0", - "fast-deep-equal": "1.0.0", + "fast-deep-equal": "1.1.0", "fast-json-stable-stringify": "2.0.0", "json-schema-traverse": "0.3.1" } @@ -143,6 +143,19 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "bitcode": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.0.1.tgz", + "integrity": "sha512-qZ4rI/ahSd3uDEORRsoMdMcLE5KINbW/rHFTS3sPG+cSC/FoVc8T3Srpf2jH+VD0G+btWHjfoFd3o0HFFsqb/g==", + "requires": { + "bitcode-builder": "1.0.0" + } + }, + "bitcode-builder": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/bitcode-builder/-/bitcode-builder-1.0.0.tgz", + "integrity": "sha512-rufP/gMS+2q9dLAGQXJuXzT7poxDI3gm9m9iwCLPcsz32/r9AP5yaGmHJkxAFAmwim7ycDpI8UGXQ9UbsgXZYw==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -154,9 +167,9 @@ } }, "browser-stdout": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", - "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, "caller-path": { @@ -175,29 +188,29 @@ "dev": true }, "chalk": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.1.tgz", - "integrity": "sha512-QUU4ofkDoMIVO7hcx1iPTISs88wsO8jA92RQIm4JAwZvFGGAV2hSAA1NX7oVj2Ej2Q6NDTcRDjPTFrMCRZoJ6g==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz", + "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==", "dev": true, "requires": { - "ansi-styles": "3.2.0", + "ansi-styles": "3.2.1", "escape-string-regexp": "1.0.5", - "supports-color": "5.2.0" + "supports-color": "5.3.0" }, "dependencies": { "ansi-styles": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.0.tgz", - "integrity": "sha512-NnSOmMEYtVR2JVMIGTzynRkkaxtiq1xnFBcdQD/DnNCYPoEPsVJhM98BDyaoNOQIi7p4okdi3E27eN7GQbsUug==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "dev": true, "requires": { "color-convert": "1.9.1" } }, "supports-color": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.2.0.tgz", - "integrity": "sha512-F39vS48la4YvTZUPVeTqsjsFNrvcMwrV3RLZINsmHo+7djCvuUzSIeXOnZ5hmjef4bajL1dNccN+tg5XAliO5Q==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz", + "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==", "dev": true, "requires": { "has-flag": "3.0.0" @@ -266,13 +279,13 @@ "dev": true }, "concat-stream": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz", - "integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.1.tgz", + "integrity": "sha512-gslSSJx03QKa59cIKqeJO9HQ/WZMotvYJCuaUULrLpjj8oG40kV2Z+gz82pVxlTkOADi4PJxQPPfhl1ELYrrXw==", "dev": true, "requires": { "inherits": "2.0.3", - "readable-stream": "2.3.4", + "readable-stream": "2.3.5", "typedarray": "0.0.6" } }, @@ -288,7 +301,7 @@ "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", "dev": true, "requires": { - "lru-cache": "4.1.1", + "lru-cache": "4.1.2", "shebang-command": "1.2.0", "which": "1.3.0" } @@ -323,9 +336,9 @@ } }, "diff": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", - "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, "doctrine": { @@ -344,21 +357,21 @@ "dev": true }, "eslint": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.18.1.tgz", - "integrity": "sha512-gPSfpSRCHre1GLxGmO68tZNxOlL2y7xBd95VcLD+Eo4S2js31YoMum3CAQIOaxY24hqYOMksMvW38xuuWKQTgw==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.18.2.tgz", + "integrity": "sha512-qy4i3wODqKMYfz9LUI8N2qYDkHkoieTbiHpMrYUI/WbjhXJQr7lI4VngixTgaG+yHX+NBCv7nW4hA0ShbvaNKw==", "dev": true, "requires": { "ajv": "5.5.2", "babel-code-frame": "6.26.0", - "chalk": "2.3.1", - "concat-stream": "1.6.0", + "chalk": "2.3.2", + "concat-stream": "1.6.1", "cross-spawn": "5.1.0", "debug": "3.1.0", "doctrine": "2.1.0", "eslint-scope": "3.7.1", "eslint-visitor-keys": "1.0.0", - "espree": "3.5.3", + "espree": "3.5.4", "esquery": "1.0.0", "esutils": "2.0.2", "file-entry-cache": "2.0.0", @@ -369,7 +382,7 @@ "imurmurhash": "0.1.4", "inquirer": "3.3.0", "is-resolvable": "1.1.0", - "js-yaml": "3.10.0", + "js-yaml": "3.11.0", "json-stable-stringify-without-jsonify": "1.0.1", "levn": "0.3.0", "lodash": "4.17.5", @@ -394,7 +407,7 @@ "integrity": "sha1-PWPD7f2gLgbgGkUq2IyqzHzctug=", "dev": true, "requires": { - "esrecurse": "4.2.0", + "esrecurse": "4.2.1", "estraverse": "4.2.0" } }, @@ -405,12 +418,12 @@ "dev": true }, "espree": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.3.tgz", - "integrity": "sha512-Zy3tAJDORxQZLl2baguiRU1syPERAIg0L+JB2MWorORgTu/CplzvxS9WWA7Xh4+Q+eOQihNs/1o1Xep8cvCxWQ==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz", + "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==", "dev": true, "requires": { - "acorn": "5.4.1", + "acorn": "5.5.3", "acorn-jsx": "3.0.1" } }, @@ -430,13 +443,12 @@ } }, "esrecurse": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.0.tgz", - "integrity": "sha1-+pVo2Y04I/mkHZHpAtyrnqblsWM=", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", "dev": true, "requires": { - "estraverse": "4.2.0", - "object-assign": "4.1.1" + "estraverse": "4.2.0" } }, "estraverse": { @@ -463,9 +475,9 @@ } }, "fast-deep-equal": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz", - "integrity": "sha1-liVqO8l1WV6zbYLpkp0GDYk0Of8=", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=", "dev": true }, "fast-json-stable-stringify": { @@ -631,7 +643,7 @@ "dev": true, "requires": { "ansi-escapes": "3.0.0", - "chalk": "2.3.1", + "chalk": "2.3.2", "cli-cursor": "2.1.0", "cli-width": "2.2.0", "external-editor": "2.1.0", @@ -707,9 +719,9 @@ "dev": true }, "js-yaml": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz", - "integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz", + "integrity": "sha512-saJstZWv7oNeOyBh3+Dx1qWzhW0+e6/8eDzo7p5rDFqxntSztloLtuKu+Ejhtq82jsilwOIZYsCz+lIjthg1Hw==", "dev": true, "requires": { "argparse": "1.0.10", @@ -739,19 +751,12 @@ } }, "llparse-test-fixture": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/llparse-test-fixture/-/llparse-test-fixture-1.2.1.tgz", - "integrity": "sha512-zIe5Eyz2sOmA3Lg0HFOy4Z3qOevLup4l24nYpgtYS1wixIr8dlZaQ7teBlIVMM90nafR6dIH8xB1/UF7XDX00w==", + "version": "git://github.com/indutny/llparse-test-fixture.git#d81b001d6cdb5ef3ef748108f20c796645e941b2", "dev": true, "requires": { "async": "2.6.0" } }, - "llvm-ir": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/llvm-ir/-/llvm-ir-1.8.0.tgz", - "integrity": "sha512-34IWteY5iEmoynLHJgwAOBGAHFfQPkSMW6ifjXAPv1F1hfVBJJgof9jP9Os8LxMcz8cYqEU4CzKFlL2yO1FBrg==" - }, "lodash": { "version": "4.17.5", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", @@ -759,9 +764,9 @@ "dev": true }, "lru-cache": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.1.tgz", - "integrity": "sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.2.tgz", + "integrity": "sha512-wgeVXhrDwAWnIF/yZARsFnMBtdFXOg1b8RIrhilp+0iDYN4mdQcNZElDZ0e4B64BhaxeQ5zN7PMyvu7we1kPeQ==", "dev": true, "requires": { "pseudomap": "1.0.2", @@ -799,15 +804,15 @@ } }, "mocha": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.0.1.tgz", - "integrity": "sha512-SpwyojlnE/WRBNGtvJSNfllfm5PqEDFxcWluSIgLeSBJtXG4DmoX2NNAeEA7rP5kK+79VgtVq8nG6HskaL1ykg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.0.4.tgz", + "integrity": "sha512-nMOpAPFosU1B4Ix1jdhx5e3q7XO55ic5a8cgYvW27CequcEY+BabS0kUVL1Cw1V5PuVHZWeNRWFLmEPexo79VA==", "dev": true, "requires": { - "browser-stdout": "1.3.0", + "browser-stdout": "1.3.1", "commander": "2.11.0", "debug": "3.1.0", - "diff": "3.3.1", + "diff": "3.5.0", "escape-string-regexp": "1.0.5", "glob": "7.1.2", "growl": "1.10.3", @@ -958,9 +963,9 @@ "dev": true }, "readable-stream": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.4.tgz", - "integrity": "sha512-vuYxeWYM+fde14+rajzqgeohAI7YoJcHE7kXDAc4Nk0EbuKnJfqtY9YtRkLo/tqkuF7MsBQRhPnPeyjYITp3ZQ==", + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.5.tgz", + "integrity": "sha512-tK0yDhrkygt/knjowCUiWP9YdV7c5R+8cR0r/kt9ZhBU906Fs6RpQJCEilamRJj1Nx2rWI6LkW9gKqjTkshhEw==", "dev": true, "requires": { "core-util-is": "1.0.2", @@ -1135,7 +1140,7 @@ "requires": { "ajv": "5.5.2", "ajv-keywords": "2.1.1", - "chalk": "2.3.1", + "chalk": "2.3.2", "lodash": "4.17.5", "slice-ansi": "1.0.0", "string-width": "2.1.1" diff --git a/package.json b/package.json index 0a9d62c..6e32e3a 100644 --- a/package.json +++ b/package.json @@ -18,14 +18,14 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "debug": "^3.1.0", - "llvm-ir": "^1.8.0" + "bitcode": "^1.0.1", + "debug": "^3.1.0" }, "devDependencies": { "async": "^2.6.0", - "eslint": "^4.18.1", - "llparse-test-fixture": "^1.2.1", - "mocha": "^5.0.1" + "eslint": "^4.18.2", + "llparse-test-fixture": "git://github.com/indutny/llparse-test-fixture.git", + "mocha": "^5.0.4" }, "directories": { "lib": "lib", diff --git a/test/code-test.js b/test/code-test.js index 74211bc..b9cb127 100644 --- a/test/code-test.js +++ b/test/code-test.js @@ -118,7 +118,7 @@ describe('LLParse/Code', function() { .select(fixtures.NUM_SELECT, p.invoke(p.code.store('counter'), check)) .otherwise(p.error(1, 'Unexpected')); - const binary = fixtures.build(p, start, 'update'); + const binary = fixtures.build(p, start, 'is-equal'); binary('010', 'off=1\noff=3\n', callback); }); From 47430d53314708d8a1749eb4cc59bb093515be78 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Mar 2018 16:54:08 -0400 Subject: [PATCH 256/281] readme: update --- README.md | 15 ++++++++------- examples/http/Makefile | 8 +++----- examples/http/index.js | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d5f360d..53226e3 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ [![Build Status](https://secure.travis-ci.org/indutny/llparse.svg)](http://travis-ci.org/indutny/llparse) [![NPM version](https://badge.fury.io/js/llparse.svg)](https://badge.fury.io/js/llparse) -An API for generating parser in LLVM IR. +An API for compiling an incremental parser into [LLVM bitcode][3]. -**NOTE: `clang` 5.0.0 and later crashes on some of the generated output. The -fixes were [submitted][0] [to][1] [upstream][2]. Please use -O0 if compilation -is crashing/failing until these fixes are landed.** +**NOTE: The resulting bitcode works only on a trunk version of `clang` now, +other versions are broken. The required fixes will be released in +`clang` 6.0.1** ## Usage @@ -54,9 +54,9 @@ http .otherwise(p.error(6, 'Expected HTTP/1.1 and two newlines')); const artifacts = p.build(method); -console.log('----- LLVM -----'); -console.log(artifacts.llvm); -console.log('----- LLVM END -----'); +console.log('----- BITCODE -----'); +console.log(artifacts.bitcode); // buffer +console.log('----- BITCODE END -----'); console.log('----- HEADER -----'); console.log(artifacts.header); console.log('----- HEADER END -----'); @@ -90,3 +90,4 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. [0]: https://reviews.llvm.org/D43729 [1]: https://reviews.llvm.org/D43708 [2]: https://reviews.llvm.org/D43695 +[3]: https://llvm.org/docs/LangRef.html diff --git a/examples/http/Makefile b/examples/http/Makefile index f23dc94..c393ebd 100644 --- a/examples/http/Makefile +++ b/examples/http/Makefile @@ -2,12 +2,10 @@ CC ?= clang all: http -# NOTE: -flto has a bug, and can't be used right now -# See: https://bugs.llvm.org/show_bug.cgi?id=36441 -http: main.c http_parser.ll - $(CC) -g3 -Os -fvisibility=hidden -Wall -I. http_parser.ll main.c -o $@ +http: main.c http_parser.bc + $(CC) -g3 -flto -Os -fvisibility=hidden -Wall -I. http_parser.bc main.c -o $@ -http_parser.ll: index.js +http_parser.bc: index.js node $< .PHONY = all diff --git a/examples/http/index.js b/examples/http/index.js index 65d7839..d1e6d23 100644 --- a/examples/http/index.js +++ b/examples/http/index.js @@ -48,4 +48,4 @@ const path = require('path'); const artifacts = p.build(method); fs.writeFileSync(path.join(__dirname, 'http_parser.h'), artifacts.header); -fs.writeFileSync(path.join(__dirname, 'http_parser.ll'), artifacts.llvm); +fs.writeFileSync(path.join(__dirname, 'http_parser.bc'), artifacts.bitcode); From 641a8bfd89702e508f5fdaba3dd9c2808d3d1839 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Mar 2018 16:54:21 -0400 Subject: [PATCH 257/281] 3.0.0-beta0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9075294..2182a27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.3", + "version": "3.0.0-beta0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 6e32e3a..db71a28 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "2.5.3", + "version": "3.0.0-beta0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From b9a6908f89f257a52389224c87f2d4eb328fee46 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 12 Mar 2018 22:06:42 -0400 Subject: [PATCH 258/281] package: bump dependencies --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2182a27..36c2301 100644 --- a/package-lock.json +++ b/package-lock.json @@ -144,17 +144,17 @@ "dev": true }, "bitcode": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.0.1.tgz", - "integrity": "sha512-qZ4rI/ahSd3uDEORRsoMdMcLE5KINbW/rHFTS3sPG+cSC/FoVc8T3Srpf2jH+VD0G+btWHjfoFd3o0HFFsqb/g==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.0.3.tgz", + "integrity": "sha512-p0R4aOO0/sd1UQxS47ZnvXpzSdmZluXijIio8jscnL3kgtcPLvVYrLLxeKF/paKXtkGa2z0qbS6UnkIoeJtMng==", "requires": { - "bitcode-builder": "1.0.0" + "bitcode-builder": "1.1.4" } }, "bitcode-builder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/bitcode-builder/-/bitcode-builder-1.0.0.tgz", - "integrity": "sha512-rufP/gMS+2q9dLAGQXJuXzT7poxDI3gm9m9iwCLPcsz32/r9AP5yaGmHJkxAFAmwim7ycDpI8UGXQ9UbsgXZYw==" + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/bitcode-builder/-/bitcode-builder-1.1.4.tgz", + "integrity": "sha512-e5t6BAtnVt4wJz1MQ3hmSJG5Glot6U9iNN/0q5Jfc96G+GpFqrdC/rjdvAioEV4i8ZqsZHbhSzl+k/KCggsJnQ==" }, "brace-expansion": { "version": "1.1.11", diff --git a/package.json b/package.json index db71a28..c683352 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "bitcode": "^1.0.1", + "bitcode": "^1.0.3", "debug": "^3.1.0" }, "devDependencies": { From b0cdbbd7045a3cf81e2842a9dc18cc4e6418166e Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 12:22:51 -0400 Subject: [PATCH 259/281] compilation: cache `cstrings` --- lib/llparse/compiler/compilation.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/compilation.js b/lib/llparse/compiler/compilation.js index b74d05c..29c60a7 100644 --- a/lib/llparse/compiler/compilation.js +++ b/lib/llparse/compiler/compilation.js @@ -64,6 +64,7 @@ class Compilation { ]); this.codeCache = new Map(); + this.cstringCache = new Map(); this.debugMethod = null; this.namespace = new Set(); @@ -264,7 +265,13 @@ class Compilation { } cstring(string) { - return this.addGlobalConst('cstr', this.ir.cstring(string)); + if (this.cstringCache.has(string)) { + return this.cstringCache.get(string); + } + + const res = this.addGlobalConst('cstr', this.ir.cstring(string)); + this.cstringCache.set(string, res); + return res; } blob(data) { From 39202e9a9ba05a00d8b6f9d83557733a92025277 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 12:23:01 -0400 Subject: [PATCH 260/281] 3.0.0-beta1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 36c2301..6254901 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta0", + "version": "3.0.0-beta1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c683352..33b6f1d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta0", + "version": "3.0.0-beta1", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From 07f5cf45a9868a5aec2df69905e13acd943fe8f0 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 14:27:58 -0400 Subject: [PATCH 261/281] invoke: supply branch weights --- lib/llparse/compiler/node/invoke.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/llparse/compiler/node/invoke.js b/lib/llparse/compiler/node/invoke.js index db99c61..32a0e6d 100644 --- a/lib/llparse/compiler/node/invoke.js +++ b/lib/llparse/compiler/node/invoke.js @@ -39,8 +39,19 @@ class Invoke extends node.Node { const call = body.call(code, args); const keys = Object.keys(this.map).map(key => key | 0); - const s = ctx.buildSwitch(body, call, keys); + const weights = new Array(1 + keys.length).fill('likely'); + + // Mark error branches as unlikely + keys.forEach((key, i) => { + if (this.map[key] instanceof node.Error) + weights[i + 1] = 'unlikely'; + }); + + if (this.otherwise instanceof node.Error) + weights[0] = 'unlikely'; + + const s = ctx.buildSwitch(body, call, keys, weights); s.cases.forEach((body, i) => { this.tailTo(ctx, body, ctx.pos.current, this.map[keys[i]]); }); From c35c67262f8ee131811b4f38203cd40a250e9b78 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 14:28:07 -0400 Subject: [PATCH 262/281] 3.0.0-beta2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6254901..24f7ca7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta1", + "version": "3.0.0-beta2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 33b6f1d..c982c1b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta1", + "version": "3.0.0-beta2", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From b2d42f453e5ac4f77707174600f81c7bb82b397c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 14:57:11 -0400 Subject: [PATCH 263/281] compiler: pass `undef` instead of `TYPE_MATCH` --- lib/llparse/compiler/compiler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llparse/compiler/compiler.js b/lib/llparse/compiler/compiler.js index 33c0630..86c4c1a 100644 --- a/lib/llparse/compiler/compiler.js +++ b/lib/llparse/compiler/compiler.js @@ -129,7 +129,7 @@ class Compiler { ctx.stateArg(execute), ctx.posArg(execute), ctx.endPosArg(execute), - TYPE_MATCH.val(0) + TYPE_MATCH.undef() ], 'normal', constants.CCONV); call.metadata.set('callees', ctx.ir.metadata(callees)); From 2338a42d4c84d271aea4e4d111625366f3c6ddc3 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 14:57:24 -0400 Subject: [PATCH 264/281] 3.0.0-beta3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 24f7ca7..c9d2dba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta2", + "version": "3.0.0-beta3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c982c1b..074f55e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta2", + "version": "3.0.0-beta3", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From e46eb3cc94f20c66c380ed999c827b3ce7fe6498 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 18:25:16 -0400 Subject: [PATCH 265/281] node: don't merge `match`/`peek` into a trampoline --- lib/llparse/compiler/node/base.js | 7 +++++-- test/api-test.js | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index 86ac64d..cacad4c 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -151,7 +151,9 @@ class Node { tailTo(ctx, body, pos, node, value = null) { const target = this.buildNode(ctx, node); - if (this.phis.has(target)) { + const isCacheable = ctx.pos.next === pos; + + if (isCacheable && this.phis.has(target)) { const cached = this.phis.get(target); if (cached.phi) { @@ -181,7 +183,8 @@ class Node { }); } - this.phis.set(target, { phi, trampoline }); + if (isCacheable) + this.phis.set(target, { phi, trampoline }); const call = trampoline.call(target, [ ctx.state, diff --git a/test/api-test.js b/test/api-test.js index 092d611..15d1f8f 100644 --- a/test/api-test.js +++ b/test/api-test.js @@ -137,6 +137,22 @@ describe('LLParse', function() { binary('aab', 'off=2 error code=42 reason="some reason"\n', callback); }); + it('should not merge `.match()` with `.peek()`', (callback) => { + const maybeCr = p.node('maybeCr'); + const lf = p.node('lf'); + + maybeCr.peek('\n', lf); + maybeCr.match('\r', lf); + maybeCr.otherwise(p.error(1, 'error')); + + lf.match('\n', printOff(p, maybeCr)); + lf.otherwise(p.error(2, 'error')); + + const binary = fixtures.build(p, maybeCr, 'no-merge'); + + binary('\r\n\n', 'off=2\noff=3\n', callback); + }); + describe('`.match()`', () => { it('should compile to a single-bit bit-check node', (callback) => { const start = p.node('start'); From e42930f1129bd7a26f3a1a47f6fea4deae1b4b3f Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 18:25:31 -0400 Subject: [PATCH 266/281] 3.0.0-beta4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index c9d2dba..b7d20ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta3", + "version": "3.0.0-beta4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 074f55e..75d02ff 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta3", + "version": "3.0.0-beta4", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From a0e704e42831fb637f6dc9cc891db44601dede13 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 20:13:11 -0400 Subject: [PATCH 267/281] node: use fieldName directly in Consume --- lib/llparse/compiler/node/consume.js | 11 +++--- lib/llparse/compiler/node/index.js | 1 - lib/llparse/compiler/node/sequence.js | 4 +- lib/llparse/compiler/node/set-index.js | 37 ------------------- lib/llparse/compiler/stage/match-sequence.js | 2 +- lib/llparse/compiler/stage/node-translator.js | 17 +-------- lib/llparse/constants.js | 2 +- lib/llparse/node/consume.js | 19 ++++------ test/consume-test.js | 2 +- 9 files changed, 21 insertions(+), 74 deletions(-) delete mode 100644 lib/llparse/compiler/node/set-index.js diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 246b1c1..920b3e0 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -5,8 +5,9 @@ const assert = require('assert'); const node = require('./'); class Consume extends node.Node { - constructor(...args) { - super('consume', ...args); + constructor(id, fieldName) { + super('consume', id); + this.fieldName = fieldName; this.noPrologueCheck = true; } @@ -16,7 +17,7 @@ class Consume extends node.Node { const pos = ctx.pos.current; - const indexPtr = ctx.stateField(body, '_index'); + const indexPtr = ctx.stateField(body, this.fieldName); const index = body.load(indexPtr); index.metadata.set('invariant.group', INVARIANT_GROUP); @@ -38,14 +39,14 @@ class Consume extends node.Node { const next = hasData.getelementptr(pos, index); assert(!this.skip); - hasData.store(ctx.TYPE_INDEX.val(0), indexPtr) + hasData.store(index.ty.val(0), indexPtr) .metadata.set('invariant.group', INVARIANT_GROUP); this.doOtherwise(ctx, hasData, { current: next, next: null }); // Pause! const left = noData.binop('sub', need, avail); - const leftTrunc = noData.cast('trunc', left, ctx.TYPE_INDEX); + const leftTrunc = noData.cast('trunc', left, index.ty); noData.store(leftTrunc, indexPtr) .metadata.set('invariant.group', INVARIANT_GROUP); diff --git a/lib/llparse/compiler/node/index.js b/lib/llparse/compiler/node/index.js index 27da776..6d0f7bc 100644 --- a/lib/llparse/compiler/node/index.js +++ b/lib/llparse/compiler/node/index.js @@ -8,7 +8,6 @@ exports.Single = require('./single'); exports.Sequence = require('./sequence'); exports.SpanStart = require('./span-start'); exports.SpanEnd = require('./span-end'); -exports.SetIndex = require('./set-index'); exports.Consume = require('./consume'); exports.Pause = require('./pause'); exports.BitCheck = require('./bit-check'); diff --git a/lib/llparse/compiler/node/sequence.js b/lib/llparse/compiler/node/sequence.js index 7dda18a..a4f45e6 100644 --- a/lib/llparse/compiler/node/sequence.js +++ b/lib/llparse/compiler/node/sequence.js @@ -3,8 +3,10 @@ const llparse = require('../../'); const constants = llparse.constants; + const node = require('./'); +const TYPE_INDEX = constants.TYPE_INDEX; const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; @@ -43,7 +45,7 @@ class Sequence extends node.Node { ctx.pos.current, ctx.endPos, cast, - INT.val(seq.ty.toPointer().to.length) + TYPE_INDEX.val(seq.ty.toPointer().to.length) ]); const status = body.extractvalue(call, diff --git a/lib/llparse/compiler/node/set-index.js b/lib/llparse/compiler/node/set-index.js deleted file mode 100644 index 18bab54..0000000 --- a/lib/llparse/compiler/node/set-index.js +++ /dev/null @@ -1,37 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const llparse = require('../../'); -const kSignature = llparse.symbols.kSignature; - -const node = require('./'); - -class SetIndex extends node.Node { - constructor(id, code) { - super('set-index', id); - - this.code = code; - this.noPrologueCheck = true; - } - - doBuild(ctx, body) { - const code = ctx.compilation.buildCode(this.code); - - const args = [ - ctx.state, - ctx.pos.current, - ctx.endPos - ]; - - assert.strictEqual(this.code[kSignature], 'match'); - const call = body.call(code, args); - - const ptr = ctx.stateField(body, '_index'); - body.store(call, ptr) - .metadata.set('invariant.group', ctx.INVARIANT_GROUP); - - this.doOtherwise(ctx, body); - } -} -module.exports = SetIndex; diff --git a/lib/llparse/compiler/stage/match-sequence.js b/lib/llparse/compiler/stage/match-sequence.js index e50a176..e6c79d8 100644 --- a/lib/llparse/compiler/stage/match-sequence.js +++ b/lib/llparse/compiler/stage/match-sequence.js @@ -40,7 +40,7 @@ class MatchSequence extends Stage { TYPE_INPUT, TYPE_INPUT, TYPE_INPUT, - INT + TYPE_INDEX ]); this.cache = new Map(); diff --git a/lib/llparse/compiler/stage/node-translator.js b/lib/llparse/compiler/stage/node-translator.js index 499d856..ad6711d 100644 --- a/lib/llparse/compiler/stage/node-translator.js +++ b/lib/llparse/compiler/stage/node-translator.js @@ -64,9 +64,7 @@ class NodeTranslator extends Stage { } else if (node instanceof llparse.node.Pause) { res = new compiler.node.Pause(this.id(node), node.code, node.reason); } else if (node instanceof llparse.node.Consume) { - res = this.buildConsume(node); - last = res.last; - res = res.first; + res = new compiler.node.Consume(this.id(node), node.fieldName); } else { assert(node instanceof llparse.node.Match); @@ -269,19 +267,6 @@ class NodeTranslator extends Stage { return res; } - buildConsume(node) { - const setIndex = new compiler.node.SetIndex(this.id(node, '_set_index'), - node[kCode]); - - const consume = new compiler.node.Consume(this.id(node, '_consume')); - setIndex.setOtherwise(consume, false); - - return { - first: setIndex, - last: consume - }; - } - checkSignature(node, value) { assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); } diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js index dad323d..a8f9ece 100644 --- a/lib/llparse/constants.js +++ b/lib/llparse/constants.js @@ -9,7 +9,7 @@ exports.INT = IR.i(32); exports.TYPE_INPUT = IR.i(8).ptr(); exports.TYPE_OUTPUT = IR.i(8).ptr(); exports.TYPE_MATCH = exports.INT; -exports.TYPE_INDEX = exports.INT; +exports.TYPE_INDEX = IR.i(64); exports.TYPE_ERROR = exports.INT; exports.TYPE_REASON = IR.i(8).ptr(); exports.TYPE_DATA = IR.i(8).ptr(); diff --git a/lib/llparse/node/consume.js b/lib/llparse/node/consume.js index 9bf3c27..1df1609 100644 --- a/lib/llparse/node/consume.js +++ b/lib/llparse/node/consume.js @@ -3,21 +3,18 @@ const assert = require('assert'); const node = require('./'); -const llparse = require('../'); -const kSignature = llparse.symbols.kSignature; -const kCode = llparse.symbols.kCode; +const kFieldName = Symbol('fieldName'); class Consume extends node.Node { - constructor(code) { - assert(code instanceof llparse.code.Code, - 'Invalid `code` argument of `.consume()`, must be a Code instance'); - assert.strictEqual(code[kSignature], 'match', - '`code` argument of `.consume()` must be have a `match` type'); + constructor(fieldName) { + assert.strictEqual(typeof fieldName, 'string', + 'Consume\'s field name must be a string'); + super('consume_' + fieldName, 'match'); - super('consume_' + code.name, 'match'); - - this[kCode] = code; + this[kFieldName] = fieldName; } + + get fieldName() { return this[kFieldName]; } } module.exports = Consume; diff --git a/test/consume-test.js b/test/consume-test.js index a45c40d..0e47101 100644 --- a/test/consume-test.js +++ b/test/consume-test.js @@ -19,7 +19,7 @@ describe('LLParse/consume', function() { p.property('i8', 'to_consume'); const start = p.node('start'); - const consume = p.consume(p.code.load('to_consume')); + const consume = p.consume('to_consume'); start.select({ '0': 0, '1': 1, '2': 2, '3': 3, '4': 4 From 81faac23746ecfcaeb48c17ccbafc59fd2265a6b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 20:29:28 -0400 Subject: [PATCH 268/281] 3.0.0-beta5 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b7d20ee..6d256f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta4", + "version": "3.0.0-beta5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 75d02ff..25a342f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta4", + "version": "3.0.0-beta5", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From d7060e3103a2604fd6ebc7014fb6d852309b972c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 20:47:35 -0400 Subject: [PATCH 269/281] consume: disallow using internal property names --- lib/llparse/node/consume.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/llparse/node/consume.js b/lib/llparse/node/consume.js index 1df1609..7358970 100644 --- a/lib/llparse/node/consume.js +++ b/lib/llparse/node/consume.js @@ -10,6 +10,8 @@ class Consume extends node.Node { constructor(fieldName) { assert.strictEqual(typeof fieldName, 'string', 'Consume\'s field name must be a string'); + assert(!/^_/.test(fieldName), 'Can\'t use internal field name in Consume'); + super('consume_' + fieldName, 'match'); this[kFieldName] = fieldName; From 6b273c80ae4407e45b3791fe18ed3c8d8d250619 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 20:58:39 -0400 Subject: [PATCH 270/281] consume: fix for i64 or larger field types --- lib/llparse/compiler/node/base.js | 1 + lib/llparse/compiler/node/consume.js | 4 ++-- test/consume-test.js | 23 ++++++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/llparse/compiler/node/base.js b/lib/llparse/compiler/node/base.js index cacad4c..d877f32 100644 --- a/lib/llparse/compiler/node/base.js +++ b/lib/llparse/compiler/node/base.js @@ -52,6 +52,7 @@ class NodeContext { cstring(...args) { return this.compilation.cstring(...args); } blob(...args) { return this.compilation.blob(...args); } addGlobalConst(...args) { return this.compilation.addGlobalConst(...args); } + truncate(...args) { return this.compilation.truncate(...args); } call(...args) { return this.compilation.call(...args); } buildSwitch(...args) { return this.compilation.buildSwitch(...args); } branch(...args) { return this.compilation.branch(...args); } diff --git a/lib/llparse/compiler/node/consume.js b/lib/llparse/compiler/node/consume.js index 920b3e0..b8f675f 100644 --- a/lib/llparse/compiler/node/consume.js +++ b/lib/llparse/compiler/node/consume.js @@ -21,7 +21,7 @@ class Consume extends node.Node { const index = body.load(indexPtr); index.metadata.set('invariant.group', INVARIANT_GROUP); - const need = body.cast('zext', index, ctx.TYPE_INTPTR); + const need = ctx.truncate(body, index, ctx.TYPE_INTPTR); const intPos = body.cast('ptrtoint', pos, ctx.TYPE_INTPTR); const intEndPos = body.cast('ptrtoint', ctx.endPos, ctx.TYPE_INTPTR); @@ -46,7 +46,7 @@ class Consume extends node.Node { // Pause! const left = noData.binop('sub', need, avail); - const leftTrunc = noData.cast('trunc', left, index.ty); + const leftTrunc = ctx.truncate(noData, left, index.ty); noData.store(leftTrunc, indexPtr) .metadata.set('invariant.group', INVARIANT_GROUP); diff --git a/test/consume-test.js b/test/consume-test.js index 0e47101..88e04ff 100644 --- a/test/consume-test.js +++ b/test/consume-test.js @@ -15,7 +15,7 @@ describe('LLParse/consume', function() { p = llparse.create('llparse'); }); - it('should consume bytes', (callback) => { + it('should consume bytes with i8 field', (callback) => { p.property('i8', 'to_consume'); const start = p.node('start'); @@ -35,4 +35,25 @@ describe('LLParse/consume', function() { binary('3aaa2bb1a01b', 'off=4\noff=7\noff=9\noff=10\noff=12\n', callback); }); + + it('should consume bytes with i64 field', (callback) => { + p.property('i64', 'to_consume'); + + const start = p.node('start'); + const consume = p.consume('to_consume'); + + start.select({ + '0': 0, '1': 1, '2': 2, '3': 3, '4': 4 + }, p.invoke(p.code.store('to_consume'), consume)); + + start + .otherwise(p.error(1, 'unexpected')); + + consume + .otherwise(printOff(p, start)); + + const binary = fixtures.build(p, start, 'consume'); + + binary('3aaa2bb1a01b', 'off=4\noff=7\noff=9\noff=10\noff=12\n', callback); + }); }); From 7491edbfd42169c23d12b53c7a7f3c21678f0059 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Tue, 13 Mar 2018 21:07:39 -0400 Subject: [PATCH 271/281] empty: disable prologue check for empty nodes If node has just `.otherwise()` - it should lead the execution there even if there's no data to process. --- lib/llparse/compiler/node/empty.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/llparse/compiler/node/empty.js b/lib/llparse/compiler/node/empty.js index e4004fb..4adc8bb 100644 --- a/lib/llparse/compiler/node/empty.js +++ b/lib/llparse/compiler/node/empty.js @@ -7,6 +7,12 @@ class Empty extends node.Node { super('empty', ...args); } + prologue(ctx, body) { + if (this.skip) + return super.prologue(ctx, body); + return body; + } + doBuild(ctx, body) { this.doOtherwise(ctx, body); } From 1ef181e5574d82333f08cf9b100dce4ad488b251 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 01:08:00 -0400 Subject: [PATCH 272/281] package: bump `bitcode` --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6d256f0..b6eb4d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -144,17 +144,17 @@ "dev": true }, "bitcode": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.0.3.tgz", - "integrity": "sha512-p0R4aOO0/sd1UQxS47ZnvXpzSdmZluXijIio8jscnL3kgtcPLvVYrLLxeKF/paKXtkGa2z0qbS6UnkIoeJtMng==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.0.4.tgz", + "integrity": "sha512-9cSlrUsQK0E0DJ8MRTHkzvlYNkQvGu9D7onwvGzaIc0jTj+9Wgu9xvadof+zhhDPeAevf9AR49YK+O5pPY+xOA==", "requires": { - "bitcode-builder": "1.1.4" + "bitcode-builder": "1.1.5" } }, "bitcode-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/bitcode-builder/-/bitcode-builder-1.1.4.tgz", - "integrity": "sha512-e5t6BAtnVt4wJz1MQ3hmSJG5Glot6U9iNN/0q5Jfc96G+GpFqrdC/rjdvAioEV4i8ZqsZHbhSzl+k/KCggsJnQ==" + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/bitcode-builder/-/bitcode-builder-1.1.5.tgz", + "integrity": "sha512-OGOdEJoTUpYgwWK6FFfM6bnIXwAQTR/putCekJdLg1GwQys9wpL8kEoSUou4pv+fRecsnvAEEGDQrjMS1S109w==" }, "brace-expansion": { "version": "1.1.11", @@ -751,7 +751,7 @@ } }, "llparse-test-fixture": { - "version": "git://github.com/indutny/llparse-test-fixture.git#d81b001d6cdb5ef3ef748108f20c796645e941b2", + "version": "git://github.com/indutny/llparse-test-fixture.git#e76dace1a2b101f84a73231c80d937ba16ec95a5", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index 25a342f..2bb8c55 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "bitcode": "^1.0.3", + "bitcode": "^1.0.4", "debug": "^3.1.0" }, "devDependencies": { From cb297eb9c92810a36c5c719803824221dd2db121 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 01:08:07 -0400 Subject: [PATCH 273/281] 3.0.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b6eb4d3..5c5055a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta5", + "version": "3.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 2bb8c55..88095aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llparse", - "version": "3.0.0-beta5", + "version": "3.0.0", "main": "lib/llparse.js", "scripts": { "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", From a15df14ba1d1918eaff5cdb5f7bb4ff0cbdee761 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 15:24:50 -0400 Subject: [PATCH 274/281] save --- .gitignore | 1 + lib/llparse/case/base.js | 13 -- lib/llparse/case/index.js | 7 - lib/llparse/case/match.js | 27 ---- lib/llparse/case/otherwise.js | 15 -- lib/llparse/case/peek.js | 27 ---- lib/llparse/case/select.js | 31 ---- lib/llparse/code/base.js | 16 --- lib/llparse/code/field-value.js | 22 --- lib/llparse/code/field.js | 16 --- lib/llparse/code/index.js | 19 --- lib/llparse/code/is-equal.js | 10 -- lib/llparse/code/load.js | 10 -- lib/llparse/code/match.js | 10 -- lib/llparse/code/mul-add.js | 35 ----- lib/llparse/code/or.js | 10 -- lib/llparse/code/span.js | 8 -- lib/llparse/code/store.js | 10 -- lib/llparse/code/test.js | 10 -- lib/llparse/code/update.js | 10 -- lib/llparse/code/value.js | 10 -- lib/llparse/constants.js | 52 ------- lib/llparse/index.js | 15 -- lib/llparse/node/base.js | 54 ------- lib/llparse/node/consume.js | 22 --- lib/llparse/node/error.js | 31 ---- lib/llparse/node/index.js | 11 -- lib/llparse/node/invoke.js | 43 ------ lib/llparse/node/match.js | 94 ------------- lib/llparse/node/pause.js | 30 ---- lib/llparse/node/span-end.js | 14 -- lib/llparse/node/span-start.js | 15 -- lib/llparse/property.js | 9 -- lib/llparse/span.js | 47 ------- lib/llparse/symbols.js | 12 -- lib/llparse/transform/base.js | 12 -- lib/llparse/transform/index.js | 3 - package-lock.json | 133 ++++++++++++++++++ package.json | 15 +- lib/llparse.js => src/llparse.ts | 3 +- src/llparse/cases/base.ts | 16 +++ src/llparse/cases/index.ts | 5 + src/llparse/cases/match.ts | 27 ++++ src/llparse/cases/otherwise.ts | 12 ++ src/llparse/cases/peek.ts | 27 ++++ src/llparse/cases/select.ts | 39 +++++ src/llparse/code/base.ts | 7 + src/llparse/code/field-value.ts | 13 ++ src/llparse/code/field.ts | 8 ++ src/llparse/code/index.ts | 17 +++ src/llparse/code/is-equal.ts | 7 + src/llparse/code/load.ts | 7 + src/llparse/code/match.ts | 7 + src/llparse/code/mul-add.ts | 40 ++++++ src/llparse/code/or.ts | 7 + src/llparse/code/span.ts | 5 + src/llparse/code/store.ts | 7 + src/llparse/code/test.ts | 7 + src/llparse/code/update.ts | 7 + src/llparse/code/value.ts | 7 + {lib => src}/llparse/compiler/code/base.js | 0 {lib => src}/llparse/compiler/code/index.js | 0 .../llparse/compiler/code/is-equal.js | 0 {lib => src}/llparse/compiler/code/load.js | 0 {lib => src}/llparse/compiler/code/match.js | 0 {lib => src}/llparse/compiler/code/mul-add.js | 0 {lib => src}/llparse/compiler/code/or.js | 0 {lib => src}/llparse/compiler/code/span.js | 0 {lib => src}/llparse/compiler/code/store.js | 0 {lib => src}/llparse/compiler/code/test.js | 0 {lib => src}/llparse/compiler/code/update.js | 0 {lib => src}/llparse/compiler/code/value.js | 0 {lib => src}/llparse/compiler/compilation.js | 0 {lib => src}/llparse/compiler/compiler.js | 0 {lib => src}/llparse/compiler/index.js | 0 {lib => src}/llparse/compiler/node/base.js | 0 .../llparse/compiler/node/bit-check.js | 0 {lib => src}/llparse/compiler/node/consume.js | 0 {lib => src}/llparse/compiler/node/empty.js | 0 {lib => src}/llparse/compiler/node/error.js | 0 {lib => src}/llparse/compiler/node/index.js | 0 {lib => src}/llparse/compiler/node/invoke.js | 0 {lib => src}/llparse/compiler/node/pause.js | 0 .../llparse/compiler/node/sequence.js | 0 {lib => src}/llparse/compiler/node/single.js | 0 .../llparse/compiler/node/span-end.js | 0 .../llparse/compiler/node/span-start.js | 0 {lib => src}/llparse/compiler/stage/base.js | 0 {lib => src}/llparse/compiler/stage/index.js | 0 .../llparse/compiler/stage/match-sequence.js | 0 .../llparse/compiler/stage/node-builder.js | 0 .../compiler/stage/node-loop-checker.js | 0 .../llparse/compiler/stage/node-translator.js | 0 .../llparse/compiler/stage/span-allocator.js | 0 .../llparse/compiler/stage/span-builder.js | 0 src/llparse/constants.ts | 54 +++++++ src/llparse/index.ts | 11 ++ src/llparse/node/base.ts | 43 ++++++ src/llparse/node/consume.ts | 10 ++ src/llparse/node/error.ts | 15 ++ src/llparse/node/index.ts | 11 ++ src/llparse/node/invoke.ts | 25 ++++ src/llparse/node/match.ts | 91 ++++++++++++ src/llparse/node/pause.ts | 16 +++ src/llparse/node/span-end.ts | 8 ++ src/llparse/node/span-start.ts | 8 ++ src/llparse/span.ts | 38 +++++ src/llparse/transform/base.ts | 4 + src/llparse/transform/index.ts | 1 + src/llparse/trie/index.ts | 5 + src/llparse/trie/next.ts | 9 ++ src/llparse/trie/node.ts | 8 ++ src/llparse/trie/sequence.ts | 10 ++ src/llparse/trie/single.ts | 15 ++ .../trie.js => src/llparse/trie/trie.ts | 86 +++++------ lib/llparse/utils.js => src/llparse/utils.ts | 39 +++-- tsconfig.json | 14 ++ tslint.json | 14 ++ 118 files changed, 889 insertions(+), 850 deletions(-) delete mode 100644 lib/llparse/case/base.js delete mode 100644 lib/llparse/case/index.js delete mode 100644 lib/llparse/case/match.js delete mode 100644 lib/llparse/case/otherwise.js delete mode 100644 lib/llparse/case/peek.js delete mode 100644 lib/llparse/case/select.js delete mode 100644 lib/llparse/code/base.js delete mode 100644 lib/llparse/code/field-value.js delete mode 100644 lib/llparse/code/field.js delete mode 100644 lib/llparse/code/index.js delete mode 100644 lib/llparse/code/is-equal.js delete mode 100644 lib/llparse/code/load.js delete mode 100644 lib/llparse/code/match.js delete mode 100644 lib/llparse/code/mul-add.js delete mode 100644 lib/llparse/code/or.js delete mode 100644 lib/llparse/code/span.js delete mode 100644 lib/llparse/code/store.js delete mode 100644 lib/llparse/code/test.js delete mode 100644 lib/llparse/code/update.js delete mode 100644 lib/llparse/code/value.js delete mode 100644 lib/llparse/constants.js delete mode 100644 lib/llparse/index.js delete mode 100644 lib/llparse/node/base.js delete mode 100644 lib/llparse/node/consume.js delete mode 100644 lib/llparse/node/error.js delete mode 100644 lib/llparse/node/index.js delete mode 100644 lib/llparse/node/invoke.js delete mode 100644 lib/llparse/node/match.js delete mode 100644 lib/llparse/node/pause.js delete mode 100644 lib/llparse/node/span-end.js delete mode 100644 lib/llparse/node/span-start.js delete mode 100644 lib/llparse/property.js delete mode 100644 lib/llparse/span.js delete mode 100644 lib/llparse/symbols.js delete mode 100644 lib/llparse/transform/base.js delete mode 100644 lib/llparse/transform/index.js rename lib/llparse.js => src/llparse.ts (97%) create mode 100644 src/llparse/cases/base.ts create mode 100644 src/llparse/cases/index.ts create mode 100644 src/llparse/cases/match.ts create mode 100644 src/llparse/cases/otherwise.ts create mode 100644 src/llparse/cases/peek.ts create mode 100644 src/llparse/cases/select.ts create mode 100644 src/llparse/code/base.ts create mode 100644 src/llparse/code/field-value.ts create mode 100644 src/llparse/code/field.ts create mode 100644 src/llparse/code/index.ts create mode 100644 src/llparse/code/is-equal.ts create mode 100644 src/llparse/code/load.ts create mode 100644 src/llparse/code/match.ts create mode 100644 src/llparse/code/mul-add.ts create mode 100644 src/llparse/code/or.ts create mode 100644 src/llparse/code/span.ts create mode 100644 src/llparse/code/store.ts create mode 100644 src/llparse/code/test.ts create mode 100644 src/llparse/code/update.ts create mode 100644 src/llparse/code/value.ts rename {lib => src}/llparse/compiler/code/base.js (100%) rename {lib => src}/llparse/compiler/code/index.js (100%) rename {lib => src}/llparse/compiler/code/is-equal.js (100%) rename {lib => src}/llparse/compiler/code/load.js (100%) rename {lib => src}/llparse/compiler/code/match.js (100%) rename {lib => src}/llparse/compiler/code/mul-add.js (100%) rename {lib => src}/llparse/compiler/code/or.js (100%) rename {lib => src}/llparse/compiler/code/span.js (100%) rename {lib => src}/llparse/compiler/code/store.js (100%) rename {lib => src}/llparse/compiler/code/test.js (100%) rename {lib => src}/llparse/compiler/code/update.js (100%) rename {lib => src}/llparse/compiler/code/value.js (100%) rename {lib => src}/llparse/compiler/compilation.js (100%) rename {lib => src}/llparse/compiler/compiler.js (100%) rename {lib => src}/llparse/compiler/index.js (100%) rename {lib => src}/llparse/compiler/node/base.js (100%) rename {lib => src}/llparse/compiler/node/bit-check.js (100%) rename {lib => src}/llparse/compiler/node/consume.js (100%) rename {lib => src}/llparse/compiler/node/empty.js (100%) rename {lib => src}/llparse/compiler/node/error.js (100%) rename {lib => src}/llparse/compiler/node/index.js (100%) rename {lib => src}/llparse/compiler/node/invoke.js (100%) rename {lib => src}/llparse/compiler/node/pause.js (100%) rename {lib => src}/llparse/compiler/node/sequence.js (100%) rename {lib => src}/llparse/compiler/node/single.js (100%) rename {lib => src}/llparse/compiler/node/span-end.js (100%) rename {lib => src}/llparse/compiler/node/span-start.js (100%) rename {lib => src}/llparse/compiler/stage/base.js (100%) rename {lib => src}/llparse/compiler/stage/index.js (100%) rename {lib => src}/llparse/compiler/stage/match-sequence.js (100%) rename {lib => src}/llparse/compiler/stage/node-builder.js (100%) rename {lib => src}/llparse/compiler/stage/node-loop-checker.js (100%) rename {lib => src}/llparse/compiler/stage/node-translator.js (100%) rename {lib => src}/llparse/compiler/stage/span-allocator.js (100%) rename {lib => src}/llparse/compiler/stage/span-builder.js (100%) create mode 100644 src/llparse/constants.ts create mode 100644 src/llparse/index.ts create mode 100644 src/llparse/node/base.ts create mode 100644 src/llparse/node/consume.ts create mode 100644 src/llparse/node/error.ts create mode 100644 src/llparse/node/index.ts create mode 100644 src/llparse/node/invoke.ts create mode 100644 src/llparse/node/match.ts create mode 100644 src/llparse/node/pause.ts create mode 100644 src/llparse/node/span-end.ts create mode 100644 src/llparse/node/span-start.ts create mode 100644 src/llparse/span.ts create mode 100644 src/llparse/transform/base.ts create mode 100644 src/llparse/transform/index.ts create mode 100644 src/llparse/trie/index.ts create mode 100644 src/llparse/trie/next.ts create mode 100644 src/llparse/trie/node.ts create mode 100644 src/llparse/trie/sequence.ts create mode 100644 src/llparse/trie/single.ts rename lib/llparse/trie.js => src/llparse/trie/trie.ts (61%) rename lib/llparse/utils.js => src/llparse/utils.ts (58%) create mode 100644 tsconfig.json create mode 100644 tslint.json diff --git a/.gitignore b/.gitignore index 5b7a295..d4fa5a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ npm-debug.log +lib/ test/tmp examples/http/http *.dSYM diff --git a/lib/llparse/case/base.js b/lib/llparse/case/base.js deleted file mode 100644 index 36ed7ca..0000000 --- a/lib/llparse/case/base.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict'; - -class Case { - constructor(type, next) { - this.type = type; - this.next = next; - } - - linearize() { - throw new Error('Not implemented'); - } -} -module.exports = Case; diff --git a/lib/llparse/case/index.js b/lib/llparse/case/index.js deleted file mode 100644 index cadd5c4..0000000 --- a/lib/llparse/case/index.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -exports.Case = require('./base'); -exports.Match = require('./match'); -exports.Peek = require('./peek'); -exports.Select = require('./select'); -exports.Otherwise = require('./otherwise'); diff --git a/lib/llparse/case/match.js b/lib/llparse/case/match.js deleted file mode 100644 index 9070849..0000000 --- a/lib/llparse/case/match.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const llparse = require('../'); - -const Case = require('./').Case; - -class Match extends Case { - constructor(key, next) { - super('match', next); - - this.key = llparse.utils.toBuffer(key); - assert(this.key.length > 0, - '`.match()` must get at least 1 byte as a first argument'); - } - - linearize() { - return [ { - key: this.key, - next: this.next, - value: null, - noAdvance: false - } ]; - } -} -module.exports = Match; diff --git a/lib/llparse/case/otherwise.js b/lib/llparse/case/otherwise.js deleted file mode 100644 index 4e10d7d..0000000 --- a/lib/llparse/case/otherwise.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -const Case = require('./').Case; - -class Otherwise extends Case { - constructor(next, skip = false) { - super('otherwise', next); - this.skip = skip; - } - - linearize() { - throw new Error('Should not be called'); - } -} -module.exports = Otherwise; diff --git a/lib/llparse/case/peek.js b/lib/llparse/case/peek.js deleted file mode 100644 index f5933f3..0000000 --- a/lib/llparse/case/peek.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const llparse = require('../'); - -const Case = require('./').Case; - -class Peek extends Case { - constructor(key, next) { - super('peek', next); - - this.key = llparse.utils.toBuffer(key); - assert.strictEqual(this.key.length, 1, - '`.peek()` must get exactly 1 byte as a first argument'); - } - - linearize() { - return [ { - key: this.key, - next: this.next, - value: null, - noAdvance: true - } ]; - } -} -module.exports = Peek; diff --git a/lib/llparse/case/select.js b/lib/llparse/case/select.js deleted file mode 100644 index 38a33a2..0000000 --- a/lib/llparse/case/select.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -const llparse = require('../'); - -const Case = require('./').Case; - -class Select extends Case { - constructor(next) { - super('select', next); - - this.map = new Map(); - } - - add(key, value) { - this.map.set(key, value); - } - - linearize() { - const res = []; - this.map.forEach((value, key) => { - res.push({ - key: llparse.utils.toBuffer(key), - next: this.next, - value, - noAdvance: false - }); - }); - return res; - } -} -module.exports = Select; diff --git a/lib/llparse/code/base.js b/lib/llparse/code/base.js deleted file mode 100644 index 2ba080d..0000000 --- a/lib/llparse/code/base.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict'; - -const llparse = require('../'); - -const kName = Symbol('name'); -const kSignature = llparse.symbols.kSignature; - -class Code { - constructor(signature, name) { - this[kSignature] = signature; - this[kName] = name; - } - - get name() { return this[kName]; } -} -module.exports = Code; diff --git a/lib/llparse/code/field-value.js b/lib/llparse/code/field-value.js deleted file mode 100644 index d3203de..0000000 --- a/lib/llparse/code/field-value.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const code = require('./'); - -const kValue = Symbol('value'); - -class FieldValue extends code.Field { - constructor(signature, name, field, value) { - super(signature, name, field); - - assert.strictEqual(typeof value, 'number', - '`value` argument must be a number'); - assert.strictEqual(value, value | 0, '`value` argument must be an integer'); - - this[kValue] = value; - } - - get value() { return this[kValue]; } -} -module.exports = FieldValue; diff --git a/lib/llparse/code/field.js b/lib/llparse/code/field.js deleted file mode 100644 index eae1529..0000000 --- a/lib/llparse/code/field.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict'; - -const code = require('./'); - -const kField = Symbol('field'); - -class Field extends code.Code { - constructor(signature, name, field) { - super(signature, name + '_' + field); - - this[kField] = field; - } - - get field() { return this[kField]; } -} -module.exports = Field; diff --git a/lib/llparse/code/index.js b/lib/llparse/code/index.js deleted file mode 100644 index ca3eecb..0000000 --- a/lib/llparse/code/index.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -exports.Code = require('./base'); -exports.Match = require('./match'); -exports.Value = require('./value'); -exports.Span = require('./span'); - -// Helpers - -exports.Field = require('./field'); -exports.FieldValue = require('./field-value'); - -exports.Store = require('./store'); -exports.Load = require('./load'); -exports.MulAdd = require('./mul-add'); -exports.Update = require('./update'); -exports.IsEqual = require('./is-equal'); -exports.Or = require('./or'); -exports.Test = require('./test'); diff --git a/lib/llparse/code/is-equal.js b/lib/llparse/code/is-equal.js deleted file mode 100644 index 16a254e..0000000 --- a/lib/llparse/code/is-equal.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class IsEqual extends code.FieldValue { - constructor(field, value) { - super('match', 'is_equal', field, value); - } -} -module.exports = IsEqual; diff --git a/lib/llparse/code/load.js b/lib/llparse/code/load.js deleted file mode 100644 index a0af7f5..0000000 --- a/lib/llparse/code/load.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Load extends code.Field { - constructor(field) { - super('match', 'load', field); - } -} -module.exports = Load; diff --git a/lib/llparse/code/match.js b/lib/llparse/code/match.js deleted file mode 100644 index f998a85..0000000 --- a/lib/llparse/code/match.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Match extends code.Code { - constructor(name) { - super('match', name); - } -} -module.exports = Match; diff --git a/lib/llparse/code/mul-add.js b/lib/llparse/code/mul-add.js deleted file mode 100644 index eb1cf88..0000000 --- a/lib/llparse/code/mul-add.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const code = require('./'); - -const kOptions = Symbol('options'); - -class MulAdd extends code.Field { - constructor(field, options) { - super('value', 'mul_add', field); - - options = Object.assign({ - max: 0, - signed: true - }, options); - assert.strictEqual(typeof options.max, 'number', - '`MulAdd.options.max` must be a number'); - assert.strictEqual(typeof options.base, 'number', - '`MulAdd.options.base` must be a number'); - assert(options.max >= 0, - '`MulAdd.options.max` must be a non-negative number'); - assert(options.base > 0, - '`MulAdd.options.base` must be a positive number'); - assert.strictEqual(options.max, options.max | 0, - '`MulAdd.options.max` must be an integer'); - assert.strictEqual(options.base, options.base | 0, - '`MulAdd.options.max` must be an integer'); - - this[kOptions] = options; - } - - get options() { return this[kOptions]; } -} -module.exports = MulAdd; diff --git a/lib/llparse/code/or.js b/lib/llparse/code/or.js deleted file mode 100644 index ea134ca..0000000 --- a/lib/llparse/code/or.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Or extends code.FieldValue { - constructor(field, value) { - super('match', 'or', field, value); - } -} -module.exports = Or; diff --git a/lib/llparse/code/span.js b/lib/llparse/code/span.js deleted file mode 100644 index d11fa85..0000000 --- a/lib/llparse/code/span.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Span extends code.Match { - // no-op -} -module.exports = Span; diff --git a/lib/llparse/code/store.js b/lib/llparse/code/store.js deleted file mode 100644 index 61380c3..0000000 --- a/lib/llparse/code/store.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Store extends code.Field { - constructor(field) { - super('value', 'store', field); - } -} -module.exports = Store; diff --git a/lib/llparse/code/test.js b/lib/llparse/code/test.js deleted file mode 100644 index e9669e3..0000000 --- a/lib/llparse/code/test.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Test extends code.FieldValue { - constructor(field, mask) { - super('match', 'test', field, mask); - } -} -module.exports = Test; diff --git a/lib/llparse/code/update.js b/lib/llparse/code/update.js deleted file mode 100644 index 5d30200..0000000 --- a/lib/llparse/code/update.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Update extends code.FieldValue { - constructor(field, value) { - super('match', 'update', field, value); - } -} -module.exports = Update; diff --git a/lib/llparse/code/value.js b/lib/llparse/code/value.js deleted file mode 100644 index eac064b..0000000 --- a/lib/llparse/code/value.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Value extends code.Code { - constructor(name) { - super('value', name); - } -} -module.exports = Value; diff --git a/lib/llparse/constants.js b/lib/llparse/constants.js deleted file mode 100644 index a8f9ece..0000000 --- a/lib/llparse/constants.js +++ /dev/null @@ -1,52 +0,0 @@ -'use strict'; - -const IR = require('bitcode').Builder; - -exports.CCONV = 'fastcc'; - -exports.BOOL = IR.i(1); -exports.INT = IR.i(32); -exports.TYPE_INPUT = IR.i(8).ptr(); -exports.TYPE_OUTPUT = IR.i(8).ptr(); -exports.TYPE_MATCH = exports.INT; -exports.TYPE_INDEX = IR.i(64); -exports.TYPE_ERROR = exports.INT; -exports.TYPE_REASON = IR.i(8).ptr(); -exports.TYPE_DATA = IR.i(8).ptr(); -exports.TYPE_STATUS = IR.i(32); - -// TODO(indutny): although it works through zero extension, is there any -// cross-platform way to do it? -exports.TYPE_INTPTR = IR.i(64); - -exports.ARG_STATE = 's'; -exports.ARG_POS = 'p'; -exports.ARG_ENDPOS = 'endp'; -exports.ARG_SEQUENCE = 'seq'; -exports.ARG_SEQUENCE_LEN = 'slen'; -exports.ARG_MATCH = 'match'; -exports.ARG_UNUSED = '_unused'; - -exports.ATTR_STATE = [ 'noalias', 'nonnull' ]; -exports.ATTR_POS = [ 'noalias', 'nonnull', 'readonly' ]; -exports.ATTR_ENDPOS = [ 'noalias', 'nonnull', 'readnone' ]; -exports.ATTR_SEQUENCE = exports.ATTR_POS; - -exports.SEQUENCE_COMPLETE = 0; -exports.SEQUENCE_PAUSE = 1; -exports.SEQUENCE_MISMATCH = 2; - -// NOTE: It is important to start them with `_`, see `lib/llparse.js` (property) -exports.SPAN_START_PREFIX = '_span_start'; -exports.SPAN_CB_PREFIX = '_span_cb'; - -exports.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE = 32; -exports.DEFAULT_TRANSLATOR_MAX_CHECK_WIDTH = 4; - -exports.USER_TYPES = { - i8: IR.i(8), - i16: IR.i(16), - i32: IR.i(32), - i64: IR.i(64), - 'ptr': IR.i(8).ptr() -}; diff --git a/lib/llparse/index.js b/lib/llparse/index.js deleted file mode 100644 index d3caa6c..0000000 --- a/lib/llparse/index.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -exports.constants = require('./constants'); -exports.symbols = require('./symbols'); -exports.utils = require('./utils'); - -exports.case = require('./case'); -exports.code = require('./code'); -exports.node = require('./node'); -exports.transform = require('./transform'); -exports.Span = require('./span'); - -exports.Trie = require('./trie'); -exports.Property = require('./property'); -exports.compiler = require('./compiler'); diff --git a/lib/llparse/node/base.js b/lib/llparse/node/base.js deleted file mode 100644 index 4ef343c..0000000 --- a/lib/llparse/node/base.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); -const llparse = require('../'); - -const kCheckIsMatch = llparse.symbols.kCheckIsMatch; -const kOtherwise = llparse.symbols.kOtherwise; -const kSignature = llparse.symbols.kSignature; - -const kName = Symbol('name'); - -class Node { - constructor(name, signature) { - this[kName] = name; - this[kSignature] = signature; - - this[kOtherwise] = null; - } - - get name() { return this[kName]; } - - otherwise(next) { - assert(next instanceof Node, 'Invalid `next` argument of `.otherwise()`'); - this[kCheckIsMatch](next, '.otherwise()'); - - assert.strictEqual(this[kOtherwise], null, - 'Duplicate `.otherwise()`/`.skipTo()`'); - this[kOtherwise] = new llparse.case.Otherwise(next); - - return this; - } - - skipTo(next) { - assert(next instanceof Node, 'Invalid `next` argument of `.skipTo()`'); - this[kCheckIsMatch](next, '.skipTo()'); - - assert.strictEqual(this[kOtherwise], null, - 'Duplicate `.skipTo()`/`.otherwise()`'); - this[kOtherwise] = new llparse.case.Otherwise(next, true); - - return this; - } - - [kCheckIsMatch](next, method) { - if (!(next instanceof node.Invoke)) - return; - - assert.strictEqual(next[kSignature], 'match', - `Invoke of "${next.name}" can't be a target of \`${method}\``); - } -} -module.exports = Node; diff --git a/lib/llparse/node/consume.js b/lib/llparse/node/consume.js deleted file mode 100644 index 7358970..0000000 --- a/lib/llparse/node/consume.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); - -const kFieldName = Symbol('fieldName'); - -class Consume extends node.Node { - constructor(fieldName) { - assert.strictEqual(typeof fieldName, 'string', - 'Consume\'s field name must be a string'); - assert(!/^_/.test(fieldName), 'Can\'t use internal field name in Consume'); - - super('consume_' + fieldName, 'match'); - - this[kFieldName] = fieldName; - } - - get fieldName() { return this[kFieldName]; } -} -module.exports = Consume; diff --git a/lib/llparse/node/error.js b/lib/llparse/node/error.js deleted file mode 100644 index a5e9c56..0000000 --- a/lib/llparse/node/error.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); - -const kCode = Symbol('code'); -const kReason = Symbol('reason'); - -class Error extends node.Node { - constructor(code, reason) { - super('error', 'match'); - - assert.strictEqual(typeof code, 'number', - 'The first argument of `.error()` must be a numeric error code'); - assert.strictEqual(code, code | 0, - 'The first argument of `.error()` must be an integer error code'); - assert.strictEqual(typeof reason, 'string', - 'The second argument of `.error()` must be a string error description'); - - this[kCode] = code; - this[kReason] = reason; - } - - get code() { return this[kCode]; } - get reason() { return this[kReason]; } - - otherwise() { throw new Error('Not supported'); } - skipTo() { throw new Error('Not supported'); } -} -module.exports = Error; diff --git a/lib/llparse/node/index.js b/lib/llparse/node/index.js deleted file mode 100644 index 9cc500d..0000000 --- a/lib/llparse/node/index.js +++ /dev/null @@ -1,11 +0,0 @@ -'use strict'; - -exports.Node = require('./base'); - -exports.Match = require('./match'); -exports.Error = require('./error'); -exports.Invoke = require('./invoke'); -exports.SpanStart = require('./span-start'); -exports.SpanEnd = require('./span-end'); -exports.Consume = require('./consume'); -exports.Pause = require('./pause'); diff --git a/lib/llparse/node/invoke.js b/lib/llparse/node/invoke.js deleted file mode 100644 index 99574d5..0000000 --- a/lib/llparse/node/invoke.js +++ /dev/null @@ -1,43 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); -const llparse = require('../'); - -const kSignature = llparse.symbols.kSignature; -const kCode = llparse.symbols.kCode; -const kMap = llparse.symbols.kMap; - -class Invoke extends node.Node { - constructor(code, map, otherwise = null) { - // `.invoke(code, otherwise)` - if (map && map instanceof node.Node) { - otherwise = map; - map = null; - } - - map = map || {}; - - assert.strictEqual(typeof map, 'object', - 'Invalid `map` argument of `.invoke()`, must be an Object'); - assert(code instanceof llparse.code.Code, - 'Invalid `code` argument of `.invoke()`, must be a Code instance'); - - Object.keys(map).forEach((key) => { - assert.equal(key, key | 0, - 'Only integer keys are allowed in `.invoke()`\'s map'); - assert(map[key] instanceof node.Node, - 'Only Node values are allowed in `.invoke()`\'s map'); - }); - - super('invoke_' + code.name, code[kSignature]); - - this[kCode] = code; - this[kMap] = map; - - if (otherwise) - this.otherwise(otherwise); - } -} -module.exports = Invoke; diff --git a/lib/llparse/node/match.js b/lib/llparse/node/match.js deleted file mode 100644 index 3829567..0000000 --- a/lib/llparse/node/match.js +++ /dev/null @@ -1,94 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); -const llparse = require('../'); - -const kCases = llparse.symbols.kCases; -const kCheckIsMatch = llparse.symbols.kCheckIsMatch; -const kSignature = llparse.symbols.kSignature; -const kTransform = llparse.symbols.kTransform; - -class Match extends node.Node { - constructor(name) { - super(name, 'match'); - - this[kTransform] = null; - - this[kCases] = []; - } - - transform(t) { - assert.strictEqual(this[kTransform], null, 'Can\'t apply transform twice'); - assert(t instanceof llparse.transform.Transform, - '`.transform()` argument must be a `Transform` instance'); - this[kTransform] = t; - - return this; - } - - peek(value, next) { - // .peek([ ... ], next) - if (Array.isArray(value)) { - value.forEach(value => this.peek(value, next)); - return this; - } - - assert(next instanceof node.Node, 'Invalid `next` argument of `.match()`'); - this[kCheckIsMatch](next, '.peek()'); - - this[kCases].push(new llparse.case.Peek(value, next)); - - return this; - } - - match(value, next) { - // .match([ ... ], next) - if (Array.isArray(value)) { - value.forEach(value => this.match(value, next)); - return this; - } - - assert(next instanceof node.Node, 'Invalid `next` argument of `.match()`'); - this[kCheckIsMatch](next, '.match()'); - - this[kCases].push(new llparse.case.Match(value, next)); - - return this; - } - - select(key, value, next) { - // .select(key, value, next) - const pairs = []; - if (Buffer.isBuffer(key) || typeof key === 'number' || - typeof key === 'string') { - assert.strictEqual(typeof value, 'number', - '`.select(key, value, next)` is a signature of the method'); - - pairs.push({ key, value }); - } else { - assert.strictEqual(typeof key, 'object', - '`.select()` first argument must be either an object or a key'); - - const map = key; - next = value; - value = null; - - Object.keys(map).forEach((key) => pairs.push({ key, value: map[key] })); - } - - assert(next instanceof node.Invoke, - 'Invalid `next` argument of `.select()`, must be an `.invoke()` node'); - assert.strictEqual(next[kSignature], 'value', - `Invoke of "${next.name}" can't be a target of \`.select()\``); - - const select = new llparse.case.Select(next); - pairs.forEach(pair => select.add(pair.key, pair.value)); - - this[kCases].push(select); - - return this; - } -} -module.exports = Match; diff --git a/lib/llparse/node/pause.js b/lib/llparse/node/pause.js deleted file mode 100644 index 16b525b..0000000 --- a/lib/llparse/node/pause.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); - -const kCode = Symbol('code'); -const kReason = Symbol('reason'); - -class Pause extends node.Node { - constructor(code, reason) { - super('pause', 'match'); - - assert.strictEqual(typeof code, 'number', - 'The first argument of `.error()` must be a numeric error code'); - assert.strictEqual(code, code | 0, - 'The first argument of `.error()` must be an integer error code'); - assert.strictEqual(typeof reason, 'string', - 'The second argument of `.error()` must be a string error description'); - - this[kCode] = code; - this[kReason] = reason; - } - - get code() { return this[kCode]; } - get reason() { return this[kReason]; } - - skipTo() { throw new Error('Not supported, please use `pause.otherwise()`'); } -} -module.exports = Pause; diff --git a/lib/llparse/node/span-end.js b/lib/llparse/node/span-end.js deleted file mode 100644 index 687b920..0000000 --- a/lib/llparse/node/span-end.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -const llparse = require('../'); -const node = require('./'); - -const kSpan = llparse.symbols.kSpan; - -class SpanEnd extends node.Node { - constructor(span, code) { - super('span_end_' + code.name, 'match'); - this[kSpan] = span; - } -} -module.exports = SpanEnd; diff --git a/lib/llparse/node/span-start.js b/lib/llparse/node/span-start.js deleted file mode 100644 index dcd13dc..0000000 --- a/lib/llparse/node/span-start.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -const llparse = require('../'); -const node = require('./'); - -const kSpan = llparse.symbols.kSpan; - -class SpanStart extends node.Node { - constructor(span, code) { - super('span_start_' + code.name, 'match'); - - this[kSpan] = span; - } -} -module.exports = SpanStart; diff --git a/lib/llparse/property.js b/lib/llparse/property.js deleted file mode 100644 index 0fbdaa4..0000000 --- a/lib/llparse/property.js +++ /dev/null @@ -1,9 +0,0 @@ -'use strict'; - -class Property { - constructor(type, name) { - this.type = type; - this.name = name; - } -} -module.exports = Property; diff --git a/lib/llparse/span.js b/lib/llparse/span.js deleted file mode 100644 index 44d0b37..0000000 --- a/lib/llparse/span.js +++ /dev/null @@ -1,47 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const llparse = require('./'); - -const kCode = llparse.symbols.kCode; -const kStartCache = Symbol('startCache'); -const kEndCache = Symbol('endCache'); - -class Span { - constructor(code) { - assert(code instanceof llparse.code.Code, - 'Invalid `code` argument of `.span()`, must be a Code instance'); - - this[kCode] = code; - this[kStartCache] = new Map(); - this[kEndCache] = new Map(); - } - - start(otherwise = null) { - const cache = this[kStartCache]; - if (otherwise && cache.has(otherwise)) - return cache.get(otherwise); - - const res = new llparse.node.SpanStart(this, this[kCode]); - if (otherwise) { - res.otherwise(otherwise); - cache.set(otherwise, res); - } - return res; - } - - end(otherwise = null) { - const cache = this[kEndCache]; - if (otherwise && cache.has(otherwise)) - return cache.get(otherwise); - - const res = new llparse.node.SpanEnd(this, this[kCode]); - if (otherwise) { - res.otherwise(otherwise); - cache.set(otherwise, res); - } - return res; - } -} -module.exports = Span; diff --git a/lib/llparse/symbols.js b/lib/llparse/symbols.js deleted file mode 100644 index a8065ac..0000000 --- a/lib/llparse/symbols.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -exports.kCases = Symbol('cases'); -exports.kCheckIsMatch = Symbol('checkIsMatch'); -exports.kCode = Symbol('code'); -exports.kMap = Symbol('map'); -exports.kName = Symbol('name'); -exports.kNoAdvance = Symbol('noAdvance'); -exports.kOtherwise = Symbol('otherwise'); -exports.kSignature = Symbol('signature'); -exports.kSpan = Symbol('span'); -exports.kTransform = Symbol('transform'); diff --git a/lib/llparse/transform/base.js b/lib/llparse/transform/base.js deleted file mode 100644 index 29c67f2..0000000 --- a/lib/llparse/transform/base.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -const kName = Symbol('name'); - -class Transform { - constructor(name) { - this[kName] = name; - } - - get name() { return this[kName]; } -} -module.exports = Transform; diff --git a/lib/llparse/transform/index.js b/lib/llparse/transform/index.js deleted file mode 100644 index 99514d5..0000000 --- a/lib/llparse/transform/index.js +++ /dev/null @@ -1,3 +0,0 @@ -'use strict'; - -exports.Transform = require('./base'); diff --git a/package-lock.json b/package-lock.json index 5c5055a..8b2517b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,18 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/mocha": { + "version": "2.2.48", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-2.2.48.tgz", + "integrity": "sha512-nlK/iyETgafGli8Zh9zJVCTicvU3iajSkRwOh3Hhiva598CMqNJ4NcVCGMTGKpGpTYj/9R8RLzS9NAykSSCqGw==", + "dev": true + }, + "@types/node": { + "version": "9.4.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-9.4.7.tgz", + "integrity": "sha512-4Ba90mWNx8ddbafuyGGwjkZMigi+AWfYLSDCpovwsE63ia8w93r3oJ8PIAQc3y8U+XHcnMOHPIzNe3o438Ywcw==", + "dev": true + }, "acorn": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.5.3.tgz", @@ -172,6 +184,12 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true + }, "caller-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz", @@ -773,6 +791,12 @@ "yallist": "2.1.2" } }, + "make-error": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.4.tgz", + "integrity": "sha512-0Dab5btKVPhibSalc9QGXb559ED7G7iLjFXBaj9Wq8O3vorueR5K5jaE3hkG6ZQINyhA/JgG6Qk4qdFQjsYV6g==", + "dev": true + }, "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -911,6 +935,12 @@ "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", "dev": true }, + "path-parse": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz", + "integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME=", + "dev": true + }, "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -987,6 +1017,15 @@ "resolve-from": "1.0.1" } }, + "resolve": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.5.0.tgz", + "integrity": "sha512-hgoSGrc3pjzAPHNBg+KnFcK2HwlHTs/YrAGUr6qgTVUZmXv1UEXXl0bZNBKMA9fud6lRYFdPGz0xXxycPzmmiw==", + "dev": true, + "requires": { + "path-parse": "1.0.5" + } + }, "resolve-from": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz", @@ -1078,6 +1117,21 @@ "is-fullwidth-code-point": "2.0.0" } }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-support": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.3.tgz", + "integrity": "sha512-eKkTgWYeBOQqFGXRfKabMFdnWepo51vWqEdoeikaEPFiJC7MCU5j2h4+6Q8npkZTeLGbSyecZvRxiSoWl3rh+w==", + "dev": true, + "requires": { + "source-map": "0.6.1" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -1167,6 +1221,73 @@ "os-tmpdir": "1.0.2" } }, + "ts-node": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-5.0.1.tgz", + "integrity": "sha512-XK7QmDcNHVmZkVtkiwNDWiERRHPyU8nBqZB1+iv2UhOG0q3RQ9HsZ2CMqISlFbxjrYFGfG2mX7bW4dAyxBVzUw==", + "dev": true, + "requires": { + "arrify": "1.0.1", + "chalk": "2.3.2", + "diff": "3.5.0", + "make-error": "1.3.4", + "minimist": "1.2.0", + "mkdirp": "0.5.1", + "source-map-support": "0.5.3", + "yn": "2.0.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + } + } + }, + "tslib": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.0.tgz", + "integrity": "sha512-f/qGG2tUkrISBlQZEjEqoZ3B2+npJjIf04H1wuAv9iA8i04Icp+61KRXxFdha22670NJopsZCIjhC3SnjPRKrQ==", + "dev": true + }, + "tslint": { + "version": "5.9.1", + "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.9.1.tgz", + "integrity": "sha1-ElX4ej/1frCw4fDmEKi0dIBGya4=", + "dev": true, + "requires": { + "babel-code-frame": "6.26.0", + "builtin-modules": "1.1.1", + "chalk": "2.3.2", + "commander": "2.15.0", + "diff": "3.5.0", + "glob": "7.1.2", + "js-yaml": "3.11.0", + "minimatch": "3.0.4", + "resolve": "1.5.0", + "semver": "5.5.0", + "tslib": "1.9.0", + "tsutils": "2.22.2" + }, + "dependencies": { + "commander": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.0.tgz", + "integrity": "sha512-7B1ilBwtYSbetCgTY1NJFg+gVpestg0fdA1MhC1Vs4ssyfSXnCAjFr+QcQM9/RedXC0EaUx1sG8Smgw2VfgKEg==", + "dev": true + } + } + }, + "tsutils": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.22.2.tgz", + "integrity": "sha512-u06FUSulCJ+Y8a2ftuqZN6kIGqdP2yJjUPEngXqmdPND4UQfb04igcotH+dw+IFr417yP6muCLE8/5/Qlfnx0w==", + "dev": true, + "requires": { + "tslib": "1.9.0" + } + }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -1182,6 +1303,12 @@ "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", "dev": true }, + "typescript": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.7.2.tgz", + "integrity": "sha512-p5TCYZDAO0m4G344hD+wx/LATebLWZNkkh2asWUFqSsD2OrDNhbAHuSjobrmsUmdzjJjEeZVU9g1h3O6vpstnw==", + "dev": true + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -1223,6 +1350,12 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", "dev": true + }, + "yn": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-2.0.0.tgz", + "integrity": "sha1-5a2ryKz0CPY4X8dklWhMiOavaJo=", + "dev": true } } } diff --git a/package.json b/package.json index 88095aa..a1dd57d 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,12 @@ "version": "3.0.0", "main": "lib/llparse.js", "scripts": { - "lint": "eslint test/*.js test/**/*.js lib/*.js lib/**/*.js lib/**/**/*.js lib/**/**/**/*.js", - "test": "mocha --reporter=spec test/*-test.js && npm run lint" + "build": "tsc", + "clean": "rm -rf lib", + "prepare": "npm run clean && npm run build", + "lint": "tslint -c tslint.json src/*.ts src/**/*.ts src/**/**/*.ts src/**/**/**/*.ts test/*.ts", + "mocha": "mocha -r ts-node/register/type-check --reporter spec test/*-test.ts", + "test": "npm run mocha && npm run lint" }, "files": [ "lib" @@ -22,10 +26,15 @@ "debug": "^3.1.0" }, "devDependencies": { + "@types/mocha": "^2.2.48", + "@types/node": "^9.4.7", "async": "^2.6.0", "eslint": "^4.18.2", "llparse-test-fixture": "git://github.com/indutny/llparse-test-fixture.git", - "mocha": "^5.0.4" + "mocha": "^5.0.4", + "ts-node": "^5.0.1", + "tslint": "^5.9.1", + "typescript": "^2.7.2" }, "directories": { "lib": "lib", diff --git a/lib/llparse.js b/src/llparse.ts similarity index 97% rename from lib/llparse.js rename to src/llparse.ts index d363474..64f3631 100644 --- a/lib/llparse.js +++ b/src/llparse.ts @@ -117,9 +117,8 @@ class LLParse { throw new Error(`Unknown property type: "${type}"`); type = internal.constants.USER_TYPES[type]; - const prop = new internal.Property(type, name); props.set.add(name); - props.list.push(prop); + props.list.push({ type, name }); } span(callback) { diff --git a/src/llparse/cases/base.ts b/src/llparse/cases/base.ts new file mode 100644 index 0000000..7cb37ce --- /dev/null +++ b/src/llparse/cases/base.ts @@ -0,0 +1,16 @@ +import * as node from '../node'; +import { Buffer } from 'buffer'; + +export interface ICaseLinearizeResult { + key: Buffer; + next: node.Node; + value: number | undefined; + noAdvance: boolean; +} + +export abstract class Case { + constructor(public readonly type: string, public readonly next: node.Node) { + } + + public abstract linearize(): ICaseLinearizeResult[]; +} diff --git a/src/llparse/cases/index.ts b/src/llparse/cases/index.ts new file mode 100644 index 0000000..07c4ee4 --- /dev/null +++ b/src/llparse/cases/index.ts @@ -0,0 +1,5 @@ +export { ICaseLinearizeResult, Case } from './base'; +export { Match } from './match'; +export { Peek } from './peek'; +export { Select } from './select'; +export { Otherwise } from './otherwise'; diff --git a/src/llparse/cases/match.ts b/src/llparse/cases/match.ts new file mode 100644 index 0000000..1829a8f --- /dev/null +++ b/src/llparse/cases/match.ts @@ -0,0 +1,27 @@ +import * as assert from 'assert'; +import { Buffer } from 'buffer'; + +import * as node from '../node'; +import * as utils from '../utils'; +import { ICaseLinearizeResult, Case } from './case'; + +export class Match extends Case { + public readonly key: Buffer; + + constructor(key: string, next: node.Node) { + super('match', next); + + this.key = utils.toBuffer(key); + assert(this.key.length > 0, + '`.match()` must get at least 1 byte as a first argument'); + } + + public linearize(): ICaseLinearizeResult[] { + return [ { + key: this.key, + next: this.next, + value: undefined, + noAdvance: false + } ]; + } +} diff --git a/src/llparse/cases/otherwise.ts b/src/llparse/cases/otherwise.ts new file mode 100644 index 0000000..b42a131 --- /dev/null +++ b/src/llparse/cases/otherwise.ts @@ -0,0 +1,12 @@ +import * as node from '../node'; +import { ICaseLinearizeResult, Case } from './case'; + +export class Otherwise extends Case { + constructor(next: node.Node, public readonly skip: boolean = false) { + super('otherwise', next); + } + + public linearize(): ICaseLinearizeResult[] { + throw new Error('Should not be called'); + } +} diff --git a/src/llparse/cases/peek.ts b/src/llparse/cases/peek.ts new file mode 100644 index 0000000..1b529e0 --- /dev/null +++ b/src/llparse/cases/peek.ts @@ -0,0 +1,27 @@ +import * as assert from 'assert'; +import { Buffer } from 'buffer'; + +import * as node from '../node'; +import * as utils from '../utils'; +import { Case, ICaseLinearizeResult } from './case'; + +export class Peek extends Case { + public readonly key: Buffer; + + constructor(key: string, next: node.Node) { + super('peek', next); + + this.key = utils.toBuffer(key); + assert.strictEqual(this.key.length, 1, + '`.peek()` must get exactly 1 byte as a first argument'); + } + + public linearize(): ICaseLinearizeResult[] { + return [ { + key: this.key, + next: this.next, + value: undefined, + noAdvance: true + } ]; + } +} diff --git a/src/llparse/cases/select.ts b/src/llparse/cases/select.ts new file mode 100644 index 0000000..a868bf9 --- /dev/null +++ b/src/llparse/cases/select.ts @@ -0,0 +1,39 @@ +import * as assert from 'assert'; +import { Buffer } from 'buffer'; + +import * as node from '../node'; +import * as utils from '../utils'; +import { ICaseLinearizeResult, Case } from './case'; + +export class Select extends Case { + private readonly privMap: Map = new Map(); + + constructor(next) { + super('select', next); + } + + public get map(): ReadonlyMap { + return this.privMap; + } + + public add(key, value): void { + this.privMap.set(key, value); + } + + public linearize(): ICaseLinearizeResult[] { + const res: ICaseLinearizeResult = []; + + this.map.forEach((value, key) => { + const buffer: Buffer = utils.toBuffer(key); + assert(buffer.length > 0, 'Select must have non-empty argument'); + + res.push({ + key: buffer, + next: this.next, + value, + noAdvance: false + }); + }); + return res; + } +} diff --git a/src/llparse/code/base.ts b/src/llparse/code/base.ts new file mode 100644 index 0000000..428b805 --- /dev/null +++ b/src/llparse/code/base.ts @@ -0,0 +1,7 @@ +export type Signature = 'match' | 'value'; + +export abstract class Code { + constructor(public readonly signature: Signature, + public readonly name: string) { + } +} diff --git a/src/llparse/code/field-value.ts b/src/llparse/code/field-value.ts new file mode 100644 index 0000000..0c0db1d --- /dev/null +++ b/src/llparse/code/field-value.ts @@ -0,0 +1,13 @@ +import * as assert from 'assert'; +import { Signature } from './code'; +import { Field } from './field'; + +export class FieldValue extends Field { + constructor(signature: Signature, name: string, field: string, + public readonly value: number) { + super(signature, name, field); + + assert.strictEqual(value, value | 0, + '`value` argument of FieldValue must be an integer'); + } +} diff --git a/src/llparse/code/field.ts b/src/llparse/code/field.ts new file mode 100644 index 0000000..9f932dd --- /dev/null +++ b/src/llparse/code/field.ts @@ -0,0 +1,8 @@ +import { Code, Signature } from './code'; + +export class Field extends Code { + constructor(signature: Signature, name: string, + public readonly field: string) { + super(signature, name + '_' + field); + } +} diff --git a/src/llparse/code/index.ts b/src/llparse/code/index.ts new file mode 100644 index 0000000..b70e72e --- /dev/null +++ b/src/llparse/code/index.ts @@ -0,0 +1,17 @@ +export { Code, Signature } from './base'; +export { Match } from './match'; +export { Value } from './value'; +export { Span } from './span'; + +// Helpers + +export { Field } from './field'; +export { FieldValue } from './field-value'; + +export { Store } from './store'; +export { Load } from './load'; +export { MulAdd } from './mul-add'; +export { Update } from './update'; +export { IsEqual } from './is-equal'; +export { Or } from './or'; +export { Test } from './test'; diff --git a/src/llparse/code/is-equal.ts b/src/llparse/code/is-equal.ts new file mode 100644 index 0000000..91bb957 --- /dev/null +++ b/src/llparse/code/is-equal.ts @@ -0,0 +1,7 @@ +import { FieldValue } from './field-value'; + +export class IsEqual extends FieldValue { + constructor(field: string, value: number) { + super('match', 'is_equal', field, value); + } +} diff --git a/src/llparse/code/load.ts b/src/llparse/code/load.ts new file mode 100644 index 0000000..9f3df2e --- /dev/null +++ b/src/llparse/code/load.ts @@ -0,0 +1,7 @@ +import { Field } from './field'; + +export class Load extends Field { + constructor(field: string) { + super('match', 'load', field); + } +} diff --git a/src/llparse/code/match.ts b/src/llparse/code/match.ts new file mode 100644 index 0000000..4f592f1 --- /dev/null +++ b/src/llparse/code/match.ts @@ -0,0 +1,7 @@ +export { Code } from './code'; + +export class Match extends Code { + constructor(name: string) { + super('match', name); + } +} diff --git a/src/llparse/code/mul-add.ts b/src/llparse/code/mul-add.ts new file mode 100644 index 0000000..cb7bfaf --- /dev/null +++ b/src/llparse/code/mul-add.ts @@ -0,0 +1,40 @@ +import * as assert from 'assert'; + +import { Field } from './field'; + +export interface IMulAddOptions { + base: number; + max?: number; + signed?: boolean; +} + +export interface IMulAddCompleteOptions { + base: number; + max: number; + signed: boolean; +} + +export class MulAdd extends Field { + public readonly options: IMulAddCompleteOptions; + + constructor(field: string, options: IMulAddOptions) { + super('value', 'mul_add', field); + + const complete: IMulAddCompleteOptions = { + base: options.base, + max: options.max === undefined ? 0 : options.max, + signed: options.signed === undefined ? true : options.signed, + }; + + assert(complete.max >= 0, + '`MulAdd.options.max` must be a non-negative number'); + assert(complete.base > 0, + '`MulAdd.options.base` must be a positive number'); + assert.strictEqual(complete.max, complete.max | 0, + '`MulAdd.options.max` must be an integer'); + assert.strictEqual(complete.base, complete.base | 0, + '`MulAdd.options.max` must be an integer'); + + this.options = complete; + } +} diff --git a/src/llparse/code/or.ts b/src/llparse/code/or.ts new file mode 100644 index 0000000..0525a6a --- /dev/null +++ b/src/llparse/code/or.ts @@ -0,0 +1,7 @@ +import { FieldValue } from './field-value'; + +export class Or extends code.FieldValue { + constructor(field: string, value: number) { + super('match', 'or', field, value); + } +} diff --git a/src/llparse/code/span.ts b/src/llparse/code/span.ts new file mode 100644 index 0000000..b97e09e --- /dev/null +++ b/src/llparse/code/span.ts @@ -0,0 +1,5 @@ +import { Match } from './match'; + +export class Span extends Match { + // no-op +} diff --git a/src/llparse/code/store.ts b/src/llparse/code/store.ts new file mode 100644 index 0000000..84abfef --- /dev/null +++ b/src/llparse/code/store.ts @@ -0,0 +1,7 @@ +import { Field } from './field'; + +export class Store extends Field { + constructor(field: string) { + super('value', 'store', field); + } +} diff --git a/src/llparse/code/test.ts b/src/llparse/code/test.ts new file mode 100644 index 0000000..3a7ac26 --- /dev/null +++ b/src/llparse/code/test.ts @@ -0,0 +1,7 @@ +import { FieldValue } from './field-value'; + +export class Test extends FieldValue { + constructor(field: string, mask: number) { + super('match', 'test', field, mask); + } +} diff --git a/src/llparse/code/update.ts b/src/llparse/code/update.ts new file mode 100644 index 0000000..de62476 --- /dev/null +++ b/src/llparse/code/update.ts @@ -0,0 +1,7 @@ +import { FieldValue } from './field-value'; + +export class Update extends FieldValue { + constructor(field: string, value: number) { + super('match', 'update', field, value); + } +} diff --git a/src/llparse/code/value.ts b/src/llparse/code/value.ts new file mode 100644 index 0000000..8000da9 --- /dev/null +++ b/src/llparse/code/value.ts @@ -0,0 +1,7 @@ +import { Code } from './code'; + +export class Value extends Code { + constructor(name: string) { + super('value', name); + } +} diff --git a/lib/llparse/compiler/code/base.js b/src/llparse/compiler/code/base.js similarity index 100% rename from lib/llparse/compiler/code/base.js rename to src/llparse/compiler/code/base.js diff --git a/lib/llparse/compiler/code/index.js b/src/llparse/compiler/code/index.js similarity index 100% rename from lib/llparse/compiler/code/index.js rename to src/llparse/compiler/code/index.js diff --git a/lib/llparse/compiler/code/is-equal.js b/src/llparse/compiler/code/is-equal.js similarity index 100% rename from lib/llparse/compiler/code/is-equal.js rename to src/llparse/compiler/code/is-equal.js diff --git a/lib/llparse/compiler/code/load.js b/src/llparse/compiler/code/load.js similarity index 100% rename from lib/llparse/compiler/code/load.js rename to src/llparse/compiler/code/load.js diff --git a/lib/llparse/compiler/code/match.js b/src/llparse/compiler/code/match.js similarity index 100% rename from lib/llparse/compiler/code/match.js rename to src/llparse/compiler/code/match.js diff --git a/lib/llparse/compiler/code/mul-add.js b/src/llparse/compiler/code/mul-add.js similarity index 100% rename from lib/llparse/compiler/code/mul-add.js rename to src/llparse/compiler/code/mul-add.js diff --git a/lib/llparse/compiler/code/or.js b/src/llparse/compiler/code/or.js similarity index 100% rename from lib/llparse/compiler/code/or.js rename to src/llparse/compiler/code/or.js diff --git a/lib/llparse/compiler/code/span.js b/src/llparse/compiler/code/span.js similarity index 100% rename from lib/llparse/compiler/code/span.js rename to src/llparse/compiler/code/span.js diff --git a/lib/llparse/compiler/code/store.js b/src/llparse/compiler/code/store.js similarity index 100% rename from lib/llparse/compiler/code/store.js rename to src/llparse/compiler/code/store.js diff --git a/lib/llparse/compiler/code/test.js b/src/llparse/compiler/code/test.js similarity index 100% rename from lib/llparse/compiler/code/test.js rename to src/llparse/compiler/code/test.js diff --git a/lib/llparse/compiler/code/update.js b/src/llparse/compiler/code/update.js similarity index 100% rename from lib/llparse/compiler/code/update.js rename to src/llparse/compiler/code/update.js diff --git a/lib/llparse/compiler/code/value.js b/src/llparse/compiler/code/value.js similarity index 100% rename from lib/llparse/compiler/code/value.js rename to src/llparse/compiler/code/value.js diff --git a/lib/llparse/compiler/compilation.js b/src/llparse/compiler/compilation.js similarity index 100% rename from lib/llparse/compiler/compilation.js rename to src/llparse/compiler/compilation.js diff --git a/lib/llparse/compiler/compiler.js b/src/llparse/compiler/compiler.js similarity index 100% rename from lib/llparse/compiler/compiler.js rename to src/llparse/compiler/compiler.js diff --git a/lib/llparse/compiler/index.js b/src/llparse/compiler/index.js similarity index 100% rename from lib/llparse/compiler/index.js rename to src/llparse/compiler/index.js diff --git a/lib/llparse/compiler/node/base.js b/src/llparse/compiler/node/base.js similarity index 100% rename from lib/llparse/compiler/node/base.js rename to src/llparse/compiler/node/base.js diff --git a/lib/llparse/compiler/node/bit-check.js b/src/llparse/compiler/node/bit-check.js similarity index 100% rename from lib/llparse/compiler/node/bit-check.js rename to src/llparse/compiler/node/bit-check.js diff --git a/lib/llparse/compiler/node/consume.js b/src/llparse/compiler/node/consume.js similarity index 100% rename from lib/llparse/compiler/node/consume.js rename to src/llparse/compiler/node/consume.js diff --git a/lib/llparse/compiler/node/empty.js b/src/llparse/compiler/node/empty.js similarity index 100% rename from lib/llparse/compiler/node/empty.js rename to src/llparse/compiler/node/empty.js diff --git a/lib/llparse/compiler/node/error.js b/src/llparse/compiler/node/error.js similarity index 100% rename from lib/llparse/compiler/node/error.js rename to src/llparse/compiler/node/error.js diff --git a/lib/llparse/compiler/node/index.js b/src/llparse/compiler/node/index.js similarity index 100% rename from lib/llparse/compiler/node/index.js rename to src/llparse/compiler/node/index.js diff --git a/lib/llparse/compiler/node/invoke.js b/src/llparse/compiler/node/invoke.js similarity index 100% rename from lib/llparse/compiler/node/invoke.js rename to src/llparse/compiler/node/invoke.js diff --git a/lib/llparse/compiler/node/pause.js b/src/llparse/compiler/node/pause.js similarity index 100% rename from lib/llparse/compiler/node/pause.js rename to src/llparse/compiler/node/pause.js diff --git a/lib/llparse/compiler/node/sequence.js b/src/llparse/compiler/node/sequence.js similarity index 100% rename from lib/llparse/compiler/node/sequence.js rename to src/llparse/compiler/node/sequence.js diff --git a/lib/llparse/compiler/node/single.js b/src/llparse/compiler/node/single.js similarity index 100% rename from lib/llparse/compiler/node/single.js rename to src/llparse/compiler/node/single.js diff --git a/lib/llparse/compiler/node/span-end.js b/src/llparse/compiler/node/span-end.js similarity index 100% rename from lib/llparse/compiler/node/span-end.js rename to src/llparse/compiler/node/span-end.js diff --git a/lib/llparse/compiler/node/span-start.js b/src/llparse/compiler/node/span-start.js similarity index 100% rename from lib/llparse/compiler/node/span-start.js rename to src/llparse/compiler/node/span-start.js diff --git a/lib/llparse/compiler/stage/base.js b/src/llparse/compiler/stage/base.js similarity index 100% rename from lib/llparse/compiler/stage/base.js rename to src/llparse/compiler/stage/base.js diff --git a/lib/llparse/compiler/stage/index.js b/src/llparse/compiler/stage/index.js similarity index 100% rename from lib/llparse/compiler/stage/index.js rename to src/llparse/compiler/stage/index.js diff --git a/lib/llparse/compiler/stage/match-sequence.js b/src/llparse/compiler/stage/match-sequence.js similarity index 100% rename from lib/llparse/compiler/stage/match-sequence.js rename to src/llparse/compiler/stage/match-sequence.js diff --git a/lib/llparse/compiler/stage/node-builder.js b/src/llparse/compiler/stage/node-builder.js similarity index 100% rename from lib/llparse/compiler/stage/node-builder.js rename to src/llparse/compiler/stage/node-builder.js diff --git a/lib/llparse/compiler/stage/node-loop-checker.js b/src/llparse/compiler/stage/node-loop-checker.js similarity index 100% rename from lib/llparse/compiler/stage/node-loop-checker.js rename to src/llparse/compiler/stage/node-loop-checker.js diff --git a/lib/llparse/compiler/stage/node-translator.js b/src/llparse/compiler/stage/node-translator.js similarity index 100% rename from lib/llparse/compiler/stage/node-translator.js rename to src/llparse/compiler/stage/node-translator.js diff --git a/lib/llparse/compiler/stage/span-allocator.js b/src/llparse/compiler/stage/span-allocator.js similarity index 100% rename from lib/llparse/compiler/stage/span-allocator.js rename to src/llparse/compiler/stage/span-allocator.js diff --git a/lib/llparse/compiler/stage/span-builder.js b/src/llparse/compiler/stage/span-builder.js similarity index 100% rename from lib/llparse/compiler/stage/span-builder.js rename to src/llparse/compiler/stage/span-builder.js diff --git a/src/llparse/constants.ts b/src/llparse/constants.ts new file mode 100644 index 0000000..fcc1ea5 --- /dev/null +++ b/src/llparse/constants.ts @@ -0,0 +1,54 @@ +import { Attribute, Builder as IR, CallingConv } from 'bitcode'; + +export const CCONV: CallingConv = 'fastcc'; + +export const BOOL = IR.i(1); +export const INT = IR.i(32); +export const TYPE_INPUT = IR.i(8).ptr(); +export const TYPE_OUTPUT = IR.i(8).ptr(); +export const TYPE_MATCH = export const INT; +export const TYPE_INDEX = IR.i(64); +export const TYPE_ERROR = export const INT; +export const TYPE_REASON = IR.i(8).ptr(); +export const TYPE_DATA = IR.i(8).ptr(); +export const TYPE_STATUS = IR.i(32); + +// TODO(indutny): although it works through zero extension, is there any +// cross-platform way to do it? +export const TYPE_INTPTR = IR.i(64); + +export const ARG_STATE = 's'; +export const ARG_POS = 'p'; +export const ARG_ENDPOS = 'endp'; +export const ARG_SEQUENCE = 'seq'; +export const ARG_SEQUENCE_LEN = 'slen'; +export const ARG_MATCH = 'match'; +export const ARG_UNUSED = '_unused'; + +export const ATTR_STATE: ReadonlyArray = [ 'noalias', 'nonnull' ]; +export const ATTR_POS: ReadonlyArray = [ + noalias', 'nonnull', 'readonly', +]; +export const ATTR_ENDPOS: ReadonlyArray = [ + 'noalias', 'nonnull', 'readnone', +]; +export const ATTR_SEQUENCE = ATTR_POS; + +export const SEQUENCE_COMPLETE = 0; +export const SEQUENCE_PAUSE = 1; +export const SEQUENCE_MISMATCH = 2; + +// NOTE: It is important to start them with `_`, see `lib/llparse.js` (property) +export const SPAN_START_PREFIX = '_span_start'; +export const SPAN_CB_PREFIX = '_span_cb'; + +export const DEFAULT_TRANSLATOR_MIN_CHECK_SIZE = 32; +export const DEFAULT_TRANSLATOR_MAX_CHECK_WIDTH = 4; + +export const USER_TYPES = { + i8: IR.i(8), + i16: IR.i(16), + i32: IR.i(32), + i64: IR.i(64), + ptr: IR.i(8).ptr(), +}; diff --git a/src/llparse/index.ts b/src/llparse/index.ts new file mode 100644 index 0000000..bf6798a --- /dev/null +++ b/src/llparse/index.ts @@ -0,0 +1,11 @@ +import * as cases from './cases'; +import * as code from './code'; +import * as compiler from './compiler'; +import * as constants from './constants'; +import * as node from './node'; +import * as transform from './transform'; +import * as utils from './utils'; + +export { cases, code, compiler, constants, node, transform, utils }; +export { Span } from './span'; +export { Trie } from './trie'; diff --git a/src/llparse/node/base.ts b/src/llparse/node/base.ts new file mode 100644 index 0000000..24c15c7 --- /dev/null +++ b/src/llparse/node/base.ts @@ -0,0 +1,43 @@ +import * as assert from 'assert'; + +import * as cases from '../cases'; +import * as code from '../code'; +import * as node from './'; + +export abstract class Node { + private privOtherwise: Node | undefinded; + + constructor(public readonly name: string, + public readonly signature: code.Signature) { + } + + public get otherwise(): Node | undefined { return this.privOtherwise; } + + public otherwise(next: Node): this { + this.checkIsMatch(next, '.otherwise()'); + + assert.strictEqual(this.privOtherwise, undefined, + 'Duplicate `.otherwise()`/`.skipTo()`'); + this.privOtherwise = new cases.Otherwise(next); + + return this; + } + + public skipTo(next: Node): this { + this.checkIsMatch(next, '.skipTo()'); + + assert.strictEqual(this.privOtherwise, undefined, + 'Duplicate `.skipTo()`/`.otherwise()`'); + this.privOtherwise = new cases.Otherwise(next, true); + + return this; + } + + protected checkIsMatch(next: Node, method: string): void { + if (!(next instanceof node.Invoke)) + return; + + assert.strictEqual(next.signature, 'match', + `Invoke of "${next.name}" can't be a target of \`${method}\``); + } +} diff --git a/src/llparse/node/consume.ts b/src/llparse/node/consume.ts new file mode 100644 index 0000000..b62a0ee --- /dev/null +++ b/src/llparse/node/consume.ts @@ -0,0 +1,10 @@ +import * as assert from 'assert'; +import { Node } from './node'; + +export class Consume extends Node { + constructor(public readonly fieldName: string) { + super('consume_' + fieldName, 'match'); + + assert(!/^_/.test(fieldName), 'Can\'t use internal field name in Consume'); + } +} diff --git a/src/llparse/node/error.ts b/src/llparse/node/error.ts new file mode 100644 index 0000000..acc930a --- /dev/null +++ b/src/llparse/node/error.ts @@ -0,0 +1,15 @@ +import * as assert from 'assert'; + +import { Node } from './node'; + +export class Error extends Node { + constructor(public readonly code: number, public readonly reason: string) { + super('error', 'match'); + + assert.strictEqual(code, code | 0, + 'The first argument of `.error()` must be an integer error code'); + } + + public otherwise(): this { throw new Error('Not supported'); } + public skipTo(): this { throw new Error('Not supported'); } +} diff --git a/src/llparse/node/index.ts b/src/llparse/node/index.ts new file mode 100644 index 0000000..e356902 --- /dev/null +++ b/src/llparse/node/index.ts @@ -0,0 +1,11 @@ +'use strict'; + +export { Node } from './base'; + +export { Match } from './match'; +export { Error } from './error'; +export { Invoke } from './invoke'; +export { SpanStart } from './span-start'; +export { SpanEnd } from './span-end'; +export { Consume } from './consume'; +export { Pause } from './pause'; diff --git a/src/llparse/node/invoke.ts b/src/llparse/node/invoke.ts new file mode 100644 index 0000000..60373a0 --- /dev/null +++ b/src/llparse/node/invoke.ts @@ -0,0 +1,25 @@ +import * as assert from 'assert'; + +import { Code } from '../code'; +import { Node } from './node'; + +export class Invoke extends Node { + public readonly map: ReadonlyMap; + + constructor(public readonly code: Code, + map: { [key: number]: Node }, otherwise?: Node) { + super('invoke_' + code.name, code.signature); + + const storedMap: Map = new Map(): + Object.keys(map).forEach((key) => { + assert.strictEqual(key, key | 0, + 'Only integer keys are allowed in `.invoke()`\'s map'); + + storedMap.set(key, map[key]!); + }); + this.map = storedMap; + + if (otherwise !== undefined) + this.otherwise(otherwise); + } +} diff --git a/src/llparse/node/match.ts b/src/llparse/node/match.ts new file mode 100644 index 0000000..4c269b4 --- /dev/null +++ b/src/llparse/node/match.ts @@ -0,0 +1,91 @@ +import * as assert from 'assert'; + +import { Case, Match, Peek, Select } from '../cases'; +import { Transform } from '../transform'; +import { Invoke } from './invoke'; +import { Node } from './node'; + +export class Match extends Node { + private privTransformation: Transform | undefined; + private privCases: Case[] = []; + + constructor(name: strings) { + super(name, 'match'); + } + + public get transformation(): Transform | undefined { + return this.privTransformation; + } + + public get cases(): ReadonlyArray { + return this.privCases; + } + + public transform(t: Transform): this { + assert.strictEqual(this.privTransformation, undefined, + 'Can\'t apply transform twice'); + + this.privTransformation = t; + return this; + } + + public peek(value: string | ReadonlyArray, next: Node): this { + // .peek([ ... ], next) + if (Array.isArray(value)) { + value.forEach(value => this.peek(value, next)); + return this; + } + + this.checkIsMatch(next, '.peek()'); + + this.privCases.push(new Peek(value, next)); + + return this; + } + + public match(value: string | ReadonlyArray, next: Node): this { + // .match([ ... ], next) + if (Array.isArray(value)) { + value.forEach(value => this.match(value, next)); + return this; + } + + this.checkIsMatch(next, '.match()'); + + this.privCases.push(new Match(value, next)); + + return this; + } + + public select(key: Buffer | number | string | { [key: string] : number }, + value: Invoke | number, next?: Invoke): this { + // .select(key, value, next) + const pairs: { key: string, value: number }[] = []; + + if (Buffer.isBuffer(key) || typeof key === 'number' || + typeof key === 'string') { + assert.strictEqual(typeof value, 'number', + '`.select(key, value, next)` is the signature of the method'); + + pairs.push({ key, value }); + } else { + assert(value instanceof Invoke, + '`.select(map, next)` is the signature of the method'); + + const map = key; + next = value as Invoke; + + Object.keys(map).forEach((key) => pairs.push({ key, value: map[key]! })); + } + + assert.strictEqual(next.signature, 'value', + `Invoke of "${next.name}" can't be a target of \`.select()\``); + + const select = new Select(next); + pairs.forEach(pair => select.add(pair.key, pair.value)); + + this.privCases.push(select); + + return this; + } +} diff --git a/src/llparse/node/pause.ts b/src/llparse/node/pause.ts new file mode 100644 index 0000000..f2edbea --- /dev/null +++ b/src/llparse/node/pause.ts @@ -0,0 +1,16 @@ +import * as assert from 'assert'; + +import { Node } from './node'; + +export class Pause extends Node { + constructor(public readonly code: number, public readonly reason: string) { + super('pause', 'match'); + + assert.strictEqual(code, code | 0, + 'The first argument of `.error()` must be an integer error code'); + } + + public skipTo(): this { + throw new Error('Not supported, please use `pause.otherwise()`'); + } +} diff --git a/src/llparse/node/span-end.ts b/src/llparse/node/span-end.ts new file mode 100644 index 0000000..76c4cef --- /dev/null +++ b/src/llparse/node/span-end.ts @@ -0,0 +1,8 @@ +import { Code } from '../code'; +import { Node } from './node'; + +export class SpanEnd extends Node { + constructor(code: Code) { + super('span_end_' + code.name, 'match'); + } +} diff --git a/src/llparse/node/span-start.ts b/src/llparse/node/span-start.ts new file mode 100644 index 0000000..bbf91a4 --- /dev/null +++ b/src/llparse/node/span-start.ts @@ -0,0 +1,8 @@ +import { Code } from '../code'; +import { Node } from './node'; + +export class SpanStart extends Node { + constructor(code: Code) { + super('span_sta,t_' + code.name, 'match'); + } +} diff --git a/src/llparse/span.ts b/src/llparse/span.ts new file mode 100644 index 0000000..a555266 --- /dev/null +++ b/src/llparse/span.ts @@ -0,0 +1,38 @@ +import * as assert from 'assert'; + +import { Code } from './code'; +import { Node, SpanStart, SpanEnd } from './node'; + +export class Span { + private readonly startCache: Map = new Map(); + private readonly endCache: Map = new Map(); + + constructor(public readonly code: Code) { + } + + public start(otherwise?: Node): SpanStart { + if (otherwise !== undefined && this.startCache.has(otherwise)) { + return this.startCache.get(otherwise)!; + } + + const res = new SpanStart(this.code); + if (otherwise !== undefined) { + res.otherwise(otherwise); + this.startCache.set(otherwise, res); + } + return res; + } + + public end(otherwise?: Node): SpanEnd { + if (otherwise !== undefined && this.endCache.has(otherwise)) { + return this.endCache.get(otherwise)!; + } + + const res = new SpanEnd(this[kCode]); + if (otherwise !== undefined) { + res.otherwise(otherwise); + cache.set(otherwise, res); + } + return res; + } +} diff --git a/src/llparse/transform/base.ts b/src/llparse/transform/base.ts new file mode 100644 index 0000000..84cb444 --- /dev/null +++ b/src/llparse/transform/base.ts @@ -0,0 +1,4 @@ +export class Transform { + constructor(public readonly name: string) { + } +} diff --git a/src/llparse/transform/index.ts b/src/llparse/transform/index.ts new file mode 100644 index 0000000..7342638 --- /dev/null +++ b/src/llparse/transform/index.ts @@ -0,0 +1 @@ +export { Transform } from './base'; diff --git a/src/llparse/trie/index.ts b/src/llparse/trie/index.ts new file mode 100644 index 0000000..e6bdda3 --- /dev/null +++ b/src/llparse/trie/index.ts @@ -0,0 +1,5 @@ +export { TrieNext } from './next'; +export { TrieNode } from './node'; +export { TrieSequence } from './sequence'; +export { TrieSingle } from './single'; +export { Trie } from './trie'; diff --git a/src/llparse/trie/next.ts b/src/llparse/trie/next.ts new file mode 100644 index 0000000..eafe669 --- /dev/null +++ b/src/llparse/trie/next.ts @@ -0,0 +1,9 @@ +import { Node } from '../node'; +import { TrieNode } from './node'; + +export class TrieNext extends TrieNode { + constructor(public readonly value?: number, + public readonly next: Node) { + super('sequence'); + } +} diff --git a/src/llparse/trie/node.ts b/src/llparse/trie/node.ts new file mode 100644 index 0000000..1fa8190 --- /dev/null +++ b/src/llparse/trie/node.ts @@ -0,0 +1,8 @@ +export type NodeType = 'single' | 'sequence' | 'next'; + +export class TrieNode { + public value: number | undefined; + + constructor(public readonly type: NodeType) { + } +} diff --git a/src/llparse/trie/sequence.ts b/src/llparse/trie/sequence.ts new file mode 100644 index 0000000..3e52019 --- /dev/null +++ b/src/llparse/trie/sequence.ts @@ -0,0 +1,10 @@ +import { Buffer } from Buffer; +import { TrieNode } from './node'; + +export class TrieSequence extends TrieNode { + public readonly child: TrieNode; + + constructor(public readonly select: Buffer) { + super('sequence'); + } +} diff --git a/src/llparse/trie/single.ts b/src/llparse/trie/single.ts new file mode 100644 index 0000000..ac12c62 --- /dev/null +++ b/src/llparse/trie/single.ts @@ -0,0 +1,15 @@ +import { TrieNode } from './node'; + +export interface ITrieSingleChild { + child: TrieNode; + key: number; + noAdvance: boolean; +} + +export class TrieSingle extends TrieNode { + public readonly children: ITrieSingleChild[] = []; + + constructor() { + super('single'); + } +} diff --git a/lib/llparse/trie.js b/src/llparse/trie/trie.ts similarity index 61% rename from lib/llparse/trie.js rename to src/llparse/trie/trie.ts index 2f07b69..60a037f 100644 --- a/lib/llparse/trie.js +++ b/src/llparse/trie/trie.ts @@ -1,59 +1,38 @@ -'use strict'; +import * as assert from 'assert'; +import { Buffer } from 'buffer'; -const assert = require('assert'); +import { Case, ICaseLinearizeResult } from '../cases'; +import { TrieNext } from './next'; +import { TrieNode } from './node'; +import { TrieSequence } from './sequence'; +import { TrieSingle } from './single'; -class Node { - constructor(type) { - this.type = type; - this.value = null; - } -} - -class Single extends Node { - constructor() { - super('single'); - this.children = []; - } -} - -class Sequence extends Node { - constructor(select) { - super('sequence'); - this.select = select; - this.child = null; - } -} - -class Next extends Node { - constructor(value, next) { - super('next'); - this.value = value === undefined ? null : value; - this.next = next; - } -} +type LinearizeList = ReadonlyArray; +type Path = ReadonlyArray; -class Trie { - constructor(name) { +export class Trie { + constructor(private readonly name: string) { this.name = name; } - combine(cases) { - const list = []; + public combine(cases: ReadonlyArray): TrieNode | undefined { + const list: ICaseLinearizeResult[] = []; cases.forEach((one) => { one.linearize().forEach(item => list.push(item)); }); - if (list.length === 0) - return null; - - return this.level(list, []); - } + if (list.length === 0) { + return undefined; + } - level(list, path) { list.sort((a, b) => { return a.key.compare(b.key); }); + return this.level(list, []); + } + + private level(list: LinearizeList, path: Path): TrieNode { // TODO(indutny): validate non-empty keys const first = list[0].key; const last = list[list.length - 1].key; @@ -65,23 +44,26 @@ class Trie { if (min === 0) { assert.strictEqual(list.length, 1, `Duplicate entries in "${this.name}" at [ ${path.join(', ')} ]`); - return new Next(list[0].value, list[0].next); + return new TrieNext(list[0].value, list[0].next); } // Find the longest common sub-string - for (; common < min; common++) - if (first[common] !== last[common]) + for (; common < min; common++) { + if (first[common] !== last[common]) { break; + } + } // Sequence - if (common > 1) + if (common > 1) { return this.sequence(list, first.slice(0, common), path); + } // Single return this.single(list, path); } - slice(list, off) { + private slice(list: LinearizeList, off: number): LinearizeList { return list.map((item) => { return { key: item.key.slice(off), @@ -92,8 +74,9 @@ class Trie { }); } - sequence(list, prefix, path) { - const res = new Sequence(prefix); + private sequence(list: LinearizeList, prefix: Buffer, path: Path) + : TrieSequence { + const res = new TrieSequence(prefix); const sliced = this.slice(list, prefix.length); const noAdvance = sliced.some(item => item.noAdvance); @@ -103,8 +86,8 @@ class Trie { return res; } - single(list, path) { - const keys = new Map(); + private single(list: LinearizeList, path: Path): TrieSingle { + const keys: Map = new Map(); for (let i = 0; i < list.length; i++) { const item = list[i]; const key = item.key[0]; @@ -115,7 +98,7 @@ class Trie { keys.set(key, [ item ]); } - const res = new Single(); + const res = new TrieSingle(); keys.forEach((sublist, key) => { const sliced = this.slice(sublist, 1); const subpath = path.concat(key); @@ -137,4 +120,3 @@ class Trie { return res; } } -module.exports = Trie; diff --git a/lib/llparse/utils.js b/src/llparse/utils.ts similarity index 58% rename from lib/llparse/utils.js rename to src/llparse/utils.ts index 9b0e8b2..4722589 100644 --- a/lib/llparse/utils.js +++ b/src/llparse/utils.ts @@ -1,25 +1,38 @@ -'use strict'; +import * as assert from 'assert'; +import { Buffer } from 'buffer'; -const assert = require('assert'); -const Buffer = require('buffer').Buffer; +import { Node } from './node'; -exports.toBuffer = (value) => { +export function toBuffer(value: number | string): Buffer { if (typeof value === 'number') { - assert.strictEqual(value, value >>> 0, - 'Invalid char value, must be integer'); assert(0 <= value <= 255, 'Invalid char value, must be between 0 and 255'); - + assert.strictEqual(value, value | 0, 'Invalid char value, must be integer'); return Buffer.from([ value ]); } else { - assert.strictEqual(typeof value, 'string', - 'Invalid value for a Buffer'); return Buffer.from(value); } -}; +} + +export function powerOfTwo(num: number): number { + return Math.pow(2, Math.ceil(Math.log2(num))); +} + +export interface ILookupListEntry { + node: Node; + noAdvance: boolean; + keys: ReadonlyArray; +} -exports.powerOfTwo = num => Math.pow(2, Math.ceil(Math.log2(num))); +export interface ILookupResult { + indexShift: number; + shiftMask: number; + shiftMul: number; + table: ReadonlyArray; + valueMask: number; +} -exports.buildLookupTable = (wordWidth, charWidth, list) => { +export function buildLookupTable(wordWidth: number, charWidth: number, + list: Readonly): ILookupTableResult { // Entry values: // 0 - no hit // 1 - map[0] @@ -57,4 +70,4 @@ exports.buildLookupTable = (wordWidth, charWidth, list) => { shiftMul, valueMask: (1 << width) - 1 }; -}; +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..4c60f42 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "strict": true, + "target": "es2017", + "module": "commonjs", + "outDir": "./lib", + "declaration": true, + "pretty": true, + "sourceMap": true + }, + "include": [ + "src/**/*.ts" + ] +} diff --git a/tslint.json b/tslint.json new file mode 100644 index 0000000..b0aaf97 --- /dev/null +++ b/tslint.json @@ -0,0 +1,14 @@ +{ + "defaultSeverity": "error", + "extends": [ + "tslint:recommended" + ], + "jsRules": {}, + "rules": { + "no-bitwise": null, + "quotemark": [ + true, "single", "avoid-escape", "avoid-template" + ] + }, + "rulesDirectory": [] +} From f03945e91c68bb9b05ced02c9834b0445b754ab2 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 15:40:59 -0400 Subject: [PATCH 275/281] api --- src/api/code.ts | 47 +++++++++++ src/api/index.ts | 2 + src/api/transform.ts | 7 ++ src/llparse.ts | 165 ++++++++++++-------------------------- src/llparse/code/index.ts | 2 +- 5 files changed, 110 insertions(+), 113 deletions(-) create mode 100644 src/api/code.ts create mode 100644 src/api/index.ts create mode 100644 src/api/transform.ts diff --git a/src/api/code.ts b/src/api/code.ts new file mode 100644 index 0000000..84ea0da --- /dev/null +++ b/src/api/code.ts @@ -0,0 +1,47 @@ +import { code } from '../llparse'; + +export class CodeAPI { + // TODO(indutny): should we allow custom bodies here? + match(name: string): code.Match { + return new code.Match(name); + } + + // TODO(indutny): should we allow custom bodies here? + value(name: string): code.Value { + return new code.Value(name); + } + + span(name: string): code.Span { + return new code.Span(name); + } + + // Helpers + + store(field: string): code.Store { + return new code.Store(field); + } + + load(field: string): code.Load { + return new code.Load(field); + } + + mulAdd(field: string, options: code.IMulAddOptions): code.MulAdd { + return new code.MulAdd(field, options); + } + + update(field: string, value: number): code.Update { + return new code.Update(field, value); + } + + isEqual(field: string, value: number): code.IsEqual { + return new code.IsEqual(field, value); + } + + or(field: string, value: number): code.Or { + return new code.Or(field, value); + } + + test(field: string, value: number): code.Test { + return new code.Test(field, value); + } +} diff --git a/src/api/index.ts b/src/api/index.ts new file mode 100644 index 0000000..77006fe --- /dev/null +++ b/src/api/index.ts @@ -0,0 +1,2 @@ +export { CodeAPI } from './code'; +export { TransformAPI } from './transform'; diff --git a/src/api/transform.ts b/src/api/transform.ts new file mode 100644 index 0000000..4b0ea5f --- /dev/null +++ b/src/api/transform.ts @@ -0,0 +1,7 @@ +import { transform } from '../internal'; + +export class TransformAPI { + public toLowerUnsafe(): transform.Transform { + return new transform.Transform('to_lower_unsafe'); + } +} diff --git a/src/llparse.ts b/src/llparse.ts index 64f3631..c06b848 100644 --- a/src/llparse.ts +++ b/src/llparse.ts @@ -1,151 +1,92 @@ -'use strict'; +import * as assert from 'assert'; -const assert = require('assert'); +import { TransformAPI, CodeAPI } from './api'; +import * as internal from './llparse/'; -const internal = require('./llparse/'); - -const kCode = Symbol('code'); -const kTransform = Symbol('transform'); -const kPrefix = Symbol('prefix'); -const kProperties = Symbol('properties'); - -// API, really - -class CodeAPI { - constructor(prefix) { - this[kPrefix] = prefix; - } - - // TODO(indutny): should we allow custom bodies here? - match(name) { - return new internal.code.Match(name, null); - } - - // TODO(indutny): should we allow custom bodies here? - value(name) { - return new internal.code.Value(name, null); - } - - span(name) { - return new internal.code.Span(name, null); - } - - // Helpers - - store(field) { - return new internal.code.Store(field); - } - - load(field) { - return new internal.code.Load(field); - } - - mulAdd(field, options) { - return new internal.code.MulAdd(field, options); - } - - update(field, value) { - return new internal.code.Update(field, value); - } - - isEqual(field, value) { - return new internal.code.IsEqual(field, value); - } - - or(field, value) { - return new internal.code.Or(field, value); - } - - test(field, value) { - return new internal.code.Test(field, value); - } +interface IProperty { + ty: string; + name: string; } -class TransformAPI { - toLowerUnsafe() { - return new internal.transform.Transform('to_lower_unsafe'); - } +export interface ILLParseOptions { + debug?: boolean; } -class LLParse { - constructor(prefix) { - this[kPrefix] = prefix || 'llparse'; +export class LLParse { + public readonly code = new CodeAPI(); + public readonly transform = new TransformAPI(); + private readonly properties: { set: Set, list: IProperty[] }; - this[kCode] = new CodeAPI(this[kPrefix]); - this[kTransform] = new TransformAPI(); + public static create(public readonly prefix: string): LLparse { + return new LLParse(prefix); + } - this[kProperties] = { + constructor(private readonly prefix: string = 'llparse') { + this.properties = { set: new Set(), list: [] }; } - static create(prefix) { - return new LLParse(prefix); + public node(name: string): internal.node.Match { + return new internal.node.Match(name); } - get prefix() { return this[kPrefix]; } - get code() { return this[kCode]; } - get transform() { return this[kTransform]; } + public error(code: number, reason: string): internal.node.Error { + return new internal.node.Error(code, reason); + } - node(name) { - return new internal.node.Match(name); + public invoke(code: internal.code.Code, + map: { [key: number]: Node } | Node | undefined, + otherwise?: Node): internal.node.Invoke { + if (map === undefined) { + return new internal.node.Invoke(code, {}); + } else if (map instanceof Node) { + return new internal.node.Invoke(code, {}, map); + } else { + return new internal.node.Invoke(code, map, otherwise); + } } - error(code, reason) { - return new internal.node.Error(code, reason); + public span(callback: internal.Code): internal.node.Span { + return new internal.Span(callback); } - invoke(name, map, otherwise) { - return new internal.node.Invoke(name, map, otherwise); + public consume(field: string): internal.node.Consume { + return new internal.node.Consume(field); } - property(type, name) { - assert.strictEqual(typeof type, 'string', - 'The first argument of `.property()` must be a type name'); - assert.strictEqual(typeof name, 'string', - 'The second argument of `.property()` must be a property name'); + public pause(code: number, reason: string): internal.node.Pause { + return new internal.node.Pause(code, reason); + } - if (/^_/.test(name)) + public property(ty: string, name: string): void { + if (/^_/.test(name)) { throw new Error(`Can't use internal property name: "${name}"`); + } - const props = this[kProperties]; - if (props.set.has(name)) + if (this.properties.set.has(name)) { throw new Error(`Duplicate property with a name: "${name}"`); + } - if (!internal.constants.USER_TYPES.hasOwnProperty(type)) - throw new Error(`Unknown property type: "${type}"`); - type = internal.constants.USER_TYPES[type]; + if (!internal.constants.USER_TYPES.hasOwnProperty(ty)) { + throw new Error(`Unknown property type: "${ty}"`); + } + const bitcodeTy = internal.constants.USER_TYPES[ty]; props.set.add(name); - props.list.push({ type, name }); - } - - span(callback) { - return new internal.Span(callback); + props.list.push({ ty: bitcodeTy, name }); } - consume(code) { - return new internal.node.Consume(code); - } - - pause(code, reason) { - return new internal.node.Pause(code, reason); - } - - build(root, options) { - assert(root, 'Missing required argument for `.build(root)`'); - assert(root instanceof internal.node.Node, - 'Invalid value of `root` in `.build(root)'); - + public build(root: internal.node.Node, options?: ILLParseOptions) + : internal.compiler.IBuildResult { options = options || {}; const c = new internal.compiler.Compiler({ prefix: this.prefix, - properties: this[kProperties].list, - debug: options.debug || false + properties: this.properties.list, + debug: options.debug === undefined ? false : options.debug }); return c.build(root); } } -module.exports = LLParse; diff --git a/src/llparse/code/index.ts b/src/llparse/code/index.ts index b70e72e..2a7f452 100644 --- a/src/llparse/code/index.ts +++ b/src/llparse/code/index.ts @@ -10,7 +10,7 @@ export { FieldValue } from './field-value'; export { Store } from './store'; export { Load } from './load'; -export { MulAdd } from './mul-add'; +export { IMulAddOptions, MulAdd } from './mul-add'; export { Update } from './update'; export { IsEqual } from './is-equal'; export { Or } from './or'; From 5a1e39a7a54ce3dcddf855e20246abcc0d4dad74 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 16:18:15 -0400 Subject: [PATCH 276/281] save --- package-lock.json | 8 +-- package.json | 2 +- src/llparse.ts | 13 ++--- src/llparse/compiler/code/base.js | 36 ------------ src/llparse/compiler/code/base.ts | 41 +++++++++++++ src/llparse/compiler/code/index.js | 19 ------- src/llparse/compiler/code/index.ts | 17 ++++++ .../code/{is-equal.js => is-equal.ts} | 17 +++--- src/llparse/compiler/code/load.js | 23 -------- src/llparse/compiler/code/load.ts | 20 +++++++ src/llparse/compiler/code/match.js | 17 ------ src/llparse/compiler/code/match.ts | 15 +++++ .../compiler/code/{mul-add.js => mul-add.ts} | 24 ++++---- src/llparse/compiler/code/{or.js => or.ts} | 18 +++--- src/llparse/compiler/code/span.js | 17 ------ src/llparse/compiler/code/span.ts | 15 +++++ .../compiler/code/{store.js => store.ts} | 15 ++--- .../compiler/code/{test.js => test.ts} | 17 +++--- src/llparse/compiler/code/update.js | 25 -------- src/llparse/compiler/code/update.ts | 22 +++++++ src/llparse/compiler/code/value.js | 17 ------ src/llparse/compiler/code/value.ts | 15 +++++ .../compiler/{compiler.js => compiler.ts} | 57 ++++++++++++------- src/llparse/compiler/index.js | 9 --- src/llparse/compiler/index.ts | 9 +++ 25 files changed, 237 insertions(+), 251 deletions(-) delete mode 100644 src/llparse/compiler/code/base.js create mode 100644 src/llparse/compiler/code/base.ts delete mode 100644 src/llparse/compiler/code/index.js create mode 100644 src/llparse/compiler/code/index.ts rename src/llparse/compiler/code/{is-equal.js => is-equal.ts} (51%) delete mode 100644 src/llparse/compiler/code/load.js create mode 100644 src/llparse/compiler/code/load.ts delete mode 100644 src/llparse/compiler/code/match.js create mode 100644 src/llparse/compiler/code/match.ts rename src/llparse/compiler/code/{mul-add.js => mul-add.ts} (84%) rename src/llparse/compiler/code/{or.js => or.ts} (53%) delete mode 100644 src/llparse/compiler/code/span.js create mode 100644 src/llparse/compiler/code/span.ts rename src/llparse/compiler/code/{store.js => store.ts} (54%) rename src/llparse/compiler/code/{test.js => test.ts} (56%) delete mode 100644 src/llparse/compiler/code/update.js create mode 100644 src/llparse/compiler/code/update.ts delete mode 100644 src/llparse/compiler/code/value.js create mode 100644 src/llparse/compiler/code/value.ts rename src/llparse/compiler/{compiler.js => compiler.ts} (78%) delete mode 100644 src/llparse/compiler/index.js create mode 100644 src/llparse/compiler/index.ts diff --git a/package-lock.json b/package-lock.json index 8b2517b..606116e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -156,9 +156,9 @@ "dev": true }, "bitcode": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.0.4.tgz", - "integrity": "sha512-9cSlrUsQK0E0DJ8MRTHkzvlYNkQvGu9D7onwvGzaIc0jTj+9Wgu9xvadof+zhhDPeAevf9AR49YK+O5pPY+xOA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/bitcode/-/bitcode-1.1.0.tgz", + "integrity": "sha512-uA2u1LsyaucwRM5Rlns/ydURJ4lcaGK4PJ+XaXSFu7VcUA8obwPvYglRNlHarg0E26Xnq7rE68Arqs9plexpag==", "requires": { "bitcode-builder": "1.1.5" } @@ -769,7 +769,7 @@ } }, "llparse-test-fixture": { - "version": "git://github.com/indutny/llparse-test-fixture.git#e76dace1a2b101f84a73231c80d937ba16ec95a5", + "version": "git://github.com/indutny/llparse-test-fixture.git#93c555d7cfced42ad7bf531bf981a9ca5ead66e8", "dev": true, "requires": { "async": "2.6.0" diff --git a/package.json b/package.json index a1dd57d..ff063cd 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "author": "Fedor Indutny (http://darksi.de/)", "license": "MIT", "dependencies": { - "bitcode": "^1.0.4", + "bitcode": "^1.1.0", "debug": "^3.1.0" }, "devDependencies": { diff --git a/src/llparse.ts b/src/llparse.ts index c06b848..fadafd9 100644 --- a/src/llparse.ts +++ b/src/llparse.ts @@ -1,13 +1,9 @@ import * as assert from 'assert'; +import { builder } from 'bitcode'; import { TransformAPI, CodeAPI } from './api'; import * as internal from './llparse/'; -interface IProperty { - ty: string; - name: string; -} - export interface ILLParseOptions { debug?: boolean; } @@ -15,7 +11,10 @@ export interface ILLParseOptions { export class LLParse { public readonly code = new CodeAPI(); public readonly transform = new TransformAPI(); - private readonly properties: { set: Set, list: IProperty[] }; + private readonly properties: { + set: Set, + list: internal.compiler.ICompilerStateProperty[], + }; public static create(public readonly prefix: string): LLparse { return new LLParse(prefix); @@ -79,7 +78,7 @@ export class LLParse { } public build(root: internal.node.Node, options?: ILLParseOptions) - : internal.compiler.IBuildResult { + : internal.compiler.ICompilerBuildResult { options = options || {}; const c = new internal.compiler.Compiler({ diff --git a/src/llparse/compiler/code/base.js b/src/llparse/compiler/code/base.js deleted file mode 100644 index 0e05fc6..0000000 --- a/src/llparse/compiler/code/base.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -class Code { - constructor(type, signature, name) { - this.type = type; - this.signature = signature; - this.name = name; - - this.isExternal = false; - this.cacheKey = this; - } - - // Just for cache key generation - numKey(num) { - if (num < 0) - return 'm' + (-num); - else - return num.toString(); - } - - getTypes(ctx, fn, field) { - const fieldType = ctx.state.lookupField(field).ty; - const returnType = fn.ty.toSignature().returnType; - assert(fieldType.isInt(), `"${field}" field is not of integer type`); - assert(returnType.isInt()); - - return { fieldType, returnType }; - } - - build() { - throw new Error('Not implemented'); - } -} -module.exports = Code; diff --git a/src/llparse/compiler/code/base.ts b/src/llparse/compiler/code/base.ts new file mode 100644 index 0000000..f18a469 --- /dev/null +++ b/src/llparse/compiler/code/base.ts @@ -0,0 +1,41 @@ +import * as assert from 'assert'; + +import { Signature } from '../../code'; +import { Compilation, types, values } from '../compilation'; + +import Func = values.constants.Func; + +// Just as a convenience +export { Func }; + +export abstract class Code { + protected privIsExternal: boolean = false; + protected privCacheKey: any = this; + + constructor(public readonly kind: string, + public readonly signature: Signature, + public readonly name: string) { + } + + public get isExternal(): boolean { return this.privIsExternal; } + public get cacheKey(): any { return this.privCacheKey; } + + public abstract build(): void; + + // Just for cache key generation + protected numKey(num): string { + if (num < 0) + return 'm' + (-num); + else + return num.toString(); + } + + protected getTypes(ctx: Compilation, fn: Func, field: string) { + const fieldType = ctx.state.lookupField(field).ty; + const returnType = fn.ty.toSignature().returnType; + assert(fieldType.isInt(), `"${field}" field is not of integer type`); + assert(returnType.isInt()); + + return { fieldType, returnType }; + } +} diff --git a/src/llparse/compiler/code/index.js b/src/llparse/compiler/code/index.js deleted file mode 100644 index 60ec343..0000000 --- a/src/llparse/compiler/code/index.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -exports.Code = require('./base'); - -// External - -exports.Match = require('./match'); -exports.Value = require('./value'); -exports.Span = require('./span'); - -// Internal - -exports.IsEqual = require('./is-equal'); -exports.Load = require('./load'); -exports.MulAdd = require('./mul-add'); -exports.Or = require('./or'); -exports.Store = require('./store'); -exports.Test = require('./test'); -exports.Update = require('./update'); diff --git a/src/llparse/compiler/code/index.ts b/src/llparse/compiler/code/index.ts new file mode 100644 index 0000000..4824c64 --- /dev/null +++ b/src/llparse/compiler/code/index.ts @@ -0,0 +1,17 @@ +export { Code } from './base'; + +// External + +export { Match } from './match'; +export { Value } from './value'; +export { Span } from './span'; + +// Internal + +export { IsEqual } from './is-equal'; +export { Load } from './load'; +export { MulAdd } from './mul-add'; +export { Or } from './or'; +export { Store } from './store'; +export { Test } from './test'; +export { Update } from './update'; diff --git a/src/llparse/compiler/code/is-equal.js b/src/llparse/compiler/code/is-equal.ts similarity index 51% rename from src/llparse/compiler/code/is-equal.js rename to src/llparse/compiler/code/is-equal.ts index ce1b417..78da7a6 100644 --- a/src/llparse/compiler/code/is-equal.js +++ b/src/llparse/compiler/code/is-equal.ts @@ -1,17 +1,15 @@ -'use strict'; +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; -const code = require('./'); - -class IsEqual extends code.Code { - constructor(name, field, value) { +class IsEqual extends Code { + constructor(name: string, private readonly field: string, + private readonly value: number) { super('is-equal', 'match', name); - this.field = field; - this.value = value; - this.cacheKey = `is_equal_${this.field}_${this.numKey(this.value)}`; + this.privCacheKey = `is_equal_${this.field}_${this.numKey(this.value)}`; } - build(ctx, fn) { + public build(ctx: Compilation, fn: Func): void { const body = fn.body; const field = this.field; const value = this.value; @@ -23,4 +21,3 @@ class IsEqual extends code.Code { body.ret(ctx.truncate(body, cmp, returnType)); } } -module.exports = IsEqual; diff --git a/src/llparse/compiler/code/load.js b/src/llparse/compiler/code/load.js deleted file mode 100644 index ca310dc..0000000 --- a/src/llparse/compiler/code/load.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Load extends code.Code { - constructor(name, field) { - super('load', 'match', name); - - this.field = field; - this.cacheKey = `load_${this.field}`; - } - - build(ctx, fn) { - const body = fn.body; - const field = this.field; - - const { returnType } = this.getTypes(ctx, fn, field); - - const adj = ctx.truncate(body, ctx.load(fn, body, field), returnType); - body.ret(adj); - } -} -module.exports = Load; diff --git a/src/llparse/compiler/code/load.ts b/src/llparse/compiler/code/load.ts new file mode 100644 index 0000000..dbf3570 --- /dev/null +++ b/src/llparse/compiler/code/load.ts @@ -0,0 +1,20 @@ +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; + +export class Load extends Code { + constructor(name: string, private readonly field: string) { + super('load', 'match', name); + + this.privCacheKey = `load_${this.field}`; + } + + public build(ctx: Compilation, fn: Func): void { + const body = fn.body; + const field = this.field; + + const { returnType } = this.getTypes(ctx, fn, field); + + const adj = ctx.truncate(body, ctx.load(fn, body, field), returnType); + body.ret(adj); + } +} diff --git a/src/llparse/compiler/code/match.js b/src/llparse/compiler/code/match.js deleted file mode 100644 index 75de9b9..0000000 --- a/src/llparse/compiler/code/match.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Match extends code.Code { - constructor(name) { - super('match', 'match', name); - - this.isExternal = true; - this.cacheKey = 'external_' + name; - } - - build() { - throw new Error('External code can\'t be built'); - } -} -module.exports = Match; diff --git a/src/llparse/compiler/code/match.ts b/src/llparse/compiler/code/match.ts new file mode 100644 index 0000000..80a699c --- /dev/null +++ b/src/llparse/compiler/code/match.ts @@ -0,0 +1,15 @@ +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; + +class Match extends Code { + constructor(name: string) { + super('match', 'match', name); + + this.privIsExternal = true; + this.privCacheKey = 'external_' + name; + } + + public build(ctx: Compilation, fn: Func): void { + throw new Error('External code can\'t be built'); + } +} diff --git a/src/llparse/compiler/code/mul-add.js b/src/llparse/compiler/code/mul-add.ts similarity index 84% rename from src/llparse/compiler/code/mul-add.js rename to src/llparse/compiler/code/mul-add.ts index a4a99f5..427c61b 100644 --- a/src/llparse/compiler/code/mul-add.js +++ b/src/llparse/compiler/code/mul-add.ts @@ -1,20 +1,17 @@ -'use strict'; - -const code = require('./'); -const llparse = require('../../'); - -const BOOL = llparse.constants.BOOL; - -class MulAdd extends code.Code { - constructor(name, field, options) { +import { IMulAddOptions } from '../../code'; +import { BOOL } from '../../constants'; +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; + +export class MulAdd extends Code { + constructor(name: string, private readonly field: string, + private readonly options: IMulAddOptions) { super('mulAdd', 'value', name); - this.field = field; - this.options = options; - this.cacheKey = `mul_add_${this.field}_${JSON.stringify(this.options)}`; + this.privCacheKey = `mul_add_${this.field}_${JSON.stringify(this.options)}`; } - build(ctx, fn) { + public build(ctx: Compilation, fn: Func): void { const ir = ctx.ir; const body = fn.body; const match = ctx.matchArg(fn); @@ -94,4 +91,3 @@ class MulAdd extends code.Code { store.ret(returnType.val(0)); } } -module.exports = MulAdd; diff --git a/src/llparse/compiler/code/or.js b/src/llparse/compiler/code/or.ts similarity index 53% rename from src/llparse/compiler/code/or.js rename to src/llparse/compiler/code/or.ts index b13a65b..1d70676 100644 --- a/src/llparse/compiler/code/or.js +++ b/src/llparse/compiler/code/or.ts @@ -1,17 +1,14 @@ -'use strict'; +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; -const code = require('./'); - -class Or extends code.Code { - constructor(name, field, value) { +export class Or extends Code { + constructor(name, private readonly field: string, + private readonly value: number) { super('or', 'match', name); - - this.field = field; - this.value = value; - this.cacheKey = `or_${this.field}_${this.numKey(this.value)}`; + this.privCacheKey = `or_${this.field}_${this.numKey(this.value)}`; } - build(ctx, fn) { + public build(ctx: Compilation, fn: Func): void { const body = fn.body; const field = this.field; const value = this.value; @@ -24,4 +21,3 @@ class Or extends code.Code { body.ret(returnType.val(0)); } } -module.exports = Or; diff --git a/src/llparse/compiler/code/span.js b/src/llparse/compiler/code/span.js deleted file mode 100644 index 788f034..0000000 --- a/src/llparse/compiler/code/span.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Span extends code.Code { - constructor(name) { - super('span', 'match', name); - - this.isExternal = true; - this.cacheKey = 'external_' + name; - } - - build() { - throw new Error('External code can\'t be built'); - } -} -module.exports = Span; diff --git a/src/llparse/compiler/code/span.ts b/src/llparse/compiler/code/span.ts new file mode 100644 index 0000000..67143df --- /dev/null +++ b/src/llparse/compiler/code/span.ts @@ -0,0 +1,15 @@ +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; + +export class Span extends Code { + constructor(name: string) { + super('span', 'match', name); + + this.privIsExternal = true; + this.privCacheKey = 'external_' + name; + } + + public build(ctx: Compilation, fn: Func): void { + throw new Error('External code can\'t be built'); + } +} diff --git a/src/llparse/compiler/code/store.js b/src/llparse/compiler/code/store.ts similarity index 54% rename from src/llparse/compiler/code/store.js rename to src/llparse/compiler/code/store.ts index c7f3146..cf1d438 100644 --- a/src/llparse/compiler/code/store.js +++ b/src/llparse/compiler/code/store.ts @@ -1,16 +1,14 @@ -'use strict'; +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; -const code = require('./'); - -class Store extends code.Code { - constructor(name, field) { +export class Store extends Code { + constructor(name: string, private readonly field: string) { super('store', 'value', name); - this.field = field; - this.cacheKey = `store_${this.field}`; + this.privCacheKey = `store_${this.field}`; } - build(ctx, fn) { + public build(ctx: Compilation, fn: Func): void { const body = fn.body; const field = this.field; @@ -24,4 +22,3 @@ class Store extends code.Code { body.ret(returnType.val(0)); } } -module.exports = Store; diff --git a/src/llparse/compiler/code/test.js b/src/llparse/compiler/code/test.ts similarity index 56% rename from src/llparse/compiler/code/test.js rename to src/llparse/compiler/code/test.ts index 0e07d41..efa5729 100644 --- a/src/llparse/compiler/code/test.js +++ b/src/llparse/compiler/code/test.ts @@ -1,17 +1,15 @@ -'use strict'; +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; -const code = require('./'); - -class Test extends code.Code { - constructor(name, field, value) { +class Test extends Code { + constructor(name: string, private readonly field: string, + private readonly value: string) { super('test', 'match', name); - this.field = field; - this.value = value; - this.cacheKey = `test_${this.field}_${this.numKey(this.value)}`; + this.privCacheKey = `test_${this.field}_${this.numKey(this.value)}`; } - build(ctx, fn) { + build(ctx: Compilation, fn: Func): void { const body = fn.body; const field = this.field; const value = this.value; @@ -26,4 +24,3 @@ class Test extends code.Code { body.ret(ctx.truncate(body, bool, returnType)); } } -module.exports = Test; diff --git a/src/llparse/compiler/code/update.js b/src/llparse/compiler/code/update.js deleted file mode 100644 index 0488cf2..0000000 --- a/src/llparse/compiler/code/update.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Update extends code.Code { - constructor(name, field, value) { - super('update', 'match', name); - - this.field = field; - this.value = value; - this.cacheKey = `update_${this.field}_${this.numKey(this.value)}`; - } - - build(ctx, fn) { - const body = fn.body; - const field = this.field; - const value = this.value; - - const { fieldType, returnType } = this.getTypes(ctx, fn, field); - - ctx.store(fn, body, field, fieldType.val(value)); - body.ret(returnType.val(0)); - } -} -module.exports = Update; diff --git a/src/llparse/compiler/code/update.ts b/src/llparse/compiler/code/update.ts new file mode 100644 index 0000000..be18958 --- /dev/null +++ b/src/llparse/compiler/code/update.ts @@ -0,0 +1,22 @@ +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; + +class Update extends Code { + constructor(name: string, private readonly field: string, + private readonly value: number) { + super('update', 'match', name); + + this.privCacheKey = `update_${this.field}_${this.numKey(this.value)}`; + } + + public build(ctx: Compilation, fn: Func): void { + const body = fn.body; + const field = this.field; + const value = this.value; + + const { fieldType, returnType } = this.getTypes(ctx, fn, field); + + ctx.store(fn, body, field, fieldType.val(value)); + body.ret(returnType.val(0)); + } +} diff --git a/src/llparse/compiler/code/value.js b/src/llparse/compiler/code/value.js deleted file mode 100644 index bcf63ce..0000000 --- a/src/llparse/compiler/code/value.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict'; - -const code = require('./'); - -class Value extends code.Code { - constructor(name) { - super('value', 'value', name); - - this.isExternal = true; - this.cacheKey = 'external_' + name; - } - - build() { - throw new Error('External code can\'t be built'); - } -} -module.exports = Value; diff --git a/src/llparse/compiler/code/value.ts b/src/llparse/compiler/code/value.ts new file mode 100644 index 0000000..b86d167 --- /dev/null +++ b/src/llparse/compiler/code/value.ts @@ -0,0 +1,15 @@ +import { Compilation } from '../compilation'; +import { Code, Func } from './base'; + +class Value extends Code { + constructor(name: string) { + super('value', 'value', name); + + this.privIsExternal = true; + this.privCacheKey = 'external_' + name; + } + + public build(ctx: Compilation, fn: Func): void { + throw new Error('External code can\'t be built'); + } +} diff --git a/src/llparse/compiler/compiler.js b/src/llparse/compiler/compiler.ts similarity index 78% rename from src/llparse/compiler/compiler.js rename to src/llparse/compiler/compiler.ts index 86c4c1a..ef61b4b 100644 --- a/src/llparse/compiler/compiler.js +++ b/src/llparse/compiler/compiler.ts @@ -1,7 +1,10 @@ -'use strict'; +import { builder } from 'bitcode'; +import { Buffer } from 'buffer'; -const llparse = require('../'); -const constants = llparse.constants; +import * as constants from '../constants'; +import * as node from '../node'; +import { Compilation } from './compilation'; +import * as stage from './stage'; const TYPE_INPUT = constants.TYPE_INPUT; const TYPE_MATCH = constants.TYPE_MATCH; @@ -15,37 +18,47 @@ const ATTR_STATE = constants.ATTR_STATE; const ATTR_POS = constants.ATTR_POS; const ATTR_ENDPOS = constants.ATTR_ENDPOS; -class Compiler { - constructor(options) { - this.options = Object.assign({}, options); +export interface ICompilerStateProperty { + name: string; + ty: builder.types.Type; +} - this.prefix = this.options.prefix; +export interface ICompilerOptions { + prefix: string; + properties: ReadonlyArray; + debug: options.debug === undefined ? false : options.debug +} - this.nodeMap = new Map(); - this.codeMap = new Map(); - this.counter = new Map(); +export interface ICompilerBuildResult { + bitcode: Buffer; + header: string; +} - // redirect blocks by `fn` and `target` - this.redirectCache = new Map(); +export class Compiler { + private readonly prefix: string; + + constructor(private readonly options: ICompilerOptions) { + this.prefix = this.options.prefix; } - build(root) { + public build(root: node.Node): ICompilerBuildResult { const compOpts = Object.assign({}, this.options, { root, stages: { before: [ - llparse.compiler.stage.NodeTranslator, - llparse.compiler.stage.MatchSequence, - llparse.compiler.stage.NodeLoopChecker, - llparse.compiler.stage.SpanAllocator, - llparse.compiler.stage.SpanBuilder + stage.NodeTranslator, + stage.MatchSequence, + stage.NodeLoopChecker, + stage.SpanAllocator, + stage.SpanBuilder ], after: [ - llparse.compiler.stage.NodeBuilder + stage.NodeBuilder ] } }); - const ctx = new llparse.compiler.Compilation(compOpts); + + const ctx = new Compilation(compOpts); ctx.build(); @@ -77,7 +90,7 @@ class Compiler { return out; } - buildInit(ctx) { + private buildInit(ctx: Compilation): string { const sig = ctx.ir.signature(ctx.ir.void(), [ ctx.state.ptr() ]); const init = ctx.defineFunction(sig, this.prefix + '_init', [ ARG_STATE ]); init.paramAttrs[0].add(ATTR_STATE); @@ -89,7 +102,7 @@ class Compiler { return `void ${this.prefix}_init(${this.prefix}_state_t* s);`; } - buildExecute(ctx) { + private buildExecute(ctx: Compilation): string { // TODO(indutny): change signature to (state*, start*, len)? const sig = ctx.ir.signature(TYPE_ERROR, [ ctx.state.ptr(), diff --git a/src/llparse/compiler/index.js b/src/llparse/compiler/index.js deleted file mode 100644 index 3a70484..0000000 --- a/src/llparse/compiler/index.js +++ /dev/null @@ -1,9 +0,0 @@ -'use strict'; - -exports.node = require('./node'); -exports.stage = require('./stage'); -exports.code = require('./code'); - -exports.Compilation = require('./compilation'); - -exports.Compiler = require('./compiler'); diff --git a/src/llparse/compiler/index.ts b/src/llparse/compiler/index.ts new file mode 100644 index 0000000..319e906 --- /dev/null +++ b/src/llparse/compiler/index.ts @@ -0,0 +1,9 @@ +import * as node from './node'; +import * as stage from './stage'; +import * as code from './code'; + +export { Compilation } from './compilation'; +export { + Compiler, ICompilerBuildResult, ICompilerOptions, ICompilerStateProperty, +} from './compiler'; +export { node, stage, code }; From a112562f2a3c79662bf25695e0b9576e37d1f644 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 16:56:12 -0400 Subject: [PATCH 277/281] save --- src/llparse/compiler/node/base.js | 218 ------------------ src/llparse/compiler/node/base.ts | 187 +++++++++++++++ .../node/{bit-check.js => bit-check.ts} | 9 +- src/llparse/compiler/node/node-context.ts | 67 ++++++ 4 files changed, 257 insertions(+), 224 deletions(-) delete mode 100644 src/llparse/compiler/node/base.js create mode 100644 src/llparse/compiler/node/base.ts rename src/llparse/compiler/node/{bit-check.js => bit-check.ts} (95%) create mode 100644 src/llparse/compiler/node/node-context.ts diff --git a/src/llparse/compiler/node/base.js b/src/llparse/compiler/node/base.js deleted file mode 100644 index d877f32..0000000 --- a/src/llparse/compiler/node/base.js +++ /dev/null @@ -1,218 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const node = require('./'); -const llparse = require('../../'); -const constants = llparse.constants; - -class NodeContext { - constructor(ctx, name, fn, nodes) { - this.compilation = ctx; - this.name = name; - this.fn = fn; - this.nodes = nodes; - - this.state = this.compilation.stateArg(this.fn); - this.pos = { - current: this.compilation.posArg(this.fn), - next: null - }; - this.endPos = this.compilation.endPosArg(this.fn); - this.match = this.compilation.matchArg(this.fn); - - // Re-export some properties - this.ir = this.compilation.ir; - this.signature = this.compilation.signature; - this.stateType = this.compilation.state; - - // Some generally useful types (to avoid unnecessary includes) - this.INT = constants.INT; - this.BOOL = constants.BOOL; - this.TYPE_MATCH = constants.TYPE_MATCH; - this.TYPE_REASON = constants.TYPE_REASON; - this.TYPE_ERROR = constants.TYPE_ERROR; - this.TYPE_INDEX = constants.TYPE_INDEX; - this.TYPE_INTPTR = constants.TYPE_INTPTR; - - this.INVARIANT_GROUP = ctx.INVARIANT_GROUP; - - this.hasPause = false; - } - - debug(body, message) { - this.compilation.debug(this.fn, body, `${this.name}: ${message}`); - } - - // Just proxy - stateField(body, name) { - return this.compilation.stateField(this.fn, body, name); - } - - cstring(...args) { return this.compilation.cstring(...args); } - blob(...args) { return this.compilation.blob(...args); } - addGlobalConst(...args) { return this.compilation.addGlobalConst(...args); } - truncate(...args) { return this.compilation.truncate(...args); } - call(...args) { return this.compilation.call(...args); } - buildSwitch(...args) { return this.compilation.buildSwitch(...args); } - branch(...args) { return this.compilation.branch(...args); } -} - -class Node { - constructor(type, id) { - this.type = type; - this.name = id.name; - this.sourceName = id.sourceName; - this.otherwise = null; - this.skip = false; - this.transform = null; - this.noPrologueCheck = false; - - this.fn = null; - this.phis = new Map(); - } - - setOtherwise(otherwise, skip) { - this.otherwise = otherwise; - this.skip = skip; - } - - getChildren() { - return [ { node: this.otherwise, noAdvance: !this.skip, key: null } ]; - } - - getResumptionTargets() { - if (this.hasPause) - return [ this ]; - else - return []; - } - - // Building - - build(compilation, nodes) { - if (nodes.has(this)) - return nodes.get(this); - - const fn = compilation.fn(compilation.signature.node, this.name); - const ctx = new NodeContext(compilation, this.name, fn, nodes); - - // Errors are assumed to be rarely called - // TODO(indutny): move this to node.Error somehow? - if (this instanceof node.Error) { - fn.attrs.add([ 'norecurse', 'cold', 'writeonly', 'noinline' ]); - } - - nodes.set(this, fn); - - let body = fn.body; - ctx.debug(body, 'enter'); - body = this.prologue(ctx, body); - - ctx.pos.next = body.getelementptr(ctx.pos.current, ctx.INT.val(1)); - - this.doBuild(ctx, body); - - return fn; - } - - prologue(ctx, body) { - if (this.noPrologueCheck) - return body; - - const pos = ctx.pos.current; - const endPos = ctx.endPos; - - // Check that we have enough chars to do the read - const cmp = body.icmp('ne', pos, endPos); - - const branch = ctx.branch(body, cmp); - - // Return self when `pos === endpos` - branch.right.name = 'no_data'; - this.pause(ctx, branch.right); - - branch.left.name = 'has_data'; - return branch.left; - } - - pause(ctx, body) { - const fn = ctx.fn; - const bitcast = body.cast('bitcast', fn, fn.ty.toSignature().returnType); - body.ret(bitcast); - - // To be used in `compiler.js` - this.hasPause = true; - } - - buildNode(ctx, node) { - return node.build(ctx.compilation, ctx.nodes); - } - - tailTo(ctx, body, pos, node, value = null) { - const target = this.buildNode(ctx, node); - - const isCacheable = ctx.pos.next === pos; - - if (isCacheable && this.phis.has(target)) { - const cached = this.phis.get(target); - - if (cached.phi) { - assert(value, '`.match()` and `.select()` with the same target'); - cached.phi.addEdge({ - fromBlock: body, - value: ctx.TYPE_MATCH.val(value) - }); - } else { - assert(!value, '`.match()` and `.select()` with the same target'); - } - - body.jmp(cached.trampoline); - return target; - } - - // Split, so that others could join us from code block above - const trampoline = body.parent.createBlock(body.name + '.trampoline'); - body.jmp(trampoline); - let phi = null; - - // Compute `match` if needed - if (value !== null) { - phi = trampoline.phi({ - fromBlock: body, - value: ctx.TYPE_MATCH.val(value) - }); - } - - if (isCacheable) - this.phis.set(target, { phi, trampoline }); - - const call = trampoline.call(target, [ - ctx.state, - pos, - ctx.endPos, - phi ? phi : ctx.TYPE_MATCH.undef() - ], 'musttail'); - - trampoline.ret(call); - - return target; - } - - doBuild() { - throw new Error('Not implemented'); - } - - doOtherwise(ctx, body, pos) { - if (!pos) - pos = ctx.pos; - - // `.skipTo()` advances by one byte - // `.otherwise()` redirects using the same byte - const next = this.skip ? pos.next : pos.current; - - assert(this.otherwise); - return this.tailTo(ctx, body, next, this.otherwise); - } -} -module.exports = Node; diff --git a/src/llparse/compiler/node/base.ts b/src/llparse/compiler/node/base.ts new file mode 100644 index 0000000..56c4de7 --- /dev/null +++ b/src/llparse/compiler/node/base.ts @@ -0,0 +1,187 @@ +import * as assert from 'assert'; +import { Buffer } from 'buffer'; + +import { Transform } from '../../transform'; +import * as node from './node'; +import { INodePosition, NodeContext } from './node-context'; +import { Compilation, INodeID, values } from './compilation'; + +import BasicBlock = values.BasicBlock; +import Func = values.constants.Func; + +interface ITrampoline { + block: values.BasicBlock; + phi?: values.instructions.Phi; +} + +interface INodeChild { + node: Node; + noAdvance: boolean; + key: Buffer; +} + +export abstract class Node { + public name: string; + public sourceName: string; + + protected otherwise: Node | undefined; + protected skip: boolean = false; + + protected transform: Transform | undefined; + protected noPrologueCheck: boolean = false; + protected hasPause: boolean = false; + private trampolines: Map = new Map(); + + constructor(public readonly kind: string, id: INodeID) { + this.name = id.name; + this.sourceName = id.sourceName; + } + + public setOtherwise(otherwise: Node, skip: boolean) { + this.otherwise = otherwise; + this.skip = skip; + } + + public getChildren(): INodeChild[] { + return [ { node: this.otherwise, noAdvance: !this.skip, key: undefined } ]; + } + + public getResumptionTargets(): Node[] { + if (this.hasPause) { + return [ this ]; + } else { + return []; + } + } + + // Building + + public build(compilation: Compilation, nodes: Map): Func { + if (nodes.has(this)) { + return nodes.get(this)!; + } + + const fn = compilation.fn(compilation.signature.node, this.name); + const ctx = new NodeContext(compilation, this.name, fn, nodes); + + // Errors are assumed to be rarely called + // TODO(indutny): move this to node.Error somehow? + if (this instanceof node.Error) { + fn.attrs.add([ 'norecurse', 'cold', 'writeonly', 'noinline' ]); + } + + nodes.set(this, fn); + + let body = fn.body; + ctx.debug(body, 'enter'); + body = this.prologue(ctx, body); + + ctx.pos.next = body.getelementptr(ctx.pos.current, ctx.INT.val(1)); + + this.doBuild(ctx, body); + + return fn; + } + + protected prologue(ctx: Compilation, body: BasicBlock): BasicBlock { + if (this.noPrologueCheck) { + return body; + } + + const pos = ctx.pos.current; + const endPos = ctx.endPos; + + // Check that we have enough chars to do the read + const cmp = body.icmp('ne', pos, endPos); + + const branch = ctx.branch(body, cmp); + + // Return self when `pos === endpos` + branch.right.name = 'no_data'; + this.pause(ctx, branch.right); + + branch.left.name = 'has_data'; + return branch.left; + } + + protected pause(ctx: Compilation, body: BasicBlock): void { + const fn = ctx.fn; + const bitcast = body.cast('bitcast', fn, fn.ty.toSignature().returnType); + body.ret(bitcast); + + // To be used in `compiler.js` + this.hasPause = true; + } + + protected buildNode(ctx: Compilation, node: Node): Func { + return node.build(ctx.compilation, ctx.nodes); + } + + protected tailTo(ctx: Compilation, body: BasicBlock, pos: INodePosition, + node: Node, value?: number): Func { + const target = this.buildNode(ctx, node); + + const isCacheable = ctx.pos.next === pos; + + if (isCacheable && this.trampolines.has(target)) { + const cached = this.trampolines.get(target); + + if (cached.phi !== undefined) { + assert(value, '`.match()` and `.select()` with the same target'); + cached.phi.addEdge({ + fromBlock: body, + value: ctx.TYPE_MATCH.val(value) + }); + } else { + assert(!value, '`.match()` and `.select()` with the same target'); + } + + body.jmp(cached.trampoline); + return target; + } + + // Split, so that others could join us from code block above + const trampoline = body.parent.createBlock(body.name + '.trampoline'); + body.jmp(trampoline); + let phi = null; + + // Compute `match` if needed + if (value !== null) { + phi = trampoline.phi({ + fromBlock: body, + value: ctx.TYPE_MATCH.val(value) + }); + } + + if (isCacheable) { + this.phis.set(target, { phi, block: trampoline }); + } + + const call = trampoline.call(target, [ + ctx.state, + pos, + ctx.endPos, + phi ? phi : ctx.TYPE_MATCH.undef() + ], 'musttail'); + + trampoline.ret(call); + + return target; + } + + protected doOtherwise(ctx: Compilation, body: BasicBlock, + pos?: INodePosition): Func { + if (pos === undefined) { + pos = ctx.pos; + } + + // `.skipTo()` advances by one byte + // `.otherwise()` redirects using the same byte + const next = this.skip ? pos!.next : pos!.current; + + assert(this.otherwise); + return this.tailTo(ctx, body, next, this.otherwise); + } + + protected abstract doBuild(ctx: Compilation, body: BasicBlock); +} diff --git a/src/llparse/compiler/node/bit-check.js b/src/llparse/compiler/node/bit-check.ts similarity index 95% rename from src/llparse/compiler/node/bit-check.js rename to src/llparse/compiler/node/bit-check.ts index 8f7c23e..eb9cae1 100644 --- a/src/llparse/compiler/node/bit-check.js +++ b/src/llparse/compiler/node/bit-check.ts @@ -1,13 +1,10 @@ -'use strict'; - -const node = require('./'); -const llparse = require('../../'); +import { Node } from './base'; const CHAR_WIDTH = 8; const WORD_WIDTH = 5; -class BitCheck extends node.Node { - constructor(id) { +class BitCheck extends Node { + constructor(id: string) { super('bit-check', id); this.map = null; diff --git a/src/llparse/compiler/node/node-context.ts b/src/llparse/compiler/node/node-context.ts new file mode 100644 index 0000000..f0de833 --- /dev/null +++ b/src/llparse/compiler/node/node-context.ts @@ -0,0 +1,67 @@ +import { Builder } from 'bitcode'; + +import { Compilation, types, values } from '../compilation'; +import { Node } from './base'; + +import Func = values.constants.Func; + +export interface INodePosition { + current: values.Value; + next?: values.Value; +} + +export class NodeContext { + // Some generally useful types (to avoid unnecessary includes) + public readonly INT = constants.INT; + public readonly BOOL = constants.BOOL; + public readonly TYPE_MATCH = constants.TYPE_MATCH; + public readonly TYPE_REASON = constants.TYPE_REASON; + public readonly TYPE_ERROR = constants.TYPE_ERROR; + public readonly TYPE_INDEX = constants.TYPE_INDEX; + public readonly TYPE_INTPTR = constants.TYPE_INTPTR; + public readonly state: values.Value; + public readonly pos: INodePosition; + public readonly endPos: values.Value; + public readonly match: values.Value; + public readonly ir: Builder; + public readonly stateType: types.Struct; + public readonly signature: { [key: string]: types.Signature }; + public readonly INVARIANT_GROUP: values.constants.Metadata; + + constructor(public readonly compilation: Compilation, + public readonly name: string, + public readonly fn: Func, + public readonly nodes: Map) { + this.state = this.compilation.stateArg(this.fn); + this.pos = { + current: this.compilation.posArg(this.fn), + next: undefined + }; + this.endPos = this.compilation.endPosArg(this.fn); + this.match = this.compilation.matchArg(this.fn); + + // Re-export some properties + this.ir = this.compilation.ir; + this.signature = this.compilation.signature; + this.stateType = this.compilation.state; + + this.INVARIANT_GROUP = ctx.INVARIANT_GROUP; + } + + debug(body, message) { + this.compilation.debug(this.fn, body, `${this.name}: ${message}`); + } + + // Just proxy + stateField(body, name) { + return this.compilation.stateField(this.fn, body, name); + } + + cstring(...args) { return this.compilation.cstring(...args); } + blob(...args) { return this.compilation.blob(...args); } + addGlobalConst(...args) { return this.compilation.addGlobalConst(...args); } + truncate(...args) { return this.compilation.truncate(...args); } + call(...args) { return this.compilation.call(...args); } + buildSwitch(...args) { return this.compilation.buildSwitch(...args); } + branch(...args) { return this.compilation.branch(...args); } +} From 95ebd4053470639bfd3182fa8095f81c39ae9773 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 17:49:51 -0400 Subject: [PATCH 278/281] save --- src/llparse/compiler/node/base.ts | 17 ++++---- src/llparse/compiler/node/bit-check.ts | 39 ++++++++++++------- .../compiler/node/{consume.js => consume.ts} | 19 ++++----- src/llparse/compiler/node/empty.js | 20 ---------- src/llparse/compiler/node/empty.ts | 18 +++++++++ .../compiler/node/{error.js => error.ts} | 21 +++++----- src/llparse/compiler/node/index.js | 13 ------- src/llparse/compiler/node/index.ts | 11 ++++++ .../compiler/node/{invoke.js => invoke.ts} | 32 ++++++++------- src/llparse/compiler/node/node-context.ts | 4 +- .../compiler/node/{pause.js => pause.ts} | 16 ++++---- .../node/{sequence.js => sequence.ts} | 36 ++++++++--------- .../compiler/node/{single.js => single.ts} | 28 ++++++------- src/llparse/compiler/node/span-end.js | 24 ------------ src/llparse/compiler/node/span-end.ts | 22 +++++++++++ src/llparse/compiler/node/span-start.js | 19 --------- src/llparse/compiler/node/span-start.ts | 21 ++++++++++ 17 files changed, 177 insertions(+), 183 deletions(-) rename src/llparse/compiler/node/{consume.js => consume.ts} (79%) delete mode 100644 src/llparse/compiler/node/empty.js create mode 100644 src/llparse/compiler/node/empty.ts rename src/llparse/compiler/node/{error.js => error.ts} (63%) delete mode 100644 src/llparse/compiler/node/index.js create mode 100644 src/llparse/compiler/node/index.ts rename src/llparse/compiler/node/{invoke.js => invoke.ts} (57%) rename src/llparse/compiler/node/{pause.js => pause.ts} (52%) rename src/llparse/compiler/node/{sequence.js => sequence.ts} (70%) rename src/llparse/compiler/node/{single.js => single.ts} (61%) delete mode 100644 src/llparse/compiler/node/span-end.js create mode 100644 src/llparse/compiler/node/span-end.ts delete mode 100644 src/llparse/compiler/node/span-start.js create mode 100644 src/llparse/compiler/node/span-start.ts diff --git a/src/llparse/compiler/node/base.ts b/src/llparse/compiler/node/base.ts index 56c4de7..7868a4f 100644 --- a/src/llparse/compiler/node/base.ts +++ b/src/llparse/compiler/node/base.ts @@ -4,17 +4,14 @@ import { Buffer } from 'buffer'; import { Transform } from '../../transform'; import * as node from './node'; import { INodePosition, NodeContext } from './node-context'; -import { Compilation, INodeID, values } from './compilation'; - -import BasicBlock = values.BasicBlock; -import Func = values.constants.Func; +import { Compilation, INodeID, BasicBlock, Func, values } from './compilation'; interface ITrampoline { - block: values.BasicBlock; + block: BasicBlock; phi?: values.instructions.Phi; } -interface INodeChild { +export interface INodeChild { node: Node; noAdvance: boolean; key: Buffer; @@ -28,7 +25,7 @@ export abstract class Node { protected skip: boolean = false; protected transform: Transform | undefined; - protected noPrologueCheck: boolean = false; + protected privNoPrologueCheck: boolean = false; protected hasPause: boolean = false; private trampolines: Map = new Map(); @@ -37,12 +34,14 @@ export abstract class Node { this.sourceName = id.sourceName; } + public get noPrologueCheck(): boolean { return this.privNoPrologueCheck; } + public setOtherwise(otherwise: Node, skip: boolean) { this.otherwise = otherwise; this.skip = skip; } - public getChildren(): INodeChild[] { + public getChildren(): ReadonlyArray { return [ { node: this.otherwise, noAdvance: !this.skip, key: undefined } ]; } @@ -84,7 +83,7 @@ export abstract class Node { } protected prologue(ctx: Compilation, body: BasicBlock): BasicBlock { - if (this.noPrologueCheck) { + if (this.privNoPrologueCheck) { return body; } diff --git a/src/llparse/compiler/node/bit-check.ts b/src/llparse/compiler/node/bit-check.ts index eb9cae1..5d028bc 100644 --- a/src/llparse/compiler/node/bit-check.ts +++ b/src/llparse/compiler/node/bit-check.ts @@ -1,18 +1,30 @@ -import { Node } from './base'; +import { Buffer } from 'buffer'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; const CHAR_WIDTH = 8; const WORD_WIDTH = 5; -class BitCheck extends Node { - constructor(id: string) { +export interface IBitCheckEntry { + node: Node; + noAdvance: boolean; + keys: Buffer[]; +} + +export class BitCheck extends Node { + public readonly entries: IBitCheckEntry[] = []; + + constructor(id: INodeID) { super('bit-check', id); + } - this.map = null; + public add(entry: IBitCheckEntry): void { + this.entries.push(entry); } - getChildren() { + public getChildren(): ReadonlyArray { const res = super.getChildren(); - this.map.forEach((entry) => { + this.entries.forEach((entry) => { entry.keys.forEach((key) => { res.push({ node: entry.node, noAdvance: entry.noAdvance, key }); }); @@ -20,9 +32,9 @@ class BitCheck extends Node { return res; } - buildTable(ctx) { + private buildTable(ctx: Compilation) { const table = llparse.utils.buildLookupTable(WORD_WIDTH, CHAR_WIDTH, - this.map.map(entry => entry.keys)); + this.entries.map(entry => entry.keys)); const cellTy = ctx.ir.i(1 << WORD_WIDTH); const arrayTy = ctx.ir.array(table.table.length, cellTy); @@ -41,7 +53,7 @@ class BitCheck extends Node { }; } - doBuild(ctx, body) { + protected doBuild(ctx: Compilation, body: BasicBlock): void { const pos = ctx.pos.current; // Load the character @@ -79,9 +91,9 @@ class BitCheck extends Node { const shr = body.binop('lshr', load, shift); const masked = body.binop('and', shr, cellTy.val(table.valueMask)); - const weights = new Array(this.map.length + 1).fill('likely'); + const weights = new Array(this.entries.length + 1).fill('likely'); - this.map.forEach((entry, i) => { + this.entries.forEach((entry, i) => { if (entry.node instanceof node.Error) weights[i + 1] = 'unlikely'; }); @@ -89,11 +101,11 @@ class BitCheck extends Node { if (this.otherwise instanceof node.Error) weights[0] = 'unlikely'; - const keys = this.map.map((entry, i) => i + 1); + const keys = this.entries.map((entry, i) => i + 1); const s = ctx.buildSwitch(body, masked, keys, weights); s.cases.forEach((body, i) => { - const child = this.map[i]; + const child = this.entries[i]; this.tailTo(ctx, body, child.noAdvance ? ctx.pos.current : ctx.pos.next, child.node, null); @@ -102,4 +114,3 @@ class BitCheck extends Node { this.doOtherwise(ctx, s.otherwise); } } -module.exports = BitCheck; diff --git a/src/llparse/compiler/node/consume.js b/src/llparse/compiler/node/consume.ts similarity index 79% rename from src/llparse/compiler/node/consume.js rename to src/llparse/compiler/node/consume.ts index b8f675f..744a533 100644 --- a/src/llparse/compiler/node/consume.js +++ b/src/llparse/compiler/node/consume.ts @@ -1,18 +1,14 @@ -'use strict'; +import * as assert from 'assert'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; -const assert = require('assert'); - -const node = require('./'); - -class Consume extends node.Node { - constructor(id, fieldName) { +export class Consume extends Node { + constructor(id: INodeID, private readonly fieldName: string) { super('consume', id); - this.fieldName = fieldName; - this.noPrologueCheck = true; + this.privNoPrologueCheck = true; } - // TODO(indutny): remove unnecessary load - doBuild(ctx, body) { + public doBuild(ctx: Compilation, body: BasicBlock): void { const INVARIANT_GROUP = ctx.INVARIANT_GROUP; const pos = ctx.pos.current; @@ -53,4 +49,3 @@ class Consume extends node.Node { this.pause(ctx, noData); } } -module.exports = Consume; diff --git a/src/llparse/compiler/node/empty.js b/src/llparse/compiler/node/empty.js deleted file mode 100644 index 4adc8bb..0000000 --- a/src/llparse/compiler/node/empty.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -const node = require('./'); - -class Empty extends node.Node { - constructor(...args) { - super('empty', ...args); - } - - prologue(ctx, body) { - if (this.skip) - return super.prologue(ctx, body); - return body; - } - - doBuild(ctx, body) { - this.doOtherwise(ctx, body); - } -} -module.exports = Empty; diff --git a/src/llparse/compiler/node/empty.ts b/src/llparse/compiler/node/empty.ts new file mode 100644 index 0000000..1f3d0e7 --- /dev/null +++ b/src/llparse/compiler/node/empty.ts @@ -0,0 +1,18 @@ +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; + +export class Empty extends Node { + constructor(id: INodeID) { + super('empty', id); + } + + protected prologue(ctx: Compilation, body: BasicBlock): BasicBlock { + if (this.skip) + return super.prologue(ctx, body); + return body; + } + + protected doBuild(ctx: Compilation, body: BasicBlock): void { + this.doOtherwise(ctx, body); + } +} diff --git a/src/llparse/compiler/node/error.js b/src/llparse/compiler/node/error.ts similarity index 63% rename from src/llparse/compiler/node/error.js rename to src/llparse/compiler/node/error.ts index 7be0e2f..65bacc9 100644 --- a/src/llparse/compiler/node/error.js +++ b/src/llparse/compiler/node/error.ts @@ -1,21 +1,19 @@ -'use strict'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; -const node = require('./'); - -class Error extends node.Node { - constructor(id, code, reason) { +export class Error extends Node { + constructor(id: NodeID, private readonly code: number, + private readonly reason: string) { super('error', id); - this.code = code; - this.reason = reason; - this.noPrologueCheck = true; + this.privNoPrologueCheck = true; } - getChildren() { + public getChildren(): ReadonlyArray { return []; } - buildStoreError(ctx, body) { + private buildStoreError(ctx: Compilation, body: BasicBlock): BasicBlock { const INT = ctx.INT; const reason = ctx.cstring(this.reason); @@ -33,7 +31,7 @@ class Error extends node.Node { return body; } - doBuild(ctx, body) { + protected doBuild(ctx: Compilation, body: BasicBlock): void { body = this.buildStoreError(ctx, body); const currentPtr = ctx.stateField(body, '_current'); @@ -46,4 +44,3 @@ class Error extends node.Node { body.ret(retType.val(null)); } } -module.exports = Error; diff --git a/src/llparse/compiler/node/index.js b/src/llparse/compiler/node/index.js deleted file mode 100644 index 6d0f7bc..0000000 --- a/src/llparse/compiler/node/index.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict'; - -exports.Node = require('./base'); -exports.Empty = require('./empty'); -exports.Error = require('./error'); -exports.Invoke = require('./invoke'); -exports.Single = require('./single'); -exports.Sequence = require('./sequence'); -exports.SpanStart = require('./span-start'); -exports.SpanEnd = require('./span-end'); -exports.Consume = require('./consume'); -exports.Pause = require('./pause'); -exports.BitCheck = require('./bit-check'); diff --git a/src/llparse/compiler/node/index.ts b/src/llparse/compiler/node/index.ts new file mode 100644 index 0000000..dcfd062 --- /dev/null +++ b/src/llparse/compiler/node/index.ts @@ -0,0 +1,11 @@ +export { Node } from './base'; +export { Empty } from './empty'; +export { Error } from './error'; +export { Invoke } from './invoke'; +export { Single } from './single'; +export { Sequence } from './sequence'; +export { SpanStart } from './span-start'; +export { SpanEnd } from './span-end'; +export { Consume } from './consume'; +export { Pause } from './pause'; +export { BitCheck } from './bit-check'; diff --git a/src/llparse/compiler/node/invoke.js b/src/llparse/compiler/node/invoke.ts similarity index 57% rename from src/llparse/compiler/node/invoke.js rename to src/llparse/compiler/node/invoke.ts index 32a0e6d..c8e117e 100644 --- a/src/llparse/compiler/node/invoke.js +++ b/src/llparse/compiler/node/invoke.ts @@ -1,28 +1,31 @@ -'use strict'; +import * as assert from 'assert'; -const assert = require('assert'); +import { Code } from '../code'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; -const llparse = require('../../'); -const kSignature = llparse.symbols.kSignature; +export class Invoke extends Node { + private readonly map: Map = new Map(); -const node = require('./'); - -class Invoke extends node.Node { - constructor(id, code) { + constructor(id: INodeID, private readonly code: Code) { super('invoke', id); this.code = code; - this.map = null; - this.noPrologueCheck = true; + this.privNoPrologueCheck = true; + } + + public add(key: number, node: Node): void { + assert(!this.map.has(key)); + this.map.set(key, node); } - getChildren() { - return super.getChildren().concat(Object.keys(this.map).map((key) => { - return { node: this.map[key], noAdvance: true, key: null }; + public getChildren(): ReadonlyArray { + return super.getChildren().concat(this.map.forEach((node) => { + return { node, noAdvance: true, key: null }; })); } - doBuild(ctx, body) { + protected doBuild(ctx: Compilation, body: BasicBlock): void { const code = ctx.compilation.buildCode(this.code); const args = [ @@ -59,4 +62,3 @@ class Invoke extends node.Node { this.doOtherwise(ctx, s.otherwise); } } -module.exports = Invoke; diff --git a/src/llparse/compiler/node/node-context.ts b/src/llparse/compiler/node/node-context.ts index f0de833..91c44dc 100644 --- a/src/llparse/compiler/node/node-context.ts +++ b/src/llparse/compiler/node/node-context.ts @@ -1,10 +1,8 @@ import { Builder } from 'bitcode'; -import { Compilation, types, values } from '../compilation'; +import { Compilation, Func, types, values } from '../compilation'; import { Node } from './base'; -import Func = values.constants.Func; - export interface INodePosition { current: values.Value; next?: values.Value; diff --git a/src/llparse/compiler/node/pause.js b/src/llparse/compiler/node/pause.ts similarity index 52% rename from src/llparse/compiler/node/pause.js rename to src/llparse/compiler/node/pause.ts index 73ddb09..e04b0a8 100644 --- a/src/llparse/compiler/node/pause.js +++ b/src/llparse/compiler/node/pause.ts @@ -1,18 +1,17 @@ -'use strict'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; +import { Error } from './error'; -const node = require('./'); - -class Pause extends node.Error { - constructor(id, code, reason) { +export class Pause extends Error { + constructor(id: INodeID, code: number, reason: string) { super(id, code, reason); - this.type = 'pause'; } - getResumptionTargets() { + public getResumptionTargets(): ReadonlyArray { return super.getResumptionTargets().concat(this.otherwise); } - doBuild(ctx, body) { + protected doBuild(ctx: Compilation, body: BasicBlock): void { body = this.buildStoreError(ctx, body); const currentPtr = ctx.stateField(body, '_current'); @@ -24,4 +23,3 @@ class Pause extends node.Error { body.ret(ctx.fn.ty.toSignature().returnType.val(null)); } } -module.exports = Pause; diff --git a/src/llparse/compiler/node/sequence.js b/src/llparse/compiler/node/sequence.ts similarity index 70% rename from src/llparse/compiler/node/sequence.js rename to src/llparse/compiler/node/sequence.ts index a4f45e6..9460bcc 100644 --- a/src/llparse/compiler/node/sequence.js +++ b/src/llparse/compiler/node/sequence.ts @@ -1,26 +1,25 @@ -'use strict'; +import { Buffer } from 'buffer'; -const llparse = require('../../'); -const constants = llparse.constants; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; +import { + TYPE_INDEX, SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH, +} from '../../constants'; +export class Sequence extends Node { + private next: Node | undefined; + private value: number | undefined; -const node = require('./'); - -const TYPE_INDEX = constants.TYPE_INDEX; -const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; -const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; -const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; - -class Sequence extends node.Node { - constructor(id, select) { + constructor(id: INodeID, private readonly select: Buffer) { super('sequence', id); + } - this.select = select; - this.next = null; - this.value = null; + public finish(next: Node, value: number) { + this.next = next; + this.value = value; } - getChildren() { + public getChildren(): ReadonlyArray { return super.getChildren().concat({ node: this.next, noAdvance: false, @@ -28,7 +27,7 @@ class Sequence extends node.Node { }); } - doBuild(ctx, body) { + protected doBuild(ctx: Compilation, body: BasicBlock): void { const INT = ctx.INT; const seq = ctx.blob(this.select); @@ -79,11 +78,10 @@ class Sequence extends node.Node { pause.name = 'pause'; mismatch.name = 'mismatch'; - this.tailTo(ctx, complete, next, this.next, this.value); + this.tailTo(ctx, complete, next, this.next!, this.value!); this.pause(ctx, pause); // Not equal this.doOtherwise(ctx, mismatch, { current, next }); } } -module.exports = Sequence; diff --git a/src/llparse/compiler/node/single.js b/src/llparse/compiler/node/single.ts similarity index 61% rename from src/llparse/compiler/node/single.js rename to src/llparse/compiler/node/single.ts index 85baac2..9ac89ee 100644 --- a/src/llparse/compiler/node/single.js +++ b/src/llparse/compiler/node/single.ts @@ -1,21 +1,22 @@ -'use strict'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; -const node = require('./'); +export class Single extends Node { + private readonly children: INodeChild[] = []; -class Single extends node.Node { - constructor(...args) { - super('single', ...args); + constructor(id: NodeID) { + super('single', id); + } - this.children = null; + public add(child: INodeChild): void { + this.children.push(child); } - getChildren() { - return super.getChildren().concat(this.children.map((child) => { - return { node: child.next, noAdvance: child.noAdvance, key: child.key }; - })); + public getChildren(): ReadonlyArray { + return super.getChildren().concat(this.children); } - doBuild(ctx, body) { + protected doBuild(ctx: Compilation, body: BasicBlock): void { const pos = ctx.pos.current; // Load the character @@ -33,7 +34,7 @@ class Single extends node.Node { // Mark error branches as unlikely this.children.forEach((child, i) => { - if (child.next instanceof node.Error) + if (child.node instanceof node.Error) weights[i + 1] = 'unlikely'; }); @@ -47,10 +48,9 @@ class Single extends node.Node { const child = this.children[i]; this.tailTo(ctx, body, child.noAdvance ? ctx.pos.current : ctx.pos.next, - child.next, child.value); + child.node, child.value); }); this.doOtherwise(ctx, s.otherwise); } } -module.exports = Single; diff --git a/src/llparse/compiler/node/span-end.js b/src/llparse/compiler/node/span-end.js deleted file mode 100644 index 26838ef..0000000 --- a/src/llparse/compiler/node/span-end.js +++ /dev/null @@ -1,24 +0,0 @@ -'use strict'; - -const node = require('./'); - -class SpanEnd extends node.Node { - constructor(id, code) { - super('span-end', id); - - this.code = code; - this.noPrologueCheck = true; - } - - getResumptionTargets() { - return super.getResumptionTargets().concat(this.otherwise); - } - - doBuild(ctx, body) { - const result = ctx.compilation.stageResults['span-builder'].spanEnd( - ctx.fn, body, this.code); - - result.updateResumptionTarget(this.doOtherwise(ctx, result.body)); - } -} -module.exports = SpanEnd; diff --git a/src/llparse/compiler/node/span-end.ts b/src/llparse/compiler/node/span-end.ts new file mode 100644 index 0000000..69b79cb --- /dev/null +++ b/src/llparse/compiler/node/span-end.ts @@ -0,0 +1,22 @@ +import { Code } from '../code'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; + +export class SpanEnd extends Node { + constructor(id: INodeID, private readonly code: Code) { + super('span-end', id); + + this.privNoPrologueCheck = true; + } + + public getResumptionTargets(): ReadonlyArray { + return super.getResumptionTargets().concat(this.otherwise); + } + + protected doBuild(ctx: Compilation, body: BasicBlock): void { + const result = ctx.compilation.stageResults['span-builder'].spanEnd( + ctx.fn, body, this.code); + + result.updateResumptionTarget(this.doOtherwise(ctx, result.body)); + } +} diff --git a/src/llparse/compiler/node/span-start.js b/src/llparse/compiler/node/span-start.js deleted file mode 100644 index e92d455..0000000 --- a/src/llparse/compiler/node/span-start.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -const node = require('./'); - -class SpanStart extends node.Node { - constructor(id, code) { - super('span-start', id); - - this.code = code; - this.noPrologueCheck = true; - } - - doBuild(ctx, body) { - body = ctx.compilation.stageResults['span-builder'].spanStart( - ctx.fn, body, this.code); - this.doOtherwise(ctx, body); - } -} -module.exports = SpanStart; diff --git a/src/llparse/compiler/node/span-start.ts b/src/llparse/compiler/node/span-start.ts new file mode 100644 index 0000000..9bf0d3c --- /dev/null +++ b/src/llparse/compiler/node/span-start.ts @@ -0,0 +1,21 @@ +import { Code } from '../code'; +import { Compilation, BasicBlock, INodeID } from '../compilation'; +import { Node, INodeChild } from './base'; + +export class SpanStart extends Node { + constructor(id: INodeID, private readonly code: Code) { + super('span-start', id); + + this.privNoPrologueCheck = true; + } + + public getResumptionTargets(): ReadonlyArray { + return super.getResumptionTargets().concat(this.otherwise); + } + + protected doBuild(ctx: Compilation, body: BasicBlock): void { + body = ctx.compilation.stageResults['span-builder'].spanStart( + ctx.fn, body, this.code); + this.doOtherwise(ctx, body); + } +} From 74b3d37e4f78c05e64804fa6c33ecc18b54a0f98 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 18:42:18 -0400 Subject: [PATCH 279/281] save --- src/llparse/compiler/compilation.js | 6 + src/llparse/compiler/node/base.ts | 6 +- src/llparse/compiler/node/node-context.ts | 9 +- src/llparse/compiler/stage/base.js | 13 -- src/llparse/compiler/stage/base.ts | 10 + src/llparse/compiler/stage/index.js | 10 - src/llparse/compiler/stage/index.ts | 8 + .../{match-sequence.js => match-sequence.ts} | 80 ++++---- src/llparse/compiler/stage/node-builder.js | 20 -- src/llparse/compiler/stage/node-builder.ts | 20 ++ ...e-loop-checker.js => node-loop-checker.ts} | 90 ++++----- ...{node-translator.js => node-translator.ts} | 174 ++++++++++-------- .../{span-allocator.js => span-allocator.ts} | 45 +++-- .../{span-builder.js => span-builder.ts} | 47 +++-- src/llparse/node/base.ts | 10 +- src/llparse/node/match.ts | 10 +- src/llparse/node/span-end.ts | 2 +- src/llparse/node/span-start.ts | 4 +- 18 files changed, 301 insertions(+), 263 deletions(-) delete mode 100644 src/llparse/compiler/stage/base.js create mode 100644 src/llparse/compiler/stage/base.ts delete mode 100644 src/llparse/compiler/stage/index.js create mode 100644 src/llparse/compiler/stage/index.ts rename src/llparse/compiler/stage/{match-sequence.js => match-sequence.ts} (77%) delete mode 100644 src/llparse/compiler/stage/node-builder.js create mode 100644 src/llparse/compiler/stage/node-builder.ts rename src/llparse/compiler/stage/{node-loop-checker.js => node-loop-checker.ts} (65%) rename src/llparse/compiler/stage/{node-translator.js => node-translator.ts} (53%) rename src/llparse/compiler/stage/{span-allocator.js => span-allocator.ts} (77%) rename src/llparse/compiler/stage/{span-builder.js => span-builder.ts} (85%) diff --git a/src/llparse/compiler/compilation.js b/src/llparse/compiler/compilation.js index 29c60a7..af80636 100644 --- a/src/llparse/compiler/compilation.js +++ b/src/llparse/compiler/compilation.js @@ -27,6 +27,12 @@ const ARG_POS = constants.ARG_POS; const ARG_ENDPOS = constants.ARG_ENDPOS; const ARG_MATCH = constants.ARG_MATCH; +export interface INodePosition { + current: values.Value; + next?: values.Value; +} + + class Compilation { constructor(options) { this.options = Object.assign({}, options); diff --git a/src/llparse/compiler/node/base.ts b/src/llparse/compiler/node/base.ts index 7868a4f..29efc34 100644 --- a/src/llparse/compiler/node/base.ts +++ b/src/llparse/compiler/node/base.ts @@ -3,8 +3,10 @@ import { Buffer } from 'buffer'; import { Transform } from '../../transform'; import * as node from './node'; -import { INodePosition, NodeContext } from './node-context'; -import { Compilation, INodeID, BasicBlock, Func, values } from './compilation'; +import { NodeContext } from './node-context'; +import { + Compilation, INodeID, INodePosition, BasicBlock, Func, values +} from './compilation'; interface ITrampoline { block: BasicBlock; diff --git a/src/llparse/compiler/node/node-context.ts b/src/llparse/compiler/node/node-context.ts index 91c44dc..babcaf4 100644 --- a/src/llparse/compiler/node/node-context.ts +++ b/src/llparse/compiler/node/node-context.ts @@ -1,13 +1,10 @@ import { Builder } from 'bitcode'; -import { Compilation, Func, types, values } from '../compilation'; +import { + Compilation, INodePosition, Func, types, values, +} from '../compilation'; import { Node } from './base'; -export interface INodePosition { - current: values.Value; - next?: values.Value; -} - export class NodeContext { // Some generally useful types (to avoid unnecessary includes) public readonly INT = constants.INT; diff --git a/src/llparse/compiler/stage/base.js b/src/llparse/compiler/stage/base.js deleted file mode 100644 index 113c347..0000000 --- a/src/llparse/compiler/stage/base.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict'; - -class Stage { - constructor(ctx, name) { - this.ctx = ctx; - this.name = name; - } - - build() { - throw new Error('Not implemented'); - } -} -module.exports = Stage; diff --git a/src/llparse/compiler/stage/base.ts b/src/llparse/compiler/stage/base.ts new file mode 100644 index 0000000..aee2399 --- /dev/null +++ b/src/llparse/compiler/stage/base.ts @@ -0,0 +1,10 @@ +import { Compilation } from './compilation'; + +export abstract class Stage { + constructor(protected readonly ctx: Compilation, + public readonly name: string) { + } + + // TODO(indutny): specify types? + public abstract build(): any; +} diff --git a/src/llparse/compiler/stage/index.js b/src/llparse/compiler/stage/index.js deleted file mode 100644 index de2bf0e..0000000 --- a/src/llparse/compiler/stage/index.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -exports.Stage = require('./base'); - -exports.MatchSequence = require('./match-sequence'); -exports.NodeTranslator = require('./node-translator'); -exports.NodeLoopChecker = require('./node-loop-checker'); -exports.NodeBuilder = require('./node-builder'); -exports.SpanAllocator = require('./span-allocator'); -exports.SpanBuilder = require('./span-builder'); diff --git a/src/llparse/compiler/stage/index.ts b/src/llparse/compiler/stage/index.ts new file mode 100644 index 0000000..15e8a63 --- /dev/null +++ b/src/llparse/compiler/stage/index.ts @@ -0,0 +1,8 @@ +export { Stage } from './base'; + +export { MatchSequence } from './match-sequence'; +export { NodeTranslator } from './node-translator'; +export { NodeLoopChecker } from './node-loop-checker'; +export { NodeBuilder } from './node-builder'; +export { SpanAllocator } from './span-allocator'; +export { SpanBuilder } from './span-builder'; diff --git a/src/llparse/compiler/stage/match-sequence.js b/src/llparse/compiler/stage/match-sequence.ts similarity index 77% rename from src/llparse/compiler/stage/match-sequence.js rename to src/llparse/compiler/stage/match-sequence.ts index e6c79d8..8b61b5d 100644 --- a/src/llparse/compiler/stage/match-sequence.js +++ b/src/llparse/compiler/stage/match-sequence.ts @@ -1,33 +1,31 @@ -'use strict'; - -const Stage = require('./').Stage; -const llparse = require('../../'); -const constants = llparse.constants; - -const CCONV = constants.CCONV; - -const INT = constants.INT; -const TYPE_INPUT = constants.TYPE_INPUT; -const TYPE_INDEX = constants.TYPE_INDEX; -const TYPE_STATUS = constants.TYPE_STATUS; - -const ATTR_STATE = constants.ATTR_STATE; -const ATTR_POS = constants.ATTR_POS; -const ATTR_ENDPOS = constants.ATTR_ENDPOS; -const ATTR_SEQUENCE = constants.ATTR_SEQUENCE; - -const ARG_STATE = constants.ARG_STATE; -const ARG_POS = constants.ARG_POS; -const ARG_ENDPOS = constants.ARG_ENDPOS; -const ARG_SEQUENCE = constants.ARG_SEQUENCE; -const ARG_SEQUENCE_LEN = constants.ARG_SEQUENCE_LEN; +import { + CCONV, + INT, TYPE_INPUT, TYPE_INDEX, TYPE_STATUS, + ATTR_STATE, ATTR_POS, ATTR_ENDPOS, ATTR_SEQUENCE, + + ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_SEQUENCE, ARG_SEQUENCE_LEN, + + SEQUENCE_COMPLETE, SEQUENCE_PAUSE, SEQUENCE_MISMATCH +} from '../../constants'; +import { Transform } from '../../transform'; +import { + Compilation, BasicBlock, INodePosition, Func, values, +} from '../compilation'; +import { Stage } from './base'; + +interface IIterationResult { + index: values.Value; + pos: INodePosition; + complete: BasicBlock; + mismatch: BasicBlock; + loop: BasicBlock; + pause: BasicBlock; +} -const SEQUENCE_COMPLETE = constants.SEQUENCE_COMPLETE; -const SEQUENCE_PAUSE = constants.SEQUENCE_PAUSE; -const SEQUENCE_MISMATCH = constants.SEQUENCE_MISMATCH; +export class MatchSequence extends Stage { + private readonly cache: Map = new Map(); -class MatchSequence extends Stage { - constructor(ctx) { + constructor(ctx: Compilation) { super(ctx, 'match-sequence'); this.returnType = this.ctx.ir.struct('match_sequence_ret'); @@ -42,20 +40,19 @@ class MatchSequence extends Stage { TYPE_INPUT, TYPE_INDEX ]); - - this.cache = new Map(); } - build() { + public build(): any { return { get: transform => this.get(transform) }; } - get(transform = null) { - const cacheKey = transform === null ? null : transform.name; - if (this.cache.has(cacheKey)) - return this.cache.get(cacheKey); + public get(transform?: Transform): Func { + const cacheKey = transform === undefined ? undefined : transform.name; + if (this.cache.has(cacheKey)) { + return this.cache.get(cacheKey)!; + } const postfix = cacheKey ? '_' + cacheKey.toLowerCase() : ''; @@ -79,8 +76,7 @@ class MatchSequence extends Stage { return fn; } - // TODO(indutny): initialize `state.index` before calling matcher? - buildBody(fn, transform) { + private buildBody(fn: Func, transform?: Transform): void { const body = fn.body; const maxSeqLen = this.ctx.stageResults['node-translator'].maxSequenceLen; @@ -124,7 +120,9 @@ class MatchSequence extends Stage { this.ret(iteration.mismatch, iteration.pos, SEQUENCE_MISMATCH); } - buildIteration(fn, body, indexField, pos, index, transform) { + private buildIteration(fn: Func, body: BasicBlock, indexField: values.Value, + pos: values.Value, index: values.Value, + transform?: Transform): IIterationResult { const seq = fn.getArgument(ARG_SEQUENCE); const seqLen = fn.getArgument(ARG_SEQUENCE_LEN); @@ -182,7 +180,7 @@ class MatchSequence extends Stage { }; } - ret(body, pos, status) { + private ret(body: BasicBlock, pos: INodePosition, status: number): void { const create = body.insertvalue(this.returnType.undef(), pos.current, this.returnType.lookupField('current').index); @@ -192,10 +190,8 @@ class MatchSequence extends Stage { body.ret(amend); } - reset(body, field) { + private reset(body: BasicBlock, field: values.Value): void { const store = body.store(TYPE_INDEX.val(0), field); store.metadata.set('invariant.group', this.ctx.INVARIANT_GROUP); - return store; } } -module.exports = MatchSequence; diff --git a/src/llparse/compiler/stage/node-builder.js b/src/llparse/compiler/stage/node-builder.js deleted file mode 100644 index 84024b0..0000000 --- a/src/llparse/compiler/stage/node-builder.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -const Stage = require('./').Stage; - -class NodeBuilder extends Stage { - constructor(ctx) { - super(ctx, 'node-builder'); - - this.nodes = new Map(); - } - - build() { - const root = this.ctx.stageResults['node-translator'].root; - return { - entry: root.build(this.ctx, this.nodes), - map: this.nodes - }; - } -} -module.exports = NodeBuilder; diff --git a/src/llparse/compiler/stage/node-builder.ts b/src/llparse/compiler/stage/node-builder.ts new file mode 100644 index 0000000..6b337c0 --- /dev/null +++ b/src/llparse/compiler/stage/node-builder.ts @@ -0,0 +1,20 @@ +import { Compilation, Func } from '../compilation'; +import { Node } from '../node'; +import { Stage } from './base'; + +export class NodeBuilder extends Stage { + private readonly nodes: Map = new Map(); + + constructor(ctx: Compilation) { + super(ctx, 'node-builder'); + } + + public build(): any { + // TODO(indutny): types + const root = this.ctx.stageResults['node-translator'].root; + return { + entry: root.build(this.ctx, this.nodes), + map: this.nodes + }; + } +} diff --git a/src/llparse/compiler/stage/node-loop-checker.js b/src/llparse/compiler/stage/node-loop-checker.ts similarity index 65% rename from src/llparse/compiler/stage/node-loop-checker.js rename to src/llparse/compiler/stage/node-loop-checker.ts index a5b1aff..b25204f 100644 --- a/src/llparse/compiler/stage/node-loop-checker.js +++ b/src/llparse/compiler/stage/node-loop-checker.ts @@ -1,47 +1,23 @@ -'use strict'; +import { Compilation } from '../compilation'; +import { Node } from '../node'; +import { Stage } from './base'; -const Stage = require('./').Stage; - -class NodeLoopChecker extends Stage { - constructor(ctx) { - super(ctx, 'node-loop-checker'); - - this.reachableMap = new Map(); - } - - reachable(from) { - if (this.reachableMap.has(from)) - return this.reachableMap.get(from); - - const res = new Set([ from ]); - this.reachableMap.set(from, res); - return res; - } - - addEdge(from, to) { - const target = this.reachable(to); - - let changed = false; - this.reachable(from).forEach((node) => { - if (to === node) { - throw new Error(`Loop detected in "${to.sourceName}", ` + - `through the backedge from "${from.sourceName}"`); - } - - if (target.has(node)) - return; +interface IQueueElem { + node: Node; + key: number | Buffer | undefined; +} - changed = true; - target.add(node); - }); +export class NodeLoopChecker extends Stage { + private readonly reachableMap > = new Map(); - return changed; + constructor(ctx: Compilation) { + super(ctx, 'node-loop-checker'); } - build() { - const queue = [ { + public build(): any { + const queue: IQueueElem = [ { node: this.ctx.stageResults['node-translator'].root, - key: null + key: undefined } ]; while (queue.length !== 0) { @@ -56,10 +32,10 @@ class NodeLoopChecker extends Stage { // `nodeA: peek(A)` => `nodeB: match(A), otherwise -> nodeA` // // should pass the check - if (lastKey !== null && typeof lastKey === 'number') { + if (typeof lastKey === 'number') { // Remove all unreachable clauses children = children.filter((child) => { - return child.key === null || child.key === lastKey; + return child.key === undefined || child.key === lastKey; }); // See if there is a matching peek clause @@ -73,12 +49,42 @@ class NodeLoopChecker extends Stage { children = children.filter(child => child.noAdvance); children.forEach((child) => { - if (this.addEdge(node, child.node)) + if (this.addEdge(node, child.node)) { queue.push({ node: child.node, key: child.key || lastKey }); + } }); } return true; } + + private reachable(from: Node): Set { + if (this.reachableMap.has(from)) { + return this.reachableMap.get(from)!; + } + + const res = new Set([ from ]); + this.reachableMap.set(from, res); + return res; + } + + private addEdge(from: Node, to: Node): boolean { + const target = this.reachable(to); + + let changed = false; + this.reachable(from).forEach((node) => { + if (to === node) { + throw new Error(`Loop detected in "${to.sourceName}", ` + + `through the backedge from "${from.sourceName}"`); + } + + if (target.has(node)) + return; + + changed = true; + target.add(node); + }); + + return changed; + } } -module.exports = NodeLoopChecker; diff --git a/src/llparse/compiler/stage/node-translator.js b/src/llparse/compiler/stage/node-translator.ts similarity index 53% rename from src/llparse/compiler/stage/node-translator.js rename to src/llparse/compiler/stage/node-translator.ts index ad6711d..a4b57a2 100644 --- a/src/llparse/compiler/stage/node-translator.js +++ b/src/llparse/compiler/stage/node-translator.ts @@ -1,78 +1,82 @@ -'use strict'; +import * as assert from 'assert'; +import * as debug from 'debug'; -const assert = require('assert'); -const debugOpt = require('debug')('llparse:opt'); +import * as node from '../../node'; +import { Trie, TrieNode, TrieSingle, TrieSequence, TrieNext } from '../../trie'; +import { Compilation } from '../compilation'; +import * as compilerNode from '../node'; +import { Stage } from './base'; -const Stage = require('./').Stage; -const compiler = require('../'); -const llparse = require('../../'); +const debugOpt = debug('llparse:opt'); -const kCases = llparse.symbols.kCases; -const kCode = llparse.symbols.kCode; -const kMap = llparse.symbols.kMap; -const kOtherwise = llparse.symbols.kOtherwise; -const kSignature = llparse.symbols.kSignature; -const kSpan = llparse.symbols.kSpan; -const kTransform = llparse.symbols.kTransform; +type TrieOutputList = compilerNode[]; -class NodeTranslator extends Stage { - constructor(ctx) { +export interface INodeTranslatorOptions { + // Minimum number of cases of `single` node to make it eligable for + // `BitCheck` optimization + minCheckSize: number | undefined; + + // Maximum width of entry in a bitfield for a `BitCheck` optimization + maxCheckWidth: number | undefined; +} + +export class NodeTranslator extends Stage { + private readonly INodeTranslatorOptions options; + private readonly nodes: Map = new Map(); + private readonly errorCache: Map = new Map(); + private maxSequenceLen: number = 0; + + constructor(ctx: Compilation, options: INodeTranslatorOptions) { super(ctx, 'node-translator'); this.options = Object.assign({ - // Minimum number of cases of `single` node to make it eligable for - // `BitCheck` optimization minCheckSize: llparse.constants.DEFAULT_TRANSLATOR_MIN_CHECK_SIZE, - // Maximum width of entry in a bitfield for a `BitCheck` optimization maxCheckWidth: llparse.constants.DEFAULT_TRANSLATOR_MAX_CHECK_WIDTH - }, this.ctx.options.translator); - - this.nodes = new Map(); - this.errorCache = new Map(); - this.maxSequenceLen = 0; + }, options); } - build() { + public build(): any { return { - root: this.buildNode(this.ctx.root, null), + root: this.buildNode(this.ctx.root, undefined), maxSequenceLen: this.maxSequenceLen }; } - id(node, postfix = '') { + private id(node: node.Node, postfix: string = '') { return this.ctx.id(node.name, 'n_', postfix); } - buildNode(node, value) { + private buildNode(node: node.Node, value: number | undefined) { this.checkSignature(node, value); - if (this.nodes.has(node)) + if (this.nodes.has(node)) { return this.nodes.get(node); + } let res; - let list; + let list: TrieOutputList || undefined; let last; - if (node instanceof llparse.node.Invoke) { - res = new compiler.node.Invoke(this.id(node), node[kCode]); - } else if (node instanceof llparse.node.Error) { + if (node instanceof node.Invoke) { + res = new compilerNode.Invoke(this.id(node), node.code); + } else if (node instanceof node.Error) { res = this.buildError(node); - } else if (node instanceof llparse.node.SpanStart) { - res = new compiler.node.SpanStart(this.id(node), node[kSpan][kCode]); - } else if (node instanceof llparse.node.SpanEnd) { - res = new compiler.node.SpanEnd(this.id(node), node[kSpan][kCode]); - } else if (node instanceof llparse.node.Pause) { - res = new compiler.node.Pause(this.id(node), node.code, node.reason); - } else if (node instanceof llparse.node.Consume) { - res = new compiler.node.Consume(this.id(node), node.fieldName); + } else if (node instanceof node.SpanStart) { + res = new compilerNode.SpanStart(this.id(node), node.code); + } else if (node instanceof node.SpanEnd) { + res = new compilerNode.SpanEnd(this.id(node), node.code); + } else if (node instanceof node.Pause) { + res = new compilerNode.Pause(this.id(node), node.code, node.reason); + } else if (node instanceof node.Consume) { + res = new compilerNode.Consume(this.id(node), node.fieldName); } else { - assert(node instanceof llparse.node.Match); + assert(node instanceof node.Match); - const trie = new llparse.Trie(node.name); + const trie = new Trie(node.name); - const combined = trie.combine(node[kCases]); + const combined = trie.combine(node.cases); if (combined === null) { - res = new compiler.node.Empty(this.id(node)); + res = new compilerNode.Empty(this.id(node)); } else { list = []; res = this.buildTrie(node, combined, list, true); @@ -87,23 +91,23 @@ class NodeTranslator extends Stage { // Build `invoke`'s map if (node instanceof llparse.node.Invoke) { - res.map = {}; - Object.keys(node[kMap]).forEach((key) => { - res.map[key] = this.buildNode(node[kMap][key], null); + Object.keys(node.map).forEach((key) => { + res.add(key, this.buildNode(node.map[key], undefined)); }); } if (node instanceof llparse.node.Error) { - assert.strictEqual(node[kOtherwise], null); + assert.strictEqual(node.getOtherwise(), undefined); } else { - assert.notStrictEqual(node[kOtherwise], null, + const otherwise = node.getOtherwise(); + assert.notStrictEqual(node.otherwise, undefined, `Node "${node.name}" must have \`.otherwise()\`/\`.skipTo()\``); if (!list) list = [ last ]; - const otherwise = this.buildNode(node[kOtherwise].next, null); + const otherwise = this.buildNode(otherwise.next, null); list.forEach((entry) => { - entry.setOtherwise(otherwise, node[kOtherwise].skip); + entry.setOtherwise(otherwise, otherwise.skip); }); } @@ -111,7 +115,7 @@ class NodeTranslator extends Stage { // not skip over the input, and lead to nodes with prologue check // (p != endp) - stitch the node straight to its `otherwise` recursively. for (;;) { - if (!(res instanceof compiler.node.Empty)) + if (!(res instanceof compilerNode.Empty)) break; if (res.skip) break; @@ -128,44 +132,52 @@ class NodeTranslator extends Stage { return res; } - buildError(node) { + private buildError(node: node.Error): complerNode.Error { const cacheKey = (node.code >>> 0) + ':' + node.reason; if (this.errorCache.has(cacheKey)) return this.errorCache.get(cacheKey); - const res = new compiler.node.Error(this.id(node), node.code, node.reason); + const res = new compilerNode.Error(this.id(node), node.code, node.reason); this.errorCache.set(cacheKey, res); return res; } - buildTrie(node, trie, list, isRoot = false) { - if (trie.type === 'next') { + private buildTrie(node: node.Match, trie: TrieNode, list: TrieOutputList, + isRoot: boolean = false): compilerNode.Node { + if (trie instanceof TrieNext) { assert(!isRoot); return this.buildNode(trie.next, trie.value); } - if (trie.type === 'single') + if (trie instanceof TrieSingle) { return this.buildSingle(node, trie, list, isRoot); + } + + if (trie instanceof TrieSequence) { + return this.buildSequence(node, trie, list, isRoot); + } - assert.strictEqual(trie.type, 'sequence'); - return this.buildSequence(node, trie, list, isRoot); + throw new Error('Unknown trie node'); } - buildSingle(node, trie, list, isRoot) { + private buildSingle(node: node.Match, trie: TrieSingle, list: TrieOutputList, + isRoot: boolean): compilerNode.Node { assert.notStrictEqual(trie.children.length, 0); // Fast case, every child leads to the same result and no values are passed const bitCheck = this.buildBitCheck(node, trie, list, isRoot); - if (bitCheck) + if (bitCheck) { return bitCheck; + } const res = new compiler.node.Single(this.id(node)); // Break loops - if (isRoot) + if (isRoot) { this.nodes.set(node, res); + } - res.transform = node[kTransform]; + res.transform = node.getTransform(); res.children = trie.children.map((child) => { return { key: child.key, @@ -179,18 +191,23 @@ class NodeTranslator extends Stage { return res; } - buildBitCheck(node, trie, list, isRoot) { - if (trie.children.length < this.options.minCheckSize) + private buildBitCheck(node: node.Match, trie: TrieSingle, + list: TrieOutputList, isRoot: boolean) + : compilerNode.Node | false { + if (trie.children.length < this.options.minCheckSize) { return false; + } const targets = new Map(); const bailout = !trie.children.every((child) => { - if (child.child.value !== null) + if (child.child.value !== null) { return false; + } - if (child.child.type !== 'next') + if (child.child.type !== 'next') { return false; + } const target = child.child.next; if (!targets.has(target)) { @@ -215,8 +232,9 @@ class NodeTranslator extends Stage { return true; }); - if (bailout) + if (bailout) { return false; + } // We've width limit for this optimization if (targets.size >= (1 << this.options.maxCheckWidth)) { @@ -228,10 +246,11 @@ class NodeTranslator extends Stage { const res = new compiler.node.BitCheck(this.id(node)); // Break loops - if (isRoot) + if (isRoot) { this.nodes.set(node, res); + } - res.transform = node[kTransform]; + res.transform = node.getTransform(); const map = []; let totalKeys = 0; @@ -249,17 +268,20 @@ class NodeTranslator extends Stage { return res; } - buildSequence(node, trie, list, isRoot) { + private buildSequence(node: node.Match, trie: TrieSequence, + list: TrieOutputList, isRoot: boolean) + : compilerNode.Node { const res = new compiler.node.Sequence(this.id(node), trie.select); const value = trie.child.type === 'next' ? trie.child.value : null; this.maxSequenceLen = Math.max(this.maxSequenceLen, trie.select.length); // Break loops - if (isRoot) + if (isRoot) { this.nodes.set(node, res); + } - res.transform = node[kTransform]; + res.transform = node.getTransform(); res.next = this.buildTrie(node, trie.child, list); res.value = value; @@ -267,8 +289,8 @@ class NodeTranslator extends Stage { return res; } - checkSignature(node, value) { - assert.strictEqual(node[kSignature], value === null ? 'match' : 'value'); + private checkSignature(node: node.Node, value: number | undefined) { + assert.strictEqual(node.signature, + value === undefined ? 'match' : 'value'); } } -module.exports = NodeTranslator; diff --git a/src/llparse/compiler/stage/span-allocator.js b/src/llparse/compiler/stage/span-allocator.ts similarity index 77% rename from src/llparse/compiler/stage/span-allocator.js rename to src/llparse/compiler/stage/span-allocator.ts index 709cd78..8c18eaf 100644 --- a/src/llparse/compiler/stage/span-allocator.js +++ b/src/llparse/compiler/stage/span-allocator.ts @@ -1,20 +1,35 @@ -'use strict'; +import { Code } from '../code'; +import { Compilation } from '../compilation'; +import { Node } from '../node'; +import { Stage } from './base'; -const assert = require('assert'); +type SpanID = Code; -const Stage = require('./').Stage; -const compiler = require('../'); +interface IActiveResult { + active: activeMap; + spans: ReadonlyArray; +} + +type OverlapMap = Map >; + +export type ColoringMap = ReadonlyMap; + +export interface IColoringResult { + map: ColoringMap; + concurrency: ReadonlyArray; + max: number; +} -class Allocator extends Stage { - constructor(ctx) { +export class Allocator extends Stage { + constructor(ctx: Compilation) { super(ctx, 'span-allocator'); } - id(node) { + private id(node): SpanID { return node.code; } - build() { + public build(): any { const root = this.ctx.stageResults['node-translator'].root; const nodes = this.getNodes(root); const info = this.computeActive(nodes); @@ -24,7 +39,7 @@ class Allocator extends Stage { return color; } - getNodes(root) { + private getNodes(root: Node): ReadonlyArray { const res = new Set(); const queue = [ root ]; while (queue.length !== 0) { @@ -38,7 +53,7 @@ class Allocator extends Stage { return Array.from(res); } - computeActive(nodes) { + private computeActive(nodes: ReadonlyArray): IActiveResult { const activeMap = new Map(); nodes.forEach(node => activeMap.set(node, new Set())); @@ -91,9 +106,9 @@ class Allocator extends Stage { return { active: activeMap, spans: Array.from(spans) }; } - computeOverlap(info) { + private computeOverlap(info: IActiveResult): OverlapMap { const active = info.active; - const overlap = new Map(); + const overlap: OverlapMap = new Map(); info.spans.forEach(span => overlap.set(span, new Set())); @@ -113,7 +128,8 @@ class Allocator extends Stage { return overlap; } - color(spans, overlapMap) { + private color(spans: ReadonlyArray, overlapMap: OverlapMap) + : IColoringResult { let max = -1; const colors = new Map(); @@ -142,7 +158,7 @@ class Allocator extends Stage { return i; }; - const res = new Map(); + const res: Map = new Map(); spans.forEach(span => res.set(span, allocate(span))); @@ -155,4 +171,3 @@ class Allocator extends Stage { return { map: res, concurrency, max }; } } -module.exports = Allocator; diff --git a/src/llparse/compiler/stage/span-builder.js b/src/llparse/compiler/stage/span-builder.ts similarity index 85% rename from src/llparse/compiler/stage/span-builder.js rename to src/llparse/compiler/stage/span-builder.ts index eed306b..5b9a9fb 100644 --- a/src/llparse/compiler/stage/span-builder.js +++ b/src/llparse/compiler/stage/span-builder.ts @@ -1,26 +1,20 @@ -'use strict'; +import * as assert from 'assert'; -const assert = require('assert'); - -const Stage = require('./').Stage; -const llparse = require('../../'); - -const constants = llparse.constants; - -const INT = constants.INT; -const TYPE_INPUT = constants.TYPE_INPUT; -const TYPE_OUTPUT = constants.TYPE_OUTPUT; -const TYPE_ERROR = constants.TYPE_ERROR; - -const SPAN_START_PREFIX = constants.SPAN_START_PREFIX; -const SPAN_CB_PREFIX = constants.SPAN_CB_PREFIX; +import { Code } from '../code'; +import { + INT, TYPE_INPUT, TYPE_OUTPUT, TYPE_ERROR, + SPAN_START_PREFIX, SPAN_CB_PREFIX +} from '../../constants'; +import { INodePosition, Compilation, Func, BasicBlock } from '../compilation'; +import { Node } from '../node'; +import { Stage } from './base'; class Builder extends Stage { - constructor(ctx) { + constructor(ctx: Compilation) { super(ctx, 'span-builder'); } - build() { + public build(): any { this.buildFields(); return { @@ -32,7 +26,7 @@ class Builder extends Stage { }; } - buildFields() { + private buildFields(): void { const colors = this.ctx.stageResults['span-allocator']; const callbackType = this.ctx.signature.callback.span.ptr(); @@ -52,7 +46,7 @@ class Builder extends Stage { // Nodes - buildSpanStart(fn, body, code) { + private buildSpanStart(fn: Func, body: BasicBlock, code: Code): BasicBlock { const colors = this.ctx.stageResults['span-allocator']; const index = colors.map.get(code); @@ -69,7 +63,7 @@ class Builder extends Stage { return body; } - buildSpanEnd(fn, body, code) { + private buildSpanEnd(fn: Func, body: BasicBlock, code: Code): BasicBlock { const colors = this.ctx.stageResults['span-allocator']; const index = colors.map.get(code); @@ -117,7 +111,8 @@ class Builder extends Stage { }; } - buildError(fn, body, pos, code) { + private buildError(fn: Func, body: BasicBlock, pos: INodePositon, + code: Code): void { const reason = this.ctx.cstring('Span callback error'); const cast = body.getelementptr(reason, INT.val(0), INT.val(0), true); @@ -132,7 +127,7 @@ class Builder extends Stage { // ${prefix}_execute - buildPrologue(fn, body) { + private buildPrologue(fn: Func, body: BasicBlock): BasicBlock { const colors = this.ctx.stageResults['span-allocator']; colors.concurrency.forEach((_, index) => { @@ -156,7 +151,7 @@ class Builder extends Stage { return body; } - buildEpilogue(fn, body) { + private buildEpilogue(fn: Func, body: BasicBlock): BasicBlock { const colors = this.ctx.stageResults['span-allocator']; const callbackType = this.ctx.signature.callback.span.ptr(); @@ -224,12 +219,12 @@ class Builder extends Stage { return body; } - startField(fn, body, index) { + private startField(fn: Func, body: BasicBlock, index: number): values.Value { return this.ctx.stateField(fn, body, SPAN_START_PREFIX + index); } - callbackField(fn, body, index) { + private callbackField(fn: Func, body: BasicBlock, index: number) + : values.Value { return this.ctx.stateField(fn, body, SPAN_CB_PREFIX + index); } } -module.exports = Builder; diff --git a/src/llparse/node/base.ts b/src/llparse/node/base.ts index 24c15c7..125ceee 100644 --- a/src/llparse/node/base.ts +++ b/src/llparse/node/base.ts @@ -5,14 +5,12 @@ import * as code from '../code'; import * as node from './'; export abstract class Node { - private privOtherwise: Node | undefinded; + private privOtherwise: cases.Otherwise | undefinded; constructor(public readonly name: string, public readonly signature: code.Signature) { } - public get otherwise(): Node | undefined { return this.privOtherwise; } - public otherwise(next: Node): this { this.checkIsMatch(next, '.otherwise()'); @@ -33,6 +31,12 @@ export abstract class Node { return this; } + // Internal API below + + public getOtherwise(): cases.Otherwise | undefined { + return this.privOtherwise; + } + protected checkIsMatch(next: Node, method: string): void { if (!(next instanceof node.Invoke)) return; diff --git a/src/llparse/node/match.ts b/src/llparse/node/match.ts index 4c269b4..4d9bd87 100644 --- a/src/llparse/node/match.ts +++ b/src/llparse/node/match.ts @@ -6,15 +6,15 @@ import { Invoke } from './invoke'; import { Node } from './node'; export class Match extends Node { - private privTransformation: Transform | undefined; + private privTransform: Transform | undefined; private privCases: Case[] = []; constructor(name: strings) { super(name, 'match'); } - public get transformation(): Transform | undefined { - return this.privTransformation; + public getTransform(): Transform | undefined { + return this.privTransform; } public get cases(): ReadonlyArray { @@ -22,10 +22,10 @@ export class Match extends Node { } public transform(t: Transform): this { - assert.strictEqual(this.privTransformation, undefined, + assert.strictEqual(this.privTransform, undefined, 'Can\'t apply transform twice'); - this.privTransformation = t; + this.privTransform = t; return this; } diff --git a/src/llparse/node/span-end.ts b/src/llparse/node/span-end.ts index 76c4cef..7d3fc33 100644 --- a/src/llparse/node/span-end.ts +++ b/src/llparse/node/span-end.ts @@ -2,7 +2,7 @@ import { Code } from '../code'; import { Node } from './node'; export class SpanEnd extends Node { - constructor(code: Code) { + constructor(public readonly code: Code) { super('span_end_' + code.name, 'match'); } } diff --git a/src/llparse/node/span-start.ts b/src/llparse/node/span-start.ts index bbf91a4..27eb25f 100644 --- a/src/llparse/node/span-start.ts +++ b/src/llparse/node/span-start.ts @@ -2,7 +2,7 @@ import { Code } from '../code'; import { Node } from './node'; export class SpanStart extends Node { - constructor(code: Code) { - super('span_sta,t_' + code.name, 'match'); + constructor(public readonly code: Code) { + super('span_start_' + code.name, 'match'); } } From 8706abd2b3872315d533efabc057745741fdc532 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 19:58:07 -0400 Subject: [PATCH 280/281] save --- .../{compilation.js => compilation.ts} | 178 ++++++++++-------- 1 file changed, 95 insertions(+), 83 deletions(-) rename src/llparse/compiler/{compilation.js => compilation.ts} (70%) diff --git a/src/llparse/compiler/compilation.js b/src/llparse/compiler/compilation.ts similarity index 70% rename from src/llparse/compiler/compilation.js rename to src/llparse/compiler/compilation.ts index af80636..85bd4fa 100644 --- a/src/llparse/compiler/compilation.js +++ b/src/llparse/compiler/compilation.ts @@ -1,40 +1,44 @@ -'use strict'; +import * as asert from 'assert'; +import * as bitcode from 'bitcode'; +import { Buffer } from 'buffer'; -const assert = require('assert'); -const bitcode = require('bitcode'); +import { + CCONV, -const llparse = require('../'); -const compiler = require('./'); -const constants = llparse.constants; + INT, TYPE_INPUT, TYPE_OUTPUT, TYPE_MATCH, TYPE_INDEX, TYPE_ERROR, TYPE_REASON, + TYPE_DATA, -const CCONV = constants.CCONV; + ATTR_STATE, ATTR_POS, ATTR_ENDPOS, -const INT = constants.INT; -const TYPE_INPUT = constants.TYPE_INPUT; -const TYPE_OUTPUT = constants.TYPE_OUTPUT; -const TYPE_MATCH = constants.TYPE_MATCH; -const TYPE_INDEX = constants.TYPE_INDEX; -const TYPE_ERROR = constants.TYPE_ERROR; -const TYPE_REASON = constants.TYPE_REASON; -const TYPE_DATA = constants.TYPE_DATA; + ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH, +} from '../constants'; -const ATTR_STATE = constants.ATTR_STATE; -const ATTR_POS = constants.ATTR_POS; -const ATTR_ENDPOS = constants.ATTR_ENDPOS; +import * as code from '../code'; +import * as compilerCode from './code'; -const ARG_STATE = constants.ARG_STATE; -const ARG_POS = constants.ARG_POS; -const ARG_ENDPOS = constants.ARG_ENDPOS; -const ARG_MATCH = constants.ARG_MATCH; +import builder = bitcode.builder; +import Declaration = builder.values.constants.Declaration; +import Func = builder.values.constants.Func; +import BasicBlock = builder.values.constants.Func; +import Value = builder.values.Value; +import Type = builder.types.Type; + +export { bitcode.values as values, Func, BasicBlock }; + +export interface INodeID { + name: string; + sourceName: string; +} export interface INodePosition { - current: values.Value; - next?: values.Value; + current: Value; + next?: Value; } +export type Weight = 'likely' | 'unlikely' | number; -class Compilation { - constructor(options) { +export class Compilation { + constructor(options: any) { this.options = Object.assign({}, options); this.bitcode = new bitcode.Module(); @@ -79,7 +83,7 @@ class Compilation { this.stageResults = {}; } - id(name, prefix = '', postfix = '') { + public id(name, prefix = '', postfix = ''): INodeID { let res = prefix + name + postfix; if (this.namespace.has(res)) { let i; @@ -93,7 +97,7 @@ class Compilation { return { name: res, sourceName: name }; } - build() { + public build(): void { // Private fields this.declareField(this.signature.node.ptr(), '_current', (type, ctx) => ctx.stageResults['node-builder'].entry); @@ -126,11 +130,11 @@ class Compilation { this.buildStages(this.options.stages.after); } - end() { + public end(): Buffer { return this.bitcode.build(this.ir); } - buildCState() { + public buildCState(): string { const out = []; out.push(`typedef struct ${this.prefix}_state_s ${this.prefix}_state_t;`); @@ -165,36 +169,38 @@ class Compilation { return out.join('\n'); } - buildStages(stages) { + private buildStages(stages): void { stages.forEach(Stage => this.buildStage(Stage)); } - buildStage(Stage) { + private buildStage(Stage): void { const stage = new Stage(this); this.stageResults[stage.name] = stage.build(); } - declareFunction(signature, name) { + public declareFunction(signature: builder.types.Signature, + name: string): Func { const res = signature.declareFunction(name); this.bitcode.add(res); return res; } - defineFunction(signature, name, paramNames) { + public defineFunction(signature: builder.types.Signature, name: string, + paramNames: ReadonlyArray): Func { const res = signature.defineFunction(name, paramNames); this.bitcode.add(res); return res; } // TODO(indutny): find better place for it? - translateCode(code) { + private translateCode(code: code.Code): compilerCode.Code { // User callbacks - if (code instanceof llparse.code.Match) - return new compiler.code.Match(code.name); - else if (code instanceof llparse.code.Value) - return new compiler.code.Value(code.name); - else if (code instanceof llparse.code.Span) - return new compiler.code.Span(code.name); + if (code instanceof code.Match) + return new compilerCode.Match(code.name); + else if (code instanceof code.Value) + return new compilerCode.Value(code.name); + else if (code instanceof code.Span) + return new compilerCode.Span(code.name); // Internal helpers let name = code.name; @@ -202,25 +208,25 @@ class Compilation { name += '_' + code.field; const id = this.id(name, 'c_').name; - if (code instanceof llparse.code.IsEqual) - return new compiler.code.IsEqual(id, code.field, code.value); - else if (code instanceof llparse.code.Load) - return new compiler.code.Load(id, code.field); - else if (code instanceof llparse.code.MulAdd) - return new compiler.code.MulAdd(id, code.field, code.options); - else if (code instanceof llparse.code.Or) - return new compiler.code.Or(id, code.field, code.value); - else if (code instanceof llparse.code.Store) - return new compiler.code.Store(id, code.field); - else if (code instanceof llparse.code.Test) - return new compiler.code.Test(id, code.field, code.value); - else if (code instanceof llparse.code.Update) - return new compiler.code.Update(id, code.field, code.value); + if (code instanceof code.IsEqual) + return new compilerCode.IsEqual(id, code.field, code.value); + else if (code instanceof code.Load) + return new compilerCode.Load(id, code.field); + else if (code instanceof code.MulAdd) + return new compilerCode.MulAdd(id, code.field, code.options); + else if (code instanceof code.Or) + return new compilerCode.Or(id, code.field, code.value); + else if (code instanceof code.Store) + return new compilerCode.Store(id, code.field); + else if (code instanceof code.Test) + return new compilerCode.Test(id, code.field, code.value); + else if (code instanceof code.Update) + return new compilerCode.Update(id, code.field, code.value); else throw new Error('Unexpected code type of: ' + code.name); } - buildCode(code) { + public buildCode(code: compilerCode.Code): Declaration { const native = this.translateCode(code); const signatures = this.signature.callback; @@ -252,7 +258,8 @@ class Compilation { return fn; } - buildCodeWithBody(code, signature) { + private buildCodeWithBody(code: compilerCode.Code, + signature: builder.types.Signature): Func { const args = [ ARG_STATE, ARG_POS, ARG_ENDPOS ]; if (code.signature === 'value') @@ -270,21 +277,21 @@ class Compilation { return fn; } - cstring(string) { - if (this.cstringCache.has(string)) { - return this.cstringCache.get(string); + public cstring(value: string): builder.values.Global { + if (this.cstringCache.has(value)) { + return this.cstringCache.get(value)!; } - const res = this.addGlobalConst('cstr', this.ir.cstring(string)); - this.cstringCache.set(string, res); + const res = this.addGlobalConst('cstr', this.ir.cstring(value)); + this.cstringCache.set(value, res); return res; } - blob(data) { + public blob(data: Buffer): builder.values.Global { return this.addGlobalConst('blob', this.ir.blob(data)); } - addGlobalConst(name, value) { + private addGlobalConst(name: string, value: Value): builder.values.Global { const id = this.id(name, 'g_').name; const glob = this.ir.global(value.ty.ptr(), id, value); glob.linkage = 'internal'; @@ -293,15 +300,15 @@ class Compilation { return glob; } - debug(fn, body, string) { + public debug(fn: Func, body: BasicBlock, message: string): BasicBlock { if (!this.options.debug) return body; - const str = this.cstring(string); + const str = this.cstring(message); const cast = body.getelementptr(str, INT.val(0), INT.val(0)); // Lazily declare debug method - if (this.debugMethod === null) { + if (this.debugMethod === undefined) { const sig = this.ir.signature(this.ir.void(), [ this.state.ptr(), TYPE_INPUT, TYPE_INPUT, TYPE_INPUT ]); @@ -320,7 +327,7 @@ class Compilation { return body; } - fn(signature, name) { + public fn(signature: builder.types.Signature, name: string): Func { name = this.prefix + '__' + name; let fn; @@ -355,7 +362,7 @@ class Compilation { return fn; } - toBranchWeight(value) { + private toBranchWeight(value: Weight): number { if (value === 'likely') return 0x10000; else if (value === 'unlikely') @@ -365,7 +372,8 @@ class Compilation { return value; } - branch(body, cmp, weights) { + // TODO(indutny): return type + public branch(body: BasicBlock, cmp: Value, weights: ReadonlyArray) { const onTrue = body.parent.createBlock('true'); const onFalse = body.parent.createBlock('false'); const branch = body.branch(cmp, onTrue, onFalse); @@ -390,7 +398,9 @@ class Compilation { }; } - buildSwitch(body, what, values, weights) { + // TODO(indutny): return type + buildSwitch(body: BasicBlock, what: Value, values: ReadonlyArray, + weights: ReadonlyArray) { const cases = []; const blocks = []; values.forEach((value, i) => { @@ -425,7 +435,9 @@ class Compilation { }; } - buildTransform(transform, body, current) { + // TODO(indutny): return type + public buildTransform(transform: Transform, body: BasicBlock, + current: Value) { if (transform.name === 'to_lower_unsafe') { current = body.binop('or', current, TYPE_INPUT.to.val(0x20)); } else { @@ -435,7 +447,8 @@ class Compilation { return { body, current }; } - truncate(body, from, toType, isSigned = false) { + public truncate(body: BasicBlock, from: Value, toType: Type, + isSigned: boolean = false): Value { const fromTy = from.ty; assert(toType.isInt()); assert(fromTy.isInt()); @@ -460,39 +473,38 @@ class Compilation { return res; } - load(fn, body, field) { + public load(fn: Func, body: BasicBlock, field: string): Value { const lookup = this.stateField(fn, body, field); return body.load(lookup); } - store(fn, body, field, value) { + public store(fn: Func, body: BasicBlock, field: string, value: Value): Value { const lookup = this.stateField(fn, body, field); body.store(value, lookup); } - declareField(type, name, init) { + public declareField(ty: Type, name: string, init: Value): void { this.state.addField(type, name); this.initializers.push({ type, name, init }); } - initFields(fn, body) { + public initFields(fn: Func, body: BasicBlock): void { this.initializers.forEach((entry) => { const field = this.stateField(fn, body, entry.name); body.store(entry.init(entry.type, this), field); }); } - stateField(fn, body, name) { + public stateField(fn: Func, body: BasicBlock, name: string): Value { const stateArg = this.stateArg(fn); return body.getelementptr(stateArg, INT.val(0), INT.val(this.state.lookupField(name).index), true); } - stateArg(fn) { return fn.getArgument(ARG_STATE); } - posArg(fn) { return fn.getArgument(ARG_POS); } - endPosArg(fn) { return fn.getArgument(ARG_ENDPOS); } - matchArg(fn) { return fn.getArgument(ARG_MATCH); } + public stateArg(fn: Func): Value { return fn.getArgument(ARG_STATE); } + public posArg(fn: Func): Value { return fn.getArgument(ARG_POS); } + public endPosArg(fn: Func): Value { return fn.getArgument(ARG_ENDPOS); } + public matchArg(fn: Func): Value { return fn.getArgument(ARG_MATCH); } } -module.exports = Compilation; From 8b78f6647616e8f4e65aa0ab4f4d63f7e6f80e1c Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Wed, 14 Mar 2018 20:34:00 -0400 Subject: [PATCH 281/281] save --- src/llparse.ts | 4 +- src/llparse/compiler/compilation.ts | 12 +++--- src/llparse/compiler/compiler.ts | 2 +- src/llparse/compiler/node/error.ts | 2 +- src/llparse/compiler/node/node-context.ts | 22 +++++----- src/llparse/compiler/node/single.ts | 2 +- .../compiler/stage/node-loop-checker.ts | 2 +- src/llparse/compiler/stage/node-translator.ts | 4 +- src/llparse/compiler/stage/span-allocator.ts | 2 +- src/llparse/constants.ts | 42 ++++++++++--------- src/llparse/node/invoke.ts | 2 +- src/llparse/node/match.ts | 2 +- src/llparse/trie/next.ts | 4 +- src/llparse/trie/sequence.ts | 4 +- src/llparse/trie/trie.ts | 4 +- src/llparse/utils.ts | 8 ++-- 16 files changed, 63 insertions(+), 55 deletions(-) diff --git a/src/llparse.ts b/src/llparse.ts index fadafd9..53b70c1 100644 --- a/src/llparse.ts +++ b/src/llparse.ts @@ -68,10 +68,10 @@ export class LLParse { throw new Error(`Duplicate property with a name: "${name}"`); } - if (!internal.constants.USER_TYPES.hasOwnProperty(ty)) { + if (!internal.constants.USER_TYPES.has(ty)) { throw new Error(`Unknown property type: "${ty}"`); } - const bitcodeTy = internal.constants.USER_TYPES[ty]; + const bitcodeTy = internal.constants.USER_TYPES.get(ty)!; props.set.add(name); props.list.push({ ty: bitcodeTy, name }); diff --git a/src/llparse/compiler/compilation.ts b/src/llparse/compiler/compilation.ts index 85bd4fa..c2d34b2 100644 --- a/src/llparse/compiler/compilation.ts +++ b/src/llparse/compiler/compilation.ts @@ -12,18 +12,20 @@ import { ARG_STATE, ARG_POS, ARG_ENDPOS, ARG_MATCH, } from '../constants'; +import { Transform } from './transform'; import * as code from '../code'; import * as compilerCode from './code'; import builder = bitcode.builder; -import Declaration = builder.values.constants.Declaration; -import Func = builder.values.constants.Func; -import BasicBlock = builder.values.constants.Func; -import Value = builder.values.Value; +import values = builder.values; +import Declaration = values.constants.Declaration; +import Func = values.constants.Func; +import BasicBlock = values.constants.Func; +import Value = values.Value; import Type = builder.types.Type; -export { bitcode.values as values, Func, BasicBlock }; +export { values, Func, BasicBlock }; export interface INodeID { name: string; diff --git a/src/llparse/compiler/compiler.ts b/src/llparse/compiler/compiler.ts index ef61b4b..d9447ac 100644 --- a/src/llparse/compiler/compiler.ts +++ b/src/llparse/compiler/compiler.ts @@ -26,7 +26,7 @@ export interface ICompilerStateProperty { export interface ICompilerOptions { prefix: string; properties: ReadonlyArray; - debug: options.debug === undefined ? false : options.debug + debug: boolean; } export interface ICompilerBuildResult { diff --git a/src/llparse/compiler/node/error.ts b/src/llparse/compiler/node/error.ts index 65bacc9..af0159a 100644 --- a/src/llparse/compiler/node/error.ts +++ b/src/llparse/compiler/node/error.ts @@ -2,7 +2,7 @@ import { Compilation, BasicBlock, INodeID } from '../compilation'; import { Node, INodeChild } from './base'; export class Error extends Node { - constructor(id: NodeID, private readonly code: number, + constructor(id: INodeID, private readonly code: number, private readonly reason: string) { super('error', id); diff --git a/src/llparse/compiler/node/node-context.ts b/src/llparse/compiler/node/node-context.ts index babcaf4..81b1658 100644 --- a/src/llparse/compiler/node/node-context.ts +++ b/src/llparse/compiler/node/node-context.ts @@ -1,7 +1,7 @@ import { Builder } from 'bitcode'; import { - Compilation, INodePosition, Func, types, values, + BasicBlock, Compilation, INodePosition, Func, types, values, } from '../compilation'; import { Node } from './base'; @@ -43,20 +43,22 @@ export class NodeContext { this.INVARIANT_GROUP = ctx.INVARIANT_GROUP; } - debug(body, message) { + public debug(body: BasicBlock, message: string): void { this.compilation.debug(this.fn, body, `${this.name}: ${message}`); } // Just proxy - stateField(body, name) { + public stateField(body: BasicBlock, name: string): values.Value { return this.compilation.stateField(this.fn, body, name); } - cstring(...args) { return this.compilation.cstring(...args); } - blob(...args) { return this.compilation.blob(...args); } - addGlobalConst(...args) { return this.compilation.addGlobalConst(...args); } - truncate(...args) { return this.compilation.truncate(...args); } - call(...args) { return this.compilation.call(...args); } - buildSwitch(...args) { return this.compilation.buildSwitch(...args); } - branch(...args) { return this.compilation.branch(...args); } + public cstring(...args) { return this.compilation.cstring(...args); } + public blob(...args) { return this.compilation.blob(...args); } + public addGlobalConst(...args) { + return this.compilation.addGlobalConst(...args); + } + public truncate(...args) { return this.compilation.truncate(...args); } + public call(...args) { return this.compilation.call(...args); } + public buildSwitch(...args) { return this.compilation.buildSwitch(...args); } + public branch(...args) { return this.compilation.branch(...args); } } diff --git a/src/llparse/compiler/node/single.ts b/src/llparse/compiler/node/single.ts index 9ac89ee..e24e782 100644 --- a/src/llparse/compiler/node/single.ts +++ b/src/llparse/compiler/node/single.ts @@ -4,7 +4,7 @@ import { Node, INodeChild } from './base'; export class Single extends Node { private readonly children: INodeChild[] = []; - constructor(id: NodeID) { + constructor(id: INodeID) { super('single', id); } diff --git a/src/llparse/compiler/stage/node-loop-checker.ts b/src/llparse/compiler/stage/node-loop-checker.ts index b25204f..276edbb 100644 --- a/src/llparse/compiler/stage/node-loop-checker.ts +++ b/src/llparse/compiler/stage/node-loop-checker.ts @@ -8,7 +8,7 @@ interface IQueueElem { } export class NodeLoopChecker extends Stage { - private readonly reachableMap > = new Map(); + private readonly reachable: Map > = new Map(); constructor(ctx: Compilation) { super(ctx, 'node-loop-checker'); diff --git a/src/llparse/compiler/stage/node-translator.ts b/src/llparse/compiler/stage/node-translator.ts index a4b57a2..6e66b0b 100644 --- a/src/llparse/compiler/stage/node-translator.ts +++ b/src/llparse/compiler/stage/node-translator.ts @@ -21,7 +21,7 @@ export interface INodeTranslatorOptions { } export class NodeTranslator extends Stage { - private readonly INodeTranslatorOptions options; + private readonly options: INodeTranslatorOptions; private readonly nodes: Map = new Map(); private readonly errorCache: Map = new Map(); private maxSequenceLen: number = 0; @@ -55,7 +55,7 @@ export class NodeTranslator extends Stage { } let res; - let list: TrieOutputList || undefined; + let list: TrieOutputList | undefined; let last; if (node instanceof node.Invoke) { res = new compilerNode.Invoke(this.id(node), node.code); diff --git a/src/llparse/compiler/stage/span-allocator.ts b/src/llparse/compiler/stage/span-allocator.ts index 8c18eaf..48ffaa7 100644 --- a/src/llparse/compiler/stage/span-allocator.ts +++ b/src/llparse/compiler/stage/span-allocator.ts @@ -3,7 +3,7 @@ import { Compilation } from '../compilation'; import { Node } from '../node'; import { Stage } from './base'; -type SpanID = Code; +export type SpanID = Code; interface IActiveResult { active: activeMap; diff --git a/src/llparse/constants.ts b/src/llparse/constants.ts index fcc1ea5..34c4183 100644 --- a/src/llparse/constants.ts +++ b/src/llparse/constants.ts @@ -1,21 +1,23 @@ -import { Attribute, Builder as IR, CallingConv } from 'bitcode'; +import { Attribute, Builder as IR, CallingConv, types } from 'bitcode'; + +import Type = types.Type; export const CCONV: CallingConv = 'fastcc'; -export const BOOL = IR.i(1); -export const INT = IR.i(32); -export const TYPE_INPUT = IR.i(8).ptr(); -export const TYPE_OUTPUT = IR.i(8).ptr(); -export const TYPE_MATCH = export const INT; -export const TYPE_INDEX = IR.i(64); -export const TYPE_ERROR = export const INT; -export const TYPE_REASON = IR.i(8).ptr(); -export const TYPE_DATA = IR.i(8).ptr(); -export const TYPE_STATUS = IR.i(32); +export const BOOL: Type = IR.i(1); +export const INT: Type = IR.i(32); +export const TYPE_INPUT: Type = IR.i(8).ptr(); +export const TYPE_OUTPUT: Type = IR.i(8).ptr(); +export const TYPE_MATCH: Type = INT; +export const TYPE_INDEX: Type = IR.i(64); +export const TYPE_ERROR: Type = INT; +export const TYPE_REASON: Type = IR.i(8).ptr(); +export const TYPE_DATA: Type = IR.i(8).ptr(); +export const TYPE_STATUS: Type = IR.i(32); // TODO(indutny): although it works through zero extension, is there any // cross-platform way to do it? -export const TYPE_INTPTR = IR.i(64); +export const TYPE_INTPTR: Type = IR.i(64); export const ARG_STATE = 's'; export const ARG_POS = 'p'; @@ -27,7 +29,7 @@ export const ARG_UNUSED = '_unused'; export const ATTR_STATE: ReadonlyArray = [ 'noalias', 'nonnull' ]; export const ATTR_POS: ReadonlyArray = [ - noalias', 'nonnull', 'readonly', + 'noalias', 'nonnull', 'readonly', ]; export const ATTR_ENDPOS: ReadonlyArray = [ 'noalias', 'nonnull', 'readnone', @@ -45,10 +47,10 @@ export const SPAN_CB_PREFIX = '_span_cb'; export const DEFAULT_TRANSLATOR_MIN_CHECK_SIZE = 32; export const DEFAULT_TRANSLATOR_MAX_CHECK_WIDTH = 4; -export const USER_TYPES = { - i8: IR.i(8), - i16: IR.i(16), - i32: IR.i(32), - i64: IR.i(64), - ptr: IR.i(8).ptr(), -}; +export const USER_TYPES: Map = new Map([ + [ 'i8', IR.i(8) ], + [ 'i16', IR.i(16) ], + [ 'i32', IR.i(32) ], + [ 'i64', IR.i(64) ], + [ 'ptr', IR.i(8).ptr() ], +]); diff --git a/src/llparse/node/invoke.ts b/src/llparse/node/invoke.ts index 60373a0..f5b06df 100644 --- a/src/llparse/node/invoke.ts +++ b/src/llparse/node/invoke.ts @@ -10,7 +10,7 @@ export class Invoke extends Node { map: { [key: number]: Node }, otherwise?: Node) { super('invoke_' + code.name, code.signature); - const storedMap: Map = new Map(): + const storedMap: Map = new Map(); Object.keys(map).forEach((key) => { assert.strictEqual(key, key | 0, 'Only integer keys are allowed in `.invoke()`\'s map'); diff --git a/src/llparse/node/match.ts b/src/llparse/node/match.ts index 4d9bd87..bfc4ad1 100644 --- a/src/llparse/node/match.ts +++ b/src/llparse/node/match.ts @@ -9,7 +9,7 @@ export class Match extends Node { private privTransform: Transform | undefined; private privCases: Case[] = []; - constructor(name: strings) { + constructor(name: string) { super(name, 'match'); } diff --git a/src/llparse/trie/next.ts b/src/llparse/trie/next.ts index eafe669..08bfc69 100644 --- a/src/llparse/trie/next.ts +++ b/src/llparse/trie/next.ts @@ -2,8 +2,8 @@ import { Node } from '../node'; import { TrieNode } from './node'; export class TrieNext extends TrieNode { - constructor(public readonly value?: number, - public readonly next: Node) { + constructor(value: number | undefined, public readonly next: Node) { super('sequence'); + this.value = value; } } diff --git a/src/llparse/trie/sequence.ts b/src/llparse/trie/sequence.ts index 3e52019..834b6d7 100644 --- a/src/llparse/trie/sequence.ts +++ b/src/llparse/trie/sequence.ts @@ -1,8 +1,8 @@ -import { Buffer } from Buffer; +import { Buffer } from 'buffer'; import { TrieNode } from './node'; export class TrieSequence extends TrieNode { - public readonly child: TrieNode; + public child: TrieNode | undefined; constructor(public readonly select: Buffer) { super('sequence'); diff --git a/src/llparse/trie/trie.ts b/src/llparse/trie/trie.ts index 60a037f..1676207 100644 --- a/src/llparse/trie/trie.ts +++ b/src/llparse/trie/trie.ts @@ -8,7 +8,7 @@ import { TrieSequence } from './sequence'; import { TrieSingle } from './single'; type LinearizeList = ReadonlyArray; -type Path = ReadonlyArray; +type Path = ReadonlyArray; export class Trie { constructor(private readonly name: string) { @@ -93,7 +93,7 @@ export class Trie { const key = item.key[0]; if (keys.has(key)) - keys.get(key).push(item); + keys.get(key)!.push(item); else keys.set(key, [ item ]); } diff --git a/src/llparse/utils.ts b/src/llparse/utils.ts index 4722589..d1b46d3 100644 --- a/src/llparse/utils.ts +++ b/src/llparse/utils.ts @@ -5,7 +5,8 @@ import { Node } from './node'; export function toBuffer(value: number | string): Buffer { if (typeof value === 'number') { - assert(0 <= value <= 255, 'Invalid char value, must be between 0 and 255'); + assert(0 <= value && value <= 255, + 'Invalid char value, must be between 0 and 255'); assert.strictEqual(value, value | 0, 'Invalid char value, must be integer'); return Buffer.from([ value ]); } else { @@ -32,7 +33,8 @@ export interface ILookupResult { } export function buildLookupTable(wordWidth: number, charWidth: number, - list: Readonly): ILookupTableResult { + list: ReadonlyArray) + : ILookupResult { // Entry values: // 0 - no hit // 1 - map[0] @@ -55,7 +57,7 @@ export function buildLookupTable(wordWidth: number, charWidth: number, list.forEach((entry, i) => { const val = i + 1; - entry.forEach((key) => { + entry.keys.forEach((key) => { const index = key >> indexShift; const shift = (key & shiftMask) * shiftMul;