Skip to content
Merged
Prev Previous commit
Next Next commit
feat(core): use Markdown pipelines in ast, metadata, and json
The configured remark plugins run on each whole document once `ast`
parses it, and `metadata` and `json` parse and serialise Markdown with
their syntax.

Assisted-by: Claude Opus 5.5 <noreply@anthropic.com>
  • Loading branch information
ovflowd committed Oct 8, 2026
commit 46a293811688ffc9dc1a9f77f469e75338b27064
45 changes: 45 additions & 0 deletions packages/core/src/generators/ast/__tests__/generate.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,19 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, sep } from 'node:path';
import { after, before, describe, it } from 'node:test';
import { pathToFileURL } from 'node:url';

import { loadGenerator } from '#generators/loader.mjs';
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';

import { STABILITY_INDEX_URL } from '../constants.mjs';
import { processChunk } from '../generate.mjs';

const ast = await loadGenerator(import.meta.resolve('../index.mjs'));

// Files are parsed with the pipeline of `ast`
await loadMarkdownPlugins(ast);

let dir;

const toPosixPath = value => value.split(sep).join('/');
Expand Down Expand Up @@ -157,6 +166,42 @@ describe('processChunk', () => {
});
});

describe('remark plugins', () => {
before(async () => {
// Records the file it runs on, and the headings of its document
await writeFile(
join(dir, 'recorder.mjs'),
`export default () => (tree, file) => {
const headings = tree.children.filter(node => node.type === 'heading');

tree.data = { recorded: file.path + ':' + headings.length };
};`
);

await loadMarkdownPlugins(ast, {
remarkPlugins: [pathToFileURL(join(dir, 'recorder.mjs')).href],
});
});

after(() => loadMarkdownPlugins(ast));

it('run on each whole document, knowing its file', async () => {
const tuple = await file('plugged.md', '# A\n\n## B\n\n## C\n');

const { tree } = await process(tuple);

assert.strictEqual(tree.data.recorded, `${tuple[0]}:3`);
});

it('run on MDX documents too', async () => {
const tuple = await file('plugged.mdx', '# A\n\n<B />\n');

const { tree } = await process(tuple);

assert.strictEqual(tree.data.recorded, `${tuple[0]}:1`);
});
});

describe('chunking', () => {
it('only processes the requested indices', async () => {
const a = await file('a.md', '# A\n');
Expand Down
10 changes: 7 additions & 3 deletions packages/core/src/generators/ast/generate.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import { parse as parseYaml } from 'yaml';

import getConfig from '#utils/configuration/index.mjs';
import { withExt } from '#utils/file.mjs';
import { getProcessor } from '#utils/markdown/processor.mjs';
import { QUERIES } from '#utils/queries/index.mjs';
import { getRemark as remark, getRemarkMdx } from '#utils/remark.mjs';

import { STABILITY_INDEX_URL } from './constants.mjs';

Expand Down Expand Up @@ -67,6 +67,8 @@ export async function processChunk(inputSlice, itemIndices) {
// The path is the relative path minus the extension
const relativePath = sep + withExt(relative(parent, path));

const processor = getProcessor('ast', { mdx });

let tree;

if (mdx) {
Expand All @@ -79,7 +81,7 @@ export async function processChunk(inputSlice, itemIndices) {
''
);

tree = getRemarkMdx().parse(source);
tree = processor.parse(source);

if (frontmatter) {
tree.children.unshift({
Expand All @@ -93,9 +95,11 @@ export async function processChunk(inputSlice, itemIndices) {
(_, yaml) => `<!-- YAML\n${yaml}\n-->`
);

tree = remark().parse(value);
tree = processor.parse(value);
}

tree = await processor.run(tree, { path });

results.push({ tree, path: relativePath, mdx });
}

Expand Down
9 changes: 9 additions & 0 deletions packages/core/src/generators/ast/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ export default {

hasParallelProcessor: true,

markdown: {
remarkPlugins: [
'remark-parse',
'#plugins/type-annotations/remark.mjs',
'remark-gfm',
'...',
],
},

generate,
processChunk,
};
16 changes: 14 additions & 2 deletions packages/core/src/generators/json/__tests__/generate.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,25 @@ import { fileURLToPath } from 'node:url';
import Ajv from 'ajv';
import { globSync } from 'tinyglobby';

import { loadGenerator } from '#generators/loader.mjs';
import { parseApiDoc } from '#generators/metadata/utils/parse.mjs';
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';
import { getProcessor } from '#utils/markdown/processor.mjs';
import { QUERIES } from '#utils/queries/index.mjs';
import { getRemark } from '#utils/remark.mjs';

import schema from '../schema.json' with { type: 'json' };
import { buildDocument } from '../utils/document.mjs';

// The documents are parsed, split into entries, and serialised with the
// pipelines of the generators doing so
for (const generator of ['ast', 'metadata', 'json']) {
await loadMarkdownPlugins(
await loadGenerator(
import.meta.resolve(`#generators/${generator}/index.mjs`)
)
);
}

const fixtures = new URL('./fixtures/', import.meta.url);

const typeMap = {
Expand Down Expand Up @@ -42,7 +54,7 @@ const buildFixture = async name => {

const path = `/${basename(name, '.md')}`;
const entries = parseApiDoc(
{ path, tree: getRemark().parse(source) },
{ path, tree: getProcessor('ast').parse(source) },
typeMap
);

Expand Down
9 changes: 9 additions & 0 deletions packages/core/src/generators/json/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,15 @@ export default {

hasParallelProcessor: true,

markdown: {
remarkPlugins: [
'#plugins/type-annotations/remark.mjs',
'remark-gfm',
'remark-stringify',
'...',
],
},

generate,
processChunk,
};
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,16 @@ import { describe, it } from 'node:test';

import { u } from 'unist-builder';

import { loadGenerator } from '#generators/loader.mjs';
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';

import { buildEntry, entryBody, takeTypedItems } from '../entry.mjs';

// Markdown is serialised with the pipeline of `json`
await loadMarkdownPlugins(
await loadGenerator(import.meta.resolve('../../index.mjs'))
);

const code = value => u('inlineCode', value);
const text = value => u('text', value);
const type = value => u('typeAnnotation', { value });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,21 @@ import { describe, it } from 'node:test';

import { u } from 'unist-builder';

import { loadGenerator } from '#generators/loader.mjs';
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';

import {
toChanges,
toNumbers,
toStability,
toVersions,
} from '../lifecycle.mjs';

// Markdown is serialised with the pipeline of `json`
await loadMarkdownPlugins(
await loadGenerator(import.meta.resolve('../../index.mjs'))
);

describe('toVersions', () => {
it('always yields an array of strings', () => {
assert.deepEqual(toVersions(undefined), []);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,20 @@ import { describe, it } from 'node:test';

import { u } from 'unist-builder';

import { loadGenerator } from '#generators/loader.mjs';
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';

import {
blocksToMarkdown,
extractExamples,
inlineToMarkdown,
} from '../markdown.mjs';

// Markdown is serialised with the pipeline of `json`
await loadMarkdownPlugins(
await loadGenerator(import.meta.resolve('../../index.mjs'))
);

describe('blocksToMarkdown', () => {
it('serialises blocks, type annotations included', () => {
const nodes = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,21 @@ import { describe, it } from 'node:test';

import { u } from 'unist-builder';

import { loadGenerator } from '#generators/loader.mjs';
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';

import {
buildEventParameters,
buildExtends,
buildPropertyType,
buildSignature,
} from '../signature.mjs';

// Markdown is serialised with the pipeline of `json`
await loadMarkdownPlugins(
await loadGenerator(import.meta.resolve('../../index.mjs'))
);

const code = value => u('inlineCode', value);
const text = value => u('text', value);
const type = (value, links = []) =>
Expand Down
4 changes: 2 additions & 2 deletions packages/core/src/generators/json/utils/markdown.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import { u as createTree } from 'unist-builder';
import { visit } from 'unist-util-visit';

import { getRemark, getRemarkMdx } from '#utils/remark.mjs';
import { getProcessor } from '#utils/markdown/processor.mjs';

import { DISPLAY_NAME } from '../constants.mjs';

Expand All @@ -19,7 +19,7 @@ export const blocksToMarkdown = (nodes, mdx = false) => {
return '';
}

const processor = mdx ? getRemarkMdx() : getRemark();
const processor = getProcessor('json', { mdx });

return processor.stringify(createTree('root', nodes)).trim();
};
Expand Down
9 changes: 9 additions & 0 deletions packages/core/src/generators/metadata/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ export default {

hasParallelProcessor: true,

markdown: {
remarkPlugins: [
'remark-parse',
'#plugins/type-annotations/remark.mjs',
'remark-gfm',
'...',
],
},

generate,
processChunk,
};
6 changes: 2 additions & 4 deletions packages/core/src/generators/metadata/utils/parse.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import {
IGNORE_STABILITY_STEMS,
} from '#generators/metadata/constants.mjs';
import { UNIST } from '#utils/queries/index.mjs';
import { getRemark as remark } from '#utils/remark.mjs';
import { relative } from '#utils/url.mjs';

import { resolveTypeAnnotations } from './resolveTypes.mjs';
Expand Down Expand Up @@ -147,9 +146,8 @@ export const parseApiDoc = ({ path, tree, mdx = false }, typeMap) => {
// Remove processed YAML nodes from the content
remove(subTree, [UNIST.isYamlNode]);

// Apply AST transformations
const parsedSubTree = remark().runSync(subTree);
metadata.content = parsedSubTree;
// The remark plugins already ran on the whole document, in `ast`
metadata.content = subTree;

// Add to collection
metadataCollection.push(metadata);
Expand Down
4 changes: 2 additions & 2 deletions packages/core/src/generators/metadata/utils/visitors.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import { SKIP } from 'unist-util-visit';

import { getProcessor } from '#utils/markdown/processor.mjs';
import { QUERIES } from '#utils/queries/index.mjs';
import { getRemark as remark } from '#utils/remark.mjs';
import { transformNodesToString } from '#utils/unist.mjs';

import { transformUnixManualToLink } from './transformers.mjs';
Expand Down Expand Up @@ -41,7 +41,7 @@ const updateReferences = (query, transformer, node, parent) => {
// and adding those nodes to the parent.
const {
children: [newNode],
} = remark().parse(replacedTypes);
} = getProcessor('metadata').parse(replacedTypes);

// Find the index of the original node in the parent
const index = parent.children.indexOf(node);
Expand Down