Skip to content

Commit 953d272

Browse files
pablogsalpablogsal
authored
Update pegen to use the latest upstream developments (GH-27586)
1 parent 8e832fb commit 953d272

26 files changed

Lines changed: 1240 additions & 670 deletions

‎Lib/test/test_peg_generator/test_c_parser.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
from test.support import os_helper
1212
from test.support.script_helper import assert_python_ok
1313

14-
_py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
15-
_pgo_flag = sysconfig.get_config_var('PGO_PROF_USE_FLAG')
14+
_py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST")
15+
_pgo_flag = sysconfig.get_config_var("PGO_PROF_USE_FLAG")
1616
if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist:
1717
raise unittest.SkipTest("peg_generator test disabled under PGO build")
1818

@@ -458,3 +458,28 @@ def test_soft_keywords_lookahead(self) -> None:
458458
self.check_input_strings_for_grammar(valid_cases, invalid_cases)
459459
"""
460460
self.run_test(grammar_source, test_source)
461+
462+
def test_forced(self) -> None:
463+
grammar_source = """
464+
start: NAME &&':' | NAME
465+
"""
466+
test_source = """
467+
self.assertEqual(parse.parse_string("number :", mode=0), None)
468+
with self.assertRaises(SyntaxError) as e:
469+
parse.parse_string("a", mode=0)
470+
self.assertIn("expected ':'", str(e.exception))
471+
"""
472+
self.run_test(grammar_source, test_source)
473+
474+
def test_forced_with_group(self) -> None:
475+
grammar_source = """
476+
start: NAME &&(':' | ';') | NAME
477+
"""
478+
test_source = """
479+
self.assertEqual(parse.parse_string("number :", mode=0), None)
480+
self.assertEqual(parse.parse_string("number ;", mode=0), None)
481+
with self.assertRaises(SyntaxError) as e:
482+
parse.parse_string("a", mode=0)
483+
self.assertIn("expected (':' | ';')", e.exception.args[0])
484+
"""
485+
self.run_test(grammar_source, test_source)

‎Lib/test/test_peg_generator/test_first_sets.py‎

Lines changed: 129 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
from test import test_tools
44
from typing import Dict, Set
55

6-
test_tools.skip_if_missing('peg_generator')
7-
with test_tools.imports_under_tool('peg_generator'):
6+
test_tools.skip_if_missing("peg_generator")
7+
with test_tools.imports_under_tool("peg_generator"):
88
from pegen.grammar_parser import GeneratedParser as GrammarParser
99
from pegen.testutil import parse_string
1010
from pegen.first_sets import FirstSetCalculator
@@ -23,126 +23,167 @@ def test_alternatives(self) -> None:
2323
A: 'a' | '-'
2424
B: 'b' | '+'
2525
"""
26-
self.assertEqual(self.calculate_first_sets(grammar), {
27-
"A": {"'a'", "'-'"},
28-
"B": {"'+'", "'b'"},
29-
"expr": {"'+'", "'a'", "'b'", "'-'"},
30-
"start": {"'+'", "'a'", "'b'", "'-'"},
31-
})
26+
self.assertEqual(
27+
self.calculate_first_sets(grammar),
28+
{
29+
"A": {"'a'", "'-'"},
30+
"B": {"'+'", "'b'"},
31+
"expr": {"'+'", "'a'", "'b'", "'-'"},
32+
"start": {"'+'", "'a'", "'b'", "'-'"},
33+
},
34+
)
3235

3336
def test_optionals(self) -> None:
3437
grammar = """
3538
start: expr NEWLINE
3639
expr: ['a'] ['b'] 'c'
3740
"""
38-
self.assertEqual(self.calculate_first_sets(grammar), {
39-
"expr": {"'c'", "'a'", "'b'"},
40-
"start": {"'c'", "'a'", "'b'"},
41-
})
41+
self.assertEqual(
42+
self.calculate_first_sets(grammar),
43+
{
44+
"expr": {"'c'", "'a'", "'b'"},
45+
"start": {"'c'", "'a'", "'b'"},
46+
},
47+
)
4248

4349
def test_repeat_with_separator(self) -> None:
4450
grammar = """
4551
start: ','.thing+ NEWLINE
4652
thing: NUMBER
4753
"""
48-
self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}})
54+
self.assertEqual(
55+
self.calculate_first_sets(grammar),
56+
{"thing": {"NUMBER"}, "start": {"NUMBER"}},
57+
)
4958

5059
def test_optional_operator(self) -> None:
5160
grammar = """
5261
start: sum NEWLINE
5362
sum: (term)? 'b'
5463
term: NUMBER
5564
"""
56-
self.assertEqual(self.calculate_first_sets(grammar), {
57-
"term": {"NUMBER"},
58-
"sum": {"NUMBER", "'b'"},
59-
"start": {"'b'", "NUMBER"},
60-
})
65+
self.assertEqual(
66+
self.calculate_first_sets(grammar),
67+
{
68+
"term": {"NUMBER"},
69+
"sum": {"NUMBER", "'b'"},
70+
"start": {"'b'", "NUMBER"},
71+
},
72+
)
6173

6274
def test_optional_literal(self) -> None:
6375
grammar = """
6476
start: sum NEWLINE
6577
sum: '+' ? term
6678
term: NUMBER
6779
"""
68-
self.assertEqual(self.calculate_first_sets(grammar), {
69-
"term": {"NUMBER"},
70-
"sum": {"'+'", "NUMBER"},
71-
"start": {"'+'", "NUMBER"},
72-
})
80+
self.assertEqual(
81+
self.calculate_first_sets(grammar),
82+
{
83+
"term": {"NUMBER"},
84+
"sum": {"'+'", "NUMBER"},
85+
"start": {"'+'", "NUMBER"},
86+
},
87+
)
7388

7489
def test_optional_after(self) -> None:
7590
grammar = """
7691
start: term NEWLINE
7792
term: NUMBER ['+']
7893
"""
79-
self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"NUMBER"}})
94+
self.assertEqual(
95+
self.calculate_first_sets(grammar),
96+
{"term": {"NUMBER"}, "start": {"NUMBER"}},
97+
)
8098

8199
def test_optional_before(self) -> None:
82100
grammar = """
83101
start: term NEWLINE
84102
term: ['+'] NUMBER
85103
"""
86-
self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}})
104+
self.assertEqual(
105+
self.calculate_first_sets(grammar),
106+
{"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}},
107+
)
87108

88109
def test_repeat_0(self) -> None:
89110
grammar = """
90111
start: thing* "+" NEWLINE
91112
thing: NUMBER
92113
"""
93-
self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}})
114+
self.assertEqual(
115+
self.calculate_first_sets(grammar),
116+
{"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}},
117+
)
94118

95119
def test_repeat_0_with_group(self) -> None:
96120
grammar = """
97121
start: ('+' '-')* term NEWLINE
98122
term: NUMBER
99123
"""
100-
self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}})
124+
self.assertEqual(
125+
self.calculate_first_sets(grammar),
126+
{"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}},
127+
)
101128

102129
def test_repeat_1(self) -> None:
103130
grammar = """
104131
start: thing+ '-' NEWLINE
105132
thing: NUMBER
106133
"""
107-
self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}})
134+
self.assertEqual(
135+
self.calculate_first_sets(grammar),
136+
{"thing": {"NUMBER"}, "start": {"NUMBER"}},
137+
)
108138

109139
def test_repeat_1_with_group(self) -> None:
110140
grammar = """
111141
start: ('+' term)+ term NEWLINE
112142
term: NUMBER
113143
"""
114-
self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}})
144+
self.assertEqual(
145+
self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}}
146+
)
115147

116148
def test_gather(self) -> None:
117149
grammar = """
118150
start: ','.thing+ NEWLINE
119151
thing: NUMBER
120152
"""
121-
self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}})
153+
self.assertEqual(
154+
self.calculate_first_sets(grammar),
155+
{"thing": {"NUMBER"}, "start": {"NUMBER"}},
156+
)
122157

123158
def test_positive_lookahead(self) -> None:
124159
grammar = """
125160
start: expr NEWLINE
126161
expr: &'a' opt
127162
opt: 'a' | 'b' | 'c'
128163
"""
129-
self.assertEqual(self.calculate_first_sets(grammar), {
130-
"expr": {"'a'"},
131-
"start": {"'a'"},
132-
"opt": {"'b'", "'c'", "'a'"},
133-
})
164+
self.assertEqual(
165+
self.calculate_first_sets(grammar),
166+
{
167+
"expr": {"'a'"},
168+
"start": {"'a'"},
169+
"opt": {"'b'", "'c'", "'a'"},
170+
},
171+
)
134172

135173
def test_negative_lookahead(self) -> None:
136174
grammar = """
137175
start: expr NEWLINE
138176
expr: !'a' opt
139177
opt: 'a' | 'b' | 'c'
140178
"""
141-
self.assertEqual(self.calculate_first_sets(grammar), {
142-
"opt": {"'b'", "'a'", "'c'"},
143-
"expr": {"'b'", "'c'"},
144-
"start": {"'b'", "'c'"},
145-
})
179+
self.assertEqual(
180+
self.calculate_first_sets(grammar),
181+
{
182+
"opt": {"'b'", "'a'", "'c'"},
183+
"expr": {"'b'", "'c'"},
184+
"start": {"'b'", "'c'"},
185+
},
186+
)
146187

147188
def test_left_recursion(self) -> None:
148189
grammar = """
@@ -153,33 +194,42 @@ def test_left_recursion(self) -> None:
153194
bar: 'bar'
154195
baz: 'baz'
155196
"""
156-
self.assertEqual(self.calculate_first_sets(grammar), {
157-
"expr": {"NUMBER", "'-'"},
158-
"term": {"NUMBER"},
159-
"start": {"NUMBER", "'-'"},
160-
"foo": {"'foo'"},
161-
"bar": {"'bar'"},
162-
"baz": {"'baz'"},
163-
})
197+
self.assertEqual(
198+
self.calculate_first_sets(grammar),
199+
{
200+
"expr": {"NUMBER", "'-'"},
201+
"term": {"NUMBER"},
202+
"start": {"NUMBER", "'-'"},
203+
"foo": {"'foo'"},
204+
"bar": {"'bar'"},
205+
"baz": {"'baz'"},
206+
},
207+
)
164208

165209
def test_advance_left_recursion(self) -> None:
166210
grammar = """
167211
start: NUMBER | sign start
168212
sign: ['-']
169213
"""
170-
self.assertEqual(self.calculate_first_sets(grammar), {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}})
214+
self.assertEqual(
215+
self.calculate_first_sets(grammar),
216+
{"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}},
217+
)
171218

172219
def test_mutual_left_recursion(self) -> None:
173220
grammar = """
174221
start: foo 'E'
175222
foo: bar 'A' | 'B'
176223
bar: foo 'C' | 'D'
177224
"""
178-
self.assertEqual(self.calculate_first_sets(grammar), {
179-
"foo": {"'D'", "'B'"},
180-
"bar": {"'D'"},
181-
"start": {"'D'", "'B'"},
182-
})
225+
self.assertEqual(
226+
self.calculate_first_sets(grammar),
227+
{
228+
"foo": {"'D'", "'B'"},
229+
"bar": {"'D'"},
230+
"start": {"'D'", "'B'"},
231+
},
232+
)
183233

184234
def test_nasty_left_recursion(self) -> None:
185235
# TODO: Validate this
@@ -188,25 +238,33 @@ def test_nasty_left_recursion(self) -> None:
188238
target: maybe '+' | NAME
189239
maybe: maybe '-' | target
190240
"""
191-
self.assertEqual(self.calculate_first_sets(grammar), {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}})
241+
self.assertEqual(
242+
self.calculate_first_sets(grammar),
243+
{"maybe": set(), "target": {"NAME"}, "start": {"NAME"}},
244+
)
192245

193246
def test_nullable_rule(self) -> None:
194247
grammar = """
195248
start: sign thing $
196249
sign: ['-']
197250
thing: NUMBER
198251
"""
199-
self.assertEqual(self.calculate_first_sets(grammar), {
200-
"sign": {"", "'-'"},
201-
"thing": {"NUMBER"},
202-
"start": {"NUMBER", "'-'"},
203-
})
252+
self.assertEqual(
253+
self.calculate_first_sets(grammar),
254+
{
255+
"sign": {"", "'-'"},
256+
"thing": {"NUMBER"},
257+
"start": {"NUMBER", "'-'"},
258+
},
259+
)
204260

205261
def test_epsilon_production_in_start_rule(self) -> None:
206262
grammar = """
207263
start: ['-'] $
208264
"""
209-
self.assertEqual(self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}})
265+
self.assertEqual(
266+
self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}}
267+
)
210268

211269
def test_multiple_nullable_rules(self) -> None:
212270
grammar = """
@@ -216,10 +274,13 @@ def test_multiple_nullable_rules(self) -> None:
216274
other: '*'
217275
another: '/'
218276
"""
219-
self.assertEqual(self.calculate_first_sets(grammar), {
220-
"sign": {"", "'-'"},
221-
"thing": {"'+'", ""},
222-
"start": {"'+'", "'-'", "'*'"},
223-
"other": {"'*'"},
224-
"another": {"'/'"},
225-
})
277+
self.assertEqual(
278+
self.calculate_first_sets(grammar),
279+
{
280+
"sign": {"", "'-'"},
281+
"thing": {"'+'", ""},
282+
"start": {"'+'", "'-'", "'*'"},
283+
"other": {"'*'"},
284+
"another": {"'/'"},
285+
},
286+
)

‎Lib/test/test_peg_generator/test_grammar_validator.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import unittest
22
from test import test_tools
33

4-
test_tools.skip_if_missing('peg_generator')
5-
with test_tools.imports_under_tool('peg_generator'):
4+
test_tools.skip_if_missing("peg_generator")
5+
with test_tools.imports_under_tool("peg_generator"):
66
from pegen.grammar_parser import GeneratedParser as GrammarParser
77
from pegen.validator import SubRuleValidator, ValidationError
88
from pegen.testutil import parse_string

0 commit comments

Comments
 (0)