1 | #ifndef PEGEN_H |
2 | #define PEGEN_H |
3 | |
4 | #define PY_SSIZE_T_CLEAN |
5 | #include <Python.h> |
6 | #include <token.h> |
7 | #include <pycore_ast.h> |
8 | |
9 | #if 0 |
10 | #define PyPARSE_YIELD_IS_KEYWORD 0x0001 |
11 | #endif |
12 | |
13 | #define PyPARSE_DONT_IMPLY_DEDENT 0x0002 |
14 | |
15 | #if 0 |
16 | #define PyPARSE_WITH_IS_KEYWORD 0x0003 |
17 | #define PyPARSE_PRINT_IS_FUNCTION 0x0004 |
18 | #define PyPARSE_UNICODE_LITERALS 0x0008 |
19 | #endif |
20 | |
21 | #define PyPARSE_IGNORE_COOKIE 0x0010 |
22 | #define PyPARSE_BARRY_AS_BDFL 0x0020 |
23 | #define 0x0040 |
24 | #define PyPARSE_ASYNC_HACKS 0x0080 |
25 | #define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100 |
26 | |
27 | typedef struct _memo { |
28 | int type; |
29 | void *node; |
30 | int mark; |
31 | struct _memo *next; |
32 | } Memo; |
33 | |
34 | typedef struct { |
35 | int type; |
36 | PyObject *bytes; |
37 | int level; |
38 | int lineno, col_offset, end_lineno, end_col_offset; |
39 | Memo *memo; |
40 | } Token; |
41 | |
42 | typedef struct { |
43 | char *str; |
44 | int type; |
45 | } KeywordToken; |
46 | |
47 | |
48 | typedef struct { |
49 | struct { |
50 | int lineno; |
51 | char *; // The " <tag>" in "# type: ignore <tag>" |
52 | } *items; |
53 | size_t size; |
54 | size_t num_items; |
55 | } ; |
56 | |
57 | typedef struct { |
58 | struct tok_state *tok; |
59 | Token **tokens; |
60 | int mark; |
61 | int fill, size; |
62 | PyArena *arena; |
63 | KeywordToken **keywords; |
64 | char **soft_keywords; |
65 | int n_keyword_lists; |
66 | int start_rule; |
67 | int *errcode; |
68 | int parsing_started; |
69 | PyObject* normalize; |
70 | int starting_lineno; |
71 | int starting_col_offset; |
72 | int error_indicator; |
73 | int flags; |
74 | int feature_version; |
75 | growable_comment_array ; |
76 | Token *known_err_token; |
77 | int level; |
78 | int call_invalid_rules; |
79 | } Parser; |
80 | |
81 | typedef struct { |
82 | cmpop_ty cmpop; |
83 | expr_ty expr; |
84 | } CmpopExprPair; |
85 | |
86 | typedef struct { |
87 | expr_ty key; |
88 | expr_ty value; |
89 | } KeyValuePair; |
90 | |
91 | typedef struct { |
92 | expr_ty key; |
93 | pattern_ty pattern; |
94 | } KeyPatternPair; |
95 | |
96 | typedef struct { |
97 | arg_ty arg; |
98 | expr_ty value; |
99 | } NameDefaultPair; |
100 | |
101 | typedef struct { |
102 | asdl_arg_seq *plain_names; |
103 | asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's |
104 | } SlashWithDefault; |
105 | |
106 | typedef struct { |
107 | arg_ty vararg; |
108 | asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's |
109 | arg_ty kwarg; |
110 | } StarEtc; |
111 | |
112 | typedef struct { operator_ty kind; } AugOperator; |
113 | typedef struct { |
114 | void *element; |
115 | int is_keyword; |
116 | } KeywordOrStarred; |
117 | |
118 | #if defined(Py_DEBUG) |
119 | void _PyPegen_clear_memo_statistics(void); |
120 | PyObject *_PyPegen_get_memo_statistics(void); |
121 | #endif |
122 | |
123 | int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node); |
124 | int _PyPegen_update_memo(Parser *p, int mark, int type, void *node); |
125 | int _PyPegen_is_memoized(Parser *p, int type, void *pres); |
126 | |
127 | |
128 | int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *); |
129 | int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); |
130 | int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*); |
131 | int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *); |
132 | |
133 | Token *_PyPegen_expect_token(Parser *p, int type); |
134 | Token *_PyPegen_expect_forced_token(Parser *p, int type, const char* expected); |
135 | expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword); |
136 | expr_ty _PyPegen_soft_keyword_token(Parser *p); |
137 | Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); |
138 | int _PyPegen_fill_token(Parser *p); |
139 | expr_ty _PyPegen_name_token(Parser *p); |
140 | expr_ty _PyPegen_number_token(Parser *p); |
141 | void *_PyPegen_string_token(Parser *p); |
142 | const char *_PyPegen_get_expr_name(expr_ty); |
143 | void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...); |
144 | void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, |
145 | Py_ssize_t lineno, Py_ssize_t col_offset, |
146 | Py_ssize_t end_lineno, Py_ssize_t end_col_offset, |
147 | const char *errmsg, va_list va); |
148 | void *_PyPegen_dummy_name(Parser *p, ...); |
149 | |
150 | void * _PyPegen_seq_last_item(asdl_seq *seq); |
151 | #define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq)) |
152 | |
153 | void * _PyPegen_seq_first_item(asdl_seq *seq); |
154 | #define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq)) |
155 | |
156 | #define CURRENT_POS (-5) |
157 | |
158 | Py_LOCAL_INLINE(void *) |
159 | RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, |
160 | Py_ssize_t lineno, Py_ssize_t col_offset, |
161 | Py_ssize_t end_lineno, Py_ssize_t end_col_offset, |
162 | const char *errmsg, ...) |
163 | { |
164 | va_list va; |
165 | va_start(va, errmsg); |
166 | Py_ssize_t _col_offset = (col_offset == CURRENT_POS ? CURRENT_POS : col_offset + 1); |
167 | Py_ssize_t _end_col_offset = (end_col_offset == CURRENT_POS ? CURRENT_POS : end_col_offset + 1); |
168 | _PyPegen_raise_error_known_location(p, errtype, lineno, _col_offset, end_lineno, _end_col_offset, errmsg, va); |
169 | va_end(va); |
170 | return NULL; |
171 | } |
172 | |
173 | #define UNUSED(expr) do { (void)(expr); } while (0) |
174 | #define (head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena |
175 | #define _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena |
176 | #define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__) |
177 | #define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__) |
178 | #define RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \ |
179 | RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (b)->end_lineno, (b)->end_col_offset, msg, ##__VA_ARGS__) |
180 | #define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \ |
181 | RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__) |
182 | #define RAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \ |
183 | RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__) |
184 | |
185 | Py_LOCAL_INLINE(void *) |
186 | CHECK_CALL(Parser *p, void *result) |
187 | { |
188 | if (result == NULL) { |
189 | assert(PyErr_Occurred()); |
190 | p->error_indicator = 1; |
191 | } |
192 | return result; |
193 | } |
194 | |
195 | /* This is needed for helper functions that are allowed to |
196 | return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */ |
197 | Py_LOCAL_INLINE(void *) |
198 | CHECK_CALL_NULL_ALLOWED(Parser *p, void *result) |
199 | { |
200 | if (result == NULL && PyErr_Occurred()) { |
201 | p->error_indicator = 1; |
202 | } |
203 | return result; |
204 | } |
205 | |
206 | #define CHECK(type, result) ((type) CHECK_CALL(p, result)) |
207 | #define CHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result)) |
208 | |
209 | PyObject *(Parser *, const char *); |
210 | |
211 | Py_LOCAL_INLINE(PyObject *) |
212 | (Parser *p, Token *tc) |
213 | { |
214 | if (tc == NULL) { |
215 | return NULL; |
216 | } |
217 | const char *bytes = PyBytes_AsString(tc->bytes); |
218 | if (bytes == NULL) { |
219 | goto error; |
220 | } |
221 | PyObject *tco = _PyPegen_new_type_comment(p, bytes); |
222 | if (tco == NULL) { |
223 | goto error; |
224 | } |
225 | return tco; |
226 | error: |
227 | p->error_indicator = 1; // Inline CHECK_CALL |
228 | return NULL; |
229 | } |
230 | |
231 | Py_LOCAL_INLINE(void *) |
232 | INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node) |
233 | { |
234 | if (node == NULL) { |
235 | p->error_indicator = 1; // Inline CHECK_CALL |
236 | return NULL; |
237 | } |
238 | if (p->feature_version < version) { |
239 | p->error_indicator = 1; |
240 | return RAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater" , |
241 | msg, version); |
242 | } |
243 | return node; |
244 | } |
245 | |
246 | #define CHECK_VERSION(type, version, msg, node) ((type) INVALID_VERSION_CHECK(p, version, msg, node)) |
247 | |
248 | arg_ty (Parser *, arg_ty, Token *); |
249 | PyObject *_PyPegen_new_identifier(Parser *, const char *); |
250 | Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *); |
251 | void _PyPegen_Parser_Free(Parser *); |
252 | mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, |
253 | const char *, const char *, PyCompilerFlags *, int *, PyArena *); |
254 | void *_PyPegen_run_parser(Parser *); |
255 | mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); |
256 | asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); |
257 | asdl_seq *_PyPegen_singleton_seq(Parser *, void *); |
258 | asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *); |
259 | asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *); |
260 | asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *); |
261 | expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty); |
262 | int _PyPegen_seq_count_dots(asdl_seq *); |
263 | alias_ty _PyPegen_alias_for_star(Parser *, int, int, int, int, PyArena *); |
264 | asdl_identifier_seq *_PyPegen_map_names_to_ids(Parser *, asdl_expr_seq *); |
265 | CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty); |
266 | asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *); |
267 | asdl_expr_seq *_PyPegen_get_exprs(Parser *, asdl_seq *); |
268 | expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty); |
269 | KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty); |
270 | asdl_expr_seq *_PyPegen_get_keys(Parser *, asdl_seq *); |
271 | asdl_expr_seq *_PyPegen_get_values(Parser *, asdl_seq *); |
272 | KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty); |
273 | asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *); |
274 | asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *); |
275 | NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *); |
276 | SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *); |
277 | StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty); |
278 | arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *, |
279 | asdl_arg_seq *, asdl_seq *, StarEtc *); |
280 | arguments_ty _PyPegen_empty_arguments(Parser *); |
281 | AugOperator *_PyPegen_augoperator(Parser*, operator_ty type); |
282 | stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty); |
283 | stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty); |
284 | KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int); |
285 | asdl_expr_seq *(Parser *, asdl_seq *); |
286 | asdl_keyword_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *); |
287 | expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, |
288 | int lineno, int col_offset, int end_lineno, |
289 | int end_col_offset, PyArena *arena); |
290 | expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *); |
291 | expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); |
292 | expr_ty _PyPegen_ensure_real(Parser *p, expr_ty); |
293 | asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *); |
294 | int _PyPegen_check_barry_as_flufl(Parser *, Token *); |
295 | int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t); |
296 | mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *); |
297 | |
298 | // Error reporting helpers |
299 | typedef enum { |
300 | STAR_TARGETS, |
301 | DEL_TARGETS, |
302 | FOR_TARGETS |
303 | } TARGETS_TYPE; |
304 | expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type); |
305 | #define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e) |
306 | |
307 | Py_LOCAL_INLINE(void *) |
308 | _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e) |
309 | { |
310 | expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type)); |
311 | if (invalid_target != NULL) { |
312 | const char *msg; |
313 | if (type == STAR_TARGETS || type == FOR_TARGETS) { |
314 | msg = "cannot assign to %s" ; |
315 | } |
316 | else { |
317 | msg = "cannot delete %s" ; |
318 | } |
319 | return RAISE_SYNTAX_ERROR_KNOWN_LOCATION( |
320 | invalid_target, |
321 | msg, |
322 | _PyPegen_get_expr_name(invalid_target) |
323 | ); |
324 | return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax" ); |
325 | } |
326 | return NULL; |
327 | } |
328 | |
329 | void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); |
330 | expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension); |
331 | void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); |
332 | |
333 | |
334 | // Generated function in parse.c - function definition in python.gram |
335 | void *_PyPegen_parse(Parser *); |
336 | |
337 | #endif |
338 | |