1#ifndef PEGEN_H
2#define PEGEN_H
3
4#define PY_SSIZE_T_CLEAN
5#include <Python.h>
6#include <token.h>
7#include <pycore_ast.h>
8
9#if 0
10#define PyPARSE_YIELD_IS_KEYWORD 0x0001
11#endif
12
13#define PyPARSE_DONT_IMPLY_DEDENT 0x0002
14
15#if 0
16#define PyPARSE_WITH_IS_KEYWORD 0x0003
17#define PyPARSE_PRINT_IS_FUNCTION 0x0004
18#define PyPARSE_UNICODE_LITERALS 0x0008
19#endif
20
21#define PyPARSE_IGNORE_COOKIE 0x0010
22#define PyPARSE_BARRY_AS_BDFL 0x0020
23#define PyPARSE_TYPE_COMMENTS 0x0040
24#define PyPARSE_ASYNC_HACKS 0x0080
25#define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100
26
27typedef struct _memo {
28 int type;
29 void *node;
30 int mark;
31 struct _memo *next;
32} Memo;
33
34typedef struct {
35 int type;
36 PyObject *bytes;
37 int level;
38 int lineno, col_offset, end_lineno, end_col_offset;
39 Memo *memo;
40} Token;
41
42typedef struct {
43 char *str;
44 int type;
45} KeywordToken;
46
47
48typedef struct {
49 struct {
50 int lineno;
51 char *comment; // The " <tag>" in "# type: ignore <tag>"
52 } *items;
53 size_t size;
54 size_t num_items;
55} growable_comment_array;
56
57typedef struct {
58 struct tok_state *tok;
59 Token **tokens;
60 int mark;
61 int fill, size;
62 PyArena *arena;
63 KeywordToken **keywords;
64 char **soft_keywords;
65 int n_keyword_lists;
66 int start_rule;
67 int *errcode;
68 int parsing_started;
69 PyObject* normalize;
70 int starting_lineno;
71 int starting_col_offset;
72 int error_indicator;
73 int flags;
74 int feature_version;
75 growable_comment_array type_ignore_comments;
76 Token *known_err_token;
77 int level;
78 int call_invalid_rules;
79} Parser;
80
81typedef struct {
82 cmpop_ty cmpop;
83 expr_ty expr;
84} CmpopExprPair;
85
86typedef struct {
87 expr_ty key;
88 expr_ty value;
89} KeyValuePair;
90
91typedef struct {
92 expr_ty key;
93 pattern_ty pattern;
94} KeyPatternPair;
95
96typedef struct {
97 arg_ty arg;
98 expr_ty value;
99} NameDefaultPair;
100
101typedef struct {
102 asdl_arg_seq *plain_names;
103 asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's
104} SlashWithDefault;
105
106typedef struct {
107 arg_ty vararg;
108 asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's
109 arg_ty kwarg;
110} StarEtc;
111
112typedef struct { operator_ty kind; } AugOperator;
113typedef struct {
114 void *element;
115 int is_keyword;
116} KeywordOrStarred;
117
118#if defined(Py_DEBUG)
119void _PyPegen_clear_memo_statistics(void);
120PyObject *_PyPegen_get_memo_statistics(void);
121#endif
122
123int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node);
124int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
125int _PyPegen_is_memoized(Parser *p, int type, void *pres);
126
127
128int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
129int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
130int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*);
131int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
132
133Token *_PyPegen_expect_token(Parser *p, int type);
134Token *_PyPegen_expect_forced_token(Parser *p, int type, const char* expected);
135expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword);
136expr_ty _PyPegen_soft_keyword_token(Parser *p);
137Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
138int _PyPegen_fill_token(Parser *p);
139expr_ty _PyPegen_name_token(Parser *p);
140expr_ty _PyPegen_number_token(Parser *p);
141void *_PyPegen_string_token(Parser *p);
142const char *_PyPegen_get_expr_name(expr_ty);
143void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...);
144void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
145 Py_ssize_t lineno, Py_ssize_t col_offset,
146 Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
147 const char *errmsg, va_list va);
148void *_PyPegen_dummy_name(Parser *p, ...);
149
150void * _PyPegen_seq_last_item(asdl_seq *seq);
151#define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq))
152
153void * _PyPegen_seq_first_item(asdl_seq *seq);
154#define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq))
155
156#define CURRENT_POS (-5)
157
158Py_LOCAL_INLINE(void *)
159RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype,
160 Py_ssize_t lineno, Py_ssize_t col_offset,
161 Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
162 const char *errmsg, ...)
163{
164 va_list va;
165 va_start(va, errmsg);
166 Py_ssize_t _col_offset = (col_offset == CURRENT_POS ? CURRENT_POS : col_offset + 1);
167 Py_ssize_t _end_col_offset = (end_col_offset == CURRENT_POS ? CURRENT_POS : end_col_offset + 1);
168 _PyPegen_raise_error_known_location(p, errtype, lineno, _col_offset, end_lineno, _end_col_offset, errmsg, va);
169 va_end(va);
170 return NULL;
171}
172
173#define UNUSED(expr) do { (void)(expr); } while (0)
174#define EXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena
175#define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
176#define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__)
177#define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__)
178#define RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \
179 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (b)->end_lineno, (b)->end_col_offset, msg, ##__VA_ARGS__)
180#define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \
181 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__)
182#define RAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \
183 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__)
184
185Py_LOCAL_INLINE(void *)
186CHECK_CALL(Parser *p, void *result)
187{
188 if (result == NULL) {
189 assert(PyErr_Occurred());
190 p->error_indicator = 1;
191 }
192 return result;
193}
194
195/* This is needed for helper functions that are allowed to
196 return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */
197Py_LOCAL_INLINE(void *)
198CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
199{
200 if (result == NULL && PyErr_Occurred()) {
201 p->error_indicator = 1;
202 }
203 return result;
204}
205
206#define CHECK(type, result) ((type) CHECK_CALL(p, result))
207#define CHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result))
208
209PyObject *_PyPegen_new_type_comment(Parser *, const char *);
210
211Py_LOCAL_INLINE(PyObject *)
212NEW_TYPE_COMMENT(Parser *p, Token *tc)
213{
214 if (tc == NULL) {
215 return NULL;
216 }
217 const char *bytes = PyBytes_AsString(tc->bytes);
218 if (bytes == NULL) {
219 goto error;
220 }
221 PyObject *tco = _PyPegen_new_type_comment(p, bytes);
222 if (tco == NULL) {
223 goto error;
224 }
225 return tco;
226 error:
227 p->error_indicator = 1; // Inline CHECK_CALL
228 return NULL;
229}
230
231Py_LOCAL_INLINE(void *)
232INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node)
233{
234 if (node == NULL) {
235 p->error_indicator = 1; // Inline CHECK_CALL
236 return NULL;
237 }
238 if (p->feature_version < version) {
239 p->error_indicator = 1;
240 return RAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater",
241 msg, version);
242 }
243 return node;
244}
245
246#define CHECK_VERSION(type, version, msg, node) ((type) INVALID_VERSION_CHECK(p, version, msg, node))
247
248arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
249PyObject *_PyPegen_new_identifier(Parser *, const char *);
250Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
251void _PyPegen_Parser_Free(Parser *);
252mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
253 const char *, const char *, PyCompilerFlags *, int *, PyArena *);
254void *_PyPegen_run_parser(Parser *);
255mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
256asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
257asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
258asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
259asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
260asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *);
261expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty);
262int _PyPegen_seq_count_dots(asdl_seq *);
263alias_ty _PyPegen_alias_for_star(Parser *, int, int, int, int, PyArena *);
264asdl_identifier_seq *_PyPegen_map_names_to_ids(Parser *, asdl_expr_seq *);
265CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty);
266asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *);
267asdl_expr_seq *_PyPegen_get_exprs(Parser *, asdl_seq *);
268expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty);
269KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty);
270asdl_expr_seq *_PyPegen_get_keys(Parser *, asdl_seq *);
271asdl_expr_seq *_PyPegen_get_values(Parser *, asdl_seq *);
272KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty);
273asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *);
274asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *);
275NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *);
276SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *);
277StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty);
278arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *,
279 asdl_arg_seq *, asdl_seq *, StarEtc *);
280arguments_ty _PyPegen_empty_arguments(Parser *);
281AugOperator *_PyPegen_augoperator(Parser*, operator_ty type);
282stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
283stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
284KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int);
285asdl_expr_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *);
286asdl_keyword_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *);
287expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
288 int lineno, int col_offset, int end_lineno,
289 int end_col_offset, PyArena *arena);
290expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *);
291expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty);
292expr_ty _PyPegen_ensure_real(Parser *p, expr_ty);
293asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
294int _PyPegen_check_barry_as_flufl(Parser *, Token *);
295int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t);
296mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *);
297
298// Error reporting helpers
299typedef enum {
300 STAR_TARGETS,
301 DEL_TARGETS,
302 FOR_TARGETS
303} TARGETS_TYPE;
304expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type);
305#define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)
306
307Py_LOCAL_INLINE(void *)
308_RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
309{
310 expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type));
311 if (invalid_target != NULL) {
312 const char *msg;
313 if (type == STAR_TARGETS || type == FOR_TARGETS) {
314 msg = "cannot assign to %s";
315 }
316 else {
317 msg = "cannot delete %s";
318 }
319 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
320 invalid_target,
321 msg,
322 _PyPegen_get_expr_name(invalid_target)
323 );
324 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax");
325 }
326 return NULL;
327}
328
329void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
330expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension);
331void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions);
332
333
334// Generated function in parse.c - function definition in python.gram
335void *_PyPegen_parse(Parser *);
336
337#endif
338