1 | /* |
2 | ** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $ |
3 | ** Lexical Analyzer |
4 | ** See Copyright Notice in lua.h |
5 | */ |
6 | |
7 | |
8 | #include <ctype.h> |
9 | #include <locale.h> |
10 | #include <string.h> |
11 | |
12 | #define llex_c |
13 | #define LUA_CORE |
14 | |
15 | #include "lua.h" |
16 | |
17 | #include "ldo.h" |
18 | #include "llex.h" |
19 | #include "lobject.h" |
20 | #include "lparser.h" |
21 | #include "lstate.h" |
22 | #include "lstring.h" |
23 | #include "ltable.h" |
24 | #include "lzio.h" |
25 | |
26 | |
27 | |
28 | #define next(ls) (ls->current = zgetc(ls->z)) |
29 | |
30 | |
31 | |
32 | |
33 | #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') |
34 | |
35 | |
36 | /* ORDER RESERVED */ |
37 | const char *const luaX_tokens [] = { |
38 | "and" , "break" , "do" , "else" , "elseif" , |
39 | "end" , "false" , "for" , "function" , "if" , |
40 | "in" , "local" , "nil" , "not" , "or" , "repeat" , |
41 | "return" , "then" , "true" , "until" , "while" , |
42 | ".." , "..." , "==" , ">=" , "<=" , "~=" , |
43 | "<number>" , "<name>" , "<string>" , "<eof>" , |
44 | NULL |
45 | }; |
46 | |
47 | |
48 | #define save_and_next(ls) (save(ls, ls->current), next(ls)) |
49 | |
50 | |
51 | static void save (LexState *ls, int c) { |
52 | Mbuffer *b = ls->buff; |
53 | if (b->n + 1 > b->buffsize) { |
54 | size_t newsize; |
55 | if (b->buffsize >= MAX_SIZET/2) |
56 | luaX_lexerror(ls, "lexical element too long" , 0); |
57 | newsize = b->buffsize * 2; |
58 | luaZ_resizebuffer(ls->L, b, newsize); |
59 | } |
60 | b->buffer[b->n++] = cast(char, c); |
61 | } |
62 | |
63 | |
64 | void luaX_init (lua_State *L) { |
65 | int i; |
66 | for (i=0; i<NUM_RESERVED; i++) { |
67 | TString *ts = luaS_new(L, luaX_tokens[i]); |
68 | luaS_fix(ts); /* reserved words are never collected */ |
69 | lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN); |
70 | ts->tsv.reserved = cast_byte(i+1); /* reserved word */ |
71 | } |
72 | } |
73 | |
74 | |
75 | #define MAXSRC 80 |
76 | |
77 | |
78 | const char *luaX_token2str (LexState *ls, int token) { |
79 | if (token < FIRST_RESERVED) { |
80 | lua_assert(token == cast(unsigned char, token)); |
81 | return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)" , token) : |
82 | luaO_pushfstring(ls->L, "%c" , token); |
83 | } |
84 | else |
85 | return luaX_tokens[token-FIRST_RESERVED]; |
86 | } |
87 | |
88 | |
89 | static const char *txtToken (LexState *ls, int token) { |
90 | switch (token) { |
91 | case TK_NAME: |
92 | case TK_STRING: |
93 | case TK_NUMBER: |
94 | save(ls, '\0'); |
95 | return luaZ_buffer(ls->buff); |
96 | default: |
97 | return luaX_token2str(ls, token); |
98 | } |
99 | } |
100 | |
101 | |
102 | void luaX_lexerror (LexState *ls, const char *msg, int token) { |
103 | char buff[MAXSRC]; |
104 | luaO_chunkid(buff, getstr(ls->source), MAXSRC); |
105 | msg = luaO_pushfstring(ls->L, "%s:%d: %s" , buff, ls->linenumber, msg); |
106 | if (token) |
107 | luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token)); |
108 | luaD_throw(ls->L, LUA_ERRSYNTAX); |
109 | } |
110 | |
111 | |
112 | void luaX_syntaxerror (LexState *ls, const char *msg) { |
113 | luaX_lexerror(ls, msg, ls->t.token); |
114 | } |
115 | |
116 | |
117 | TString *luaX_newstring (LexState *ls, const char *str, size_t l) { |
118 | lua_State *L = ls->L; |
119 | TString *ts = luaS_newlstr(L, str, l); |
120 | TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */ |
121 | if (ttisnil(o)) { |
122 | setbvalue(o, 1); /* make sure `str' will not be collected */ |
123 | luaC_checkGC(L); |
124 | } |
125 | return ts; |
126 | } |
127 | |
128 | |
129 | static void inclinenumber (LexState *ls) { |
130 | int old = ls->current; |
131 | lua_assert(currIsNewline(ls)); |
132 | next(ls); /* skip `\n' or `\r' */ |
133 | if (currIsNewline(ls) && ls->current != old) |
134 | next(ls); /* skip `\n\r' or `\r\n' */ |
135 | if (++ls->linenumber >= MAX_INT) |
136 | luaX_syntaxerror(ls, "chunk has too many lines" ); |
137 | } |
138 | |
139 | |
140 | void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) { |
141 | ls->decpoint = '.'; |
142 | ls->L = L; |
143 | ls->lookahead.token = TK_EOS; /* no look-ahead token */ |
144 | ls->z = z; |
145 | ls->fs = NULL; |
146 | ls->linenumber = 1; |
147 | ls->lastline = 1; |
148 | ls->source = source; |
149 | luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */ |
150 | next(ls); /* read first char */ |
151 | } |
152 | |
153 | |
154 | |
155 | /* |
156 | ** ======================================================= |
157 | ** LEXICAL ANALYZER |
158 | ** ======================================================= |
159 | */ |
160 | |
161 | |
162 | |
163 | static int check_next (LexState *ls, const char *set) { |
164 | if (!strchr(set, ls->current)) |
165 | return 0; |
166 | save_and_next(ls); |
167 | return 1; |
168 | } |
169 | |
170 | |
171 | static void buffreplace (LexState *ls, char from, char to) { |
172 | size_t n = luaZ_bufflen(ls->buff); |
173 | char *p = luaZ_buffer(ls->buff); |
174 | while (n--) |
175 | if (p[n] == from) p[n] = to; |
176 | } |
177 | |
178 | |
179 | static void trydecpoint (LexState *ls, SemInfo *seminfo) { |
180 | /* format error: try to update decimal point separator */ |
181 | struct lconv *cv = localeconv(); |
182 | char old = ls->decpoint; |
183 | ls->decpoint = (cv ? cv->decimal_point[0] : '.'); |
184 | buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */ |
185 | if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) { |
186 | /* format error with correct decimal point: no more options */ |
187 | buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */ |
188 | luaX_lexerror(ls, "malformed number" , TK_NUMBER); |
189 | } |
190 | } |
191 | |
192 | |
193 | /* LUA_NUMBER */ |
194 | static void read_numeral (LexState *ls, SemInfo *seminfo) { |
195 | lua_assert(isdigit(ls->current)); |
196 | do { |
197 | save_and_next(ls); |
198 | } while (isdigit(ls->current) || ls->current == '.'); |
199 | if (check_next(ls, "Ee" )) /* `E'? */ |
200 | check_next(ls, "+-" ); /* optional exponent sign */ |
201 | while (isalnum(ls->current) || ls->current == '_') |
202 | save_and_next(ls); |
203 | save(ls, '\0'); |
204 | buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ |
205 | if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */ |
206 | trydecpoint(ls, seminfo); /* try to update decimal point separator */ |
207 | } |
208 | |
209 | |
210 | static int skip_sep (LexState *ls) { |
211 | int count = 0; |
212 | int s = ls->current; |
213 | lua_assert(s == '[' || s == ']'); |
214 | save_and_next(ls); |
215 | while (ls->current == '=') { |
216 | save_and_next(ls); |
217 | count++; |
218 | } |
219 | return (ls->current == s) ? count : (-count) - 1; |
220 | } |
221 | |
222 | |
223 | static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { |
224 | int cont = 0; |
225 | (void)(cont); /* avoid warnings when `cont' is not used */ |
226 | save_and_next(ls); /* skip 2nd `[' */ |
227 | if (currIsNewline(ls)) /* string starts with a newline? */ |
228 | inclinenumber(ls); /* skip it */ |
229 | for (;;) { |
230 | switch (ls->current) { |
231 | case EOZ: |
232 | luaX_lexerror(ls, (seminfo) ? "unfinished long string" : |
233 | "unfinished long comment" , TK_EOS); |
234 | break; /* to avoid warnings */ |
235 | #if defined(LUA_COMPAT_LSTR) |
236 | case '[': { |
237 | if (skip_sep(ls) == sep) { |
238 | save_and_next(ls); /* skip 2nd `[' */ |
239 | cont++; |
240 | #if LUA_COMPAT_LSTR == 1 |
241 | if (sep == 0) |
242 | luaX_lexerror(ls, "nesting of [[...]] is deprecated" , '['); |
243 | #endif |
244 | } |
245 | break; |
246 | } |
247 | #endif |
248 | case ']': { |
249 | if (skip_sep(ls) == sep) { |
250 | save_and_next(ls); /* skip 2nd `]' */ |
251 | #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2 |
252 | cont--; |
253 | if (sep == 0 && cont >= 0) break; |
254 | #endif |
255 | goto endloop; |
256 | } |
257 | break; |
258 | } |
259 | case '\n': |
260 | case '\r': { |
261 | save(ls, '\n'); |
262 | inclinenumber(ls); |
263 | if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ |
264 | break; |
265 | } |
266 | default: { |
267 | if (seminfo) save_and_next(ls); |
268 | else next(ls); |
269 | } |
270 | } |
271 | } endloop: |
272 | if (seminfo) |
273 | seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), |
274 | luaZ_bufflen(ls->buff) - 2*(2 + sep)); |
275 | } |
276 | |
277 | |
278 | static void read_string (LexState *ls, int del, SemInfo *seminfo) { |
279 | save_and_next(ls); |
280 | while (ls->current != del) { |
281 | switch (ls->current) { |
282 | case EOZ: |
283 | luaX_lexerror(ls, "unfinished string" , TK_EOS); |
284 | continue; /* to avoid warnings */ |
285 | case '\n': |
286 | case '\r': |
287 | luaX_lexerror(ls, "unfinished string" , TK_STRING); |
288 | continue; /* to avoid warnings */ |
289 | case '\\': { |
290 | int c; |
291 | next(ls); /* do not save the `\' */ |
292 | switch (ls->current) { |
293 | case 'a': c = '\a'; break; |
294 | case 'b': c = '\b'; break; |
295 | case 'f': c = '\f'; break; |
296 | case 'n': c = '\n'; break; |
297 | case 'r': c = '\r'; break; |
298 | case 't': c = '\t'; break; |
299 | case 'v': c = '\v'; break; |
300 | case '\n': /* go through */ |
301 | case '\r': save(ls, '\n'); inclinenumber(ls); continue; |
302 | case EOZ: continue; /* will raise an error next loop */ |
303 | default: { |
304 | if (!isdigit(ls->current)) |
305 | save_and_next(ls); /* handles \\, \", \', and \? */ |
306 | else { /* \xxx */ |
307 | int i = 0; |
308 | c = 0; |
309 | do { |
310 | c = 10*c + (ls->current-'0'); |
311 | next(ls); |
312 | } while (++i<3 && isdigit(ls->current)); |
313 | if (c > UCHAR_MAX) |
314 | luaX_lexerror(ls, "escape sequence too large" , TK_STRING); |
315 | save(ls, c); |
316 | } |
317 | continue; |
318 | } |
319 | } |
320 | save(ls, c); |
321 | next(ls); |
322 | continue; |
323 | } |
324 | default: |
325 | save_and_next(ls); |
326 | } |
327 | } |
328 | save_and_next(ls); /* skip delimiter */ |
329 | seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, |
330 | luaZ_bufflen(ls->buff) - 2); |
331 | } |
332 | |
333 | |
334 | static int llex (LexState *ls, SemInfo *seminfo) { |
335 | luaZ_resetbuffer(ls->buff); |
336 | for (;;) { |
337 | switch (ls->current) { |
338 | case '\n': |
339 | case '\r': { |
340 | inclinenumber(ls); |
341 | continue; |
342 | } |
343 | case '-': { |
344 | next(ls); |
345 | if (ls->current != '-') return '-'; |
346 | /* else is a comment */ |
347 | next(ls); |
348 | if (ls->current == '[') { |
349 | int sep = skip_sep(ls); |
350 | luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ |
351 | if (sep >= 0) { |
352 | read_long_string(ls, NULL, sep); /* long comment */ |
353 | luaZ_resetbuffer(ls->buff); |
354 | continue; |
355 | } |
356 | } |
357 | /* else short comment */ |
358 | while (!currIsNewline(ls) && ls->current != EOZ) |
359 | next(ls); |
360 | continue; |
361 | } |
362 | case '[': { |
363 | int sep = skip_sep(ls); |
364 | if (sep >= 0) { |
365 | read_long_string(ls, seminfo, sep); |
366 | return TK_STRING; |
367 | } |
368 | else if (sep == -1) return '['; |
369 | else luaX_lexerror(ls, "invalid long string delimiter" , TK_STRING); |
370 | } |
371 | case '=': { |
372 | next(ls); |
373 | if (ls->current != '=') return '='; |
374 | else { next(ls); return TK_EQ; } |
375 | } |
376 | case '<': { |
377 | next(ls); |
378 | if (ls->current != '=') return '<'; |
379 | else { next(ls); return TK_LE; } |
380 | } |
381 | case '>': { |
382 | next(ls); |
383 | if (ls->current != '=') return '>'; |
384 | else { next(ls); return TK_GE; } |
385 | } |
386 | case '~': { |
387 | next(ls); |
388 | if (ls->current != '=') return '~'; |
389 | else { next(ls); return TK_NE; } |
390 | } |
391 | case '"': |
392 | case '\'': { |
393 | read_string(ls, ls->current, seminfo); |
394 | return TK_STRING; |
395 | } |
396 | case '.': { |
397 | save_and_next(ls); |
398 | if (check_next(ls, "." )) { |
399 | if (check_next(ls, "." )) |
400 | return TK_DOTS; /* ... */ |
401 | else return TK_CONCAT; /* .. */ |
402 | } |
403 | else if (!isdigit(ls->current)) return '.'; |
404 | else { |
405 | read_numeral(ls, seminfo); |
406 | return TK_NUMBER; |
407 | } |
408 | } |
409 | case EOZ: { |
410 | return TK_EOS; |
411 | } |
412 | default: { |
413 | if (isspace(ls->current)) { |
414 | lua_assert(!currIsNewline(ls)); |
415 | next(ls); |
416 | continue; |
417 | } |
418 | else if (isdigit(ls->current)) { |
419 | read_numeral(ls, seminfo); |
420 | return TK_NUMBER; |
421 | } |
422 | else if (isalpha(ls->current) || ls->current == '_') { |
423 | /* identifier or reserved word */ |
424 | TString *ts; |
425 | do { |
426 | save_and_next(ls); |
427 | } while (isalnum(ls->current) || ls->current == '_'); |
428 | ts = luaX_newstring(ls, luaZ_buffer(ls->buff), |
429 | luaZ_bufflen(ls->buff)); |
430 | if (ts->tsv.reserved > 0) /* reserved word? */ |
431 | return ts->tsv.reserved - 1 + FIRST_RESERVED; |
432 | else { |
433 | seminfo->ts = ts; |
434 | return TK_NAME; |
435 | } |
436 | } |
437 | else { |
438 | int c = ls->current; |
439 | next(ls); |
440 | return c; /* single-char tokens (+ - / ...) */ |
441 | } |
442 | } |
443 | } |
444 | } |
445 | } |
446 | |
447 | |
448 | void luaX_next (LexState *ls) { |
449 | ls->lastline = ls->linenumber; |
450 | if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */ |
451 | ls->t = ls->lookahead; /* use this one */ |
452 | ls->lookahead.token = TK_EOS; /* and discharge it */ |
453 | } |
454 | else |
455 | ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */ |
456 | } |
457 | |
458 | |
459 | void luaX_lookahead (LexState *ls) { |
460 | lua_assert(ls->lookahead.token == TK_EOS); |
461 | ls->lookahead.token = llex(ls, &ls->lookahead.seminfo); |
462 | } |
463 | |
464 | |