1/*
2 * Copyright 2014 Google Inc. All rights reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <algorithm>
18#include <cmath>
19#include <list>
20#include <string>
21#include <utility>
22
23#include "flatbuffers/base.h"
24#include "flatbuffers/idl.h"
25#include "flatbuffers/util.h"
26
27namespace flatbuffers {
28
29// Reflects the version at the compiling time of binary(lib/dll/so).
30const char *FLATBUFFERS_VERSION() {
31 // clang-format off
32 return
33 FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "."
34 FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MINOR) "."
35 FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION);
36 // clang-format on
37}
38
39const double kPi = 3.14159265358979323846;
40
41// clang-format off
42const char *const kTypeNames[] = {
43 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ...) \
44 IDLTYPE,
45 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
46 #undef FLATBUFFERS_TD
47 nullptr
48};
49
50const char kTypeSizes[] = {
51 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
52 sizeof(CTYPE),
53 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
54 #undef FLATBUFFERS_TD
55};
56// clang-format on
57
58// The enums in the reflection schema should match the ones we use internally.
59// Compare the last element to check if these go out of sync.
60static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union),
61 "enums don't match");
62
63// Any parsing calls have to be wrapped in this macro, which automates
64// handling of recursive error checking a bit. It will check the received
65// CheckedError object, and return straight away on error.
66#define ECHECK(call) \
67 { \
68 auto ce = (call); \
69 if (ce.Check()) return ce; \
70 }
71
72// These two functions are called hundreds of times below, so define a short
73// form:
74#define NEXT() ECHECK(Next())
75#define EXPECT(tok) ECHECK(Expect(tok))
76
77static bool ValidateUTF8(const std::string &str) {
78 const char *s = &str[0];
79 const char *const sEnd = s + str.length();
80 while (s < sEnd) {
81 if (FromUTF8(&s) < 0) { return false; }
82 }
83 return true;
84}
85
86static bool IsLowerSnakeCase(const std::string &str) {
87 for (size_t i = 0; i < str.length(); i++) {
88 char c = str[i];
89 if (!check_ascii_range(c, 'a', 'z') && !is_digit(c) && c != '_') {
90 return false;
91 }
92 }
93 return true;
94}
95
96// Convert an underscore_based_identifier in to camelCase.
97// Also uppercases the first character if first is true.
98std::string MakeCamel(const std::string &in, bool first) {
99 std::string s;
100 for (size_t i = 0; i < in.length(); i++) {
101 if (!i && first)
102 s += CharToUpper(in[0]);
103 else if (in[i] == '_' && i + 1 < in.length())
104 s += CharToUpper(in[++i]);
105 else
106 s += in[i];
107 }
108 return s;
109}
110
111// Convert an underscore_based_identifier in to screaming snake case.
112std::string MakeScreamingCamel(const std::string &in) {
113 std::string s;
114 for (size_t i = 0; i < in.length(); i++) {
115 if (in[i] != '_')
116 s += CharToUpper(in[i]);
117 else
118 s += in[i];
119 }
120 return s;
121}
122
123void DeserializeDoc(std::vector<std::string> &doc,
124 const Vector<Offset<String>> *documentation) {
125 if (documentation == nullptr) return;
126 for (uoffset_t index = 0; index < documentation->size(); index++)
127 doc.push_back(documentation->Get(index)->str());
128}
129
130void Parser::Message(const std::string &msg) {
131 if (!error_.empty()) error_ += "\n"; // log all warnings and errors
132 error_ += file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
133 // clang-format off
134
135 #ifdef _WIN32 // MSVC alike
136 error_ +=
137 "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")";
138 #else // gcc alike
139 if (file_being_parsed_.length()) error_ += ":";
140 error_ += NumToString(line_) + ": " + NumToString(CursorPosition());
141 #endif
142 // clang-format on
143 error_ += ": " + msg;
144}
145
146void Parser::Warning(const std::string &msg) {
147 if (!opts.no_warnings) {
148 Message("warning: " + msg);
149 has_warning_ = true; // for opts.warnings_as_errors
150 }
151}
152
153CheckedError Parser::Error(const std::string &msg) {
154 Message("error: " + msg);
155 return CheckedError(true);
156}
157
158inline CheckedError NoError() { return CheckedError(false); }
159
160CheckedError Parser::RecurseError() {
161 return Error("maximum parsing depth " + NumToString(parse_depth_counter_) +
162 " reached");
163}
164
165const std::string &Parser::GetPooledString(const std::string &s) const {
166 return *(string_cache_.insert(s).first);
167}
168
169class Parser::ParseDepthGuard {
170 public:
171 explicit ParseDepthGuard(Parser *parser_not_null)
172 : parser_(*parser_not_null), caller_depth_(parser_.parse_depth_counter_) {
173 FLATBUFFERS_ASSERT(caller_depth_ <= (FLATBUFFERS_MAX_PARSING_DEPTH) &&
174 "Check() must be called to prevent stack overflow");
175 parser_.parse_depth_counter_ += 1;
176 }
177
178 ~ParseDepthGuard() { parser_.parse_depth_counter_ -= 1; }
179
180 CheckedError Check() {
181 return caller_depth_ >= (FLATBUFFERS_MAX_PARSING_DEPTH)
182 ? parser_.RecurseError()
183 : CheckedError(false);
184 }
185
186 FLATBUFFERS_DELETE_FUNC(ParseDepthGuard(const ParseDepthGuard &));
187 FLATBUFFERS_DELETE_FUNC(ParseDepthGuard &operator=(const ParseDepthGuard &));
188
189 private:
190 Parser &parser_;
191 const int caller_depth_;
192};
193
194template<typename T> std::string TypeToIntervalString() {
195 return "[" + NumToString((flatbuffers::numeric_limits<T>::lowest)()) + "; " +
196 NumToString((flatbuffers::numeric_limits<T>::max)()) + "]";
197}
198
199// atot: template version of atoi/atof: convert a string to an instance of T.
200template<typename T>
201bool atot_scalar(const char *s, T *val, bool_constant<false>) {
202 return StringToNumber(s, val);
203}
204
205template<typename T>
206bool atot_scalar(const char *s, T *val, bool_constant<true>) {
207 // Normalize NaN parsed from fbs or json to unsigned NaN.
208 if (false == StringToNumber(s, val)) return false;
209 *val = (*val != *val) ? std::fabs(*val) : *val;
210 return true;
211}
212
213template<typename T> CheckedError atot(const char *s, Parser &parser, T *val) {
214 auto done = atot_scalar(s, val, bool_constant<is_floating_point<T>::value>());
215 if (done) return NoError();
216 if (0 == *val)
217 return parser.Error("invalid number: \"" + std::string(s) + "\"");
218 else
219 return parser.Error("invalid number: \"" + std::string(s) + "\"" +
220 ", constant does not fit " + TypeToIntervalString<T>());
221}
222template<>
223inline CheckedError atot<Offset<void>>(const char *s, Parser &parser,
224 Offset<void> *val) {
225 (void)parser;
226 *val = Offset<void>(atoi(s));
227 return NoError();
228}
229
230std::string Namespace::GetFullyQualifiedName(const std::string &name,
231 size_t max_components) const {
232 // Early exit if we don't have a defined namespace.
233 if (components.empty() || !max_components) { return name; }
234 std::string stream_str;
235 for (size_t i = 0; i < std::min(components.size(), max_components); i++) {
236 stream_str += components[i];
237 stream_str += '.';
238 }
239 if (!stream_str.empty()) stream_str.pop_back();
240 if (name.length()) {
241 stream_str += '.';
242 stream_str += name;
243 }
244 return stream_str;
245}
246
247template<typename T>
248T *LookupTableByName(const SymbolTable<T> &table, const std::string &name,
249 const Namespace &current_namespace, size_t skip_top) {
250 const auto &components = current_namespace.components;
251 if (table.dict.empty()) return nullptr;
252 if (components.size() < skip_top) return nullptr;
253 const auto N = components.size() - skip_top;
254 std::string full_name;
255 for (size_t i = 0; i < N; i++) {
256 full_name += components[i];
257 full_name += '.';
258 }
259 for (size_t i = N; i > 0; i--) {
260 full_name += name;
261 auto obj = table.Lookup(full_name);
262 if (obj) return obj;
263 auto len = full_name.size() - components[i - 1].size() - 1 - name.size();
264 full_name.resize(len);
265 }
266 FLATBUFFERS_ASSERT(full_name.empty());
267 return table.Lookup(name); // lookup in global namespace
268}
269
270// Declare tokens we'll use. Single character tokens are represented by their
271// ascii character code (e.g. '{'), others above 256.
272// clang-format off
273#define FLATBUFFERS_GEN_TOKENS(TD) \
274 TD(Eof, 256, "end of file") \
275 TD(StringConstant, 257, "string constant") \
276 TD(IntegerConstant, 258, "integer constant") \
277 TD(FloatConstant, 259, "float constant") \
278 TD(Identifier, 260, "identifier")
279#ifdef __GNUC__
280__extension__ // Stop GCC complaining about trailing comma with -Wpendantic.
281#endif
282enum {
283 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE,
284 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
285 #undef FLATBUFFERS_TOKEN
286};
287
288static std::string TokenToString(int t) {
289 static const char * const tokens[] = {
290 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING,
291 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
292 #undef FLATBUFFERS_TOKEN
293 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ...) \
294 IDLTYPE,
295 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
296 #undef FLATBUFFERS_TD
297 };
298 if (t < 256) { // A single ascii char token.
299 std::string s;
300 s.append(1, static_cast<char>(t));
301 return s;
302 } else { // Other tokens.
303 return tokens[t - 256];
304 }
305}
306// clang-format on
307
308std::string Parser::TokenToStringId(int t) const {
309 return t == kTokenIdentifier ? attribute_ : TokenToString(t);
310}
311
312// Parses exactly nibbles worth of hex digits into a number, or error.
313CheckedError Parser::ParseHexNum(int nibbles, uint64_t *val) {
314 FLATBUFFERS_ASSERT(nibbles > 0);
315 for (int i = 0; i < nibbles; i++)
316 if (!is_xdigit(cursor_[i]))
317 return Error("escape code must be followed by " + NumToString(nibbles) +
318 " hex digits");
319 std::string target(cursor_, cursor_ + nibbles);
320 *val = StringToUInt(target.c_str(), 16);
321 cursor_ += nibbles;
322 return NoError();
323}
324
325CheckedError Parser::SkipByteOrderMark() {
326 if (static_cast<unsigned char>(*cursor_) != 0xef) return NoError();
327 cursor_++;
328 if (static_cast<unsigned char>(*cursor_) != 0xbb)
329 return Error("invalid utf-8 byte order mark");
330 cursor_++;
331 if (static_cast<unsigned char>(*cursor_) != 0xbf)
332 return Error("invalid utf-8 byte order mark");
333 cursor_++;
334 return NoError();
335}
336
337static inline bool IsIdentifierStart(char c) {
338 return is_alpha(c) || (c == '_');
339}
340
341CheckedError Parser::Next() {
342 doc_comment_.clear();
343 bool seen_newline = cursor_ == source_;
344 attribute_.clear();
345 attr_is_trivial_ascii_string_ = true;
346 for (;;) {
347 char c = *cursor_++;
348 token_ = c;
349 switch (c) {
350 case '\0':
351 cursor_--;
352 token_ = kTokenEof;
353 return NoError();
354 case ' ':
355 case '\r':
356 case '\t': break;
357 case '\n':
358 MarkNewLine();
359 seen_newline = true;
360 break;
361 case '{':
362 case '}':
363 case '(':
364 case ')':
365 case '[':
366 case ']':
367 case ',':
368 case ':':
369 case ';':
370 case '=': return NoError();
371 case '\"':
372 case '\'': {
373 int unicode_high_surrogate = -1;
374
375 while (*cursor_ != c) {
376 if (*cursor_ < ' ' && static_cast<signed char>(*cursor_) >= 0)
377 return Error("illegal character in string constant");
378 if (*cursor_ == '\\') {
379 attr_is_trivial_ascii_string_ = false; // has escape sequence
380 cursor_++;
381 if (unicode_high_surrogate != -1 && *cursor_ != 'u') {
382 return Error(
383 "illegal Unicode sequence (unpaired high surrogate)");
384 }
385 switch (*cursor_) {
386 case 'n':
387 attribute_ += '\n';
388 cursor_++;
389 break;
390 case 't':
391 attribute_ += '\t';
392 cursor_++;
393 break;
394 case 'r':
395 attribute_ += '\r';
396 cursor_++;
397 break;
398 case 'b':
399 attribute_ += '\b';
400 cursor_++;
401 break;
402 case 'f':
403 attribute_ += '\f';
404 cursor_++;
405 break;
406 case '\"':
407 attribute_ += '\"';
408 cursor_++;
409 break;
410 case '\'':
411 attribute_ += '\'';
412 cursor_++;
413 break;
414 case '\\':
415 attribute_ += '\\';
416 cursor_++;
417 break;
418 case '/':
419 attribute_ += '/';
420 cursor_++;
421 break;
422 case 'x': { // Not in the JSON standard
423 cursor_++;
424 uint64_t val;
425 ECHECK(ParseHexNum(2, &val));
426 attribute_ += static_cast<char>(val);
427 break;
428 }
429 case 'u': {
430 cursor_++;
431 uint64_t val;
432 ECHECK(ParseHexNum(4, &val));
433 if (val >= 0xD800 && val <= 0xDBFF) {
434 if (unicode_high_surrogate != -1) {
435 return Error(
436 "illegal Unicode sequence (multiple high surrogates)");
437 } else {
438 unicode_high_surrogate = static_cast<int>(val);
439 }
440 } else if (val >= 0xDC00 && val <= 0xDFFF) {
441 if (unicode_high_surrogate == -1) {
442 return Error(
443 "illegal Unicode sequence (unpaired low surrogate)");
444 } else {
445 int code_point = 0x10000 +
446 ((unicode_high_surrogate & 0x03FF) << 10) +
447 (val & 0x03FF);
448 ToUTF8(code_point, &attribute_);
449 unicode_high_surrogate = -1;
450 }
451 } else {
452 if (unicode_high_surrogate != -1) {
453 return Error(
454 "illegal Unicode sequence (unpaired high surrogate)");
455 }
456 ToUTF8(static_cast<int>(val), &attribute_);
457 }
458 break;
459 }
460 default: return Error("unknown escape code in string constant");
461 }
462 } else { // printable chars + UTF-8 bytes
463 if (unicode_high_surrogate != -1) {
464 return Error(
465 "illegal Unicode sequence (unpaired high surrogate)");
466 }
467 // reset if non-printable
468 attr_is_trivial_ascii_string_ &=
469 check_ascii_range(*cursor_, ' ', '~');
470
471 attribute_ += *cursor_++;
472 }
473 }
474 if (unicode_high_surrogate != -1) {
475 return Error("illegal Unicode sequence (unpaired high surrogate)");
476 }
477 cursor_++;
478 if (!attr_is_trivial_ascii_string_ && !opts.allow_non_utf8 &&
479 !ValidateUTF8(attribute_)) {
480 return Error("illegal UTF-8 sequence");
481 }
482 token_ = kTokenStringConstant;
483 return NoError();
484 }
485 case '/':
486 if (*cursor_ == '/') {
487 const char *start = ++cursor_;
488 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++;
489 if (*start == '/') { // documentation comment
490 if (!seen_newline)
491 return Error(
492 "a documentation comment should be on a line on its own");
493 doc_comment_.push_back(std::string(start + 1, cursor_));
494 }
495 break;
496 } else if (*cursor_ == '*') {
497 cursor_++;
498 // TODO: make nested.
499 while (*cursor_ != '*' || cursor_[1] != '/') {
500 if (*cursor_ == '\n') MarkNewLine();
501 if (!*cursor_) return Error("end of file in comment");
502 cursor_++;
503 }
504 cursor_ += 2;
505 break;
506 }
507 FLATBUFFERS_FALLTHROUGH(); // else fall thru
508 default:
509 if (IsIdentifierStart(c)) {
510 // Collect all chars of an identifier:
511 const char *start = cursor_ - 1;
512 while (IsIdentifierStart(*cursor_) || is_digit(*cursor_)) cursor_++;
513 attribute_.append(start, cursor_);
514 token_ = kTokenIdentifier;
515 return NoError();
516 }
517
518 const auto has_sign = (c == '+') || (c == '-');
519 if (has_sign && IsIdentifierStart(*cursor_)) {
520 // '-'/'+' and following identifier - it could be a predefined
521 // constant. Return the sign in token_, see ParseSingleValue.
522 return NoError();
523 }
524
525 auto dot_lvl =
526 (c == '.') ? 0 : 1; // dot_lvl==0 <=> exactly one '.' seen
527 if (!dot_lvl && !is_digit(*cursor_)) return NoError(); // enum?
528 // Parser accepts hexadecimal-floating-literal (see C++ 5.13.4).
529 if (is_digit(c) || has_sign || !dot_lvl) {
530 const auto start = cursor_ - 1;
531 auto start_digits = !is_digit(c) ? cursor_ : cursor_ - 1;
532 if (!is_digit(c) && is_digit(*cursor_)) {
533 start_digits = cursor_; // see digit in cursor_ position
534 c = *cursor_++;
535 }
536 // hex-float can't begind with '.'
537 auto use_hex = dot_lvl && (c == '0') && is_alpha_char(*cursor_, 'X');
538 if (use_hex) start_digits = ++cursor_; // '0x' is the prefix, skip it
539 // Read an integer number or mantisa of float-point number.
540 do {
541 if (use_hex) {
542 while (is_xdigit(*cursor_)) cursor_++;
543 } else {
544 while (is_digit(*cursor_)) cursor_++;
545 }
546 } while ((*cursor_ == '.') && (++cursor_) && (--dot_lvl >= 0));
547 // Exponent of float-point number.
548 if ((dot_lvl >= 0) && (cursor_ > start_digits)) {
549 // The exponent suffix of hexadecimal float number is mandatory.
550 if (use_hex && !dot_lvl) start_digits = cursor_;
551 if ((use_hex && is_alpha_char(*cursor_, 'P')) ||
552 is_alpha_char(*cursor_, 'E')) {
553 dot_lvl = 0; // Emulate dot to signal about float-point number.
554 cursor_++;
555 if (*cursor_ == '+' || *cursor_ == '-') cursor_++;
556 start_digits = cursor_; // the exponent-part has to have digits
557 // Exponent is decimal integer number
558 while (is_digit(*cursor_)) cursor_++;
559 if (*cursor_ == '.') {
560 cursor_++; // If see a dot treat it as part of invalid number.
561 dot_lvl = -1; // Fall thru to Error().
562 }
563 }
564 }
565 // Finalize.
566 if ((dot_lvl >= 0) && (cursor_ > start_digits)) {
567 attribute_.append(start, cursor_);
568 token_ = dot_lvl ? kTokenIntegerConstant : kTokenFloatConstant;
569 return NoError();
570 } else {
571 return Error("invalid number: " + std::string(start, cursor_));
572 }
573 }
574 std::string ch;
575 ch = c;
576 if (false == check_ascii_range(c, ' ', '~'))
577 ch = "code: " + NumToString(c);
578 return Error("illegal character: " + ch);
579 }
580 }
581}
582
583// Check if a given token is next.
584bool Parser::Is(int t) const { return t == token_; }
585
586bool Parser::IsIdent(const char *id) const {
587 return token_ == kTokenIdentifier && attribute_ == id;
588}
589
590// Expect a given token to be next, consume it, or error if not present.
591CheckedError Parser::Expect(int t) {
592 if (t != token_) {
593 return Error("expecting: " + TokenToString(t) +
594 " instead got: " + TokenToStringId(token_));
595 }
596 NEXT();
597 return NoError();
598}
599
600CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) {
601 while (Is('.')) {
602 NEXT();
603 *id += ".";
604 *id += attribute_;
605 if (last) *last = attribute_;
606 EXPECT(kTokenIdentifier);
607 }
608 return NoError();
609}
610
611EnumDef *Parser::LookupEnum(const std::string &id) {
612 // Search thru parent namespaces.
613 return LookupTableByName(enums_, id, *current_namespace_, 0);
614}
615
616StructDef *Parser::LookupStruct(const std::string &id) const {
617 auto sd = structs_.Lookup(id);
618 if (sd) sd->refcount++;
619 return sd;
620}
621
622StructDef *Parser::LookupStructThruParentNamespaces(
623 const std::string &id) const {
624 auto sd = LookupTableByName(structs_, id, *current_namespace_, 1);
625 if (sd) sd->refcount++;
626 return sd;
627}
628
629CheckedError Parser::ParseTypeIdent(Type &type) {
630 std::string id = attribute_;
631 EXPECT(kTokenIdentifier);
632 ECHECK(ParseNamespacing(&id, nullptr));
633 auto enum_def = LookupEnum(id);
634 if (enum_def) {
635 type = enum_def->underlying_type;
636 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
637 } else {
638 type.base_type = BASE_TYPE_STRUCT;
639 type.struct_def = LookupCreateStruct(id);
640 }
641 return NoError();
642}
643
644// Parse any IDL type.
645CheckedError Parser::ParseType(Type &type) {
646 if (token_ == kTokenIdentifier) {
647 if (IsIdent("bool")) {
648 type.base_type = BASE_TYPE_BOOL;
649 NEXT();
650 } else if (IsIdent("byte") || IsIdent("int8")) {
651 type.base_type = BASE_TYPE_CHAR;
652 NEXT();
653 } else if (IsIdent("ubyte") || IsIdent("uint8")) {
654 type.base_type = BASE_TYPE_UCHAR;
655 NEXT();
656 } else if (IsIdent("short") || IsIdent("int16")) {
657 type.base_type = BASE_TYPE_SHORT;
658 NEXT();
659 } else if (IsIdent("ushort") || IsIdent("uint16")) {
660 type.base_type = BASE_TYPE_USHORT;
661 NEXT();
662 } else if (IsIdent("int") || IsIdent("int32")) {
663 type.base_type = BASE_TYPE_INT;
664 NEXT();
665 } else if (IsIdent("uint") || IsIdent("uint32")) {
666 type.base_type = BASE_TYPE_UINT;
667 NEXT();
668 } else if (IsIdent("long") || IsIdent("int64")) {
669 type.base_type = BASE_TYPE_LONG;
670 NEXT();
671 } else if (IsIdent("ulong") || IsIdent("uint64")) {
672 type.base_type = BASE_TYPE_ULONG;
673 NEXT();
674 } else if (IsIdent("float") || IsIdent("float32")) {
675 type.base_type = BASE_TYPE_FLOAT;
676 NEXT();
677 } else if (IsIdent("double") || IsIdent("float64")) {
678 type.base_type = BASE_TYPE_DOUBLE;
679 NEXT();
680 } else if (IsIdent("string")) {
681 type.base_type = BASE_TYPE_STRING;
682 NEXT();
683 } else {
684 ECHECK(ParseTypeIdent(type));
685 }
686 } else if (token_ == '[') {
687 ParseDepthGuard depth_guard(this);
688 ECHECK(depth_guard.Check());
689 NEXT();
690 Type subtype;
691 ECHECK(ParseType(subtype));
692 if (IsSeries(subtype)) {
693 // We could support this, but it will complicate things, and it's
694 // easier to work around with a struct around the inner vector.
695 return Error("nested vector types not supported (wrap in table first)");
696 }
697 if (token_ == ':') {
698 NEXT();
699 if (token_ != kTokenIntegerConstant) {
700 return Error("length of fixed-length array must be an integer value");
701 }
702 uint16_t fixed_length = 0;
703 bool check = StringToNumber(attribute_.c_str(), &fixed_length);
704 if (!check || fixed_length < 1) {
705 return Error(
706 "length of fixed-length array must be positive and fit to "
707 "uint16_t type");
708 }
709 type = Type(BASE_TYPE_ARRAY, subtype.struct_def, subtype.enum_def,
710 fixed_length);
711 NEXT();
712 } else {
713 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def);
714 }
715 type.element = subtype.base_type;
716 EXPECT(']');
717 } else {
718 return Error("illegal type syntax");
719 }
720 return NoError();
721}
722
723CheckedError Parser::AddField(StructDef &struct_def, const std::string &name,
724 const Type &type, FieldDef **dest) {
725 auto &field = *new FieldDef();
726 field.value.offset =
727 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
728 field.name = name;
729 field.file = struct_def.file;
730 field.value.type = type;
731 if (struct_def.fixed) { // statically compute the field offset
732 auto size = InlineSize(type);
733 auto alignment = InlineAlignment(type);
734 // structs_ need to have a predictable format, so we need to align to
735 // the largest scalar
736 struct_def.minalign = std::max(struct_def.minalign, alignment);
737 struct_def.PadLastField(alignment);
738 field.value.offset = static_cast<voffset_t>(struct_def.bytesize);
739 struct_def.bytesize += size;
740 }
741 if (struct_def.fields.Add(name, &field))
742 return Error("field already exists: " + name);
743 *dest = &field;
744 return NoError();
745}
746
747CheckedError Parser::ParseField(StructDef &struct_def) {
748 std::string name = attribute_;
749
750 if (LookupCreateStruct(name, false, false))
751 return Error("field name can not be the same as table/struct name");
752
753 if (!IsLowerSnakeCase(name)) {
754 Warning("field names should be lowercase snake_case, got: " + name);
755 }
756
757 std::vector<std::string> dc = doc_comment_;
758 EXPECT(kTokenIdentifier);
759 EXPECT(':');
760 Type type;
761 ECHECK(ParseType(type));
762
763 if (struct_def.fixed) {
764 auto valid = IsScalar(type.base_type) || IsStruct(type);
765 if (!valid && IsArray(type)) {
766 const auto &elem_type = type.VectorType();
767 valid |= IsScalar(elem_type.base_type) || IsStruct(elem_type);
768 }
769 if (!valid)
770 return Error("structs may contain only scalar or struct fields");
771 }
772
773 if (!struct_def.fixed && IsArray(type))
774 return Error("fixed-length array in table must be wrapped in struct");
775
776 if (IsArray(type)) {
777 advanced_features_ |= reflection::AdvancedArrayFeatures;
778 if (!SupportsAdvancedArrayFeatures()) {
779 return Error(
780 "Arrays are not yet supported in all "
781 "the specified programming languages.");
782 }
783 }
784
785 FieldDef *typefield = nullptr;
786 if (type.base_type == BASE_TYPE_UNION) {
787 // For union fields, add a second auto-generated field to hold the type,
788 // with a special suffix.
789 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(),
790 type.enum_def->underlying_type, &typefield));
791 } else if (IsVector(type) && type.element == BASE_TYPE_UNION) {
792 advanced_features_ |= reflection::AdvancedUnionFeatures;
793 // Only cpp, js and ts supports the union vector feature so far.
794 if (!SupportsAdvancedUnionFeatures()) {
795 return Error(
796 "Vectors of unions are not yet supported in at least one of "
797 "the specified programming languages.");
798 }
799 // For vector of union fields, add a second auto-generated vector field to
800 // hold the types, with a special suffix.
801 Type union_vector(BASE_TYPE_VECTOR, nullptr, type.enum_def);
802 union_vector.element = BASE_TYPE_UTYPE;
803 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), union_vector,
804 &typefield));
805 }
806
807 FieldDef *field;
808 ECHECK(AddField(struct_def, name, type, &field));
809
810 if (token_ == '=') {
811 NEXT();
812 ECHECK(ParseSingleValue(&field->name, field->value, true));
813 if (IsStruct(type) || (struct_def.fixed && field->value.constant != "0"))
814 return Error(
815 "default values are not supported for struct fields, table fields, "
816 "or in structs.");
817 if (IsString(type) || IsVector(type)) {
818 advanced_features_ |= reflection::DefaultVectorsAndStrings;
819 if (field->value.constant != "0" && !SupportsDefaultVectorsAndStrings()) {
820 return Error(
821 "Default values for strings and vectors are not supported in one "
822 "of the specified programming languages");
823 }
824 }
825
826 if (IsVector(type) && field->value.constant != "0" &&
827 field->value.constant != "[]") {
828 return Error("The only supported default for vectors is `[]`.");
829 }
830 }
831
832 // Append .0 if the value has not it (skip hex and scientific floats).
833 // This suffix needed for generated C++ code.
834 if (IsFloat(type.base_type)) {
835 auto &text = field->value.constant;
836 FLATBUFFERS_ASSERT(false == text.empty());
837 auto s = text.c_str();
838 while (*s == ' ') s++;
839 if (*s == '-' || *s == '+') s++;
840 // 1) A float constants (nan, inf, pi, etc) is a kind of identifier.
841 // 2) A float number needn't ".0" at the end if it has exponent.
842 if ((false == IsIdentifierStart(*s)) &&
843 (std::string::npos == field->value.constant.find_first_of(".eEpP"))) {
844 field->value.constant += ".0";
845 }
846 }
847
848 field->doc_comment = dc;
849 ECHECK(ParseMetaData(&field->attributes));
850 field->deprecated = field->attributes.Lookup("deprecated") != nullptr;
851 auto hash_name = field->attributes.Lookup("hash");
852 if (hash_name) {
853 switch ((IsVector(type)) ? type.element : type.base_type) {
854 case BASE_TYPE_SHORT:
855 case BASE_TYPE_USHORT: {
856 if (FindHashFunction16(hash_name->constant.c_str()) == nullptr)
857 return Error("Unknown hashing algorithm for 16 bit types: " +
858 hash_name->constant);
859 break;
860 }
861 case BASE_TYPE_INT:
862 case BASE_TYPE_UINT: {
863 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
864 return Error("Unknown hashing algorithm for 32 bit types: " +
865 hash_name->constant);
866 break;
867 }
868 case BASE_TYPE_LONG:
869 case BASE_TYPE_ULONG: {
870 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
871 return Error("Unknown hashing algorithm for 64 bit types: " +
872 hash_name->constant);
873 break;
874 }
875 default:
876 return Error(
877 "only short, ushort, int, uint, long and ulong data types support "
878 "hashing.");
879 }
880 }
881
882 // For historical convenience reasons, string keys are assumed required.
883 // Scalars are kDefault unless otherwise specified.
884 // Nonscalars are kOptional unless required;
885 field->key = field->attributes.Lookup("key") != nullptr;
886 const bool required = field->attributes.Lookup("required") != nullptr ||
887 (IsString(type) && field->key);
888 const bool default_str_or_vec =
889 ((IsString(type) || IsVector(type)) && field->value.constant != "0");
890 const bool optional = IsScalar(type.base_type)
891 ? (field->value.constant == "null")
892 : !(required || default_str_or_vec);
893 if (required && optional) {
894 return Error("Fields cannot be both optional and required.");
895 }
896 field->presence = FieldDef::MakeFieldPresence(optional, required);
897
898 if (required && (struct_def.fixed || IsScalar(type.base_type))) {
899 return Error("only non-scalar fields in tables may be 'required'");
900 }
901 if (field->key) {
902 if (struct_def.has_key) return Error("only one field may be set as 'key'");
903 struct_def.has_key = true;
904 if (!IsScalar(type.base_type) && !IsString(type)) {
905 return Error("'key' field must be string or scalar type");
906 }
907 }
908
909 if (field->IsScalarOptional()) {
910 advanced_features_ |= reflection::OptionalScalars;
911 if (type.enum_def && type.enum_def->Lookup("null")) {
912 FLATBUFFERS_ASSERT(IsInteger(type.base_type));
913 return Error(
914 "the default 'null' is reserved for declaring optional scalar "
915 "fields, it conflicts with declaration of enum '" +
916 type.enum_def->name + "'.");
917 }
918 if (field->attributes.Lookup("key")) {
919 return Error(
920 "only a non-optional scalar field can be used as a 'key' field");
921 }
922 if (!SupportsOptionalScalars()) {
923 return Error(
924 "Optional scalars are not yet supported in at least one of "
925 "the specified programming languages.");
926 }
927 }
928
929 if (type.enum_def) {
930 // Verify the enum's type and default value.
931 const std::string &constant = field->value.constant;
932 if (type.base_type == BASE_TYPE_UNION) {
933 if (constant != "0") { return Error("Union defaults must be NONE"); }
934 } else if (IsVector(type)) {
935 if (constant != "0" && constant != "[]") {
936 return Error("Vector defaults may only be `[]`.");
937 }
938 } else if (IsArray(type)) {
939 if (constant != "0") {
940 return Error("Array defaults are not supported yet.");
941 }
942 } else {
943 if (!IsInteger(type.base_type)) {
944 return Error("Enums must have integer base types");
945 }
946 // Optional and bitflags enums may have default constants that are not
947 // their specified variants.
948 if (!field->IsOptional() &&
949 type.enum_def->attributes.Lookup("bit_flags") == nullptr) {
950 if (type.enum_def->FindByValue(constant) == nullptr) {
951 return Error("default value of `" + constant + "` for " + "field `" +
952 name + "` is not part of enum `" + type.enum_def->name +
953 "`.");
954 }
955 }
956 }
957 }
958
959 if (field->deprecated && struct_def.fixed)
960 return Error("can't deprecate fields in a struct");
961
962 auto cpp_type = field->attributes.Lookup("cpp_type");
963 if (cpp_type) {
964 if (!hash_name)
965 return Error("cpp_type can only be used with a hashed field");
966 /// forcing cpp_ptr_type to 'naked' if unset
967 auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type");
968 if (!cpp_ptr_type) {
969 auto val = new Value();
970 val->type = cpp_type->type;
971 val->constant = "naked";
972 field->attributes.Add("cpp_ptr_type", val);
973 }
974 }
975
976 field->shared = field->attributes.Lookup("shared") != nullptr;
977 if (field->shared && field->value.type.base_type != BASE_TYPE_STRING)
978 return Error("shared can only be defined on strings");
979
980 auto field_native_custom_alloc =
981 field->attributes.Lookup("native_custom_alloc");
982 if (field_native_custom_alloc)
983 return Error(
984 "native_custom_alloc can only be used with a table or struct "
985 "definition");
986
987 field->native_inline = field->attributes.Lookup("native_inline") != nullptr;
988 if (field->native_inline && !IsStruct(field->value.type))
989 return Error("native_inline can only be defined on structs");
990
991 auto nested = field->attributes.Lookup("nested_flatbuffer");
992 if (nested) {
993 if (nested->type.base_type != BASE_TYPE_STRING)
994 return Error(
995 "nested_flatbuffer attribute must be a string (the root type)");
996 if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
997 return Error(
998 "nested_flatbuffer attribute may only apply to a vector of ubyte");
999 // This will cause an error if the root type of the nested flatbuffer
1000 // wasn't defined elsewhere.
1001 field->nested_flatbuffer = LookupCreateStruct(nested->constant);
1002 }
1003
1004 if (field->attributes.Lookup("flexbuffer")) {
1005 field->flexbuffer = true;
1006 uses_flexbuffers_ = true;
1007 if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
1008 return Error("flexbuffer attribute may only apply to a vector of ubyte");
1009 }
1010
1011 if (typefield) {
1012 if (!IsScalar(typefield->value.type.base_type)) {
1013 // this is a union vector field
1014 typefield->presence = field->presence;
1015 }
1016 // If this field is a union, and it has a manually assigned id,
1017 // the automatically added type field should have an id as well (of N - 1).
1018 auto attr = field->attributes.Lookup("id");
1019 if (attr) {
1020 const auto &id_str = attr->constant;
1021 voffset_t id = 0;
1022 const auto done = !atot(id_str.c_str(), *this, &id).Check();
1023 if (done && id > 0) {
1024 auto val = new Value();
1025 val->type = attr->type;
1026 val->constant = NumToString(id - 1);
1027 typefield->attributes.Add("id", val);
1028 } else {
1029 return Error(
1030 "a union type effectively adds two fields with non-negative ids, "
1031 "its id must be that of the second field (the first field is "
1032 "the type field and not explicitly declared in the schema);\n"
1033 "field: " +
1034 field->name + ", id: " + id_str);
1035 }
1036 }
1037 // if this field is a union that is deprecated,
1038 // the automatically added type field should be deprecated as well
1039 if (field->deprecated) { typefield->deprecated = true; }
1040 }
1041
1042 EXPECT(';');
1043 return NoError();
1044}
1045
1046CheckedError Parser::ParseString(Value &val, bool use_string_pooling) {
1047 auto s = attribute_;
1048 EXPECT(kTokenStringConstant);
1049 if (use_string_pooling) {
1050 val.constant = NumToString(builder_.CreateSharedString(s).o);
1051 } else {
1052 val.constant = NumToString(builder_.CreateString(s).o);
1053 }
1054 return NoError();
1055}
1056
1057CheckedError Parser::ParseComma() {
1058 if (!opts.protobuf_ascii_alike) EXPECT(',');
1059 return NoError();
1060}
1061
1062CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
1063 size_t parent_fieldn,
1064 const StructDef *parent_struct_def,
1065 uoffset_t count, bool inside_vector) {
1066 switch (val.type.base_type) {
1067 case BASE_TYPE_UNION: {
1068 FLATBUFFERS_ASSERT(field);
1069 std::string constant;
1070 Vector<uint8_t> *vector_of_union_types = nullptr;
1071 // Find corresponding type field we may have already parsed.
1072 for (auto elem = field_stack_.rbegin() + count;
1073 elem != field_stack_.rbegin() + parent_fieldn + count; ++elem) {
1074 auto &type = elem->second->value.type;
1075 if (type.enum_def == val.type.enum_def) {
1076 if (inside_vector) {
1077 if (IsVector(type) && type.element == BASE_TYPE_UTYPE) {
1078 // Vector of union type field.
1079 uoffset_t offset;
1080 ECHECK(atot(elem->first.constant.c_str(), *this, &offset));
1081 vector_of_union_types = reinterpret_cast<Vector<uint8_t> *>(
1082 builder_.GetCurrentBufferPointer() + builder_.GetSize() -
1083 offset);
1084 break;
1085 }
1086 } else {
1087 if (type.base_type == BASE_TYPE_UTYPE) {
1088 // Union type field.
1089 constant = elem->first.constant;
1090 break;
1091 }
1092 }
1093 }
1094 }
1095 if (constant.empty() && !inside_vector) {
1096 // We haven't seen the type field yet. Sadly a lot of JSON writers
1097 // output these in alphabetical order, meaning it comes after this
1098 // value. So we scan past the value to find it, then come back here.
1099 // We currently don't do this for vectors of unions because the
1100 // scanning/serialization logic would get very complicated.
1101 auto type_name = field->name + UnionTypeFieldSuffix();
1102 FLATBUFFERS_ASSERT(parent_struct_def);
1103 auto type_field = parent_struct_def->fields.Lookup(type_name);
1104 FLATBUFFERS_ASSERT(type_field); // Guaranteed by ParseField().
1105 // Remember where we are in the source file, so we can come back here.
1106 auto backup = *static_cast<ParserState *>(this);
1107 ECHECK(SkipAnyJsonValue()); // The table.
1108 ECHECK(ParseComma());
1109 auto next_name = attribute_;
1110 if (Is(kTokenStringConstant)) {
1111 NEXT();
1112 } else {
1113 EXPECT(kTokenIdentifier);
1114 }
1115 if (next_name == type_name) {
1116 EXPECT(':');
1117 ParseDepthGuard depth_guard(this);
1118 ECHECK(depth_guard.Check());
1119 Value type_val = type_field->value;
1120 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr, 0));
1121 constant = type_val.constant;
1122 // Got the information we needed, now rewind:
1123 *static_cast<ParserState *>(this) = backup;
1124 }
1125 }
1126 if (constant.empty() && !vector_of_union_types) {
1127 return Error("missing type field for this union value: " + field->name);
1128 }
1129 uint8_t enum_idx;
1130 if (vector_of_union_types) {
1131 if (vector_of_union_types->size() <= count)
1132 return Error("union types vector smaller than union values vector"
1133 " for: " + field->name);
1134 enum_idx = vector_of_union_types->Get(count);
1135 } else {
1136 ECHECK(atot(constant.c_str(), *this, &enum_idx));
1137 }
1138 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx, true);
1139 if (!enum_val) return Error("illegal type id for: " + field->name);
1140 if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) {
1141 ECHECK(ParseTable(*enum_val->union_type.struct_def, &val.constant,
1142 nullptr));
1143 if (enum_val->union_type.struct_def->fixed) {
1144 // All BASE_TYPE_UNION values are offsets, so turn this into one.
1145 SerializeStruct(*enum_val->union_type.struct_def, val);
1146 builder_.ClearOffsets();
1147 val.constant = NumToString(builder_.GetSize());
1148 }
1149 } else if (IsString(enum_val->union_type)) {
1150 ECHECK(ParseString(val, field->shared));
1151 } else {
1152 FLATBUFFERS_ASSERT(false);
1153 }
1154 break;
1155 }
1156 case BASE_TYPE_STRUCT:
1157 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
1158 break;
1159 case BASE_TYPE_STRING: {
1160 ECHECK(ParseString(val, field->shared));
1161 break;
1162 }
1163 case BASE_TYPE_VECTOR: {
1164 uoffset_t off;
1165 ECHECK(ParseVector(val.type.VectorType(), &off, field, parent_fieldn));
1166 val.constant = NumToString(off);
1167 break;
1168 }
1169 case BASE_TYPE_ARRAY: {
1170 ECHECK(ParseArray(val));
1171 break;
1172 }
1173 case BASE_TYPE_INT:
1174 case BASE_TYPE_UINT:
1175 case BASE_TYPE_LONG:
1176 case BASE_TYPE_ULONG: {
1177 if (field && field->attributes.Lookup("hash") &&
1178 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
1179 ECHECK(ParseHash(val, field));
1180 } else {
1181 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false));
1182 }
1183 break;
1184 }
1185 default:
1186 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false));
1187 break;
1188 }
1189 return NoError();
1190}
1191
1192void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
1193 SerializeStruct(builder_, struct_def, val);
1194}
1195
1196void Parser::SerializeStruct(FlatBufferBuilder &builder,
1197 const StructDef &struct_def, const Value &val) {
1198 FLATBUFFERS_ASSERT(val.constant.length() == struct_def.bytesize);
1199 builder.Align(struct_def.minalign);
1200 builder.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()),
1201 struct_def.bytesize);
1202 builder.AddStructOffset(val.offset, builder.GetSize());
1203}
1204
1205template<typename F>
1206CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
1207 const StructDef *struct_def, F body) {
1208 // We allow tables both as JSON object{ .. } with field names
1209 // or vector[..] with all fields in order
1210 char terminator = '}';
1211 bool is_nested_vector = struct_def && Is('[');
1212 if (is_nested_vector) {
1213 NEXT();
1214 terminator = ']';
1215 } else {
1216 EXPECT('{');
1217 }
1218 for (;;) {
1219 if ((!opts.strict_json || !fieldn) && Is(terminator)) break;
1220 std::string name;
1221 if (is_nested_vector) {
1222 if (fieldn >= struct_def->fields.vec.size()) {
1223 return Error("too many unnamed fields in nested array");
1224 }
1225 name = struct_def->fields.vec[fieldn]->name;
1226 } else {
1227 name = attribute_;
1228 if (Is(kTokenStringConstant)) {
1229 NEXT();
1230 } else {
1231 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
1232 }
1233 if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':');
1234 }
1235 ECHECK(body(name, fieldn, struct_def));
1236 if (Is(terminator)) break;
1237 ECHECK(ParseComma());
1238 }
1239 NEXT();
1240 if (is_nested_vector && fieldn != struct_def->fields.vec.size()) {
1241 return Error("wrong number of unnamed fields in table vector");
1242 }
1243 return NoError();
1244}
1245
1246CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
1247 uoffset_t *ovalue) {
1248 ParseDepthGuard depth_guard(this);
1249 ECHECK(depth_guard.Check());
1250
1251 size_t fieldn_outer = 0;
1252 auto err = ParseTableDelimiters(
1253 fieldn_outer, &struct_def,
1254 [&](const std::string &name, size_t &fieldn,
1255 const StructDef *struct_def_inner) -> CheckedError {
1256 if (name == "$schema") {
1257 ECHECK(Expect(kTokenStringConstant));
1258 return NoError();
1259 }
1260 auto field = struct_def_inner->fields.Lookup(name);
1261 if (!field) {
1262 if (!opts.skip_unexpected_fields_in_json) {
1263 return Error("unknown field: " + name);
1264 } else {
1265 ECHECK(SkipAnyJsonValue());
1266 }
1267 } else {
1268 if (IsIdent("null") && !IsScalar(field->value.type.base_type)) {
1269 ECHECK(Next()); // Ignore this field.
1270 } else {
1271 Value val = field->value;
1272 if (field->flexbuffer) {
1273 flexbuffers::Builder builder(1024,
1274 flexbuffers::BUILDER_FLAG_SHARE_ALL);
1275 ECHECK(ParseFlexBufferValue(&builder));
1276 builder.Finish();
1277 // Force alignment for nested flexbuffer
1278 builder_.ForceVectorAlignment(builder.GetSize(), sizeof(uint8_t),
1279 sizeof(largest_scalar_t));
1280 auto off = builder_.CreateVector(builder.GetBuffer());
1281 val.constant = NumToString(off.o);
1282 } else if (field->nested_flatbuffer) {
1283 ECHECK(
1284 ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner));
1285 } else {
1286 ECHECK(ParseAnyValue(val, field, fieldn, struct_def_inner, 0));
1287 }
1288 // Hardcoded insertion-sort with error-check.
1289 // If fields are specified in order, then this loop exits
1290 // immediately.
1291 auto elem = field_stack_.rbegin();
1292 for (; elem != field_stack_.rbegin() + fieldn; ++elem) {
1293 auto existing_field = elem->second;
1294 if (existing_field == field)
1295 return Error("field set more than once: " + field->name);
1296 if (existing_field->value.offset < field->value.offset) break;
1297 }
1298 // Note: elem points to before the insertion point, thus .base()
1299 // points to the correct spot.
1300 field_stack_.insert(elem.base(), std::make_pair(val, field));
1301 fieldn++;
1302 }
1303 }
1304 return NoError();
1305 });
1306 ECHECK(err);
1307
1308 // Check if all required fields are parsed.
1309 for (auto field_it = struct_def.fields.vec.begin();
1310 field_it != struct_def.fields.vec.end(); ++field_it) {
1311 auto required_field = *field_it;
1312 if (!required_field->IsRequired()) { continue; }
1313 bool found = false;
1314 for (auto pf_it = field_stack_.end() - fieldn_outer;
1315 pf_it != field_stack_.end(); ++pf_it) {
1316 auto parsed_field = pf_it->second;
1317 if (parsed_field == required_field) {
1318 found = true;
1319 break;
1320 }
1321 }
1322 if (!found) {
1323 return Error("required field is missing: " + required_field->name +
1324 " in " + struct_def.name);
1325 }
1326 }
1327
1328 if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size())
1329 return Error("struct: wrong number of initializers: " + struct_def.name);
1330
1331 auto start = struct_def.fixed ? builder_.StartStruct(struct_def.minalign)
1332 : builder_.StartTable();
1333
1334 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size;
1335 size /= 2) {
1336 // Go through elements in reverse, since we're building the data backwards.
1337 for (auto it = field_stack_.rbegin();
1338 it != field_stack_.rbegin() + fieldn_outer; ++it) {
1339 auto &field_value = it->first;
1340 auto field = it->second;
1341 if (!struct_def.sortbysize ||
1342 size == SizeOf(field_value.type.base_type)) {
1343 switch (field_value.type.base_type) {
1344 // clang-format off
1345 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1346 case BASE_TYPE_ ## ENUM: \
1347 builder_.Pad(field->padding); \
1348 if (struct_def.fixed) { \
1349 CTYPE val; \
1350 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1351 builder_.PushElement(val); \
1352 } else { \
1353 CTYPE val, valdef; \
1354 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1355 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
1356 builder_.AddElement(field_value.offset, val, valdef); \
1357 } \
1358 break;
1359 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
1360 #undef FLATBUFFERS_TD
1361 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1362 case BASE_TYPE_ ## ENUM: \
1363 builder_.Pad(field->padding); \
1364 if (IsStruct(field->value.type)) { \
1365 SerializeStruct(*field->value.type.struct_def, field_value); \
1366 } else { \
1367 CTYPE val; \
1368 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1369 builder_.AddOffset(field_value.offset, val); \
1370 } \
1371 break;
1372 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD)
1373 #undef FLATBUFFERS_TD
1374 case BASE_TYPE_ARRAY:
1375 builder_.Pad(field->padding);
1376 builder_.PushBytes(
1377 reinterpret_cast<const uint8_t*>(field_value.constant.c_str()),
1378 InlineSize(field_value.type));
1379 break;
1380 // clang-format on
1381 }
1382 }
1383 }
1384 }
1385 for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back();
1386
1387 if (struct_def.fixed) {
1388 builder_.ClearOffsets();
1389 builder_.EndStruct();
1390 FLATBUFFERS_ASSERT(value);
1391 // Temporarily store this struct in the value string, since it is to
1392 // be serialized in-place elsewhere.
1393 value->assign(
1394 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()),
1395 struct_def.bytesize);
1396 builder_.PopBytes(struct_def.bytesize);
1397 FLATBUFFERS_ASSERT(!ovalue);
1398 } else {
1399 auto val = builder_.EndTable(start);
1400 if (ovalue) *ovalue = val;
1401 if (value) *value = NumToString(val);
1402 }
1403 return NoError();
1404}
1405
1406template<typename F>
1407CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) {
1408 EXPECT('[');
1409 for (;;) {
1410 if ((!opts.strict_json || !count) && Is(']')) break;
1411 ECHECK(body(count));
1412 count++;
1413 if (Is(']')) break;
1414 ECHECK(ParseComma());
1415 }
1416 NEXT();
1417 return NoError();
1418}
1419
1420static bool CompareSerializedScalars(const uint8_t *a, const uint8_t *b,
1421 const FieldDef &key) {
1422 switch (key.value.type.base_type) {
1423#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1424 case BASE_TYPE_##ENUM: { \
1425 CTYPE def = static_cast<CTYPE>(0); \
1426 if (!a || !b) { StringToNumber(key.value.constant.c_str(), &def); } \
1427 const auto av = a ? ReadScalar<CTYPE>(a) : def; \
1428 const auto bv = b ? ReadScalar<CTYPE>(b) : def; \
1429 return av < bv; \
1430 }
1431 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
1432#undef FLATBUFFERS_TD
1433 default: {
1434 FLATBUFFERS_ASSERT(false && "scalar type expected");
1435 return false;
1436 }
1437 }
1438}
1439
1440static bool CompareTablesByScalarKey(const Offset<Table> *_a,
1441 const Offset<Table> *_b,
1442 const FieldDef &key) {
1443 const voffset_t offset = key.value.offset;
1444 // Indirect offset pointer to table pointer.
1445 auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
1446 auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
1447 // Fetch field address from table.
1448 a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
1449 b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
1450 return CompareSerializedScalars(a, b, key);
1451}
1452
1453static bool CompareTablesByStringKey(const Offset<Table> *_a,
1454 const Offset<Table> *_b,
1455 const FieldDef &key) {
1456 const voffset_t offset = key.value.offset;
1457 // Indirect offset pointer to table pointer.
1458 auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
1459 auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
1460 // Fetch field address from table.
1461 a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
1462 b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
1463 if (a && b) {
1464 // Indirect offset pointer to string pointer.
1465 a += ReadScalar<uoffset_t>(a);
1466 b += ReadScalar<uoffset_t>(b);
1467 return *reinterpret_cast<const String *>(a) <
1468 *reinterpret_cast<const String *>(b);
1469 } else {
1470 return a ? true : false;
1471 }
1472}
1473
1474static void SwapSerializedTables(Offset<Table> *a, Offset<Table> *b) {
1475 // These are serialized offsets, so are relative where they are
1476 // stored in memory, so compute the distance between these pointers:
1477 ptrdiff_t diff = (b - a) * sizeof(Offset<Table>);
1478 FLATBUFFERS_ASSERT(diff >= 0); // Guaranteed by SimpleQsort.
1479 auto udiff = static_cast<uoffset_t>(diff);
1480 a->o = EndianScalar(ReadScalar<uoffset_t>(a) - udiff);
1481 b->o = EndianScalar(ReadScalar<uoffset_t>(b) + udiff);
1482 std::swap(*a, *b);
1483}
1484
1485// See below for why we need our own sort :(
1486template<typename T, typename F, typename S>
1487void SimpleQsort(T *begin, T *end, size_t width, F comparator, S swapper) {
1488 if (end - begin <= static_cast<ptrdiff_t>(width)) return;
1489 auto l = begin + width;
1490 auto r = end;
1491 while (l < r) {
1492 if (comparator(begin, l)) {
1493 r -= width;
1494 swapper(l, r);
1495 } else {
1496 l += width;
1497 }
1498 }
1499 l -= width;
1500 swapper(begin, l);
1501 SimpleQsort(begin, l, width, comparator, swapper);
1502 SimpleQsort(r, end, width, comparator, swapper);
1503}
1504
1505CheckedError Parser::ParseAlignAttribute(const std::string &align_constant,
1506 size_t min_align, size_t *align) {
1507 // Use uint8_t to avoid problems with size_t==`unsigned long` on LP64.
1508 uint8_t align_value;
1509 if (StringToNumber(align_constant.c_str(), &align_value) &&
1510 VerifyAlignmentRequirements(static_cast<size_t>(align_value),
1511 min_align)) {
1512 *align = align_value;
1513 return NoError();
1514 }
1515 return Error("unexpected force_align value '" + align_constant +
1516 "', alignment must be a power of two integer ranging from the "
1517 "type\'s natural alignment " +
1518 NumToString(min_align) + " to " +
1519 NumToString(FLATBUFFERS_MAX_ALIGNMENT));
1520}
1521
1522CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
1523 FieldDef *field, size_t fieldn) {
1524 uoffset_t count = 0;
1525 auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
1526 Value val;
1527 val.type = type;
1528 ECHECK(ParseAnyValue(val, field, fieldn, nullptr, count, true));
1529 field_stack_.push_back(std::make_pair(val, nullptr));
1530 return NoError();
1531 });
1532 ECHECK(err);
1533
1534 const size_t len = count * InlineSize(type) / InlineAlignment(type);
1535 const size_t elemsize = InlineAlignment(type);
1536 const auto force_align = field->attributes.Lookup("force_align");
1537 if (force_align) {
1538 size_t align;
1539 ECHECK(ParseAlignAttribute(force_align->constant, 1, &align));
1540 if (align > 1) { builder_.ForceVectorAlignment(len, elemsize, align); }
1541 }
1542
1543 builder_.StartVector(len, elemsize);
1544 for (uoffset_t i = 0; i < count; i++) {
1545 // start at the back, since we're building the data backwards.
1546 auto &val = field_stack_.back().first;
1547 switch (val.type.base_type) {
1548 // clang-format off
1549 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE,...) \
1550 case BASE_TYPE_ ## ENUM: \
1551 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
1552 else { \
1553 CTYPE elem; \
1554 ECHECK(atot(val.constant.c_str(), *this, &elem)); \
1555 builder_.PushElement(elem); \
1556 } \
1557 break;
1558 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1559 #undef FLATBUFFERS_TD
1560 // clang-format on
1561 }
1562 field_stack_.pop_back();
1563 }
1564
1565 builder_.ClearOffsets();
1566 *ovalue = builder_.EndVector(count);
1567
1568 if (type.base_type == BASE_TYPE_STRUCT && type.struct_def->has_key) {
1569 // We should sort this vector. Find the key first.
1570 const FieldDef *key = nullptr;
1571 for (auto it = type.struct_def->fields.vec.begin();
1572 it != type.struct_def->fields.vec.end(); ++it) {
1573 if ((*it)->key) {
1574 key = (*it);
1575 break;
1576 }
1577 }
1578 FLATBUFFERS_ASSERT(key);
1579 // Now sort it.
1580 // We can't use std::sort because for structs the size is not known at
1581 // compile time, and for tables our iterators dereference offsets, so can't
1582 // be used to swap elements.
1583 // And we can't use C qsort either, since that would force use to use
1584 // globals, making parsing thread-unsafe.
1585 // So for now, we use SimpleQsort above.
1586 // TODO: replace with something better, preferably not recursive.
1587
1588 if (type.struct_def->fixed) {
1589 const voffset_t offset = key->value.offset;
1590 const size_t struct_size = type.struct_def->bytesize;
1591 auto v =
1592 reinterpret_cast<VectorOfAny *>(builder_.GetCurrentBufferPointer());
1593 SimpleQsort<uint8_t>(
1594 v->Data(), v->Data() + v->size() * type.struct_def->bytesize,
1595 type.struct_def->bytesize,
1596 [offset, key](const uint8_t *a, const uint8_t *b) -> bool {
1597 return CompareSerializedScalars(a + offset, b + offset, *key);
1598 },
1599 [struct_size](uint8_t *a, uint8_t *b) {
1600 // FIXME: faster?
1601 for (size_t i = 0; i < struct_size; i++) { std::swap(a[i], b[i]); }
1602 });
1603 } else {
1604 auto v = reinterpret_cast<Vector<Offset<Table>> *>(
1605 builder_.GetCurrentBufferPointer());
1606 // Here also can't use std::sort. We do have an iterator type for it,
1607 // but it is non-standard as it will dereference the offsets, and thus
1608 // can't be used to swap elements.
1609 if (key->value.type.base_type == BASE_TYPE_STRING) {
1610 SimpleQsort<Offset<Table>>(
1611 v->data(), v->data() + v->size(), 1,
1612 [key](const Offset<Table> *_a, const Offset<Table> *_b) -> bool {
1613 return CompareTablesByStringKey(_a, _b, *key);
1614 },
1615 SwapSerializedTables);
1616 } else {
1617 SimpleQsort<Offset<Table>>(
1618 v->data(), v->data() + v->size(), 1,
1619 [key](const Offset<Table> *_a, const Offset<Table> *_b) -> bool {
1620 return CompareTablesByScalarKey(_a, _b, *key);
1621 },
1622 SwapSerializedTables);
1623 }
1624 }
1625 }
1626 return NoError();
1627}
1628
1629CheckedError Parser::ParseArray(Value &array) {
1630 std::vector<Value> stack;
1631 FlatBufferBuilder builder;
1632 const auto &type = array.type.VectorType();
1633 auto length = array.type.fixed_length;
1634 uoffset_t count = 0;
1635 auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
1636 stack.emplace_back(Value());
1637 auto &val = stack.back();
1638 val.type = type;
1639 if (IsStruct(type)) {
1640 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
1641 } else {
1642 ECHECK(ParseSingleValue(nullptr, val, false));
1643 }
1644 return NoError();
1645 });
1646 ECHECK(err);
1647 if (length != count) return Error("Fixed-length array size is incorrect.");
1648
1649 for (auto it = stack.rbegin(); it != stack.rend(); ++it) {
1650 auto &val = *it;
1651 // clang-format off
1652 switch (val.type.base_type) {
1653 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1654 case BASE_TYPE_ ## ENUM: \
1655 if (IsStruct(val.type)) { \
1656 SerializeStruct(builder, *val.type.struct_def, val); \
1657 } else { \
1658 CTYPE elem; \
1659 ECHECK(atot(val.constant.c_str(), *this, &elem)); \
1660 builder.PushElement(elem); \
1661 } \
1662 break;
1663 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1664 #undef FLATBUFFERS_TD
1665 default: FLATBUFFERS_ASSERT(0);
1666 }
1667 // clang-format on
1668 }
1669
1670 array.constant.assign(
1671 reinterpret_cast<const char *>(builder.GetCurrentBufferPointer()),
1672 InlineSize(array.type));
1673 return NoError();
1674}
1675
1676CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
1677 size_t fieldn,
1678 const StructDef *parent_struct_def) {
1679 if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers
1680 if (opts.json_nested_legacy_flatbuffers) {
1681 ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0));
1682 } else {
1683 return Error(
1684 "cannot parse nested_flatbuffer as bytes unless"
1685 " --json-nested-bytes is set");
1686 }
1687 } else {
1688 auto cursor_at_value_begin = cursor_;
1689 ECHECK(SkipAnyJsonValue());
1690 std::string substring(cursor_at_value_begin - 1, cursor_ - 1);
1691
1692 // Create and initialize new parser
1693 Parser nested_parser;
1694 FLATBUFFERS_ASSERT(field->nested_flatbuffer);
1695 nested_parser.root_struct_def_ = field->nested_flatbuffer;
1696 nested_parser.enums_ = enums_;
1697 nested_parser.opts = opts;
1698 nested_parser.uses_flexbuffers_ = uses_flexbuffers_;
1699 nested_parser.parse_depth_counter_ = parse_depth_counter_;
1700 // Parse JSON substring into new flatbuffer builder using nested_parser
1701 bool ok = nested_parser.Parse(substring.c_str(), nullptr, nullptr);
1702
1703 // Clean nested_parser to avoid deleting the elements in
1704 // the SymbolTables on destruction
1705 nested_parser.enums_.dict.clear();
1706 nested_parser.enums_.vec.clear();
1707
1708 if (!ok) { ECHECK(Error(nested_parser.error_)); }
1709 // Force alignment for nested flatbuffer
1710 builder_.ForceVectorAlignment(
1711 nested_parser.builder_.GetSize(), sizeof(uint8_t),
1712 nested_parser.builder_.GetBufferMinAlignment());
1713
1714 auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(),
1715 nested_parser.builder_.GetSize());
1716 val.constant = NumToString(off.o);
1717 }
1718 return NoError();
1719}
1720
1721CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) {
1722 if (Is('(')) {
1723 NEXT();
1724 for (;;) {
1725 auto name = attribute_;
1726 if (false == (Is(kTokenIdentifier) || Is(kTokenStringConstant)))
1727 return Error("attribute name must be either identifier or string: " +
1728 name);
1729 if (known_attributes_.find(name) == known_attributes_.end())
1730 return Error("user define attributes must be declared before use: " +
1731 name);
1732 NEXT();
1733 auto e = new Value();
1734 if (attributes->Add(name, e)) Warning("attribute already found: " + name);
1735 if (Is(':')) {
1736 NEXT();
1737 ECHECK(ParseSingleValue(&name, *e, true));
1738 }
1739 if (Is(')')) {
1740 NEXT();
1741 break;
1742 }
1743 EXPECT(',');
1744 }
1745 }
1746 return NoError();
1747}
1748
1749CheckedError Parser::ParseEnumFromString(const Type &type,
1750 std::string *result) {
1751 const auto base_type =
1752 type.enum_def ? type.enum_def->underlying_type.base_type : type.base_type;
1753 if (!IsInteger(base_type)) return Error("not a valid value for this field");
1754 uint64_t u64 = 0;
1755 for (size_t pos = 0; pos != std::string::npos;) {
1756 const auto delim = attribute_.find_first_of(' ', pos);
1757 const auto last = (std::string::npos == delim);
1758 auto word = attribute_.substr(pos, !last ? delim - pos : std::string::npos);
1759 pos = !last ? delim + 1 : std::string::npos;
1760 const EnumVal *ev = nullptr;
1761 if (type.enum_def) {
1762 ev = type.enum_def->Lookup(word);
1763 } else {
1764 auto dot = word.find_first_of('.');
1765 if (std::string::npos == dot)
1766 return Error("enum values need to be qualified by an enum type");
1767 auto enum_def_str = word.substr(0, dot);
1768 const auto enum_def = LookupEnum(enum_def_str);
1769 if (!enum_def) return Error("unknown enum: " + enum_def_str);
1770 auto enum_val_str = word.substr(dot + 1);
1771 ev = enum_def->Lookup(enum_val_str);
1772 }
1773 if (!ev) return Error("unknown enum value: " + word);
1774 u64 |= ev->GetAsUInt64();
1775 }
1776 *result = IsUnsigned(base_type) ? NumToString(u64)
1777 : NumToString(static_cast<int64_t>(u64));
1778 return NoError();
1779}
1780
1781CheckedError Parser::ParseHash(Value &e, FieldDef *field) {
1782 FLATBUFFERS_ASSERT(field);
1783 Value *hash_name = field->attributes.Lookup("hash");
1784 switch (e.type.base_type) {
1785 case BASE_TYPE_SHORT: {
1786 auto hash = FindHashFunction16(hash_name->constant.c_str());
1787 int16_t hashed_value = static_cast<int16_t>(hash(attribute_.c_str()));
1788 e.constant = NumToString(hashed_value);
1789 break;
1790 }
1791 case BASE_TYPE_USHORT: {
1792 auto hash = FindHashFunction16(hash_name->constant.c_str());
1793 uint16_t hashed_value = hash(attribute_.c_str());
1794 e.constant = NumToString(hashed_value);
1795 break;
1796 }
1797 case BASE_TYPE_INT: {
1798 auto hash = FindHashFunction32(hash_name->constant.c_str());
1799 int32_t hashed_value = static_cast<int32_t>(hash(attribute_.c_str()));
1800 e.constant = NumToString(hashed_value);
1801 break;
1802 }
1803 case BASE_TYPE_UINT: {
1804 auto hash = FindHashFunction32(hash_name->constant.c_str());
1805 uint32_t hashed_value = hash(attribute_.c_str());
1806 e.constant = NumToString(hashed_value);
1807 break;
1808 }
1809 case BASE_TYPE_LONG: {
1810 auto hash = FindHashFunction64(hash_name->constant.c_str());
1811 int64_t hashed_value = static_cast<int64_t>(hash(attribute_.c_str()));
1812 e.constant = NumToString(hashed_value);
1813 break;
1814 }
1815 case BASE_TYPE_ULONG: {
1816 auto hash = FindHashFunction64(hash_name->constant.c_str());
1817 uint64_t hashed_value = hash(attribute_.c_str());
1818 e.constant = NumToString(hashed_value);
1819 break;
1820 }
1821 default: FLATBUFFERS_ASSERT(0);
1822 }
1823 NEXT();
1824 return NoError();
1825}
1826
1827CheckedError Parser::TokenError() {
1828 return Error("cannot parse value starting with: " + TokenToStringId(token_));
1829}
1830
1831// Re-pack helper (ParseSingleValue) to normalize defaults of scalars.
1832template<typename T> inline void SingleValueRepack(Value &e, T val) {
1833 // Remove leading zeros.
1834 if (IsInteger(e.type.base_type)) { e.constant = NumToString(val); }
1835}
1836#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
1837// Normalize defaults NaN to unsigned quiet-NaN(0) if value was parsed from
1838// hex-float literal.
1839static inline void SingleValueRepack(Value &e, float val) {
1840 if (val != val) e.constant = "nan";
1841}
1842static inline void SingleValueRepack(Value &e, double val) {
1843 if (val != val) e.constant = "nan";
1844}
1845#endif
1846
1847CheckedError Parser::ParseFunction(const std::string *name, Value &e) {
1848 ParseDepthGuard depth_guard(this);
1849 ECHECK(depth_guard.Check());
1850
1851 // Copy name, attribute will be changed on NEXT().
1852 const auto functionname = attribute_;
1853 if (!IsFloat(e.type.base_type)) {
1854 return Error(functionname + ": type of argument mismatch, expecting: " +
1855 kTypeNames[BASE_TYPE_DOUBLE] +
1856 ", found: " + kTypeNames[e.type.base_type] +
1857 ", name: " + (name ? *name : "") + ", value: " + e.constant);
1858 }
1859 NEXT();
1860 EXPECT('(');
1861 ECHECK(ParseSingleValue(name, e, false));
1862 EXPECT(')');
1863 // calculate with double precision
1864 double x, y = 0.0;
1865 ECHECK(atot(e.constant.c_str(), *this, &x));
1866 // clang-format off
1867 auto func_match = false;
1868 #define FLATBUFFERS_FN_DOUBLE(name, op) \
1869 if (!func_match && functionname == name) { y = op; func_match = true; }
1870 FLATBUFFERS_FN_DOUBLE("deg", x / kPi * 180);
1871 FLATBUFFERS_FN_DOUBLE("rad", x * kPi / 180);
1872 FLATBUFFERS_FN_DOUBLE("sin", sin(x));
1873 FLATBUFFERS_FN_DOUBLE("cos", cos(x));
1874 FLATBUFFERS_FN_DOUBLE("tan", tan(x));
1875 FLATBUFFERS_FN_DOUBLE("asin", asin(x));
1876 FLATBUFFERS_FN_DOUBLE("acos", acos(x));
1877 FLATBUFFERS_FN_DOUBLE("atan", atan(x));
1878 // TODO(wvo): add more useful conversion functions here.
1879 #undef FLATBUFFERS_FN_DOUBLE
1880 // clang-format on
1881 if (true != func_match) {
1882 return Error(std::string("Unknown conversion function: ") + functionname +
1883 ", field name: " + (name ? *name : "") +
1884 ", value: " + e.constant);
1885 }
1886 e.constant = NumToString(y);
1887 return NoError();
1888}
1889
1890CheckedError Parser::TryTypedValue(const std::string *name, int dtoken,
1891 bool check, Value &e, BaseType req,
1892 bool *destmatch) {
1893 FLATBUFFERS_ASSERT(*destmatch == false && dtoken == token_);
1894 *destmatch = true;
1895 e.constant = attribute_;
1896 // Check token match
1897 if (!check) {
1898 if (e.type.base_type == BASE_TYPE_NONE) {
1899 e.type.base_type = req;
1900 } else {
1901 return Error(std::string("type mismatch: expecting: ") +
1902 kTypeNames[e.type.base_type] +
1903 ", found: " + kTypeNames[req] +
1904 ", name: " + (name ? *name : "") + ", value: " + e.constant);
1905 }
1906 }
1907 // The exponent suffix of hexadecimal float-point number is mandatory.
1908 // A hex-integer constant is forbidden as an initializer of float number.
1909 if ((kTokenFloatConstant != dtoken) && IsFloat(e.type.base_type)) {
1910 const auto &s = e.constant;
1911 const auto k = s.find_first_of("0123456789.");
1912 if ((std::string::npos != k) && (s.length() > (k + 1)) &&
1913 (s[k] == '0' && is_alpha_char(s[k + 1], 'X')) &&
1914 (std::string::npos == s.find_first_of("pP", k + 2))) {
1915 return Error(
1916 "invalid number, the exponent suffix of hexadecimal "
1917 "floating-point literals is mandatory: \"" +
1918 s + "\"");
1919 }
1920 }
1921 NEXT();
1922 return NoError();
1923}
1924
1925CheckedError Parser::ParseSingleValue(const std::string *name, Value &e,
1926 bool check_now) {
1927 if (token_ == '+' || token_ == '-') {
1928 const char sign = static_cast<char>(token_);
1929 // Get an indentifier: NAN, INF, or function name like cos/sin/deg.
1930 NEXT();
1931 if (token_ != kTokenIdentifier) return Error("constant name expected");
1932 attribute_.insert(0, 1, sign);
1933 }
1934
1935 const auto in_type = e.type.base_type;
1936 const auto is_tok_ident = (token_ == kTokenIdentifier);
1937 const auto is_tok_string = (token_ == kTokenStringConstant);
1938
1939 // First see if this could be a conversion function.
1940 if (is_tok_ident && *cursor_ == '(') { return ParseFunction(name, e); }
1941
1942 // clang-format off
1943 auto match = false;
1944
1945 #define IF_ECHECK_(force, dtoken, check, req) \
1946 if (!match && ((dtoken) == token_) && ((check) || IsConstTrue(force))) \
1947 ECHECK(TryTypedValue(name, dtoken, check, e, req, &match))
1948 #define TRY_ECHECK(dtoken, check, req) IF_ECHECK_(false, dtoken, check, req)
1949 #define FORCE_ECHECK(dtoken, check, req) IF_ECHECK_(true, dtoken, check, req)
1950 // clang-format on
1951
1952 if (is_tok_ident || is_tok_string) {
1953 const auto kTokenStringOrIdent = token_;
1954 // The string type is a most probable type, check it first.
1955 TRY_ECHECK(kTokenStringConstant, in_type == BASE_TYPE_STRING,
1956 BASE_TYPE_STRING);
1957
1958 // avoid escaped and non-ascii in the string
1959 if (!match && is_tok_string && IsScalar(in_type) &&
1960 !attr_is_trivial_ascii_string_) {
1961 return Error(
1962 std::string("type mismatch or invalid value, an initializer of "
1963 "non-string field must be trivial ASCII string: type: ") +
1964 kTypeNames[in_type] + ", name: " + (name ? *name : "") +
1965 ", value: " + attribute_);
1966 }
1967
1968 // A boolean as true/false. Boolean as Integer check below.
1969 if (!match && IsBool(in_type)) {
1970 auto is_true = attribute_ == "true";
1971 if (is_true || attribute_ == "false") {
1972 attribute_ = is_true ? "1" : "0";
1973 // accepts both kTokenStringConstant and kTokenIdentifier
1974 TRY_ECHECK(kTokenStringOrIdent, IsBool(in_type), BASE_TYPE_BOOL);
1975 }
1976 }
1977 // Check for optional scalars.
1978 if (!match && IsScalar(in_type) && attribute_ == "null") {
1979 e.constant = "null";
1980 NEXT();
1981 match = true;
1982 }
1983 // Check if this could be a string/identifier enum value.
1984 // Enum can have only true integer base type.
1985 if (!match && IsInteger(in_type) && !IsBool(in_type) &&
1986 IsIdentifierStart(*attribute_.c_str())) {
1987 ECHECK(ParseEnumFromString(e.type, &e.constant));
1988 NEXT();
1989 match = true;
1990 }
1991 // Parse a float/integer number from the string.
1992 // A "scalar-in-string" value needs extra checks.
1993 if (!match && is_tok_string && IsScalar(in_type)) {
1994 // Strip trailing whitespaces from attribute_.
1995 auto last_non_ws = attribute_.find_last_not_of(' ');
1996 if (std::string::npos != last_non_ws) attribute_.resize(last_non_ws + 1);
1997 if (IsFloat(e.type.base_type)) {
1998 // The functions strtod() and strtof() accept both 'nan' and
1999 // 'nan(number)' literals. While 'nan(number)' is rejected by the parser
2000 // as an unsupported function if is_tok_ident is true.
2001 if (attribute_.find_last_of(')') != std::string::npos) {
2002 return Error("invalid number: " + attribute_);
2003 }
2004 }
2005 }
2006 // Float numbers or nan, inf, pi, etc.
2007 TRY_ECHECK(kTokenStringOrIdent, IsFloat(in_type), BASE_TYPE_FLOAT);
2008 // An integer constant in string.
2009 TRY_ECHECK(kTokenStringOrIdent, IsInteger(in_type), BASE_TYPE_INT);
2010 // Unknown tokens will be interpreted as string type.
2011 // An attribute value may be a scalar or string constant.
2012 FORCE_ECHECK(kTokenStringConstant, in_type == BASE_TYPE_STRING,
2013 BASE_TYPE_STRING);
2014 } else {
2015 // Try a float number.
2016 TRY_ECHECK(kTokenFloatConstant, IsFloat(in_type), BASE_TYPE_FLOAT);
2017 // Integer token can init any scalar (integer of float).
2018 FORCE_ECHECK(kTokenIntegerConstant, IsScalar(in_type), BASE_TYPE_INT);
2019 }
2020 // Match empty vectors for default-empty-vectors.
2021 if (!match && IsVector(e.type) && token_ == '[') {
2022 NEXT();
2023 if (token_ != ']') { return Error("Expected `]` in vector default"); }
2024 NEXT();
2025 match = true;
2026 e.constant = "[]";
2027 }
2028
2029#undef FORCE_ECHECK
2030#undef TRY_ECHECK
2031#undef IF_ECHECK_
2032
2033 if (!match) {
2034 std::string msg;
2035 msg += "Cannot assign token starting with '" + TokenToStringId(token_) +
2036 "' to value of <" + std::string(kTypeNames[in_type]) + "> type.";
2037 return Error(msg);
2038 }
2039 const auto match_type = e.type.base_type; // may differ from in_type
2040 // The check_now flag must be true when parse a fbs-schema.
2041 // This flag forces to check default scalar values or metadata of field.
2042 // For JSON parser the flag should be false.
2043 // If it is set for JSON each value will be checked twice (see ParseTable).
2044 // Special case 'null' since atot can't handle that.
2045 if (check_now && IsScalar(match_type) && e.constant != "null") {
2046 // clang-format off
2047 switch (match_type) {
2048 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
2049 case BASE_TYPE_ ## ENUM: {\
2050 CTYPE val; \
2051 ECHECK(atot(e.constant.c_str(), *this, &val)); \
2052 SingleValueRepack(e, val); \
2053 break; }
2054 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
2055 #undef FLATBUFFERS_TD
2056 default: break;
2057 }
2058 // clang-format on
2059 }
2060 return NoError();
2061}
2062
2063StructDef *Parser::LookupCreateStruct(const std::string &name,
2064 bool create_if_new, bool definition) {
2065 std::string qualified_name = current_namespace_->GetFullyQualifiedName(name);
2066 // See if it exists pre-declared by an unqualified use.
2067 auto struct_def = LookupStruct(name);
2068 if (struct_def && struct_def->predecl) {
2069 if (definition) {
2070 // Make sure it has the current namespace, and is registered under its
2071 // qualified name.
2072 struct_def->defined_namespace = current_namespace_;
2073 structs_.Move(name, qualified_name);
2074 }
2075 return struct_def;
2076 }
2077 // See if it exists pre-declared by an qualified use.
2078 struct_def = LookupStruct(qualified_name);
2079 if (struct_def && struct_def->predecl) {
2080 if (definition) {
2081 // Make sure it has the current namespace.
2082 struct_def->defined_namespace = current_namespace_;
2083 }
2084 return struct_def;
2085 }
2086 if (!definition && !struct_def) {
2087 struct_def = LookupStructThruParentNamespaces(name);
2088 }
2089 if (!struct_def && create_if_new) {
2090 struct_def = new StructDef();
2091 if (definition) {
2092 structs_.Add(qualified_name, struct_def);
2093 struct_def->name = name;
2094 struct_def->defined_namespace = current_namespace_;
2095 } else {
2096 // Not a definition.
2097 // Rather than failing, we create a "pre declared" StructDef, due to
2098 // circular references, and check for errors at the end of parsing.
2099 // It is defined in the current namespace, as the best guess what the
2100 // final namespace will be.
2101 structs_.Add(name, struct_def);
2102 struct_def->name = name;
2103 struct_def->defined_namespace = current_namespace_;
2104 struct_def->original_location.reset(
2105 new std::string(file_being_parsed_ + ":" + NumToString(line_)));
2106 }
2107 }
2108 return struct_def;
2109}
2110
2111const EnumVal *EnumDef::MinValue() const {
2112 return vals.vec.empty() ? nullptr : vals.vec.front();
2113}
2114const EnumVal *EnumDef::MaxValue() const {
2115 return vals.vec.empty() ? nullptr : vals.vec.back();
2116}
2117
2118template<typename T> static uint64_t EnumDistanceImpl(T e1, T e2) {
2119 if (e1 < e2) { std::swap(e1, e2); } // use std for scalars
2120 // Signed overflow may occur, use unsigned calculation.
2121 // The unsigned overflow is well-defined by C++ standard (modulo 2^n).
2122 return static_cast<uint64_t>(e1) - static_cast<uint64_t>(e2);
2123}
2124
2125uint64_t EnumDef::Distance(const EnumVal *v1, const EnumVal *v2) const {
2126 return IsUInt64() ? EnumDistanceImpl(v1->GetAsUInt64(), v2->GetAsUInt64())
2127 : EnumDistanceImpl(v1->GetAsInt64(), v2->GetAsInt64());
2128}
2129
2130std::string EnumDef::AllFlags() const {
2131 FLATBUFFERS_ASSERT(attributes.Lookup("bit_flags"));
2132 uint64_t u64 = 0;
2133 for (auto it = Vals().begin(); it != Vals().end(); ++it) {
2134 u64 |= (*it)->GetAsUInt64();
2135 }
2136 return IsUInt64() ? NumToString(u64) : NumToString(static_cast<int64_t>(u64));
2137}
2138
2139EnumVal *EnumDef::ReverseLookup(int64_t enum_idx,
2140 bool skip_union_default) const {
2141 auto skip_first = static_cast<int>(is_union && skip_union_default);
2142 for (auto it = Vals().begin() + skip_first; it != Vals().end(); ++it) {
2143 if ((*it)->GetAsInt64() == enum_idx) { return *it; }
2144 }
2145 return nullptr;
2146}
2147
2148EnumVal *EnumDef::FindByValue(const std::string &constant) const {
2149 int64_t i64;
2150 auto done = false;
2151 if (IsUInt64()) {
2152 uint64_t u64; // avoid reinterpret_cast of pointers
2153 done = StringToNumber(constant.c_str(), &u64);
2154 i64 = static_cast<int64_t>(u64);
2155 } else {
2156 done = StringToNumber(constant.c_str(), &i64);
2157 }
2158 FLATBUFFERS_ASSERT(done);
2159 if (!done) return nullptr;
2160 return ReverseLookup(i64, false);
2161}
2162
2163void EnumDef::SortByValue() {
2164 auto &v = vals.vec;
2165 if (IsUInt64())
2166 std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) {
2167 if (e1->GetAsUInt64() == e2->GetAsUInt64()) {
2168 return e1->name < e2->name;
2169 }
2170 return e1->GetAsUInt64() < e2->GetAsUInt64();
2171 });
2172 else
2173 std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) {
2174 if (e1->GetAsInt64() == e2->GetAsInt64()) { return e1->name < e2->name; }
2175 return e1->GetAsInt64() < e2->GetAsInt64();
2176 });
2177}
2178
2179void EnumDef::RemoveDuplicates() {
2180 // This method depends form SymbolTable implementation!
2181 // 1) vals.vec - owner (raw pointer)
2182 // 2) vals.dict - access map
2183 auto first = vals.vec.begin();
2184 auto last = vals.vec.end();
2185 if (first == last) return;
2186 auto result = first;
2187 while (++first != last) {
2188 if ((*result)->value != (*first)->value) {
2189 *(++result) = *first;
2190 } else {
2191 auto ev = *first;
2192 for (auto it = vals.dict.begin(); it != vals.dict.end(); ++it) {
2193 if (it->second == ev) it->second = *result; // reassign
2194 }
2195 delete ev; // delete enum value
2196 *first = nullptr;
2197 }
2198 }
2199 vals.vec.erase(++result, last);
2200}
2201
2202template<typename T> void EnumDef::ChangeEnumValue(EnumVal *ev, T new_value) {
2203 ev->value = static_cast<int64_t>(new_value);
2204}
2205
2206namespace EnumHelper {
2207template<BaseType E> struct EnumValType { typedef int64_t type; };
2208template<> struct EnumValType<BASE_TYPE_ULONG> { typedef uint64_t type; };
2209} // namespace EnumHelper
2210
2211struct EnumValBuilder {
2212 EnumVal *CreateEnumerator(const std::string &ev_name) {
2213 FLATBUFFERS_ASSERT(!temp);
2214 auto first = enum_def.vals.vec.empty();
2215 user_value = first;
2216 temp = new EnumVal(ev_name, first ? 0 : enum_def.vals.vec.back()->value);
2217 return temp;
2218 }
2219
2220 EnumVal *CreateEnumerator(const std::string &ev_name, int64_t val) {
2221 FLATBUFFERS_ASSERT(!temp);
2222 user_value = true;
2223 temp = new EnumVal(ev_name, val);
2224 return temp;
2225 }
2226
2227 FLATBUFFERS_CHECKED_ERROR AcceptEnumerator(const std::string &name) {
2228 FLATBUFFERS_ASSERT(temp);
2229 ECHECK(ValidateValue(&temp->value, false == user_value));
2230 FLATBUFFERS_ASSERT((temp->union_type.enum_def == nullptr) ||
2231 (temp->union_type.enum_def == &enum_def));
2232 auto not_unique = enum_def.vals.Add(name, temp);
2233 temp = nullptr;
2234 if (not_unique) return parser.Error("enum value already exists: " + name);
2235 return NoError();
2236 }
2237
2238 FLATBUFFERS_CHECKED_ERROR AcceptEnumerator() {
2239 return AcceptEnumerator(temp->name);
2240 }
2241
2242 FLATBUFFERS_CHECKED_ERROR AssignEnumeratorValue(const std::string &value) {
2243 user_value = true;
2244 auto fit = false;
2245 if (enum_def.IsUInt64()) {
2246 uint64_t u64;
2247 fit = StringToNumber(value.c_str(), &u64);
2248 temp->value = static_cast<int64_t>(u64); // well-defined since C++20.
2249 } else {
2250 int64_t i64;
2251 fit = StringToNumber(value.c_str(), &i64);
2252 temp->value = i64;
2253 }
2254 if (!fit) return parser.Error("enum value does not fit, \"" + value + "\"");
2255 return NoError();
2256 }
2257
2258 template<BaseType E, typename CTYPE>
2259 inline FLATBUFFERS_CHECKED_ERROR ValidateImpl(int64_t *ev, int m) {
2260 typedef typename EnumHelper::EnumValType<E>::type T; // int64_t or uint64_t
2261 static_assert(sizeof(T) == sizeof(int64_t), "invalid EnumValType");
2262 const auto v = static_cast<T>(*ev);
2263 auto up = static_cast<T>((flatbuffers::numeric_limits<CTYPE>::max)());
2264 auto dn = static_cast<T>((flatbuffers::numeric_limits<CTYPE>::lowest)());
2265 if (v < dn || v > (up - m)) {
2266 return parser.Error("enum value does not fit, \"" + NumToString(v) +
2267 (m ? " + 1\"" : "\"") + " out of " +
2268 TypeToIntervalString<CTYPE>());
2269 }
2270 *ev = static_cast<int64_t>(v + m); // well-defined since C++20.
2271 return NoError();
2272 }
2273
2274 FLATBUFFERS_CHECKED_ERROR ValidateValue(int64_t *ev, bool next) {
2275 // clang-format off
2276 switch (enum_def.underlying_type.base_type) {
2277 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
2278 case BASE_TYPE_##ENUM: { \
2279 if (!IsInteger(BASE_TYPE_##ENUM)) break; \
2280 return ValidateImpl<BASE_TYPE_##ENUM, CTYPE>(ev, next ? 1 : 0); \
2281 }
2282 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
2283 #undef FLATBUFFERS_TD
2284 default: break;
2285 }
2286 // clang-format on
2287 return parser.Error("fatal: invalid enum underlying type");
2288 }
2289
2290 EnumValBuilder(Parser &_parser, EnumDef &_enum_def)
2291 : parser(_parser),
2292 enum_def(_enum_def),
2293 temp(nullptr),
2294 user_value(false) {}
2295
2296 ~EnumValBuilder() { delete temp; }
2297
2298 Parser &parser;
2299 EnumDef &enum_def;
2300 EnumVal *temp;
2301 bool user_value;
2302};
2303
2304CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest,
2305 const char *filename) {
2306 std::vector<std::string> enum_comment = doc_comment_;
2307 NEXT();
2308 std::string enum_name = attribute_;
2309 EXPECT(kTokenIdentifier);
2310 EnumDef *enum_def;
2311 ECHECK(StartEnum(enum_name, is_union, &enum_def));
2312 if (filename != nullptr && !opts.project_root.empty()) {
2313 enum_def->declaration_file =
2314 &GetPooledString(RelativeToRootPath(opts.project_root, filename));
2315 }
2316 enum_def->doc_comment = enum_comment;
2317 if (!is_union && !opts.proto_mode) {
2318 // Give specialized error message, since this type spec used to
2319 // be optional in the first FlatBuffers release.
2320 if (!Is(':')) {
2321 return Error(
2322 "must specify the underlying integer type for this"
2323 " enum (e.g. \': short\', which was the default).");
2324 } else {
2325 NEXT();
2326 }
2327 // Specify the integer type underlying this enum.
2328 ECHECK(ParseType(enum_def->underlying_type));
2329 if (!IsInteger(enum_def->underlying_type.base_type) ||
2330 IsBool(enum_def->underlying_type.base_type))
2331 return Error("underlying enum type must be integral");
2332 // Make this type refer back to the enum it was derived from.
2333 enum_def->underlying_type.enum_def = enum_def;
2334 }
2335 ECHECK(ParseMetaData(&enum_def->attributes));
2336 const auto underlying_type = enum_def->underlying_type.base_type;
2337 if (enum_def->attributes.Lookup("bit_flags") &&
2338 !IsUnsigned(underlying_type)) {
2339 // todo: Convert to the Error in the future?
2340 Warning("underlying type of bit_flags enum must be unsigned");
2341 }
2342 EnumValBuilder evb(*this, *enum_def);
2343 EXPECT('{');
2344 // A lot of code generatos expect that an enum is not-empty.
2345 if ((is_union || Is('}')) && !opts.proto_mode) {
2346 evb.CreateEnumerator("NONE");
2347 ECHECK(evb.AcceptEnumerator());
2348 }
2349 std::set<std::pair<BaseType, StructDef *>> union_types;
2350 while (!Is('}')) {
2351 if (opts.proto_mode && attribute_ == "option") {
2352 ECHECK(ParseProtoOption());
2353 } else {
2354 auto &ev = *evb.CreateEnumerator(attribute_);
2355 auto full_name = ev.name;
2356 ev.doc_comment = doc_comment_;
2357 EXPECT(kTokenIdentifier);
2358 if (is_union) {
2359 ECHECK(ParseNamespacing(&full_name, &ev.name));
2360 if (opts.union_value_namespacing) {
2361 // Since we can't namespace the actual enum identifiers, turn
2362 // namespace parts into part of the identifier.
2363 ev.name = full_name;
2364 std::replace(ev.name.begin(), ev.name.end(), '.', '_');
2365 }
2366 if (Is(':')) {
2367 NEXT();
2368 ECHECK(ParseType(ev.union_type));
2369 if (ev.union_type.base_type != BASE_TYPE_STRUCT &&
2370 ev.union_type.base_type != BASE_TYPE_STRING)
2371 return Error("union value type may only be table/struct/string");
2372 } else {
2373 ev.union_type = Type(BASE_TYPE_STRUCT, LookupCreateStruct(full_name));
2374 }
2375 if (!enum_def->uses_multiple_type_instances) {
2376 auto ins = union_types.insert(std::make_pair(
2377 ev.union_type.base_type, ev.union_type.struct_def));
2378 enum_def->uses_multiple_type_instances = (false == ins.second);
2379 }
2380 }
2381
2382 if (Is('=')) {
2383 NEXT();
2384 ECHECK(evb.AssignEnumeratorValue(attribute_));
2385 EXPECT(kTokenIntegerConstant);
2386 }
2387
2388 ECHECK(evb.AcceptEnumerator());
2389
2390 if (opts.proto_mode && Is('[')) {
2391 NEXT();
2392 // ignore attributes on enums.
2393 while (token_ != ']') NEXT();
2394 NEXT();
2395 }
2396 }
2397 if (!Is(opts.proto_mode ? ';' : ',')) break;
2398 NEXT();
2399 }
2400 EXPECT('}');
2401
2402 // At this point, the enum can be empty if input is invalid proto-file.
2403 if (!enum_def->size())
2404 return Error("incomplete enum declaration, values not found");
2405
2406 if (enum_def->attributes.Lookup("bit_flags")) {
2407 const auto base_width = static_cast<uint64_t>(8 * SizeOf(underlying_type));
2408 for (auto it = enum_def->Vals().begin(); it != enum_def->Vals().end();
2409 ++it) {
2410 auto ev = *it;
2411 const auto u = ev->GetAsUInt64();
2412 // Stop manipulations with the sign.
2413 if (!IsUnsigned(underlying_type) && u == (base_width - 1))
2414 return Error("underlying type of bit_flags enum must be unsigned");
2415 if (u >= base_width)
2416 return Error("bit flag out of range of underlying integral type");
2417 enum_def->ChangeEnumValue(ev, 1ULL << u);
2418 }
2419 }
2420
2421 enum_def->SortByValue(); // Must be sorted to use MinValue/MaxValue.
2422
2423 // Ensure enum value uniqueness.
2424 auto prev_it = enum_def->Vals().begin();
2425 for (auto it = prev_it + 1; it != enum_def->Vals().end(); ++it) {
2426 auto prev_ev = *prev_it;
2427 auto ev = *it;
2428 if (prev_ev->GetAsUInt64() == ev->GetAsUInt64())
2429 return Error("all enum values must be unique: " + prev_ev->name +
2430 " and " + ev->name + " are both " +
2431 NumToString(ev->GetAsInt64()));
2432 }
2433
2434 if (dest) *dest = enum_def;
2435 const auto qualified_name =
2436 current_namespace_->GetFullyQualifiedName(enum_def->name);
2437 if (types_.Add(qualified_name, new Type(BASE_TYPE_UNION, nullptr, enum_def)))
2438 return Error("datatype already exists: " + qualified_name);
2439 return NoError();
2440}
2441
2442CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
2443 auto &struct_def = *LookupCreateStruct(name, true, true);
2444 if (!struct_def.predecl)
2445 return Error("datatype already exists: " +
2446 current_namespace_->GetFullyQualifiedName(name));
2447 struct_def.predecl = false;
2448 struct_def.name = name;
2449 struct_def.file = file_being_parsed_;
2450 // Move this struct to the back of the vector just in case it was predeclared,
2451 // to preserve declaration order.
2452 *std::remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) =
2453 &struct_def;
2454 *dest = &struct_def;
2455 return NoError();
2456}
2457
2458CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields,
2459 StructDef *struct_def, const char *suffix,
2460 BaseType basetype) {
2461 auto len = strlen(suffix);
2462 for (auto it = fields.begin(); it != fields.end(); ++it) {
2463 auto &fname = (*it)->name;
2464 if (fname.length() > len &&
2465 fname.compare(fname.length() - len, len, suffix) == 0 &&
2466 (*it)->value.type.base_type != BASE_TYPE_UTYPE) {
2467 auto field =
2468 struct_def->fields.Lookup(fname.substr(0, fname.length() - len));
2469 if (field && field->value.type.base_type == basetype)
2470 return Error("Field " + fname +
2471 " would clash with generated functions for field " +
2472 field->name);
2473 }
2474 }
2475 return NoError();
2476}
2477
2478bool Parser::SupportsOptionalScalars(const flatbuffers::IDLOptions &opts) {
2479 static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
2480 IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster |
2481 IDLOptions::kKotlin | IDLOptions::kCpp | IDLOptions::kJava |
2482 IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary;
2483 unsigned long langs = opts.lang_to_generate;
2484 return (langs > 0 && langs < IDLOptions::kMAX) && !(langs & ~supported_langs);
2485}
2486bool Parser::SupportsOptionalScalars() const {
2487 // Check in general if a language isn't specified.
2488 return opts.lang_to_generate == 0 || SupportsOptionalScalars(opts);
2489}
2490
2491bool Parser::SupportsDefaultVectorsAndStrings() const {
2492 static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
2493 IDLOptions::kRust | IDLOptions::kSwift;
2494 return !(opts.lang_to_generate & ~supported_langs);
2495}
2496
2497bool Parser::SupportsAdvancedUnionFeatures() const {
2498 return (opts.lang_to_generate &
2499 ~(IDLOptions::kCpp | IDLOptions::kTs | IDLOptions::kPhp |
2500 IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kKotlin |
2501 IDLOptions::kBinary | IDLOptions::kSwift)) == 0;
2502}
2503
2504bool Parser::SupportsAdvancedArrayFeatures() const {
2505 return (opts.lang_to_generate &
2506 ~(IDLOptions::kCpp | IDLOptions::kPython | IDLOptions::kJava |
2507 IDLOptions::kCSharp | IDLOptions::kJsonSchema | IDLOptions::kJson |
2508 IDLOptions::kBinary | IDLOptions::kRust)) == 0;
2509}
2510
2511Namespace *Parser::UniqueNamespace(Namespace *ns) {
2512 for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) {
2513 if (ns->components == (*it)->components) {
2514 delete ns;
2515 return *it;
2516 }
2517 }
2518 namespaces_.push_back(ns);
2519 return ns;
2520}
2521
2522std::string Parser::UnqualifiedName(const std::string &full_qualified_name) {
2523 Namespace *ns = new Namespace();
2524
2525 std::size_t current, previous = 0;
2526 current = full_qualified_name.find('.');
2527 while (current != std::string::npos) {
2528 ns->components.push_back(
2529 full_qualified_name.substr(previous, current - previous));
2530 previous = current + 1;
2531 current = full_qualified_name.find('.', previous);
2532 }
2533 current_namespace_ = UniqueNamespace(ns);
2534 return full_qualified_name.substr(previous, current - previous);
2535}
2536
2537static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) {
2538 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str());
2539 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str());
2540 return a_id < b_id;
2541}
2542
2543CheckedError Parser::ParseDecl(const char *filename) {
2544 std::vector<std::string> dc = doc_comment_;
2545 bool fixed = IsIdent("struct");
2546 if (!fixed && !IsIdent("table")) return Error("declaration expected");
2547 NEXT();
2548 std::string name = attribute_;
2549 EXPECT(kTokenIdentifier);
2550 StructDef *struct_def;
2551 ECHECK(StartStruct(name, &struct_def));
2552 struct_def->doc_comment = dc;
2553 struct_def->fixed = fixed;
2554 if (filename && !opts.project_root.empty()) {
2555 struct_def->declaration_file =
2556 &GetPooledString(RelativeToRootPath(opts.project_root, filename));
2557 }
2558 ECHECK(ParseMetaData(&struct_def->attributes));
2559 struct_def->sortbysize =
2560 struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
2561 EXPECT('{');
2562 while (token_ != '}') ECHECK(ParseField(*struct_def));
2563 if (fixed) {
2564 const auto force_align = struct_def->attributes.Lookup("force_align");
2565 if (force_align) {
2566 size_t align;
2567 ECHECK(ParseAlignAttribute(force_align->constant, struct_def->minalign,
2568 &align));
2569 struct_def->minalign = align;
2570 }
2571 if (!struct_def->bytesize) return Error("size 0 structs not allowed");
2572 }
2573 struct_def->PadLastField(struct_def->minalign);
2574 // Check if this is a table that has manual id assignments
2575 auto &fields = struct_def->fields.vec;
2576 if (!fixed && fields.size()) {
2577 size_t num_id_fields = 0;
2578 for (auto it = fields.begin(); it != fields.end(); ++it) {
2579 if ((*it)->attributes.Lookup("id")) num_id_fields++;
2580 }
2581 // If any fields have ids..
2582 if (num_id_fields || opts.require_explicit_ids) {
2583 // Then all fields must have them.
2584 if (num_id_fields != fields.size()) {
2585 if (opts.require_explicit_ids) {
2586 return Error(
2587 "all fields must have an 'id' attribute when "
2588 "--require-explicit-ids is used");
2589 } else {
2590 return Error(
2591 "either all fields or no fields must have an 'id' attribute");
2592 }
2593 }
2594 // Simply sort by id, then the fields are the same as if no ids had
2595 // been specified.
2596 std::sort(fields.begin(), fields.end(), compareFieldDefs);
2597 // Verify we have a contiguous set, and reassign vtable offsets.
2598 FLATBUFFERS_ASSERT(fields.size() <=
2599 flatbuffers::numeric_limits<voffset_t>::max());
2600 for (voffset_t i = 0; i < static_cast<voffset_t>(fields.size()); i++) {
2601 auto &field = *fields[i];
2602 const auto &id_str = field.attributes.Lookup("id")->constant;
2603 // Metadata values have a dynamic type, they can be `float`, 'int', or
2604 // 'string`.
2605 // The FieldIndexToOffset(i) expects the voffset_t so `id` is limited by
2606 // this type.
2607 voffset_t id = 0;
2608 const auto done = !atot(id_str.c_str(), *this, &id).Check();
2609 if (!done)
2610 return Error("field id\'s must be non-negative number, field: " +
2611 field.name + ", id: " + id_str);
2612 if (i != id)
2613 return Error("field id\'s must be consecutive from 0, id " +
2614 NumToString(i) + " missing or set twice, field: " +
2615 field.name + ", id: " + id_str);
2616 field.value.offset = FieldIndexToOffset(i);
2617 }
2618 }
2619 }
2620
2621 ECHECK(
2622 CheckClash(fields, struct_def, UnionTypeFieldSuffix(), BASE_TYPE_UNION));
2623 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION));
2624 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR));
2625 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR));
2626 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING));
2627 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING));
2628 EXPECT('}');
2629 const auto qualified_name =
2630 current_namespace_->GetFullyQualifiedName(struct_def->name);
2631 if (types_.Add(qualified_name,
2632 new Type(BASE_TYPE_STRUCT, struct_def, nullptr)))
2633 return Error("datatype already exists: " + qualified_name);
2634 return NoError();
2635}
2636
2637CheckedError Parser::ParseService(const char *filename) {
2638 std::vector<std::string> service_comment = doc_comment_;
2639 NEXT();
2640 auto service_name = attribute_;
2641 EXPECT(kTokenIdentifier);
2642 auto &service_def = *new ServiceDef();
2643 service_def.name = service_name;
2644 service_def.file = file_being_parsed_;
2645 service_def.doc_comment = service_comment;
2646 service_def.defined_namespace = current_namespace_;
2647 if (filename != nullptr && !opts.project_root.empty()) {
2648 service_def.declaration_file =
2649 &GetPooledString(RelativeToRootPath(opts.project_root, filename));
2650 }
2651 if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name),
2652 &service_def))
2653 return Error("service already exists: " + service_name);
2654 ECHECK(ParseMetaData(&service_def.attributes));
2655 EXPECT('{');
2656 do {
2657 std::vector<std::string> doc_comment = doc_comment_;
2658 auto rpc_name = attribute_;
2659 EXPECT(kTokenIdentifier);
2660 EXPECT('(');
2661 Type reqtype, resptype;
2662 ECHECK(ParseTypeIdent(reqtype));
2663 EXPECT(')');
2664 EXPECT(':');
2665 ECHECK(ParseTypeIdent(resptype));
2666 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed ||
2667 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed)
2668 return Error("rpc request and response types must be tables");
2669 auto &rpc = *new RPCCall();
2670 rpc.name = rpc_name;
2671 rpc.request = reqtype.struct_def;
2672 rpc.response = resptype.struct_def;
2673 rpc.doc_comment = doc_comment;
2674 if (service_def.calls.Add(rpc_name, &rpc))
2675 return Error("rpc already exists: " + rpc_name);
2676 ECHECK(ParseMetaData(&rpc.attributes));
2677 EXPECT(';');
2678 } while (token_ != '}');
2679 NEXT();
2680 return NoError();
2681}
2682
2683bool Parser::SetRootType(const char *name) {
2684 root_struct_def_ = LookupStruct(name);
2685 if (!root_struct_def_)
2686 root_struct_def_ =
2687 LookupStruct(current_namespace_->GetFullyQualifiedName(name));
2688 return root_struct_def_ != nullptr;
2689}
2690
2691void Parser::MarkGenerated() {
2692 // This function marks all existing definitions as having already
2693 // been generated, which signals no code for included files should be
2694 // generated.
2695 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
2696 (*it)->generated = true;
2697 }
2698 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
2699 if (!(*it)->predecl) { (*it)->generated = true; }
2700 }
2701 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
2702 (*it)->generated = true;
2703 }
2704}
2705
2706CheckedError Parser::ParseNamespace() {
2707 NEXT();
2708 auto ns = new Namespace();
2709 namespaces_.push_back(ns); // Store it here to not leak upon error.
2710 if (token_ != ';') {
2711 for (;;) {
2712 ns->components.push_back(attribute_);
2713 EXPECT(kTokenIdentifier);
2714 if (Is('.')) NEXT() else break;
2715 }
2716 }
2717 namespaces_.pop_back();
2718 current_namespace_ = UniqueNamespace(ns);
2719 EXPECT(';');
2720 return NoError();
2721}
2722
2723// Best effort parsing of .proto declarations, with the aim to turn them
2724// in the closest corresponding FlatBuffer equivalent.
2725// We parse everything as identifiers instead of keywords, since we don't
2726// want protobuf keywords to become invalid identifiers in FlatBuffers.
2727CheckedError Parser::ParseProtoDecl() {
2728 bool isextend = IsIdent("extend");
2729 if (IsIdent("package")) {
2730 // These are identical in syntax to FlatBuffer's namespace decl.
2731 ECHECK(ParseNamespace());
2732 } else if (IsIdent("message") || isextend) {
2733 std::vector<std::string> struct_comment = doc_comment_;
2734 NEXT();
2735 StructDef *struct_def = nullptr;
2736 Namespace *parent_namespace = nullptr;
2737 if (isextend) {
2738 if (Is('.')) NEXT(); // qualified names may start with a . ?
2739 auto id = attribute_;
2740 EXPECT(kTokenIdentifier);
2741 ECHECK(ParseNamespacing(&id, nullptr));
2742 struct_def = LookupCreateStruct(id, false);
2743 if (!struct_def)
2744 return Error("cannot extend unknown message type: " + id);
2745 } else {
2746 std::string name = attribute_;
2747 EXPECT(kTokenIdentifier);
2748 ECHECK(StartStruct(name, &struct_def));
2749 // Since message definitions can be nested, we create a new namespace.
2750 auto ns = new Namespace();
2751 // Copy of current namespace.
2752 *ns = *current_namespace_;
2753 // But with current message name.
2754 ns->components.push_back(name);
2755 ns->from_table++;
2756 parent_namespace = current_namespace_;
2757 current_namespace_ = UniqueNamespace(ns);
2758 }
2759 struct_def->doc_comment = struct_comment;
2760 ECHECK(ParseProtoFields(struct_def, isextend, false));
2761 if (!isextend) { current_namespace_ = parent_namespace; }
2762 if (Is(';')) NEXT();
2763 } else if (IsIdent("enum")) {
2764 // These are almost the same, just with different terminator:
2765 EnumDef *enum_def;
2766 ECHECK(ParseEnum(false, &enum_def, nullptr));
2767 if (Is(';')) NEXT();
2768 // Temp: remove any duplicates, as .fbs files can't handle them.
2769 enum_def->RemoveDuplicates();
2770 } else if (IsIdent("syntax")) { // Skip these.
2771 NEXT();
2772 EXPECT('=');
2773 EXPECT(kTokenStringConstant);
2774 EXPECT(';');
2775 } else if (IsIdent("option")) { // Skip these.
2776 ECHECK(ParseProtoOption());
2777 EXPECT(';');
2778 } else if (IsIdent("service")) { // Skip these.
2779 NEXT();
2780 EXPECT(kTokenIdentifier);
2781 ECHECK(ParseProtoCurliesOrIdent());
2782 } else {
2783 return Error("don\'t know how to parse .proto declaration starting with " +
2784 TokenToStringId(token_));
2785 }
2786 return NoError();
2787}
2788
2789CheckedError Parser::StartEnum(const std::string &name, bool is_union,
2790 EnumDef **dest) {
2791 auto &enum_def = *new EnumDef();
2792 enum_def.name = name;
2793 enum_def.file = file_being_parsed_;
2794 enum_def.doc_comment = doc_comment_;
2795 enum_def.is_union = is_union;
2796 enum_def.defined_namespace = current_namespace_;
2797 const auto qualified_name = current_namespace_->GetFullyQualifiedName(name);
2798 if (enums_.Add(qualified_name, &enum_def))
2799 return Error("enum already exists: " + qualified_name);
2800 enum_def.underlying_type.base_type =
2801 is_union ? BASE_TYPE_UTYPE : BASE_TYPE_INT;
2802 enum_def.underlying_type.enum_def = &enum_def;
2803 if (dest) *dest = &enum_def;
2804 return NoError();
2805}
2806
2807CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
2808 bool inside_oneof) {
2809 EXPECT('{');
2810 while (token_ != '}') {
2811 if (IsIdent("message") || IsIdent("extend") || IsIdent("enum")) {
2812 // Nested declarations.
2813 ECHECK(ParseProtoDecl());
2814 } else if (IsIdent("extensions")) { // Skip these.
2815 NEXT();
2816 EXPECT(kTokenIntegerConstant);
2817 if (Is(kTokenIdentifier)) {
2818 NEXT(); // to
2819 NEXT(); // num
2820 }
2821 EXPECT(';');
2822 } else if (IsIdent("option")) { // Skip these.
2823 ECHECK(ParseProtoOption());
2824 EXPECT(';');
2825 } else if (IsIdent("reserved")) { // Skip these.
2826 NEXT();
2827 while (!Is(';')) { NEXT(); } // A variety of formats, just skip.
2828 NEXT();
2829 } else {
2830 std::vector<std::string> field_comment = doc_comment_;
2831 // Parse the qualifier.
2832 bool required = false;
2833 bool repeated = false;
2834 bool oneof = false;
2835 if (!inside_oneof) {
2836 if (IsIdent("optional")) {
2837 // This is the default.
2838 NEXT();
2839 } else if (IsIdent("required")) {
2840 required = true;
2841 NEXT();
2842 } else if (IsIdent("repeated")) {
2843 repeated = true;
2844 NEXT();
2845 } else if (IsIdent("oneof")) {
2846 oneof = true;
2847 NEXT();
2848 } else {
2849 // can't error, proto3 allows decls without any of the above.
2850 }
2851 }
2852 StructDef *anonymous_struct = nullptr;
2853 EnumDef *oneof_union = nullptr;
2854 Type type;
2855 if (IsIdent("group") || oneof) {
2856 if (!oneof) NEXT();
2857 if (oneof && opts.proto_oneof_union) {
2858 auto name = MakeCamel(attribute_, true) + "Union";
2859 ECHECK(StartEnum(name, true, &oneof_union));
2860 type = Type(BASE_TYPE_UNION, nullptr, oneof_union);
2861 } else {
2862 auto name = "Anonymous" + NumToString(anonymous_counter_++);
2863 ECHECK(StartStruct(name, &anonymous_struct));
2864 type = Type(BASE_TYPE_STRUCT, anonymous_struct);
2865 }
2866 } else {
2867 ECHECK(ParseTypeFromProtoType(&type));
2868 }
2869 // Repeated elements get mapped to a vector.
2870 if (repeated) {
2871 type.element = type.base_type;
2872 type.base_type = BASE_TYPE_VECTOR;
2873 if (type.element == BASE_TYPE_VECTOR) {
2874 // We have a vector or vectors, which FlatBuffers doesn't support.
2875 // For now make it a vector of string (since the source is likely
2876 // "repeated bytes").
2877 // TODO(wvo): A better solution would be to wrap this in a table.
2878 type.element = BASE_TYPE_STRING;
2879 }
2880 }
2881 std::string name = attribute_;
2882 EXPECT(kTokenIdentifier);
2883 if (!oneof) {
2884 // Parse the field id. Since we're just translating schemas, not
2885 // any kind of binary compatibility, we can safely ignore these, and
2886 // assign our own.
2887 EXPECT('=');
2888 EXPECT(kTokenIntegerConstant);
2889 }
2890 FieldDef *field = nullptr;
2891 if (isextend) {
2892 // We allow a field to be re-defined when extending.
2893 // TODO: are there situations where that is problematic?
2894 field = struct_def->fields.Lookup(name);
2895 }
2896 if (!field) ECHECK(AddField(*struct_def, name, type, &field));
2897 field->doc_comment = field_comment;
2898 if (!IsScalar(type.base_type) && required) {
2899 field->presence = FieldDef::kRequired;
2900 }
2901 // See if there's a default specified.
2902 if (Is('[')) {
2903 NEXT();
2904 for (;;) {
2905 auto key = attribute_;
2906 ECHECK(ParseProtoKey());
2907 EXPECT('=');
2908 auto val = attribute_;
2909 ECHECK(ParseProtoCurliesOrIdent());
2910 if (key == "default") {
2911 // Temp: skip non-numeric and non-boolean defaults (enums).
2912 auto numeric = strpbrk(val.c_str(), "0123456789-+.");
2913 if (IsScalar(type.base_type) && numeric == val.c_str()) {
2914 field->value.constant = val;
2915 } else if (val == "true") {
2916 field->value.constant = val;
2917 } // "false" is default, no need to handle explicitly.
2918 } else if (key == "deprecated") {
2919 field->deprecated = val == "true";
2920 }
2921 if (!Is(',')) break;
2922 NEXT();
2923 }
2924 EXPECT(']');
2925 }
2926 if (anonymous_struct) {
2927 ECHECK(ParseProtoFields(anonymous_struct, false, oneof));
2928 if (Is(';')) NEXT();
2929 } else if (oneof_union) {
2930 // Parse into a temporary StructDef, then transfer fields into an
2931 // EnumDef describing the oneof as a union.
2932 StructDef oneof_struct;
2933 ECHECK(ParseProtoFields(&oneof_struct, false, oneof));
2934 if (Is(';')) NEXT();
2935 for (auto field_it = oneof_struct.fields.vec.begin();
2936 field_it != oneof_struct.fields.vec.end(); ++field_it) {
2937 const auto &oneof_field = **field_it;
2938 const auto &oneof_type = oneof_field.value.type;
2939 if (oneof_type.base_type != BASE_TYPE_STRUCT ||
2940 !oneof_type.struct_def || oneof_type.struct_def->fixed)
2941 return Error("oneof '" + name +
2942 "' cannot be mapped to a union because member '" +
2943 oneof_field.name + "' is not a table type.");
2944 EnumValBuilder evb(*this, *oneof_union);
2945 auto ev = evb.CreateEnumerator(oneof_type.struct_def->name);
2946 ev->union_type = oneof_type;
2947 ev->doc_comment = oneof_field.doc_comment;
2948 ECHECK(evb.AcceptEnumerator(oneof_field.name));
2949 }
2950 } else {
2951 EXPECT(';');
2952 }
2953 }
2954 }
2955 NEXT();
2956 return NoError();
2957}
2958
2959CheckedError Parser::ParseProtoKey() {
2960 if (token_ == '(') {
2961 NEXT();
2962 // Skip "(a.b)" style custom attributes.
2963 while (token_ == '.' || token_ == kTokenIdentifier) NEXT();
2964 EXPECT(')');
2965 while (Is('.')) {
2966 NEXT();
2967 EXPECT(kTokenIdentifier);
2968 }
2969 } else {
2970 EXPECT(kTokenIdentifier);
2971 }
2972 return NoError();
2973}
2974
2975CheckedError Parser::ParseProtoCurliesOrIdent() {
2976 if (Is('{')) {
2977 NEXT();
2978 for (int nesting = 1; nesting;) {
2979 if (token_ == '{')
2980 nesting++;
2981 else if (token_ == '}')
2982 nesting--;
2983 NEXT();
2984 }
2985 } else {
2986 NEXT(); // Any single token.
2987 }
2988 return NoError();
2989}
2990
2991CheckedError Parser::ParseProtoOption() {
2992 NEXT();
2993 ECHECK(ParseProtoKey());
2994 EXPECT('=');
2995 ECHECK(ParseProtoCurliesOrIdent());
2996 return NoError();
2997}
2998
2999// Parse a protobuf type, and map it to the corresponding FlatBuffer one.
3000CheckedError Parser::ParseTypeFromProtoType(Type *type) {
3001 struct type_lookup {
3002 const char *proto_type;
3003 BaseType fb_type, element;
3004 };
3005 static type_lookup lookup[] = {
3006 { "float", BASE_TYPE_FLOAT, BASE_TYPE_NONE },
3007 { "double", BASE_TYPE_DOUBLE, BASE_TYPE_NONE },
3008 { "int32", BASE_TYPE_INT, BASE_TYPE_NONE },
3009 { "int64", BASE_TYPE_LONG, BASE_TYPE_NONE },
3010 { "uint32", BASE_TYPE_UINT, BASE_TYPE_NONE },
3011 { "uint64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
3012 { "sint32", BASE_TYPE_INT, BASE_TYPE_NONE },
3013 { "sint64", BASE_TYPE_LONG, BASE_TYPE_NONE },
3014 { "fixed32", BASE_TYPE_UINT, BASE_TYPE_NONE },
3015 { "fixed64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
3016 { "sfixed32", BASE_TYPE_INT, BASE_TYPE_NONE },
3017 { "sfixed64", BASE_TYPE_LONG, BASE_TYPE_NONE },
3018 { "bool", BASE_TYPE_BOOL, BASE_TYPE_NONE },
3019 { "string", BASE_TYPE_STRING, BASE_TYPE_NONE },
3020 { "bytes", BASE_TYPE_VECTOR, BASE_TYPE_UCHAR },
3021 { nullptr, BASE_TYPE_NONE, BASE_TYPE_NONE }
3022 };
3023 for (auto tl = lookup; tl->proto_type; tl++) {
3024 if (attribute_ == tl->proto_type) {
3025 type->base_type = tl->fb_type;
3026 type->element = tl->element;
3027 NEXT();
3028 return NoError();
3029 }
3030 }
3031 if (Is('.')) NEXT(); // qualified names may start with a . ?
3032 ECHECK(ParseTypeIdent(*type));
3033 return NoError();
3034}
3035
3036CheckedError Parser::SkipAnyJsonValue() {
3037 ParseDepthGuard depth_guard(this);
3038 ECHECK(depth_guard.Check());
3039
3040 switch (token_) {
3041 case '{': {
3042 size_t fieldn_outer = 0;
3043 return ParseTableDelimiters(fieldn_outer, nullptr,
3044 [&](const std::string &, size_t &fieldn,
3045 const StructDef *) -> CheckedError {
3046 ECHECK(SkipAnyJsonValue());
3047 fieldn++;
3048 return NoError();
3049 });
3050 }
3051 case '[': {
3052 uoffset_t count = 0;
3053 return ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
3054 return SkipAnyJsonValue();
3055 });
3056 }
3057 case kTokenStringConstant:
3058 case kTokenIntegerConstant:
3059 case kTokenFloatConstant: NEXT(); break;
3060 default:
3061 if (IsIdent("true") || IsIdent("false") || IsIdent("null")) {
3062 NEXT();
3063 } else
3064 return TokenError();
3065 }
3066 return NoError();
3067}
3068
3069CheckedError Parser::ParseFlexBufferNumericConstant(
3070 flexbuffers::Builder *builder) {
3071 double d;
3072 if (!StringToNumber(attribute_.c_str(), &d))
3073 return Error("unexpected floating-point constant: " + attribute_);
3074 builder->Double(d);
3075 return NoError();
3076}
3077
3078CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
3079 ParseDepthGuard depth_guard(this);
3080 ECHECK(depth_guard.Check());
3081
3082 switch (token_) {
3083 case '{': {
3084 auto start = builder->StartMap();
3085 size_t fieldn_outer = 0;
3086 auto err =
3087 ParseTableDelimiters(fieldn_outer, nullptr,
3088 [&](const std::string &name, size_t &fieldn,
3089 const StructDef *) -> CheckedError {
3090 builder->Key(name);
3091 ECHECK(ParseFlexBufferValue(builder));
3092 fieldn++;
3093 return NoError();
3094 });
3095 ECHECK(err);
3096 builder->EndMap(start);
3097 if (builder->HasDuplicateKeys())
3098 return Error("FlexBuffers map has duplicate keys");
3099 break;
3100 }
3101 case '[': {
3102 auto start = builder->StartVector();
3103 uoffset_t count = 0;
3104 ECHECK(ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
3105 return ParseFlexBufferValue(builder);
3106 }));
3107 builder->EndVector(start, false, false);
3108 break;
3109 }
3110 case kTokenStringConstant:
3111 builder->String(attribute_);
3112 EXPECT(kTokenStringConstant);
3113 break;
3114 case kTokenIntegerConstant:
3115 builder->Int(StringToInt(attribute_.c_str()));
3116 EXPECT(kTokenIntegerConstant);
3117 break;
3118 case kTokenFloatConstant: {
3119 double d;
3120 StringToNumber(attribute_.c_str(), &d);
3121 builder->Double(d);
3122 EXPECT(kTokenFloatConstant);
3123 break;
3124 }
3125 case '-':
3126 case '+': {
3127 // `[-+]?(nan|inf|infinity)`, see ParseSingleValue().
3128 const auto sign = static_cast<char>(token_);
3129 NEXT();
3130 if (token_ != kTokenIdentifier)
3131 return Error("floating-point constant expected");
3132 attribute_.insert(0, 1, sign);
3133 ECHECK(ParseFlexBufferNumericConstant(builder));
3134 NEXT();
3135 break;
3136 }
3137 default:
3138 if (IsIdent("true")) {
3139 builder->Bool(true);
3140 NEXT();
3141 } else if (IsIdent("false")) {
3142 builder->Bool(false);
3143 NEXT();
3144 } else if (IsIdent("null")) {
3145 builder->Null();
3146 NEXT();
3147 } else if (IsIdent("inf") || IsIdent("infinity") || IsIdent("nan")) {
3148 ECHECK(ParseFlexBufferNumericConstant(builder));
3149 NEXT();
3150 } else
3151 return TokenError();
3152 }
3153 return NoError();
3154}
3155
3156bool Parser::ParseFlexBuffer(const char *source, const char *source_filename,
3157 flexbuffers::Builder *builder) {
3158 const auto initial_depth = parse_depth_counter_;
3159 (void)initial_depth;
3160 auto ok = !StartParseFile(source, source_filename).Check() &&
3161 !ParseFlexBufferValue(builder).Check();
3162 if (ok) builder->Finish();
3163 FLATBUFFERS_ASSERT(initial_depth == parse_depth_counter_);
3164 return ok;
3165}
3166
3167bool Parser::Parse(const char *source, const char **include_paths,
3168 const char *source_filename) {
3169 const auto initial_depth = parse_depth_counter_;
3170 (void)initial_depth;
3171 bool r;
3172
3173 if (opts.use_flexbuffers) {
3174 r = ParseFlexBuffer(source, source_filename, &flex_builder_);
3175 } else {
3176 r = !ParseRoot(source, include_paths, source_filename).Check();
3177 }
3178 FLATBUFFERS_ASSERT(initial_depth == parse_depth_counter_);
3179 return r;
3180}
3181
3182bool Parser::ParseJson(const char *json, const char *json_filename) {
3183 const auto initial_depth = parse_depth_counter_;
3184 (void)initial_depth;
3185 builder_.Clear();
3186 const auto done =
3187 !StartParseFile(json, json_filename).Check() && !DoParseJson().Check();
3188 FLATBUFFERS_ASSERT(initial_depth == parse_depth_counter_);
3189 return done;
3190}
3191
3192CheckedError Parser::StartParseFile(const char *source,
3193 const char *source_filename) {
3194 file_being_parsed_ = source_filename ? source_filename : "";
3195 source_ = source;
3196 ResetState(source_);
3197 error_.clear();
3198 ECHECK(SkipByteOrderMark());
3199 NEXT();
3200 if (Is(kTokenEof)) return Error("input file is empty");
3201 return NoError();
3202}
3203
3204CheckedError Parser::ParseRoot(const char *source, const char **include_paths,
3205 const char *source_filename) {
3206 ECHECK(DoParse(source, include_paths, source_filename, nullptr));
3207
3208 // Check that all types were defined.
3209 for (auto it = structs_.vec.begin(); it != structs_.vec.end();) {
3210 auto &struct_def = **it;
3211 if (struct_def.predecl) {
3212 if (opts.proto_mode) {
3213 // Protos allow enums to be used before declaration, so check if that
3214 // is the case here.
3215 EnumDef *enum_def = nullptr;
3216 for (size_t components =
3217 struct_def.defined_namespace->components.size() + 1;
3218 components && !enum_def; components--) {
3219 auto qualified_name =
3220 struct_def.defined_namespace->GetFullyQualifiedName(
3221 struct_def.name, components - 1);
3222 enum_def = LookupEnum(qualified_name);
3223 }
3224 if (enum_def) {
3225 // This is pretty slow, but a simple solution for now.
3226 auto initial_count = struct_def.refcount;
3227 for (auto struct_it = structs_.vec.begin();
3228 struct_it != structs_.vec.end(); ++struct_it) {
3229 auto &sd = **struct_it;
3230 for (auto field_it = sd.fields.vec.begin();
3231 field_it != sd.fields.vec.end(); ++field_it) {
3232 auto &field = **field_it;
3233 if (field.value.type.struct_def == &struct_def) {
3234 field.value.type.struct_def = nullptr;
3235 field.value.type.enum_def = enum_def;
3236 auto &bt = IsVector(field.value.type)
3237 ? field.value.type.element
3238 : field.value.type.base_type;
3239 FLATBUFFERS_ASSERT(bt == BASE_TYPE_STRUCT);
3240 bt = enum_def->underlying_type.base_type;
3241 struct_def.refcount--;
3242 enum_def->refcount++;
3243 }
3244 }
3245 }
3246 if (struct_def.refcount)
3247 return Error("internal: " + NumToString(struct_def.refcount) + "/" +
3248 NumToString(initial_count) +
3249 " use(s) of pre-declaration enum not accounted for: " +
3250 enum_def->name);
3251 structs_.dict.erase(structs_.dict.find(struct_def.name));
3252 it = structs_.vec.erase(it);
3253 delete &struct_def;
3254 continue; // Skip error.
3255 }
3256 }
3257 auto err = "type referenced but not defined (check namespace): " +
3258 struct_def.name;
3259 if (struct_def.original_location)
3260 err += ", originally at: " + *struct_def.original_location;
3261 return Error(err);
3262 }
3263 ++it;
3264 }
3265
3266 // This check has to happen here and not earlier, because only now do we
3267 // know for sure what the type of these are.
3268 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
3269 auto &enum_def = **it;
3270 if (enum_def.is_union) {
3271 for (auto val_it = enum_def.Vals().begin();
3272 val_it != enum_def.Vals().end(); ++val_it) {
3273 auto &val = **val_it;
3274 if (!(opts.lang_to_generate != 0 && SupportsAdvancedUnionFeatures()) &&
3275 (IsStruct(val.union_type) || IsString(val.union_type)))
3276 return Error(
3277 "only tables can be union elements in the generated language: " +
3278 val.name);
3279 }
3280 }
3281 }
3282 // Parse JSON object only if the scheme has been parsed.
3283 if (token_ == '{') { ECHECK(DoParseJson()); }
3284 EXPECT(kTokenEof);
3285 return NoError();
3286}
3287
3288// Generate a unique hash for a file based on its name and contents (if any).
3289static uint64_t HashFile(const char *source_filename, const char *source) {
3290 uint64_t hash = 0;
3291
3292 if (source_filename)
3293 hash = HashFnv1a<uint64_t>(StripPath(source_filename).c_str());
3294
3295 if (source && *source) hash ^= HashFnv1a<uint64_t>(source);
3296
3297 return hash;
3298}
3299
3300CheckedError Parser::DoParse(const char *source, const char **include_paths,
3301 const char *source_filename,
3302 const char *include_filename) {
3303 uint64_t source_hash = 0;
3304 if (source_filename) {
3305 // If the file is in-memory, don't include its contents in the hash as we
3306 // won't be able to load them later.
3307 if (FileExists(source_filename))
3308 source_hash = HashFile(source_filename, source);
3309 else
3310 source_hash = HashFile(source_filename, nullptr);
3311
3312 if (included_files_.find(source_hash) == included_files_.end()) {
3313 included_files_[source_hash] = include_filename ? include_filename : "";
3314 files_included_per_file_[source_filename] = std::set<std::string>();
3315 } else {
3316 return NoError();
3317 }
3318 }
3319 if (!include_paths) {
3320 static const char *current_directory[] = { "", nullptr };
3321 include_paths = current_directory;
3322 }
3323 field_stack_.clear();
3324 builder_.Clear();
3325 // Start with a blank namespace just in case this file doesn't have one.
3326 current_namespace_ = empty_namespace_;
3327
3328 ECHECK(StartParseFile(source, source_filename));
3329
3330 // Includes must come before type declarations:
3331 for (;;) {
3332 // Parse pre-include proto statements if any:
3333 if (opts.proto_mode && (attribute_ == "option" || attribute_ == "syntax" ||
3334 attribute_ == "package")) {
3335 ECHECK(ParseProtoDecl());
3336 } else if (IsIdent("native_include")) {
3337 NEXT();
3338 native_included_files_.emplace_back(attribute_);
3339 EXPECT(kTokenStringConstant);
3340 EXPECT(';');
3341 } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) {
3342 NEXT();
3343 if (opts.proto_mode && attribute_ == "public") NEXT();
3344 auto name = flatbuffers::PosixPath(attribute_.c_str());
3345 EXPECT(kTokenStringConstant);
3346 // Look for the file relative to the directory of the current file.
3347 std::string filepath;
3348 if (source_filename) {
3349 auto source_file_directory =
3350 flatbuffers::StripFileName(source_filename);
3351 filepath = flatbuffers::ConCatPathFileName(source_file_directory, name);
3352 }
3353 if (filepath.empty() || !FileExists(filepath.c_str())) {
3354 // Look for the file in include_paths.
3355 for (auto paths = include_paths; paths && *paths; paths++) {
3356 filepath = flatbuffers::ConCatPathFileName(*paths, name);
3357 if (FileExists(filepath.c_str())) break;
3358 }
3359 }
3360 if (filepath.empty())
3361 return Error("unable to locate include file: " + name);
3362 if (source_filename)
3363 files_included_per_file_[source_filename].insert(filepath);
3364
3365 std::string contents;
3366 bool file_loaded = LoadFile(filepath.c_str(), true, &contents);
3367 if (included_files_.find(HashFile(filepath.c_str(), contents.c_str())) ==
3368 included_files_.end()) {
3369 // We found an include file that we have not parsed yet.
3370 // Parse it.
3371 if (!file_loaded) return Error("unable to load include file: " + name);
3372 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str(),
3373 name.c_str()));
3374 // We generally do not want to output code for any included files:
3375 if (!opts.generate_all) MarkGenerated();
3376 // Reset these just in case the included file had them, and the
3377 // parent doesn't.
3378 root_struct_def_ = nullptr;
3379 file_identifier_.clear();
3380 file_extension_.clear();
3381 // This is the easiest way to continue this file after an include:
3382 // instead of saving and restoring all the state, we simply start the
3383 // file anew. This will cause it to encounter the same include
3384 // statement again, but this time it will skip it, because it was
3385 // entered into included_files_.
3386 // This is recursive, but only go as deep as the number of include
3387 // statements.
3388 included_files_.erase(source_hash);
3389 return DoParse(source, include_paths, source_filename,
3390 include_filename);
3391 }
3392 EXPECT(';');
3393 } else {
3394 break;
3395 }
3396 }
3397 // Now parse all other kinds of declarations:
3398 while (token_ != kTokenEof) {
3399 if (opts.proto_mode) {
3400 ECHECK(ParseProtoDecl());
3401 } else if (IsIdent("namespace")) {
3402 ECHECK(ParseNamespace());
3403 } else if (token_ == '{') {
3404 return NoError();
3405 } else if (IsIdent("enum")) {
3406 ECHECK(ParseEnum(false, nullptr, source_filename));
3407 } else if (IsIdent("union")) {
3408 ECHECK(ParseEnum(true, nullptr, source_filename));
3409 } else if (IsIdent("root_type")) {
3410 NEXT();
3411 auto root_type = attribute_;
3412 EXPECT(kTokenIdentifier);
3413 ECHECK(ParseNamespacing(&root_type, nullptr));
3414 if (opts.root_type.empty()) {
3415 if (!SetRootType(root_type.c_str()))
3416 return Error("unknown root type: " + root_type);
3417 if (root_struct_def_->fixed) return Error("root type must be a table");
3418 }
3419 EXPECT(';');
3420 } else if (IsIdent("file_identifier")) {
3421 NEXT();
3422 file_identifier_ = attribute_;
3423 EXPECT(kTokenStringConstant);
3424 if (file_identifier_.length() != flatbuffers::kFileIdentifierLength)
3425 return Error("file_identifier must be exactly " +
3426 NumToString(flatbuffers::kFileIdentifierLength) +
3427 " characters");
3428 EXPECT(';');
3429 } else if (IsIdent("file_extension")) {
3430 NEXT();
3431 file_extension_ = attribute_;
3432 EXPECT(kTokenStringConstant);
3433 EXPECT(';');
3434 } else if (IsIdent("include")) {
3435 return Error("includes must come before declarations");
3436 } else if (IsIdent("attribute")) {
3437 NEXT();
3438 auto name = attribute_;
3439 if (Is(kTokenIdentifier)) {
3440 NEXT();
3441 } else {
3442 EXPECT(kTokenStringConstant);
3443 }
3444 EXPECT(';');
3445 known_attributes_[name] = false;
3446 } else if (IsIdent("rpc_service")) {
3447 ECHECK(ParseService(source_filename));
3448 } else {
3449 ECHECK(ParseDecl(source_filename));
3450 }
3451 }
3452 if (opts.warnings_as_errors && has_warning_) {
3453 return Error("treating warnings as errors, failed due to above warnings");
3454 }
3455 return NoError();
3456}
3457
3458CheckedError Parser::DoParseJson() {
3459 if (token_ != '{') {
3460 EXPECT('{');
3461 } else {
3462 if (!root_struct_def_) return Error("no root type set to parse json with");
3463 if (builder_.GetSize()) {
3464 return Error("cannot have more than one json object in a file");
3465 }
3466 uoffset_t toff;
3467 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff));
3468 if (opts.size_prefixed) {
3469 builder_.FinishSizePrefixed(
3470 Offset<Table>(toff),
3471 file_identifier_.length() ? file_identifier_.c_str() : nullptr);
3472 } else {
3473 builder_.Finish(Offset<Table>(toff), file_identifier_.length()
3474 ? file_identifier_.c_str()
3475 : nullptr);
3476 }
3477 }
3478 // Check that JSON file doesn't contain more objects or IDL directives.
3479 // Comments after JSON are allowed.
3480 EXPECT(kTokenEof);
3481 return NoError();
3482}
3483
3484std::set<std::string> Parser::GetIncludedFilesRecursive(
3485 const std::string &file_name) const {
3486 std::set<std::string> included_files;
3487 std::list<std::string> to_process;
3488
3489 if (file_name.empty()) return included_files;
3490 to_process.push_back(file_name);
3491
3492 while (!to_process.empty()) {
3493 std::string current = to_process.front();
3494 to_process.pop_front();
3495 included_files.insert(current);
3496
3497 // Workaround the lack of const accessor in C++98 maps.
3498 auto &new_files =
3499 (*const_cast<std::map<std::string, std::set<std::string>> *>(
3500 &files_included_per_file_))[current];
3501 for (auto it = new_files.begin(); it != new_files.end(); ++it) {
3502 if (included_files.find(*it) == included_files.end())
3503 to_process.push_back(*it);
3504 }
3505 }
3506
3507 return included_files;
3508}
3509
3510// Schema serialization functionality:
3511
3512template<typename T> bool compareName(const T *a, const T *b) {
3513 return a->defined_namespace->GetFullyQualifiedName(a->name) <
3514 b->defined_namespace->GetFullyQualifiedName(b->name);
3515}
3516
3517template<typename T> void AssignIndices(const std::vector<T *> &defvec) {
3518 // Pre-sort these vectors, such that we can set the correct indices for them.
3519 auto vec = defvec;
3520 std::sort(vec.begin(), vec.end(), compareName<T>);
3521 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
3522}
3523
3524void Parser::Serialize() {
3525 builder_.Clear();
3526 AssignIndices(structs_.vec);
3527 AssignIndices(enums_.vec);
3528 std::vector<Offset<reflection::Object>> object_offsets;
3529 std::set<std::string> files;
3530 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
3531 auto offset = (*it)->Serialize(&builder_, *this);
3532 object_offsets.push_back(offset);
3533 (*it)->serialized_location = offset.o;
3534 const std::string *file = (*it)->declaration_file;
3535 if (file) files.insert(*file);
3536 }
3537 std::vector<Offset<reflection::Enum>> enum_offsets;
3538 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
3539 auto offset = (*it)->Serialize(&builder_, *this);
3540 enum_offsets.push_back(offset);
3541 const std::string *file = (*it)->declaration_file;
3542 if (file) files.insert(*file);
3543 }
3544 std::vector<Offset<reflection::Service>> service_offsets;
3545 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
3546 auto offset = (*it)->Serialize(&builder_, *this);
3547 service_offsets.push_back(offset);
3548 const std::string *file = (*it)->declaration_file;
3549 if (file) files.insert(*file);
3550 }
3551
3552 // Create Schemafiles vector of tables.
3553 flatbuffers::Offset<
3554 flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>>
3555 schema_files__;
3556 if (!opts.project_root.empty()) {
3557 std::vector<Offset<reflection::SchemaFile>> schema_files;
3558 std::vector<Offset<flatbuffers::String>> included_files;
3559 for (auto f = files_included_per_file_.begin();
3560 f != files_included_per_file_.end(); f++) {
3561 const auto filename__ = builder_.CreateSharedString(
3562 RelativeToRootPath(opts.project_root, f->first));
3563 for (auto i = f->second.begin(); i != f->second.end(); i++) {
3564 included_files.push_back(builder_.CreateSharedString(
3565 RelativeToRootPath(opts.project_root, *i)));
3566 }
3567 const auto included_files__ = builder_.CreateVector(included_files);
3568 included_files.clear();
3569
3570 schema_files.push_back(
3571 reflection::CreateSchemaFile(builder_, filename__, included_files__));
3572 }
3573 schema_files__ = builder_.CreateVectorOfSortedTables(&schema_files);
3574 }
3575
3576 const auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets);
3577 const auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets);
3578 const auto fiid__ = builder_.CreateString(file_identifier_);
3579 const auto fext__ = builder_.CreateString(file_extension_);
3580 const auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets);
3581 const auto schema_offset = reflection::CreateSchema(
3582 builder_, objs__, enum__, fiid__, fext__,
3583 (root_struct_def_ ? root_struct_def_->serialized_location : 0), serv__,
3584 static_cast<reflection::AdvancedFeatures>(advanced_features_),
3585 schema_files__);
3586 if (opts.size_prefixed) {
3587 builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier());
3588 } else {
3589 builder_.Finish(schema_offset, reflection::SchemaIdentifier());
3590 }
3591}
3592
3593static Namespace *GetNamespace(
3594 const std::string &qualified_name, std::vector<Namespace *> &namespaces,
3595 std::map<std::string, Namespace *> &namespaces_index) {
3596 size_t dot = qualified_name.find_last_of('.');
3597 std::string namespace_name = (dot != std::string::npos)
3598 ? std::string(qualified_name.c_str(), dot)
3599 : "";
3600 Namespace *&ns = namespaces_index[namespace_name];
3601
3602 if (!ns) {
3603 ns = new Namespace();
3604 namespaces.push_back(ns);
3605
3606 size_t pos = 0;
3607
3608 for (;;) {
3609 dot = qualified_name.find('.', pos);
3610 if (dot == std::string::npos) { break; }
3611 ns->components.push_back(qualified_name.substr(pos, dot - pos));
3612 pos = dot + 1;
3613 }
3614 }
3615
3616 return ns;
3617}
3618
3619Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
3620 const Parser &parser) const {
3621 std::vector<Offset<reflection::Field>> field_offsets;
3622 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) {
3623 field_offsets.push_back((*it)->Serialize(
3624 builder, static_cast<uint16_t>(it - fields.vec.begin()), parser));
3625 }
3626 const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
3627 const auto name__ = builder->CreateString(qualified_name);
3628 const auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets);
3629 const auto attr__ = SerializeAttributes(builder, parser);
3630 const auto docs__ = parser.opts.binary_schema_comments
3631 ? builder->CreateVectorOfStrings(doc_comment)
3632 : 0;
3633 std::string decl_file_in_project = declaration_file ? *declaration_file : "";
3634 const auto file__ = builder->CreateSharedString(decl_file_in_project);
3635 return reflection::CreateObject(
3636 *builder, name__, flds__, fixed, static_cast<int>(minalign),
3637 static_cast<int>(bytesize), attr__, docs__, file__);
3638}
3639
3640bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) {
3641 if (!DeserializeAttributes(parser, object->attributes())) return false;
3642 DeserializeDoc(doc_comment, object->documentation());
3643 name = parser.UnqualifiedName(object->name()->str());
3644 predecl = false;
3645 sortbysize = attributes.Lookup("original_order") == nullptr && !fixed;
3646 const auto &of = *(object->fields());
3647 auto indexes = std::vector<uoffset_t>(of.size());
3648 for (uoffset_t i = 0; i < of.size(); i++) indexes[of.Get(i)->id()] = i;
3649 size_t tmp_struct_size = 0;
3650 for (size_t i = 0; i < indexes.size(); i++) {
3651 auto field = of.Get(indexes[i]);
3652 auto field_def = new FieldDef();
3653 if (!field_def->Deserialize(parser, field) ||
3654 fields.Add(field_def->name, field_def)) {
3655 delete field_def;
3656 return false;
3657 }
3658 if (fixed) {
3659 // Recompute padding since that's currently not serialized.
3660 auto size = InlineSize(field_def->value.type);
3661 auto next_field =
3662 i + 1 < indexes.size() ? of.Get(indexes[i + 1]) : nullptr;
3663 tmp_struct_size += size;
3664 field_def->padding =
3665 next_field ? (next_field->offset() - field_def->value.offset) - size
3666 : PaddingBytes(tmp_struct_size, minalign);
3667 tmp_struct_size += field_def->padding;
3668 }
3669 }
3670 FLATBUFFERS_ASSERT(static_cast<int>(tmp_struct_size) == object->bytesize());
3671 return true;
3672}
3673
3674Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
3675 uint16_t id,
3676 const Parser &parser) const {
3677 auto name__ = builder->CreateString(name);
3678 auto type__ = value.type.Serialize(builder);
3679 auto attr__ = SerializeAttributes(builder, parser);
3680 auto docs__ = parser.opts.binary_schema_comments
3681 ? builder->CreateVectorOfStrings(doc_comment)
3682 : 0;
3683 double d;
3684 StringToNumber(value.constant.c_str(), &d);
3685 return reflection::CreateField(
3686 *builder, name__, type__, id, value.offset,
3687 // Is uint64>max(int64) tested?
3688 IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0,
3689 // result may be platform-dependent if underlying is float (not double)
3690 IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key,
3691 attr__, docs__, IsOptional(), static_cast<uint16_t>(padding));
3692 // TODO: value.constant is almost always "0", we could save quite a bit of
3693 // space by sharing it. Same for common values of value.type.
3694}
3695
3696bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) {
3697 name = field->name()->str();
3698 defined_namespace = parser.current_namespace_;
3699 if (!value.type.Deserialize(parser, field->type())) return false;
3700 value.offset = field->offset();
3701 if (IsInteger(value.type.base_type)) {
3702 value.constant = NumToString(field->default_integer());
3703 } else if (IsFloat(value.type.base_type)) {
3704 value.constant = FloatToString(field->default_real(), 16);
3705 }
3706 presence = FieldDef::MakeFieldPresence(field->optional(), field->required());
3707 padding = field->padding();
3708 key = field->key();
3709 if (!DeserializeAttributes(parser, field->attributes())) return false;
3710 // TODO: this should probably be handled by a separate attribute
3711 if (attributes.Lookup("flexbuffer")) {
3712 flexbuffer = true;
3713 parser.uses_flexbuffers_ = true;
3714 if (value.type.base_type != BASE_TYPE_VECTOR ||
3715 value.type.element != BASE_TYPE_UCHAR)
3716 return false;
3717 }
3718 if (auto nested = attributes.Lookup("nested_flatbuffer")) {
3719 auto nested_qualified_name =
3720 parser.current_namespace_->GetFullyQualifiedName(nested->constant);
3721 nested_flatbuffer = parser.LookupStruct(nested_qualified_name);
3722 if (!nested_flatbuffer) return false;
3723 }
3724 shared = attributes.Lookup("shared") != nullptr;
3725 DeserializeDoc(doc_comment, field->documentation());
3726 return true;
3727}
3728
3729Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder,
3730 const Parser &parser) const {
3731 auto name__ = builder->CreateString(name);
3732 auto attr__ = SerializeAttributes(builder, parser);
3733 auto docs__ = parser.opts.binary_schema_comments
3734 ? builder->CreateVectorOfStrings(doc_comment)
3735 : 0;
3736 return reflection::CreateRPCCall(
3737 *builder, name__, request->serialized_location,
3738 response->serialized_location, attr__, docs__);
3739}
3740
3741bool RPCCall::Deserialize(Parser &parser, const reflection::RPCCall *call) {
3742 name = call->name()->str();
3743 if (!DeserializeAttributes(parser, call->attributes())) return false;
3744 DeserializeDoc(doc_comment, call->documentation());
3745 request = parser.structs_.Lookup(call->request()->name()->str());
3746 response = parser.structs_.Lookup(call->response()->name()->str());
3747 if (!request || !response) { return false; }
3748 return true;
3749}
3750
3751Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder,
3752 const Parser &parser) const {
3753 std::vector<Offset<reflection::RPCCall>> servicecall_offsets;
3754 for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) {
3755 servicecall_offsets.push_back((*it)->Serialize(builder, parser));
3756 }
3757 const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
3758 const auto name__ = builder->CreateString(qualified_name);
3759 const auto call__ = builder->CreateVector(servicecall_offsets);
3760 const auto attr__ = SerializeAttributes(builder, parser);
3761 const auto docs__ = parser.opts.binary_schema_comments
3762 ? builder->CreateVectorOfStrings(doc_comment)
3763 : 0;
3764 std::string decl_file_in_project = declaration_file ? *declaration_file : "";
3765 const auto file__ = builder->CreateSharedString(decl_file_in_project);
3766 return reflection::CreateService(*builder, name__, call__, attr__, docs__,
3767 file__);
3768}
3769
3770bool ServiceDef::Deserialize(Parser &parser,
3771 const reflection::Service *service) {
3772 name = parser.UnqualifiedName(service->name()->str());
3773 if (service->calls()) {
3774 for (uoffset_t i = 0; i < service->calls()->size(); ++i) {
3775 auto call = new RPCCall();
3776 if (!call->Deserialize(parser, service->calls()->Get(i)) ||
3777 calls.Add(call->name, call)) {
3778 delete call;
3779 return false;
3780 }
3781 }
3782 }
3783 if (!DeserializeAttributes(parser, service->attributes())) return false;
3784 DeserializeDoc(doc_comment, service->documentation());
3785 return true;
3786}
3787
3788Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder,
3789 const Parser &parser) const {
3790 std::vector<Offset<reflection::EnumVal>> enumval_offsets;
3791 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
3792 enumval_offsets.push_back((*it)->Serialize(builder, parser));
3793 }
3794 const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
3795 const auto name__ = builder->CreateString(qualified_name);
3796 const auto vals__ = builder->CreateVector(enumval_offsets);
3797 const auto type__ = underlying_type.Serialize(builder);
3798 const auto attr__ = SerializeAttributes(builder, parser);
3799 const auto docs__ = parser.opts.binary_schema_comments
3800 ? builder->CreateVectorOfStrings(doc_comment)
3801 : 0;
3802 std::string decl_file_in_project = declaration_file ? *declaration_file : "";
3803 const auto file__ = builder->CreateSharedString(decl_file_in_project);
3804 return reflection::CreateEnum(*builder, name__, vals__, is_union, type__,
3805 attr__, docs__, file__);
3806}
3807
3808bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) {
3809 name = parser.UnqualifiedName(_enum->name()->str());
3810 for (uoffset_t i = 0; i < _enum->values()->size(); ++i) {
3811 auto val = new EnumVal();
3812 if (!val->Deserialize(parser, _enum->values()->Get(i)) ||
3813 vals.Add(val->name, val)) {
3814 delete val;
3815 return false;
3816 }
3817 }
3818 is_union = _enum->is_union();
3819 if (!underlying_type.Deserialize(parser, _enum->underlying_type())) {
3820 return false;
3821 }
3822 if (!DeserializeAttributes(parser, _enum->attributes())) return false;
3823 DeserializeDoc(doc_comment, _enum->documentation());
3824 return true;
3825}
3826
3827Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
3828 const Parser &parser) const {
3829 auto name__ = builder->CreateString(name);
3830 auto type__ = union_type.Serialize(builder);
3831 auto docs__ = parser.opts.binary_schema_comments
3832 ? builder->CreateVectorOfStrings(doc_comment)
3833 : 0;
3834 return reflection::CreateEnumVal(*builder, name__, value, type__, docs__);
3835}
3836
3837bool EnumVal::Deserialize(const Parser &parser,
3838 const reflection::EnumVal *val) {
3839 name = val->name()->str();
3840 value = val->value();
3841 if (!union_type.Deserialize(parser, val->union_type())) return false;
3842 DeserializeDoc(doc_comment, val->documentation());
3843 return true;
3844}
3845
3846Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
3847 return reflection::CreateType(
3848 *builder, static_cast<reflection::BaseType>(base_type),
3849 static_cast<reflection::BaseType>(element),
3850 struct_def ? struct_def->index : (enum_def ? enum_def->index : -1),
3851 fixed_length, static_cast<uint32_t>(SizeOf(base_type)),
3852 static_cast<uint32_t>(SizeOf(element)));
3853}
3854
3855bool Type::Deserialize(const Parser &parser, const reflection::Type *type) {
3856 if (type == nullptr) return true;
3857 base_type = static_cast<BaseType>(type->base_type());
3858 element = static_cast<BaseType>(type->element());
3859 fixed_length = type->fixed_length();
3860 if (type->index() >= 0) {
3861 bool is_series = type->base_type() == reflection::Vector ||
3862 type->base_type() == reflection::Array;
3863 if (type->base_type() == reflection::Obj ||
3864 (is_series && type->element() == reflection::Obj)) {
3865 if (static_cast<size_t>(type->index()) < parser.structs_.vec.size()) {
3866 struct_def = parser.structs_.vec[type->index()];
3867 struct_def->refcount++;
3868 } else {
3869 return false;
3870 }
3871 } else {
3872 if (static_cast<size_t>(type->index()) < parser.enums_.vec.size()) {
3873 enum_def = parser.enums_.vec[type->index()];
3874 } else {
3875 return false;
3876 }
3877 }
3878 }
3879 return true;
3880}
3881
3882flatbuffers::Offset<
3883 flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
3884Definition::SerializeAttributes(FlatBufferBuilder *builder,
3885 const Parser &parser) const {
3886 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
3887 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
3888 auto it = parser.known_attributes_.find(kv->first);
3889 FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
3890 if (parser.opts.binary_schema_builtins || !it->second) {
3891 auto key = builder->CreateString(kv->first);
3892 auto val = builder->CreateString(kv->second->constant);
3893 attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
3894 }
3895 }
3896 if (attrs.size()) {
3897 return builder->CreateVectorOfSortedTables(&attrs);
3898 } else {
3899 return 0;
3900 }
3901}
3902
3903bool Definition::DeserializeAttributes(
3904 Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
3905 if (attrs == nullptr) return true;
3906 for (uoffset_t i = 0; i < attrs->size(); ++i) {
3907 auto kv = attrs->Get(i);
3908 auto value = new Value();
3909 if (kv->value()) { value->constant = kv->value()->str(); }
3910 if (attributes.Add(kv->key()->str(), value)) {
3911 delete value;
3912 return false;
3913 }
3914 parser.known_attributes_[kv->key()->str()];
3915 }
3916 return true;
3917}
3918
3919/************************************************************************/
3920/* DESERIALIZATION */
3921/************************************************************************/
3922bool Parser::Deserialize(const uint8_t *buf, const size_t size) {
3923 flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(buf), size);
3924 bool size_prefixed = false;
3925 if (!reflection::SchemaBufferHasIdentifier(buf)) {
3926 if (!flatbuffers::BufferHasIdentifier(buf, reflection::SchemaIdentifier(),
3927 true))
3928 return false;
3929 else
3930 size_prefixed = true;
3931 }
3932 auto verify_fn = size_prefixed ? &reflection::VerifySizePrefixedSchemaBuffer
3933 : &reflection::VerifySchemaBuffer;
3934 if (!verify_fn(verifier)) { return false; }
3935 auto schema = size_prefixed ? reflection::GetSizePrefixedSchema(buf)
3936 : reflection::GetSchema(buf);
3937 return Deserialize(schema);
3938}
3939
3940bool Parser::Deserialize(const reflection::Schema *schema) {
3941 file_identifier_ = schema->file_ident() ? schema->file_ident()->str() : "";
3942 file_extension_ = schema->file_ext() ? schema->file_ext()->str() : "";
3943 std::map<std::string, Namespace *> namespaces_index;
3944
3945 // Create defs without deserializing so references from fields to structs and
3946 // enums can be resolved.
3947 for (auto it = schema->objects()->begin(); it != schema->objects()->end();
3948 ++it) {
3949 auto struct_def = new StructDef();
3950 struct_def->bytesize = it->bytesize();
3951 struct_def->fixed = it->is_struct();
3952 struct_def->minalign = it->minalign();
3953 if (structs_.Add(it->name()->str(), struct_def)) {
3954 delete struct_def;
3955 return false;
3956 }
3957 auto type = new Type(BASE_TYPE_STRUCT, struct_def, nullptr);
3958 if (types_.Add(it->name()->str(), type)) {
3959 delete type;
3960 return false;
3961 }
3962 }
3963 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) {
3964 auto enum_def = new EnumDef();
3965 if (enums_.Add(it->name()->str(), enum_def)) {
3966 delete enum_def;
3967 return false;
3968 }
3969 auto type = new Type(BASE_TYPE_UNION, nullptr, enum_def);
3970 if (types_.Add(it->name()->str(), type)) {
3971 delete type;
3972 return false;
3973 }
3974 }
3975
3976 // Now fields can refer to structs and enums by index.
3977 for (auto it = schema->objects()->begin(); it != schema->objects()->end();
3978 ++it) {
3979 std::string qualified_name = it->name()->str();
3980 auto struct_def = structs_.Lookup(qualified_name);
3981 struct_def->defined_namespace =
3982 GetNamespace(qualified_name, namespaces_, namespaces_index);
3983 if (!struct_def->Deserialize(*this, *it)) { return false; }
3984 if (schema->root_table() == *it) { root_struct_def_ = struct_def; }
3985 }
3986 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) {
3987 std::string qualified_name = it->name()->str();
3988 auto enum_def = enums_.Lookup(qualified_name);
3989 enum_def->defined_namespace =
3990 GetNamespace(qualified_name, namespaces_, namespaces_index);
3991 if (!enum_def->Deserialize(*this, *it)) { return false; }
3992 }
3993
3994 if (schema->services()) {
3995 for (auto it = schema->services()->begin(); it != schema->services()->end();
3996 ++it) {
3997 std::string qualified_name = it->name()->str();
3998 auto service_def = new ServiceDef();
3999 service_def->defined_namespace =
4000 GetNamespace(qualified_name, namespaces_, namespaces_index);
4001 if (!service_def->Deserialize(*this, *it) ||
4002 services_.Add(qualified_name, service_def)) {
4003 delete service_def;
4004 return false;
4005 }
4006 }
4007 }
4008 advanced_features_ = schema->advanced_features();
4009
4010 if (schema->fbs_files())
4011 for (auto s = schema->fbs_files()->begin(); s != schema->fbs_files()->end();
4012 ++s) {
4013 for (auto f = s->included_filenames()->begin();
4014 f != s->included_filenames()->end(); ++f) {
4015 files_included_per_file_[s->filename()->str()].insert(f->str());
4016 }
4017 }
4018
4019 return true;
4020}
4021
4022std::string Parser::ConformTo(const Parser &base) {
4023 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) {
4024 auto &struct_def = **sit;
4025 auto qualified_name =
4026 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name);
4027 auto struct_def_base = base.LookupStruct(qualified_name);
4028 if (!struct_def_base) continue;
4029 for (auto fit = struct_def.fields.vec.begin();
4030 fit != struct_def.fields.vec.end(); ++fit) {
4031 auto &field = **fit;
4032 auto field_base = struct_def_base->fields.Lookup(field.name);
4033 if (field_base) {
4034 if (field.value.offset != field_base->value.offset)
4035 return "offsets differ for field: " + field.name;
4036 if (field.value.constant != field_base->value.constant)
4037 return "defaults differ for field: " + field.name;
4038 if (!EqualByName(field.value.type, field_base->value.type))
4039 return "types differ for field: " + field.name;
4040 } else {
4041 // Doesn't have to exist, deleting fields is fine.
4042 // But we should check if there is a field that has the same offset
4043 // but is incompatible (in the case of field renaming).
4044 for (auto fbit = struct_def_base->fields.vec.begin();
4045 fbit != struct_def_base->fields.vec.end(); ++fbit) {
4046 field_base = *fbit;
4047 if (field.value.offset == field_base->value.offset) {
4048 if (!EqualByName(field.value.type, field_base->value.type))
4049 return "field renamed to different type: " + field.name;
4050 break;
4051 }
4052 }
4053 }
4054 }
4055 }
4056 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) {
4057 auto &enum_def = **eit;
4058 auto qualified_name =
4059 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name);
4060 auto enum_def_base = base.enums_.Lookup(qualified_name);
4061 if (!enum_def_base) continue;
4062 for (auto evit = enum_def.Vals().begin(); evit != enum_def.Vals().end();
4063 ++evit) {
4064 auto &enum_val = **evit;
4065 auto enum_val_base = enum_def_base->Lookup(enum_val.name);
4066 if (enum_val_base) {
4067 if (enum_val != *enum_val_base)
4068 return "values differ for enum: " + enum_val.name;
4069 }
4070 }
4071 }
4072 return "";
4073}
4074
4075} // namespace flatbuffers
4076