1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author jiliang.ljl
17 * \date Feb 2021
18 * \brief implementation of protobuf helper
19 */
20
21#include "protobuf_helper.h"
22#include <ailego/encoding/base64.h>
23#include <ailego/encoding/json.h>
24#include "common/logger.h"
25
26using google::protobuf::Descriptor;
27using google::protobuf::EnumDescriptor;
28using google::protobuf::EnumValueDescriptor;
29using google::protobuf::FieldDescriptor;
30using google::protobuf::Message;
31using google::protobuf::Reflection;
32
33namespace proxima {
34namespace be {
35
36#define RETURN_ON_NULL(ptr, msg) \
37 if (!ptr) { \
38 LOG_ERROR(msg); \
39 return false; \
40 }
41
42static bool ParseMessage(const Message &msg, ailego::JsonValue *json,
43 const ProtobufHelper::PrintOptions &options);
44static bool ParseJson(const ailego::JsonObject &json,
45 const ProtobufHelper::JsonParseOptions &options,
46 google::protobuf::Message *msg);
47
48static bool FieldToJson(const Message &msg, const FieldDescriptor *field,
49 const ProtobufHelper::PrintOptions &options,
50 ailego::JsonValue *json) {
51 const Reflection *ref = msg.GetReflection();
52 bool repeated = field->is_repeated();
53
54 ailego::JsonArray json_array;
55 size_t array_size = 0;
56 if (repeated) {
57 array_size = ref->FieldSize(msg, field);
58 json_array.reserve(array_size);
59 }
60 switch (field->cpp_type()) {
61 case FieldDescriptor::CPPTYPE_DOUBLE:
62 if (repeated) {
63 for (size_t i = 0; i != array_size; ++i) {
64 double value = ref->GetRepeatedDouble(msg, field, i);
65 ailego::JsonValue v(value);
66 json_array.push(v);
67 }
68 } else {
69 *json = ref->GetDouble(msg, field);
70 }
71 break;
72 case FieldDescriptor::CPPTYPE_FLOAT:
73 if (repeated) {
74 for (size_t i = 0; i != array_size; ++i) {
75 float value = ref->GetRepeatedFloat(msg, field, i);
76 ailego::JsonValue v(value);
77 json_array.push(v);
78 }
79 } else {
80 *json = ref->GetFloat(msg, field);
81 }
82 break;
83 case FieldDescriptor::CPPTYPE_INT64:
84 if (repeated) {
85 for (size_t i = 0; i != array_size; ++i) {
86 int64_t value = ref->GetRepeatedInt64(msg, field, i);
87 ailego::JsonValue v(std::to_string(value));
88 json_array.push(v);
89 }
90 } else {
91 *json = std::to_string(ref->GetInt64(msg, field));
92 }
93 break;
94 case FieldDescriptor::CPPTYPE_UINT64:
95 if (repeated) {
96 for (size_t i = 0; i != array_size; ++i) {
97 uint64_t value = ref->GetRepeatedUInt64(msg, field, i);
98 ailego::JsonValue v(std::to_string(value));
99 json_array.push(v);
100 }
101 } else {
102 *json = std::to_string(ref->GetUInt64(msg, field));
103 }
104 break;
105 case FieldDescriptor::CPPTYPE_INT32:
106 if (repeated) {
107 for (size_t i = 0; i != array_size; ++i) {
108 int32_t value = ref->GetRepeatedInt32(msg, field, i);
109 ailego::JsonValue v(value);
110 json_array.push(v);
111 }
112 } else {
113 *json = ref->GetInt32(msg, field);
114 }
115 break;
116 case FieldDescriptor::CPPTYPE_UINT32:
117 if (repeated) {
118 for (size_t i = 0; i != array_size; ++i) {
119 uint32_t value = ref->GetRepeatedUInt32(msg, field, i);
120 ailego::JsonValue v(value);
121 json_array.push(v);
122 }
123 } else {
124 *json = ref->GetUInt32(msg, field);
125 }
126 break;
127 case FieldDescriptor::CPPTYPE_BOOL:
128 if (repeated) {
129 for (size_t i = 0; i != array_size; ++i) {
130 bool value = ref->GetRepeatedBool(msg, field, i);
131 ailego::JsonValue v(value);
132 json_array.push(v);
133 }
134 } else {
135 *json = ref->GetBool(msg, field);
136 }
137 break;
138 case FieldDescriptor::CPPTYPE_STRING: {
139 bool is_binary = field->type() == FieldDescriptor::TYPE_BYTES;
140 if (repeated) {
141 for (size_t i = 0; i != array_size; ++i) {
142 std::string value = ref->GetRepeatedString(msg, field, i);
143 if (is_binary) {
144 value = ailego::Base64::Encode(value);
145 }
146 ailego::JsonValue v(ailego::JsonString(value).encode());
147 json_array.push(v);
148 }
149 } else {
150 std::string value = ref->GetString(msg, field);
151 if (is_binary) {
152 value = ailego::Base64::Encode(value);
153 }
154 *json = ailego::JsonString(value).encode();
155 }
156 break;
157 }
158 case FieldDescriptor::CPPTYPE_MESSAGE:
159 if (repeated) {
160 for (size_t i = 0; i != array_size; ++i) {
161 const Message &value = ref->GetRepeatedMessage(msg, field, i);
162 ailego::JsonValue v;
163 bool ret = ParseMessage(value, &v, options);
164 if (!ret) {
165 return ret;
166 }
167 json_array.push(v);
168 }
169 } else {
170 const Message &value = ref->GetMessage(msg, field);
171 return ParseMessage(value, json, options);
172 }
173 break;
174 case FieldDescriptor::CPPTYPE_ENUM:
175 if (repeated) {
176 for (size_t i = 0; i != array_size; ++i) {
177 const EnumValueDescriptor *value =
178 ref->GetRepeatedEnum(msg, field, i);
179 ailego::JsonValue v(value->name());
180 json_array.push(v);
181 }
182 } else {
183 auto enum_desp = ref->GetEnum(msg, field);
184 *json = enum_desp->name();
185 }
186 break;
187 default:
188 LOG_ERROR("Unexpected type: %d", static_cast<int>(field->cpp_type()));
189 return false;
190 }
191 if (repeated) {
192 *json = json_array;
193 }
194 return true;
195}
196
197#define RETURN_ON_INVALID_JSON(ERROR_MSG) \
198 LOG_ERROR("Json is " ERROR_MSG ". json[%s] field[%s]", \
199 json.as_json_string().as_stl_string().c_str(), \
200 field->full_name().c_str()); \
201 return false
202
203static bool JsonToField(const ailego::JsonValue &json,
204 const ProtobufHelper::JsonParseOptions &options,
205 google::protobuf::Message *msg,
206 const FieldDescriptor *field) {
207 const Reflection *ref = msg->GetReflection();
208 bool repeated = field->is_repeated();
209 switch (field->cpp_type()) {
210#define CONVERT(type, ctype, checkfun, asfun, sfunc, afunc, error) \
211 case FieldDescriptor::type: { \
212 if (!json.checkfun()) { \
213 RETURN_ON_INVALID_JSON(error); \
214 } \
215 if (repeated) { \
216 ref->afunc(msg, field, static_cast<ctype>(json.asfun())); \
217 } else { \
218 ref->sfunc(msg, field, static_cast<ctype>(json.asfun())); \
219 } \
220 break; \
221 }
222
223#define CONVERT_FLOAT(type, ctype, sfunc, afunc) \
224 case FieldDescriptor::type: { \
225 ctype val = 0.0; \
226 if (json.is_integer()) { \
227 val = static_cast<ctype>(json.as_integer()); \
228 } else if (json.is_float()) { \
229 val = static_cast<ctype>(json.as_float()); \
230 } else if (json.is_string()) { \
231 const char *value = json.as_c_string(); \
232 if (strncmp(value, "NaN", 3) == 0) { \
233 val = std::numeric_limits<ctype>::quiet_NaN(); \
234 } else if (strncmp(value, "Infinity", 8) == 0) { \
235 val = std::numeric_limits<ctype>::infinity(); \
236 } else if (strncmp(value, "-Infinity", 9) == 0) { \
237 val = -std::numeric_limits<ctype>::infinity(); \
238 } else { \
239 RETURN_ON_INVALID_JSON( \
240 "float string only allow Nan, Infinity, -Infinity"); \
241 } \
242 } else { \
243 RETURN_ON_INVALID_JSON("not integer, float or string"); \
244 } \
245 if (repeated) { \
246 ref->afunc(msg, field, val); \
247 } else { \
248 ref->sfunc(msg, field, val); \
249 } \
250 break; \
251 }
252
253#define CONVERT_LONG(type, ctype, sfunc, afunc, strtofun) \
254 case FieldDescriptor::type: { \
255 ctype val = 0; \
256 if (json.is_integer()) { \
257 val = static_cast<ctype>(json.as_integer()); \
258 } else if (json.is_string()) { \
259 val = static_cast<ctype>(strtofun(json.as_c_string(), nullptr, 10)); \
260 } else { \
261 RETURN_ON_INVALID_JSON("not integer or string"); \
262 } \
263 if (repeated) { \
264 ref->afunc(msg, field, val); \
265 } else { \
266 ref->sfunc(msg, field, val); \
267 } \
268 break; \
269 }
270
271
272 CONVERT(CPPTYPE_INT32, int32_t, is_integer, as_integer, SetInt32, AddInt32,
273 "not number");
274
275 CONVERT(CPPTYPE_UINT32, uint32_t, is_integer, as_integer, SetUInt32,
276 AddUInt32, "not number");
277
278 CONVERT_FLOAT(CPPTYPE_FLOAT, float, SetFloat, AddFloat);
279
280 CONVERT_FLOAT(CPPTYPE_DOUBLE, double, SetDouble, AddDouble);
281
282 CONVERT(CPPTYPE_BOOL, bool, is_boolean, as_bool, SetBool, AddBool,
283 "not bool");
284
285 CONVERT_LONG(CPPTYPE_INT64, int64_t, SetInt64, AddInt64, strtoll);
286
287 CONVERT_LONG(CPPTYPE_UINT64, uint64_t, SetUInt64, AddUInt64, strtoull);
288
289 case FieldDescriptor::CPPTYPE_STRING: {
290 if (!json.is_string()) {
291 RETURN_ON_INVALID_JSON("not string");
292 }
293 const char *value = json.as_c_string();
294 bool is_bytes = field->type() == FieldDescriptor::TYPE_BYTES;
295 if (repeated) {
296 ref->AddString(
297 msg, field,
298 is_bytes ? ailego::Base64::Decode(value) : json.as_stl_string());
299 } else {
300 ref->SetString(
301 msg, field,
302 is_bytes ? ailego::Base64::Decode(value) : json.as_stl_string());
303 }
304 break;
305 }
306 case FieldDescriptor::CPPTYPE_MESSAGE: {
307 if (!json.is_object()) {
308 RETURN_ON_INVALID_JSON("not object");
309 }
310 Message *mf = (repeated) ? ref->AddMessage(msg, field)
311 : ref->MutableMessage(msg, field);
312 return ParseJson(json.as_object(), options, mf);
313 }
314 case FieldDescriptor::CPPTYPE_ENUM: {
315 const EnumDescriptor *ed = field->enum_type();
316 const EnumValueDescriptor *ev = 0;
317 if (json.is_integer()) {
318 ev = ed->FindValueByNumber(json.as_integer());
319 } else if (json.is_string()) {
320 ev = ed->FindValueByName(json.as_stl_string());
321 } else {
322 RETURN_ON_INVALID_JSON("not integer or string");
323 }
324 if (!ev) {
325 RETURN_ON_INVALID_JSON("Enum value not found");
326 }
327 if (repeated) {
328 ref->AddEnum(msg, field, ev);
329 } else {
330 ref->SetEnum(msg, field, ev);
331 }
332 break;
333 }
334 default:
335 LOG_ERROR("Unexpected type: %d", static_cast<int>(field->cpp_type()));
336 return false;
337 }
338 return true;
339}
340
341static bool ParseJson(const ailego::JsonObject &json,
342 const ProtobufHelper::JsonParseOptions &options,
343 google::protobuf::Message *msg) {
344 const Descriptor *d = msg->GetDescriptor();
345 const Reflection *ref = msg->GetReflection();
346 RETURN_ON_NULL(d, "null descriptor");
347 RETURN_ON_NULL(ref, "null reflection");
348 for (auto it = json.cbegin(); it != json.cend(); ++it) {
349 const char *name = it->key().c_str();
350 const FieldDescriptor *field = d->FindFieldByName(name);
351 if (!field) {
352 field = d->FindFieldByCamelcaseName(name);
353 if (!field) {
354 field = ref->FindKnownExtensionByName(name);
355 }
356 }
357 if (!field) {
358 if (options.ignore_unknown_fields) {
359 continue;
360 }
361 LOG_ERROR("Unknown field in protobuf. json key[%s] message[%s]", name,
362 d->full_name().c_str());
363 return false;
364 }
365 auto &value = it->value();
366
367 if (value.is_null()) {
368 ref->ClearField(msg, field);
369 // no need check required as protobuf3 is all optional
370 continue;
371 }
372 if (field->is_repeated()) {
373 if (!value.is_array()) {
374 LOG_ERROR("Input json is not array. key[%s] value[%s] field[%s]", name,
375 value.as_json_string().as_stl_string().c_str(),
376 field->full_name().c_str());
377 return false;
378 }
379 ref->ClearField(msg, field);
380 auto &array = value.as_array();
381 for (auto ait = array.cbegin(); ait != array.cend(); ++ait) {
382 if (!JsonToField(*ait, options, msg, field)) {
383 return false;
384 }
385 }
386 } else {
387 if (!JsonToField(value, options, msg, field)) {
388 return false;
389 }
390 }
391 }
392 return true;
393}
394
395static bool ParseMessage(const Message &msg, ailego::JsonValue *json,
396 const ProtobufHelper::PrintOptions &options) {
397 const Descriptor *d = msg.GetDescriptor();
398 RETURN_ON_NULL(d, "null descriptor")
399 const Reflection *ref = msg.GetReflection();
400 RETURN_ON_NULL(ref, "null reflection")
401 size_t count = d->field_count();
402 ailego::JsonObject obj;
403 for (size_t i = 0; i != count; ++i) {
404 const FieldDescriptor *field = d->field(i);
405 RETURN_ON_NULL(field, "null field descriptor")
406 if (field->is_optional() && !ref->HasField(msg, field)) {
407 // do not print empty message
408 if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
409 continue;
410 }
411 if (!options.always_print_primitive_fields) {
412 continue;
413 }
414 }
415 auto oneof_des = field->containing_oneof();
416 if (oneof_des) {
417 bool has_oneof = false;
418 for (int fi = 0; fi < oneof_des->field_count(); fi++) {
419 if (ref->HasField(msg, oneof_des->field(fi))) {
420 has_oneof = true;
421 break;
422 }
423 }
424 if (!has_oneof || !ref->HasField(msg, field)) {
425 continue;
426 }
427 }
428 ailego::JsonValue field_json;
429 bool ret = FieldToJson(msg, field, options, &field_json);
430 if (!ret) {
431 return ret;
432 }
433 ailego::JsonString field_name(field->name());
434 obj.set(field_name.encode(), field_json);
435 }
436 *json = obj;
437 return true;
438}
439
440bool ProtobufHelper::MessageToJson(const google::protobuf::Message &message,
441 const ProtobufHelper::PrintOptions &options,
442 std::string *json) {
443 ailego::JsonValue root;
444 bool ret = ParseMessage(message, &root, options);
445 if (!ret) {
446 return ret;
447 }
448 *json = root.as_json_string().as_stl_string();
449 return true;
450}
451
452bool ProtobufHelper::JsonToMessage(
453 const std::string &json, const ProtobufHelper::JsonParseOptions &options,
454 google::protobuf::Message *message) {
455 ailego::JsonValue node;
456 if (!node.parse(json.c_str())) {
457 LOG_ERROR("Parse json value failed. json[%s]", json.c_str());
458 return false;
459 }
460 if (!node.is_object()) {
461 LOG_ERROR("Json is not object. json[%s]", json.c_str());
462 return false;
463 }
464 return ParseJson(node.as_object(), options, message);
465}
466
467#undef RETURN_ON_NULL
468#undef RETURN_ON_INVALID_JSON
469#undef CONVERT
470#undef CONVERT_FLOAT
471#undef CONVERT_LONG
472
473} // namespace be
474} // namespace proxima
475