1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author Hongqing.hu
17 * \date Oct 2020
18 * \brief Proto converter interface implementation for bilin engine
19 */
20
21#include "proto_converter.h"
22#include "common/transformer.h"
23#include "common/types_helper.h"
24
25namespace proxima {
26namespace be {
27namespace server {
28
29int ProtoConverter::ConvertIndexData(
30 const std::string &index_value, const meta::ColumnMeta &column_meta,
31 const proto::WriteRequest::IndexColumnMeta &proto_meta, bool is_bytes,
32 index::ColumnData *column_data) {
33 IndexTypes index_type = column_meta.index_type();
34 if (index_type == IndexTypes::PROXIMA_GRAPH_INDEX) {
35 column_data->column_name = column_meta.name();
36 column_data->data_type = column_meta.data_type();
37 column_data->dimension = column_meta.dimension();
38 if (is_bytes) {
39 int ret = ParseBytesIndexColumnValue(index_value, column_meta, proto_meta,
40 &(column_data->data));
41 if (ret != 0) {
42 LOG_ERROR("Set index column value failed. column[%s].",
43 column_meta.name().c_str());
44 return ret;
45 }
46 } else {
47 int ret = ParseJsonIndexColumnValue(index_value, column_meta, proto_meta,
48 &(column_data->data));
49 if (ret != 0) {
50 LOG_ERROR("Parse index column value failed. column[%s].",
51 column_meta.name().c_str());
52 return ret;
53 }
54 }
55 } else {
56 LOG_ERROR("Invalid index type %u.", (uint32_t)index_type);
57 return ErrorCode_InvalidIndexType;
58 }
59
60 return 0;
61}
62
63int ProtoConverter::ParseBytesIndexColumnValue(
64 const std::string &column_value, const meta::ColumnMeta &meta,
65 const proto::WriteRequest::IndexColumnMeta &proto_meta,
66 std::string *serialized_value) {
67 DataTypes in_data_type = DataTypeCodeBook::Get(proto_meta.data_type());
68 if (Transformer::NeedTransform(in_data_type, meta.data_type())) {
69 std::string dst_column_value;
70 int ret = Transformer::Transform(in_data_type, column_value,
71 meta.data_type(), &dst_column_value);
72 if (ret != 0) {
73 LOG_ERROR("Transform vector failed. in[%d] out[%d]", (int)in_data_type,
74 (int)meta.data_type());
75 return ret;
76 }
77 return CopyBytesIndexColumnValue(dst_column_value, meta, serialized_value);
78 } else {
79 return CopyBytesIndexColumnValue(column_value, meta, serialized_value);
80 }
81}
82
83int ProtoConverter::CopyBytesIndexColumnValue(const std::string &column_value,
84 const meta::ColumnMeta &meta,
85 std::string *serialized_value) {
86 int ret = 0;
87 switch (meta.data_type()) {
88 case DataTypes::VECTOR_FP32:
89 ret = ValidateTypedIndexColumnValue<float>(column_value, meta);
90 break;
91 case DataTypes::VECTOR_FP16:
92 ret = ValidateTypedIndexColumnValue<float>(column_value, meta);
93 break;
94 case DataTypes::VECTOR_INT16:
95 ret = ValidateTypedIndexColumnValue<int16_t>(column_value, meta);
96 break;
97 case DataTypes::VECTOR_INT8:
98 ret = ValidateTypedIndexColumnValue<int8_t>(column_value, meta);
99 break;
100 case DataTypes::VECTOR_INT4:
101 ret = ValidateTypedIndexColumnValue<int8_t>(column_value, meta);
102 break;
103 case DataTypes::VECTOR_BINARY32:
104 ret = ValidateTypedIndexColumnValue<uint32_t>(column_value, meta);
105 break;
106 case DataTypes::VECTOR_BINARY64:
107 ret = ValidateTypedIndexColumnValue<uint64_t>(column_value, meta);
108 break;
109 default:
110 LOG_ERROR("Invalid data type %u.", (uint32_t)meta.data_type());
111 return ErrorCode_InvalidDataType;
112 }
113
114 if (ret != 0) {
115 LOG_ERROR("Index value is invalid");
116 return ret;
117 }
118
119 *serialized_value = column_value;
120
121 return 0;
122}
123
124int ProtoConverter::ParseJsonIndexColumnValue(
125 const std::string &column_value, const meta::ColumnMeta &meta,
126 const proto::WriteRequest::IndexColumnMeta &proto_meta,
127 std::string *serialized_value) {
128 DataTypes src_data_type = DataTypeCodeBook::Get(proto_meta.data_type());
129 int ret = Transformer::SupportTransform(src_data_type, meta.data_type());
130 if (ret != 0) {
131 LOG_ERROR("Not support current transform. src[%d] dst[%d]",
132 (int)src_data_type, (int)meta.data_type());
133 return ret;
134 }
135 switch (src_data_type) {
136 case DataTypes::VECTOR_FP32:
137 ret = ParseTypedIndexColumnValue<float>(column_value, meta,
138 serialized_value);
139 break;
140 case DataTypes::VECTOR_FP16:
141 ret = ParseTypedIndexColumnValue<float>(column_value, meta,
142 serialized_value);
143 break;
144 case DataTypes::VECTOR_INT16:
145 ret = ParseTypedIndexColumnValue<int16_t>(column_value, meta,
146 serialized_value);
147 break;
148 case DataTypes::VECTOR_INT8:
149 ret = ParseTypedIndexColumnValue<int8_t>(column_value, meta,
150 serialized_value);
151 break;
152 case DataTypes::VECTOR_INT4:
153 ret = ParseTypedIndexColumnValue<int8_t>(column_value, meta,
154 serialized_value);
155 break;
156 case DataTypes::VECTOR_BINARY32:
157 ret = ParseTypedIndexColumnValue<uint32_t>(column_value, meta,
158 serialized_value);
159 break;
160 case DataTypes::VECTOR_BINARY64:
161 ret = ParseTypedIndexColumnValue<uint64_t>(column_value, meta,
162 serialized_value);
163 break;
164 default:
165 LOG_ERROR("Unsupported data type %u.", (uint32_t)src_data_type);
166 return ErrorCode_InvalidDataType;
167 }
168
169 return ret;
170}
171
172} // end namespace server
173} // namespace be
174} // end namespace proxima
175