1 | /** |
2 | * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | |
16 | * \author Hongqing.hu |
17 | * \date Oct 2020 |
18 | * \brief Proto converter interface implementation for bilin engine |
19 | */ |
20 | |
21 | #include "proto_converter.h" |
22 | #include "common/transformer.h" |
23 | #include "common/types_helper.h" |
24 | |
25 | namespace proxima { |
26 | namespace be { |
27 | namespace server { |
28 | |
29 | int ProtoConverter::ConvertIndexData( |
30 | const std::string &index_value, const meta::ColumnMeta &column_meta, |
31 | const proto::WriteRequest::IndexColumnMeta &proto_meta, bool is_bytes, |
32 | index::ColumnData *column_data) { |
33 | IndexTypes index_type = column_meta.index_type(); |
34 | if (index_type == IndexTypes::PROXIMA_GRAPH_INDEX) { |
35 | column_data->column_name = column_meta.name(); |
36 | column_data->data_type = column_meta.data_type(); |
37 | column_data->dimension = column_meta.dimension(); |
38 | if (is_bytes) { |
39 | int ret = ParseBytesIndexColumnValue(index_value, column_meta, proto_meta, |
40 | &(column_data->data)); |
41 | if (ret != 0) { |
42 | LOG_ERROR("Set index column value failed. column[%s]." , |
43 | column_meta.name().c_str()); |
44 | return ret; |
45 | } |
46 | } else { |
47 | int ret = ParseJsonIndexColumnValue(index_value, column_meta, proto_meta, |
48 | &(column_data->data)); |
49 | if (ret != 0) { |
50 | LOG_ERROR("Parse index column value failed. column[%s]." , |
51 | column_meta.name().c_str()); |
52 | return ret; |
53 | } |
54 | } |
55 | } else { |
56 | LOG_ERROR("Invalid index type %u." , (uint32_t)index_type); |
57 | return ErrorCode_InvalidIndexType; |
58 | } |
59 | |
60 | return 0; |
61 | } |
62 | |
63 | int ProtoConverter::ParseBytesIndexColumnValue( |
64 | const std::string &column_value, const meta::ColumnMeta &meta, |
65 | const proto::WriteRequest::IndexColumnMeta &proto_meta, |
66 | std::string *serialized_value) { |
67 | DataTypes in_data_type = DataTypeCodeBook::Get(proto_meta.data_type()); |
68 | if (Transformer::NeedTransform(in_data_type, meta.data_type())) { |
69 | std::string dst_column_value; |
70 | int ret = Transformer::Transform(in_data_type, column_value, |
71 | meta.data_type(), &dst_column_value); |
72 | if (ret != 0) { |
73 | LOG_ERROR("Transform vector failed. in[%d] out[%d]" , (int)in_data_type, |
74 | (int)meta.data_type()); |
75 | return ret; |
76 | } |
77 | return CopyBytesIndexColumnValue(dst_column_value, meta, serialized_value); |
78 | } else { |
79 | return CopyBytesIndexColumnValue(column_value, meta, serialized_value); |
80 | } |
81 | } |
82 | |
83 | int ProtoConverter::CopyBytesIndexColumnValue(const std::string &column_value, |
84 | const meta::ColumnMeta &meta, |
85 | std::string *serialized_value) { |
86 | int ret = 0; |
87 | switch (meta.data_type()) { |
88 | case DataTypes::VECTOR_FP32: |
89 | ret = ValidateTypedIndexColumnValue<float>(column_value, meta); |
90 | break; |
91 | case DataTypes::VECTOR_FP16: |
92 | ret = ValidateTypedIndexColumnValue<float>(column_value, meta); |
93 | break; |
94 | case DataTypes::VECTOR_INT16: |
95 | ret = ValidateTypedIndexColumnValue<int16_t>(column_value, meta); |
96 | break; |
97 | case DataTypes::VECTOR_INT8: |
98 | ret = ValidateTypedIndexColumnValue<int8_t>(column_value, meta); |
99 | break; |
100 | case DataTypes::VECTOR_INT4: |
101 | ret = ValidateTypedIndexColumnValue<int8_t>(column_value, meta); |
102 | break; |
103 | case DataTypes::VECTOR_BINARY32: |
104 | ret = ValidateTypedIndexColumnValue<uint32_t>(column_value, meta); |
105 | break; |
106 | case DataTypes::VECTOR_BINARY64: |
107 | ret = ValidateTypedIndexColumnValue<uint64_t>(column_value, meta); |
108 | break; |
109 | default: |
110 | LOG_ERROR("Invalid data type %u." , (uint32_t)meta.data_type()); |
111 | return ErrorCode_InvalidDataType; |
112 | } |
113 | |
114 | if (ret != 0) { |
115 | LOG_ERROR("Index value is invalid" ); |
116 | return ret; |
117 | } |
118 | |
119 | *serialized_value = column_value; |
120 | |
121 | return 0; |
122 | } |
123 | |
124 | int ProtoConverter::ParseJsonIndexColumnValue( |
125 | const std::string &column_value, const meta::ColumnMeta &meta, |
126 | const proto::WriteRequest::IndexColumnMeta &proto_meta, |
127 | std::string *serialized_value) { |
128 | DataTypes src_data_type = DataTypeCodeBook::Get(proto_meta.data_type()); |
129 | int ret = Transformer::SupportTransform(src_data_type, meta.data_type()); |
130 | if (ret != 0) { |
131 | LOG_ERROR("Not support current transform. src[%d] dst[%d]" , |
132 | (int)src_data_type, (int)meta.data_type()); |
133 | return ret; |
134 | } |
135 | switch (src_data_type) { |
136 | case DataTypes::VECTOR_FP32: |
137 | ret = ParseTypedIndexColumnValue<float>(column_value, meta, |
138 | serialized_value); |
139 | break; |
140 | case DataTypes::VECTOR_FP16: |
141 | ret = ParseTypedIndexColumnValue<float>(column_value, meta, |
142 | serialized_value); |
143 | break; |
144 | case DataTypes::VECTOR_INT16: |
145 | ret = ParseTypedIndexColumnValue<int16_t>(column_value, meta, |
146 | serialized_value); |
147 | break; |
148 | case DataTypes::VECTOR_INT8: |
149 | ret = ParseTypedIndexColumnValue<int8_t>(column_value, meta, |
150 | serialized_value); |
151 | break; |
152 | case DataTypes::VECTOR_INT4: |
153 | ret = ParseTypedIndexColumnValue<int8_t>(column_value, meta, |
154 | serialized_value); |
155 | break; |
156 | case DataTypes::VECTOR_BINARY32: |
157 | ret = ParseTypedIndexColumnValue<uint32_t>(column_value, meta, |
158 | serialized_value); |
159 | break; |
160 | case DataTypes::VECTOR_BINARY64: |
161 | ret = ParseTypedIndexColumnValue<uint64_t>(column_value, meta, |
162 | serialized_value); |
163 | break; |
164 | default: |
165 | LOG_ERROR("Unsupported data type %u." , (uint32_t)src_data_type); |
166 | return ErrorCode_InvalidDataType; |
167 | } |
168 | |
169 | return ret; |
170 | } |
171 | |
172 | } // end namespace server |
173 | } // namespace be |
174 | } // end namespace proxima |
175 | |