1 | /** |
2 | * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | |
16 | * \author Jiliang.ljl |
17 | * \date Mar 2021 |
18 | * \brief |
19 | * \detail |
20 | */ |
21 | |
22 | #include "admin_proto_converter.h" |
23 | #include "common/types_helper.h" |
24 | namespace proxima { |
25 | namespace be { |
26 | namespace admin { |
27 | |
28 | namespace { |
29 | |
30 | //! Status Code Book |
31 | struct StatusCodeBook { |
32 | //! Convert C++ CollectionStatus to protobuf CollectionStatus |
33 | static proto::CollectionInfo::CollectionStatus Get( |
34 | meta::CollectionStatus status) { |
35 | switch (status) { |
36 | case meta::CollectionStatus::INITIALIZED: |
37 | return proto::CollectionInfo_CollectionStatus_CS_INITIALIZED; |
38 | case meta::CollectionStatus::SERVING: |
39 | return proto::CollectionInfo_CollectionStatus_CS_SERVING; |
40 | case meta::CollectionStatus::DROPPED: |
41 | return proto::CollectionInfo_CollectionStatus_CS_DROPPED; |
42 | } |
43 | return proto::CollectionInfo_CollectionStatus_CS_INITIALIZED; |
44 | } |
45 | }; |
46 | |
47 | //! Index Type Codebook |
48 | struct IndexTypeCodeBook { |
49 | //! convert protobuf IndexType to C++ IndexTypes |
50 | static IndexTypes Get(be::proto::IndexType type) { |
51 | switch (type) { |
52 | case be::proto::IT_PROXIMA_GRAPH_INDEX: |
53 | return IndexTypes::PROXIMA_GRAPH_INDEX; |
54 | default: |
55 | break; |
56 | } |
57 | return IndexTypes::UNDEFINED; |
58 | } |
59 | |
60 | //! Convert C++ IndexTypes to protobuf IndexType |
61 | static be::proto::IndexType Get(IndexTypes type) { |
62 | switch (type) { |
63 | case IndexTypes::PROXIMA_GRAPH_INDEX: |
64 | return be::proto::IT_PROXIMA_GRAPH_INDEX; |
65 | default: |
66 | break; |
67 | } |
68 | return be::proto::IT_UNDEFINED; |
69 | } |
70 | }; |
71 | |
72 | } // namespace |
73 | |
74 | void AdminProtoConverter::PBToColumnMeta( |
75 | const proto::CollectionConfig::IndexColumnParam *request, |
76 | meta::ColumnMeta *column) { |
77 | column->set_name(request->column_name()); |
78 | column->set_index_type(IndexTypeCodeBook::Get(request->index_type())); |
79 | column->set_data_type(DataTypeCodeBook::Get(request->data_type())); |
80 | |
81 | IndexParamsHelper::SerializeToParams(request->extra_params(), |
82 | column->mutable_parameters()); |
83 | column->set_dimension(request->dimension()); |
84 | } |
85 | |
86 | int AdminProtoConverter::PBToCollectionBase( |
87 | const proto::CollectionConfig &request, meta::CollectionBase *param) { |
88 | param->mutable_name()->assign(request.collection_name()); |
89 | if (request.has_repository_config()) { |
90 | const auto &repo_config = request.repository_config(); |
91 | switch (repo_config.repository_type()) { |
92 | case proto:: |
93 | CollectionConfig_RepositoryConfig_RepositoryType_RT_DATABASE: { |
94 | auto &database = repo_config.database(); |
95 | auto db_repo = std::make_shared<meta::DatabaseRepositoryMeta>(); |
96 | db_repo->set_connection(database.connection_uri()); |
97 | db_repo->set_table_name(database.table_name()); |
98 | db_repo->set_password(database.password()); |
99 | db_repo->set_user(database.user()); |
100 | db_repo->set_name(repo_config.repository_name()); |
101 | db_repo->set_type(meta::RepositoryTypes::DATABASE); |
102 | param->set_repository(db_repo); |
103 | break; |
104 | } |
105 | default: |
106 | LOG_ERROR("Invalid repository type. repository_type[%d] collection[%s]" , |
107 | static_cast<int>(repo_config.repository_type()), |
108 | request.collection_name().c_str()); |
109 | return PROXIMA_BE_ERROR_CODE(InvalidRepositoryType); |
110 | } |
111 | } |
112 | // Should we recommend some of default value for users |
113 | param->set_max_docs_per_segment(request.max_docs_per_segment()); |
114 | // Serialize forward_columns |
115 | param->mutable_forward_columns()->insert( |
116 | param->forward_columns().begin(), request.forward_column_names().begin(), |
117 | request.forward_column_names().end()); |
118 | int columns = request.index_column_params_size(); |
119 | while (columns--) { |
120 | auto column = std::make_shared<meta::ColumnMeta>(); |
121 | PBToColumnMeta(&request.index_column_params(columns), column.get()); |
122 | param->append(std::move(column)); |
123 | } |
124 | return 0; |
125 | } |
126 | |
127 | void AdminProtoConverter::ColumnMetaToPB( |
128 | const meta::ColumnMetaPtr &column, |
129 | proto::CollectionConfig::IndexColumnParam *param) { |
130 | param->set_column_name(column->name()); |
131 | param->set_index_type(IndexTypeCodeBook::Get(column->index_type())); |
132 | param->set_data_type(DataTypeCodeBook::Get(column->data_type())); |
133 | |
134 | be::IndexParamsHelper::SerializeToPB(column->parameters(), |
135 | param->mutable_extra_params()); |
136 | param->set_dimension(column->dimension()); |
137 | } |
138 | |
139 | void AdminProtoConverter::RepositoryToPB( |
140 | std::shared_ptr<meta::RepositoryBase> repo, |
141 | proto::CollectionConfig *config) { |
142 | if (!repo) { |
143 | return; |
144 | } |
145 | auto *repo_config = config->mutable_repository_config(); |
146 | repo_config->set_repository_name(repo->name()); |
147 | switch (repo->type()) { |
148 | case meta::RepositoryTypes::DATABASE: { |
149 | auto db_repo = |
150 | std::dynamic_pointer_cast<meta::DatabaseRepositoryMeta>(repo); |
151 | if (!db_repo) { |
152 | LOG_ERROR("Mismatched repository. type[%d] name[%s]" , |
153 | static_cast<int>(repo->type()), repo->name().c_str()); |
154 | return; |
155 | } |
156 | repo_config->set_repository_type( |
157 | proto::CollectionConfig_RepositoryConfig_RepositoryType_RT_DATABASE); |
158 | auto *db = repo_config->mutable_database(); |
159 | db->set_connection_uri(db_repo->connection()); |
160 | db->set_table_name(db_repo->table_name()); |
161 | db->set_user(db_repo->user()); |
162 | db->set_password(db_repo->password()); |
163 | break; |
164 | } |
165 | |
166 | default: |
167 | LOG_FATAL("Unexpected repository type. type[%d]" , |
168 | static_cast<int>(repo->type())); |
169 | } |
170 | } |
171 | |
172 | void AdminProtoConverter::CollectionMetaToPB( |
173 | const meta::CollectionMeta &collection, proto::CollectionInfo *info) { |
174 | info->set_status(StatusCodeBook::Get(collection.status())); |
175 | info->set_uuid(collection.uid()); |
176 | auto *config = info->mutable_config(); |
177 | config->set_collection_name(collection.name()); |
178 | config->set_max_docs_per_segment(collection.max_docs_per_segment()); |
179 | |
180 | for (auto &forward : collection.forward_columns()) { |
181 | config->add_forward_column_names(forward); |
182 | } |
183 | for (auto &column : collection.index_columns()) { |
184 | auto *column_meta = config->add_index_column_params(); |
185 | ColumnMetaToPB(column, column_meta); |
186 | } |
187 | RepositoryToPB(collection.repository(), config); |
188 | } |
189 | |
190 | #define SET_STATS_FIELD(name) pb_stats->set_##name(stats.name) |
191 | void AdminProtoConverter::SegmentStatsToPB( |
192 | const index::SegmentStats &stats, |
193 | proto::CollectionStats::SegmentStats *pb_stats) { |
194 | SET_STATS_FIELD(segment_id); |
195 | SET_STATS_FIELD(doc_count); |
196 | SET_STATS_FIELD(index_file_count); |
197 | SET_STATS_FIELD(index_file_size); |
198 | SET_STATS_FIELD(min_doc_id); |
199 | SET_STATS_FIELD(max_doc_id); |
200 | SET_STATS_FIELD(min_primary_key); |
201 | SET_STATS_FIELD(max_primary_key); |
202 | SET_STATS_FIELD(min_timestamp); |
203 | SET_STATS_FIELD(max_timestamp); |
204 | SET_STATS_FIELD(min_lsn); |
205 | SET_STATS_FIELD(max_lsn); |
206 | pb_stats->set_state( |
207 | static_cast<proto::CollectionStats_SegmentStats_SegmentState>( |
208 | stats.state)); |
209 | } |
210 | |
211 | void AdminProtoConverter::CollectionStatsToPB( |
212 | const index::CollectionStats &stats, proto::CollectionStats *pb_stats) { |
213 | SET_STATS_FIELD(collection_name); |
214 | SET_STATS_FIELD(collection_path); |
215 | SET_STATS_FIELD(total_doc_count); |
216 | SET_STATS_FIELD(total_segment_count); |
217 | SET_STATS_FIELD(total_index_file_count); |
218 | SET_STATS_FIELD(total_index_file_size); |
219 | for (const auto &segment : stats.segment_stats) { |
220 | SegmentStatsToPB(segment, pb_stats->add_segment_stats()); |
221 | } |
222 | } |
223 | #undef SET_STATS_FIELD |
224 | |
225 | } // namespace admin |
226 | } // namespace be |
227 | } // namespace proxima |
228 | |