1 | /** |
2 | * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | |
16 | * \author Hongqing.hu |
17 | * \date Nov 2020 |
18 | * \brief Implementation of Config |
19 | */ |
20 | |
21 | #include "config.h" |
22 | #include <thread> |
23 | #include <ailego/io/file.h> |
24 | #include <ailego/utility/string_helper.h> |
25 | #include <google/protobuf/text_format.h> |
26 | #include "common/logger.h" |
27 | #include "error_code.h" |
28 | |
29 | namespace proxima { |
30 | namespace be { |
31 | |
32 | int Config::load_config(const std::string &file_name) { |
33 | ailego::File file; |
34 | if (!file.open(file_name.c_str(), true, false)) { |
35 | LOG_ERROR("Open file %s failed, maybe file not exist." , file_name.c_str()); |
36 | return ErrorCode_LoadConfig; |
37 | } |
38 | |
39 | std::string file_content; |
40 | file_content.resize(file.size()); |
41 | size_t read_size = file.read(&(file_content[0]), file.size()); |
42 | if (read_size != file.size()) { |
43 | LOG_ERROR("File %s read error, expected size: %zu, actual size: %zu." , |
44 | file_name.c_str(), file.size(), read_size); |
45 | return ErrorCode_LoadConfig; |
46 | } |
47 | |
48 | bool ret = |
49 | google::protobuf::TextFormat::ParseFromString(file_content, &config_); |
50 | if (!ret) { |
51 | LOG_ERROR("Parse file %s content %s failed." , file_name.c_str(), |
52 | file_content.c_str()); |
53 | |
54 | return ErrorCode_LoadConfig; |
55 | } |
56 | |
57 | config_file_ = file_name; |
58 | |
59 | LOG_INFO( |
60 | "Load config complete. protocol[%s] grpc_listen_port[%u] " |
61 | "http_listen_port[%u] log_directory[%s] log_file[%s] log_level[%u] " |
62 | "build_thread_count[%u] dump_thread_count[%u] " |
63 | "max_build_qps[%u] index_directory[%s] " |
64 | "flush_internal[%u] optimize_internal[%u] meta_uri[%s] " |
65 | "query_thread_count[%u]" , |
66 | this->get_protocol().c_str(), this->get_grpc_listen_port(), |
67 | this->get_http_listen_port(), this->get_log_dir().c_str(), |
68 | this->get_log_file().c_str(), this->get_log_level() + 1, |
69 | this->get_index_build_thread_count(), this->get_index_dump_thread_count(), |
70 | this->get_index_max_build_qps(), this->get_index_directory().c_str(), |
71 | this->get_index_flush_internal(), this->get_index_optimize_internal(), |
72 | this->get_meta_uri().c_str(), this->get_query_thread_count()); |
73 | |
74 | return 0; |
75 | } |
76 | |
77 | int Config::cleanup() { |
78 | config_.Clear(); |
79 | return 0; |
80 | } |
81 | |
82 | bool Config::validate_config() const { |
83 | /** ========== valid Common Config========= **/ |
84 | std::string protocol = this->get_protocol(); |
85 | std::vector<std::string> prots; |
86 | ailego::StringHelper::Split<std::string>(protocol, '|', &prots); |
87 | auto it = |
88 | std::find_if(prots.begin(), prots.end(), [](const std::string &str) { |
89 | return (str == "grpc" ) || (str == "http" ); |
90 | }); |
91 | if (it == prots.end()) { |
92 | LOG_ERROR( |
93 | "Config error, protocol must contains grpc or http at least. " |
94 | "protocol[%s]" , |
95 | protocol.c_str()); |
96 | return false; |
97 | } |
98 | |
99 | if (this->get_grpc_listen_port() > 65535) { |
100 | LOG_ERROR("Config error, grpc_listen_port must be [0, 65535]. port[%u]" , |
101 | this->get_grpc_listen_port()); |
102 | return false; |
103 | } |
104 | |
105 | if (this->get_http_listen_port() > 65535) { |
106 | LOG_ERROR("Config error, http_listen_port must be [0, 65535]. port[%u]" , |
107 | this->get_http_listen_port()); |
108 | return false; |
109 | } |
110 | |
111 | if (this->get_logger_type() != "ConsoleLogger" && |
112 | this->get_logger_type() != "AppendLogger" && |
113 | this->get_logger_type() != "SysLogger" ) { |
114 | LOG_ERROR("Config error, unknown logger type. logger[%s]" , |
115 | this->get_logger_type().c_str()); |
116 | return false; |
117 | } |
118 | |
119 | /** ========== valid Index Config========= **/ |
120 | if (this->get_index_build_thread_count() > 500) { |
121 | LOG_ERROR( |
122 | "Config error, build_thread_count must be [1, 500]. thread_count[%u]" , |
123 | this->get_index_build_thread_count()); |
124 | return false; |
125 | } |
126 | |
127 | if (this->get_index_dump_thread_count() > 500) { |
128 | LOG_ERROR( |
129 | "Config error, dump_thread_count must be [2, 500]. thread_count[%u]" , |
130 | this->get_index_dump_thread_count()); |
131 | return false; |
132 | } |
133 | |
134 | /** ========== valid Query Config========= **/ |
135 | if (this->get_query_thread_count() > 500) { |
136 | LOG_ERROR( |
137 | "Config error, query_thread_count must be [1, 500]. thread_count[%u]" , |
138 | this->get_query_thread_count()); |
139 | return false; |
140 | } |
141 | |
142 | /** ========== valid Meta Config========= **/ |
143 | return true; |
144 | } |
145 | |
146 | std::string Config::get_protocol() const { |
147 | std::string protocol = "grpc|http" ; |
148 | if (config_.has_common_config() && |
149 | !config_.common_config().protocol().empty()) { |
150 | protocol = config_.common_config().protocol(); |
151 | } |
152 | return protocol; |
153 | } |
154 | |
155 | uint32_t Config::get_grpc_listen_port() const { |
156 | uint32_t listen_port = 16000; |
157 | if (config_.has_common_config() && |
158 | config_.common_config().grpc_listen_port() != 0U) { |
159 | listen_port = config_.common_config().grpc_listen_port(); |
160 | } |
161 | return listen_port; |
162 | } |
163 | |
164 | uint32_t Config::get_http_listen_port() const { |
165 | uint32_t listen_port = 16001; |
166 | if (config_.has_common_config() && |
167 | config_.common_config().http_listen_port() != 0U) { |
168 | listen_port = config_.common_config().http_listen_port(); |
169 | } |
170 | return listen_port; |
171 | } |
172 | |
173 | std::string Config::get_log_dir() const { |
174 | std::string log_dir; |
175 | ailego::FileHelper::GetWorkingDirectory(&log_dir); |
176 | log_dir.append("/log/" ); |
177 | if (config_.has_common_config() && |
178 | !config_.common_config().log_directory().empty()) { |
179 | log_dir = config_.common_config().log_directory(); |
180 | } |
181 | return log_dir; |
182 | } |
183 | |
184 | std::string Config::get_log_file() const { |
185 | std::string log_file = "proxima_be.log" ; |
186 | if (config_.has_common_config() && |
187 | !config_.common_config().log_file().empty()) { |
188 | log_file = config_.common_config().log_file(); |
189 | } |
190 | return log_file; |
191 | } |
192 | |
193 | uint32_t Config::get_log_level() const { |
194 | uint32_t actual_log_level = 2; |
195 | if (config_.has_common_config()) { |
196 | uint32_t input_log_level = config_.common_config().log_level(); |
197 | if (input_log_level < 1 || input_log_level > 5) { |
198 | actual_log_level = 1; |
199 | } else { |
200 | actual_log_level = input_log_level - 1; |
201 | } |
202 | } |
203 | return actual_log_level; |
204 | } |
205 | |
206 | std::string Config::get_logger_type() const { |
207 | std::string logger_type = "AppendLogger" ; |
208 | if (!config_.common_config().logger_type().empty()) { |
209 | logger_type = config_.common_config().logger_type(); |
210 | } |
211 | return logger_type; |
212 | } |
213 | |
214 | uint32_t Config::get_index_build_thread_count(void) const { |
215 | uint32_t thread_count = 10U; |
216 | if (config_.has_index_config() && |
217 | config_.index_config().build_thread_count() != 0) { |
218 | thread_count = config_.index_config().build_thread_count(); |
219 | } |
220 | return thread_count; |
221 | } |
222 | |
223 | uint32_t Config::get_index_dump_thread_count(void) const { |
224 | uint32_t thread_count = 3U; |
225 | if (config_.has_index_config() && |
226 | config_.index_config().dump_thread_count() != 0) { |
227 | thread_count = config_.index_config().dump_thread_count(); |
228 | } |
229 | return thread_count; |
230 | } |
231 | |
232 | uint32_t Config::get_index_max_build_qps(void) const { |
233 | uint32_t max_build_qps = 0; |
234 | if (config_.has_index_config()) { |
235 | max_build_qps = config_.index_config().max_build_qps(); |
236 | } |
237 | return max_build_qps; |
238 | } |
239 | |
240 | std::string Config::get_index_directory(void) const { |
241 | std::string index_directory; |
242 | // Set default path to current binary path |
243 | ailego::FileHelper::GetWorkingDirectory(&index_directory); |
244 | if (config_.has_index_config() && |
245 | !config_.index_config().index_directory().empty()) { |
246 | index_directory = config_.index_config().index_directory(); |
247 | } |
248 | return index_directory; |
249 | } |
250 | |
251 | uint32_t Config::get_index_flush_internal(void) const { |
252 | uint32_t flush_internal = 300U; |
253 | if (config_.has_index_config() && |
254 | config_.index_config().flush_internal() != 0) { |
255 | flush_internal = config_.index_config().flush_internal(); |
256 | } |
257 | return flush_internal; |
258 | } |
259 | |
260 | uint32_t Config::get_index_optimize_internal(void) const { |
261 | uint32_t optimize_internal = 0U; |
262 | if (config_.has_index_config() && |
263 | config_.index_config().optimize_internal() != 0) { |
264 | optimize_internal = config_.index_config().optimize_internal(); |
265 | } |
266 | return optimize_internal; |
267 | } |
268 | |
269 | std::string Config::get_meta_uri(void) const { |
270 | if (config_.has_meta_config() && !config_.meta_config().meta_uri().empty()) { |
271 | return config_.meta_config().meta_uri(); |
272 | } |
273 | std::string meta_uri = "sqlite://" ; |
274 | std::string work_directory; |
275 | ailego::FileHelper::GetWorkingDirectory(&work_directory); |
276 | meta_uri.append(work_directory); |
277 | meta_uri.append("/proxima_be_meta.sqlite" ); |
278 | return meta_uri; |
279 | } |
280 | |
281 | uint32_t Config::get_query_thread_count(void) const { |
282 | /// Default set thread count to the number of cpu cores |
283 | uint32_t thread_count = std::thread::hardware_concurrency(); |
284 | if (config_.has_query_config() && |
285 | config_.query_config().query_thread_count() != 0) { |
286 | thread_count = config_.query_config().query_thread_count(); |
287 | } |
288 | return thread_count; |
289 | } |
290 | |
291 | } // namespace be |
292 | } // end namespace proxima |
293 | |