1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author Hongqing.hu
17 * \date Nov 2020
18 * \brief Implementation of Config
19 */
20
21#include "config.h"
22#include <thread>
23#include <ailego/io/file.h>
24#include <ailego/utility/string_helper.h>
25#include <google/protobuf/text_format.h>
26#include "common/logger.h"
27#include "error_code.h"
28
29namespace proxima {
30namespace be {
31
32int Config::load_config(const std::string &file_name) {
33 ailego::File file;
34 if (!file.open(file_name.c_str(), true, false)) {
35 LOG_ERROR("Open file %s failed, maybe file not exist.", file_name.c_str());
36 return ErrorCode_LoadConfig;
37 }
38
39 std::string file_content;
40 file_content.resize(file.size());
41 size_t read_size = file.read(&(file_content[0]), file.size());
42 if (read_size != file.size()) {
43 LOG_ERROR("File %s read error, expected size: %zu, actual size: %zu.",
44 file_name.c_str(), file.size(), read_size);
45 return ErrorCode_LoadConfig;
46 }
47
48 bool ret =
49 google::protobuf::TextFormat::ParseFromString(file_content, &config_);
50 if (!ret) {
51 LOG_ERROR("Parse file %s content %s failed.", file_name.c_str(),
52 file_content.c_str());
53
54 return ErrorCode_LoadConfig;
55 }
56
57 config_file_ = file_name;
58
59 LOG_INFO(
60 "Load config complete. protocol[%s] grpc_listen_port[%u] "
61 "http_listen_port[%u] log_directory[%s] log_file[%s] log_level[%u] "
62 "build_thread_count[%u] dump_thread_count[%u] "
63 "max_build_qps[%u] index_directory[%s] "
64 "flush_internal[%u] optimize_internal[%u] meta_uri[%s] "
65 "query_thread_count[%u]",
66 this->get_protocol().c_str(), this->get_grpc_listen_port(),
67 this->get_http_listen_port(), this->get_log_dir().c_str(),
68 this->get_log_file().c_str(), this->get_log_level() + 1,
69 this->get_index_build_thread_count(), this->get_index_dump_thread_count(),
70 this->get_index_max_build_qps(), this->get_index_directory().c_str(),
71 this->get_index_flush_internal(), this->get_index_optimize_internal(),
72 this->get_meta_uri().c_str(), this->get_query_thread_count());
73
74 return 0;
75}
76
77int Config::cleanup() {
78 config_.Clear();
79 return 0;
80}
81
82bool Config::validate_config() const {
83 /** ========== valid Common Config========= **/
84 std::string protocol = this->get_protocol();
85 std::vector<std::string> prots;
86 ailego::StringHelper::Split<std::string>(protocol, '|', &prots);
87 auto it =
88 std::find_if(prots.begin(), prots.end(), [](const std::string &str) {
89 return (str == "grpc") || (str == "http");
90 });
91 if (it == prots.end()) {
92 LOG_ERROR(
93 "Config error, protocol must contains grpc or http at least. "
94 "protocol[%s]",
95 protocol.c_str());
96 return false;
97 }
98
99 if (this->get_grpc_listen_port() > 65535) {
100 LOG_ERROR("Config error, grpc_listen_port must be [0, 65535]. port[%u]",
101 this->get_grpc_listen_port());
102 return false;
103 }
104
105 if (this->get_http_listen_port() > 65535) {
106 LOG_ERROR("Config error, http_listen_port must be [0, 65535]. port[%u]",
107 this->get_http_listen_port());
108 return false;
109 }
110
111 if (this->get_logger_type() != "ConsoleLogger" &&
112 this->get_logger_type() != "AppendLogger" &&
113 this->get_logger_type() != "SysLogger") {
114 LOG_ERROR("Config error, unknown logger type. logger[%s]",
115 this->get_logger_type().c_str());
116 return false;
117 }
118
119 /** ========== valid Index Config========= **/
120 if (this->get_index_build_thread_count() > 500) {
121 LOG_ERROR(
122 "Config error, build_thread_count must be [1, 500]. thread_count[%u]",
123 this->get_index_build_thread_count());
124 return false;
125 }
126
127 if (this->get_index_dump_thread_count() > 500) {
128 LOG_ERROR(
129 "Config error, dump_thread_count must be [2, 500]. thread_count[%u]",
130 this->get_index_dump_thread_count());
131 return false;
132 }
133
134 /** ========== valid Query Config========= **/
135 if (this->get_query_thread_count() > 500) {
136 LOG_ERROR(
137 "Config error, query_thread_count must be [1, 500]. thread_count[%u]",
138 this->get_query_thread_count());
139 return false;
140 }
141
142 /** ========== valid Meta Config========= **/
143 return true;
144}
145
146std::string Config::get_protocol() const {
147 std::string protocol = "grpc|http";
148 if (config_.has_common_config() &&
149 !config_.common_config().protocol().empty()) {
150 protocol = config_.common_config().protocol();
151 }
152 return protocol;
153}
154
155uint32_t Config::get_grpc_listen_port() const {
156 uint32_t listen_port = 16000;
157 if (config_.has_common_config() &&
158 config_.common_config().grpc_listen_port() != 0U) {
159 listen_port = config_.common_config().grpc_listen_port();
160 }
161 return listen_port;
162}
163
164uint32_t Config::get_http_listen_port() const {
165 uint32_t listen_port = 16001;
166 if (config_.has_common_config() &&
167 config_.common_config().http_listen_port() != 0U) {
168 listen_port = config_.common_config().http_listen_port();
169 }
170 return listen_port;
171}
172
173std::string Config::get_log_dir() const {
174 std::string log_dir;
175 ailego::FileHelper::GetWorkingDirectory(&log_dir);
176 log_dir.append("/log/");
177 if (config_.has_common_config() &&
178 !config_.common_config().log_directory().empty()) {
179 log_dir = config_.common_config().log_directory();
180 }
181 return log_dir;
182}
183
184std::string Config::get_log_file() const {
185 std::string log_file = "proxima_be.log";
186 if (config_.has_common_config() &&
187 !config_.common_config().log_file().empty()) {
188 log_file = config_.common_config().log_file();
189 }
190 return log_file;
191}
192
193uint32_t Config::get_log_level() const {
194 uint32_t actual_log_level = 2;
195 if (config_.has_common_config()) {
196 uint32_t input_log_level = config_.common_config().log_level();
197 if (input_log_level < 1 || input_log_level > 5) {
198 actual_log_level = 1;
199 } else {
200 actual_log_level = input_log_level - 1;
201 }
202 }
203 return actual_log_level;
204}
205
206std::string Config::get_logger_type() const {
207 std::string logger_type = "AppendLogger";
208 if (!config_.common_config().logger_type().empty()) {
209 logger_type = config_.common_config().logger_type();
210 }
211 return logger_type;
212}
213
214uint32_t Config::get_index_build_thread_count(void) const {
215 uint32_t thread_count = 10U;
216 if (config_.has_index_config() &&
217 config_.index_config().build_thread_count() != 0) {
218 thread_count = config_.index_config().build_thread_count();
219 }
220 return thread_count;
221}
222
223uint32_t Config::get_index_dump_thread_count(void) const {
224 uint32_t thread_count = 3U;
225 if (config_.has_index_config() &&
226 config_.index_config().dump_thread_count() != 0) {
227 thread_count = config_.index_config().dump_thread_count();
228 }
229 return thread_count;
230}
231
232uint32_t Config::get_index_max_build_qps(void) const {
233 uint32_t max_build_qps = 0;
234 if (config_.has_index_config()) {
235 max_build_qps = config_.index_config().max_build_qps();
236 }
237 return max_build_qps;
238}
239
240std::string Config::get_index_directory(void) const {
241 std::string index_directory;
242 // Set default path to current binary path
243 ailego::FileHelper::GetWorkingDirectory(&index_directory);
244 if (config_.has_index_config() &&
245 !config_.index_config().index_directory().empty()) {
246 index_directory = config_.index_config().index_directory();
247 }
248 return index_directory;
249}
250
251uint32_t Config::get_index_flush_internal(void) const {
252 uint32_t flush_internal = 300U;
253 if (config_.has_index_config() &&
254 config_.index_config().flush_internal() != 0) {
255 flush_internal = config_.index_config().flush_internal();
256 }
257 return flush_internal;
258}
259
260uint32_t Config::get_index_optimize_internal(void) const {
261 uint32_t optimize_internal = 0U;
262 if (config_.has_index_config() &&
263 config_.index_config().optimize_internal() != 0) {
264 optimize_internal = config_.index_config().optimize_internal();
265 }
266 return optimize_internal;
267}
268
269std::string Config::get_meta_uri(void) const {
270 if (config_.has_meta_config() && !config_.meta_config().meta_uri().empty()) {
271 return config_.meta_config().meta_uri();
272 }
273 std::string meta_uri = "sqlite://";
274 std::string work_directory;
275 ailego::FileHelper::GetWorkingDirectory(&work_directory);
276 meta_uri.append(work_directory);
277 meta_uri.append("/proxima_be_meta.sqlite");
278 return meta_uri;
279}
280
281uint32_t Config::get_query_thread_count(void) const {
282 /// Default set thread count to the number of cpu cores
283 uint32_t thread_count = std::thread::hardware_concurrency();
284 if (config_.has_query_config() &&
285 config_.query_config().query_thread_count() != 0) {
286 thread_count = config_.query_config().query_thread_count();
287 }
288 return thread_count;
289}
290
291} // namespace be
292} // end namespace proxima
293