1 | /** |
2 | * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | |
16 | * \author Hechong.xyf |
17 | * \date Oct 2019 |
18 | * \brief Interface of AiTheta Index Container |
19 | */ |
20 | |
21 | #ifndef __AITHETA2_INDEX_CONTAINER_H__ |
22 | #define __AITHETA2_INDEX_CONTAINER_H__ |
23 | |
24 | #include "index_module.h" |
25 | #include "index_unpacker.h" |
26 | |
27 | namespace aitheta2 { |
28 | |
29 | /*! Index Container |
30 | */ |
31 | class IndexContainer : public IndexModule { |
32 | public: |
33 | //! Index Container Pointer |
34 | typedef std::shared_ptr<IndexContainer> Pointer; |
35 | |
36 | /*! Index Container Segment Data |
37 | */ |
38 | struct SegmentData { |
39 | //! Constructor |
40 | SegmentData(void) : offset(0u), length(0u), data(nullptr) {} |
41 | |
42 | //! Constructor |
43 | SegmentData(size_t off, size_t len) |
44 | : offset(off), length(len), data(nullptr) {} |
45 | |
46 | //! Members |
47 | size_t offset; |
48 | size_t length; |
49 | const void *data; |
50 | }; |
51 | |
52 | /*! Index Container Segment |
53 | */ |
54 | struct Segment { |
55 | //! Index Container Pointer |
56 | typedef std::shared_ptr<Segment> Pointer; |
57 | |
58 | //! Destructor |
59 | virtual ~Segment(void) {} |
60 | |
61 | //! Retrieve size of data |
62 | virtual size_t data_size(void) const = 0; |
63 | |
64 | //! Retrieve crc of data |
65 | virtual uint32_t data_crc(void) const = 0; |
66 | |
67 | //! Retrieve size of padding |
68 | virtual size_t padding_size(void) const = 0; |
69 | |
70 | //! Fetch data from segment (with own buffer) |
71 | virtual size_t fetch(size_t offset, void *buf, size_t len) const = 0; |
72 | |
73 | //! Read data from segment |
74 | virtual size_t read(size_t offset, const void **data, size_t len) = 0; |
75 | |
76 | //! Read data from segment |
77 | virtual bool read(SegmentData *iovec, size_t count) = 0; |
78 | |
79 | //! Clone the segment |
80 | virtual Pointer clone(void) = 0; |
81 | }; |
82 | |
83 | //! Destructor |
84 | virtual ~IndexContainer(void) {} |
85 | |
86 | //! Initialize container |
87 | virtual int init(const IndexParams ¶ms) = 0; |
88 | |
89 | //! Cleanup container |
90 | virtual int cleanup(void) = 0; |
91 | |
92 | //! Load a index file into container |
93 | virtual int load(const std::string &path) = 0; |
94 | |
95 | //! Load the current index into container |
96 | virtual int load(void) = 0; |
97 | |
98 | //! Unload all indexes |
99 | virtual int unload(void) = 0; |
100 | |
101 | //! Retrieve a segment by id |
102 | virtual Segment::Pointer get(const std::string &id) const = 0; |
103 | |
104 | //! Test if it a segment exists |
105 | virtual bool has(const std::string &id) const = 0; |
106 | |
107 | //! Retrieve all segments |
108 | virtual std::map<std::string, Segment::Pointer> get_all(void) const = 0; |
109 | |
110 | //! Retrieve magic number of index |
111 | virtual uint32_t magic(void) const = 0; |
112 | |
113 | //! Fetch a segment by id with level (0 high, 1 normal, 2 low) |
114 | virtual Segment::Pointer fetch(const std::string &id, int /*level*/) const { |
115 | return this->get(id); |
116 | } |
117 | }; |
118 | |
119 | /*! Index Segment Container |
120 | */ |
121 | class IndexSegmentContainer : public IndexContainer { |
122 | public: |
123 | //! Index Segment Container Pointer |
124 | typedef std::shared_ptr<IndexSegmentContainer> Pointer; |
125 | |
126 | /*! Index Container Segment |
127 | */ |
128 | class Segment : public IndexContainer::Segment { |
129 | public: |
130 | //! Index Container Pointer |
131 | typedef std::shared_ptr<Segment> Pointer; |
132 | |
133 | //! Constructor |
134 | Segment(const Segment &rhs) |
135 | : data_offset_(rhs.data_offset_), |
136 | data_size_(rhs.data_size_), |
137 | padding_size_(rhs.padding_size_), |
138 | region_size_(rhs.region_size_), |
139 | data_crc_(rhs.data_crc_), |
140 | parent_(rhs.parent_->clone()) {} |
141 | |
142 | //! Constructor |
143 | Segment(const IndexContainer::Segment::Pointer &parent, |
144 | const IndexUnpacker::SegmentMeta &segment) |
145 | : data_offset_(segment.data_offset()), |
146 | data_size_(segment.data_size()), |
147 | padding_size_(segment.padding_size()), |
148 | region_size_(segment.data_size() + segment.padding_size()), |
149 | data_crc_(segment.data_crc()), |
150 | parent_(parent->clone()) {} |
151 | |
152 | //! Destructor |
153 | virtual ~Segment(void) {} |
154 | |
155 | //! Retrieve size of data |
156 | size_t data_size(void) const override { |
157 | return data_size_; |
158 | } |
159 | |
160 | //! Retrieve crc of data |
161 | uint32_t data_crc(void) const override { |
162 | return data_crc_; |
163 | } |
164 | |
165 | //! Retrieve size of padding |
166 | size_t padding_size(void) const override { |
167 | return padding_size_; |
168 | } |
169 | |
170 | //! Fetch data from segment (with own buffer) |
171 | size_t fetch(size_t offset, void *buf, size_t len) const override { |
172 | return parent_->fetch(data_offset_ + offset, buf, len); |
173 | } |
174 | |
175 | //! Read data from segment |
176 | size_t read(size_t offset, const void **data, size_t len) override { |
177 | return parent_->read(data_offset_ + offset, data, len); |
178 | } |
179 | |
180 | //! Read data from segment |
181 | bool read(SegmentData *iovec, size_t count) override { |
182 | for (SegmentData *it = iovec, *end = iovec + count; it != end; ++it) { |
183 | it->offset += data_offset_; |
184 | } |
185 | bool success = parent_->read(iovec, count); |
186 | for (SegmentData *it = iovec, *end = iovec + count; it != end; ++it) { |
187 | it->offset -= data_offset_; |
188 | } |
189 | return success; |
190 | } |
191 | |
192 | //! Clone the segment |
193 | IndexContainer::Segment::Pointer clone(void) override { |
194 | return std::make_shared<Segment>(*this); |
195 | } |
196 | |
197 | private: |
198 | size_t data_offset_{0u}; |
199 | size_t data_size_{0u}; |
200 | size_t padding_size_{0u}; |
201 | size_t region_size_{0u}; |
202 | uint32_t data_crc_{0u}; |
203 | IndexContainer::Segment::Pointer parent_{nullptr}; |
204 | }; |
205 | |
206 | //! Constructor |
207 | IndexSegmentContainer(IndexContainer::Segment::Pointer &&seg) |
208 | : parent_(std::move(seg)) {} |
209 | |
210 | //! Constructor |
211 | IndexSegmentContainer(const IndexContainer::Segment::Pointer &seg) |
212 | : parent_(seg) {} |
213 | |
214 | //! Destructor |
215 | virtual ~IndexSegmentContainer(void) {} |
216 | |
217 | //! Initialize container |
218 | int init(const IndexParams &) override { |
219 | return 0; |
220 | } |
221 | |
222 | //! Cleanup container |
223 | int cleanup(void) override { |
224 | return 0; |
225 | } |
226 | |
227 | //! Load the current index into container |
228 | int load(void) override { |
229 | if (!parent_) { |
230 | LOG_ERROR("Failed to load an empty segment" ); |
231 | return IndexError_NoReady; |
232 | } |
233 | |
234 | auto read_data = [this](size_t offset, const void **data, size_t len) { |
235 | return this->parent_->read(offset, data, len); |
236 | }; |
237 | |
238 | IndexUnpacker unpacker; |
239 | if (!unpacker.unpack(read_data, parent_->data_size(), false)) { |
240 | LOG_ERROR("Failed to unpack segment data" ); |
241 | return IndexError_UnpackIndex; |
242 | } |
243 | segments_ = std::move(*unpacker.mutable_segments()); |
244 | magic_ = unpacker.magic(); |
245 | return 0; |
246 | } |
247 | |
248 | //! Load the current segment, ignore path |
249 | int load(const std::string &) override { |
250 | return this->load(); |
251 | } |
252 | |
253 | //! Retrieve a segment by id |
254 | IndexContainer::Segment::Pointer get(const std::string &id) const override { |
255 | if (!parent_) { |
256 | return IndexContainer::Segment::Pointer(); |
257 | } |
258 | auto it = segments_.find(id); |
259 | if (it == segments_.end()) { |
260 | return IndexContainer::Segment::Pointer(); |
261 | } |
262 | return std::make_shared<IndexSegmentContainer::Segment>(parent_, |
263 | it->second); |
264 | } |
265 | |
266 | //! Test if it a segment exists |
267 | bool has(const std::string &id) const override { |
268 | return (segments_.find(id) != segments_.end()); |
269 | } |
270 | |
271 | //! Retrieve all segments |
272 | std::map<std::string, IndexContainer::Segment::Pointer> get_all( |
273 | void) const override { |
274 | std::map<std::string, IndexContainer::Segment::Pointer> result; |
275 | if (parent_) { |
276 | for (const auto &it : segments_) { |
277 | result.emplace(it.first, |
278 | std::make_shared<IndexSegmentContainer::Segment>( |
279 | parent_, it.second)); |
280 | } |
281 | } |
282 | return result; |
283 | } |
284 | |
285 | //! Unload all indexes |
286 | int unload(void) override { |
287 | parent_ = nullptr; |
288 | segments_.clear(); |
289 | return 0; |
290 | } |
291 | |
292 | //! Retrieve magic number of index |
293 | uint32_t magic(void) const override { |
294 | return magic_; |
295 | } |
296 | |
297 | private: |
298 | uint32_t magic_{0}; |
299 | std::map<std::string, IndexUnpacker::SegmentMeta> segments_{}; |
300 | IndexContainer::Segment::Pointer parent_{}; |
301 | }; |
302 | |
303 | } // namespace aitheta2 |
304 | |
305 | #endif // __AITHETA2_INDEX_CONTAINER_H__ |
306 | |