1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author Hechong.xyf
17 * \date Oct 2019
18 * \brief Interface of AiTheta Index Container
19 */
20
21#ifndef __AITHETA2_INDEX_CONTAINER_H__
22#define __AITHETA2_INDEX_CONTAINER_H__
23
24#include "index_module.h"
25#include "index_unpacker.h"
26
27namespace aitheta2 {
28
29/*! Index Container
30 */
31class IndexContainer : public IndexModule {
32 public:
33 //! Index Container Pointer
34 typedef std::shared_ptr<IndexContainer> Pointer;
35
36 /*! Index Container Segment Data
37 */
38 struct SegmentData {
39 //! Constructor
40 SegmentData(void) : offset(0u), length(0u), data(nullptr) {}
41
42 //! Constructor
43 SegmentData(size_t off, size_t len)
44 : offset(off), length(len), data(nullptr) {}
45
46 //! Members
47 size_t offset;
48 size_t length;
49 const void *data;
50 };
51
52 /*! Index Container Segment
53 */
54 struct Segment {
55 //! Index Container Pointer
56 typedef std::shared_ptr<Segment> Pointer;
57
58 //! Destructor
59 virtual ~Segment(void) {}
60
61 //! Retrieve size of data
62 virtual size_t data_size(void) const = 0;
63
64 //! Retrieve crc of data
65 virtual uint32_t data_crc(void) const = 0;
66
67 //! Retrieve size of padding
68 virtual size_t padding_size(void) const = 0;
69
70 //! Fetch data from segment (with own buffer)
71 virtual size_t fetch(size_t offset, void *buf, size_t len) const = 0;
72
73 //! Read data from segment
74 virtual size_t read(size_t offset, const void **data, size_t len) = 0;
75
76 //! Read data from segment
77 virtual bool read(SegmentData *iovec, size_t count) = 0;
78
79 //! Clone the segment
80 virtual Pointer clone(void) = 0;
81 };
82
83 //! Destructor
84 virtual ~IndexContainer(void) {}
85
86 //! Initialize container
87 virtual int init(const IndexParams &params) = 0;
88
89 //! Cleanup container
90 virtual int cleanup(void) = 0;
91
92 //! Load a index file into container
93 virtual int load(const std::string &path) = 0;
94
95 //! Load the current index into container
96 virtual int load(void) = 0;
97
98 //! Unload all indexes
99 virtual int unload(void) = 0;
100
101 //! Retrieve a segment by id
102 virtual Segment::Pointer get(const std::string &id) const = 0;
103
104 //! Test if it a segment exists
105 virtual bool has(const std::string &id) const = 0;
106
107 //! Retrieve all segments
108 virtual std::map<std::string, Segment::Pointer> get_all(void) const = 0;
109
110 //! Retrieve magic number of index
111 virtual uint32_t magic(void) const = 0;
112
113 //! Fetch a segment by id with level (0 high, 1 normal, 2 low)
114 virtual Segment::Pointer fetch(const std::string &id, int /*level*/) const {
115 return this->get(id);
116 }
117};
118
119/*! Index Segment Container
120 */
121class IndexSegmentContainer : public IndexContainer {
122 public:
123 //! Index Segment Container Pointer
124 typedef std::shared_ptr<IndexSegmentContainer> Pointer;
125
126 /*! Index Container Segment
127 */
128 class Segment : public IndexContainer::Segment {
129 public:
130 //! Index Container Pointer
131 typedef std::shared_ptr<Segment> Pointer;
132
133 //! Constructor
134 Segment(const Segment &rhs)
135 : data_offset_(rhs.data_offset_),
136 data_size_(rhs.data_size_),
137 padding_size_(rhs.padding_size_),
138 region_size_(rhs.region_size_),
139 data_crc_(rhs.data_crc_),
140 parent_(rhs.parent_->clone()) {}
141
142 //! Constructor
143 Segment(const IndexContainer::Segment::Pointer &parent,
144 const IndexUnpacker::SegmentMeta &segment)
145 : data_offset_(segment.data_offset()),
146 data_size_(segment.data_size()),
147 padding_size_(segment.padding_size()),
148 region_size_(segment.data_size() + segment.padding_size()),
149 data_crc_(segment.data_crc()),
150 parent_(parent->clone()) {}
151
152 //! Destructor
153 virtual ~Segment(void) {}
154
155 //! Retrieve size of data
156 size_t data_size(void) const override {
157 return data_size_;
158 }
159
160 //! Retrieve crc of data
161 uint32_t data_crc(void) const override {
162 return data_crc_;
163 }
164
165 //! Retrieve size of padding
166 size_t padding_size(void) const override {
167 return padding_size_;
168 }
169
170 //! Fetch data from segment (with own buffer)
171 size_t fetch(size_t offset, void *buf, size_t len) const override {
172 return parent_->fetch(data_offset_ + offset, buf, len);
173 }
174
175 //! Read data from segment
176 size_t read(size_t offset, const void **data, size_t len) override {
177 return parent_->read(data_offset_ + offset, data, len);
178 }
179
180 //! Read data from segment
181 bool read(SegmentData *iovec, size_t count) override {
182 for (SegmentData *it = iovec, *end = iovec + count; it != end; ++it) {
183 it->offset += data_offset_;
184 }
185 bool success = parent_->read(iovec, count);
186 for (SegmentData *it = iovec, *end = iovec + count; it != end; ++it) {
187 it->offset -= data_offset_;
188 }
189 return success;
190 }
191
192 //! Clone the segment
193 IndexContainer::Segment::Pointer clone(void) override {
194 return std::make_shared<Segment>(*this);
195 }
196
197 private:
198 size_t data_offset_{0u};
199 size_t data_size_{0u};
200 size_t padding_size_{0u};
201 size_t region_size_{0u};
202 uint32_t data_crc_{0u};
203 IndexContainer::Segment::Pointer parent_{nullptr};
204 };
205
206 //! Constructor
207 IndexSegmentContainer(IndexContainer::Segment::Pointer &&seg)
208 : parent_(std::move(seg)) {}
209
210 //! Constructor
211 IndexSegmentContainer(const IndexContainer::Segment::Pointer &seg)
212 : parent_(seg) {}
213
214 //! Destructor
215 virtual ~IndexSegmentContainer(void) {}
216
217 //! Initialize container
218 int init(const IndexParams &) override {
219 return 0;
220 }
221
222 //! Cleanup container
223 int cleanup(void) override {
224 return 0;
225 }
226
227 //! Load the current index into container
228 int load(void) override {
229 if (!parent_) {
230 LOG_ERROR("Failed to load an empty segment");
231 return IndexError_NoReady;
232 }
233
234 auto read_data = [this](size_t offset, const void **data, size_t len) {
235 return this->parent_->read(offset, data, len);
236 };
237
238 IndexUnpacker unpacker;
239 if (!unpacker.unpack(read_data, parent_->data_size(), false)) {
240 LOG_ERROR("Failed to unpack segment data");
241 return IndexError_UnpackIndex;
242 }
243 segments_ = std::move(*unpacker.mutable_segments());
244 magic_ = unpacker.magic();
245 return 0;
246 }
247
248 //! Load the current segment, ignore path
249 int load(const std::string &) override {
250 return this->load();
251 }
252
253 //! Retrieve a segment by id
254 IndexContainer::Segment::Pointer get(const std::string &id) const override {
255 if (!parent_) {
256 return IndexContainer::Segment::Pointer();
257 }
258 auto it = segments_.find(id);
259 if (it == segments_.end()) {
260 return IndexContainer::Segment::Pointer();
261 }
262 return std::make_shared<IndexSegmentContainer::Segment>(parent_,
263 it->second);
264 }
265
266 //! Test if it a segment exists
267 bool has(const std::string &id) const override {
268 return (segments_.find(id) != segments_.end());
269 }
270
271 //! Retrieve all segments
272 std::map<std::string, IndexContainer::Segment::Pointer> get_all(
273 void) const override {
274 std::map<std::string, IndexContainer::Segment::Pointer> result;
275 if (parent_) {
276 for (const auto &it : segments_) {
277 result.emplace(it.first,
278 std::make_shared<IndexSegmentContainer::Segment>(
279 parent_, it.second));
280 }
281 }
282 return result;
283 }
284
285 //! Unload all indexes
286 int unload(void) override {
287 parent_ = nullptr;
288 segments_.clear();
289 return 0;
290 }
291
292 //! Retrieve magic number of index
293 uint32_t magic(void) const override {
294 return magic_;
295 }
296
297 private:
298 uint32_t magic_{0};
299 std::map<std::string, IndexUnpacker::SegmentMeta> segments_{};
300 IndexContainer::Segment::Pointer parent_{};
301};
302
303} // namespace aitheta2
304
305#endif // __AITHETA2_INDEX_CONTAINER_H__
306