1 | /** |
2 | * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | |
16 | * \author Hechong.xyf |
17 | * \date Oct 2019 |
18 | * \brief Interface of AiTheta Index Dumper |
19 | */ |
20 | |
21 | #ifndef __AITHETA2_INDEX_DUMPER_H__ |
22 | #define __AITHETA2_INDEX_DUMPER_H__ |
23 | |
24 | #include "index_module.h" |
25 | #include "index_packer.h" |
26 | #include "index_params.h" |
27 | |
28 | namespace aitheta2 { |
29 | |
30 | /*! Index Dumper |
31 | */ |
32 | class IndexDumper : public IndexModule { |
33 | public: |
34 | //! Index Dumper Pointer |
35 | typedef std::shared_ptr<IndexDumper> Pointer; |
36 | |
37 | //! Destructor |
38 | virtual ~IndexDumper(void) {} |
39 | |
40 | //! Initialize dumper |
41 | virtual int init(const IndexParams ¶ms) = 0; |
42 | |
43 | //! Cleanup dumper |
44 | virtual int cleanup(void) = 0; |
45 | |
46 | //! Create a file for dumping |
47 | virtual int create(const std::string &path) = 0; |
48 | |
49 | //! Close file |
50 | virtual int close(void) = 0; |
51 | |
52 | //! Append a segment meta into table |
53 | virtual int append(const std::string &id, size_t data_size, |
54 | size_t padding_size, uint32_t crc) = 0; |
55 | |
56 | //! Write data to the storage |
57 | virtual size_t write(const void *data, size_t len) = 0; |
58 | |
59 | //! Retrieve magic number of index |
60 | virtual uint32_t magic(void) const = 0; |
61 | }; |
62 | |
63 | /*! Index Segment Dumper |
64 | */ |
65 | class IndexSegmentDumper : public IndexDumper { |
66 | public: |
67 | //! Index Segment Dumper Pointer |
68 | typedef std::shared_ptr<IndexSegmentDumper> Pointer; |
69 | |
70 | //! Constructor |
71 | IndexSegmentDumper(IndexDumper::Pointer dumper, std::string segid) |
72 | : segment_id_(std::move(segid)), dumper_(std::move(dumper)) {} |
73 | |
74 | //! Destructor |
75 | virtual ~IndexSegmentDumper(void) { |
76 | this->close_index(); |
77 | } |
78 | |
79 | //! Initialize dumper |
80 | int init(const IndexParams &) override { |
81 | return 0; |
82 | } |
83 | |
84 | //! Cleanup dumper |
85 | int cleanup(void) override { |
86 | return 0; |
87 | } |
88 | |
89 | //! Create a file for dumping |
90 | int create(const std::string &segid) override { |
91 | if (dumped_size_ != 0) { |
92 | return IndexError_NoReady; |
93 | } |
94 | |
95 | auto write_data = [&](const void *buf, size_t size) { |
96 | return this->write_to_dumper(buf, size); |
97 | }; |
98 | if (!packer_.setup(write_data)) { |
99 | return IndexError_WriteData; |
100 | } |
101 | segment_id_ = segid; |
102 | return 0; |
103 | } |
104 | |
105 | //! Close file |
106 | int close(void) override { |
107 | return this->close_index(); |
108 | } |
109 | |
110 | //! Append a segment meta into table |
111 | int append(const std::string &id, size_t data_size, size_t padding_size, |
112 | uint32_t crc) override { |
113 | stab_.emplace_back(id, data_size, padding_size, crc); |
114 | return 0; |
115 | } |
116 | |
117 | //! Write data to the storage |
118 | size_t write(const void *data, size_t len) override { |
119 | auto write_data = [&](const void *buf, size_t size) { |
120 | return this->write_to_dumper(buf, size); |
121 | }; |
122 | |
123 | if (dumped_size_ == 0 && !packer_.setup(write_data)) { |
124 | return 0; |
125 | } |
126 | return packer_.pack(write_data, data, len); |
127 | } |
128 | |
129 | //! Retrieve magic number of index |
130 | uint32_t magic(void) const override { |
131 | return packer_.magic(); |
132 | } |
133 | |
134 | protected: |
135 | //! Write data to dumper |
136 | size_t write_to_dumper(const void *data, size_t len) { |
137 | size_t wrlen = dumper_->write(data, len); |
138 | dumped_size_ += wrlen; |
139 | return wrlen; |
140 | } |
141 | |
142 | //! Close index file |
143 | int close_index(void) { |
144 | if (dumped_size_ == 0) { |
145 | return 0; |
146 | } |
147 | |
148 | auto write_data = [&](const void *buf, size_t size) { |
149 | return this->write_to_dumper(buf, size); |
150 | }; |
151 | if (!packer_.finish(write_data, stab_)) { |
152 | return IndexError_WriteData; |
153 | } |
154 | stab_.clear(); |
155 | |
156 | int ret = dumper_->append(segment_id_, dumped_size_, 0, 0); |
157 | dumped_size_ = 0u; |
158 | return ret; |
159 | } |
160 | |
161 | private: |
162 | size_t dumped_size_{0}; |
163 | std::string segment_id_{}; |
164 | IndexDumper::Pointer dumper_{}; |
165 | IndexPacker packer_{}; |
166 | std::vector<IndexPacker::SegmentMeta> stab_{}; |
167 | }; |
168 | |
169 | |
170 | } // namespace aitheta2 |
171 | |
172 | #endif // __AITHETA2_INDEX_DUMPER_H__ |
173 | |