1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author Hechong.xyf
17 * \date Oct 2019
18 * \brief Interface of AiTheta Index Dumper
19 */
20
21#ifndef __AITHETA2_INDEX_DUMPER_H__
22#define __AITHETA2_INDEX_DUMPER_H__
23
24#include "index_module.h"
25#include "index_packer.h"
26#include "index_params.h"
27
28namespace aitheta2 {
29
30/*! Index Dumper
31 */
32class IndexDumper : public IndexModule {
33 public:
34 //! Index Dumper Pointer
35 typedef std::shared_ptr<IndexDumper> Pointer;
36
37 //! Destructor
38 virtual ~IndexDumper(void) {}
39
40 //! Initialize dumper
41 virtual int init(const IndexParams &params) = 0;
42
43 //! Cleanup dumper
44 virtual int cleanup(void) = 0;
45
46 //! Create a file for dumping
47 virtual int create(const std::string &path) = 0;
48
49 //! Close file
50 virtual int close(void) = 0;
51
52 //! Append a segment meta into table
53 virtual int append(const std::string &id, size_t data_size,
54 size_t padding_size, uint32_t crc) = 0;
55
56 //! Write data to the storage
57 virtual size_t write(const void *data, size_t len) = 0;
58
59 //! Retrieve magic number of index
60 virtual uint32_t magic(void) const = 0;
61};
62
63/*! Index Segment Dumper
64 */
65class IndexSegmentDumper : public IndexDumper {
66 public:
67 //! Index Segment Dumper Pointer
68 typedef std::shared_ptr<IndexSegmentDumper> Pointer;
69
70 //! Constructor
71 IndexSegmentDumper(IndexDumper::Pointer dumper, std::string segid)
72 : segment_id_(std::move(segid)), dumper_(std::move(dumper)) {}
73
74 //! Destructor
75 virtual ~IndexSegmentDumper(void) {
76 this->close_index();
77 }
78
79 //! Initialize dumper
80 int init(const IndexParams &) override {
81 return 0;
82 }
83
84 //! Cleanup dumper
85 int cleanup(void) override {
86 return 0;
87 }
88
89 //! Create a file for dumping
90 int create(const std::string &segid) override {
91 if (dumped_size_ != 0) {
92 return IndexError_NoReady;
93 }
94
95 auto write_data = [&](const void *buf, size_t size) {
96 return this->write_to_dumper(buf, size);
97 };
98 if (!packer_.setup(write_data)) {
99 return IndexError_WriteData;
100 }
101 segment_id_ = segid;
102 return 0;
103 }
104
105 //! Close file
106 int close(void) override {
107 return this->close_index();
108 }
109
110 //! Append a segment meta into table
111 int append(const std::string &id, size_t data_size, size_t padding_size,
112 uint32_t crc) override {
113 stab_.emplace_back(id, data_size, padding_size, crc);
114 return 0;
115 }
116
117 //! Write data to the storage
118 size_t write(const void *data, size_t len) override {
119 auto write_data = [&](const void *buf, size_t size) {
120 return this->write_to_dumper(buf, size);
121 };
122
123 if (dumped_size_ == 0 && !packer_.setup(write_data)) {
124 return 0;
125 }
126 return packer_.pack(write_data, data, len);
127 }
128
129 //! Retrieve magic number of index
130 uint32_t magic(void) const override {
131 return packer_.magic();
132 }
133
134 protected:
135 //! Write data to dumper
136 size_t write_to_dumper(const void *data, size_t len) {
137 size_t wrlen = dumper_->write(data, len);
138 dumped_size_ += wrlen;
139 return wrlen;
140 }
141
142 //! Close index file
143 int close_index(void) {
144 if (dumped_size_ == 0) {
145 return 0;
146 }
147
148 auto write_data = [&](const void *buf, size_t size) {
149 return this->write_to_dumper(buf, size);
150 };
151 if (!packer_.finish(write_data, stab_)) {
152 return IndexError_WriteData;
153 }
154 stab_.clear();
155
156 int ret = dumper_->append(segment_id_, dumped_size_, 0, 0);
157 dumped_size_ = 0u;
158 return ret;
159 }
160
161 private:
162 size_t dumped_size_{0};
163 std::string segment_id_{};
164 IndexDumper::Pointer dumper_{};
165 IndexPacker packer_{};
166 std::vector<IndexPacker::SegmentMeta> stab_{};
167};
168
169
170} // namespace aitheta2
171
172#endif // __AITHETA2_INDEX_DUMPER_H__
173