1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author Hechong.xyf
17 * \date Mar 2018
18 * \brief Interface of AiTheta Index Meta
19 */
20
21#ifndef __AITHETA2_INDEX_META_H__
22#define __AITHETA2_INDEX_META_H__
23
24#include "index_params.h"
25
26namespace aitheta2 {
27
28/*! Index Meta
29 */
30class IndexMeta {
31 public:
32 /*! Feature Types
33 */
34 enum FeatureTypes {
35 FT_UNDEFINED = 0,
36 FT_BINARY32 = 1,
37 FT_BINARY64 = 2,
38 FT_FP16 = 3,
39 FT_FP32 = 4,
40 FT_FP64 = 5,
41 FT_INT8 = 6,
42 FT_INT16 = 7,
43 FT_INT4 = 8,
44 };
45
46 /*! Major Orders
47 */
48 enum MajorOrders {
49 MO_UNDEFINED = 0,
50 MO_ROW = 1,
51 MO_COLUMN = 2,
52 };
53
54 //! Constructor
55 IndexMeta(void) {
56 this->set_meta(FeatureTypes::FT_FP32, 128u);
57 this->set_measure("SquaredEuclidean", 0, IndexParams());
58 }
59
60 //! Constructor
61 IndexMeta(FeatureTypes tp, uint32_t dim) {
62 this->set_meta(tp, dim);
63 this->set_measure("SquaredEuclidean", 0, IndexParams());
64 }
65
66 //! Constructor
67 IndexMeta(const IndexMeta &rhs)
68 : major_order_(rhs.major_order_),
69 type_(rhs.type_),
70 dimension_(rhs.dimension_),
71 unit_size_(rhs.unit_size_),
72 element_size_(rhs.element_size_),
73 space_id_(rhs.space_id_),
74 measure_revision_(rhs.measure_revision_),
75 converter_revision_(rhs.converter_revision_),
76 reformer_revision_(rhs.reformer_revision_),
77 trainer_revision_(rhs.trainer_revision_),
78 builder_revision_(rhs.builder_revision_),
79 reducer_revision_(rhs.reducer_revision_),
80 searcher_revision_(rhs.searcher_revision_),
81 streamer_revision_(rhs.streamer_revision_),
82 measure_name_(rhs.measure_name_),
83 converter_name_(rhs.converter_name_),
84 reformer_name_(rhs.reformer_name_),
85 trainer_name_(rhs.trainer_name_),
86 builder_name_(rhs.builder_name_),
87 reducer_name_(rhs.reducer_name_),
88 searcher_name_(rhs.searcher_name_),
89 streamer_name_(rhs.streamer_name_),
90 measure_params_(rhs.measure_params_),
91 converter_params_(rhs.converter_params_),
92 reformer_params_(rhs.reformer_params_),
93 trainer_params_(rhs.trainer_params_),
94 builder_params_(rhs.builder_params_),
95 reducer_params_(rhs.reducer_params_),
96 searcher_params_(rhs.searcher_params_),
97 streamer_params_(rhs.streamer_params_),
98 attributes_(rhs.attributes_) {}
99
100 //! Constructor
101 IndexMeta(IndexMeta &&rhs)
102 : major_order_(rhs.major_order_),
103 type_(rhs.type_),
104 dimension_(rhs.dimension_),
105 unit_size_(rhs.unit_size_),
106 element_size_(rhs.element_size_),
107 space_id_(rhs.space_id_),
108 measure_revision_(rhs.measure_revision_),
109 converter_revision_(rhs.converter_revision_),
110 reformer_revision_(rhs.reformer_revision_),
111 trainer_revision_(rhs.trainer_revision_),
112 builder_revision_(rhs.builder_revision_),
113 reducer_revision_(rhs.reducer_revision_),
114 searcher_revision_(rhs.searcher_revision_),
115 streamer_revision_(rhs.streamer_revision_),
116 measure_name_(std::move(rhs.measure_name_)),
117 converter_name_(std::move(rhs.converter_name_)),
118 reformer_name_(std::move(rhs.reformer_name_)),
119 trainer_name_(std::move(rhs.trainer_name_)),
120 builder_name_(std::move(rhs.builder_name_)),
121 reducer_name_(std::move(rhs.reducer_name_)),
122 searcher_name_(std::move(rhs.searcher_name_)),
123 streamer_name_(std::move(rhs.streamer_name_)),
124 measure_params_(std::move(rhs.measure_params_)),
125 converter_params_(std::move(rhs.converter_params_)),
126 reformer_params_(std::move(rhs.reformer_params_)),
127 trainer_params_(std::move(rhs.trainer_params_)),
128 builder_params_(std::move(rhs.builder_params_)),
129 reducer_params_(std::move(rhs.reducer_params_)),
130 searcher_params_(std::move(rhs.searcher_params_)),
131 streamer_params_(std::move(rhs.streamer_params_)),
132 attributes_(std::move(rhs.attributes_)) {}
133
134 //! Assignment
135 IndexMeta &operator=(const IndexMeta &rhs) {
136 major_order_ = rhs.major_order_;
137 type_ = rhs.type_;
138 dimension_ = rhs.dimension_;
139 unit_size_ = rhs.unit_size_;
140 element_size_ = rhs.element_size_;
141 measure_revision_ = rhs.measure_revision_;
142 converter_revision_ = rhs.converter_revision_;
143 reformer_revision_ = rhs.reformer_revision_;
144 trainer_revision_ = rhs.trainer_revision_;
145 builder_revision_ = rhs.builder_revision_;
146 reducer_revision_ = rhs.reducer_revision_;
147 searcher_revision_ = rhs.searcher_revision_;
148 streamer_revision_ = rhs.streamer_revision_;
149 measure_name_ = rhs.measure_name_;
150 converter_name_ = rhs.converter_name_;
151 reformer_name_ = rhs.reformer_name_;
152 trainer_name_ = rhs.trainer_name_;
153 builder_name_ = rhs.builder_name_;
154 reducer_name_ = rhs.reducer_name_;
155 searcher_name_ = rhs.searcher_name_;
156 streamer_name_ = rhs.streamer_name_;
157 measure_params_ = rhs.measure_params_;
158 converter_params_ = rhs.converter_params_;
159 reformer_params_ = rhs.reformer_params_;
160 trainer_params_ = rhs.trainer_params_;
161 builder_params_ = rhs.builder_params_;
162 reducer_params_ = rhs.reducer_params_;
163 searcher_params_ = rhs.searcher_params_;
164 streamer_params_ = rhs.streamer_params_;
165 attributes_ = rhs.attributes_;
166 return *this;
167 }
168
169 //! Assignment
170 IndexMeta &operator=(IndexMeta &&rhs) {
171 major_order_ = rhs.major_order_;
172 type_ = rhs.type_;
173 dimension_ = rhs.dimension_;
174 unit_size_ = rhs.unit_size_;
175 element_size_ = rhs.element_size_;
176 space_id_ = rhs.space_id_;
177 measure_revision_ = rhs.measure_revision_;
178 converter_revision_ = rhs.converter_revision_;
179 reformer_revision_ = rhs.reformer_revision_;
180 trainer_revision_ = rhs.trainer_revision_;
181 builder_revision_ = rhs.builder_revision_;
182 reducer_revision_ = rhs.reducer_revision_;
183 searcher_revision_ = rhs.searcher_revision_;
184 streamer_revision_ = rhs.streamer_revision_;
185 measure_name_ = std::move(rhs.measure_name_);
186 converter_name_ = std::move(rhs.converter_name_);
187 reformer_name_ = std::move(rhs.reformer_name_);
188 trainer_name_ = std::move(rhs.trainer_name_);
189 builder_name_ = std::move(rhs.builder_name_);
190 reducer_name_ = std::move(rhs.reducer_name_);
191 searcher_name_ = std::move(rhs.searcher_name_);
192 streamer_name_ = std::move(rhs.streamer_name_);
193 measure_params_ = std::move(rhs.measure_params_);
194 converter_params_ = std::move(rhs.converter_params_);
195 reformer_params_ = std::move(rhs.reformer_params_);
196 trainer_params_ = std::move(rhs.trainer_params_);
197 builder_params_ = std::move(rhs.builder_params_);
198 reducer_params_ = std::move(rhs.reducer_params_);
199 searcher_params_ = std::move(rhs.searcher_params_);
200 streamer_params_ = std::move(rhs.streamer_params_);
201 attributes_ = std::move(rhs.attributes_);
202 return *this;
203 }
204
205 //! Reset the meta
206 void clear(void) {
207 major_order_ = MajorOrders::MO_UNDEFINED;
208 type_ = FeatureTypes::FT_UNDEFINED;
209 dimension_ = 0;
210 unit_size_ = 0;
211 element_size_ = 0;
212 space_id_ = 0;
213 measure_revision_ = 0;
214 converter_revision_ = 0;
215 reformer_revision_ = 0;
216 trainer_revision_ = 0;
217 builder_revision_ = 0;
218 reducer_revision_ = 0;
219 searcher_revision_ = 0;
220 streamer_revision_ = 0;
221 measure_name_.clear();
222 converter_name_.clear();
223 reformer_name_.clear();
224 trainer_name_.clear();
225 builder_name_.clear();
226 reducer_name_.clear();
227 searcher_name_.clear();
228 streamer_name_.clear();
229 measure_params_.clear();
230 converter_params_.clear();
231 reformer_params_.clear();
232 trainer_params_.clear();
233 builder_params_.clear();
234 reducer_params_.clear();
235 searcher_params_.clear();
236 streamer_params_.clear();
237 attributes_.clear();
238 }
239
240 //! Retrieve major order information
241 MajorOrders major_order(void) const {
242 return major_order_;
243 }
244
245 //! Retrieve type information
246 FeatureTypes type(void) const {
247 return type_;
248 }
249
250 //! Retrieve dimension
251 uint32_t dimension(void) const {
252 return dimension_;
253 }
254
255 //! Retrieve unit size in bytes
256 uint32_t unit_size(void) const {
257 return unit_size_;
258 }
259
260 //! Retrieve element size in bytes
261 uint32_t element_size(void) const {
262 return element_size_;
263 }
264
265 //! Retrieve space id
266 uint64_t space_id(void) const {
267 return space_id_;
268 }
269
270 //! Retrieve revision of measure
271 uint32_t measure_revision(void) const {
272 return measure_revision_;
273 }
274
275 //! Retrieve revision of converter
276 uint32_t converter_revision(void) const {
277 return converter_revision_;
278 }
279
280 //! Retrieve revision of reformer
281 uint32_t reformer_revision(void) const {
282 return reformer_revision_;
283 }
284
285 //! Retrieve revision of trainer
286 uint32_t trainer_revision(void) const {
287 return trainer_revision_;
288 }
289
290 //! Retrieve revision of builder
291 uint32_t builder_revision(void) const {
292 return builder_revision_;
293 }
294
295 //! Retrieve revision of searcher
296 uint32_t searcher_revision(void) const {
297 return searcher_revision_;
298 }
299
300 //! Retrieve revision of reducer
301 uint32_t reducer_revision(void) const {
302 return reducer_revision_;
303 }
304
305 //! Retrieve revision of streamer
306 uint32_t streamer_revision(void) const {
307 return streamer_revision_;
308 }
309
310 //! Retrieve name of measure
311 const std::string &measure_name(void) const {
312 return measure_name_;
313 }
314
315 //! Retrieve name of converter
316 const std::string &converter_name(void) const {
317 return converter_name_;
318 }
319
320 //! Retrieve name of reformer
321 const std::string &reformer_name(void) const {
322 return reformer_name_;
323 }
324
325 //! Retrieve name of trainer
326 const std::string &trainer_name(void) const {
327 return trainer_name_;
328 }
329
330 //! Retrieve name of builder
331 const std::string &builder_name(void) const {
332 return builder_name_;
333 }
334
335 //! Retrieve name of reducer
336 const std::string &reducer_name(void) const {
337 return reducer_name_;
338 }
339
340 //! Retrieve name of searcher
341 const std::string &searcher_name(void) const {
342 return searcher_name_;
343 }
344
345 //! Retrieve name of streamer
346 const std::string &streamer_name(void) const {
347 return streamer_name_;
348 }
349
350 //! Retrieve measure params
351 const IndexParams &measure_params(void) const {
352 return measure_params_;
353 }
354
355 //! Retrieve converter params
356 const IndexParams &converter_params(void) const {
357 return converter_params_;
358 }
359
360 //! Retrieve reformer params
361 const IndexParams &reformer_params(void) const {
362 return reformer_params_;
363 }
364
365 //! Retrieve trainer params
366 const IndexParams &trainer_params(void) const {
367 return trainer_params_;
368 }
369
370 //! Retrieve builder params
371 const IndexParams &builder_params(void) const {
372 return builder_params_;
373 }
374
375 //! Retrieve reducer params
376 const IndexParams &reducer_params(void) const {
377 return reducer_params_;
378 }
379
380 //! Retrieve searcher params
381 const IndexParams &searcher_params(void) const {
382 return searcher_params_;
383 }
384
385 //! Retrieve streamer params
386 const IndexParams &streamer_params(void) const {
387 return streamer_params_;
388 }
389
390 //! Retrieve attributes
391 const IndexParams &attributes(void) const {
392 return attributes_;
393 }
394
395 //! Retrieve mutable attributes
396 IndexParams *mutable_attributes(void) {
397 return &attributes_;
398 }
399
400 //! Set major order of features
401 void set_major_order(MajorOrders order) {
402 major_order_ = order;
403 }
404
405 //! Set dimension of feature
406 void set_dimension(uint32_t dim) {
407 dimension_ = dim;
408 element_size_ = IndexMeta::ElementSizeof(type_, unit_size_, dim);
409 }
410
411 //! Set meta information of feature
412 void set_meta(FeatureTypes tp, uint32_t unit, uint32_t dim) {
413 type_ = tp;
414 dimension_ = dim;
415 unit_size_ = unit;
416 element_size_ = ElementSizeof(tp, unit, dim);
417 }
418
419 //! Set meta information of feature
420 void set_meta(FeatureTypes tp, uint32_t dim) {
421 this->set_meta(tp, UnitSizeof(tp), dim);
422 }
423
424 //! Set space id of index
425 void set_space_id(uint64_t val) {
426 space_id_ = val;
427 }
428
429 //! Set information of measure
430 template <typename TName, typename TParams>
431 void set_measure(TName &&name, uint32_t rev, TParams &&params) {
432 measure_name_ = std::forward<TName>(name);
433 measure_revision_ = rev;
434 measure_params_ = std::forward<TParams>(params);
435 }
436
437 //! Set information of converter
438 template <typename TName, typename TParams>
439 void set_converter(TName &&name, uint32_t rev, TParams &&params) {
440 converter_name_ = std::forward<TName>(name);
441 converter_revision_ = rev;
442 converter_params_ = std::forward<TParams>(params);
443 }
444
445 //! Set information of reformer
446 template <typename TName, typename TParams>
447 void set_reformer(TName &&name, uint32_t rev, TParams &&params) {
448 reformer_name_ = std::forward<TName>(name);
449 reformer_revision_ = rev;
450 reformer_params_ = std::forward<TParams>(params);
451 }
452
453 //! Set information of trainer
454 template <typename TName, typename TParams>
455 void set_trainer(TName &&name, uint32_t rev, TParams &&params) {
456 trainer_name_ = std::forward<TName>(name);
457 trainer_revision_ = rev;
458 trainer_params_ = std::forward<TParams>(params);
459 }
460
461 //! Set information of builder
462 template <typename TName, typename TParams>
463 void set_builder(TName &&name, uint32_t rev, TParams &&params) {
464 builder_name_ = std::forward<TName>(name);
465 builder_revision_ = rev;
466 builder_params_ = std::forward<TParams>(params);
467 }
468
469 //! Set information of reducer
470 template <typename TName, typename TParams>
471 void set_reducer(TName &&name, uint32_t rev, TParams &&params) {
472 reducer_name_ = std::forward<TName>(name);
473 reducer_revision_ = rev;
474 reducer_params_ = std::forward<TParams>(params);
475 }
476
477 //! Set information of searcher
478 template <typename TName, typename TParams>
479 void set_searcher(TName &&name, uint32_t rev, TParams &&params) {
480 searcher_name_ = std::forward<TName>(name);
481 searcher_revision_ = rev;
482 searcher_params_ = std::forward<TParams>(params);
483 }
484
485 //! Set information of streamer
486 template <typename TName, typename TParams>
487 void set_streamer(TName &&name, uint32_t rev, TParams &&params) {
488 streamer_name_ = std::forward<TName>(name);
489 streamer_revision_ = rev;
490 streamer_params_ = std::forward<TParams>(params);
491 }
492
493 //! Serialize meta information into buffer
494 void serialize(std::string *out) const;
495
496 //! Derialize meta information from buffer
497 bool deserialize(const void *data, size_t len);
498
499 //! Retrieve debug information
500 std::string debug_string(void) const;
501
502 //! Calculate unit size of feature
503 static uint32_t UnitSizeof(FeatureTypes ft) {
504 static const uint32_t unit_size_table[] = {
505 0u, // FT_UNDEFINED
506 sizeof(uint32_t), // FT_BINARY32
507 sizeof(uint64_t), // FT_BINARY64
508 sizeof(uint16_t), // FT_FP16
509 sizeof(float), // FT_FP32
510 sizeof(double), // FT_FP64
511 sizeof(int8_t), // FT_INT8
512 sizeof(int16_t), // FT_INT16
513 sizeof(uint8_t) // FT_INT4
514 };
515 return unit_size_table[ft];
516 }
517
518 //! Calculate align size of feature
519 static uint32_t AlignSizeof(FeatureTypes ft) {
520 static const uint32_t align_size_table[] = {
521 0u, // FT_UNDEFINED
522 sizeof(uint32_t), // FT_BINARY32
523 sizeof(uint64_t), // FT_BINARY64
524 sizeof(uint16_t), // FT_FP16
525 sizeof(float), // FT_FP32
526 sizeof(double), // FT_FP64
527 sizeof(int8_t) * 4, // FT_INT8
528 sizeof(int16_t), // FT_INT16
529 sizeof(uint8_t) * 4 // FT_INT4
530 };
531 return align_size_table[ft];
532 }
533
534 //! Calculate element size of feature
535 static uint32_t ElementSizeof(FeatureTypes ft, uint32_t unit, uint32_t dim) {
536 switch (ft) {
537 case FeatureTypes::FT_UNDEFINED:
538 return 0;
539 case FeatureTypes::FT_BINARY32:
540 case FeatureTypes::FT_BINARY64:
541 return (dim + unit * 8 - 1) / (unit * 8) * unit;
542 case FeatureTypes::FT_FP16:
543 case FeatureTypes::FT_FP32:
544 case FeatureTypes::FT_FP64:
545 case FeatureTypes::FT_INT8:
546 case FeatureTypes::FT_INT16:
547 return (dim * unit);
548 case FeatureTypes::FT_INT4:
549 return (dim + unit * 2 - 1) / (unit * 2) * unit;
550 }
551 return 0;
552 }
553
554 //! Calculate element size of feature
555 static uint32_t ElementSizeof(FeatureTypes ft, uint32_t dim) {
556 return ElementSizeof(ft, UnitSizeof(ft), dim);
557 }
558
559 private:
560 MajorOrders major_order_{MajorOrders::MO_UNDEFINED};
561 FeatureTypes type_{FeatureTypes::FT_UNDEFINED};
562 uint32_t dimension_{0};
563 uint32_t unit_size_{0};
564 uint32_t element_size_{0};
565 uint64_t space_id_{0};
566 uint32_t measure_revision_{0};
567 uint32_t converter_revision_{0};
568 uint32_t reformer_revision_{0};
569 uint32_t trainer_revision_{0};
570 uint32_t builder_revision_{0};
571 uint32_t reducer_revision_{0};
572 uint32_t searcher_revision_{0};
573 uint32_t streamer_revision_{0};
574 std::string measure_name_{};
575 std::string converter_name_{};
576 std::string reformer_name_{};
577 std::string trainer_name_{};
578 std::string builder_name_{};
579 std::string reducer_name_{};
580 std::string searcher_name_{};
581 std::string streamer_name_{};
582 IndexParams measure_params_{};
583 IndexParams converter_params_{};
584 IndexParams reformer_params_{};
585 IndexParams trainer_params_{};
586 IndexParams builder_params_{};
587 IndexParams reducer_params_{};
588 IndexParams searcher_params_{};
589 IndexParams streamer_params_{};
590 IndexParams attributes_{};
591};
592
593/*! Index Query Meta
594 */
595class IndexQueryMeta {
596 public:
597 //! Constructor
598 IndexQueryMeta(void) {}
599
600 //! Constructor
601 IndexQueryMeta(IndexMeta::FeatureTypes ft, uint32_t unit, uint32_t dim)
602 : type_(ft),
603 dimension_(dim),
604 unit_size_(unit),
605 element_size_(IndexMeta::ElementSizeof(ft, unit, dim)) {}
606
607 //! Constructor
608 IndexQueryMeta(IndexMeta::FeatureTypes ft, uint32_t dim)
609 : IndexQueryMeta{ft, IndexMeta::UnitSizeof(ft), dim} {}
610
611 //! Retrieve type of features
612 IndexMeta::FeatureTypes type(void) const {
613 return type_;
614 }
615
616 //! Retrieve dimension of features
617 uint32_t dimension(void) const {
618 return dimension_;
619 }
620
621 //! Retrieve unit size of feature
622 uint32_t unit_size(void) const {
623 return unit_size_;
624 }
625
626 //! Retrieve element size of feature
627 uint32_t element_size(void) const {
628 return element_size_;
629 }
630
631 //! Set dimension of feature
632 void set_dimension(uint32_t dim) {
633 dimension_ = dim;
634 element_size_ = IndexMeta::ElementSizeof(type_, unit_size_, dim);
635 }
636
637 //! Set meta information of feature
638 void set_meta(IndexMeta::FeatureTypes tp, uint32_t unit, uint32_t dim) {
639 type_ = tp;
640 dimension_ = dim;
641 unit_size_ = unit;
642 element_size_ = IndexMeta::ElementSizeof(tp, unit, dim);
643 }
644
645 //! Set meta information of feature
646 void set_meta(IndexMeta::FeatureTypes tp, uint32_t dim) {
647 this->set_meta(tp, IndexMeta::UnitSizeof(tp), dim);
648 }
649
650 private:
651 IndexMeta::FeatureTypes type_{IndexMeta::FT_UNDEFINED};
652 uint32_t dimension_{0};
653 uint32_t unit_size_{0};
654 uint32_t element_size_{0};
655};
656
657} // namespace aitheta2
658
659#endif // __AITHETA2_INDEX_META_H__
660