1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *
16 * \author guonix
17 * \date Oct 2020
18 * \brief
19 */
20
21#pragma once
22
23#include <memory>
24#include <vector>
25#include <ailego/encoding/uri.h>
26#include <aitheta2/index_params.h>
27#include "common/error_code.h"
28#include "common/logger.h"
29#include "common/types.h"
30#include "common/uuid_helper.h"
31
32namespace proxima {
33namespace be {
34namespace meta {
35
36// Predefine classes
37class RepositoryBase;
38class DatabaseRepositoryMeta;
39class ColumnMeta;
40class CollectionBase;
41class CollectionMeta;
42
43// Alias for meta types
44using RepositoryBasePtr = std::shared_ptr<RepositoryBase>;
45using DatabaseRepositoryMetaPtr = std::shared_ptr<DatabaseRepositoryMeta>;
46using ColumnMetaPtr = std::shared_ptr<ColumnMeta>;
47using ColumnMetaPtrList = std::vector<ColumnMetaPtr>;
48using CollectionMetaPtr = std::shared_ptr<CollectionMeta>;
49using CollectionMetaPtrList = std::vector<CollectionMetaPtr>;
50
51//! Alias
52using IndexParams = aitheta2::IndexParams;
53
54#define META_VERIFY_ARGUMENTS(COND, CODE, MSG) \
55 if ((COND)) { \
56 LOG_ERROR(MSG); \
57 return CODE; \
58 }
59
60
61/*!
62 * ColumnMeta illustrate Column in Collection
63 */
64class ColumnMeta {
65 public:
66 //! Constructors
67 ColumnMeta() = default;
68
69 explicit ColumnMeta(std::string &&column_name)
70 : name_(std::move(column_name)),
71 uid_(gen_uuid()),
72 index_type_(IndexTypes::PROXIMA_GRAPH_INDEX),
73 data_type_(DataTypes::UNDEFINED),
74 dimension_(0) {}
75
76 explicit ColumnMeta(const ColumnMeta &param)
77 : name_(param.name_),
78 uid_(param.uid_),
79 index_type_(param.index_type_),
80 data_type_(param.data_type_),
81 dimension_(param.dimension_),
82 parameters_(param.parameters_) {}
83
84 //! Destructor
85 ~ColumnMeta() = default;
86
87 public:
88 int validate() const {
89 META_VERIFY_ARGUMENTS(
90 name_.empty(), PROXIMA_BE_ERROR_CODE(EmptyColumnName),
91 "Invalid arguments for create collection, Name of collection "
92 "can't be empty")
93
94 META_VERIFY_ARGUMENTS(data_type_ == DataTypes::UNDEFINED,
95 PROXIMA_BE_ERROR_CODE(InvalidDataType),
96 "Invalid data types")
97 return 0;
98 }
99
100 //! Retrieve name field
101 const std::string &name() const {
102 return name_;
103 }
104
105 //! Set name
106 void set_name(const std::string &new_name) {
107 name_ = new_name;
108 }
109
110 //! Retrieve mutable name field
111 std::string *mutable_name() {
112 return &name_;
113 }
114
115 //! Retrieve column uid
116 const std::string &uid() const {
117 return uid_;
118 }
119
120 //! Retrieve mutable column uid
121 std::string *mutable_uid() {
122 return &uid_;
123 }
124
125 //! Set column uid
126 void set_uid(const std::string &new_uid) {
127 uid_ = new_uid;
128 }
129
130 //! Retrieve index type
131 IndexTypes index_type() const {
132 return index_type_;
133 }
134
135 //! Set index type
136 void set_index_type(IndexTypes type) {
137 index_type_ = type;
138 }
139
140 //! Retrieve data type
141 DataTypes data_type() const {
142 return data_type_;
143 }
144
145 //! Set data type
146 void set_data_type(DataTypes type) {
147 data_type_ = type;
148 }
149
150 //! Retrieve index type
151 uint32_t dimension() const {
152 return dimension_;
153 }
154
155 //! Set index type
156 void set_dimension(uint32_t new_dimension) {
157 dimension_ = new_dimension;
158 }
159
160 //! Retrieve params
161 const IndexParams &parameters() const {
162 return parameters_;
163 }
164
165 //! Get mutable params
166 IndexParams *mutable_parameters() {
167 return &parameters_;
168 }
169
170 int check_updated_field(const ColumnMeta &param) const {
171 if (name_ != param.name()) {
172 return PROXIMA_BE_ERROR_CODE(UpdateColumnNameField);
173 } else if (index_type_ != param.index_type()) {
174 return PROXIMA_BE_ERROR_CODE(UpdateIndexTypeField);
175 } else if (data_type_ != param.data_type()) {
176 return PROXIMA_BE_ERROR_CODE(UpdateDataTypeField);
177 } else if (dimension_ != param.dimension()) {
178 } else {
179 std::string meta_params, params;
180 IndexParams::SerializeToBuffer(param.parameters(), &meta_params);
181 IndexParams::SerializeToBuffer(parameters_, &params);
182 if (params != meta_params) {
183 return PROXIMA_BE_ERROR_CODE(UpdateParametersField);
184 }
185 }
186 LOG_INFO("Input column was check_updated_field");
187 return 0;
188 }
189
190 private:
191 //! Column name
192 std::string name_{};
193
194 //! Unique ID of column
195 std::string uid_{};
196
197 //! The type of column, possible value listed bellow:
198 //! 1. PROXIMA_GRAPH_INDEX: vector index, default value
199 //! 2. ...
200 IndexTypes index_type_{IndexTypes::PROXIMA_GRAPH_INDEX};
201
202 //! Data Types
203 DataTypes data_type_{DataTypes::UNDEFINED};
204
205 //! Dimension of data
206 uint32_t dimension_{0u};
207
208 //! Dictionary of params
209 IndexParams parameters_{};
210};
211
212
213/*!
214 * Collection Status
215 */
216enum class CollectionStatus : uint32_t {
217 INITIALIZED = 0, // Collection has been initialized, ready to serving
218 SERVING, // Collection is serving
219 DROPPED, // Collection has been dropped
220};
221
222
223//! Alias ForwardColumn
224using ForwardColumn = std::string;
225//! Alias ForwardColumns
226using ForwardColumns = std::vector<ForwardColumn>;
227
228
229/*!
230 * Types of Repository
231 */
232enum class RepositoryTypes : int {
233 //! Undefined repo type
234 UNDEFINED,
235 //! DATABASE
236 DATABASE
237};
238
239/*!
240 * Repository Base Object
241 */
242class RepositoryBase {
243 public:
244 //! Constructors
245 RepositoryBase() = default;
246
247 explicit RepositoryBase(RepositoryTypes repo_type) : type_(repo_type) {}
248
249 explicit RepositoryBase(const RepositoryBase &repo)
250 : name_(repo.name_), type_(repo.type_) {}
251
252 //! Destructor
253 virtual ~RepositoryBase() = default;
254
255 public:
256 //! Validate repository object
257 virtual int validate() const {
258 META_VERIFY_ARGUMENTS(
259 name_.empty(), PROXIMA_BE_ERROR_CODE(EmptyRepositoryName),
260 "Invalid arguments for create collection, empty repository name.")
261 return 0;
262 }
263
264 //! Retrieve repository name
265 const std::string &name() const {
266 return name_;
267 }
268
269 //! Retrieve mutable repository name
270 std::string *mutable_name() {
271 return &name_;
272 }
273
274 //! Set repository name
275 void set_name(const std::string &new_name) {
276 name_ = new_name;
277 }
278
279 RepositoryTypes type() const {
280 return type_;
281 }
282
283 void set_type(RepositoryTypes new_type) {
284 type_ = new_type;
285 }
286
287 private:
288 //! Name of repository
289 std::string name_{};
290
291 //! Type of repository
292 RepositoryTypes type_{RepositoryTypes::UNDEFINED};
293};
294
295
296/*!
297 * Repository Meta
298 */
299class DatabaseRepositoryMeta : public RepositoryBase {
300 public:
301 //! Constructors
302 DatabaseRepositoryMeta() : RepositoryBase(RepositoryTypes::DATABASE) {}
303
304 explicit DatabaseRepositoryMeta(const DatabaseRepositoryMeta &param)
305 : RepositoryBase(param),
306 connection_(param.connection_),
307 user_(param.user_),
308 password_(param.password_),
309 table_name_(param.table_name_) {}
310
311 //! Destructor
312 ~DatabaseRepositoryMeta() override = default;
313
314 public:
315 //! Validate repository object
316 int validate() const override {
317 int code = RepositoryBase::validate();
318 if (code != 0) {
319 return code;
320 }
321
322 ailego::Uri uri(connection_);
323 META_VERIFY_ARGUMENTS(!uri.is_valid(), PROXIMA_BE_ERROR_CODE(InvalidURI),
324 "Invalid arguments for create collection, URI is "
325 "invalid.")
326
327 META_VERIFY_ARGUMENTS(
328 user_.empty(), PROXIMA_BE_ERROR_CODE(EmptyUserName),
329 "Invalid arguments for create collection, empty user name.")
330
331 META_VERIFY_ARGUMENTS(
332 password_.empty(), PROXIMA_BE_ERROR_CODE(EmptyPassword),
333 "Invalid arguments for create collection, empty password.")
334
335 META_VERIFY_ARGUMENTS(table_name_.empty(),
336 PROXIMA_BE_ERROR_CODE(EmptyRepositoryTable),
337 "Invalid arguments for create collection, "
338 "repository_table can't be empty")
339 return 0;
340 }
341
342 //! merge repository
343 int merge_repository(DatabaseRepositoryMetaPtr repo) {
344 set_name(repo->name());
345 connection_.assign(repo->connection());
346 user_.assign(repo->user());
347 password_.assign(repo->password());
348 table_name_.assign(repo->table_name());
349 return 0;
350 }
351
352 //! Retrieve connection uri
353 const std::string &connection() const {
354 return connection_;
355 }
356
357 //! Retrieve mutable connection uris
358 std::string *mutable_connection() {
359 return &connection_;
360 }
361
362 //! Set connection uri
363 void set_connection(const std::string &uri) {
364 connection_ = uri;
365 }
366
367 //! Retrieve user of repository
368 const std::string &user() const {
369 return user_;
370 }
371
372 //! Retrieve mutable user of repository
373 std::string *mutable_user() {
374 return &user_;
375 }
376
377 //! Set user of repository
378 void set_user(const std::string &user_name) {
379 user_ = user_name;
380 }
381
382 //! Retrieve password of user
383 const std::string &password() const {
384 return password_;
385 }
386
387 //! Retrieve mutable password of user
388 std::string *mutable_password() {
389 return &password_;
390 }
391
392 //! Set password
393 void set_password(const std::string &pass) {
394 password_ = pass;
395 }
396
397 //! Retrieve the table of repository
398 const std::string &table_name() const {
399 return table_name_;
400 }
401
402 //! Retrieve the mutable table of repository
403 std::string *mutable_table_name() {
404 return &table_name_;
405 }
406
407 //! Set repository_table
408 void set_table_name(const std::string &new_table_name) {
409 table_name_ = new_table_name;
410 }
411
412 private:
413 //! JDBC Connection URI
414 std::string connection_{};
415
416 //! User name
417 std::string user_{};
418
419 //! User password
420 std::string password_{};
421
422 //! Source table of repository
423 std::string table_name_{};
424};
425
426//! Repository type helper
427struct RepositoryHelper {
428 //! Get child pointer from base pointer
429 template <typename REPO, typename = typename std::enable_if<std::is_base_of<
430 RepositoryBase, REPO>::value>::type>
431 static std::shared_ptr<typename std::remove_cv<REPO>::type> Child(
432 RepositoryBasePtr repo) {
433 typedef typename std::remove_cv<REPO>::type CHILD;
434 return std::dynamic_pointer_cast<CHILD>(repo);
435 }
436
437 //! Construct one child from base pointer
438 template <typename REPO, typename = typename std::enable_if<std::is_base_of<
439 RepositoryBase, REPO>::value>::type>
440 static std::shared_ptr<typename std::remove_cv<REPO>::type> NewChild(
441 RepositoryBasePtr repo) {
442 typedef typename std::remove_cv<REPO>::type CHILD;
443 auto child = RepositoryHelper::Child<CHILD>(repo);
444 return std::make_shared<CHILD>(*child);
445 }
446
447 //! Copy child repository from base pointer
448 static RepositoryBasePtr CopyRepository(RepositoryBasePtr base) {
449 if (base) {
450 switch (base->type()) {
451 case RepositoryTypes::DATABASE:
452 return RepositoryHelper::NewChild<DatabaseRepositoryMeta>(base);
453 case RepositoryTypes::UNDEFINED:
454 LOG_WARN("Ignore undefined repository type");
455 }
456 }
457 return nullptr;
458 }
459};
460
461/*!
462 * CollectionBase object
463 */
464class CollectionBase {
465 public:
466 constexpr static uint64_t UNLIMITED_DOCS_PER_SEGMENT =
467 std::numeric_limits<uint64_t>::max();
468
469 public:
470 //! Constructors
471 CollectionBase() = default;
472
473 explicit CollectionBase(const CollectionBase &param)
474 : name_(param.name_),
475 max_docs_per_segment_(param.max_docs_per_segment_),
476 forward_columns_(param.forward_columns_) {
477 for (auto column : param.index_columns_) {
478 index_columns_.emplace_back(std::make_shared<ColumnMeta>(*column));
479 }
480
481 if (param.repository_) {
482 repository_ = RepositoryHelper::CopyRepository(param.repository_);
483 }
484 }
485
486 //! Destructor
487 virtual ~CollectionBase() = default;
488
489 public:
490 //! Validate params, 0 for valid, otherwise invalid
491 virtual int validate() const {
492 META_VERIFY_ARGUMENTS(name_.empty(),
493 PROXIMA_BE_ERROR_CODE(EmptyCollectionName),
494 "Invalid name of collection")
495 META_VERIFY_ARGUMENTS(max_docs_per_segment_ == 0,
496 PROXIMA_BE_ERROR_CODE(ZeroDocsPerSegment),
497 "Max doc per segment can't be 0")
498 META_VERIFY_ARGUMENTS(index_columns_.empty(),
499 PROXIMA_BE_ERROR_CODE(EmptyColumns), "Empty Columns")
500
501 int code = 0;
502 for (auto &index : index_columns_) {
503 code = index->validate();
504 if (code != 0) {
505 break;
506 }
507 }
508
509 if (code == 0 && repository_) {
510 code = repository_->validate();
511 }
512 return code;
513 }
514
515 //! Retrieve name
516 const std::string &name() const {
517 return name_;
518 }
519
520 //! Retrieve mutable name
521 std::string *mutable_name() {
522 return &name_;
523 }
524
525 //! Set name
526 void set_name(const std::string &new_name) {
527 name_ = new_name;
528 }
529
530 //! Retrieve split index size
531 uint64_t max_docs_per_segment() const {
532 return max_docs_per_segment_;
533 }
534
535 //! Set split index size, 0 means unlimited segment size
536 void set_max_docs_per_segment(uint64_t count) {
537 if (count == 0) {
538 count = UNLIMITED_DOCS_PER_SEGMENT;
539 }
540 max_docs_per_segment_ = count;
541 }
542
543 //! Retrieve forward columns
544 const ForwardColumns &forward_columns() const {
545 return forward_columns_;
546 }
547
548 //! Retrieve mutable forward columns
549 ForwardColumns *mutable_forward_columns() {
550 return &forward_columns_;
551 }
552
553 //! Retrieve index columns
554 const ColumnMetaPtrList &index_columns() const {
555 return index_columns_;
556 }
557
558 //! Retrieve mutable index columns
559 ColumnMetaPtrList *mutable_index_columns() {
560 return &index_columns_;
561 }
562
563 //! Append column
564 void append(ColumnMetaPtr param) {
565 index_columns_.emplace_back(param);
566 }
567
568 //! Append column
569 void append(const ColumnMeta &param) {
570 index_columns_.emplace_back(std::make_shared<ColumnMeta>(param));
571 }
572
573 //! Retrieve repository member
574 const RepositoryBasePtr repository() const {
575 return repository_;
576 }
577
578 //! Set repository
579 void set_repository(RepositoryBasePtr repo) {
580 repository_ = repo;
581 }
582
583 const std::string &repository_name() const {
584 static const std::string kEmpty = "";
585 if (repository_) {
586 return repository_->name();
587 }
588 return kEmpty;
589 }
590
591 public:
592 //! Update columns
593 int update_columns(const ColumnMetaPtrList &columns) {
594 ColumnMetaPtrList merged_columns;
595 for (auto column : columns) {
596 auto ptr = column_by_name(column->name());
597 if (ptr) {
598 // Does not support update any of fields of column yet, so copy column
599 // to newer collection.
600 int code = ptr->check_updated_field(*column);
601 if (code != 0) {
602 return code;
603 }
604 merged_columns.emplace_back(std::make_shared<ColumnMeta>(*ptr));
605 } else {
606 auto new_column = std::make_shared<ColumnMeta>(*column);
607 new_column->set_uid(gen_uuid());
608 merged_columns.emplace_back(new_column);
609 }
610 }
611 index_columns_ = merged_columns;
612 return 0;
613 }
614
615 //! Find column which named by name
616 ColumnMetaPtr column_by_name(const std::string &column_name) const {
617 return find_column_by_filter(
618 [&column_name](ColumnMetaPtr column_ptr) -> bool {
619 return column_ptr->name() == column_name;
620 });
621 }
622
623 private:
624 //! Find column by filter
625 ColumnMetaPtr find_column_by_filter(
626 std::function<bool(ColumnMetaPtr)> filter) const {
627 for (auto &c : index_columns_) {
628 if (filter(c)) {
629 return c;
630 }
631 }
632 return nullptr;
633 }
634
635 private:
636 //! The name of collection, unique name
637 std::string name_{};
638
639 //! The max document count of segments, optional field, default is Max
640 // value of system.
641 uint64_t max_docs_per_segment_{0};
642
643 //! Forward columns
644 ForwardColumns forward_columns_{};
645
646 //! Indices
647 ColumnMetaPtrList index_columns_{};
648
649 //! Repository of collection, Optional field
650 RepositoryBasePtr repository_{nullptr};
651};
652
653
654/*!
655 * CollectionMeta
656 */
657class CollectionMeta : public CollectionBase {
658 public:
659 //! InvalidRevision
660 constexpr static uint32_t INVALID_REVISION =
661 std::numeric_limits<uint32_t>::max();
662
663 public:
664 //! Constructors
665 CollectionMeta()
666 : CollectionBase(),
667 uid_(gen_uuid()),
668 readable_(true),
669 writable_(true),
670 current_(true) {}
671
672 explicit CollectionMeta(const CollectionMeta &meta)
673 : CollectionBase(meta),
674 uid_(meta.uid_),
675 readable_(meta.readable_),
676 writable_(meta.writable_),
677 revision_(meta.revision_),
678 status_(meta.status_),
679 current_(meta.current_) {}
680
681 explicit CollectionMeta(const CollectionBase &param)
682 : CollectionBase(param),
683 uid_(gen_uuid()),
684 readable_(true),
685 writable_(true),
686 current_(true) {}
687
688 //! Destructor
689 ~CollectionMeta() override = default;
690
691 public:
692 //! Retrieve collection uid, indicate a group of collections
693 const std::string &uid() const {
694 return uid_;
695 }
696
697 //! Retrieve mutable collection uid, indicate a group of collections
698 std::string *mutable_uid() {
699 return &uid_;
700 }
701
702 //! Set collection uid
703 void set_uid(const std::string &new_uid) {
704 uid_ = new_uid;
705 }
706
707 //! Check readable
708 bool readable() const {
709 return readable_;
710 }
711
712 //! Set readable
713 void set_readable(bool flag) {
714 readable_ = flag;
715 }
716
717 //! Check writable
718 bool writable() const {
719 return writable_;
720 }
721
722 //! Set writable
723 void set_writable(bool flag) {
724 writable_ = flag;
725 }
726
727 //! Retrieve revision
728 uint32_t revision() const {
729 return revision_;
730 }
731
732 //! Set revision
733 void set_revision(uint32_t new_revision) {
734 revision_ = new_revision;
735 }
736
737 //! Invalid revision
738 bool invalid_revision() const {
739 return revision_ == INVALID_REVISION;
740 }
741
742 //! Increase revision number
743 void increase_revision(uint32_t step = 1) {
744 revision_ += step;
745 }
746
747 //! Retrieve status
748 CollectionStatus status() const {
749 return status_;
750 }
751
752 //! Check is INITIALIZED
753 bool initialized() const {
754 return status_ == CollectionStatus::INITIALIZED;
755 }
756
757 //! Check is serving
758 bool serving() const {
759 return status_ == CollectionStatus::SERVING;
760 }
761
762 //! Set status
763 void set_status(CollectionStatus new_status) {
764 status_ = new_status;
765 }
766
767 //! Check current flag
768 bool is_current() const {
769 return current_;
770 }
771
772 //! Set current flag
773 void set_current(bool flag = true) {
774 current_ = flag;
775 }
776
777 //! Merge update param
778 int merge_update_param(const CollectionBase &param) {
779 set_max_docs_per_segment(param.max_docs_per_segment());
780 mutable_forward_columns()->assign(param.forward_columns().begin(),
781 param.forward_columns().end());
782
783 if (repository() && param.repository()) {
784 auto repo = RepositoryHelper::Child<DatabaseRepositoryMeta>(repository());
785 repo->merge_repository(
786 RepositoryHelper::Child<DatabaseRepositoryMeta>(param.repository()));
787 } else if (!repository() && !param.repository()) {
788 } else {
789 return PROXIMA_BE_ERROR_CODE(UpdateRepositoryTypeField);
790 }
791
792 return update_columns(param.index_columns());
793 }
794
795 private:
796 //! Unique id of collection
797 std::string uid_{};
798
799 //! Readable flag, default is true
800 bool readable_{false};
801
802 //! Writable flag, default is true
803 bool writable_{false};
804
805 //! Revision number
806 uint32_t revision_{0};
807
808 //! One of values: initialized, serving ...
809 CollectionStatus status_{CollectionStatus::INITIALIZED};
810
811 //! True for indicated current used version, otherwise false
812 bool current_{false};
813};
814
815#undef META_VERIFY_ARGUMENTS
816
817} // namespace meta
818} // namespace be
819} // namespace proxima
820