1 | /** |
2 | * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | |
16 | * \author Haichao.chc |
17 | * \date Oct 2020 |
18 | * \brief IndexService mainly provides collection management and writing |
19 | * records |
20 | */ |
21 | |
22 | #pragma once |
23 | |
24 | #include <ailego/parallel/thread_pool.h> |
25 | #include "common/config.h" |
26 | #include "common/interface/service.h" |
27 | #include "common/wait_notifier.h" |
28 | #include "collection.h" |
29 | #include "concurrent_hash_map.h" |
30 | #include "typedef.h" |
31 | |
32 | namespace proxima { |
33 | namespace be { |
34 | namespace index { |
35 | |
36 | class IndexService; |
37 | using IndexServicePtr = std::shared_ptr<IndexService>; |
38 | |
39 | /* |
40 | * IndexService is mainly responsible for collection management |
41 | * and record indexing. |
42 | * Collection operations includes create/drop/update .etc |
43 | * Record operations includes insert/delete/query/update .etc |
44 | * And it regularly do snapshot from collection to persist storage. |
45 | */ |
46 | class IndexService : public Service { |
47 | public: |
48 | PROXIMA_DISALLOW_COPY_AND_ASSIGN(IndexService); |
49 | |
50 | //! Constructor |
51 | IndexService() = default; |
52 | |
53 | //! Destructor |
54 | virtual ~IndexService(); |
55 | |
56 | public: |
57 | //! Create collection with schema |
58 | virtual int create_collection(const std::string &collection_name, |
59 | const meta::CollectionMetaPtr &schema); |
60 | |
61 | //! Drop collection by name |
62 | virtual int drop_collection(const std::string &collection_name); |
63 | |
64 | //! Update collection schema |
65 | virtual int update_collection(const std::string &collection_name, |
66 | const meta::CollectionMetaPtr &new_schema); |
67 | |
68 | //! Check if collection exist |
69 | virtual bool has_collection(const std::string &collection_name); |
70 | |
71 | //! Load collections from storage |
72 | virtual int load_collections( |
73 | const std::vector<std::string> &collection_names, |
74 | const std::vector<meta::CollectionMetaPtr> &schemas); |
75 | |
76 | //! List all collection names |
77 | virtual int list_collections(std::vector<std::string> *collection_names); |
78 | |
79 | //! Get collection statistics |
80 | virtual int get_collection_stats(const std::string &collection_name, |
81 | CollectionStats *collection_stats); |
82 | |
83 | //! List all collection segments |
84 | virtual int list_segments(const std::string &collection_name, |
85 | std::vector<SegmentPtr> *segments); |
86 | |
87 | //! Get collection latest lsn and context |
88 | virtual int get_latest_lsn(const std::string &collection_name, uint64_t *lsn, |
89 | std::string *lsn_context); |
90 | |
91 | //! Write records to some collection |
92 | virtual int write_records(const std::string &collection_name, |
93 | const CollectionDatasetPtr &records); |
94 | |
95 | protected: |
96 | //! Initialize inner members |
97 | int init_impl() override; |
98 | |
99 | //! Cleanup and destroy objects |
100 | int cleanup_impl() override; |
101 | |
102 | //! Start worker thread |
103 | int start_impl() override; |
104 | |
105 | //! Stop worker thread |
106 | int stop_impl() override; |
107 | |
108 | private: |
109 | bool load_config(); |
110 | |
111 | void do_routine_flush(); |
112 | |
113 | void do_routine_optimize(); |
114 | |
115 | private: |
116 | ThreadPoolPtr thread_pool_{}; |
117 | ConcurrentHashMap<std::string, CollectionPtr> collections_{}; |
118 | |
119 | std::string index_directory_{}; |
120 | uint32_t thread_count_{0U}; |
121 | uint32_t flush_internal_{0U}; |
122 | uint32_t optimize_internal_{0U}; |
123 | uint32_t concurrency_{0U}; |
124 | bool use_mmap_read_{false}; |
125 | |
126 | WaitNotifier flush_notifier_{}; |
127 | std::atomic<bool> flush_flag_{false}; |
128 | |
129 | WaitNotifier optimize_notifier_{}; |
130 | std::atomic<bool> optimize_flag_{false}; |
131 | }; |
132 | |
133 | |
134 | } // end namespace index |
135 | } // namespace be |
136 | } // end namespace proxima |
137 | |