1/**
2 * Copyright 2021 Alibaba, Inc. and its affiliates. All Rights Reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15
16 * \author Haichao.chc
17 * \date Oct 2020
18 * \brief IndexService mainly provides collection management and writing
19 * records
20 */
21
22#pragma once
23
24#include <ailego/parallel/thread_pool.h>
25#include "common/config.h"
26#include "common/interface/service.h"
27#include "common/wait_notifier.h"
28#include "collection.h"
29#include "concurrent_hash_map.h"
30#include "typedef.h"
31
32namespace proxima {
33namespace be {
34namespace index {
35
36class IndexService;
37using IndexServicePtr = std::shared_ptr<IndexService>;
38
39/*
40 * IndexService is mainly responsible for collection management
41 * and record indexing.
42 * Collection operations includes create/drop/update .etc
43 * Record operations includes insert/delete/query/update .etc
44 * And it regularly do snapshot from collection to persist storage.
45 */
46class IndexService : public Service {
47 public:
48 PROXIMA_DISALLOW_COPY_AND_ASSIGN(IndexService);
49
50 //! Constructor
51 IndexService() = default;
52
53 //! Destructor
54 virtual ~IndexService();
55
56 public:
57 //! Create collection with schema
58 virtual int create_collection(const std::string &collection_name,
59 const meta::CollectionMetaPtr &schema);
60
61 //! Drop collection by name
62 virtual int drop_collection(const std::string &collection_name);
63
64 //! Update collection schema
65 virtual int update_collection(const std::string &collection_name,
66 const meta::CollectionMetaPtr &new_schema);
67
68 //! Check if collection exist
69 virtual bool has_collection(const std::string &collection_name);
70
71 //! Load collections from storage
72 virtual int load_collections(
73 const std::vector<std::string> &collection_names,
74 const std::vector<meta::CollectionMetaPtr> &schemas);
75
76 //! List all collection names
77 virtual int list_collections(std::vector<std::string> *collection_names);
78
79 //! Get collection statistics
80 virtual int get_collection_stats(const std::string &collection_name,
81 CollectionStats *collection_stats);
82
83 //! List all collection segments
84 virtual int list_segments(const std::string &collection_name,
85 std::vector<SegmentPtr> *segments);
86
87 //! Get collection latest lsn and context
88 virtual int get_latest_lsn(const std::string &collection_name, uint64_t *lsn,
89 std::string *lsn_context);
90
91 //! Write records to some collection
92 virtual int write_records(const std::string &collection_name,
93 const CollectionDatasetPtr &records);
94
95 protected:
96 //! Initialize inner members
97 int init_impl() override;
98
99 //! Cleanup and destroy objects
100 int cleanup_impl() override;
101
102 //! Start worker thread
103 int start_impl() override;
104
105 //! Stop worker thread
106 int stop_impl() override;
107
108 private:
109 bool load_config();
110
111 void do_routine_flush();
112
113 void do_routine_optimize();
114
115 private:
116 ThreadPoolPtr thread_pool_{};
117 ConcurrentHashMap<std::string, CollectionPtr> collections_{};
118
119 std::string index_directory_{};
120 uint32_t thread_count_{0U};
121 uint32_t flush_internal_{0U};
122 uint32_t optimize_internal_{0U};
123 uint32_t concurrency_{0U};
124 bool use_mmap_read_{false};
125
126 WaitNotifier flush_notifier_{};
127 std::atomic<bool> flush_flag_{false};
128
129 WaitNotifier optimize_notifier_{};
130 std::atomic<bool> optimize_flag_{false};
131};
132
133
134} // end namespace index
135} // namespace be
136} // end namespace proxima
137