1 | /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. |
14 | ==============================================================================*/ |
15 | #ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ |
16 | #define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ |
17 | |
18 | #include <memory> |
19 | |
20 | #include "tensorflow/c/experimental/filesystem/filesystem_interface.h" |
21 | #include "tensorflow/core/platform/file_system.h" |
22 | |
23 | /// This file builds classes needed to hold a filesystem implementation in the |
24 | /// modular world. Once all TensorFlow filesystems are converted to use the |
25 | /// plugin based approach, this file will replace the one in core/platform and |
26 | /// the names will lose the `Modular` part. Until that point, the `Modular*` |
27 | /// classes here are experimental and subject to breaking changes. |
28 | /// For documentation on these methods, consult `core/platform/filesystem.h`. |
29 | |
30 | namespace tensorflow { |
31 | |
32 | // TODO(b/143949615): After all filesystems are converted, this file will be |
33 | // moved to core/platform, and this class can become a singleton and replace the |
34 | // need for `Env::Default()`. At that time, we might decide to remove the need |
35 | // for `Env::Default()` altogether, but that's a different project, not in |
36 | // scope for now. I'm just mentioning this here as that transition will mean |
37 | // removal of the registration part from `Env` and adding it here instead: we |
38 | // will need tables to hold for each scheme the function tables that implement |
39 | // the needed functionality instead of the current `FileSystemRegistry` code in |
40 | // `core/platform/env.cc`. |
41 | class ModularFileSystem final : public FileSystem { |
42 | public: |
43 | ModularFileSystem( |
44 | std::unique_ptr<TF_Filesystem> filesystem, |
45 | std::unique_ptr<const TF_FilesystemOps> filesystem_ops, |
46 | std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops, |
47 | std::unique_ptr<const TF_WritableFileOps> writable_file_ops, |
48 | std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> |
49 | read_only_memory_region_ops, |
50 | std::function<void*(size_t)> plugin_memory_allocate, |
51 | std::function<void(void*)> plugin_memory_free) |
52 | : filesystem_(std::move(filesystem)), |
53 | ops_(std::move(filesystem_ops)), |
54 | random_access_file_ops_(std::move(random_access_file_ops)), |
55 | writable_file_ops_(std::move(writable_file_ops)), |
56 | read_only_memory_region_ops_(std::move(read_only_memory_region_ops)), |
57 | plugin_memory_allocate_(std::move(plugin_memory_allocate)), |
58 | plugin_memory_free_(std::move(plugin_memory_free)) {} |
59 | |
60 | ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); } |
61 | |
62 | TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; |
63 | |
64 | Status NewRandomAccessFile( |
65 | const std::string& fname, TransactionToken* token, |
66 | std::unique_ptr<RandomAccessFile>* result) override; |
67 | Status NewWritableFile(const std::string& fname, TransactionToken* token, |
68 | std::unique_ptr<WritableFile>* result) override; |
69 | Status NewAppendableFile(const std::string& fname, TransactionToken* token, |
70 | std::unique_ptr<WritableFile>* result) override; |
71 | Status NewReadOnlyMemoryRegionFromFile( |
72 | const std::string& fname, TransactionToken* token, |
73 | std::unique_ptr<ReadOnlyMemoryRegion>* result) override; |
74 | Status FileExists(const std::string& fname, TransactionToken* token) override; |
75 | bool FilesExist(const std::vector<std::string>& files, |
76 | TransactionToken* token, |
77 | std::vector<Status>* status) override; |
78 | Status GetChildren(const std::string& dir, TransactionToken* token, |
79 | std::vector<std::string>* result) override; |
80 | Status GetMatchingPaths(const std::string& pattern, TransactionToken* token, |
81 | std::vector<std::string>* results) override; |
82 | Status DeleteFile(const std::string& fname, TransactionToken* token) override; |
83 | Status DeleteRecursively(const std::string& dirname, TransactionToken* token, |
84 | int64_t* undeleted_files, |
85 | int64_t* undeleted_dirs) override; |
86 | Status DeleteDir(const std::string& dirname, |
87 | TransactionToken* token) override; |
88 | Status RecursivelyCreateDir(const std::string& dirname, |
89 | TransactionToken* token) override; |
90 | Status CreateDir(const std::string& dirname, |
91 | TransactionToken* token) override; |
92 | Status Stat(const std::string& fname, TransactionToken* token, |
93 | FileStatistics* stat) override; |
94 | Status IsDirectory(const std::string& fname, |
95 | TransactionToken* token) override; |
96 | Status GetFileSize(const std::string& fname, TransactionToken* token, |
97 | uint64* file_size) override; |
98 | Status RenameFile(const std::string& src, const std::string& target, |
99 | TransactionToken* token) override; |
100 | Status CopyFile(const std::string& src, const std::string& target, |
101 | TransactionToken* token) override; |
102 | std::string TranslateName(const std::string& name) const override; |
103 | void FlushCaches(TransactionToken* token) override; |
104 | Status SetOption(const std::string& name, |
105 | const std::vector<string>& values) override; |
106 | Status SetOption(const std::string& name, |
107 | const std::vector<int64_t>& values) override; |
108 | Status SetOption(const std::string& name, |
109 | const std::vector<double>& values) override; |
110 | |
111 | private: |
112 | std::unique_ptr<TF_Filesystem> filesystem_; |
113 | std::unique_ptr<const TF_FilesystemOps> ops_; |
114 | std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops_; |
115 | std::unique_ptr<const TF_WritableFileOps> writable_file_ops_; |
116 | std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> |
117 | read_only_memory_region_ops_; |
118 | std::function<void*(size_t)> plugin_memory_allocate_; |
119 | std::function<void(void*)> plugin_memory_free_; |
120 | TF_DISALLOW_COPY_AND_ASSIGN(ModularFileSystem); |
121 | }; |
122 | |
123 | class ModularRandomAccessFile final : public RandomAccessFile { |
124 | public: |
125 | ModularRandomAccessFile(const std::string& filename, |
126 | std::unique_ptr<TF_RandomAccessFile> file, |
127 | const TF_RandomAccessFileOps* ops) |
128 | : filename_(filename), file_(std::move(file)), ops_(ops) {} |
129 | |
130 | ~ModularRandomAccessFile() override { ops_->cleanup(file_.get()); } |
131 | |
132 | Status Read(uint64 offset, size_t n, StringPiece* result, |
133 | char* scratch) const override; |
134 | Status Name(StringPiece* result) const override; |
135 | |
136 | private: |
137 | std::string filename_; |
138 | std::unique_ptr<TF_RandomAccessFile> file_; |
139 | const TF_RandomAccessFileOps* ops_; // not owned |
140 | TF_DISALLOW_COPY_AND_ASSIGN(ModularRandomAccessFile); |
141 | }; |
142 | |
143 | class ModularWritableFile final : public WritableFile { |
144 | public: |
145 | ModularWritableFile(const std::string& filename, |
146 | std::unique_ptr<TF_WritableFile> file, |
147 | const TF_WritableFileOps* ops) |
148 | : filename_(filename), file_(std::move(file)), ops_(ops) {} |
149 | |
150 | ~ModularWritableFile() override { ops_->cleanup(file_.get()); } |
151 | |
152 | Status Append(StringPiece data) override; |
153 | Status Close() override; |
154 | Status Flush() override; |
155 | Status Sync() override; |
156 | Status Name(StringPiece* result) const override; |
157 | Status Tell(int64_t* position) override; |
158 | |
159 | private: |
160 | std::string filename_; |
161 | std::unique_ptr<TF_WritableFile> file_; |
162 | const TF_WritableFileOps* ops_; // not owned |
163 | TF_DISALLOW_COPY_AND_ASSIGN(ModularWritableFile); |
164 | }; |
165 | |
166 | class ModularReadOnlyMemoryRegion final : public ReadOnlyMemoryRegion { |
167 | public: |
168 | ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region, |
169 | const TF_ReadOnlyMemoryRegionOps* ops) |
170 | : region_(std::move(region)), ops_(ops) {} |
171 | |
172 | ~ModularReadOnlyMemoryRegion() override { ops_->cleanup(region_.get()); }; |
173 | |
174 | const void* data() override { return ops_->data(region_.get()); } |
175 | uint64 length() override { return ops_->length(region_.get()); } |
176 | |
177 | private: |
178 | std::unique_ptr<TF_ReadOnlyMemoryRegion> region_; |
179 | const TF_ReadOnlyMemoryRegionOps* ops_; // not owned |
180 | TF_DISALLOW_COPY_AND_ASSIGN(ModularReadOnlyMemoryRegion); |
181 | }; |
182 | |
183 | // Registers a filesystem plugin so that core TensorFlow can use it. |
184 | Status RegisterFilesystemPlugin(const std::string& dso_path); |
185 | |
186 | } // namespace tensorflow |
187 | |
188 | #endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ |
189 | |