1/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is distributed on an "AS IS" BASIS,
11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15#ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_
16#define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_
17
18#include <memory>
19
20#include "tensorflow/c/experimental/filesystem/filesystem_interface.h"
21#include "tensorflow/core/platform/file_system.h"
22
23/// This file builds classes needed to hold a filesystem implementation in the
24/// modular world. Once all TensorFlow filesystems are converted to use the
25/// plugin based approach, this file will replace the one in core/platform and
26/// the names will lose the `Modular` part. Until that point, the `Modular*`
27/// classes here are experimental and subject to breaking changes.
28/// For documentation on these methods, consult `core/platform/filesystem.h`.
29
30namespace tensorflow {
31
32// TODO(b/143949615): After all filesystems are converted, this file will be
33// moved to core/platform, and this class can become a singleton and replace the
34// need for `Env::Default()`. At that time, we might decide to remove the need
35// for `Env::Default()` altogether, but that's a different project, not in
36// scope for now. I'm just mentioning this here as that transition will mean
37// removal of the registration part from `Env` and adding it here instead: we
38// will need tables to hold for each scheme the function tables that implement
39// the needed functionality instead of the current `FileSystemRegistry` code in
40// `core/platform/env.cc`.
41class ModularFileSystem final : public FileSystem {
42 public:
43 ModularFileSystem(
44 std::unique_ptr<TF_Filesystem> filesystem,
45 std::unique_ptr<const TF_FilesystemOps> filesystem_ops,
46 std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops,
47 std::unique_ptr<const TF_WritableFileOps> writable_file_ops,
48 std::unique_ptr<const TF_ReadOnlyMemoryRegionOps>
49 read_only_memory_region_ops,
50 std::function<void*(size_t)> plugin_memory_allocate,
51 std::function<void(void*)> plugin_memory_free)
52 : filesystem_(std::move(filesystem)),
53 ops_(std::move(filesystem_ops)),
54 random_access_file_ops_(std::move(random_access_file_ops)),
55 writable_file_ops_(std::move(writable_file_ops)),
56 read_only_memory_region_ops_(std::move(read_only_memory_region_ops)),
57 plugin_memory_allocate_(std::move(plugin_memory_allocate)),
58 plugin_memory_free_(std::move(plugin_memory_free)) {}
59
60 ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); }
61
62 TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
63
64 Status NewRandomAccessFile(
65 const std::string& fname, TransactionToken* token,
66 std::unique_ptr<RandomAccessFile>* result) override;
67 Status NewWritableFile(const std::string& fname, TransactionToken* token,
68 std::unique_ptr<WritableFile>* result) override;
69 Status NewAppendableFile(const std::string& fname, TransactionToken* token,
70 std::unique_ptr<WritableFile>* result) override;
71 Status NewReadOnlyMemoryRegionFromFile(
72 const std::string& fname, TransactionToken* token,
73 std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
74 Status FileExists(const std::string& fname, TransactionToken* token) override;
75 bool FilesExist(const std::vector<std::string>& files,
76 TransactionToken* token,
77 std::vector<Status>* status) override;
78 Status GetChildren(const std::string& dir, TransactionToken* token,
79 std::vector<std::string>* result) override;
80 Status GetMatchingPaths(const std::string& pattern, TransactionToken* token,
81 std::vector<std::string>* results) override;
82 Status DeleteFile(const std::string& fname, TransactionToken* token) override;
83 Status DeleteRecursively(const std::string& dirname, TransactionToken* token,
84 int64_t* undeleted_files,
85 int64_t* undeleted_dirs) override;
86 Status DeleteDir(const std::string& dirname,
87 TransactionToken* token) override;
88 Status RecursivelyCreateDir(const std::string& dirname,
89 TransactionToken* token) override;
90 Status CreateDir(const std::string& dirname,
91 TransactionToken* token) override;
92 Status Stat(const std::string& fname, TransactionToken* token,
93 FileStatistics* stat) override;
94 Status IsDirectory(const std::string& fname,
95 TransactionToken* token) override;
96 Status GetFileSize(const std::string& fname, TransactionToken* token,
97 uint64* file_size) override;
98 Status RenameFile(const std::string& src, const std::string& target,
99 TransactionToken* token) override;
100 Status CopyFile(const std::string& src, const std::string& target,
101 TransactionToken* token) override;
102 std::string TranslateName(const std::string& name) const override;
103 void FlushCaches(TransactionToken* token) override;
104 Status SetOption(const std::string& name,
105 const std::vector<string>& values) override;
106 Status SetOption(const std::string& name,
107 const std::vector<int64_t>& values) override;
108 Status SetOption(const std::string& name,
109 const std::vector<double>& values) override;
110
111 private:
112 std::unique_ptr<TF_Filesystem> filesystem_;
113 std::unique_ptr<const TF_FilesystemOps> ops_;
114 std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops_;
115 std::unique_ptr<const TF_WritableFileOps> writable_file_ops_;
116 std::unique_ptr<const TF_ReadOnlyMemoryRegionOps>
117 read_only_memory_region_ops_;
118 std::function<void*(size_t)> plugin_memory_allocate_;
119 std::function<void(void*)> plugin_memory_free_;
120 TF_DISALLOW_COPY_AND_ASSIGN(ModularFileSystem);
121};
122
123class ModularRandomAccessFile final : public RandomAccessFile {
124 public:
125 ModularRandomAccessFile(const std::string& filename,
126 std::unique_ptr<TF_RandomAccessFile> file,
127 const TF_RandomAccessFileOps* ops)
128 : filename_(filename), file_(std::move(file)), ops_(ops) {}
129
130 ~ModularRandomAccessFile() override { ops_->cleanup(file_.get()); }
131
132 Status Read(uint64 offset, size_t n, StringPiece* result,
133 char* scratch) const override;
134 Status Name(StringPiece* result) const override;
135
136 private:
137 std::string filename_;
138 std::unique_ptr<TF_RandomAccessFile> file_;
139 const TF_RandomAccessFileOps* ops_; // not owned
140 TF_DISALLOW_COPY_AND_ASSIGN(ModularRandomAccessFile);
141};
142
143class ModularWritableFile final : public WritableFile {
144 public:
145 ModularWritableFile(const std::string& filename,
146 std::unique_ptr<TF_WritableFile> file,
147 const TF_WritableFileOps* ops)
148 : filename_(filename), file_(std::move(file)), ops_(ops) {}
149
150 ~ModularWritableFile() override { ops_->cleanup(file_.get()); }
151
152 Status Append(StringPiece data) override;
153 Status Close() override;
154 Status Flush() override;
155 Status Sync() override;
156 Status Name(StringPiece* result) const override;
157 Status Tell(int64_t* position) override;
158
159 private:
160 std::string filename_;
161 std::unique_ptr<TF_WritableFile> file_;
162 const TF_WritableFileOps* ops_; // not owned
163 TF_DISALLOW_COPY_AND_ASSIGN(ModularWritableFile);
164};
165
166class ModularReadOnlyMemoryRegion final : public ReadOnlyMemoryRegion {
167 public:
168 ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region,
169 const TF_ReadOnlyMemoryRegionOps* ops)
170 : region_(std::move(region)), ops_(ops) {}
171
172 ~ModularReadOnlyMemoryRegion() override { ops_->cleanup(region_.get()); };
173
174 const void* data() override { return ops_->data(region_.get()); }
175 uint64 length() override { return ops_->length(region_.get()); }
176
177 private:
178 std::unique_ptr<TF_ReadOnlyMemoryRegion> region_;
179 const TF_ReadOnlyMemoryRegionOps* ops_; // not owned
180 TF_DISALLOW_COPY_AND_ASSIGN(ModularReadOnlyMemoryRegion);
181};
182
183// Registers a filesystem plugin so that core TensorFlow can use it.
184Status RegisterFilesystemPlugin(const std::string& dso_path);
185
186} // namespace tensorflow
187
188#endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_
189