1/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is distributed on an "AS IS" BASIS,
11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15
16#ifndef TENSORFLOW_TSL_PLATFORM_ENV_H_
17#define TENSORFLOW_TSL_PLATFORM_ENV_H_
18
19#include <stdint.h>
20
21#include <memory>
22#include <string>
23#include <unordered_map>
24#include <vector>
25
26#include "tensorflow/tsl/platform/env_time.h"
27#include "tensorflow/tsl/platform/errors.h"
28#include "tensorflow/tsl/platform/file_system.h"
29#include "tensorflow/tsl/platform/macros.h"
30#include "tensorflow/tsl/platform/mutex.h"
31#include "tensorflow/tsl/platform/numa.h"
32#include "tensorflow/tsl/platform/platform.h"
33#include "tensorflow/tsl/platform/protobuf.h"
34#include "tensorflow/tsl/platform/status.h"
35#include "tensorflow/tsl/platform/stringpiece.h"
36#include "tensorflow/tsl/platform/types.h"
37
38// Delete leaked Windows definitions.
39#ifdef PLATFORM_WINDOWS
40#undef CopyFile
41#undef DeleteFile
42#endif
43
44namespace tsl {
45
46class Thread;
47struct ThreadOptions;
48
49/// \brief An interface used by the tensorflow implementation to
50/// access operating system functionality like the filesystem etc.
51///
52/// Callers may wish to provide a custom Env object to get fine grain
53/// control.
54///
55/// All Env implementations are safe for concurrent access from
56/// multiple threads without any external synchronization.
57class Env {
58 public:
59 Env();
60 virtual ~Env() = default;
61
62 /// \brief Returns a default environment suitable for the current operating
63 /// system.
64 ///
65 /// Sophisticated users may wish to provide their own Env
66 /// implementation instead of relying on this default environment.
67 ///
68 /// The result of Default() belongs to this library and must never be deleted.
69 static Env* Default();
70
71 /// \brief Returns the FileSystem object to handle operations on the file
72 /// specified by 'fname'. The FileSystem object is used as the implementation
73 /// for the file system related (non-virtual) functions that follow.
74 /// Returned FileSystem object is still owned by the Env object and will
75 // (might) be destroyed when the environment is destroyed.
76 virtual Status GetFileSystemForFile(const std::string& fname,
77 FileSystem** result);
78
79 /// \brief Returns the file system schemes registered for this Env.
80 virtual Status GetRegisteredFileSystemSchemes(
81 std::vector<std::string>* schemes);
82
83 /// \brief Register a file system for a scheme.
84 virtual Status RegisterFileSystem(const std::string& scheme,
85 FileSystemRegistry::Factory factory);
86
87 /// \brief Register a modular file system for a scheme.
88 ///
89 /// Same as `RegisterFileSystem` but for filesystems provided by plugins.
90 ///
91 /// TODO(b/139060984): After all filesystems are converted, make this be the
92 /// canonical registration function.
93 virtual Status RegisterFileSystem(const std::string& scheme,
94 std::unique_ptr<FileSystem> filesystem);
95
96 Status SetOption(const std::string& scheme, const std::string& key,
97 const std::string& value);
98
99 Status SetOption(const std::string& scheme, const std::string& key,
100 const std::vector<string>& values);
101
102 Status SetOption(const std::string& scheme, const std::string& key,
103 const std::vector<int64_t>& values);
104
105 Status SetOption(const std::string& scheme, const std::string& key,
106 const std::vector<double>& values);
107
108 /// \brief Flush filesystem caches for all registered filesystems.
109 Status FlushFileSystemCaches();
110
111 /// \brief Creates a brand new random access read-only file with the
112 /// specified name.
113
114 /// On success, stores a pointer to the new file in
115 /// *result and returns OK. On failure stores NULL in *result and
116 /// returns non-OK. If the file does not exist, returns a non-OK
117 /// status.
118 ///
119 /// The returned file may be concurrently accessed by multiple threads.
120 ///
121 /// The ownership of the returned RandomAccessFile is passed to the caller
122 /// and the object should be deleted when is not used. The file object
123 /// shouldn't live longer than the Env object.
124 Status NewRandomAccessFile(const std::string& fname,
125 std::unique_ptr<RandomAccessFile>* result);
126
127 Status NewRandomAccessFile(const std::string& fname, TransactionToken* token,
128 std::unique_ptr<RandomAccessFile>* result) {
129 // We duplicate these methods due to Google internal coding style prevents
130 // virtual functions with default arguments. See PR #41615.
131 return OkStatus();
132 }
133
134 /// \brief Creates an object that writes to a new file with the specified
135 /// name.
136 ///
137 /// Deletes any existing file with the same name and creates a
138 /// new file. On success, stores a pointer to the new file in
139 /// *result and returns OK. On failure stores NULL in *result and
140 /// returns non-OK.
141 ///
142 /// The returned file will only be accessed by one thread at a time.
143 ///
144 /// The ownership of the returned WritableFile is passed to the caller
145 /// and the object should be deleted when is not used. The file object
146 /// shouldn't live longer than the Env object.
147 Status NewWritableFile(const std::string& fname,
148 std::unique_ptr<WritableFile>* result);
149
150 Status NewWritableFile(const std::string& fname, TransactionToken* token,
151 std::unique_ptr<WritableFile>* result) {
152 return OkStatus();
153 }
154
155 /// \brief Creates an object that either appends to an existing file, or
156 /// writes to a new file (if the file does not exist to begin with).
157 ///
158 /// On success, stores a pointer to the new file in *result and
159 /// returns OK. On failure stores NULL in *result and returns
160 /// non-OK.
161 ///
162 /// The returned file will only be accessed by one thread at a time.
163 ///
164 /// The ownership of the returned WritableFile is passed to the caller
165 /// and the object should be deleted when is not used. The file object
166 /// shouldn't live longer than the Env object.
167 Status NewAppendableFile(const std::string& fname,
168 std::unique_ptr<WritableFile>* result);
169
170 Status NewAppendableFile(const std::string& fname, TransactionToken* token,
171 std::unique_ptr<WritableFile>* result) {
172 return OkStatus();
173 }
174 /// \brief Creates a readonly region of memory with the file context.
175 ///
176 /// On success, it returns a pointer to read-only memory region
177 /// from the content of file fname. The ownership of the region is passed to
178 /// the caller. On failure stores nullptr in *result and returns non-OK.
179 ///
180 /// The returned memory region can be accessed from many threads in parallel.
181 ///
182 /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller
183 /// and the object should be deleted when is not used. The memory region
184 /// object shouldn't live longer than the Env object.
185 Status NewReadOnlyMemoryRegionFromFile(
186 const std::string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result);
187
188 Status NewReadOnlyMemoryRegionFromFile(
189 const std::string& fname, TransactionToken* token,
190 std::unique_ptr<ReadOnlyMemoryRegion>* result) {
191 return OkStatus();
192 }
193
194 /// Returns OK if the named path exists and NOT_FOUND otherwise.
195 Status FileExists(const std::string& fname);
196
197 Status FileExists(const std::string& fname, TransactionToken* token) {
198 return OkStatus();
199 }
200
201 /// Returns true if all the listed files exist, false otherwise.
202 /// if status is not null, populate the vector with a detailed status
203 /// for each file.
204 bool FilesExist(const std::vector<string>& files,
205 std::vector<Status>* status);
206
207 bool FilesExist(const std::vector<string>& files, TransactionToken* token,
208 std::vector<Status>* status) {
209 return true;
210 }
211
212 /// \brief Stores in *result the names of the children of the specified
213 /// directory. The names are relative to "dir".
214 ///
215 /// Original contents of *results are dropped.
216 Status GetChildren(const std::string& dir, std::vector<string>* result);
217
218 Status GetChildren(const std::string& dir, TransactionToken* token,
219 std::vector<string>* result) {
220 return OkStatus();
221 }
222
223 /// \brief Returns true if the path matches the given pattern. The wildcards
224 /// allowed in pattern are described in FileSystem::GetMatchingPaths.
225 virtual bool MatchPath(const std::string& path,
226 const std::string& pattern) = 0;
227
228 /// \brief Given a pattern, stores in *results the set of paths that matches
229 /// that pattern. *results is cleared.
230 ///
231 /// More details about `pattern` in FileSystem::GetMatchingPaths.
232 virtual Status GetMatchingPaths(const std::string& pattern,
233 std::vector<string>* results);
234
235 Status GetMatchingPaths(const std::string& pattern, TransactionToken* token,
236 std::vector<string>* results) {
237 return OkStatus();
238 }
239
240 /// Deletes the named file.
241 Status DeleteFile(const std::string& fname);
242
243 Status DeleteFile(const std::string& fname, TransactionToken* token) {
244 return OkStatus();
245 }
246
247 /// \brief Deletes the specified directory and all subdirectories and files
248 /// underneath it. This is accomplished by traversing the directory tree
249 /// rooted at dirname and deleting entries as they are encountered.
250 ///
251 /// If dirname itself is not readable or does not exist, *undeleted_dir_count
252 /// is set to 1, *undeleted_file_count is set to 0 and an appropriate status
253 /// (e.g. NOT_FOUND) is returned.
254 ///
255 /// If dirname and all its descendants were successfully deleted, TF_OK is
256 /// returned and both error counters are set to zero.
257 ///
258 /// Otherwise, while traversing the tree, undeleted_file_count and
259 /// undeleted_dir_count are updated if an entry of the corresponding type
260 /// could not be deleted. The returned error status represents the reason that
261 /// any one of these entries could not be deleted.
262 ///
263 /// REQUIRES: undeleted_files, undeleted_dirs to be not null.
264 ///
265 /// Typical return codes:
266 /// * OK - dirname exists and we were able to delete everything underneath.
267 /// * NOT_FOUND - dirname doesn't exist
268 /// * PERMISSION_DENIED - dirname or some descendant is not writable
269 /// * UNIMPLEMENTED - Some underlying functions (like Delete) are not
270 /// implemented
271 Status DeleteRecursively(const std::string& dirname, int64_t* undeleted_files,
272 int64_t* undeleted_dirs);
273
274 Status DeleteRecursively(const std::string& dirname, TransactionToken* token,
275 int64_t* undeleted_files, int64_t* undeleted_dirs) {
276 return OkStatus();
277 }
278
279 /// \brief Creates the specified directory and all the necessary
280 /// subdirectories. Typical return codes.
281 /// * OK - successfully created the directory and sub directories, even if
282 /// they were already created.
283 /// * PERMISSION_DENIED - dirname or some subdirectory is not writable.
284 Status RecursivelyCreateDir(const std::string& dirname);
285
286 Status RecursivelyCreateDir(const std::string& dirname,
287 TransactionToken* token) {
288 return OkStatus();
289 }
290 /// \brief Creates the specified directory. Typical return codes
291 /// * OK - successfully created the directory.
292 /// * ALREADY_EXISTS - directory already exists.
293 /// * PERMISSION_DENIED - dirname is not writable.
294 Status CreateDir(const std::string& dirname);
295
296 Status CreateDir(const std::string& dirname, TransactionToken* token) {
297 return OkStatus();
298 }
299
300 /// Deletes the specified directory.
301 Status DeleteDir(const std::string& dirname);
302
303 Status DeleteDir(const std::string& dirname, TransactionToken* token) {
304 return OkStatus();
305 }
306
307 /// Obtains statistics for the given path.
308 Status Stat(const std::string& fname, FileStatistics* stat);
309
310 Status Stat(const std::string& fname, TransactionToken* token,
311 FileStatistics* stat) {
312 return OkStatus();
313 }
314
315 /// \brief Returns whether the given path is a directory or not.
316 /// Typical return codes (not guaranteed exhaustive):
317 /// * OK - The path exists and is a directory.
318 /// * FAILED_PRECONDITION - The path exists and is not a directory.
319 /// * NOT_FOUND - The path entry does not exist.
320 /// * PERMISSION_DENIED - Insufficient permissions.
321 /// * UNIMPLEMENTED - The file factory doesn't support directories.
322 Status IsDirectory(const std::string& fname);
323
324 /// \brief Returns whether the given path is on a file system
325 /// that has atomic move capabilities. This can be used
326 /// to determine if there needs to be a temp location to safely write objects.
327 /// The second boolean argument has_atomic_move contains this information.
328 ///
329 /// Returns one of the following status codes (not guaranteed exhaustive):
330 /// * OK - The path is on a recognized file system,
331 /// so has_atomic_move holds the above information.
332 /// * UNIMPLEMENTED - The file system of the path hasn't been implemented in
333 /// TF
334 Status HasAtomicMove(const std::string& path, bool* has_atomic_move);
335
336 /// Stores the size of `fname` in `*file_size`.
337 Status GetFileSize(const std::string& fname, uint64* file_size);
338
339 Status GetFileSize(const std::string& fname, TransactionToken* token,
340 uint64* file_size) {
341 return OkStatus();
342 }
343
344 /// \brief Renames file src to target. If target already exists, it will be
345 /// replaced.
346 Status RenameFile(const std::string& src, const std::string& target);
347
348 Status RenameFile(const std::string& src, const std::string& target,
349 TransactionToken* token) {
350 return OkStatus();
351 }
352
353 /// \brief Copy the src to target.
354 Status CopyFile(const std::string& src, const std::string& target);
355
356 Status CopyFile(const std::string& src, const std::string& target,
357 TransactionToken* token) {
358 return OkStatus();
359 }
360
361 /// \brief starts a new transaction on the filesystem that handles filename
362 Status StartTransaction(const std::string& filename,
363 TransactionToken** token) {
364 *token = nullptr;
365 return OkStatus();
366 }
367
368 /// \brief Adds `path` to transaction in `token` if token belongs to
369 /// filesystem that handles the path.
370 Status AddToTransaction(const std::string& path, TransactionToken* token) {
371 return OkStatus();
372 }
373
374 /// \brief Get token for `path` or start a new transaction and add `path` to
375 /// it.
376 Status GetTokenOrStartTransaction(const std::string& path,
377 TransactionToken** token) {
378 *token = nullptr;
379 return OkStatus();
380 }
381
382 /// \brief Returns the transaction for `path` or nullptr in `token`
383 Status GetTransactionForPath(const std::string& path,
384 TransactionToken** token) {
385 *token = nullptr;
386 return OkStatus();
387 }
388
389 /// \brief Finalizes the transaction
390 Status EndTransaction(TransactionToken* token) { return OkStatus(); }
391
392 /// \brief Returns the absolute path of the current executable. It resolves
393 /// symlinks if there is any.
394 std::string GetExecutablePath();
395
396 /// Creates a local unique temporary file name. Returns true if success.
397 bool LocalTempFilename(std::string* filename);
398
399 /// Creates a local unique file name that starts with |prefix| and ends with
400 /// |suffix|. Returns true if success.
401 bool CreateUniqueFileName(std::string* prefix, const std::string& suffix);
402
403 /// \brief Return the runfiles directory if running under bazel. Returns
404 /// the directory the executable is located in if not running under bazel.
405 virtual std::string GetRunfilesDir() = 0;
406
407 // TODO(jeff,sanjay): Add back thread/thread-pool support if needed.
408 // TODO(jeff,sanjay): if needed, tighten spec so relative to epoch, or
409 // provide a routine to get the absolute time.
410
411 /// \brief Returns the number of nano-seconds since the Unix epoch.
412 virtual uint64 NowNanos() const { return EnvTime::NowNanos(); }
413
414 /// \brief Returns the number of micro-seconds since the Unix epoch.
415 virtual uint64 NowMicros() const { return EnvTime::NowMicros(); }
416
417 /// \brief Returns the number of seconds since the Unix epoch.
418 virtual uint64 NowSeconds() const { return EnvTime::NowSeconds(); }
419
420 /// Sleeps/delays the thread for the prescribed number of micro-seconds.
421 virtual void SleepForMicroseconds(int64_t micros) = 0;
422
423 /// Returns the process ID of the calling process.
424 int32 GetProcessId();
425
426 /// \brief Returns a new thread that is running fn() and is identified
427 /// (for debugging/performance-analysis) by "name".
428 ///
429 /// Caller takes ownership of the result and must delete it eventually
430 /// (the deletion will block until fn() stops running).
431 virtual Thread* StartThread(const ThreadOptions& thread_options,
432 const std::string& name,
433 std::function<void()> fn) TF_MUST_USE_RESULT = 0;
434
435 // Returns the thread id of calling thread.
436 // Posix: Returns pthread id which is only guaranteed to be unique within a
437 // process.
438 // Windows: Returns thread id which is unique.
439 virtual int32 GetCurrentThreadId() = 0;
440
441 // Copies current thread name to "name". Returns true if success.
442 virtual bool GetCurrentThreadName(std::string* name) = 0;
443
444 // \brief Schedules the given closure on a thread-pool.
445 //
446 // NOTE(mrry): This closure may block.
447 virtual void SchedClosure(std::function<void()> closure) = 0;
448
449 // \brief Schedules the given closure on a thread-pool after the given number
450 // of microseconds.
451 //
452 // NOTE(mrry): This closure must not block.
453 virtual void SchedClosureAfter(int64_t micros,
454 std::function<void()> closure) = 0;
455
456 // \brief Load a dynamic library.
457 //
458 // Pass "library_filename" to a platform-specific mechanism for dynamically
459 // loading a library. The rules for determining the exact location of the
460 // library are platform-specific and are not documented here.
461 //
462 // On success, returns a handle to the library in "*handle" and returns
463 // OK from the function.
464 // Otherwise returns nullptr in "*handle" and an error status from the
465 // function.
466 virtual Status LoadDynamicLibrary(const char* library_filename,
467 void** handle) = 0;
468
469 // \brief Get a pointer to a symbol from a dynamic library.
470 //
471 // "handle" should be a pointer returned from a previous call to LoadLibrary.
472 // On success, store a pointer to the located symbol in "*symbol" and return
473 // OK from the function. Otherwise, returns nullptr in "*symbol" and an error
474 // status from the function.
475 virtual Status GetSymbolFromLibrary(void* handle, const char* symbol_name,
476 void** symbol) = 0;
477
478 // \brief build the name of dynamic library.
479 //
480 // "name" should be name of the library.
481 // "version" should be the version of the library or NULL
482 // returns the name that LoadLibrary() can use
483 virtual std::string FormatLibraryFileName(const std::string& name,
484 const std::string& version) = 0;
485
486 // Returns a possible list of local temporary directories.
487 virtual void GetLocalTempDirectories(std::vector<string>* list) = 0;
488
489 private:
490 std::unique_ptr<FileSystemRegistry> file_system_registry_;
491 TF_DISALLOW_COPY_AND_ASSIGN(Env);
492};
493
494/// \brief An implementation of Env that forwards all calls to another Env.
495///
496/// May be useful to clients who wish to override just part of the
497/// functionality of another Env.
498class EnvWrapper : public Env {
499 public:
500 /// Initializes an EnvWrapper that delegates all calls to *t
501 explicit EnvWrapper(Env* t) : target_(t) {}
502 ~EnvWrapper() override;
503
504 /// Returns the target to which this Env forwards all calls
505 Env* target() const { return target_; }
506
507 Status GetFileSystemForFile(const std::string& fname,
508 FileSystem** result) override {
509 return target_->GetFileSystemForFile(fname, result);
510 }
511
512 Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes) override {
513 return target_->GetRegisteredFileSystemSchemes(schemes);
514 }
515
516 Status RegisterFileSystem(const std::string& scheme,
517 FileSystemRegistry::Factory factory) override {
518 return target_->RegisterFileSystem(scheme, factory);
519 }
520
521 bool MatchPath(const std::string& path, const std::string& pattern) override {
522 return target_->MatchPath(path, pattern);
523 }
524
525 uint64 NowMicros() const override { return target_->NowMicros(); }
526 void SleepForMicroseconds(int64_t micros) override {
527 target_->SleepForMicroseconds(micros);
528 }
529 Thread* StartThread(const ThreadOptions& thread_options,
530 const std::string& name,
531 std::function<void()> fn) override {
532 return target_->StartThread(thread_options, name, fn);
533 }
534 int32 GetCurrentThreadId() override { return target_->GetCurrentThreadId(); }
535 bool GetCurrentThreadName(std::string* name) override {
536 return target_->GetCurrentThreadName(name);
537 }
538 void SchedClosure(std::function<void()> closure) override {
539 target_->SchedClosure(closure);
540 }
541 void SchedClosureAfter(int64_t micros,
542 std::function<void()> closure) override {
543 target_->SchedClosureAfter(micros, closure);
544 }
545 Status LoadDynamicLibrary(const char* library_filename,
546 void** handle) override {
547 return target_->LoadDynamicLibrary(library_filename, handle);
548 }
549 Status GetSymbolFromLibrary(void* handle, const char* symbol_name,
550 void** symbol) override {
551 return target_->GetSymbolFromLibrary(handle, symbol_name, symbol);
552 }
553 std::string FormatLibraryFileName(const std::string& name,
554 const std::string& version) override {
555 return target_->FormatLibraryFileName(name, version);
556 }
557
558 std::string GetRunfilesDir() override { return target_->GetRunfilesDir(); }
559
560 private:
561 void GetLocalTempDirectories(std::vector<string>* list) override {
562 target_->GetLocalTempDirectories(list);
563 }
564
565 Env* target_;
566};
567
568/// Represents a thread used to run a TSL function.
569class Thread {
570 public:
571 Thread() {}
572
573 /// Blocks until the thread of control stops running.
574 virtual ~Thread();
575
576 private:
577 TF_DISALLOW_COPY_AND_ASSIGN(Thread);
578};
579
580/// \brief Cross-platform setenv.
581///
582/// Since setenv() is not available on windows, we provide an
583/// alternative with platform specific implementations here.
584int setenv(const char* name, const char* value, int overwrite);
585
586/// Cross-platform unsetenv.
587int unsetenv(const char* name);
588
589/// \brief Options to configure a Thread.
590///
591/// Note that the options are all hints, and the
592/// underlying implementation may choose to ignore it.
593struct ThreadOptions {
594 /// Thread stack size to use (in bytes).
595 size_t stack_size = 0; // 0: use system default value
596 /// Guard area size to use near thread stacks to use (in bytes)
597 size_t guard_size = 0; // 0: use system default value
598 int numa_node = port::kNUMANoAffinity;
599};
600
601/// A utility routine: copy contents of `src` in file system `src_fs`
602/// to `target` in file system `target_fs`.
603Status FileSystemCopyFile(FileSystem* src_fs, const std::string& src,
604 FileSystem* target_fs, const std::string& target);
605
606/// A utility routine: reads contents of named file into `*data`
607Status ReadFileToString(Env* env, const std::string& fname, std::string* data);
608
609/// A utility routine: write contents of `data` to file named `fname`
610/// (overwriting existing contents, if any).
611Status WriteStringToFile(Env* env, const std::string& fname,
612 const StringPiece& data);
613
614/// Write binary representation of "proto" to the named file.
615Status WriteBinaryProto(Env* env, const std::string& fname,
616 const protobuf::MessageLite& proto);
617
618/// Reads contents of named file and parse as binary encoded proto data
619/// and store into `*proto`.
620Status ReadBinaryProto(Env* env, const std::string& fname,
621 protobuf::MessageLite* proto);
622
623/// Write the text representation of "proto" to the named file.
624inline Status WriteTextProto(Env* /* env */, const std::string& /* fname */,
625 const protobuf::MessageLite& /* proto */) {
626 return errors::Unimplemented("Can't write text protos with protolite.");
627}
628Status WriteTextProto(Env* env, const std::string& fname,
629 const protobuf::Message& proto);
630
631/// Read contents of named file and parse as text encoded proto data
632/// and store into `*proto`.
633inline Status ReadTextProto(Env* /* env */, const std::string& /* fname */,
634 protobuf::MessageLite* /* proto */) {
635 return errors::Unimplemented("Can't parse text protos with protolite.");
636}
637Status ReadTextProto(Env* env, const std::string& fname,
638 protobuf::Message* proto);
639
640/// Read contents of named file and parse as either text or binary encoded proto
641/// data and store into `*proto`.
642Status ReadTextOrBinaryProto(Env* env, const std::string& fname,
643 protobuf::Message* proto);
644Status ReadTextOrBinaryProto(Env* env, const std::string& fname,
645 protobuf::MessageLite* proto);
646
647// START_SKIP_DOXYGEN
648
649// The following approach to register filesystems is deprecated and will be
650// replaced with modular filesystem plugins registration.
651// TODO(b/139060984): After all filesystems are converted, remove this.
652namespace register_file_system {
653
654template <typename Factory>
655struct Register {
656 Register(Env* env, const std::string& scheme, bool try_modular_filesystems) {
657 // TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
658 // after TF 2.6+.
659 if (try_modular_filesystems) {
660 const char* env_value = getenv("TF_USE_MODULAR_FILESYSTEM");
661 string load_plugin = env_value ? absl::AsciiStrToLower(env_value) : "";
662 if (load_plugin == "true" || load_plugin == "1") {
663 // We don't register the static filesystem and wait for SIG IO one
664 LOG(WARNING) << "Using modular file system for '" << scheme << "'."
665 << " Please switch to tensorflow-io"
666 << " (https://github.com/tensorflow/io) for file system"
667 << " support of '" << scheme << "'.";
668 return;
669 }
670 // If the envvar is missing or not "true"/"1", then fall back to legacy
671 // implementation to be backwards compatible.
672 }
673 // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
674 env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
675 .IgnoreError();
676 }
677};
678
679} // namespace register_file_system
680
681// END_SKIP_DOXYGEN
682
683} // namespace tsl
684
685// Register a FileSystem implementation for a scheme. Files with names that have
686// "scheme://" prefixes are routed to use this implementation.
687#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, modular) \
688 REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, modular)
689#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, modular) \
690 REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular)
691#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular) \
692 static ::tsl::register_file_system::Register<factory> register_ff##ctr \
693 TF_ATTRIBUTE_UNUSED = \
694 ::tsl::register_file_system::Register<factory>(env, scheme, modular)
695
696#define REGISTER_FILE_SYSTEM(scheme, factory) \
697 REGISTER_FILE_SYSTEM_ENV(::tsl::Env::Default(), scheme, factory, false);
698
699#define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \
700 REGISTER_FILE_SYSTEM_ENV(::tsl::Env::Default(), scheme, factory, true);
701
702#endif // TENSORFLOW_TSL_PLATFORM_ENV_H_
703