1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. |
14 | ==============================================================================*/ |
15 | |
16 | #ifndef TENSORFLOW_TSL_PLATFORM_ENV_H_ |
17 | #define TENSORFLOW_TSL_PLATFORM_ENV_H_ |
18 | |
19 | #include <stdint.h> |
20 | |
21 | #include <memory> |
22 | #include <string> |
23 | #include <unordered_map> |
24 | #include <vector> |
25 | |
26 | #include "tensorflow/tsl/platform/env_time.h" |
27 | #include "tensorflow/tsl/platform/errors.h" |
28 | #include "tensorflow/tsl/platform/file_system.h" |
29 | #include "tensorflow/tsl/platform/macros.h" |
30 | #include "tensorflow/tsl/platform/mutex.h" |
31 | #include "tensorflow/tsl/platform/numa.h" |
32 | #include "tensorflow/tsl/platform/platform.h" |
33 | #include "tensorflow/tsl/platform/protobuf.h" |
34 | #include "tensorflow/tsl/platform/status.h" |
35 | #include "tensorflow/tsl/platform/stringpiece.h" |
36 | #include "tensorflow/tsl/platform/types.h" |
37 | |
38 | // Delete leaked Windows definitions. |
39 | #ifdef PLATFORM_WINDOWS |
40 | #undef CopyFile |
41 | #undef DeleteFile |
42 | #endif |
43 | |
44 | namespace tsl { |
45 | |
46 | class Thread; |
47 | struct ThreadOptions; |
48 | |
49 | /// \brief An interface used by the tensorflow implementation to |
50 | /// access operating system functionality like the filesystem etc. |
51 | /// |
52 | /// Callers may wish to provide a custom Env object to get fine grain |
53 | /// control. |
54 | /// |
55 | /// All Env implementations are safe for concurrent access from |
56 | /// multiple threads without any external synchronization. |
57 | class Env { |
58 | public: |
59 | Env(); |
60 | virtual ~Env() = default; |
61 | |
62 | /// \brief Returns a default environment suitable for the current operating |
63 | /// system. |
64 | /// |
65 | /// Sophisticated users may wish to provide their own Env |
66 | /// implementation instead of relying on this default environment. |
67 | /// |
68 | /// The result of Default() belongs to this library and must never be deleted. |
69 | static Env* Default(); |
70 | |
71 | /// \brief Returns the FileSystem object to handle operations on the file |
72 | /// specified by 'fname'. The FileSystem object is used as the implementation |
73 | /// for the file system related (non-virtual) functions that follow. |
74 | /// Returned FileSystem object is still owned by the Env object and will |
75 | // (might) be destroyed when the environment is destroyed. |
76 | virtual Status GetFileSystemForFile(const std::string& fname, |
77 | FileSystem** result); |
78 | |
79 | /// \brief Returns the file system schemes registered for this Env. |
80 | virtual Status GetRegisteredFileSystemSchemes( |
81 | std::vector<std::string>* schemes); |
82 | |
83 | /// \brief Register a file system for a scheme. |
84 | virtual Status RegisterFileSystem(const std::string& scheme, |
85 | FileSystemRegistry::Factory factory); |
86 | |
87 | /// \brief Register a modular file system for a scheme. |
88 | /// |
89 | /// Same as `RegisterFileSystem` but for filesystems provided by plugins. |
90 | /// |
91 | /// TODO(b/139060984): After all filesystems are converted, make this be the |
92 | /// canonical registration function. |
93 | virtual Status RegisterFileSystem(const std::string& scheme, |
94 | std::unique_ptr<FileSystem> filesystem); |
95 | |
96 | Status SetOption(const std::string& scheme, const std::string& key, |
97 | const std::string& value); |
98 | |
99 | Status SetOption(const std::string& scheme, const std::string& key, |
100 | const std::vector<string>& values); |
101 | |
102 | Status SetOption(const std::string& scheme, const std::string& key, |
103 | const std::vector<int64_t>& values); |
104 | |
105 | Status SetOption(const std::string& scheme, const std::string& key, |
106 | const std::vector<double>& values); |
107 | |
108 | /// \brief Flush filesystem caches for all registered filesystems. |
109 | Status FlushFileSystemCaches(); |
110 | |
111 | /// \brief Creates a brand new random access read-only file with the |
112 | /// specified name. |
113 | |
114 | /// On success, stores a pointer to the new file in |
115 | /// *result and returns OK. On failure stores NULL in *result and |
116 | /// returns non-OK. If the file does not exist, returns a non-OK |
117 | /// status. |
118 | /// |
119 | /// The returned file may be concurrently accessed by multiple threads. |
120 | /// |
121 | /// The ownership of the returned RandomAccessFile is passed to the caller |
122 | /// and the object should be deleted when is not used. The file object |
123 | /// shouldn't live longer than the Env object. |
124 | Status NewRandomAccessFile(const std::string& fname, |
125 | std::unique_ptr<RandomAccessFile>* result); |
126 | |
127 | Status NewRandomAccessFile(const std::string& fname, TransactionToken* token, |
128 | std::unique_ptr<RandomAccessFile>* result) { |
129 | // We duplicate these methods due to Google internal coding style prevents |
130 | // virtual functions with default arguments. See PR #41615. |
131 | return OkStatus(); |
132 | } |
133 | |
134 | /// \brief Creates an object that writes to a new file with the specified |
135 | /// name. |
136 | /// |
137 | /// Deletes any existing file with the same name and creates a |
138 | /// new file. On success, stores a pointer to the new file in |
139 | /// *result and returns OK. On failure stores NULL in *result and |
140 | /// returns non-OK. |
141 | /// |
142 | /// The returned file will only be accessed by one thread at a time. |
143 | /// |
144 | /// The ownership of the returned WritableFile is passed to the caller |
145 | /// and the object should be deleted when is not used. The file object |
146 | /// shouldn't live longer than the Env object. |
147 | Status NewWritableFile(const std::string& fname, |
148 | std::unique_ptr<WritableFile>* result); |
149 | |
150 | Status NewWritableFile(const std::string& fname, TransactionToken* token, |
151 | std::unique_ptr<WritableFile>* result) { |
152 | return OkStatus(); |
153 | } |
154 | |
155 | /// \brief Creates an object that either appends to an existing file, or |
156 | /// writes to a new file (if the file does not exist to begin with). |
157 | /// |
158 | /// On success, stores a pointer to the new file in *result and |
159 | /// returns OK. On failure stores NULL in *result and returns |
160 | /// non-OK. |
161 | /// |
162 | /// The returned file will only be accessed by one thread at a time. |
163 | /// |
164 | /// The ownership of the returned WritableFile is passed to the caller |
165 | /// and the object should be deleted when is not used. The file object |
166 | /// shouldn't live longer than the Env object. |
167 | Status NewAppendableFile(const std::string& fname, |
168 | std::unique_ptr<WritableFile>* result); |
169 | |
170 | Status NewAppendableFile(const std::string& fname, TransactionToken* token, |
171 | std::unique_ptr<WritableFile>* result) { |
172 | return OkStatus(); |
173 | } |
174 | /// \brief Creates a readonly region of memory with the file context. |
175 | /// |
176 | /// On success, it returns a pointer to read-only memory region |
177 | /// from the content of file fname. The ownership of the region is passed to |
178 | /// the caller. On failure stores nullptr in *result and returns non-OK. |
179 | /// |
180 | /// The returned memory region can be accessed from many threads in parallel. |
181 | /// |
182 | /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller |
183 | /// and the object should be deleted when is not used. The memory region |
184 | /// object shouldn't live longer than the Env object. |
185 | Status NewReadOnlyMemoryRegionFromFile( |
186 | const std::string& fname, std::unique_ptr<ReadOnlyMemoryRegion>* result); |
187 | |
188 | Status NewReadOnlyMemoryRegionFromFile( |
189 | const std::string& fname, TransactionToken* token, |
190 | std::unique_ptr<ReadOnlyMemoryRegion>* result) { |
191 | return OkStatus(); |
192 | } |
193 | |
194 | /// Returns OK if the named path exists and NOT_FOUND otherwise. |
195 | Status FileExists(const std::string& fname); |
196 | |
197 | Status FileExists(const std::string& fname, TransactionToken* token) { |
198 | return OkStatus(); |
199 | } |
200 | |
201 | /// Returns true if all the listed files exist, false otherwise. |
202 | /// if status is not null, populate the vector with a detailed status |
203 | /// for each file. |
204 | bool FilesExist(const std::vector<string>& files, |
205 | std::vector<Status>* status); |
206 | |
207 | bool FilesExist(const std::vector<string>& files, TransactionToken* token, |
208 | std::vector<Status>* status) { |
209 | return true; |
210 | } |
211 | |
212 | /// \brief Stores in *result the names of the children of the specified |
213 | /// directory. The names are relative to "dir". |
214 | /// |
215 | /// Original contents of *results are dropped. |
216 | Status GetChildren(const std::string& dir, std::vector<string>* result); |
217 | |
218 | Status GetChildren(const std::string& dir, TransactionToken* token, |
219 | std::vector<string>* result) { |
220 | return OkStatus(); |
221 | } |
222 | |
223 | /// \brief Returns true if the path matches the given pattern. The wildcards |
224 | /// allowed in pattern are described in FileSystem::GetMatchingPaths. |
225 | virtual bool MatchPath(const std::string& path, |
226 | const std::string& pattern) = 0; |
227 | |
228 | /// \brief Given a pattern, stores in *results the set of paths that matches |
229 | /// that pattern. *results is cleared. |
230 | /// |
231 | /// More details about `pattern` in FileSystem::GetMatchingPaths. |
232 | virtual Status GetMatchingPaths(const std::string& pattern, |
233 | std::vector<string>* results); |
234 | |
235 | Status GetMatchingPaths(const std::string& pattern, TransactionToken* token, |
236 | std::vector<string>* results) { |
237 | return OkStatus(); |
238 | } |
239 | |
240 | /// Deletes the named file. |
241 | Status DeleteFile(const std::string& fname); |
242 | |
243 | Status DeleteFile(const std::string& fname, TransactionToken* token) { |
244 | return OkStatus(); |
245 | } |
246 | |
247 | /// \brief Deletes the specified directory and all subdirectories and files |
248 | /// underneath it. This is accomplished by traversing the directory tree |
249 | /// rooted at dirname and deleting entries as they are encountered. |
250 | /// |
251 | /// If dirname itself is not readable or does not exist, *undeleted_dir_count |
252 | /// is set to 1, *undeleted_file_count is set to 0 and an appropriate status |
253 | /// (e.g. NOT_FOUND) is returned. |
254 | /// |
255 | /// If dirname and all its descendants were successfully deleted, TF_OK is |
256 | /// returned and both error counters are set to zero. |
257 | /// |
258 | /// Otherwise, while traversing the tree, undeleted_file_count and |
259 | /// undeleted_dir_count are updated if an entry of the corresponding type |
260 | /// could not be deleted. The returned error status represents the reason that |
261 | /// any one of these entries could not be deleted. |
262 | /// |
263 | /// REQUIRES: undeleted_files, undeleted_dirs to be not null. |
264 | /// |
265 | /// Typical return codes: |
266 | /// * OK - dirname exists and we were able to delete everything underneath. |
267 | /// * NOT_FOUND - dirname doesn't exist |
268 | /// * PERMISSION_DENIED - dirname or some descendant is not writable |
269 | /// * UNIMPLEMENTED - Some underlying functions (like Delete) are not |
270 | /// implemented |
271 | Status DeleteRecursively(const std::string& dirname, int64_t* undeleted_files, |
272 | int64_t* undeleted_dirs); |
273 | |
274 | Status DeleteRecursively(const std::string& dirname, TransactionToken* token, |
275 | int64_t* undeleted_files, int64_t* undeleted_dirs) { |
276 | return OkStatus(); |
277 | } |
278 | |
279 | /// \brief Creates the specified directory and all the necessary |
280 | /// subdirectories. Typical return codes. |
281 | /// * OK - successfully created the directory and sub directories, even if |
282 | /// they were already created. |
283 | /// * PERMISSION_DENIED - dirname or some subdirectory is not writable. |
284 | Status RecursivelyCreateDir(const std::string& dirname); |
285 | |
286 | Status RecursivelyCreateDir(const std::string& dirname, |
287 | TransactionToken* token) { |
288 | return OkStatus(); |
289 | } |
290 | /// \brief Creates the specified directory. Typical return codes |
291 | /// * OK - successfully created the directory. |
292 | /// * ALREADY_EXISTS - directory already exists. |
293 | /// * PERMISSION_DENIED - dirname is not writable. |
294 | Status CreateDir(const std::string& dirname); |
295 | |
296 | Status CreateDir(const std::string& dirname, TransactionToken* token) { |
297 | return OkStatus(); |
298 | } |
299 | |
300 | /// Deletes the specified directory. |
301 | Status DeleteDir(const std::string& dirname); |
302 | |
303 | Status DeleteDir(const std::string& dirname, TransactionToken* token) { |
304 | return OkStatus(); |
305 | } |
306 | |
307 | /// Obtains statistics for the given path. |
308 | Status Stat(const std::string& fname, FileStatistics* stat); |
309 | |
310 | Status Stat(const std::string& fname, TransactionToken* token, |
311 | FileStatistics* stat) { |
312 | return OkStatus(); |
313 | } |
314 | |
315 | /// \brief Returns whether the given path is a directory or not. |
316 | /// Typical return codes (not guaranteed exhaustive): |
317 | /// * OK - The path exists and is a directory. |
318 | /// * FAILED_PRECONDITION - The path exists and is not a directory. |
319 | /// * NOT_FOUND - The path entry does not exist. |
320 | /// * PERMISSION_DENIED - Insufficient permissions. |
321 | /// * UNIMPLEMENTED - The file factory doesn't support directories. |
322 | Status IsDirectory(const std::string& fname); |
323 | |
324 | /// \brief Returns whether the given path is on a file system |
325 | /// that has atomic move capabilities. This can be used |
326 | /// to determine if there needs to be a temp location to safely write objects. |
327 | /// The second boolean argument has_atomic_move contains this information. |
328 | /// |
329 | /// Returns one of the following status codes (not guaranteed exhaustive): |
330 | /// * OK - The path is on a recognized file system, |
331 | /// so has_atomic_move holds the above information. |
332 | /// * UNIMPLEMENTED - The file system of the path hasn't been implemented in |
333 | /// TF |
334 | Status HasAtomicMove(const std::string& path, bool* has_atomic_move); |
335 | |
336 | /// Stores the size of `fname` in `*file_size`. |
337 | Status GetFileSize(const std::string& fname, uint64* file_size); |
338 | |
339 | Status GetFileSize(const std::string& fname, TransactionToken* token, |
340 | uint64* file_size) { |
341 | return OkStatus(); |
342 | } |
343 | |
344 | /// \brief Renames file src to target. If target already exists, it will be |
345 | /// replaced. |
346 | Status RenameFile(const std::string& src, const std::string& target); |
347 | |
348 | Status RenameFile(const std::string& src, const std::string& target, |
349 | TransactionToken* token) { |
350 | return OkStatus(); |
351 | } |
352 | |
353 | /// \brief Copy the src to target. |
354 | Status CopyFile(const std::string& src, const std::string& target); |
355 | |
356 | Status CopyFile(const std::string& src, const std::string& target, |
357 | TransactionToken* token) { |
358 | return OkStatus(); |
359 | } |
360 | |
361 | /// \brief starts a new transaction on the filesystem that handles filename |
362 | Status StartTransaction(const std::string& filename, |
363 | TransactionToken** token) { |
364 | *token = nullptr; |
365 | return OkStatus(); |
366 | } |
367 | |
368 | /// \brief Adds `path` to transaction in `token` if token belongs to |
369 | /// filesystem that handles the path. |
370 | Status AddToTransaction(const std::string& path, TransactionToken* token) { |
371 | return OkStatus(); |
372 | } |
373 | |
374 | /// \brief Get token for `path` or start a new transaction and add `path` to |
375 | /// it. |
376 | Status GetTokenOrStartTransaction(const std::string& path, |
377 | TransactionToken** token) { |
378 | *token = nullptr; |
379 | return OkStatus(); |
380 | } |
381 | |
382 | /// \brief Returns the transaction for `path` or nullptr in `token` |
383 | Status GetTransactionForPath(const std::string& path, |
384 | TransactionToken** token) { |
385 | *token = nullptr; |
386 | return OkStatus(); |
387 | } |
388 | |
389 | /// \brief Finalizes the transaction |
390 | Status EndTransaction(TransactionToken* token) { return OkStatus(); } |
391 | |
392 | /// \brief Returns the absolute path of the current executable. It resolves |
393 | /// symlinks if there is any. |
394 | std::string GetExecutablePath(); |
395 | |
396 | /// Creates a local unique temporary file name. Returns true if success. |
397 | bool LocalTempFilename(std::string* filename); |
398 | |
399 | /// Creates a local unique file name that starts with |prefix| and ends with |
400 | /// |suffix|. Returns true if success. |
401 | bool CreateUniqueFileName(std::string* prefix, const std::string& suffix); |
402 | |
403 | /// \brief Return the runfiles directory if running under bazel. Returns |
404 | /// the directory the executable is located in if not running under bazel. |
405 | virtual std::string GetRunfilesDir() = 0; |
406 | |
407 | // TODO(jeff,sanjay): Add back thread/thread-pool support if needed. |
408 | // TODO(jeff,sanjay): if needed, tighten spec so relative to epoch, or |
409 | // provide a routine to get the absolute time. |
410 | |
411 | /// \brief Returns the number of nano-seconds since the Unix epoch. |
412 | virtual uint64 NowNanos() const { return EnvTime::NowNanos(); } |
413 | |
414 | /// \brief Returns the number of micro-seconds since the Unix epoch. |
415 | virtual uint64 NowMicros() const { return EnvTime::NowMicros(); } |
416 | |
417 | /// \brief Returns the number of seconds since the Unix epoch. |
418 | virtual uint64 NowSeconds() const { return EnvTime::NowSeconds(); } |
419 | |
420 | /// Sleeps/delays the thread for the prescribed number of micro-seconds. |
421 | virtual void SleepForMicroseconds(int64_t micros) = 0; |
422 | |
423 | /// Returns the process ID of the calling process. |
424 | int32 GetProcessId(); |
425 | |
426 | /// \brief Returns a new thread that is running fn() and is identified |
427 | /// (for debugging/performance-analysis) by "name". |
428 | /// |
429 | /// Caller takes ownership of the result and must delete it eventually |
430 | /// (the deletion will block until fn() stops running). |
431 | virtual Thread* StartThread(const ThreadOptions& thread_options, |
432 | const std::string& name, |
433 | std::function<void()> fn) TF_MUST_USE_RESULT = 0; |
434 | |
435 | // Returns the thread id of calling thread. |
436 | // Posix: Returns pthread id which is only guaranteed to be unique within a |
437 | // process. |
438 | // Windows: Returns thread id which is unique. |
439 | virtual int32 GetCurrentThreadId() = 0; |
440 | |
441 | // Copies current thread name to "name". Returns true if success. |
442 | virtual bool GetCurrentThreadName(std::string* name) = 0; |
443 | |
444 | // \brief Schedules the given closure on a thread-pool. |
445 | // |
446 | // NOTE(mrry): This closure may block. |
447 | virtual void SchedClosure(std::function<void()> closure) = 0; |
448 | |
449 | // \brief Schedules the given closure on a thread-pool after the given number |
450 | // of microseconds. |
451 | // |
452 | // NOTE(mrry): This closure must not block. |
453 | virtual void SchedClosureAfter(int64_t micros, |
454 | std::function<void()> closure) = 0; |
455 | |
456 | // \brief Load a dynamic library. |
457 | // |
458 | // Pass "library_filename" to a platform-specific mechanism for dynamically |
459 | // loading a library. The rules for determining the exact location of the |
460 | // library are platform-specific and are not documented here. |
461 | // |
462 | // On success, returns a handle to the library in "*handle" and returns |
463 | // OK from the function. |
464 | // Otherwise returns nullptr in "*handle" and an error status from the |
465 | // function. |
466 | virtual Status LoadDynamicLibrary(const char* library_filename, |
467 | void** handle) = 0; |
468 | |
469 | // \brief Get a pointer to a symbol from a dynamic library. |
470 | // |
471 | // "handle" should be a pointer returned from a previous call to LoadLibrary. |
472 | // On success, store a pointer to the located symbol in "*symbol" and return |
473 | // OK from the function. Otherwise, returns nullptr in "*symbol" and an error |
474 | // status from the function. |
475 | virtual Status GetSymbolFromLibrary(void* handle, const char* symbol_name, |
476 | void** symbol) = 0; |
477 | |
478 | // \brief build the name of dynamic library. |
479 | // |
480 | // "name" should be name of the library. |
481 | // "version" should be the version of the library or NULL |
482 | // returns the name that LoadLibrary() can use |
483 | virtual std::string FormatLibraryFileName(const std::string& name, |
484 | const std::string& version) = 0; |
485 | |
486 | // Returns a possible list of local temporary directories. |
487 | virtual void GetLocalTempDirectories(std::vector<string>* list) = 0; |
488 | |
489 | private: |
490 | std::unique_ptr<FileSystemRegistry> file_system_registry_; |
491 | TF_DISALLOW_COPY_AND_ASSIGN(Env); |
492 | }; |
493 | |
494 | /// \brief An implementation of Env that forwards all calls to another Env. |
495 | /// |
496 | /// May be useful to clients who wish to override just part of the |
497 | /// functionality of another Env. |
498 | class EnvWrapper : public Env { |
499 | public: |
500 | /// Initializes an EnvWrapper that delegates all calls to *t |
501 | explicit EnvWrapper(Env* t) : target_(t) {} |
502 | ~EnvWrapper() override; |
503 | |
504 | /// Returns the target to which this Env forwards all calls |
505 | Env* target() const { return target_; } |
506 | |
507 | Status GetFileSystemForFile(const std::string& fname, |
508 | FileSystem** result) override { |
509 | return target_->GetFileSystemForFile(fname, result); |
510 | } |
511 | |
512 | Status GetRegisteredFileSystemSchemes(std::vector<string>* schemes) override { |
513 | return target_->GetRegisteredFileSystemSchemes(schemes); |
514 | } |
515 | |
516 | Status RegisterFileSystem(const std::string& scheme, |
517 | FileSystemRegistry::Factory factory) override { |
518 | return target_->RegisterFileSystem(scheme, factory); |
519 | } |
520 | |
521 | bool MatchPath(const std::string& path, const std::string& pattern) override { |
522 | return target_->MatchPath(path, pattern); |
523 | } |
524 | |
525 | uint64 NowMicros() const override { return target_->NowMicros(); } |
526 | void SleepForMicroseconds(int64_t micros) override { |
527 | target_->SleepForMicroseconds(micros); |
528 | } |
529 | Thread* StartThread(const ThreadOptions& thread_options, |
530 | const std::string& name, |
531 | std::function<void()> fn) override { |
532 | return target_->StartThread(thread_options, name, fn); |
533 | } |
534 | int32 GetCurrentThreadId() override { return target_->GetCurrentThreadId(); } |
535 | bool GetCurrentThreadName(std::string* name) override { |
536 | return target_->GetCurrentThreadName(name); |
537 | } |
538 | void SchedClosure(std::function<void()> closure) override { |
539 | target_->SchedClosure(closure); |
540 | } |
541 | void SchedClosureAfter(int64_t micros, |
542 | std::function<void()> closure) override { |
543 | target_->SchedClosureAfter(micros, closure); |
544 | } |
545 | Status LoadDynamicLibrary(const char* library_filename, |
546 | void** handle) override { |
547 | return target_->LoadDynamicLibrary(library_filename, handle); |
548 | } |
549 | Status GetSymbolFromLibrary(void* handle, const char* symbol_name, |
550 | void** symbol) override { |
551 | return target_->GetSymbolFromLibrary(handle, symbol_name, symbol); |
552 | } |
553 | std::string FormatLibraryFileName(const std::string& name, |
554 | const std::string& version) override { |
555 | return target_->FormatLibraryFileName(name, version); |
556 | } |
557 | |
558 | std::string GetRunfilesDir() override { return target_->GetRunfilesDir(); } |
559 | |
560 | private: |
561 | void GetLocalTempDirectories(std::vector<string>* list) override { |
562 | target_->GetLocalTempDirectories(list); |
563 | } |
564 | |
565 | Env* target_; |
566 | }; |
567 | |
568 | /// Represents a thread used to run a TSL function. |
569 | class Thread { |
570 | public: |
571 | Thread() {} |
572 | |
573 | /// Blocks until the thread of control stops running. |
574 | virtual ~Thread(); |
575 | |
576 | private: |
577 | TF_DISALLOW_COPY_AND_ASSIGN(Thread); |
578 | }; |
579 | |
580 | /// \brief Cross-platform setenv. |
581 | /// |
582 | /// Since setenv() is not available on windows, we provide an |
583 | /// alternative with platform specific implementations here. |
584 | int setenv(const char* name, const char* value, int overwrite); |
585 | |
586 | /// Cross-platform unsetenv. |
587 | int unsetenv(const char* name); |
588 | |
589 | /// \brief Options to configure a Thread. |
590 | /// |
591 | /// Note that the options are all hints, and the |
592 | /// underlying implementation may choose to ignore it. |
593 | struct ThreadOptions { |
594 | /// Thread stack size to use (in bytes). |
595 | size_t stack_size = 0; // 0: use system default value |
596 | /// Guard area size to use near thread stacks to use (in bytes) |
597 | size_t guard_size = 0; // 0: use system default value |
598 | int numa_node = port::kNUMANoAffinity; |
599 | }; |
600 | |
601 | /// A utility routine: copy contents of `src` in file system `src_fs` |
602 | /// to `target` in file system `target_fs`. |
603 | Status FileSystemCopyFile(FileSystem* src_fs, const std::string& src, |
604 | FileSystem* target_fs, const std::string& target); |
605 | |
606 | /// A utility routine: reads contents of named file into `*data` |
607 | Status ReadFileToString(Env* env, const std::string& fname, std::string* data); |
608 | |
609 | /// A utility routine: write contents of `data` to file named `fname` |
610 | /// (overwriting existing contents, if any). |
611 | Status WriteStringToFile(Env* env, const std::string& fname, |
612 | const StringPiece& data); |
613 | |
614 | /// Write binary representation of "proto" to the named file. |
615 | Status WriteBinaryProto(Env* env, const std::string& fname, |
616 | const protobuf::MessageLite& proto); |
617 | |
618 | /// Reads contents of named file and parse as binary encoded proto data |
619 | /// and store into `*proto`. |
620 | Status ReadBinaryProto(Env* env, const std::string& fname, |
621 | protobuf::MessageLite* proto); |
622 | |
623 | /// Write the text representation of "proto" to the named file. |
624 | inline Status WriteTextProto(Env* /* env */, const std::string& /* fname */, |
625 | const protobuf::MessageLite& /* proto */) { |
626 | return errors::Unimplemented("Can't write text protos with protolite." ); |
627 | } |
628 | Status WriteTextProto(Env* env, const std::string& fname, |
629 | const protobuf::Message& proto); |
630 | |
631 | /// Read contents of named file and parse as text encoded proto data |
632 | /// and store into `*proto`. |
633 | inline Status ReadTextProto(Env* /* env */, const std::string& /* fname */, |
634 | protobuf::MessageLite* /* proto */) { |
635 | return errors::Unimplemented("Can't parse text protos with protolite." ); |
636 | } |
637 | Status ReadTextProto(Env* env, const std::string& fname, |
638 | protobuf::Message* proto); |
639 | |
640 | /// Read contents of named file and parse as either text or binary encoded proto |
641 | /// data and store into `*proto`. |
642 | Status ReadTextOrBinaryProto(Env* env, const std::string& fname, |
643 | protobuf::Message* proto); |
644 | Status ReadTextOrBinaryProto(Env* env, const std::string& fname, |
645 | protobuf::MessageLite* proto); |
646 | |
647 | // START_SKIP_DOXYGEN |
648 | |
649 | // The following approach to register filesystems is deprecated and will be |
650 | // replaced with modular filesystem plugins registration. |
651 | // TODO(b/139060984): After all filesystems are converted, remove this. |
652 | namespace register_file_system { |
653 | |
654 | template <typename Factory> |
655 | struct Register { |
656 | Register(Env* env, const std::string& scheme, bool try_modular_filesystems) { |
657 | // TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs |
658 | // after TF 2.6+. |
659 | if (try_modular_filesystems) { |
660 | const char* env_value = getenv("TF_USE_MODULAR_FILESYSTEM" ); |
661 | string load_plugin = env_value ? absl::AsciiStrToLower(env_value) : "" ; |
662 | if (load_plugin == "true" || load_plugin == "1" ) { |
663 | // We don't register the static filesystem and wait for SIG IO one |
664 | LOG(WARNING) << "Using modular file system for '" << scheme << "'." |
665 | << " Please switch to tensorflow-io" |
666 | << " (https://github.com/tensorflow/io) for file system" |
667 | << " support of '" << scheme << "'." ; |
668 | return; |
669 | } |
670 | // If the envvar is missing or not "true"/"1", then fall back to legacy |
671 | // implementation to be backwards compatible. |
672 | } |
673 | // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object! |
674 | env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; }) |
675 | .IgnoreError(); |
676 | } |
677 | }; |
678 | |
679 | } // namespace register_file_system |
680 | |
681 | // END_SKIP_DOXYGEN |
682 | |
683 | } // namespace tsl |
684 | |
685 | // Register a FileSystem implementation for a scheme. Files with names that have |
686 | // "scheme://" prefixes are routed to use this implementation. |
687 | #define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, modular) \ |
688 | REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, modular) |
689 | #define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, modular) \ |
690 | REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular) |
691 | #define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular) \ |
692 | static ::tsl::register_file_system::Register<factory> register_ff##ctr \ |
693 | TF_ATTRIBUTE_UNUSED = \ |
694 | ::tsl::register_file_system::Register<factory>(env, scheme, modular) |
695 | |
696 | #define REGISTER_FILE_SYSTEM(scheme, factory) \ |
697 | REGISTER_FILE_SYSTEM_ENV(::tsl::Env::Default(), scheme, factory, false); |
698 | |
699 | #define REGISTER_LEGACY_FILE_SYSTEM(scheme, factory) \ |
700 | REGISTER_FILE_SYSTEM_ENV(::tsl::Env::Default(), scheme, factory, true); |
701 | |
702 | #endif // TENSORFLOW_TSL_PLATFORM_ENV_H_ |
703 | |