1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | #ifndef GLOW_BACKENDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H |
17 | #define GLOW_BACKENDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H |
18 | |
19 | #include "glow/Backends/QueueBackedDeviceManager.h" |
20 | #include "glow/Runtime/StatsExporter.h" |
21 | |
22 | namespace glow { |
23 | namespace runtime { |
24 | |
25 | /// A class controlling a single "Interpreter Device", a thread of execution in |
26 | /// the IR-Interpreter. Many InterpreterFunctions may be added, but only one |
27 | /// inference is executed at a time. |
28 | class InterpreterDeviceManager : public QueueBackedDeviceManager { |
29 | /// Compiled function list by name. |
30 | FunctionMapTy functions_; |
31 | |
32 | /// Map from PH to functionName for static placeholders. |
33 | std::unordered_map<Placeholder *, std::vector<std::string>> |
34 | staticPlaceholderToFunctions_; |
35 | |
36 | /// String constant for logging number of in-use devices. |
37 | static constexpr const char *kDevicesUsedInterpreter = |
38 | "glow.devices_used.interpreter" ; |
39 | |
40 | public: |
41 | explicit InterpreterDeviceManager(const DeviceConfig &config) |
42 | : QueueBackedDeviceManager(config) { |
43 | statsExporterRegistry_->incrementCounter(kDevicesUsedInterpreter); |
44 | exportMemoryCounters(); |
45 | } |
46 | |
47 | ~InterpreterDeviceManager() override { |
48 | statsExporterRegistry_->incrementCounter(kDevicesUsedInterpreter, -1); |
49 | zeroMemoryCounters(); |
50 | } |
51 | |
52 | /// Returns the amount of memory in bytes available on the device when no |
53 | /// models are loaded. |
54 | uint64_t getMaximumMemory() const override; |
55 | |
56 | /// Returns the amount of memory in bytes currently availbe on the device. |
57 | uint64_t getAvailableMemory() const override; |
58 | |
59 | /// Returns true if a function requiring the \p estimate size will fit on the |
60 | /// device. This is not a promise as memory cost could vary due to alignment, |
61 | /// etc. |
62 | bool isMemoryAvailable(uint64_t estimate) const override; |
63 | |
64 | /// Returns the DeviceInfo for this device containing peak limits for |
65 | /// compute and bandwidths (used in partitioning). |
66 | DeviceInfo getDeviceInfo() const override; |
67 | |
68 | /// Copies the contents of Tensor \p T to the device resource allocated to |
69 | /// Placeholder \p PH. once finished calls \p resultCB with the result of the |
70 | /// operation. |
71 | void transferStaticPlaceholderToDevice( |
72 | Placeholder *PH, Tensor *T, std::function<void(Error)> resultCB) override; |
73 | |
74 | protected: |
75 | void addNetworkImpl(const Module *module, FunctionMapTy functions, |
76 | ReadyCBTy cb) override; |
77 | void evictNetworkImpl(std::string functionName, |
78 | EvictFunctionCBTy evictCB) override; |
79 | void runFunctionImpl(runtime::RunIdentifierTy id, std::string functionName, |
80 | std::unique_ptr<ExecutionContext> context, |
81 | ResultCBTy cb) override; |
82 | }; |
83 | |
84 | DeviceManager *createInterpreterDeviceManager(const DeviceConfig &config); |
85 | |
86 | } // namespace runtime |
87 | } // namespace glow |
88 | |
89 | #endif // GLOW_BACKENBDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H |
90 | |