1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#ifndef GLOW_BACKENDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H
17#define GLOW_BACKENDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H
18
19#include "glow/Backends/QueueBackedDeviceManager.h"
20#include "glow/Runtime/StatsExporter.h"
21
22namespace glow {
23namespace runtime {
24
25/// A class controlling a single "Interpreter Device", a thread of execution in
26/// the IR-Interpreter. Many InterpreterFunctions may be added, but only one
27/// inference is executed at a time.
28class InterpreterDeviceManager : public QueueBackedDeviceManager {
29 /// Compiled function list by name.
30 FunctionMapTy functions_;
31
32 /// Map from PH to functionName for static placeholders.
33 std::unordered_map<Placeholder *, std::vector<std::string>>
34 staticPlaceholderToFunctions_;
35
36 /// String constant for logging number of in-use devices.
37 static constexpr const char *kDevicesUsedInterpreter =
38 "glow.devices_used.interpreter";
39
40public:
41 explicit InterpreterDeviceManager(const DeviceConfig &config)
42 : QueueBackedDeviceManager(config) {
43 statsExporterRegistry_->incrementCounter(kDevicesUsedInterpreter);
44 exportMemoryCounters();
45 }
46
47 ~InterpreterDeviceManager() override {
48 statsExporterRegistry_->incrementCounter(kDevicesUsedInterpreter, -1);
49 zeroMemoryCounters();
50 }
51
52 /// Returns the amount of memory in bytes available on the device when no
53 /// models are loaded.
54 uint64_t getMaximumMemory() const override;
55
56 /// Returns the amount of memory in bytes currently availbe on the device.
57 uint64_t getAvailableMemory() const override;
58
59 /// Returns true if a function requiring the \p estimate size will fit on the
60 /// device. This is not a promise as memory cost could vary due to alignment,
61 /// etc.
62 bool isMemoryAvailable(uint64_t estimate) const override;
63
64 /// Returns the DeviceInfo for this device containing peak limits for
65 /// compute and bandwidths (used in partitioning).
66 DeviceInfo getDeviceInfo() const override;
67
68 /// Copies the contents of Tensor \p T to the device resource allocated to
69 /// Placeholder \p PH. once finished calls \p resultCB with the result of the
70 /// operation.
71 void transferStaticPlaceholderToDevice(
72 Placeholder *PH, Tensor *T, std::function<void(Error)> resultCB) override;
73
74protected:
75 void addNetworkImpl(const Module *module, FunctionMapTy functions,
76 ReadyCBTy cb) override;
77 void evictNetworkImpl(std::string functionName,
78 EvictFunctionCBTy evictCB) override;
79 void runFunctionImpl(runtime::RunIdentifierTy id, std::string functionName,
80 std::unique_ptr<ExecutionContext> context,
81 ResultCBTy cb) override;
82};
83
84DeviceManager *createInterpreterDeviceManager(const DeviceConfig &config);
85
86} // namespace runtime
87} // namespace glow
88
89#endif // GLOW_BACKENBDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H
90