1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. |
14 | ==============================================================================*/ |
15 | |
16 | #include "tensorflow/core/framework/op_kernel.h" |
17 | #include "tensorflow/core/framework/resource_mgr.h" |
18 | #include "tensorflow/core/framework/tensor.h" |
19 | #include "tensorflow/core/framework/tensor_shape.h" |
20 | #include "tensorflow/core/kernels/record_yielder.h" |
21 | #include "tensorflow/core/lib/strings/strcat.h" |
22 | #include "tensorflow/core/platform/env.h" |
23 | |
24 | namespace tensorflow { |
25 | |
26 | class RecordInputOp : public OpKernel { |
27 | public: |
28 | explicit RecordInputOp(OpKernelConstruction* ctx) : OpKernel(ctx) { |
29 | #define GETATTR(TYPE, FIELD) \ |
30 | TYPE FIELD; \ |
31 | OP_REQUIRES_OK(ctx, ctx->GetAttr(#FIELD, &FIELD)); |
32 | |
33 | GETATTR(string, file_pattern); |
34 | GETATTR(int64_t, file_random_seed); |
35 | GETATTR(float, file_shuffle_shift_ratio); |
36 | GETATTR(int64_t, file_buffer_size); |
37 | GETATTR(int64_t, file_parallelism); |
38 | GETATTR(int64_t, batch_size); |
39 | GETATTR(string, compression_type); |
40 | #undef GETATTR |
41 | |
42 | OP_REQUIRES_OK(ctx, ctx->GetAttr("compression_type" , &compression_type)); |
43 | |
44 | RecordYielder::Options yopts; |
45 | yopts.file_pattern = file_pattern; |
46 | yopts.seed = file_random_seed; |
47 | yopts.bufsize = file_buffer_size; |
48 | yopts.file_shuffle_shift_ratio = file_shuffle_shift_ratio; |
49 | yopts.parallelism = file_parallelism; |
50 | yopts.compression_type = compression_type; |
51 | yielder_ = std::unique_ptr<RecordYielder>(new RecordYielder(ctx, yopts)); |
52 | |
53 | batch_size_ = batch_size; |
54 | } |
55 | |
56 | void Compute(OpKernelContext* ctx) override { |
57 | Tensor out(DT_STRING, {batch_size_}); |
58 | auto t_out = out.flat<tstring>(); |
59 | for (int i = 0; i < batch_size_; ++i) { |
60 | OP_REQUIRES_OK(ctx, yielder_->YieldOne(&t_out(i))); |
61 | } |
62 | ctx->set_output(0, out); |
63 | } |
64 | |
65 | private: |
66 | int64_t batch_size_; |
67 | std::unique_ptr<RecordYielder> yielder_; |
68 | }; |
69 | |
70 | REGISTER_KERNEL_BUILDER(Name("RecordInput" ).Device(DEVICE_CPU), RecordInputOp); |
71 | } // namespace tensorflow |
72 | |