Apache Arrow (C++)
A columnar in-memory analytics layer designed to accelerate big data.
memory.h
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing,
12 // software distributed under the License is distributed on an
13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 // KIND, either express or implied. See the License for the
15 // specific language governing permissions and limitations
16 // under the License.
17 
18 // Public API for different memory sharing / IO mechanisms
19 
20 #ifndef ARROW_IO_MEMORY_H
21 #define ARROW_IO_MEMORY_H
22 
23 #include <cstdint>
24 #include <memory>
25 
26 #include "arrow/io/interfaces.h"
27 #include "arrow/util/visibility.h"
28 
29 namespace arrow {
30 
31 class Buffer;
32 class MemoryPool;
33 class ResizableBuffer;
34 class Status;
35 
36 namespace io {
37 
38 // \brief An output stream that writes to a resizable buffer
39 class ARROW_EXPORT BufferOutputStream : public OutputStream {
40  public:
41  explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
42 
43  static Status Create(int64_t initial_capacity, MemoryPool* pool,
44  std::shared_ptr<BufferOutputStream>* out);
45 
46  ~BufferOutputStream() override;
47 
48  // Implement the OutputStream interface
49  Status Close() override;
50  Status Tell(int64_t* position) const override;
51  Status Write(const uint8_t* data, int64_t nbytes) override;
52 
54  Status Finish(std::shared_ptr<Buffer>* result);
55 
56  private:
57  // Ensures there is sufficient space available to write nbytes
58  Status Reserve(int64_t nbytes);
59 
60  std::shared_ptr<ResizableBuffer> buffer_;
61  bool is_open_;
62  int64_t capacity_;
63  int64_t position_;
64  uint8_t* mutable_data_;
65 };
66 
67 // \brief A helper class to tracks the size of allocations
68 class ARROW_EXPORT MockOutputStream : public OutputStream {
69  public:
70  MockOutputStream() : extent_bytes_written_(0) {}
71 
72  // Implement the OutputStream interface
73  Status Close() override;
74  Status Tell(int64_t* position) const override;
75  Status Write(const uint8_t* data, int64_t nbytes) override;
76 
77  int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
78 
79  private:
80  int64_t extent_bytes_written_;
81 };
82 
84 class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile {
85  public:
87  explicit FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer);
88  ~FixedSizeBufferWriter() override;
89 
90  Status Close() override;
91  Status Seek(int64_t position) override;
92  Status Tell(int64_t* position) const override;
93  Status Write(const uint8_t* data, int64_t nbytes) override;
94  Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override;
95 
96  void set_memcopy_threads(int num_threads);
97  void set_memcopy_blocksize(int64_t blocksize);
98  void set_memcopy_threshold(int64_t threshold);
99 
100  protected:
101  class FixedSizeBufferWriterImpl;
102  std::unique_ptr<FixedSizeBufferWriterImpl> impl_;
103 };
104 
107 class ARROW_EXPORT BufferReader : public RandomAccessFile {
108  public:
109  explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
110  BufferReader(const uint8_t* data, int64_t size);
111 
112  Status Close() override;
113  Status Tell(int64_t* position) const override;
114  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override;
115 
116  // Zero copy read
117  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
118  Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
119  uint8_t* out) override;
120 
122  Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr<Buffer>* out) override;
123 
124  Status GetSize(int64_t* size) override;
125  Status Seek(int64_t position) override;
126 
127  bool supports_zero_copy() const override;
128 
129  std::shared_ptr<Buffer> buffer() const { return buffer_; }
130 
131  protected:
132  std::shared_ptr<Buffer> buffer_;
133  const uint8_t* data_;
134  int64_t size_;
135  int64_t position_;
136 };
137 
138 } // namespace io
139 } // namespace arrow
140 
141 #endif // ARROW_IO_MEMORY_H
Definition: interfaces.h:111
Definition: interfaces.h:163
std::shared_ptr< Buffer > buffer() const
Definition: memory.h:129
Definition: interfaces.h:121
Random access zero-copy reads on an arrow::Buffer.
Definition: memory.h:107
Definition: memory.h:39
const uint8_t * data_
Definition: memory.h:133
Definition: status.h:106
MockOutputStream()
Definition: memory.h:70
int64_t GetExtentBytesWritten() const
Definition: memory.h:77
Definition: memory.h:68
Top-level namespace for Apache Arrow C++ API.
Definition: allocator.h:29
std::unique_ptr< FixedSizeBufferWriterImpl > impl_
Definition: memory.h:101
std::shared_ptr< Buffer > buffer_
Definition: memory.h:132
Base class for memory allocation.
Definition: memory_pool.h:34
Enables random writes into a fixed-size mutable buffer.
Definition: memory.h:84
int64_t position_
Definition: memory.h:135
int64_t size_
Definition: memory.h:134