Apache Arrow (C++)
A columnar in-memory analytics layer designed to accelerate big data.
hdfs-internal.h
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing,
12 // software distributed under the License is distributed on an
13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 // KIND, either express or implied. See the License for the
15 // specific language governing permissions and limitations
16 // under the License.
17 
18 #ifndef ARROW_IO_HDFS_INTERNAL
19 #define ARROW_IO_HDFS_INTERNAL
20 
21 #include <cstddef>
22 #include <cstdint>
23 
24 #include <hdfs.h>
25 
26 #include "arrow/io/windows_compatibility.h" // IWYU pragma: keep
27 #include "arrow/util/visibility.h"
28 
29 using std::size_t;
30 
31 struct hdfsBuilder;
32 
33 namespace arrow {
34 
35 class Status;
36 
37 namespace io {
38 namespace internal {
39 
40 // NOTE(wesm): cpplint does not like use of short and other imprecise C types
41 struct LibHdfsShim {
42 #ifndef _WIN32
43  void* handle;
44 #else
45  HINSTANCE handle;
46 #endif
47 
48  hdfsBuilder* (*hdfsNewBuilder)(void);
49  void (*hdfsBuilderSetNameNode)(hdfsBuilder* bld, const char* nn);
50  void (*hdfsBuilderSetNameNodePort)(hdfsBuilder* bld, tPort port);
51  void (*hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName);
52  void (*hdfsBuilderSetKerbTicketCachePath)(hdfsBuilder* bld,
53  const char* kerbTicketCachePath);
54  void (*hdfsBuilderSetForceNewInstance)(hdfsBuilder* bld);
55  hdfsFS (*hdfsBuilderConnect)(hdfsBuilder* bld);
56  int (*hdfsBuilderConfSetStr)(hdfsBuilder* bld, const char* key, const char* val);
57 
58  int (*hdfsDisconnect)(hdfsFS fs);
59 
60  hdfsFile (*hdfsOpenFile)(hdfsFS fs, const char* path, int flags, int bufferSize,
61  short replication, tSize blocksize); // NOLINT
62 
63  int (*hdfsCloseFile)(hdfsFS fs, hdfsFile file);
64  int (*hdfsExists)(hdfsFS fs, const char* path);
65  int (*hdfsSeek)(hdfsFS fs, hdfsFile file, tOffset desiredPos);
66  tOffset (*hdfsTell)(hdfsFS fs, hdfsFile file);
67  tSize (*hdfsRead)(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
68  tSize (*hdfsPread)(hdfsFS fs, hdfsFile file, tOffset position, void* buffer,
69  tSize length);
70  tSize (*hdfsWrite)(hdfsFS fs, hdfsFile file, const void* buffer, tSize length);
71  int (*hdfsFlush)(hdfsFS fs, hdfsFile file);
72  int (*hdfsAvailable)(hdfsFS fs, hdfsFile file);
73  int (*hdfsCopy)(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
74  int (*hdfsMove)(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
75  int (*hdfsDelete)(hdfsFS fs, const char* path, int recursive);
76  int (*hdfsRename)(hdfsFS fs, const char* oldPath, const char* newPath);
77  char* (*hdfsGetWorkingDirectory)(hdfsFS fs, char* buffer, size_t bufferSize);
78  int (*hdfsSetWorkingDirectory)(hdfsFS fs, const char* path);
79  int (*hdfsCreateDirectory)(hdfsFS fs, const char* path);
80  int (*hdfsSetReplication)(hdfsFS fs, const char* path, int16_t replication);
81  hdfsFileInfo* (*hdfsListDirectory)(hdfsFS fs, const char* path, int* numEntries);
82  hdfsFileInfo* (*hdfsGetPathInfo)(hdfsFS fs, const char* path);
83  void (*hdfsFreeFileInfo)(hdfsFileInfo* hdfsFileInfo, int numEntries);
84  char*** (*hdfsGetHosts)(hdfsFS fs, const char* path, tOffset start, tOffset length);
85  void (*hdfsFreeHosts)(char*** blockHosts);
86  tOffset (*hdfsGetDefaultBlockSize)(hdfsFS fs);
87  tOffset (*hdfsGetCapacity)(hdfsFS fs);
88  tOffset (*hdfsGetUsed)(hdfsFS fs);
89  int (*hdfsChown)(hdfsFS fs, const char* path, const char* owner, const char* group);
90  int (*hdfsChmod)(hdfsFS fs, const char* path, short mode); // NOLINT
91  int (*hdfsUtime)(hdfsFS fs, const char* path, tTime mtime, tTime atime);
92 
93  void Initialize() {
94  this->handle = nullptr;
95  this->hdfsNewBuilder = nullptr;
96  this->hdfsBuilderSetNameNode = nullptr;
97  this->hdfsBuilderSetNameNodePort = nullptr;
98  this->hdfsBuilderSetUserName = nullptr;
99  this->hdfsBuilderSetKerbTicketCachePath = nullptr;
100  this->hdfsBuilderSetForceNewInstance = nullptr;
101  this->hdfsBuilderConfSetStr = nullptr;
102  this->hdfsBuilderConnect = nullptr;
103  this->hdfsDisconnect = nullptr;
104  this->hdfsOpenFile = nullptr;
105  this->hdfsCloseFile = nullptr;
106  this->hdfsExists = nullptr;
107  this->hdfsSeek = nullptr;
108  this->hdfsTell = nullptr;
109  this->hdfsRead = nullptr;
110  this->hdfsPread = nullptr;
111  this->hdfsWrite = nullptr;
112  this->hdfsFlush = nullptr;
113  this->hdfsAvailable = nullptr;
114  this->hdfsCopy = nullptr;
115  this->hdfsMove = nullptr;
116  this->hdfsDelete = nullptr;
117  this->hdfsRename = nullptr;
118  this->hdfsGetWorkingDirectory = nullptr;
119  this->hdfsSetWorkingDirectory = nullptr;
120  this->hdfsCreateDirectory = nullptr;
121  this->hdfsSetReplication = nullptr;
122  this->hdfsListDirectory = nullptr;
123  this->hdfsGetPathInfo = nullptr;
124  this->hdfsFreeFileInfo = nullptr;
125  this->hdfsGetHosts = nullptr;
126  this->hdfsFreeHosts = nullptr;
127  this->hdfsGetDefaultBlockSize = nullptr;
128  this->hdfsGetCapacity = nullptr;
129  this->hdfsGetUsed = nullptr;
130  this->hdfsChown = nullptr;
131  this->hdfsChmod = nullptr;
132  this->hdfsUtime = nullptr;
133  }
134 
135  hdfsBuilder* NewBuilder(void);
136 
137  void BuilderSetNameNode(hdfsBuilder* bld, const char* nn);
138 
139  void BuilderSetNameNodePort(hdfsBuilder* bld, tPort port);
140 
141  void BuilderSetUserName(hdfsBuilder* bld, const char* userName);
142 
143  void BuilderSetKerbTicketCachePath(hdfsBuilder* bld, const char* kerbTicketCachePath);
144 
145  void BuilderSetForceNewInstance(hdfsBuilder* bld);
146 
147  int BuilderConfSetStr(hdfsBuilder* bld, const char* key, const char* val);
148 
149  hdfsFS BuilderConnect(hdfsBuilder* bld);
150 
151  int Disconnect(hdfsFS fs);
152 
153  hdfsFile OpenFile(hdfsFS fs, const char* path, int flags, int bufferSize,
154  short replication, tSize blocksize); // NOLINT
155 
156  int CloseFile(hdfsFS fs, hdfsFile file);
157 
158  int Exists(hdfsFS fs, const char* path);
159 
160  int Seek(hdfsFS fs, hdfsFile file, tOffset desiredPos);
161 
162  tOffset Tell(hdfsFS fs, hdfsFile file);
163 
164  tSize Read(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
165 
166  bool HasPread();
167 
168  tSize Pread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length);
169 
170  tSize Write(hdfsFS fs, hdfsFile file, const void* buffer, tSize length);
171 
172  int Flush(hdfsFS fs, hdfsFile file);
173 
174  int Available(hdfsFS fs, hdfsFile file);
175 
176  int Copy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
177 
178  int Move(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
179 
180  int Delete(hdfsFS fs, const char* path, int recursive);
181 
182  int Rename(hdfsFS fs, const char* oldPath, const char* newPath);
183 
184  char* GetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize);
185 
186  int SetWorkingDirectory(hdfsFS fs, const char* path);
187 
188  int MakeDirectory(hdfsFS fs, const char* path);
189 
190  int SetReplication(hdfsFS fs, const char* path, int16_t replication);
191 
192  hdfsFileInfo* ListDirectory(hdfsFS fs, const char* path, int* numEntries);
193 
194  hdfsFileInfo* GetPathInfo(hdfsFS fs, const char* path);
195 
196  void FreeFileInfo(hdfsFileInfo* hdfsFileInfo, int numEntries);
197 
198  char*** GetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length);
199 
200  void FreeHosts(char*** blockHosts);
201 
202  tOffset GetDefaultBlockSize(hdfsFS fs);
203  tOffset GetCapacity(hdfsFS fs);
204 
205  tOffset GetUsed(hdfsFS fs);
206 
207  int Chown(hdfsFS fs, const char* path, const char* owner, const char* group);
208 
209  int Chmod(hdfsFS fs, const char* path, short mode); // NOLINT
210 
211  int Utime(hdfsFS fs, const char* path, tTime mtime, tTime atime);
212 
213  Status GetRequiredSymbols();
214 };
215 
216 // TODO(wesm): Remove these exports when we are linking statically
217 Status ARROW_EXPORT ConnectLibHdfs(LibHdfsShim** driver);
218 Status ARROW_EXPORT ConnectLibHdfs3(LibHdfsShim** driver);
219 
220 } // namespace internal
221 } // namespace io
222 } // namespace arrow
223 
224 #endif // ARROW_IO_HDFS_INTERNAL
Top-level namespace for Apache Arrow C++ API.
Definition: adapter.h:32