Libosmium  2.11.3
Fast and flexible C++ library for working with OpenStreetMap data
gzip_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_GZIP_COMPRESSION_HPP
2 #define OSMIUM_IO_GZIP_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
45 #include <cstddef>
46 #include <string>
47 
48 #ifndef _MSC_VER
49 # include <unistd.h>
50 #endif
51 
52 #include <errno.h>
53 #include <zlib.h>
54 
56 #include <osmium/io/error.hpp>
58 #include <osmium/io/detail/read_write.hpp>
60 #include <osmium/util/cast.hpp>
62 
63 namespace osmium {
64 
69  struct gzip_error : public io_error {
70 
73 
74  gzip_error(const std::string& what, int error_code) :
75  io_error(what),
76  gzip_error_code(error_code),
77  system_errno(error_code == Z_ERRNO ? errno : 0) {
78  }
79 
80  }; // struct gzip_error
81 
82  namespace io {
83 
84  namespace detail {
85 
86  OSMIUM_NORETURN inline void throw_gzip_error(gzFile gzfile, const char* msg, int zlib_error = 0) {
87  std::string error("gzip error: ");
88  error += msg;
89  error += ": ";
90  int errnum = zlib_error;
91  if (zlib_error) {
92  error += std::to_string(zlib_error);
93  } else {
94  error += ::gzerror(gzfile, &errnum);
95  }
96  throw osmium::gzip_error(error, errnum);
97  }
98 
99  } // namespace detail
100 
101  class GzipCompressor : public Compressor {
102 
103  int m_fd;
104  gzFile m_gzfile;
105 
106  public:
107 
108  explicit GzipCompressor(int fd, fsync sync) :
109  Compressor(sync),
110  m_fd(::dup(fd)),
111  m_gzfile(::gzdopen(fd, "w")) {
112  if (!m_gzfile) {
113  detail::throw_gzip_error(m_gzfile, "write initialization failed");
114  }
115  }
116 
117  ~GzipCompressor() noexcept final {
118  try {
119  close();
120  } catch (...) {
121  // Ignore any exceptions because destructor must not throw.
122  }
123  }
124 
125  void write(const std::string& data) final {
126  if (!data.empty()) {
127  int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast_with_assert<unsigned int>(data.size()));
128  if (nwrite == 0) {
129  detail::throw_gzip_error(m_gzfile, "write failed");
130  }
131  }
132  }
133 
134  void close() final {
135  if (m_gzfile) {
136  int result = ::gzclose(m_gzfile);
137  m_gzfile = nullptr;
138  if (result != Z_OK) {
139  detail::throw_gzip_error(m_gzfile, "write close failed", result);
140  }
141  if (do_fsync()) {
142  osmium::io::detail::reliable_fsync(m_fd);
143  }
144  osmium::io::detail::reliable_close(m_fd);
145  }
146  }
147 
148  }; // class GzipCompressor
149 
151 
152  gzFile m_gzfile;
153 
154  public:
155 
156  explicit GzipDecompressor(int fd) :
157  Decompressor(),
158  m_gzfile(::gzdopen(fd, "r")) {
159  if (!m_gzfile) {
160  detail::throw_gzip_error(m_gzfile, "read initialization failed");
161  }
162  }
163 
164  ~GzipDecompressor() noexcept final {
165  try {
166  close();
167  } catch (...) {
168  // Ignore any exceptions because destructor must not throw.
169  }
170  }
171 
172  std::string read() final {
173  std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0');
174  int nread = ::gzread(m_gzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<unsigned int>(buffer.size()));
175  if (nread < 0) {
176  detail::throw_gzip_error(m_gzfile, "read failed");
177  }
178  buffer.resize(static_cast<std::string::size_type>(nread));
179 #if ZLIB_VERNUM >= 0x1240
180  set_offset(size_t(::gzoffset(m_gzfile)));
181 #endif
182  return buffer;
183  }
184 
185  void close() final {
186  if (m_gzfile) {
187  int result = ::gzclose(m_gzfile);
188  m_gzfile = nullptr;
189  if (result != Z_OK) {
190  detail::throw_gzip_error(m_gzfile, "read close failed", result);
191  }
192  }
193  }
194 
195  }; // class GzipDecompressor
196 
198 
199  const char* m_buffer;
201  z_stream m_zstream;
202 
203  public:
204 
205  GzipBufferDecompressor(const char* buffer, size_t size) :
206  m_buffer(buffer),
207  m_buffer_size(size),
208  m_zstream() {
209  m_zstream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(buffer));
210  m_zstream.avail_in = static_cast_with_assert<unsigned int>(size);
211  int result = inflateInit2(&m_zstream, MAX_WBITS | 32);
212  if (result != Z_OK) {
213  std::string message("gzip error: decompression init failed: ");
214  if (m_zstream.msg) {
215  message.append(m_zstream.msg);
216  }
217  throw osmium::gzip_error(message, result);
218  }
219  }
220 
221  ~GzipBufferDecompressor() noexcept final {
222  try {
223  close();
224  } catch (...) {
225  // Ignore any exceptions because destructor must not throw.
226  }
227  }
228 
229  std::string read() final {
230  std::string output;
231 
232  if (m_buffer) {
233  const size_t buffer_size = 10240;
234  output.append(buffer_size, '\0');
235  m_zstream.next_out = reinterpret_cast<unsigned char*>(const_cast<char*>(output.data()));
236  m_zstream.avail_out = buffer_size;
237  int result = inflate(&m_zstream, Z_SYNC_FLUSH);
238 
239  if (result != Z_OK) {
240  m_buffer = nullptr;
241  m_buffer_size = 0;
242  }
243 
244  if (result != Z_OK && result != Z_STREAM_END) {
245  std::string message("gzip error: inflate failed: ");
246  if (m_zstream.msg) {
247  message.append(m_zstream.msg);
248  }
249  throw osmium::gzip_error(message, result);
250  }
251 
252  output.resize(static_cast<unsigned long>(m_zstream.next_out - reinterpret_cast<const unsigned char*>(output.data())));
253  }
254 
255  return output;
256  }
257 
258  void close() final {
259  inflateEnd(&m_zstream);
260  }
261 
262  }; // class GzipBufferDecompressor
263 
264  namespace detail {
265 
266  // we want the register_compression() function to run, setting
267  // the variable is only a side-effect, it will never be used
269  [](int fd, fsync sync) { return new osmium::io::GzipCompressor(fd, sync); },
270  [](int fd) { return new osmium::io::GzipDecompressor(fd); },
271  [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); }
272  );
273 
274  // dummy function to silence the unused variable warning from above
275  inline bool get_registered_gzip_compression() noexcept {
276  return registered_gzip_compression;
277  }
278 
279  } // namespace detail
280 
281  } // namespace io
282 
283 } // namespace osmium
284 
285 #endif // OSMIUM_IO_GZIP_COMPRESSION_HPP
gzFile m_gzfile
Definition: gzip_compression.hpp:104
Definition: gzip_compression.hpp:197
gzip_error(const std::string &what, int error_code)
Definition: gzip_compression.hpp:74
gzFile m_gzfile
Definition: gzip_compression.hpp:152
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
Definition: gzip_compression.hpp:101
z_stream m_zstream
Definition: gzip_compression.hpp:201
static CompressionFactory & instance()
Definition: compression.hpp:179
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:96
GzipCompressor(int fd, fsync sync)
Definition: gzip_compression.hpp:108
size_t m_buffer_size
Definition: gzip_compression.hpp:200
Definition: gzip_compression.hpp:150
~GzipDecompressor() noexcept final
Definition: gzip_compression.hpp:164
Definition: compression.hpp:89
int m_fd
Definition: gzip_compression.hpp:103
GzipBufferDecompressor(const char *buffer, size_t size)
Definition: gzip_compression.hpp:205
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
const char * m_buffer
Definition: gzip_compression.hpp:199
Definition: attr.hpp:333
void close() final
Definition: gzip_compression.hpp:258
fsync
Definition: writer_options.hpp:51
void close() final
Definition: gzip_compression.hpp:134
Definition: error.hpp:44
int system_errno
Definition: gzip_compression.hpp:72
Definition: gzip_compression.hpp:69
std::string read() final
Definition: gzip_compression.hpp:172
int gzip_error_code
Definition: gzip_compression.hpp:71
GzipDecompressor(int fd)
Definition: gzip_compression.hpp:156
Definition: compression.hpp:64
void write(const std::string &data) final
Definition: gzip_compression.hpp:125
~GzipCompressor() noexcept final
Definition: gzip_compression.hpp:117
~GzipBufferDecompressor() noexcept final
Definition: gzip_compression.hpp:221
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:184
std::string read() final
Definition: gzip_compression.hpp:229
void close() final
Definition: gzip_compression.hpp:185