1 #ifndef OSMIUM_IO_READER_HPP
2 #define OSMIUM_IO_READER_HPP
37 #include <osmium/io/detail/input_format.hpp>
38 #include <osmium/io/detail/queue_util.hpp>
39 #include <osmium/io/detail/read_thread.hpp>
40 #include <osmium/io/detail/read_write.hpp>
56 #include <system_error>
61 # include <sys/wait.h>
74 inline std::size_t get_input_queue_size() noexcept {
78 inline std::size_t get_osmdata_queue_size() noexcept {
159 const detail::ParserFactory::create_parser_type& creator,
160 detail::future_string_queue_type& input_queue,
161 detail::future_buffer_queue_type& osmdata_queue,
162 std::promise<osmium::io::Header>&& header_promise,
163 std::atomic<std::size_t>* offset_ptr,
167 bool want_buffered_pages_removed) {
168 std::promise<osmium::io::Header> promise{std::move(header_promise)};
169 osmium::io::detail::parser_arguments args = {
179 want_buffered_pages_removed
181 creator(args)->parse();
196 static int execute(
const std::string& command,
const std::string& filename,
int* childpid) {
198 if (pipe(pipefd) < 0) {
199 throw std::system_error{errno, std::system_category(),
"opening pipe failed"};
201 const pid_t pid = fork();
203 throw std::system_error{errno, std::system_category(),
"fork failed"};
207 for (
int i = 0; i < 32; ++i) {
208 if (i != pipefd[1]) {
212 if (dup2(pipefd[1], 1) < 0) {
216 ::open(
"/dev/null", O_RDONLY);
217 ::open(
"/dev/null", O_WRONLY);
222 if (::execlp(command.c_str(), command.c_str(),
"-g", filename.c_str(),
nullptr) < 0) {
242 const std::string protocol{filename.substr(0, filename.find_first_of(
':'))};
243 if (protocol ==
"http" || protocol ==
"https" || protocol ==
"ftp" || protocol ==
"file") {
245 return execute(
"curl", filename, childpid);
247 throw io_error{
"Reading OSM files from the network currently not supported on Windows."};
250 const int fd = osmium::io::detail::open_for_reading(filename);
254 ::posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
270 return factory.create_decompressor(file.
compression(), fd);
315 template <
typename... TArgs>
327 (void)std::initializer_list<int>{
337 std::promise<osmium::io::Header> header_promise;
353 template <
typename... TArgs>
354 explicit Reader(
const std::string& filename, TArgs&&... args) :
358 template <
typename... TArgs>
359 explicit Reader(
const char* filename, TArgs&&... args) :
402 #pragma GCC diagnostic push
403 #pragma GCC diagnostic ignored "-Wold-style-cast"
404 if (pid < 0 || !WIFEXITED(
status) || WEXITSTATUS(
status) != 0) {
405 throw std::system_error{errno, std::system_category(),
"subprocess returned error"};
407 #pragma GCC diagnostic pop
421 throw io_error{
"Can not get header from reader when in status 'error'"};
445 osmium::memory::Buffer
read() {
446 osmium::memory::Buffer buffer;
460 throw io_error{
"Can not read from reader when in status 'closed', 'eof', or 'error'"};
475 if (detail::at_end_of_data(buffer)) {
480 if (buffer.has_nested_buffers()) {
484 if (buffer.committed() > 0) {
539 template <
typename... TArgs>
541 osmium::memory::Buffer buffer{1024 * 1024, osmium::memory::Buffer::auto_grow::yes};
543 Reader reader{std::forward<TArgs>(args)...};
544 while (
auto read_buffer = reader.read()) {
545 buffer.add_buffer(read_buffer);
static CompressionFactory & instance()
Definition: compression.hpp:191
Definition: compression.hpp:287
size_t buffer_size() const noexcept
Definition: file.hpp:147
bool has_multiple_object_versions() const noexcept
Definition: file.hpp:303
const char * buffer() const noexcept
Definition: file.hpp:143
file_compression compression() const noexcept
Definition: file.hpp:294
file_format format() const noexcept
Definition: file.hpp:285
Definition: reader.hpp:90
osmium::memory::Buffer read()
Definition: reader.hpp:445
osmium::io::buffers_type m_buffers_kind
Definition: reader.hpp:133
detail::future_string_queue_type m_input_queue
Definition: reader.hpp:112
osmium::memory::Buffer m_back_buffers
Definition: reader.hpp:95
static void parser_thread(osmium::thread::Pool &pool, int fd, const detail::ParserFactory::create_parser_type &creator, detail::future_string_queue_type &input_queue, detail::future_buffer_queue_type &osmdata_queue, std::promise< osmium::io::Header > &&header_promise, std::atomic< std::size_t > *offset_ptr, osmium::osm_entity_bits::type read_which_entities, osmium::io::read_meta read_metadata, osmium::io::buffers_type buffers_kind, bool want_buffered_pages_removed)
Definition: reader.hpp:157
int m_childpid
Definition: reader.hpp:110
void set_option(osmium::io::read_meta value) noexcept
Definition: reader.hpp:143
detail::future_buffer_queue_type m_osmdata_queue
Definition: reader.hpp:120
std::size_t m_file_size
Definition: reader.hpp:128
Reader & operator=(Reader &&)=delete
void set_option(osmium::thread::Pool &pool) noexcept
Definition: reader.hpp:135
void set_option(osmium::io::buffers_type value) noexcept
Definition: reader.hpp:152
static int execute(const std::string &command, const std::string &filename, int *childpid)
Definition: reader.hpp:196
enum osmium::io::Reader::status m_status
static int open_input_file_or_url(const std::string &filename, int *childpid)
Definition: reader.hpp:241
std::unique_ptr< osmium::io::Decompressor > m_decompressor
Definition: reader.hpp:116
status
Definition: reader.hpp:103
Reader(const char *filename, TArgs &&... args)
Definition: reader.hpp:359
osmium::io::Header m_header
Definition: reader.hpp:124
detail::ParserFactory::create_parser_type m_creator
Definition: reader.hpp:101
Reader & operator=(const Reader &)=delete
osmium::io::Header header()
Definition: reader.hpp:419
detail::queue_wrapper< osmium::memory::Buffer > m_osmdata_queue_wrapper
Definition: reader.hpp:121
Reader(const osmium::io::File &file, TArgs &&... args)
Definition: reader.hpp:316
std::size_t file_size() const noexcept
Definition: reader.hpp:507
void set_option(osmium::osm_entity_bits::type value) noexcept
Definition: reader.hpp:139
std::future< osmium::io::Header > m_header_future
Definition: reader.hpp:123
std::unique_ptr< Decompressor > make_decompressor(const osmium::io::File &file, int fd)
Definition: reader.hpp:260
osmium::io::detail::ReadThreadManager m_read_thread_manager
Definition: reader.hpp:118
bool eof() const
Definition: reader.hpp:499
Reader(const Reader &)=delete
std::size_t offset() const noexcept
Definition: reader.hpp:525
std::atomic< std::size_t > m_offset
Definition: reader.hpp:129
osmium::thread::thread_handler m_thread
Definition: reader.hpp:126
void close()
Definition: reader.hpp:385
~Reader() noexcept
Definition: reader.hpp:369
osmium::io::File m_file
Definition: reader.hpp:97
osmium::osm_entity_bits::type m_read_which_entities
Definition: reader.hpp:131
Reader(const std::string &filename, TArgs &&... args)
Definition: reader.hpp:354
osmium::io::read_meta m_read_metadata
Definition: reader.hpp:132
int m_fd
Definition: reader.hpp:114
osmium::thread::Pool * m_pool
Definition: reader.hpp:99
static Pool & default_instance()
Definition: pool.hpp:186
std::size_t get_max_queue_size(const char *queue_name, const std::size_t default_value) noexcept
Definition: config.hpp:83
int8_t clean_page_cache_after_read() noexcept
Definition: config.hpp:106
osmium::memory::Buffer read_file(TArgs &&... args)
Definition: reader.hpp:540
buffers_type
Definition: file_format.hpp:60
read_meta
Definition: file_format.hpp:55
type
Definition: entity_bits.hpp:63
@ all
object or changeset
Definition: entity_bits.hpp:76
@ nothing
Definition: entity_bits.hpp:67
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: location.hpp:551