ygm::io::parquet_parser

class parquet_parser

Public Types

using parquet_type_variant = std::variant<std::monostate, bool, int32_t, int64_t, float, double, std::string>

Public Functions

inline parquet_parser(ygm::comm &_comm, const std::vector<std::string> &stringpaths, const bool recursive = false)

inline ~parquet_parser()

inline const std::vector<column_schema_type> &get_schema() const

Returns a list of column schema (simpler version). The order of the schema is the same as the order of Parquet column indices (ascending order). This function assumes that all files have the same schema. Returns an empty vector if there is no file the rank can read.

inline const std::string &schema_to_string() const

template<typename Function>
inline void for_all(Function fn, const size_t num_rows = std::numeric_limits<size_t>::max())

Read all rows and call the function for each row.

Parameters:

• fn – A function to call for every row. Expected signature is void(const std::vector<parquet_type_variant>&). The value of an unsupported column is set to std::monostate.

• num_rows – Max number of rows the rank to read.

template<typename Function>
inline void for_all(const std::vector<std::string> &columns, Function fn, const size_t num_rows = std::numeric_limits<size_t>::max())

for_all(), read only the specified columns.

inline std::optional<std::vector<parquet_type_variant>> peek()

Return the first row assigned to the rank. Return nullopt if no row was assgined.

inline size_t num_files() const

Return the total number of files.

inline size_t num_rows() const

Return the number of rows in all files.

struct column_schema_type

Public Members

detail::parquet_data_type type

std::string name

bool unsupported = {false}