You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

tokenize.hpp 8.8KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. /// \file
  2. // Range v3 library
  3. //
  4. // Copyright Eric Niebler 2013-present
  5. //
  6. // Use, modification and distribution is subject to the
  7. // Boost Software License, Version 1.0. (See accompanying
  8. // file LICENSE_1_0.txt or copy at
  9. // http://www.boost.org/LICENSE_1_0.txt)
  10. //
  11. // Project home: https://github.com/ericniebler/range-v3
  12. //
  13. #ifndef RANGES_V3_VIEW_TOKENIZE_HPP
  14. #define RANGES_V3_VIEW_TOKENIZE_HPP
  15. #include <initializer_list>
  16. #include <regex>
  17. #include <type_traits>
  18. #include <utility>
  19. #include <vector>
  20. #include <range/v3/range_fwd.hpp>
  21. #include <range/v3/functional/bind_back.hpp>
  22. #include <range/v3/range/access.hpp>
  23. #include <range/v3/range/concepts.hpp>
  24. #include <range/v3/utility/static_const.hpp>
  25. #include <range/v3/view/all.hpp>
  26. #include <range/v3/view/interface.hpp>
  27. #include <range/v3/view/view.hpp>
  28. namespace ranges
  29. {
  30. /// \addtogroup group-views
  31. /// @{
  32. template<typename Rng, typename Regex, typename SubMatchRange>
  33. struct tokenize_view
  34. : view_interface<tokenize_view<Rng, Regex, SubMatchRange>,
  35. is_finite<Rng>::value ? finite : range_cardinality<Rng>::value>
  36. {
  37. private:
  38. CPP_assert(bidirectional_range<Rng> && view_<Rng> && common_range<Rng>);
  39. CPP_assert(semiregular<Regex>);
  40. CPP_assert(semiregular<SubMatchRange>);
  41. Rng rng_;
  42. Regex rex_;
  43. SubMatchRange subs_;
  44. std::regex_constants::match_flag_type flags_;
  45. template<bool Const>
  46. using iterator_t =
  47. std::regex_token_iterator<iterator_t<meta::const_if_c<Const, Rng>>>;
  48. public:
  49. tokenize_view() = default;
  50. tokenize_view(Rng rng, Regex rex, SubMatchRange subs,
  51. std::regex_constants::match_flag_type flags)
  52. : rng_(std::move(rng))
  53. , rex_(std::move(rex))
  54. , subs_(std::move(subs))
  55. , flags_(flags)
  56. {}
  57. iterator_t<simple_view<Rng>()> begin()
  58. {
  59. meta::const_if_c<simple_view<Rng>(), Rng> & rng = rng_;
  60. return {ranges::begin(rng), ranges::end(rng), rex_, subs_, flags_};
  61. }
  62. template<bool Const = true>
  63. auto begin() const -> CPP_ret(iterator_t<Const>)( //
  64. requires range<Rng const>)
  65. {
  66. return {ranges::begin(rng_), ranges::end(rng_), rex_, subs_, flags_};
  67. }
  68. iterator_t<simple_view<Rng>()> end()
  69. {
  70. return {};
  71. }
  72. template<bool Const = true>
  73. auto end() const -> CPP_ret(iterator_t<Const>)( //
  74. requires range<Rng const>)
  75. {
  76. return {};
  77. }
  78. Rng base() const
  79. {
  80. return rng_;
  81. }
  82. };
  83. #if RANGES_CXX_DEDUCTION_GUIDES >= RANGES_CXX_DEDUCTION_GUIDES_17
  84. CPP_template(typename Rng, typename Regex, typename SubMatchRange)(
  85. requires copy_constructible<Regex> && copy_constructible<SubMatchRange>)
  86. tokenize_view(Rng &&, Regex, SubMatchRange)
  87. ->tokenize_view<views::all_t<Rng>, Regex, SubMatchRange>;
  88. #endif
  89. namespace views
  90. {
  91. struct tokenizer_impl_fn
  92. {
  93. template<typename Rng, typename Regex>
  94. tokenize_view<all_t<Rng>, detail::decay_t<Regex>, int> operator()(
  95. Rng && rng, Regex && rex, int sub = 0,
  96. std::regex_constants::match_flag_type flags =
  97. std::regex_constants::match_default) const
  98. {
  99. CPP_assert(bidirectional_range<Rng>);
  100. CPP_assert(common_range<Rng>);
  101. static_assert(
  102. RANGES_IS_SAME(range_value_t<Rng>,
  103. typename detail::decay_t<Regex>::value_type),
  104. "The character range and the regex have different character types");
  105. return {all(static_cast<Rng &&>(rng)),
  106. static_cast<Regex &&>(rex),
  107. sub,
  108. flags};
  109. }
  110. template<typename Rng, typename Regex>
  111. tokenize_view<all_t<Rng>, detail::decay_t<Regex>, std::vector<int>>
  112. operator()(Rng && rng, Regex && rex, std::vector<int> subs,
  113. std::regex_constants::match_flag_type flags =
  114. std::regex_constants::match_default) const
  115. {
  116. CPP_assert(bidirectional_range<Rng>);
  117. CPP_assert(common_range<Rng>);
  118. static_assert(
  119. RANGES_IS_SAME(range_value_t<Rng>,
  120. typename detail::decay_t<Regex>::value_type),
  121. "The character range and the regex have different character types");
  122. return {all(static_cast<Rng &&>(rng)),
  123. static_cast<Regex &&>(rex),
  124. std::move(subs),
  125. flags};
  126. }
  127. template<typename Rng, typename Regex>
  128. tokenize_view<all_t<Rng>, detail::decay_t<Regex>, std::initializer_list<int>>
  129. operator()(Rng && rng, Regex && rex, std::initializer_list<int> subs,
  130. std::regex_constants::match_flag_type flags =
  131. std::regex_constants::match_default) const
  132. {
  133. CPP_assert(bidirectional_range<Rng>);
  134. CPP_assert(common_range<Rng>);
  135. static_assert(
  136. RANGES_IS_SAME(range_value_t<Rng>,
  137. typename detail::decay_t<Regex>::value_type),
  138. "The character range and the regex have different character types");
  139. return {all(static_cast<Rng &&>(rng)),
  140. static_cast<Regex &&>(rex),
  141. std::move(subs),
  142. flags};
  143. }
  144. template<typename Regex>
  145. auto operator()(Regex && rex, int sub = 0,
  146. std::regex_constants::match_flag_type flags =
  147. std::regex_constants::match_default) const
  148. {
  149. return make_pipeable(
  150. bind_back(*this, static_cast<Regex &&>(rex), sub, flags));
  151. }
  152. template<typename Regex>
  153. auto operator()(Regex && rex, std::vector<int> subs,
  154. std::regex_constants::match_flag_type flags =
  155. std::regex_constants::match_default) const
  156. {
  157. return make_pipeable(
  158. bind_back(*this, static_cast<Regex &&>(rex), std::move(subs), flags));
  159. }
  160. template<typename Regex>
  161. auto operator()(Regex && rex, std::initializer_list<int> subs,
  162. std::regex_constants::match_flag_type flags =
  163. std::regex_constants::match_default) const
  164. {
  165. return make_pipeable(
  166. bind_back(*this, static_cast<Regex &&>(rex), subs, flags));
  167. }
  168. };
  169. // Damn C++ and its imperfect forwarding of initializer_list.
  170. struct tokenize_fn : tokenizer_impl_fn
  171. {
  172. private:
  173. tokenizer_impl_fn const & base() const
  174. {
  175. return *this;
  176. }
  177. public:
  178. template<typename... Args>
  179. auto operator()(Args &&... args) const
  180. -> decltype(base()(static_cast<Args &&>(args)...))
  181. {
  182. return base()(static_cast<Args &&>(args)...);
  183. }
  184. template<typename Arg0, typename... Args>
  185. auto operator()(Arg0 && arg0, std::initializer_list<int> subs,
  186. Args &&... args) const
  187. -> decltype(base()(static_cast<Arg0 &&>(arg0), std::move(subs),
  188. static_cast<Args &&>(args)...))
  189. {
  190. return base()(static_cast<Arg0 &&>(arg0),
  191. std::move(subs),
  192. static_cast<Args &&>(args)...);
  193. }
  194. template<typename Arg0, typename Arg1, typename... Args>
  195. auto operator()(Arg0 && arg0, Arg1 && arg1, std::initializer_list<int> subs,
  196. Args &&... args) const
  197. -> decltype(base()(static_cast<Arg0 &&>(arg0), static_cast<Arg1 &&>(arg1),
  198. std::move(subs), static_cast<Args &&>(args)...))
  199. {
  200. return base()(static_cast<Arg0 &&>(arg0),
  201. static_cast<Arg1 &&>(arg1),
  202. std::move(subs),
  203. static_cast<Args &&>(args)...);
  204. }
  205. };
  206. /// \relates tokenize_fn
  207. /// \ingroup group-views
  208. RANGES_INLINE_VARIABLE(tokenize_fn, tokenize)
  209. } // namespace views
  210. /// @}
  211. } // namespace ranges
  212. #include <range/v3/detail/satisfy_boost_range.hpp>
  213. RANGES_SATISFY_BOOST_RANGE(::ranges::tokenize_view)
  214. #endif