id,summary,reporter,owner,description,type,status,milestone,component,version,severity,resolution,keywords,cc
12497,Errors with QI and Karma rules managing 64 bit integers,Gilles Brunet <gillesb68@…>,Joel de Guzman,"MSVC++ 14 used to build the following source code enclosing a trio of spirit components (Lex, QI and Karma) performing three steps:

1. Uses “Spirit Lex” to make tokens from input text; 
1. Uses “Spirit QI” for parsing tokens to generate an AST; and
1. Uses “Spirit Karma” for generating back the original text.

The “QI” and “Karma” grammars are compiling and running fine with tokens enclosing 32 bit integers. However, the compiler generates thousands of errors with tokens enclosing 64 bit integers.

For reproducing the problem, we move comments in front of the alias type named “int_lit_type” for defining it as a 64-bit integer. The code compiles and runs fine with 32 bit integers, and do not compile with 64 bit ones. The same result carried out whenever the compiler is configured for targeting “x86” or “x64”. 

Here is the source code:
{{{
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/karma.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/variant.hpp>
#include <boost/fusion/adapted/struct.hpp>
#include <boost/phoenix.hpp>
#include <boost/phoenix/object/construct.hpp>


#include <string>
#include <vector>

namespace spirit = boost::spirit;
namespace phoenix = boost::phoenix;
namespace lex = spirit::lex;
namespace qi = spirit::qi;
namespace karma = spirit::karma;

///////////////////////////////////
// Lexer

using real_lit_type = long double;
//using int_lit_type = long;
using int_lit_type = std::int32_t;
//using int_lit_type = long long;
//using int_lit_type = std::int64_t;

template <typename Lexer>
struct lexer_grammar : lex::lexer<Lexer>
{
	lexer_grammar()
		: int_lit{""[0-9]+""}
		  , real_lit{""[0-9]+\\.([0-9]+([Ee][\\-\\+]?[0-9]*)?)?""}
		  , identifier{""[_a-zA-Z][_a-zA-Z0-9]*""}
	{
		this->self
			= lex::token_def<>(""[ \\t\\n\\r]+"")[lex::_pass = lex::pass_flags::pass_ignore]
			| real_lit
			| int_lit
			| identifier
			| lex::token_def<>('(') | ')' | '=' | ';';
	}

	lex::token_def<real_lit_type> real_lit;
	lex::token_def<int_lit_type> int_lit;
	lex::token_def<std::string> identifier;
};

template <typename Iterator>
struct tokens_factory
{
	using iterator_type = Iterator;
	using value_type = boost::mpl::vector<int_lit_type, real_lit_type, std::string>;
	using token_type = lex::lexertl::token<iterator_type, value_type, boost::mpl::true_>;
	using lexer_type = lexer_grammar<lex::lexertl::actor_lexer<token_type>>;
};

///////////////////////////////////
// AST

namespace ast {

	using literal_type = boost::variant<int_lit_type, real_lit_type>;
	using identifier_type = std::string;

	using expr_type = boost::variant<literal_type, identifier_type>;

	struct assign_stmt_type
	{
		assign_stmt_type() {}

		template <typename Stmt>
		explicit assign_stmt_type(const Stmt& stmt);

		identifier_type var_ref;
		expr_type expr;
	};

	using stmt_type = boost::variant<assign_stmt_type>;
	using stmt_list_type = std::vector<stmt_type>;

	template <>
	inline assign_stmt_type::assign_stmt_type(const stmt_type& stmt)
	{
		*this = boost::get<assign_stmt_type>(stmt);
	}

}

BOOST_FUSION_ADAPT_STRUCT(
	ast::assign_stmt_type,
	var_ref,
	expr
)

template <typename Iterator, typename Lexer>
struct parser_grammar : qi::grammar<Iterator, ast::stmt_list_type()>
{
	explicit parser_grammar(const Lexer& tokens)
		: parser_grammar::base_type{stmt_list}
	{
		stmt_list = *(stmt >> ';');

		stmt = assign_stmt;

		assign_stmt = var_ref_expr >> '=' >> expr;

		expr
			= ('(' >> expr >> ')')
			| var_ref_expr
			| literal_expr;

		var_ref_expr
			= tokens.identifier;

		literal_expr
			= tokens.int_lit
			| tokens.real_lit;
	}

	qi::rule<Iterator, ast::stmt_list_type()> stmt_list;
	qi::rule<Iterator, ast::stmt_type()> stmt;
	qi::rule<Iterator, ast::assign_stmt_type()> assign_stmt;
	qi::rule<Iterator, ast::expr_type()> expr;
	qi::rule<Iterator, ast::identifier_type()> var_ref_expr;
	qi::rule<Iterator, ast::literal_type()> literal_expr;
};

///////////////////////////////////
// Generator

template <typename Iterator>
struct generator_grammar
	: karma::grammar<Iterator, ast::stmt_list_type()>
{
	generator_grammar() : generator_grammar::base_type(stmt_list)
	{
		stmt_list = *(stmt << karma::lit(';') << karma::eol);

		stmt = assign_stmt;

		assign_stmt = var_ref_expr << karma::lit("" = "") << expr;

		expr
			= var_ref_expr
			| literal_expr;

		var_ref_expr = karma::string;

		literal_expr
			= int_literal_expr
			| real_literal_expr;

		int_literal_expr
			= karma::long_long
			| karma::long_;

		real_literal_expr
			= karma::long_double;
	}

	karma::rule<Iterator, ast::stmt_list_type()> stmt_list;
	karma::rule<Iterator, ast::stmt_type()> stmt;
	karma::rule<Iterator, ast::assign_stmt_type()> assign_stmt;
	karma::rule<Iterator, ast::expr_type()> expr;
	karma::rule<Iterator, ast::identifier_type()> var_ref_expr;
	karma::rule<Iterator, ast::literal_type()> literal_expr;
	karma::rule<Iterator, int_lit_type()> int_literal_expr;
	karma::rule<Iterator, real_lit_type()> real_literal_expr;
};

///////////////////////////////////
// main

int main()
{
	static const auto text = std::string{
		""var1 = 25.3;\n""
		""var2 = 455;\n""
		""var3 = var2;""
	};

	using lexer_type = tokens_factory<std::string::const_iterator>::lexer_type;
	lexer_type lexer{};
	auto tokens = std::vector<lexer_type::token_type>{};
	parser_grammar<lexer_type::iterator_type, lexer_type> parser{lexer};

	auto it = text.begin();
	auto lex_it = lexer.begin(it, text.end());
	auto lex_end = lexer.end();

	auto stmt_list = ast::stmt_list_type{};
	auto parser_success = qi::parse(lex_it, lex_end, parser, stmt_list);

	if (parser_success) {
		std::cout << ""parser success. Then, generates back the source."" << std::endl << std::endl;

		using output_iterator_type = std::back_insert_iterator<std::string>;
		using generator_type = generator_grammar<output_iterator_type>;

		std::string generated;
		output_iterator_type outit(generated);
		generator_type generator{};
		auto generator_success = karma::generate(outit, generator, stmt_list);

		if (generator_success) {
			std::cout
				<< ""generator success:"" << std::endl
				<< generated << std::endl;
		} else
			std::cout << ""generator failure"" << std::endl;
	} else
		std::cout << ""parser failure"" << std::endl;

	return 0;
}
}}}
",Bugs,closed,To Be Determined,spirit,Boost 1.61.0,Showstopper,fixed,,