forked from standardese/cppast
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcpp_token.hpp
More file actions
135 lines (108 loc) · 3.42 KB
/
Copy pathcpp_token.hpp
File metadata and controls
135 lines (108 loc) · 3.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
// Copyright (C) 2017-2018 Jonathan Müller <[email protected]>
// This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution.
#ifndef CPPAST_CPP_TOKEN_HPP_INCLUDED
#define CPPAST_CPP_TOKEN_HPP_INCLUDED
#include <string>
#include <vector>
#include <type_safe/reference.hpp>
namespace cppast
{
/// The kinds of C++ tokens.
enum class cpp_token_kind
{
identifier, //< Any identifier.
keyword, //< Any keyword.
int_literal, //< An integer literal.
float_literal, //< A floating point literal.
char_literal, //< A character literal.
string_literal, //< A string literal.
punctuation //< Any other punctuation.
};
/// A C++ token.
struct cpp_token
{
std::string spelling;
cpp_token_kind kind;
cpp_token(cpp_token_kind kind, std::string spelling) : spelling(std::move(spelling)), kind(kind)
{}
friend bool operator==(const cpp_token& lhs, const cpp_token& rhs) noexcept
{
return lhs.spelling == rhs.spelling;
}
friend bool operator!=(const cpp_token& lhs, const cpp_token& rhs) noexcept
{
return !(rhs == lhs);
}
};
/// A combination of multiple C++ tokens.
class cpp_token_string
{
public:
/// Builds a token string.
class builder
{
public:
builder() = default;
/// \effects Adds a token.
void add_token(cpp_token tok)
{
tokens_.push_back(std::move(tok));
}
/// \effects Converts a trailing `>>` to `>` token.
void unmunch();
/// \returns The finished string.
cpp_token_string finish()
{
return cpp_token_string(std::move(tokens_));
}
private:
std::vector<cpp_token> tokens_;
};
/// Tokenizes a string.
/// \effects Splits the string into C++ tokens.
/// The string must contain valid tokens and must already be preprocessed (i.e. translation
/// phase 6 is already done). \returns The tokenized string.
static cpp_token_string tokenize(std::string str);
/// \effects Creates it from a sequence of tokens.
cpp_token_string(std::vector<cpp_token> tokens) : tokens_(std::move(tokens)) {}
/// \exclude target
using iterator = std::vector<cpp_token>::const_iterator;
/// \returns An iterator to the first token.
iterator begin() const noexcept
{
return tokens_.begin();
}
/// \returns An iterator one past the last token.
iterator end() const noexcept
{
return tokens_.end();
}
/// \returns Whether or not the string is empty.
bool empty() const noexcept
{
return tokens_.empty();
}
/// \returns A reference to the first token.
const cpp_token& front() const noexcept
{
return tokens_.front();
}
/// \returns A reference to the last token.
const cpp_token& back() const noexcept
{
return tokens_.back();
}
/// \returns The string representation of the tokens, without any whitespace.
std::string as_string() const;
private:
std::vector<cpp_token> tokens_;
friend bool operator==(const cpp_token_string& lhs, const cpp_token_string& rhs);
};
bool operator==(const cpp_token_string& lhs, const cpp_token_string& rhs);
inline bool operator!=(const cpp_token_string& lhs, const cpp_token_string& rhs)
{
return !(lhs == rhs);
}
} // namespace cppast
#endif // CPPAST_CPP_TOKEN_HPP_INCLUDED