Line data Source code
1 : // Copyright (C) 2020-2026 Free Software Foundation, Inc.
2 :
3 : // This file is part of GCC.
4 :
5 : // GCC is free software; you can redistribute it and/or modify it under
6 : // the terms of the GNU General Public License as published by the Free
7 : // Software Foundation; either version 3, or (at your option) any later
8 : // version.
9 :
10 : // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
11 : // WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 : // for more details.
14 :
15 : // You should have received a copy of the GNU General Public License
16 : // along with GCC; see the file COPYING3. If not see
17 : // <http://www.gnu.org/licenses/>.
18 :
19 : #ifndef RUST_LEX_H
20 : #define RUST_LEX_H
21 :
22 : #include "rust-linemap.h"
23 : #include "rust-buffered-queue.h"
24 : #include "rust-token.h"
25 : #include "optional.h"
26 : #include "selftest.h"
27 : #include "rust-input-source.h"
28 :
29 : namespace Rust {
30 : // Simple wrapper for FILE* that simplifies destruction.
31 : struct RAIIFile
32 : {
33 : private:
34 : FILE *file;
35 : const char *filename;
36 :
37 15055 : void close ()
38 : {
39 15055 : if (file != nullptr && file != stdin)
40 5010 : fclose (file);
41 15055 : }
42 :
43 5014 : static bool allowed_filetype (const struct stat &statbuf)
44 : {
45 : // The file could be either
46 : // - a regular file
47 : // - a char device (/dev/null...)
48 5014 : return S_ISREG (statbuf.st_mode) || S_ISCHR (statbuf.st_mode);
49 : }
50 :
51 : public:
52 5027 : RAIIFile (const char *filename) : filename (filename)
53 : {
54 5027 : if (strcmp (filename, "-") == 0)
55 : {
56 0 : file = stdin;
57 : }
58 : else
59 : {
60 5027 : struct stat statbuf;
61 5027 : if (!(file = fopen (filename, "r")))
62 : {
63 13 : return;
64 : }
65 :
66 5014 : if (-1 == fstat (fileno (file), &statbuf)
67 5014 : || !allowed_filetype (statbuf))
68 : {
69 2 : fclose (file);
70 2 : file = nullptr;
71 2 : errno = EISDIR;
72 : }
73 : }
74 : }
75 :
76 : /**
77 : * Create a RAIIFile from an existing instance of FILE*
78 : */
79 104 : RAIIFile (FILE *raw, const char *filename = nullptr)
80 104 : : file (raw), filename (filename)
81 : {}
82 :
83 : RAIIFile (const RAIIFile &other) = delete;
84 : RAIIFile &operator= (const RAIIFile &other) = delete;
85 :
86 : // have to specify setting file to nullptr, otherwise unintended fclose occurs
87 9928 : RAIIFile (RAIIFile &&other) : file (other.file), filename (other.filename)
88 : {
89 9928 : other.file = nullptr;
90 : }
91 :
92 : RAIIFile &operator= (RAIIFile &&other)
93 : {
94 : close ();
95 : file = other.file;
96 : filename = other.filename;
97 : other.file = nullptr;
98 :
99 : return *this;
100 : }
101 :
102 104 : static RAIIFile create_error () { return RAIIFile (nullptr, nullptr); }
103 :
104 15055 : ~RAIIFile () { close (); }
105 :
106 5016 : FILE *get_raw () { return file; }
107 856 : const char *get_filename () { return filename; }
108 :
109 4975 : bool ok () const { return file; }
110 : };
111 :
112 : class Lexer
113 : {
114 : private:
115 : // Request new Location for current column in line_table
116 : location_t get_current_location ();
117 :
118 : // Skips the current input character.
119 : void skip_input ();
120 : // Advances current input character to n + 1 characters ahead of current
121 : // position.
122 : void skip_input (int n);
123 :
124 : // Peeks the current character.
125 : Codepoint peek_input ();
126 : // Returns character n characters ahead of current position.
127 : Codepoint peek_input (int n);
128 :
129 : // Classifies keyword (i.e. gets id for keyword).
130 : TokenId classify_keyword (const std::string &str);
131 :
132 : std::tuple<std::string, int, bool> parse_in_decimal ();
133 : std::pair<std::string, int> parse_in_exponent_part ();
134 : std::pair<std::string, int> parse_in_suffix ();
135 : std::tuple<char, int, bool> parse_escape (char opening_char);
136 : std::tuple<Codepoint, int, bool> parse_utf8_escape ();
137 : int parse_partial_string_continue ();
138 : std::pair<long, int> parse_partial_hex_escape ();
139 : std::pair<Codepoint, int> parse_partial_unicode_escape ();
140 :
141 : void skip_broken_string_input (Codepoint current_char);
142 :
143 : TokenPtr parse_byte_char (location_t loc);
144 : TokenPtr parse_byte_string (location_t loc);
145 : TokenPtr parse_raw_byte_string (location_t loc);
146 : TokenPtr parse_raw_identifier (location_t loc);
147 : TokenPtr parse_string (location_t loc);
148 : TokenPtr maybe_parse_raw_string (location_t loc);
149 : TokenPtr parse_raw_string (location_t loc, int initial_hash_count);
150 : TokenPtr parse_c_string (location_t loc);
151 : TokenPtr parse_non_decimal_int_literals (location_t loc);
152 : TokenPtr parse_decimal_int_or_float (location_t loc);
153 : TokenPtr parse_char_or_lifetime (location_t loc);
154 : TokenPtr parse_identifier_or_keyword (location_t loc);
155 :
156 : template <typename IsDigitFunc>
157 : TokenPtr parse_non_decimal_int_literal (location_t loc,
158 : IsDigitFunc is_digit_func,
159 : IntegerLiteralBase base);
160 :
161 : public:
162 : // Construct lexer with input file and filename provided
163 : Lexer (const char *filename, RAIIFile input, Linemap *linemap,
164 : tl::optional<std::ofstream &> dump_lex_opt = tl::nullopt);
165 :
166 : // Lex the contents of a string instead of a file
167 : Lexer (const std::string &input, Linemap *linemap);
168 :
169 : // dtor
170 : ~Lexer ();
171 :
172 : // don't allow copy semantics (for now, at least)
173 : Lexer (const Lexer &other) = delete;
174 : Lexer &operator= (const Lexer &other) = delete;
175 :
176 : // enable move semantics
177 : Lexer (Lexer &&other) = default;
178 : Lexer &operator= (Lexer &&other) = default;
179 :
180 : bool input_source_is_valid_utf8 ();
181 :
182 : // Returns token n tokens ahead of current position.
183 246386 : const_TokenPtr peek_token (int n) { return token_queue.peek (n); }
184 : // Peeks the current token.
185 2898750 : const_TokenPtr peek_token () { return peek_token (0); }
186 :
187 : // Builds a token from the input queue.
188 : TokenPtr build_token ();
189 :
190 : // Advances current token to n + 1 tokens ahead of current position.
191 : void skip_token (int n);
192 : // Skips the current token.
193 736000 : void skip_token () { skip_token (0); }
194 :
195 : // Dumps and advances by n + 1 tokens.
196 : void dump_and_skip (int n);
197 :
198 : // Replaces the current token with a specified token.
199 : void replace_current_token (TokenPtr replacement);
200 : // FIXME: don't use anymore
201 :
202 : /* Splits the current token into two. Intended for use with nested generics
203 : * closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
204 : * this will only work with "simple" tokens like punctuation. */
205 : void split_current_token (TokenId new_left, TokenId new_right);
206 :
207 : void split_current_token (std::vector<TokenPtr> new_tokens);
208 :
209 : Linemap *get_line_map () { return line_map; }
210 856 : std::string get_filename () { return std::string (input.get_filename ()); }
211 :
212 : private:
213 : void start_line (int current_line, int current_column);
214 :
215 : // File for use as input.
216 : RAIIFile input;
217 : // TODO is this actually required? could just have file storage in InputSource
218 :
219 : // Current line number.
220 : int current_line;
221 : // Current column number.
222 : int current_column;
223 : // Current character.
224 : Codepoint current_char;
225 : // Line map.
226 : Linemap *line_map;
227 :
228 : /* Max column number that can be quickly allocated - higher may require
229 : * allocating new linemap */
230 : static const int max_column_hint = 80;
231 :
232 : tl::optional<std::ofstream &> dump_lex_out;
233 :
234 : // The input source for the lexer.
235 : // InputSource input_source;
236 : // Input file queue.
237 : std::unique_ptr<InputSource> raw_input_source;
238 : buffered_queue<Codepoint, std::reference_wrapper<InputSource>> input_queue;
239 :
240 : // Token source wrapper thing.
241 : struct TokenSource
242 : {
243 : // The lexer object that will use this TokenSource.
244 : Lexer *lexer;
245 :
246 : // Create a new TokenSource with given lexer.
247 4964 : TokenSource (Lexer *parLexer) : lexer (parLexer) {}
248 :
249 : // Used to mimic std::reference_wrapper that is used for InputSource.
250 : TokenSource &get () { return *this; }
251 :
252 : // Overload operator () to build token in lexer.
253 769702 : TokenPtr next () { return lexer->build_token (); }
254 : };
255 :
256 : // The token source for the lexer.
257 : // TokenSource token_source;
258 : // Token stream queue.
259 : buffered_queue<std::shared_ptr<Token>, TokenSource> token_queue;
260 : };
261 :
262 : } // namespace Rust
263 :
264 : #if CHECKING_P
265 :
266 : namespace selftest {
267 : void rust_input_source_test ();
268 :
269 : } // namespace selftest
270 :
271 : #endif // CHECKING_P
272 :
273 : #endif
|
