Branch data Line data Source code
1 : : /* Instruction scheduling pass. Selective scheduler and pipeliner.
2 : : Copyright (C) 2006-2024 Free Software Foundation, Inc.
3 : :
4 : : This file is part of GCC.
5 : :
6 : : GCC is free software; you can redistribute it and/or modify it under
7 : : the terms of the GNU General Public License as published by the Free
8 : : Software Foundation; either version 3, or (at your option) any later
9 : : version.
10 : :
11 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : : for more details.
15 : :
16 : : You should have received a copy of the GNU General Public License
17 : : along with GCC; see the file COPYING3. If not see
18 : : <http://www.gnu.org/licenses/>. */
19 : :
20 : : #include "config.h"
21 : : #include "system.h"
22 : : #include "coretypes.h"
23 : : #include "backend.h"
24 : : #include "tree.h"
25 : : #include "rtl.h"
26 : : #include "df.h"
27 : : #include "memmodel.h"
28 : : #include "tm_p.h"
29 : : #include "regs.h"
30 : : #include "cfgbuild.h"
31 : : #include "cfgcleanup.h"
32 : : #include "insn-config.h"
33 : : #include "insn-attr.h"
34 : : #include "target.h"
35 : : #include "sched-int.h"
36 : : #include "rtlhooks-def.h"
37 : : #include "ira.h"
38 : : #include "ira-int.h"
39 : : #include "rtl-iter.h"
40 : :
41 : : #ifdef INSN_SCHEDULING
42 : : #include "regset.h"
43 : : #include "cfgloop.h"
44 : : #include "sel-sched-ir.h"
45 : : #include "sel-sched-dump.h"
46 : : #include "sel-sched.h"
47 : : #include "dbgcnt.h"
48 : : #include "function-abi.h"
49 : :
50 : : /* Implementation of selective scheduling approach.
51 : : The below implementation follows the original approach with the following
52 : : changes:
53 : :
54 : : o the scheduler works after register allocation (but can be also tuned
55 : : to work before RA);
56 : : o some instructions are not copied or register renamed;
57 : : o conditional jumps are not moved with code duplication;
58 : : o several jumps in one parallel group are not supported;
59 : : o when pipelining outer loops, code motion through inner loops
60 : : is not supported;
61 : : o control and data speculation are supported;
62 : : o some improvements for better compile time/performance were made.
63 : :
64 : : Terminology
65 : : ===========
66 : :
67 : : A vinsn, or virtual insn, is an insn with additional data characterizing
68 : : insn pattern, such as LHS, RHS, register sets used/set/clobbered, etc.
69 : : Vinsns also act as smart pointers to save memory by reusing them in
70 : : different expressions. A vinsn is described by vinsn_t type.
71 : :
72 : : An expression is a vinsn with additional data characterizing its properties
73 : : at some point in the control flow graph. The data may be its usefulness,
74 : : priority, speculative status, whether it was renamed/subsituted, etc.
75 : : An expression is described by expr_t type.
76 : :
77 : : Availability set (av_set) is a set of expressions at a given control flow
78 : : point. It is represented as av_set_t. The expressions in av sets are kept
79 : : sorted in the terms of expr_greater_p function. It allows to truncate
80 : : the set while leaving the best expressions.
81 : :
82 : : A fence is a point through which code motion is prohibited. On each step,
83 : : we gather a parallel group of insns at a fence. It is possible to have
84 : : multiple fences. A fence is represented via fence_t.
85 : :
86 : : A boundary is the border between the fence group and the rest of the code.
87 : : Currently, we never have more than one boundary per fence, as we finalize
88 : : the fence group when a jump is scheduled. A boundary is represented
89 : : via bnd_t.
90 : :
91 : : High-level overview
92 : : ===================
93 : :
94 : : The scheduler finds regions to schedule, schedules each one, and finalizes.
95 : : The regions are formed starting from innermost loops, so that when the inner
96 : : loop is pipelined, its prologue can be scheduled together with yet unprocessed
97 : : outer loop. The rest of acyclic regions are found using extend_rgns:
98 : : the blocks that are not yet allocated to any regions are traversed in top-down
99 : : order, and a block is added to a region to which all its predecessors belong;
100 : : otherwise, the block starts its own region.
101 : :
102 : : The main scheduling loop (sel_sched_region_2) consists of just
103 : : scheduling on each fence and updating fences. For each fence,
104 : : we fill a parallel group of insns (fill_insns) until some insns can be added.
105 : : First, we compute available exprs (av-set) at the boundary of the current
106 : : group. Second, we choose the best expression from it. If the stall is
107 : : required to schedule any of the expressions, we advance the current cycle
108 : : appropriately. So, the final group does not exactly correspond to a VLIW
109 : : word. Third, we move the chosen expression to the boundary (move_op)
110 : : and update the intermediate av sets and liveness sets. We quit fill_insns
111 : : when either no insns left for scheduling or we have scheduled enough insns
112 : : so we feel like advancing a scheduling point.
113 : :
114 : : Computing available expressions
115 : : ===============================
116 : :
117 : : The computation (compute_av_set) is a bottom-up traversal. At each insn,
118 : : we're moving the union of its successors' sets through it via
119 : : moveup_expr_set. The dependent expressions are removed. Local
120 : : transformations (substitution, speculation) are applied to move more
121 : : exprs. Then the expr corresponding to the current insn is added.
122 : : The result is saved on each basic block header.
123 : :
124 : : When traversing the CFG, we're moving down for no more than max_ws insns.
125 : : Also, we do not move down to ineligible successors (is_ineligible_successor),
126 : : which include moving along a back-edge, moving to already scheduled code,
127 : : and moving to another fence. The first two restrictions are lifted during
128 : : pipelining, which allows us to move insns along a back-edge. We always have
129 : : an acyclic region for scheduling because we forbid motion through fences.
130 : :
131 : : Choosing the best expression
132 : : ============================
133 : :
134 : : We sort the final availability set via sel_rank_for_schedule, then we remove
135 : : expressions which are not yet ready (tick_check_p) or which dest registers
136 : : cannot be used. For some of them, we choose another register via
137 : : find_best_reg. To do this, we run find_used_regs to calculate the set of
138 : : registers which cannot be used. The find_used_regs function performs
139 : : a traversal of code motion paths for an expr. We consider for renaming
140 : : only registers which are from the same regclass as the original one and
141 : : using which does not interfere with any live ranges. Finally, we convert
142 : : the resulting set to the ready list format and use max_issue and reorder*
143 : : hooks similarly to the Haifa scheduler.
144 : :
145 : : Scheduling the best expression
146 : : ==============================
147 : :
148 : : We run the move_op routine to perform the same type of code motion paths
149 : : traversal as in find_used_regs. (These are working via the same driver,
150 : : code_motion_path_driver.) When moving down the CFG, we look for original
151 : : instruction that gave birth to a chosen expression. We undo
152 : : the transformations performed on an expression via the history saved in it.
153 : : When found, we remove the instruction or leave a reg-reg copy/speculation
154 : : check if needed. On a way up, we insert bookkeeping copies at each join
155 : : point. If a copy is not needed, it will be removed later during this
156 : : traversal. We update the saved av sets and liveness sets on the way up, too.
157 : :
158 : : Finalizing the schedule
159 : : =======================
160 : :
161 : : When pipelining, we reschedule the blocks from which insns were pipelined
162 : : to get a tighter schedule. On Itanium, we also perform bundling via
163 : : the same routine from ia64.cc.
164 : :
165 : : Dependence analysis changes
166 : : ===========================
167 : :
168 : : We augmented the sched-deps.cc with hooks that get called when a particular
169 : : dependence is found in a particular part of an insn. Using these hooks, we
170 : : can do several actions such as: determine whether an insn can be moved through
171 : : another (has_dependence_p, moveup_expr); find out whether an insn can be
172 : : scheduled on the current cycle (tick_check_p); find out registers that
173 : : are set/used/clobbered by an insn and find out all the strange stuff that
174 : : restrict its movement, like SCHED_GROUP_P or CANT_MOVE (done in
175 : : init_global_and_expr_for_insn).
176 : :
177 : : Initialization changes
178 : : ======================
179 : :
180 : : There are parts of haifa-sched.cc, sched-deps.cc, and sched-rgn.cc that are
181 : : reused in all of the schedulers. We have split up the initialization of data
182 : : of such parts into different functions prefixed with scheduler type and
183 : : postfixed with the type of data initialized: {,sel_,haifa_}sched_{init,finish},
184 : : sched_rgn_init/finish, sched_deps_init/finish, sched_init_{luids/bbs}, etc.
185 : : The same splitting is done with current_sched_info structure:
186 : : dependence-related parts are in sched_deps_info, common part is in
187 : : common_sched_info, and haifa/sel/etc part is in current_sched_info.
188 : :
189 : : Target contexts
190 : : ===============
191 : :
192 : : As we now have multiple-point scheduling, this would not work with backends
193 : : which save some of the scheduler state to use it in the target hooks.
194 : : For this purpose, we introduce a concept of target contexts, which
195 : : encapsulate such information. The backend should implement simple routines
196 : : of allocating/freeing/setting such a context. The scheduler calls these
197 : : as target hooks and handles the target context as an opaque pointer (similar
198 : : to the DFA state type, state_t).
199 : :
200 : : Various speedups
201 : : ================
202 : :
203 : : As the correct data dependence graph is not supported during scheduling (which
204 : : is to be changed in mid-term), we cache as much of the dependence analysis
205 : : results as possible to avoid reanalyzing. This includes: bitmap caches on
206 : : each insn in stream of the region saying yes/no for a query with a pair of
207 : : UIDs; hashtables with the previously done transformations on each insn in
208 : : stream; a vector keeping a history of transformations on each expr.
209 : :
210 : : Also, we try to minimize the dependence context used on each fence to check
211 : : whether the given expression is ready for scheduling by removing from it
212 : : insns that are definitely completed the execution. The results of
213 : : tick_check_p checks are also cached in a vector on each fence.
214 : :
215 : : We keep a valid liveness set on each insn in a region to avoid the high
216 : : cost of recomputation on large basic blocks.
217 : :
218 : : Finally, we try to minimize the number of needed updates to the availability
219 : : sets. The updates happen in two cases: when fill_insns terminates,
220 : : we advance all fences and increase the stage number to show that the region
221 : : has changed and the sets are to be recomputed; and when the next iteration
222 : : of a loop in fill_insns happens (but this one reuses the saved av sets
223 : : on bb headers.) Thus, we try to break the fill_insns loop only when
224 : : "significant" number of insns from the current scheduling window was
225 : : scheduled. This should be made a target param.
226 : :
227 : :
228 : : TODO: correctly support the data dependence graph at all stages and get rid
229 : : of all caches. This should speed up the scheduler.
230 : : TODO: implement moving cond jumps with bookkeeping copies on both targets.
231 : : TODO: tune the scheduler before RA so it does not create too much pseudos.
232 : :
233 : :
234 : : References:
235 : : S.-M. Moon and K. Ebcioglu. Parallelizing nonnumerical code with
236 : : selective scheduling and software pipelining.
237 : : ACM TOPLAS, Vol 19, No. 6, pages 853--898, Nov. 1997.
238 : :
239 : : Andrey Belevantsev, Maxim Kuvyrkov, Vladimir Makarov, Dmitry Melnik,
240 : : and Dmitry Zhurikhin. An interblock VLIW-targeted instruction scheduler
241 : : for GCC. In Proceedings of GCC Developers' Summit 2006.
242 : :
243 : : Arutyun Avetisyan, Andrey Belevantsev, and Dmitry Melnik. GCC Instruction
244 : : Scheduler and Software Pipeliner on the Itanium Platform. EPIC-7 Workshop.
245 : : http://rogue.colorado.edu/EPIC7/.
246 : :
247 : : */
248 : :
249 : : /* True when pipelining is enabled. */
250 : : bool pipelining_p;
251 : :
252 : : /* True if bookkeeping is enabled. */
253 : : bool bookkeeping_p;
254 : :
255 : : /* Maximum number of insns that are eligible for renaming. */
256 : : int max_insns_to_rename;
257 : :
258 : :
259 : : /* Definitions of local types and macros. */
260 : :
261 : : /* Represents possible outcomes of moving an expression through an insn. */
262 : : enum MOVEUP_EXPR_CODE
263 : : {
264 : : /* The expression is not changed. */
265 : : MOVEUP_EXPR_SAME,
266 : :
267 : : /* Not changed, but requires a new destination register. */
268 : : MOVEUP_EXPR_AS_RHS,
269 : :
270 : : /* Cannot be moved. */
271 : : MOVEUP_EXPR_NULL,
272 : :
273 : : /* Changed (substituted or speculated). */
274 : : MOVEUP_EXPR_CHANGED
275 : : };
276 : :
277 : : /* The container to be passed into rtx search & replace functions. */
278 : : struct rtx_search_arg
279 : : {
280 : : /* What we are searching for. */
281 : : rtx x;
282 : :
283 : : /* The occurrence counter. */
284 : : int n;
285 : : };
286 : :
287 : : typedef struct rtx_search_arg *rtx_search_arg_p;
288 : :
289 : : /* This struct contains precomputed hard reg sets that are needed when
290 : : computing registers available for renaming. */
291 : : struct hard_regs_data
292 : : {
293 : : /* For every mode, this stores registers available for use with
294 : : that mode. */
295 : : HARD_REG_SET regs_for_mode[NUM_MACHINE_MODES];
296 : :
297 : : /* True when regs_for_mode[mode] is initialized. */
298 : : bool regs_for_mode_ok[NUM_MACHINE_MODES];
299 : :
300 : : /* For every register, it has regs that are ok to rename into it.
301 : : The register in question is always set. If not, this means
302 : : that the whole set is not computed yet. */
303 : : HARD_REG_SET regs_for_rename[FIRST_PSEUDO_REGISTER];
304 : :
305 : : /* All registers that are used or call used. */
306 : : HARD_REG_SET regs_ever_used;
307 : :
308 : : #ifdef STACK_REGS
309 : : /* Stack registers. */
310 : : HARD_REG_SET stack_regs;
311 : : #endif
312 : : };
313 : :
314 : : /* Holds the results of computation of available for renaming and
315 : : unavailable hard registers. */
316 : : struct reg_rename
317 : : {
318 : : /* These are unavailable due to calls crossing, globalness, etc. */
319 : : HARD_REG_SET unavailable_hard_regs;
320 : :
321 : : /* These are *available* for renaming. */
322 : : HARD_REG_SET available_for_renaming;
323 : :
324 : : /* The set of ABIs used by calls that the code motion path crosses. */
325 : : unsigned int crossed_call_abis : NUM_ABI_IDS;
326 : : };
327 : :
328 : : /* A global structure that contains the needed information about harg
329 : : regs. */
330 : : static struct hard_regs_data sel_hrd;
331 : :
332 : :
333 : : /* This structure holds local data used in code_motion_path_driver hooks on
334 : : the same or adjacent levels of recursion. Here we keep those parameters
335 : : that are not used in code_motion_path_driver routine itself, but only in
336 : : its hooks. Moreover, all parameters that can be modified in hooks are
337 : : in this structure, so all other parameters passed explicitly to hooks are
338 : : read-only. */
339 : : struct cmpd_local_params
340 : : {
341 : : /* Local params used in move_op_* functions. */
342 : :
343 : : /* Edges for bookkeeping generation. */
344 : : edge e1, e2;
345 : :
346 : : /* C_EXPR merged from all successors and locally allocated temporary C_EXPR. */
347 : : expr_t c_expr_merged, c_expr_local;
348 : :
349 : : /* Local params used in fur_* functions. */
350 : : /* Copy of the ORIGINAL_INSN list, stores the original insns already
351 : : found before entering the current level of code_motion_path_driver. */
352 : : def_list_t old_original_insns;
353 : :
354 : : /* Local params used in move_op_* functions. */
355 : : /* True when we have removed last insn in the block which was
356 : : also a boundary. Do not update anything or create bookkeeping copies. */
357 : : BOOL_BITFIELD removed_last_insn : 1;
358 : : };
359 : :
360 : : /* Stores the static parameters for move_op_* calls. */
361 : : struct moveop_static_params
362 : : {
363 : : /* Destination register. */
364 : : rtx dest;
365 : :
366 : : /* Current C_EXPR. */
367 : : expr_t c_expr;
368 : :
369 : : /* An UID of expr_vliw which is to be moved up. If we find other exprs,
370 : : they are to be removed. */
371 : : int uid;
372 : :
373 : : /* This is initialized to the insn on which the driver stopped its traversal. */
374 : : insn_t failed_insn;
375 : :
376 : : /* True if we scheduled an insn with different register. */
377 : : bool was_renamed;
378 : : };
379 : :
380 : : /* Stores the static parameters for fur_* calls. */
381 : : struct fur_static_params
382 : : {
383 : : /* Set of registers unavailable on the code motion path. */
384 : : regset used_regs;
385 : :
386 : : /* Pointer to the list of original insns definitions. */
387 : : def_list_t *original_insns;
388 : :
389 : : /* The set of ABIs used by calls that the code motion path crosses. */
390 : : unsigned int crossed_call_abis : NUM_ABI_IDS;
391 : : };
392 : :
393 : : typedef struct fur_static_params *fur_static_params_p;
394 : : typedef struct cmpd_local_params *cmpd_local_params_p;
395 : : typedef struct moveop_static_params *moveop_static_params_p;
396 : :
397 : : /* Set of hooks and parameters that determine behavior specific to
398 : : move_op or find_used_regs functions. */
399 : : struct code_motion_path_driver_info_def
400 : : {
401 : : /* Called on enter to the basic block. */
402 : : int (*on_enter) (insn_t, cmpd_local_params_p, void *, bool);
403 : :
404 : : /* Called when original expr is found. */
405 : : void (*orig_expr_found) (insn_t, expr_t, cmpd_local_params_p, void *);
406 : :
407 : : /* Called while descending current basic block if current insn is not
408 : : the original EXPR we're searching for. */
409 : : bool (*orig_expr_not_found) (insn_t, av_set_t, void *);
410 : :
411 : : /* Function to merge C_EXPRes from different successors. */
412 : : void (*merge_succs) (insn_t, insn_t, int, cmpd_local_params_p, void *);
413 : :
414 : : /* Function to finalize merge from different successors and possibly
415 : : deallocate temporary data structures used for merging. */
416 : : void (*after_merge_succs) (cmpd_local_params_p, void *);
417 : :
418 : : /* Called on the backward stage of recursion to do moveup_expr.
419 : : Used only with move_op_*. */
420 : : void (*ascend) (insn_t, void *);
421 : :
422 : : /* Called on the ascending pass, before returning from the current basic
423 : : block or from the whole traversal. */
424 : : void (*at_first_insn) (insn_t, cmpd_local_params_p, void *);
425 : :
426 : : /* When processing successors in move_op we need only descend into
427 : : SUCCS_NORMAL successors, while in find_used_regs we need SUCCS_ALL. */
428 : : int succ_flags;
429 : :
430 : : /* The routine name to print in dumps ("move_op" of "find_used_regs"). */
431 : : const char *routine_name;
432 : : };
433 : :
434 : : /* Global pointer to current hooks, either points to MOVE_OP_HOOKS or
435 : : FUR_HOOKS. */
436 : : struct code_motion_path_driver_info_def *code_motion_path_driver_info;
437 : :
438 : : /* Set of hooks for performing move_op and find_used_regs routines with
439 : : code_motion_path_driver. */
440 : : extern struct code_motion_path_driver_info_def move_op_hooks, fur_hooks;
441 : :
442 : : /* True if/when we want to emulate Haifa scheduler in the common code.
443 : : This is used in sched_rgn_local_init and in various places in
444 : : sched-deps.cc. */
445 : : int sched_emulate_haifa_p;
446 : :
447 : : /* GLOBAL_LEVEL is used to discard information stored in basic block headers
448 : : av_sets. Av_set of bb header is valid if its (bb header's) level is equal
449 : : to GLOBAL_LEVEL. And invalid if lesser. This is primarily used to advance
450 : : scheduling window. */
451 : : int global_level;
452 : :
453 : : /* Current fences. */
454 : : flist_t fences;
455 : :
456 : : /* True when separable insns should be scheduled as RHSes. */
457 : : static bool enable_schedule_as_rhs_p;
458 : :
459 : : /* Used in verify_target_availability to assert that target reg is reported
460 : : unavailabile by both TARGET_UNAVAILABLE and find_used_regs only if
461 : : we haven't scheduled anything on the previous fence.
462 : : if scheduled_something_on_previous_fence is true, TARGET_UNAVAILABLE can
463 : : have more conservative value than the one returned by the
464 : : find_used_regs, thus we shouldn't assert that these values are equal. */
465 : : static bool scheduled_something_on_previous_fence;
466 : :
467 : : /* All newly emitted insns will have their uids greater than this value. */
468 : : static int first_emitted_uid;
469 : :
470 : : /* Set of basic blocks that are forced to start new ebbs. This is a subset
471 : : of all the ebb heads. */
472 : : bitmap forced_ebb_heads;
473 : :
474 : : /* Blocks that need to be rescheduled after pipelining. */
475 : : bitmap blocks_to_reschedule = NULL;
476 : :
477 : : /* True when the first lv set should be ignored when updating liveness. */
478 : : static bool ignore_first = false;
479 : :
480 : : /* Number of insns max_issue has initialized data structures for. */
481 : : static int max_issue_size = 0;
482 : :
483 : : /* Whether we can issue more instructions. */
484 : : static int can_issue_more;
485 : :
486 : : /* Maximum software lookahead window size, reduced when rescheduling after
487 : : pipelining. */
488 : : static int max_ws;
489 : :
490 : : /* Number of insns scheduled in current region. */
491 : : static int num_insns_scheduled;
492 : :
493 : : /* A vector of expressions is used to be able to sort them. */
494 : : static vec<expr_t> vec_av_set;
495 : :
496 : : /* A vector of vinsns is used to hold temporary lists of vinsns. */
497 : : typedef vec<vinsn_t> vinsn_vec_t;
498 : :
499 : : /* This vector has the exprs which may still present in av_sets, but actually
500 : : can't be moved up due to bookkeeping created during code motion to another
501 : : fence. See comment near the call to update_and_record_unavailable_insns
502 : : for the detailed explanations. */
503 : : static vinsn_vec_t vec_bookkeeping_blocked_vinsns = vinsn_vec_t ();
504 : :
505 : : /* This vector has vinsns which are scheduled with renaming on the first fence
506 : : and then seen on the second. For expressions with such vinsns, target
507 : : availability information may be wrong. */
508 : : static vinsn_vec_t vec_target_unavailable_vinsns = vinsn_vec_t ();
509 : :
510 : : /* Vector to store temporary nops inserted in move_op to prevent removal
511 : : of empty bbs. */
512 : : static vec<insn_t> vec_temp_moveop_nops;
513 : :
514 : : /* These bitmaps record original instructions scheduled on the current
515 : : iteration and bookkeeping copies created by them. */
516 : : static bitmap current_originators = NULL;
517 : : static bitmap current_copies = NULL;
518 : :
519 : : /* This bitmap marks the blocks visited by code_motion_path_driver so we don't
520 : : visit them afterwards. */
521 : : static bitmap code_motion_visited_blocks = NULL;
522 : :
523 : : /* Variables to accumulate different statistics. */
524 : :
525 : : /* The number of bookkeeping copies created. */
526 : : static int stat_bookkeeping_copies;
527 : :
528 : : /* The number of insns that required bookkeeiping for their scheduling. */
529 : : static int stat_insns_needed_bookkeeping;
530 : :
531 : : /* The number of insns that got renamed. */
532 : : static int stat_renamed_scheduled;
533 : :
534 : : /* The number of substitutions made during scheduling. */
535 : : static int stat_substitutions_total;
536 : :
537 : :
538 : : /* Forward declarations of static functions. */
539 : : static bool rtx_ok_for_substitution_p (rtx, rtx);
540 : : static int sel_rank_for_schedule (const void *, const void *);
541 : : static av_set_t find_sequential_best_exprs (bnd_t, expr_t, bool);
542 : : static basic_block find_block_for_bookkeeping (edge e1, edge e2, bool lax);
543 : :
544 : : static rtx get_dest_from_orig_ops (av_set_t);
545 : : static basic_block generate_bookkeeping_insn (expr_t, edge, edge);
546 : : static bool find_used_regs (insn_t, av_set_t, regset, struct reg_rename *,
547 : : def_list_t *);
548 : : static bool move_op (insn_t, av_set_t, expr_t, rtx, expr_t, bool*);
549 : : static int code_motion_path_driver (insn_t, av_set_t, ilist_t,
550 : : cmpd_local_params_p, void *);
551 : : static void sel_sched_region_1 (void);
552 : : static void sel_sched_region_2 (int);
553 : : static av_set_t compute_av_set_inside_bb (insn_t, ilist_t, int, bool);
554 : :
555 : : static void debug_state (state_t);
556 : :
557 : :
558 : : /* Functions that work with fences. */
559 : :
560 : : /* Advance one cycle on FENCE. */
561 : : static void
562 : 3341 : advance_one_cycle (fence_t fence)
563 : : {
564 : 3341 : unsigned i;
565 : 3341 : int cycle;
566 : 3341 : rtx_insn *insn;
567 : :
568 : 3341 : advance_state (FENCE_STATE (fence));
569 : 3341 : cycle = ++FENCE_CYCLE (fence);
570 : 3341 : FENCE_ISSUED_INSNS (fence) = 0;
571 : 3341 : FENCE_STARTS_CYCLE_P (fence) = 1;
572 : 3341 : can_issue_more = issue_rate;
573 : 3341 : FENCE_ISSUE_MORE (fence) = can_issue_more;
574 : :
575 : 14132 : for (i = 0; vec_safe_iterate (FENCE_EXECUTING_INSNS (fence), i, &insn); )
576 : : {
577 : 10791 : if (INSN_READY_CYCLE (insn) < cycle)
578 : : {
579 : 2285 : remove_from_deps (FENCE_DC (fence), insn);
580 : 2285 : FENCE_EXECUTING_INSNS (fence)->unordered_remove (i);
581 : 2285 : continue;
582 : : }
583 : 8506 : i++;
584 : : }
585 : 3341 : if (sched_verbose >= 2)
586 : : {
587 : 0 : sel_print ("Finished a cycle. Current cycle = %d\n", FENCE_CYCLE (fence));
588 : 0 : debug_state (FENCE_STATE (fence));
589 : : }
590 : 3341 : }
591 : :
592 : : /* Returns true when SUCC in a fallthru bb of INSN, possibly
593 : : skipping empty basic blocks. */
594 : : static bool
595 : 116 : in_fallthru_bb_p (rtx_insn *insn, rtx succ)
596 : : {
597 : 116 : basic_block bb = BLOCK_FOR_INSN (insn);
598 : 116 : edge e;
599 : :
600 : 116 : if (bb == BLOCK_FOR_INSN (succ))
601 : : return true;
602 : :
603 : 116 : e = find_fallthru_edge_from (bb);
604 : 116 : if (e)
605 : 100 : bb = e->dest;
606 : : else
607 : : return false;
608 : :
609 : 100 : while (sel_bb_empty_p (bb))
610 : 0 : bb = bb->next_bb;
611 : :
612 : 100 : return bb == BLOCK_FOR_INSN (succ);
613 : : }
614 : :
615 : : /* Construct successor fences from OLD_FENCEs and put them in NEW_FENCES.
616 : : When a successor will continue a ebb, transfer all parameters of a fence
617 : : to the new fence. ORIG_MAX_SEQNO is the maximal seqno before this round
618 : : of scheduling helping to distinguish between the old and the new code. */
619 : : static void
620 : 957 : extract_new_fences_from (flist_t old_fences, flist_tail_t new_fences,
621 : : int orig_max_seqno)
622 : : {
623 : 957 : bool was_here_p = false;
624 : 957 : insn_t insn = NULL;
625 : 957 : insn_t succ;
626 : 957 : succ_iterator si;
627 : 957 : ilist_iterator ii;
628 : 957 : fence_t fence = FLIST_FENCE (old_fences);
629 : 957 : basic_block bb;
630 : :
631 : : /* Get the only element of FENCE_BNDS (fence). */
632 : 1914 : FOR_EACH_INSN (insn, ii, FENCE_BNDS (fence))
633 : : {
634 : 957 : gcc_assert (!was_here_p);
635 : 957 : was_here_p = true;
636 : : }
637 : 957 : gcc_assert (was_here_p && insn != NULL_RTX);
638 : :
639 : : /* When in the "middle" of the block, just move this fence
640 : : to the new list. */
641 : 957 : bb = BLOCK_FOR_INSN (insn);
642 : 957 : if (! sel_bb_end_p (insn)
643 : 1291 : || (single_succ_p (bb)
644 : 334 : && single_pred_p (single_succ (bb))))
645 : : {
646 : 626 : insn_t succ;
647 : :
648 : 626 : succ = (sel_bb_end_p (insn)
649 : 629 : ? sel_bb_head (single_succ (bb))
650 : 623 : : NEXT_INSN (insn));
651 : :
652 : 626 : if (INSN_SEQNO (succ) > 0
653 : 626 : && INSN_SEQNO (succ) <= orig_max_seqno
654 : 1251 : && INSN_SCHED_TIMES (succ) <= 0)
655 : : {
656 : 625 : FENCE_INSN (fence) = succ;
657 : 625 : move_fence_to_fences (old_fences, new_fences);
658 : :
659 : 625 : if (sched_verbose >= 1)
660 : 0 : sel_print ("Fence %d continues as %d[%d] (state continue)\n",
661 : 0 : INSN_UID (insn), INSN_UID (succ), BLOCK_NUM (succ));
662 : : }
663 : 626 : return;
664 : : }
665 : :
666 : : /* Otherwise copy fence's structures to (possibly) multiple successors. */
667 : 746 : FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
668 : : {
669 : 415 : int seqno = INSN_SEQNO (succ);
670 : :
671 : 415 : if (seqno > 0 && seqno <= orig_max_seqno
672 : 415 : && (pipelining_p || INSN_SCHED_TIMES (succ) <= 0))
673 : : {
674 : 280 : bool b = (in_same_ebb_p (insn, succ)
675 : 280 : || in_fallthru_bb_p (insn, succ));
676 : :
677 : 280 : if (sched_verbose >= 1)
678 : 0 : sel_print ("Fence %d continues as %d[%d] (state %s)\n",
679 : 0 : INSN_UID (insn), INSN_UID (succ),
680 : 0 : BLOCK_NUM (succ), b ? "continue" : "reset");
681 : :
682 : 280 : if (b)
683 : 179 : add_dirty_fence_to_fences (new_fences, succ, fence);
684 : : else
685 : : {
686 : : /* Mark block of the SUCC as head of the new ebb. */
687 : 101 : bitmap_set_bit (forced_ebb_heads, BLOCK_NUM (succ));
688 : 101 : add_clean_fence_to_fences (new_fences, succ, fence);
689 : : }
690 : : }
691 : : }
692 : : }
693 : :
694 : :
695 : : /* Functions to support substitution. */
696 : :
697 : : /* Returns whether INSN with dependence status DS is eligible for
698 : : substitution, i.e. it's a copy operation x := y, and RHS that is
699 : : moved up through this insn should be substituted. */
700 : : static bool
701 : 1093 : can_substitute_through_p (insn_t insn, ds_t ds)
702 : : {
703 : : /* We can substitute only true dependencies. */
704 : 1093 : if ((ds & DEP_OUTPUT)
705 : 1093 : || (ds & DEP_ANTI)
706 : 960 : || ! INSN_RHS (insn)
707 : 1826 : || ! INSN_LHS (insn))
708 : : return false;
709 : :
710 : : /* Now we just need to make sure the INSN_RHS consists of only one
711 : : simple REG rtx. */
712 : 733 : if (REG_P (INSN_LHS (insn))
713 : 733 : && REG_P (INSN_RHS (insn)))
714 : 125 : return true;
715 : : return false;
716 : : }
717 : :
718 : : /* Substitute all occurrences of INSN's destination in EXPR' vinsn with INSN's
719 : : source (if INSN is eligible for substitution). Returns TRUE if
720 : : substitution was actually performed, FALSE otherwise. Substitution might
721 : : be not performed because it's either EXPR' vinsn doesn't contain INSN's
722 : : destination or the resulting insn is invalid for the target machine.
723 : : When UNDO is true, perform unsubstitution instead (the difference is in
724 : : the part of rtx on which validate_replace_rtx is called). */
725 : : static bool
726 : 125 : substitute_reg_in_expr (expr_t expr, insn_t insn, bool undo)
727 : : {
728 : 125 : rtx *where;
729 : 125 : bool new_insn_valid;
730 : 125 : vinsn_t *vi = &EXPR_VINSN (expr);
731 : 125 : bool has_rhs = VINSN_RHS (*vi) != NULL;
732 : 125 : rtx old, new_rtx;
733 : :
734 : : /* Do not try to replace in SET_DEST. Although we'll choose new
735 : : register for the RHS, we don't want to change RHS' original reg.
736 : : If the insn is not SET, we may still be able to substitute something
737 : : in it, and if we're here (don't have deps), it doesn't write INSN's
738 : : dest. */
739 : 250 : where = (has_rhs
740 : 125 : ? &VINSN_RHS (*vi)
741 : 0 : : &PATTERN (VINSN_INSN_RTX (*vi)));
742 : 125 : old = undo ? INSN_RHS (insn) : INSN_LHS (insn);
743 : :
744 : : /* Substitute if INSN has a form of x:=y and LHS(INSN) occurs in *VI. */
745 : 125 : if (rtx_ok_for_substitution_p (old, *where))
746 : : {
747 : 103 : rtx_insn *new_insn;
748 : 103 : rtx *where_replace;
749 : :
750 : : /* We should copy these rtxes before substitution. */
751 : 103 : new_rtx = copy_rtx (undo ? INSN_LHS (insn) : INSN_RHS (insn));
752 : 103 : new_insn = create_copy_of_insn_rtx (VINSN_INSN_RTX (*vi));
753 : :
754 : : /* Where we'll replace.
755 : : WHERE_REPLACE should point inside NEW_INSN, so INSN_RHS couldn't be
756 : : used instead of SET_SRC. */
757 : 206 : where_replace = (has_rhs
758 : 103 : ? &SET_SRC (PATTERN (new_insn))
759 : 0 : : &PATTERN (new_insn));
760 : :
761 : 103 : new_insn_valid
762 : 103 : = validate_replace_rtx_part_nosimplify (old, new_rtx, where_replace,
763 : : new_insn);
764 : :
765 : : /* ??? Actually, constrain_operands result depends upon choice of
766 : : destination register. E.g. if we allow single register to be an rhs,
767 : : and if we try to move dx=ax(as rhs) through ax=dx, we'll result
768 : : in invalid insn dx=dx, so we'll loose this rhs here.
769 : : Just can't come up with significant testcase for this, so just
770 : : leaving it for now. */
771 : 103 : if (new_insn_valid)
772 : : {
773 : 79 : change_vinsn_in_expr (expr,
774 : : create_vinsn_from_insn_rtx (new_insn, false));
775 : :
776 : : /* Do not allow clobbering the address register of speculative
777 : : insns. */
778 : 79 : if ((EXPR_SPEC_DONE_DS (expr) & SPECULATIVE)
779 : 79 : && register_unavailable_p (VINSN_REG_USES (EXPR_VINSN (expr)),
780 : : expr_dest_reg (expr)))
781 : 0 : EXPR_TARGET_AVAILABLE (expr) = false;
782 : :
783 : 79 : return true;
784 : : }
785 : : else
786 : : return false;
787 : : }
788 : : else
789 : : return false;
790 : : }
791 : :
792 : : /* Return the number of places WHAT appears within WHERE.
793 : : Bail out when we found a reference occupying several hard registers. */
794 : : static int
795 : 125 : count_occurrences_equiv (const_rtx what, const_rtx where)
796 : : {
797 : 125 : int count = 0;
798 : 125 : subrtx_iterator::array_type array;
799 : 462 : FOR_EACH_SUBRTX (iter, array, where, NONCONST)
800 : : {
801 : 359 : const_rtx x = *iter;
802 : 359 : if (REG_P (x) && REGNO (x) == REGNO (what))
803 : : {
804 : : /* Bail out if mode is different or more than one register is
805 : : used. */
806 : 123 : if (GET_MODE (x) != GET_MODE (what) || REG_NREGS (x) > 1)
807 : 22 : return 0;
808 : 103 : count += 1;
809 : : }
810 : 236 : else if (GET_CODE (x) == SUBREG
811 : 236 : && (!REG_P (SUBREG_REG (x))
812 : 3 : || REGNO (SUBREG_REG (x)) == REGNO (what)))
813 : : /* ??? Do not support substituting regs inside subregs. In that case,
814 : : simplify_subreg will be called by validate_replace_rtx, and
815 : : unsubstitution will fail later. */
816 : : return 0;
817 : : }
818 : 103 : return count;
819 : 125 : }
820 : :
821 : : /* Returns TRUE if WHAT is found in WHERE rtx tree. */
822 : : static bool
823 : 125 : rtx_ok_for_substitution_p (rtx what, rtx where)
824 : : {
825 : 125 : return (count_occurrences_equiv (what, where) > 0);
826 : : }
827 : :
828 : :
829 : : /* Functions to support register renaming. */
830 : :
831 : : /* Substitute VI's set source with REGNO. Returns newly created pattern
832 : : that has REGNO as its source. */
833 : : static rtx_insn *
834 : 31 : create_insn_rtx_with_rhs (vinsn_t vi, rtx rhs_rtx)
835 : : {
836 : 31 : rtx lhs_rtx;
837 : 31 : rtx pattern;
838 : 31 : rtx_insn *insn_rtx;
839 : :
840 : 31 : lhs_rtx = copy_rtx (VINSN_LHS (vi));
841 : :
842 : 31 : pattern = gen_rtx_SET (lhs_rtx, rhs_rtx);
843 : 31 : insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
844 : :
845 : 31 : return insn_rtx;
846 : : }
847 : :
848 : : /* Returns whether INSN's src can be replaced with register number
849 : : NEW_SRC_REG. E.g. the following insn is valid for i386:
850 : :
851 : : (insn:HI 2205 6585 2207 727 ../../gcc/libiberty/regex.c:3337
852 : : (set (mem/s:QI (plus:SI (plus:SI (reg/f:SI 7 sp)
853 : : (reg:SI 0 ax [orig:770 c1 ] [770]))
854 : : (const_int 288 [0x120])) [0 str S1 A8])
855 : : (const_int 0 [0x0])) 43 {*movqi_1} (nil)
856 : : (nil))
857 : :
858 : : But if we change (const_int 0 [0x0]) to (reg:QI 4 si), it will be invalid
859 : : because of operand constraints:
860 : :
861 : : (define_insn "*movqi_1"
862 : : [(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m")
863 : : (match_operand:QI 1 "general_operand" " q,qn,qm,q,rn,qm,qn")
864 : : )]
865 : :
866 : : So do constrain_operands here, before choosing NEW_SRC_REG as best
867 : : reg for rhs. */
868 : :
869 : : static bool
870 : 162 : replace_src_with_reg_ok_p (insn_t insn, rtx new_src_reg)
871 : : {
872 : 162 : vinsn_t vi = INSN_VINSN (insn);
873 : 162 : machine_mode mode;
874 : 162 : rtx dst_loc;
875 : 162 : bool res;
876 : :
877 : 162 : gcc_assert (VINSN_SEPARABLE_P (vi));
878 : :
879 : 162 : get_dest_and_mode (insn, &dst_loc, &mode);
880 : 162 : gcc_assert (mode == GET_MODE (new_src_reg));
881 : :
882 : 162 : if (REG_P (dst_loc) && REGNO (new_src_reg) == REGNO (dst_loc))
883 : : return true;
884 : :
885 : : /* See whether SET_SRC can be replaced with this register. */
886 : 162 : validate_change (insn, &SET_SRC (PATTERN (insn)), new_src_reg, 1);
887 : 162 : res = verify_changes (0);
888 : 162 : cancel_changes (0);
889 : :
890 : 162 : return res;
891 : : }
892 : :
893 : : /* Returns whether INSN still be valid after replacing it's DEST with
894 : : register NEW_REG. */
895 : : static bool
896 : 162 : replace_dest_with_reg_ok_p (insn_t insn, rtx new_reg)
897 : : {
898 : 162 : vinsn_t vi = INSN_VINSN (insn);
899 : 162 : bool res;
900 : :
901 : : /* We should deal here only with separable insns. */
902 : 162 : gcc_assert (VINSN_SEPARABLE_P (vi));
903 : 162 : gcc_assert (GET_MODE (VINSN_LHS (vi)) == GET_MODE (new_reg));
904 : :
905 : : /* See whether SET_DEST can be replaced with this register. */
906 : 162 : validate_change (insn, &SET_DEST (PATTERN (insn)), new_reg, 1);
907 : 162 : res = verify_changes (0);
908 : 162 : cancel_changes (0);
909 : :
910 : 162 : return res;
911 : : }
912 : :
913 : : /* Create a pattern with rhs of VI and lhs of LHS_RTX. */
914 : : static rtx_insn *
915 : 213 : create_insn_rtx_with_lhs (vinsn_t vi, rtx lhs_rtx)
916 : : {
917 : 213 : rtx rhs_rtx;
918 : 213 : rtx pattern;
919 : 213 : rtx_insn *insn_rtx;
920 : :
921 : 213 : rhs_rtx = copy_rtx (VINSN_RHS (vi));
922 : :
923 : 213 : pattern = gen_rtx_SET (lhs_rtx, rhs_rtx);
924 : 213 : insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
925 : :
926 : 213 : return insn_rtx;
927 : : }
928 : :
929 : : /* Substitute lhs in the given expression EXPR for the register with number
930 : : NEW_REGNO. SET_DEST may be arbitrary rtx, not only register. */
931 : : static void
932 : 213 : replace_dest_with_reg_in_expr (expr_t expr, rtx new_reg)
933 : : {
934 : 213 : rtx_insn *insn_rtx;
935 : 213 : vinsn_t vinsn;
936 : :
937 : 213 : insn_rtx = create_insn_rtx_with_lhs (EXPR_VINSN (expr), new_reg);
938 : 213 : vinsn = create_vinsn_from_insn_rtx (insn_rtx, false);
939 : :
940 : 213 : change_vinsn_in_expr (expr, vinsn);
941 : 213 : EXPR_WAS_RENAMED (expr) = 1;
942 : 213 : EXPR_TARGET_AVAILABLE (expr) = 1;
943 : 213 : }
944 : :
945 : : /* Returns whether VI writes either one of the USED_REGS registers or,
946 : : if a register is a hard one, one of the UNAVAILABLE_HARD_REGS registers. */
947 : : static bool
948 : 73 : vinsn_writes_one_of_regs_p (vinsn_t vi, regset used_regs,
949 : : HARD_REG_SET unavailable_hard_regs)
950 : : {
951 : 73 : unsigned regno;
952 : 73 : reg_set_iterator rsi;
953 : :
954 : 86 : EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_SETS (vi), 0, regno, rsi)
955 : : {
956 : 73 : if (REGNO_REG_SET_P (used_regs, regno))
957 : : return true;
958 : 13 : if (HARD_REGISTER_NUM_P (regno)
959 : 13 : && TEST_HARD_REG_BIT (unavailable_hard_regs, regno))
960 : : return true;
961 : : }
962 : :
963 : 13 : EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_CLOBBERS (vi), 0, regno, rsi)
964 : : {
965 : 0 : if (REGNO_REG_SET_P (used_regs, regno))
966 : : return true;
967 : 0 : if (HARD_REGISTER_NUM_P (regno)
968 : 0 : && TEST_HARD_REG_BIT (unavailable_hard_regs, regno))
969 : : return true;
970 : : }
971 : :
972 : : return false;
973 : : }
974 : :
975 : : /* Returns register class of the output register in INSN.
976 : : Returns NO_REGS for call insns because some targets have constraints on
977 : : destination register of a call insn.
978 : :
979 : : Code adopted from regrename.cc::build_def_use. */
980 : : static enum reg_class
981 : 2824 : get_reg_class (rtx_insn *insn)
982 : : {
983 : 2824 : int i, n_ops;
984 : :
985 : 2824 : extract_constrain_insn (insn);
986 : 2824 : preprocess_constraints (insn);
987 : 2824 : n_ops = recog_data.n_operands;
988 : :
989 : 2824 : const operand_alternative *op_alt = which_op_alt ();
990 : 2824 : if (asm_noperands (PATTERN (insn)) > 0)
991 : : {
992 : 0 : for (i = 0; i < n_ops; i++)
993 : 0 : if (recog_data.operand_type[i] == OP_OUT)
994 : : {
995 : 0 : rtx *loc = recog_data.operand_loc[i];
996 : 0 : rtx op = *loc;
997 : 0 : enum reg_class cl = alternative_class (op_alt, i);
998 : :
999 : 0 : if (REG_P (op)
1000 : 0 : && REGNO (op) == ORIGINAL_REGNO (op))
1001 : 0 : continue;
1002 : :
1003 : : return cl;
1004 : : }
1005 : : }
1006 : 2824 : else if (!CALL_P (insn))
1007 : : {
1008 : 3898 : for (i = 0; i < n_ops + recog_data.n_dups; i++)
1009 : : {
1010 : 3361 : int opn = i < n_ops ? i : recog_data.dup_num[i - n_ops];
1011 : 3361 : enum reg_class cl = alternative_class (op_alt, opn);
1012 : :
1013 : 3361 : if (recog_data.operand_type[opn] == OP_OUT ||
1014 : : recog_data.operand_type[opn] == OP_INOUT)
1015 : : return cl;
1016 : : }
1017 : : }
1018 : :
1019 : : /* Insns like
1020 : : (insn (set (reg:CCZ 17 flags) (compare:CCZ ...)))
1021 : : may result in returning NO_REGS, cause flags is written implicitly through
1022 : : CMP insn, which has no OP_OUT | OP_INOUT operands. */
1023 : : return NO_REGS;
1024 : : }
1025 : :
1026 : : /* Calculate HARD_REGNO_RENAME_OK data for REGNO. */
1027 : : static void
1028 : 69 : init_hard_regno_rename (int regno)
1029 : : {
1030 : 69 : int cur_reg;
1031 : :
1032 : 69 : SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], regno);
1033 : :
1034 : 6417 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1035 : : {
1036 : : /* We are not interested in renaming in other regs. */
1037 : 6348 : if (!TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg))
1038 : 356 : continue;
1039 : :
1040 : 5992 : if (HARD_REGNO_RENAME_OK (regno, cur_reg))
1041 : 4888 : SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], cur_reg);
1042 : : }
1043 : 69 : }
1044 : :
1045 : : /* A wrapper around HARD_REGNO_RENAME_OK that will look into the hard regs
1046 : : data first. */
1047 : : static inline bool
1048 : 22527 : sel_hard_regno_rename_ok (int from ATTRIBUTE_UNUSED, int to ATTRIBUTE_UNUSED)
1049 : : {
1050 : : /* Check whether this is all calculated. */
1051 : 22527 : if (TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], from))
1052 : 22458 : return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to);
1053 : :
1054 : 69 : init_hard_regno_rename (from);
1055 : :
1056 : 69 : return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to);
1057 : : }
1058 : :
1059 : : /* Calculate set of registers that are capable of holding MODE. */
1060 : : static void
1061 : 36 : init_regs_for_mode (machine_mode mode)
1062 : : {
1063 : 36 : int cur_reg;
1064 : :
1065 : 36 : CLEAR_HARD_REG_SET (sel_hrd.regs_for_mode[mode]);
1066 : :
1067 : 3348 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1068 : : {
1069 : 3312 : int nregs;
1070 : 3312 : int i;
1071 : :
1072 : : /* See whether it accepts all modes that occur in
1073 : : original insns. */
1074 : 3312 : if (!targetm.hard_regno_mode_ok (cur_reg, mode))
1075 : 1204 : continue;
1076 : :
1077 : 2108 : nregs = hard_regno_nregs (cur_reg, mode);
1078 : :
1079 : 3063 : for (i = nregs - 1; i >= 0; --i)
1080 : 2108 : if (fixed_regs[cur_reg + i]
1081 : 1388 : || global_regs[cur_reg + i]
1082 : : /* Can't use regs which aren't saved by
1083 : : the prologue. */
1084 : 1388 : || !TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg + i)
1085 : : /* Can't use regs with non-null REG_BASE_VALUE, because adjusting
1086 : : it affects aliasing globally and invalidates all AV sets. */
1087 : 3385 : || get_reg_base_value (cur_reg + i)
1088 : : #ifdef LEAF_REGISTERS
1089 : : /* We can't use a non-leaf register if we're in a
1090 : : leaf function. */
1091 : : || (crtl->is_leaf
1092 : : && !LEAF_REGISTERS[cur_reg + i])
1093 : : #endif
1094 : : )
1095 : : break;
1096 : :
1097 : 2108 : if (i >= 0)
1098 : 1153 : continue;
1099 : :
1100 : : /* If the CUR_REG passed all the checks above,
1101 : : then it's ok. */
1102 : 955 : SET_HARD_REG_BIT (sel_hrd.regs_for_mode[mode], cur_reg);
1103 : : }
1104 : :
1105 : 36 : sel_hrd.regs_for_mode_ok[mode] = true;
1106 : 36 : }
1107 : :
1108 : : /* Init all register sets gathered in HRD. */
1109 : : static void
1110 : 131 : init_hard_regs_data (void)
1111 : : {
1112 : 131 : int cur_reg = 0;
1113 : 131 : int cur_mode = 0;
1114 : :
1115 : 131 : CLEAR_HARD_REG_SET (sel_hrd.regs_ever_used);
1116 : 12183 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1117 : 12052 : if (df_regs_ever_live_p (cur_reg)
1118 : 12052 : || crtl->abi->clobbers_full_reg_p (cur_reg))
1119 : 11047 : SET_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg);
1120 : :
1121 : : /* Initialize registers that are valid based on mode when this is
1122 : : really needed. */
1123 : 17161 : for (cur_mode = 0; cur_mode < NUM_MACHINE_MODES; cur_mode++)
1124 : 17030 : sel_hrd.regs_for_mode_ok[cur_mode] = false;
1125 : :
1126 : : /* Mark that all HARD_REGNO_RENAME_OK is not calculated. */
1127 : 12183 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1128 : 12052 : CLEAR_HARD_REG_SET (sel_hrd.regs_for_rename[cur_reg]);
1129 : :
1130 : : #ifdef STACK_REGS
1131 : 1179 : CLEAR_HARD_REG_SET (sel_hrd.stack_regs);
1132 : :
1133 : 1179 : for (cur_reg = FIRST_STACK_REG; cur_reg <= LAST_STACK_REG; cur_reg++)
1134 : 1048 : SET_HARD_REG_BIT (sel_hrd.stack_regs, cur_reg);
1135 : : #endif
1136 : 131 : }
1137 : :
1138 : : /* Mark hardware regs in REG_RENAME_P that are not suitable
1139 : : for renaming rhs in INSN due to hardware restrictions (register class,
1140 : : modes compatibility etc). This doesn't affect original insn's dest reg,
1141 : : if it isn't in USED_REGS. DEF is a definition insn of rhs for which the
1142 : : destination register is sought. LHS (DEF->ORIG_INSN) may be REG or MEM.
1143 : : Registers that are in used_regs are always marked in
1144 : : unavailable_hard_regs as well. */
1145 : :
1146 : : static void
1147 : 3361 : mark_unavailable_hard_regs (def_t def, struct reg_rename *reg_rename_p,
1148 : : regset used_regs ATTRIBUTE_UNUSED)
1149 : : {
1150 : 3361 : machine_mode mode;
1151 : 3361 : enum reg_class cl = NO_REGS;
1152 : 3361 : rtx orig_dest;
1153 : 3361 : unsigned cur_reg, regno;
1154 : 3361 : hard_reg_set_iterator hrsi;
1155 : :
1156 : 3361 : gcc_assert (GET_CODE (PATTERN (def->orig_insn)) == SET);
1157 : 3361 : gcc_assert (reg_rename_p);
1158 : :
1159 : 3361 : orig_dest = SET_DEST (PATTERN (def->orig_insn));
1160 : :
1161 : : /* We have decided not to rename 'mem = something;' insns, as 'something'
1162 : : is usually a register. */
1163 : 3361 : if (!REG_P (orig_dest))
1164 : 1074 : return;
1165 : :
1166 : 3361 : regno = REGNO (orig_dest);
1167 : :
1168 : : /* If before reload, don't try to work with pseudos. */
1169 : 3361 : if (!reload_completed && !HARD_REGISTER_NUM_P (regno))
1170 : : return;
1171 : :
1172 : 2824 : if (reload_completed)
1173 : 2824 : cl = get_reg_class (def->orig_insn);
1174 : :
1175 : : /* Stop if the original register is one of the fixed_regs, global_regs or
1176 : : frame pointer, or we could not discover its class. */
1177 : 2824 : if (fixed_regs[regno]
1178 : 2287 : || global_regs[regno]
1179 : 2287 : || (!HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed
1180 : 21 : && regno == HARD_FRAME_POINTER_REGNUM)
1181 : : || (HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed
1182 : : && regno == FRAME_POINTER_REGNUM)
1183 : 2287 : || (reload_completed && cl == NO_REGS))
1184 : : {
1185 : 537 : SET_HARD_REG_SET (reg_rename_p->unavailable_hard_regs);
1186 : :
1187 : : /* Give a chance for original register, if it isn't in used_regs. */
1188 : 537 : if (!def->crossed_call_abis)
1189 : 537 : CLEAR_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno);
1190 : :
1191 : 537 : return;
1192 : : }
1193 : :
1194 : : /* If something allocated on stack in this function, mark frame pointer
1195 : : register unavailable, considering also modes.
1196 : : FIXME: it is enough to do this once per all original defs. */
1197 : 2287 : if (frame_pointer_needed)
1198 : : {
1199 : 21 : add_to_hard_reg_set (®_rename_p->unavailable_hard_regs,
1200 : 21 : Pmode, FRAME_POINTER_REGNUM);
1201 : :
1202 : 21 : if (!HARD_FRAME_POINTER_IS_FRAME_POINTER)
1203 : 21 : add_to_hard_reg_set (®_rename_p->unavailable_hard_regs,
1204 : 21 : Pmode, HARD_FRAME_POINTER_REGNUM);
1205 : : }
1206 : :
1207 : : #ifdef STACK_REGS
1208 : : /* For the stack registers the presence of FIRST_STACK_REG in USED_REGS
1209 : : is equivalent to as if all stack regs were in this set.
1210 : : I.e. no stack register can be renamed, and even if it's an original
1211 : : register here we make sure it won't be lifted over it's previous def
1212 : : (it's previous def will appear as if it's a FIRST_STACK_REG def.
1213 : : The HARD_REGNO_RENAME_OK covers other cases in condition below. */
1214 : 2287 : if (IN_RANGE (REGNO (orig_dest), FIRST_STACK_REG, LAST_STACK_REG)
1215 : 2287 : && REGNO_REG_SET_P (used_regs, FIRST_STACK_REG))
1216 : 2287 : reg_rename_p->unavailable_hard_regs |= sel_hrd.stack_regs;
1217 : : #endif
1218 : :
1219 : 2287 : mode = GET_MODE (orig_dest);
1220 : :
1221 : : /* If there's a call on this path, make regs from full_reg_clobbers
1222 : : unavailable.
1223 : :
1224 : : ??? It would be better to track the set of clobbered registers
1225 : : directly, but that would be quite expensive in a def_t. */
1226 : 2287 : if (def->crossed_call_abis)
1227 : 676 : reg_rename_p->unavailable_hard_regs
1228 : 1352 : |= call_clobbers_in_region (def->crossed_call_abis,
1229 : 1352 : reg_class_contents[ALL_REGS], mode);
1230 : :
1231 : : /* Stop here before reload: we need FRAME_REGS, STACK_REGS, and
1232 : : crossed_call_abis, but not register classes. */
1233 : 2287 : if (!reload_completed)
1234 : : return;
1235 : :
1236 : : /* Leave regs as 'available' only from the current
1237 : : register class. */
1238 : 2287 : reg_rename_p->available_for_renaming = reg_class_contents[cl];
1239 : :
1240 : : /* Leave only registers available for this mode. */
1241 : 2287 : if (!sel_hrd.regs_for_mode_ok[mode])
1242 : 36 : init_regs_for_mode (mode);
1243 : 2287 : reg_rename_p->available_for_renaming &= sel_hrd.regs_for_mode[mode];
1244 : :
1245 : : /* Leave only those that are ok to rename. */
1246 : 24814 : EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1247 : : 0, cur_reg, hrsi)
1248 : : {
1249 : 22527 : int nregs;
1250 : 22527 : int i;
1251 : :
1252 : 22527 : nregs = hard_regno_nregs (cur_reg, mode);
1253 : 22527 : gcc_assert (nregs > 0);
1254 : :
1255 : 45054 : for (i = nregs - 1; i >= 0; --i)
1256 : 22527 : if (! sel_hard_regno_rename_ok (regno + i, cur_reg + i))
1257 : : break;
1258 : :
1259 : 22527 : if (i >= 0)
1260 : 0 : CLEAR_HARD_REG_BIT (reg_rename_p->available_for_renaming,
1261 : : cur_reg);
1262 : : }
1263 : :
1264 : 2287 : reg_rename_p->available_for_renaming &= ~reg_rename_p->unavailable_hard_regs;
1265 : :
1266 : : /* Regno is always ok from the renaming part of view, but it really
1267 : : could be in *unavailable_hard_regs already, so set it here instead
1268 : : of there. */
1269 : 2287 : SET_HARD_REG_BIT (reg_rename_p->available_for_renaming, regno);
1270 : : }
1271 : :
1272 : : /* reg_rename_tick[REG1] > reg_rename_tick[REG2] if REG1 was chosen as the
1273 : : best register more recently than REG2. */
1274 : : static int reg_rename_tick[FIRST_PSEUDO_REGISTER];
1275 : :
1276 : : /* Indicates the number of times renaming happened before the current one. */
1277 : : static int reg_rename_this_tick;
1278 : :
1279 : : /* Choose the register among free, that is suitable for storing
1280 : : the rhs value.
1281 : :
1282 : : ORIGINAL_INSNS is the list of insns where the operation (rhs)
1283 : : originally appears. There could be multiple original operations
1284 : : for single rhs since we moving it up and merging along different
1285 : : paths.
1286 : :
1287 : : Some code is adapted from regrename.cc (regrename_optimize).
1288 : : If original register is available, function returns it.
1289 : : Otherwise it performs the checks, so the new register should
1290 : : comply with the following:
1291 : : - it should not violate any live ranges (such registers are in
1292 : : REG_RENAME_P->available_for_renaming set);
1293 : : - it should not be in the HARD_REGS_USED regset;
1294 : : - it should be in the class compatible with original uses;
1295 : : - it should not be clobbered through reference with different mode;
1296 : : - if we're in the leaf function, then the new register should
1297 : : not be in the LEAF_REGISTERS;
1298 : : - etc.
1299 : :
1300 : : If several registers meet the conditions, the register with smallest
1301 : : tick is returned to achieve more even register allocation.
1302 : :
1303 : : If original register seems to be ok, we set *IS_ORIG_REG_P_PTR to true.
1304 : :
1305 : : If no register satisfies the above conditions, NULL_RTX is returned. */
1306 : : static rtx
1307 : 2508 : choose_best_reg_1 (HARD_REG_SET hard_regs_used,
1308 : : struct reg_rename *reg_rename_p,
1309 : : def_list_t original_insns, bool *is_orig_reg_p_ptr)
1310 : : {
1311 : 2508 : int best_new_reg;
1312 : 2508 : unsigned cur_reg;
1313 : 2508 : machine_mode mode = VOIDmode;
1314 : 2508 : unsigned regno, i, n;
1315 : 2508 : hard_reg_set_iterator hrsi;
1316 : 2508 : def_list_iterator di;
1317 : 2508 : def_t def;
1318 : :
1319 : : /* If original register is available, return it. */
1320 : 2508 : *is_orig_reg_p_ptr = true;
1321 : :
1322 : 5314 : FOR_EACH_DEF (def, di, original_insns)
1323 : : {
1324 : 2812 : rtx orig_dest = SET_DEST (PATTERN (def->orig_insn));
1325 : :
1326 : 2812 : gcc_assert (REG_P (orig_dest));
1327 : :
1328 : : /* Check that all original operations have the same mode.
1329 : : This is done for the next loop; if we'd return from this
1330 : : loop, we'd check only part of them, but in this case
1331 : : it doesn't matter. */
1332 : 2812 : if (mode == VOIDmode)
1333 : 2508 : mode = GET_MODE (orig_dest);
1334 : 2812 : gcc_assert (mode == GET_MODE (orig_dest));
1335 : :
1336 : 2812 : regno = REGNO (orig_dest);
1337 : 2818 : for (i = 0, n = REG_NREGS (orig_dest); i < n; i++)
1338 : 2812 : if (TEST_HARD_REG_BIT (hard_regs_used, regno + i))
1339 : : break;
1340 : :
1341 : : /* All hard registers are available. */
1342 : 2812 : if (i == n)
1343 : : {
1344 : 6 : gcc_assert (mode != VOIDmode);
1345 : :
1346 : : /* Hard registers should not be shared. */
1347 : 6 : return gen_rtx_REG (mode, regno);
1348 : : }
1349 : : }
1350 : :
1351 : 2502 : *is_orig_reg_p_ptr = false;
1352 : 2502 : best_new_reg = -1;
1353 : :
1354 : : /* Among all available regs choose the register that was
1355 : : allocated earliest. */
1356 : 21134 : EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1357 : : 0, cur_reg, hrsi)
1358 : 19430 : if (! TEST_HARD_REG_BIT (hard_regs_used, cur_reg))
1359 : : {
1360 : : /* Check that all hard regs for mode are available. */
1361 : 814 : for (i = 1, n = hard_regno_nregs (cur_reg, mode); i < n; i++)
1362 : 0 : if (TEST_HARD_REG_BIT (hard_regs_used, cur_reg + i)
1363 : 0 : || !TEST_HARD_REG_BIT (reg_rename_p->available_for_renaming,
1364 : : cur_reg + i))
1365 : : break;
1366 : :
1367 : 814 : if (i < n)
1368 : 0 : continue;
1369 : :
1370 : : /* All hard registers are available. */
1371 : 814 : if (best_new_reg < 0
1372 : 1 : || reg_rename_tick[cur_reg] < reg_rename_tick[best_new_reg])
1373 : : {
1374 : 814 : best_new_reg = cur_reg;
1375 : :
1376 : : /* Return immediately when we know there's no better reg. */
1377 : 814 : if (! reg_rename_tick[best_new_reg])
1378 : : break;
1379 : : }
1380 : : }
1381 : :
1382 : 2502 : if (best_new_reg >= 0)
1383 : : {
1384 : : /* Use the check from the above loop. */
1385 : 813 : gcc_assert (mode != VOIDmode);
1386 : 813 : return gen_rtx_REG (mode, best_new_reg);
1387 : : }
1388 : :
1389 : : return NULL_RTX;
1390 : : }
1391 : :
1392 : : /* A wrapper around choose_best_reg_1 () to verify that we make correct
1393 : : assumptions about available registers in the function. */
1394 : : static rtx
1395 : 2508 : choose_best_reg (HARD_REG_SET hard_regs_used, struct reg_rename *reg_rename_p,
1396 : : def_list_t original_insns, bool *is_orig_reg_p_ptr)
1397 : : {
1398 : 2508 : rtx best_reg = choose_best_reg_1 (hard_regs_used, reg_rename_p,
1399 : : original_insns, is_orig_reg_p_ptr);
1400 : :
1401 : : /* FIXME loop over hard_regno_nregs here. */
1402 : 2508 : gcc_assert (best_reg == NULL_RTX
1403 : : || TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, REGNO (best_reg)));
1404 : :
1405 : 2508 : return best_reg;
1406 : : }
1407 : :
1408 : : /* Choose the pseudo register for storing rhs value. As this is supposed
1409 : : to work before reload, we return either the original register or make
1410 : : the new one. The parameters are the same that in choose_nest_reg_1
1411 : : functions, except that USED_REGS may contain pseudos.
1412 : : If we work with hard regs, check also REG_RENAME_P->UNAVAILABLE_HARD_REGS.
1413 : :
1414 : : TODO: take into account register pressure while doing this. Up to this
1415 : : moment, this function would never return NULL for pseudos, but we should
1416 : : not rely on this. */
1417 : : static rtx
1418 : 240 : choose_best_pseudo_reg (regset used_regs,
1419 : : struct reg_rename *reg_rename_p,
1420 : : def_list_t original_insns, bool *is_orig_reg_p_ptr)
1421 : : {
1422 : 240 : def_list_iterator i;
1423 : 240 : def_t def;
1424 : 240 : machine_mode mode = VOIDmode;
1425 : 240 : bool bad_hard_regs = false;
1426 : :
1427 : : /* We should not use this after reload. */
1428 : 240 : gcc_assert (!reload_completed);
1429 : :
1430 : : /* If original register is available, return it. */
1431 : 240 : *is_orig_reg_p_ptr = true;
1432 : :
1433 : 777 : FOR_EACH_DEF (def, i, original_insns)
1434 : : {
1435 : 537 : rtx dest = SET_DEST (PATTERN (def->orig_insn));
1436 : 537 : int orig_regno;
1437 : :
1438 : 537 : gcc_assert (REG_P (dest));
1439 : :
1440 : : /* Check that all original operations have the same mode. */
1441 : 537 : if (mode == VOIDmode)
1442 : 240 : mode = GET_MODE (dest);
1443 : : else
1444 : 297 : gcc_assert (mode == GET_MODE (dest));
1445 : 537 : orig_regno = REGNO (dest);
1446 : :
1447 : : /* Check that nothing in used_regs intersects with orig_regno. When
1448 : : we have a hard reg here, still loop over hard_regno_nregs. */
1449 : 537 : if (HARD_REGISTER_NUM_P (orig_regno))
1450 : : {
1451 : 0 : int j, n;
1452 : 0 : for (j = 0, n = REG_NREGS (dest); j < n; j++)
1453 : 0 : if (REGNO_REG_SET_P (used_regs, orig_regno + j))
1454 : : break;
1455 : 0 : if (j < n)
1456 : 0 : continue;
1457 : : }
1458 : : else
1459 : : {
1460 : 537 : if (REGNO_REG_SET_P (used_regs, orig_regno))
1461 : 537 : continue;
1462 : : }
1463 : 0 : if (HARD_REGISTER_NUM_P (orig_regno))
1464 : : {
1465 : 0 : gcc_assert (df_regs_ever_live_p (orig_regno));
1466 : :
1467 : : /* For hard registers, we have to check hardware imposed
1468 : : limitations (frame/stack registers, calls crossed). */
1469 : 0 : if (!TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs,
1470 : : orig_regno))
1471 : : {
1472 : : /* Don't let register cross a call if it doesn't already
1473 : : cross one. This condition is written in accordance with
1474 : : that in sched-deps.cc sched_analyze_reg(). */
1475 : 0 : if (!reg_rename_p->crossed_call_abis
1476 : 0 : || REG_N_CALLS_CROSSED (orig_regno) > 0)
1477 : 0 : return gen_rtx_REG (mode, orig_regno);
1478 : : }
1479 : :
1480 : : bad_hard_regs = true;
1481 : : }
1482 : : else
1483 : : return dest;
1484 : : }
1485 : :
1486 : 240 : *is_orig_reg_p_ptr = false;
1487 : :
1488 : : /* We had some original hard registers that couldn't be used.
1489 : : Those were likely special. Don't try to create a pseudo. */
1490 : 240 : if (bad_hard_regs)
1491 : : return NULL_RTX;
1492 : :
1493 : : /* We haven't found a register from original operations. Get a new one.
1494 : : FIXME: control register pressure somehow. */
1495 : 240 : {
1496 : 240 : rtx new_reg = gen_reg_rtx (mode);
1497 : :
1498 : 240 : gcc_assert (mode != VOIDmode);
1499 : :
1500 : 240 : max_regno = max_reg_num ();
1501 : 240 : maybe_extend_reg_info_p ();
1502 : 240 : REG_N_CALLS_CROSSED (REGNO (new_reg))
1503 : 240 : = reg_rename_p->crossed_call_abis ? 1 : 0;
1504 : :
1505 : 240 : return new_reg;
1506 : : }
1507 : : }
1508 : :
1509 : : /* True when target of EXPR is available due to EXPR_TARGET_AVAILABLE,
1510 : : USED_REGS and REG_RENAME_P->UNAVAILABLE_HARD_REGS. */
1511 : : static void
1512 : 2748 : verify_target_availability (expr_t expr, regset used_regs,
1513 : : struct reg_rename *reg_rename_p)
1514 : : {
1515 : 2748 : unsigned n, i, regno;
1516 : 2748 : machine_mode mode;
1517 : 2748 : bool target_available, live_available, hard_available;
1518 : :
1519 : 2748 : if (!REG_P (EXPR_LHS (expr)) || EXPR_TARGET_AVAILABLE (expr) < 0)
1520 : : return;
1521 : :
1522 : 2673 : regno = expr_dest_regno (expr);
1523 : 2673 : mode = GET_MODE (EXPR_LHS (expr));
1524 : 2673 : target_available = EXPR_TARGET_AVAILABLE (expr) == 1;
1525 : 2673 : n = HARD_REGISTER_NUM_P (regno) ? hard_regno_nregs (regno, mode) : 1;
1526 : :
1527 : 2673 : live_available = hard_available = true;
1528 : 5346 : for (i = 0; i < n; i++)
1529 : : {
1530 : 2673 : if (bitmap_bit_p (used_regs, regno + i))
1531 : 2587 : live_available = false;
1532 : 2673 : if (TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno + i))
1533 : 349 : hard_available = false;
1534 : : }
1535 : :
1536 : : /* When target is not available, it may be due to hard register
1537 : : restrictions, e.g. crosses calls, so we check hard_available too. */
1538 : 2673 : if (target_available)
1539 : 11 : gcc_assert (live_available);
1540 : : else
1541 : : /* Check only if we haven't scheduled something on the previous fence,
1542 : : cause due to MAX_SOFTWARE_LOOKAHEAD_WINDOW_SIZE issues
1543 : : and having more than one fence, we may end having targ_un in a block
1544 : : in which successors target register is actually available.
1545 : :
1546 : : The last condition handles the case when a dependence from a call insn
1547 : : was created in sched-deps.cc for insns with destination registers that
1548 : : never crossed a call before, but do cross one after our code motion.
1549 : :
1550 : : FIXME: in the latter case, we just uselessly called find_used_regs,
1551 : : because we can't move this expression with any other register
1552 : : as well. */
1553 : 2662 : gcc_assert (scheduled_something_on_previous_fence || !live_available
1554 : : || !hard_available
1555 : : || (!reload_completed
1556 : : && reg_rename_p->crossed_call_abis
1557 : : && REG_N_CALLS_CROSSED (regno) == 0));
1558 : : }
1559 : :
1560 : : /* Collect unavailable registers due to liveness for EXPR from BNDS
1561 : : into USED_REGS. Save additional information about available
1562 : : registers and unavailable due to hardware restriction registers
1563 : : into REG_RENAME_P structure. Save original insns into ORIGINAL_INSNS
1564 : : list. */
1565 : : static void
1566 : 2821 : collect_unavailable_regs_from_bnds (expr_t expr, blist_t bnds, regset used_regs,
1567 : : struct reg_rename *reg_rename_p,
1568 : : def_list_t *original_insns)
1569 : : {
1570 : 5642 : for (; bnds; bnds = BLIST_NEXT (bnds))
1571 : : {
1572 : 2821 : bool res;
1573 : 2821 : av_set_t orig_ops = NULL;
1574 : 2821 : bnd_t bnd = BLIST_BND (bnds);
1575 : :
1576 : : /* If the chosen best expr doesn't belong to current boundary,
1577 : : skip it. */
1578 : 2821 : if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr)))
1579 : 0 : continue;
1580 : :
1581 : : /* Put in ORIG_OPS all exprs from this boundary that became
1582 : : RES on top. */
1583 : 2821 : orig_ops = find_sequential_best_exprs (bnd, expr, false);
1584 : :
1585 : : /* Compute used regs and OR it into the USED_REGS. */
1586 : 2821 : res = find_used_regs (BND_TO (bnd), orig_ops, used_regs,
1587 : : reg_rename_p, original_insns);
1588 : :
1589 : : /* FIXME: the assert is true until we'd have several boundaries. */
1590 : 2821 : gcc_assert (res);
1591 : 2821 : av_set_clear (&orig_ops);
1592 : : }
1593 : 2821 : }
1594 : :
1595 : : /* Return TRUE if it is possible to replace LHSes of ORIG_INSNS with BEST_REG.
1596 : : If BEST_REG is valid, replace LHS of EXPR with it. */
1597 : : static bool
1598 : 160 : try_replace_dest_reg (ilist_t orig_insns, rtx best_reg, expr_t expr)
1599 : : {
1600 : : /* Try whether we'll be able to generate the insn
1601 : : 'dest := best_reg' at the place of the original operation. */
1602 : 334 : for (; orig_insns; orig_insns = ILIST_NEXT (orig_insns))
1603 : : {
1604 : 174 : insn_t orig_insn = DEF_LIST_DEF (orig_insns)->orig_insn;
1605 : :
1606 : 174 : gcc_assert (EXPR_SEPARABLE_P (INSN_EXPR (orig_insn)));
1607 : :
1608 : 174 : if (REGNO (best_reg) != REGNO (INSN_LHS (orig_insn))
1609 : 174 : && (! replace_src_with_reg_ok_p (orig_insn, best_reg)
1610 : 162 : || ! replace_dest_with_reg_ok_p (orig_insn, best_reg)))
1611 : 0 : return false;
1612 : : }
1613 : :
1614 : : /* Make sure that EXPR has the right destination
1615 : : register. */
1616 : 160 : if (expr_dest_regno (expr) != REGNO (best_reg))
1617 : : {
1618 : 154 : rtx_insn *vinsn = EXPR_INSN_RTX (expr);
1619 : 154 : validate_change (vinsn, &SET_DEST (PATTERN (vinsn)), best_reg, 1);
1620 : 154 : bool res = verify_changes (0);
1621 : 154 : cancel_changes (0);
1622 : 154 : if (!res)
1623 : : return false;
1624 : 154 : replace_dest_with_reg_in_expr (expr, best_reg);
1625 : : }
1626 : : else
1627 : 6 : EXPR_TARGET_AVAILABLE (expr) = 1;
1628 : :
1629 : : return true;
1630 : : }
1631 : :
1632 : : /* Select and assign best register to EXPR searching from BNDS.
1633 : : Set *IS_ORIG_REG_P to TRUE if original register was selected.
1634 : : Return FALSE if no register can be chosen, which could happen when:
1635 : : * EXPR_SEPARABLE_P is true but we were unable to find suitable register;
1636 : : * EXPR_SEPARABLE_P is false but the insn sets/clobbers one of the registers
1637 : : that are used on the moving path. */
1638 : : static bool
1639 : 2821 : find_best_reg_for_expr (expr_t expr, blist_t bnds, bool *is_orig_reg_p)
1640 : : {
1641 : 2821 : static struct reg_rename reg_rename_data;
1642 : :
1643 : 2821 : regset used_regs;
1644 : 2821 : def_list_t original_insns = NULL;
1645 : 2821 : bool reg_ok;
1646 : :
1647 : 2821 : *is_orig_reg_p = false;
1648 : :
1649 : : /* Don't bother to do anything if this insn doesn't set any registers. */
1650 : 2821 : if (bitmap_empty_p (VINSN_REG_SETS (EXPR_VINSN (expr)))
1651 : 2821 : && bitmap_empty_p (VINSN_REG_CLOBBERS (EXPR_VINSN (expr))))
1652 : : return true;
1653 : :
1654 : 2821 : used_regs = get_clear_regset_from_pool ();
1655 : 2821 : CLEAR_HARD_REG_SET (reg_rename_data.unavailable_hard_regs);
1656 : :
1657 : 2821 : collect_unavailable_regs_from_bnds (expr, bnds, used_regs, ®_rename_data,
1658 : : &original_insns);
1659 : :
1660 : : /* If after reload, make sure we're working with hard regs here. */
1661 : 2821 : if (flag_checking && reload_completed)
1662 : : {
1663 : 2521 : reg_set_iterator rsi;
1664 : 2521 : unsigned i;
1665 : :
1666 : 2521 : EXECUTE_IF_SET_IN_REG_SET (used_regs, FIRST_PSEUDO_REGISTER, i, rsi)
1667 : 0 : gcc_unreachable ();
1668 : : }
1669 : :
1670 : 2821 : if (EXPR_SEPARABLE_P (expr))
1671 : : {
1672 : 2748 : rtx best_reg = NULL_RTX;
1673 : : /* Check that we have computed availability of a target register
1674 : : correctly. */
1675 : 2748 : verify_target_availability (expr, used_regs, ®_rename_data);
1676 : :
1677 : : /* Turn everything in hard regs after reload. */
1678 : 2748 : if (reload_completed)
1679 : : {
1680 : : HARD_REG_SET hard_regs_used;
1681 : 2508 : REG_SET_TO_HARD_REG_SET (hard_regs_used, used_regs);
1682 : :
1683 : : /* Join hard registers unavailable due to register class
1684 : : restrictions and live range intersection. */
1685 : 2508 : hard_regs_used |= reg_rename_data.unavailable_hard_regs;
1686 : :
1687 : 2508 : best_reg = choose_best_reg (hard_regs_used, ®_rename_data,
1688 : : original_insns, is_orig_reg_p);
1689 : : }
1690 : : else
1691 : 240 : best_reg = choose_best_pseudo_reg (used_regs, ®_rename_data,
1692 : : original_insns, is_orig_reg_p);
1693 : :
1694 : 2748 : if (!best_reg)
1695 : : reg_ok = false;
1696 : 1059 : else if (*is_orig_reg_p)
1697 : : {
1698 : : /* In case of unification BEST_REG may be different from EXPR's LHS
1699 : : when EXPR's LHS is unavailable, and there is another LHS among
1700 : : ORIGINAL_INSNS. */
1701 : 6 : reg_ok = try_replace_dest_reg (original_insns, best_reg, expr);
1702 : : }
1703 : : else
1704 : : {
1705 : : /* Forbid renaming of low-cost insns. */
1706 : 1053 : if (sel_vinsn_cost (EXPR_VINSN (expr)) < 2)
1707 : : reg_ok = false;
1708 : : else
1709 : 154 : reg_ok = try_replace_dest_reg (original_insns, best_reg, expr);
1710 : : }
1711 : : }
1712 : : else
1713 : : {
1714 : : /* If !EXPR_SCHEDULE_AS_RHS (EXPR), just make sure INSN doesn't set
1715 : : any of the HARD_REGS_USED set. */
1716 : 73 : if (vinsn_writes_one_of_regs_p (EXPR_VINSN (expr), used_regs,
1717 : : reg_rename_data.unavailable_hard_regs))
1718 : : {
1719 : 60 : reg_ok = false;
1720 : 60 : gcc_assert (EXPR_TARGET_AVAILABLE (expr) <= 0);
1721 : : }
1722 : : else
1723 : : {
1724 : 13 : reg_ok = true;
1725 : 13 : gcc_assert (EXPR_TARGET_AVAILABLE (expr) != 0);
1726 : : }
1727 : : }
1728 : :
1729 : 2821 : ilist_clear (&original_insns);
1730 : 2821 : return_regset_to_pool (used_regs);
1731 : :
1732 : 2821 : return reg_ok;
1733 : : }
1734 : :
1735 : :
1736 : : /* Return true if dependence described by DS can be overcomed. */
1737 : : static bool
1738 : 4449 : can_speculate_dep_p (ds_t ds)
1739 : : {
1740 : 4449 : if (spec_info == NULL)
1741 : : return false;
1742 : :
1743 : : /* Leave only speculative data. */
1744 : 0 : ds &= SPECULATIVE;
1745 : :
1746 : 0 : if (ds == 0)
1747 : : return false;
1748 : :
1749 : 0 : {
1750 : : /* FIXME: make sched-deps.cc produce only those non-hard dependencies,
1751 : : that we can overcome. */
1752 : 0 : ds_t spec_mask = spec_info->mask;
1753 : :
1754 : 0 : if ((ds & spec_mask) != ds)
1755 : : return false;
1756 : : }
1757 : :
1758 : 0 : if (ds_weak (ds) < spec_info->data_weakness_cutoff)
1759 : : return false;
1760 : :
1761 : : return true;
1762 : : }
1763 : :
1764 : : /* Get a speculation check instruction.
1765 : : C_EXPR is a speculative expression,
1766 : : CHECK_DS describes speculations that should be checked,
1767 : : ORIG_INSN is the original non-speculative insn in the stream. */
1768 : : static insn_t
1769 : 0 : create_speculation_check (expr_t c_expr, ds_t check_ds, insn_t orig_insn)
1770 : : {
1771 : 0 : rtx check_pattern;
1772 : 0 : rtx_insn *insn_rtx;
1773 : 0 : insn_t insn;
1774 : 0 : basic_block recovery_block;
1775 : 0 : rtx_insn *label;
1776 : :
1777 : : /* Create a recovery block if target is going to emit branchy check, or if
1778 : : ORIG_INSN was speculative already. */
1779 : 0 : if (targetm.sched.needs_block_p (check_ds)
1780 : 0 : || EXPR_SPEC_DONE_DS (INSN_EXPR (orig_insn)) != 0)
1781 : : {
1782 : 0 : recovery_block = sel_create_recovery_block (orig_insn);
1783 : 0 : label = BB_HEAD (recovery_block);
1784 : : }
1785 : : else
1786 : : {
1787 : : recovery_block = NULL;
1788 : : label = NULL;
1789 : : }
1790 : :
1791 : : /* Get pattern of the check. */
1792 : 0 : check_pattern = targetm.sched.gen_spec_check (EXPR_INSN_RTX (c_expr), label,
1793 : : check_ds);
1794 : :
1795 : 0 : gcc_assert (check_pattern != NULL);
1796 : :
1797 : : /* Emit check. */
1798 : 0 : insn_rtx = create_insn_rtx_from_pattern (check_pattern, label);
1799 : :
1800 : 0 : insn = sel_gen_insn_from_rtx_after (insn_rtx, INSN_EXPR (orig_insn),
1801 : 0 : INSN_SEQNO (orig_insn), orig_insn);
1802 : :
1803 : : /* Make check to be non-speculative. */
1804 : 0 : EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
1805 : 0 : INSN_SPEC_CHECKED_DS (insn) = check_ds;
1806 : :
1807 : : /* Decrease priority of check by difference of load/check instruction
1808 : : latencies. */
1809 : 0 : EXPR_PRIORITY (INSN_EXPR (insn)) -= (sel_vinsn_cost (INSN_VINSN (orig_insn))
1810 : 0 : - sel_vinsn_cost (INSN_VINSN (insn)));
1811 : :
1812 : : /* Emit copy of original insn (though with replaced target register,
1813 : : if needed) to the recovery block. */
1814 : 0 : if (recovery_block != NULL)
1815 : : {
1816 : 0 : rtx twin_rtx;
1817 : :
1818 : 0 : twin_rtx = copy_rtx (PATTERN (EXPR_INSN_RTX (c_expr)));
1819 : 0 : twin_rtx = create_insn_rtx_from_pattern (twin_rtx, NULL_RTX);
1820 : 0 : sel_gen_recovery_insn_from_rtx_after (twin_rtx,
1821 : 0 : INSN_EXPR (orig_insn),
1822 : 0 : INSN_SEQNO (insn),
1823 : 0 : bb_note (recovery_block));
1824 : : }
1825 : :
1826 : : /* If we've generated a data speculation check, make sure
1827 : : that all the bookkeeping instruction we'll create during
1828 : : this move_op () will allocate an ALAT entry so that the
1829 : : check won't fail.
1830 : : In case of control speculation we must convert C_EXPR to control
1831 : : speculative mode, because failing to do so will bring us an exception
1832 : : thrown by the non-control-speculative load. */
1833 : 0 : check_ds = ds_get_max_dep_weak (check_ds);
1834 : 0 : speculate_expr (c_expr, check_ds);
1835 : :
1836 : 0 : return insn;
1837 : : }
1838 : :
1839 : : /* True when INSN is a "regN = regN" copy. */
1840 : : static bool
1841 : 28 : identical_copy_p (rtx_insn *insn)
1842 : : {
1843 : 28 : rtx lhs, rhs, pat;
1844 : :
1845 : 28 : pat = PATTERN (insn);
1846 : :
1847 : 28 : if (GET_CODE (pat) != SET)
1848 : : return false;
1849 : :
1850 : 25 : lhs = SET_DEST (pat);
1851 : 25 : if (!REG_P (lhs))
1852 : : return false;
1853 : :
1854 : 25 : rhs = SET_SRC (pat);
1855 : 25 : if (!REG_P (rhs))
1856 : : return false;
1857 : :
1858 : 25 : return REGNO (lhs) == REGNO (rhs);
1859 : : }
1860 : :
1861 : : /* Undo all transformations on *AV_PTR that were done when
1862 : : moving through INSN. */
1863 : : static void
1864 : 39885 : undo_transformations (av_set_t *av_ptr, rtx_insn *insn)
1865 : : {
1866 : 39885 : av_set_iterator av_iter;
1867 : 39885 : expr_t expr;
1868 : 39885 : av_set_t new_set = NULL;
1869 : :
1870 : : /* First, kill any EXPR that uses registers set by an insn. This is
1871 : : required for correctness. */
1872 : 120523 : FOR_EACH_EXPR_1 (expr, av_iter, av_ptr)
1873 : 40319 : if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (expr))
1874 : 40319 : && bitmap_intersect_p (INSN_REG_SETS (insn),
1875 : 40319 : VINSN_REG_USES (EXPR_VINSN (expr)))
1876 : : /* When an insn looks like 'r1 = r1', we could substitute through
1877 : : it, but the above condition will still hold. This happened with
1878 : : gcc.c-torture/execute/961125-1.c. */
1879 : 40347 : && !identical_copy_p (insn))
1880 : : {
1881 : 28 : if (sched_verbose >= 6)
1882 : 0 : sel_print ("Expr %d removed due to use/set conflict\n",
1883 : 0 : INSN_UID (EXPR_INSN_RTX (expr)));
1884 : 28 : av_set_iter_remove (&av_iter);
1885 : : }
1886 : :
1887 : : /* Undo transformations looking at the history vector. */
1888 : 120467 : FOR_EACH_EXPR (expr, av_iter, *av_ptr)
1889 : : {
1890 : 40291 : int index = find_in_history_vect (EXPR_HISTORY_OF_CHANGES (expr),
1891 : : insn, EXPR_VINSN (expr), true);
1892 : :
1893 : 40291 : if (index >= 0)
1894 : : {
1895 : 230 : expr_history_def *phist;
1896 : :
1897 : 230 : phist = &EXPR_HISTORY_OF_CHANGES (expr)[index];
1898 : :
1899 : 230 : switch (phist->type)
1900 : : {
1901 : 0 : case TRANS_SPECULATION:
1902 : 0 : {
1903 : 0 : ds_t old_ds, new_ds;
1904 : :
1905 : : /* Compute the difference between old and new speculative
1906 : : statuses: that's what we need to check.
1907 : : Earlier we used to assert that the status will really
1908 : : change. This no longer works because only the probability
1909 : : bits in the status may have changed during compute_av_set,
1910 : : and in the case of merging different probabilities of the
1911 : : same speculative status along different paths we do not
1912 : : record this in the history vector. */
1913 : 0 : old_ds = phist->spec_ds;
1914 : 0 : new_ds = EXPR_SPEC_DONE_DS (expr);
1915 : :
1916 : 0 : old_ds &= SPECULATIVE;
1917 : 0 : new_ds &= SPECULATIVE;
1918 : 0 : new_ds &= ~old_ds;
1919 : :
1920 : 0 : EXPR_SPEC_TO_CHECK_DS (expr) |= new_ds;
1921 : 0 : break;
1922 : : }
1923 : 230 : case TRANS_SUBSTITUTION:
1924 : 230 : {
1925 : 230 : expr_def _tmp_expr, *tmp_expr = &_tmp_expr;
1926 : 230 : vinsn_t new_vi;
1927 : 230 : bool add = true;
1928 : :
1929 : 230 : new_vi = phist->old_expr_vinsn;
1930 : :
1931 : 230 : gcc_assert (VINSN_SEPARABLE_P (new_vi)
1932 : : == EXPR_SEPARABLE_P (expr));
1933 : 230 : copy_expr (tmp_expr, expr);
1934 : :
1935 : 230 : if (vinsn_equal_p (phist->new_expr_vinsn,
1936 : : EXPR_VINSN (tmp_expr)))
1937 : 230 : change_vinsn_in_expr (tmp_expr, new_vi);
1938 : : else
1939 : : /* This happens when we're unsubstituting on a bookkeeping
1940 : : copy, which was in turn substituted. The history is wrong
1941 : : in this case. Do it the hard way. */
1942 : 0 : add = substitute_reg_in_expr (tmp_expr, insn, true);
1943 : 230 : if (add)
1944 : 230 : av_set_add (&new_set, tmp_expr);
1945 : 230 : clear_expr (tmp_expr);
1946 : 230 : break;
1947 : : }
1948 : 0 : default:
1949 : 0 : gcc_unreachable ();
1950 : : }
1951 : : }
1952 : :
1953 : : }
1954 : :
1955 : 39885 : av_set_union_and_clear (av_ptr, &new_set, NULL);
1956 : 39885 : }
1957 : :
1958 : :
1959 : : /* Moveup_* helpers for code motion and computing av sets. */
1960 : :
1961 : : /* Propagates EXPR inside an insn group through THROUGH_INSN.
1962 : : The difference from the below function is that only substitution is
1963 : : performed. */
1964 : : static enum MOVEUP_EXPR_CODE
1965 : 0 : moveup_expr_inside_insn_group (expr_t expr, insn_t through_insn)
1966 : : {
1967 : 0 : vinsn_t vi = EXPR_VINSN (expr);
1968 : 0 : ds_t *has_dep_p;
1969 : 0 : ds_t full_ds;
1970 : :
1971 : : /* Do this only inside insn group. */
1972 : 0 : gcc_assert (INSN_SCHED_CYCLE (through_insn) > 0);
1973 : :
1974 : 0 : full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
1975 : 0 : if (full_ds == 0)
1976 : : return MOVEUP_EXPR_SAME;
1977 : :
1978 : : /* Substitution is the possible choice in this case. */
1979 : 0 : if (has_dep_p[DEPS_IN_RHS])
1980 : : {
1981 : : /* Can't substitute UNIQUE VINSNs. */
1982 : 0 : gcc_assert (!VINSN_UNIQUE_P (vi));
1983 : :
1984 : 0 : if (can_substitute_through_p (through_insn,
1985 : : has_dep_p[DEPS_IN_RHS])
1986 : 0 : && substitute_reg_in_expr (expr, through_insn, false))
1987 : : {
1988 : 0 : EXPR_WAS_SUBSTITUTED (expr) = true;
1989 : 0 : return MOVEUP_EXPR_CHANGED;
1990 : : }
1991 : :
1992 : : /* Don't care about this, as even true dependencies may be allowed
1993 : : in an insn group. */
1994 : 0 : return MOVEUP_EXPR_SAME;
1995 : : }
1996 : :
1997 : : /* This can catch output dependencies in COND_EXECs. */
1998 : 0 : if (has_dep_p[DEPS_IN_INSN])
1999 : : return MOVEUP_EXPR_NULL;
2000 : :
2001 : : /* This is either an output or an anti dependence, which usually have
2002 : : a zero latency. Allow this here, if we'd be wrong, tick_check_p
2003 : : will fix this. */
2004 : 0 : gcc_assert (has_dep_p[DEPS_IN_LHS]);
2005 : : return MOVEUP_EXPR_AS_RHS;
2006 : : }
2007 : :
2008 : : /* True when a trapping EXPR cannot be moved through THROUGH_INSN. */
2009 : : #define CANT_MOVE_TRAPPING(expr, through_insn) \
2010 : : (VINSN_MAY_TRAP_P (EXPR_VINSN (expr)) \
2011 : : && !sel_insn_has_single_succ_p ((through_insn), SUCCS_ALL) \
2012 : : && !sel_insn_is_speculation_check (through_insn))
2013 : :
2014 : : /* True when a conflict on a target register was found during moveup_expr. */
2015 : : static bool was_target_conflict = false;
2016 : :
2017 : : /* Return true when moving a debug INSN across THROUGH_INSN will
2018 : : create a bookkeeping block. We don't want to create such blocks,
2019 : : for they would cause codegen differences between compilations with
2020 : : and without debug info. */
2021 : :
2022 : : static bool
2023 : 79 : moving_insn_creates_bookkeeping_block_p (insn_t insn,
2024 : : insn_t through_insn)
2025 : : {
2026 : 79 : basic_block bbi, bbt;
2027 : 79 : edge e1, e2;
2028 : 79 : edge_iterator ei1, ei2;
2029 : :
2030 : 79 : if (!bookkeeping_can_be_created_if_moved_through_p (through_insn))
2031 : : {
2032 : 59 : if (sched_verbose >= 9)
2033 : 0 : sel_print ("no bookkeeping required: ");
2034 : 59 : return false;
2035 : : }
2036 : :
2037 : 20 : bbi = BLOCK_FOR_INSN (insn);
2038 : :
2039 : 20 : if (EDGE_COUNT (bbi->preds) == 1)
2040 : : {
2041 : 1 : if (sched_verbose >= 9)
2042 : 0 : sel_print ("only one pred edge: ");
2043 : 1 : return true;
2044 : : }
2045 : :
2046 : 19 : bbt = BLOCK_FOR_INSN (through_insn);
2047 : :
2048 : 36 : FOR_EACH_EDGE (e1, ei1, bbt->succs)
2049 : : {
2050 : 62 : FOR_EACH_EDGE (e2, ei2, bbi->preds)
2051 : : {
2052 : 45 : if (find_block_for_bookkeeping (e1, e2, true))
2053 : : {
2054 : 10 : if (sched_verbose >= 9)
2055 : 0 : sel_print ("found existing block: ");
2056 : 10 : return false;
2057 : : }
2058 : : }
2059 : : }
2060 : :
2061 : 9 : if (sched_verbose >= 9)
2062 : 0 : sel_print ("would create bookkeeping block: ");
2063 : :
2064 : : return true;
2065 : : }
2066 : :
2067 : : /* Return true when the conflict with newly created implicit clobbers
2068 : : between EXPR and THROUGH_INSN is found because of renaming. */
2069 : : static bool
2070 : 28 : implicit_clobber_conflict_p (insn_t through_insn, expr_t expr)
2071 : : {
2072 : 28 : HARD_REG_SET temp;
2073 : 28 : rtx_insn *insn;
2074 : 28 : rtx reg, rhs, pat;
2075 : 28 : hard_reg_set_iterator hrsi;
2076 : 28 : unsigned regno;
2077 : 28 : bool valid;
2078 : :
2079 : : /* Make a new pseudo register. */
2080 : 28 : reg = gen_reg_rtx (GET_MODE (EXPR_LHS (expr)));
2081 : 28 : max_regno = max_reg_num ();
2082 : 28 : maybe_extend_reg_info_p ();
2083 : :
2084 : : /* Validate a change and bail out early. */
2085 : 28 : insn = EXPR_INSN_RTX (expr);
2086 : 28 : validate_change (insn, &SET_DEST (PATTERN (insn)), reg, true);
2087 : 28 : valid = verify_changes (0);
2088 : 28 : cancel_changes (0);
2089 : 28 : if (!valid)
2090 : : {
2091 : 28 : if (sched_verbose >= 6)
2092 : 0 : sel_print ("implicit clobbers failed validation, ");
2093 : 28 : return true;
2094 : : }
2095 : :
2096 : : /* Make a new insn with it. */
2097 : 0 : rhs = copy_rtx (VINSN_RHS (EXPR_VINSN (expr)));
2098 : 0 : pat = gen_rtx_SET (reg, rhs);
2099 : 0 : start_sequence ();
2100 : 0 : insn = emit_insn (pat);
2101 : 0 : end_sequence ();
2102 : :
2103 : : /* Calculate implicit clobbers. */
2104 : 0 : extract_insn (insn);
2105 : 0 : preprocess_constraints (insn);
2106 : 0 : alternative_mask prefrred = get_preferred_alternatives (insn);
2107 : 0 : ira_implicitly_set_insn_hard_regs (&temp, prefrred);
2108 : 0 : temp &= ~ira_no_alloc_regs;
2109 : :
2110 : : /* If any implicit clobber registers intersect with regular ones in
2111 : : through_insn, we have a dependency and thus bail out. */
2112 : 0 : EXECUTE_IF_SET_IN_HARD_REG_SET (temp, 0, regno, hrsi)
2113 : : {
2114 : 0 : vinsn_t vi = INSN_VINSN (through_insn);
2115 : 0 : if (bitmap_bit_p (VINSN_REG_SETS (vi), regno)
2116 : 0 : || bitmap_bit_p (VINSN_REG_CLOBBERS (vi), regno)
2117 : 0 : || bitmap_bit_p (VINSN_REG_USES (vi), regno))
2118 : 0 : return true;
2119 : : }
2120 : :
2121 : : return false;
2122 : : }
2123 : :
2124 : : /* Modifies EXPR so it can be moved through the THROUGH_INSN,
2125 : : performing necessary transformations. Record the type of transformation
2126 : : made in PTRANS_TYPE, when it is not NULL. When INSIDE_INSN_GROUP,
2127 : : permit all dependencies except true ones, and try to remove those
2128 : : too via forward substitution. All cases when a non-eliminable
2129 : : non-zero cost dependency exists inside an insn group will be fixed
2130 : : in tick_check_p instead. */
2131 : : static enum MOVEUP_EXPR_CODE
2132 : 14797 : moveup_expr (expr_t expr, insn_t through_insn, bool inside_insn_group,
2133 : : enum local_trans_type *ptrans_type)
2134 : : {
2135 : 14797 : vinsn_t vi = EXPR_VINSN (expr);
2136 : 14797 : insn_t insn = VINSN_INSN_RTX (vi);
2137 : 14797 : bool was_changed = false;
2138 : 14797 : bool as_rhs = false;
2139 : 14797 : ds_t *has_dep_p;
2140 : 14797 : ds_t full_ds;
2141 : :
2142 : : /* ??? We use dependencies of non-debug insns on debug insns to
2143 : : indicate that the debug insns need to be reset if the non-debug
2144 : : insn is pulled ahead of it. It's hard to figure out how to
2145 : : introduce such a notion in sel-sched, but it already fails to
2146 : : support debug insns in other ways, so we just go ahead and
2147 : : let the deug insns go corrupt for now. */
2148 : 14797 : if (DEBUG_INSN_P (through_insn) && !DEBUG_INSN_P (insn))
2149 : : return MOVEUP_EXPR_SAME;
2150 : :
2151 : : /* When inside_insn_group, delegate to the helper. */
2152 : 14721 : if (inside_insn_group)
2153 : 0 : return moveup_expr_inside_insn_group (expr, through_insn);
2154 : :
2155 : : /* Deal with unique insns and control dependencies. */
2156 : 14721 : if (VINSN_UNIQUE_P (vi))
2157 : : {
2158 : : /* We can move jumps without side-effects or jumps that are
2159 : : mutually exclusive with instruction THROUGH_INSN (all in cases
2160 : : dependencies allow to do so and jump is not speculative). */
2161 : 1904 : if (control_flow_insn_p (insn))
2162 : : {
2163 : 860 : basic_block fallthru_bb;
2164 : :
2165 : : /* Do not move checks and do not move jumps through other
2166 : : jumps. */
2167 : 860 : if (control_flow_insn_p (through_insn)
2168 : 860 : || sel_insn_is_speculation_check (insn))
2169 : 7 : return MOVEUP_EXPR_NULL;
2170 : :
2171 : : /* Don't move jumps through CFG joins. */
2172 : 853 : if (bookkeeping_can_be_created_if_moved_through_p (through_insn))
2173 : : return MOVEUP_EXPR_NULL;
2174 : :
2175 : : /* The jump should have a clear fallthru block, and
2176 : : this block should be in the current region. */
2177 : 839 : if ((fallthru_bb = fallthru_bb_of_jump (insn)) == NULL
2178 : 839 : || ! in_current_region_p (fallthru_bb))
2179 : 668 : return MOVEUP_EXPR_NULL;
2180 : :
2181 : : /* And it should be mutually exclusive with through_insn. */
2182 : 171 : if (! sched_insns_conditions_mutex_p (insn, through_insn)
2183 : 171 : && ! DEBUG_INSN_P (through_insn))
2184 : : return MOVEUP_EXPR_NULL;
2185 : : }
2186 : :
2187 : : /* Don't move what we can't move. */
2188 : 1044 : if (EXPR_CANT_MOVE (expr)
2189 : 1044 : && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn))
2190 : : return MOVEUP_EXPR_NULL;
2191 : :
2192 : : /* Don't move SCHED_GROUP instruction through anything.
2193 : : If we don't force this, then it will be possible to start
2194 : : scheduling a sched_group before all its dependencies are
2195 : : resolved.
2196 : : ??? Haifa deals with this issue by delaying the SCHED_GROUP
2197 : : as late as possible through rank_for_schedule. */
2198 : 945 : if (SCHED_GROUP_P (insn))
2199 : : return MOVEUP_EXPR_NULL;
2200 : : }
2201 : : else
2202 : 12817 : gcc_assert (!control_flow_insn_p (insn));
2203 : :
2204 : : /* Don't move debug insns if this would require bookkeeping. */
2205 : 13732 : if (DEBUG_INSN_P (insn)
2206 : 120 : && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn)
2207 : 13811 : && moving_insn_creates_bookkeeping_block_p (insn, through_insn))
2208 : : return MOVEUP_EXPR_NULL;
2209 : :
2210 : : /* Deal with data dependencies. */
2211 : 13722 : was_target_conflict = false;
2212 : 13722 : full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
2213 : 13722 : if (full_ds == 0)
2214 : : {
2215 : 9730 : if (!CANT_MOVE_TRAPPING (expr, through_insn))
2216 : 9729 : return MOVEUP_EXPR_SAME;
2217 : : }
2218 : : else
2219 : : {
2220 : : /* We can move UNIQUE insn up only as a whole and unchanged,
2221 : : so it shouldn't have any dependencies. */
2222 : 3992 : if (VINSN_UNIQUE_P (vi))
2223 : : return MOVEUP_EXPR_NULL;
2224 : : }
2225 : :
2226 : 3357 : if (full_ds != 0 && can_speculate_dep_p (full_ds))
2227 : : {
2228 : 0 : int res;
2229 : :
2230 : 0 : res = speculate_expr (expr, full_ds);
2231 : 0 : if (res >= 0)
2232 : : {
2233 : : /* Speculation was successful. */
2234 : 0 : full_ds = 0;
2235 : 0 : was_changed = (res > 0);
2236 : 0 : if (res == 2)
2237 : 0 : was_target_conflict = true;
2238 : 0 : if (ptrans_type)
2239 : 0 : *ptrans_type = TRANS_SPECULATION;
2240 : 0 : sel_clear_has_dependence ();
2241 : : }
2242 : : }
2243 : :
2244 : 3357 : if (has_dep_p[DEPS_IN_INSN])
2245 : : /* We have some dependency that cannot be discarded. */
2246 : : return MOVEUP_EXPR_NULL;
2247 : :
2248 : 2588 : if (has_dep_p[DEPS_IN_LHS])
2249 : : {
2250 : : /* Only separable insns can be moved up with the new register.
2251 : : Anyways, we should mark that the original register is
2252 : : unavailable. */
2253 : 1738 : if (!enable_schedule_as_rhs_p || !EXPR_SEPARABLE_P (expr))
2254 : : return MOVEUP_EXPR_NULL;
2255 : :
2256 : : /* When renaming a hard register to a pseudo before reload, extra
2257 : : dependencies can occur from the implicit clobbers of the insn.
2258 : : Filter out such cases here. */
2259 : 198 : if (!reload_completed && REG_P (EXPR_LHS (expr))
2260 : 198 : && HARD_REGISTER_P (EXPR_LHS (expr))
2261 : 1363 : && implicit_clobber_conflict_p (through_insn, expr))
2262 : : {
2263 : 28 : if (sched_verbose >= 6)
2264 : 0 : sel_print ("implicit clobbers conflict detected, ");
2265 : 28 : return MOVEUP_EXPR_NULL;
2266 : : }
2267 : 1307 : EXPR_TARGET_AVAILABLE (expr) = false;
2268 : 1307 : was_target_conflict = true;
2269 : 1307 : as_rhs = true;
2270 : : }
2271 : :
2272 : : /* At this point we have either separable insns, that will be lifted
2273 : : up only as RHSes, or non-separable insns with no dependency in lhs.
2274 : : If dependency is in RHS, then try to perform substitution and move up
2275 : : substituted RHS:
2276 : :
2277 : : Ex. 1: Ex.2
2278 : : y = x; y = x;
2279 : : z = y*2; y = y*2;
2280 : :
2281 : : In Ex.1 y*2 can be substituted for x*2 and the whole operation can be
2282 : : moved above y=x assignment as z=x*2.
2283 : :
2284 : : In Ex.2 y*2 also can be substituted for x*2, but only the right hand
2285 : : side can be moved because of the output dependency. The operation was
2286 : : cropped to its rhs above. */
2287 : 2157 : if (has_dep_p[DEPS_IN_RHS])
2288 : : {
2289 : 1093 : ds_t *rhs_dsp = &has_dep_p[DEPS_IN_RHS];
2290 : :
2291 : : /* Can't substitute UNIQUE VINSNs. */
2292 : 1093 : gcc_assert (!VINSN_UNIQUE_P (vi));
2293 : :
2294 : 1093 : if (can_speculate_dep_p (*rhs_dsp))
2295 : : {
2296 : 0 : int res;
2297 : :
2298 : 0 : res = speculate_expr (expr, *rhs_dsp);
2299 : 0 : if (res >= 0)
2300 : : {
2301 : : /* Speculation was successful. */
2302 : 0 : *rhs_dsp = 0;
2303 : 0 : was_changed = (res > 0);
2304 : 0 : if (res == 2)
2305 : 0 : was_target_conflict = true;
2306 : 0 : if (ptrans_type)
2307 : 0 : *ptrans_type = TRANS_SPECULATION;
2308 : : }
2309 : : else
2310 : : return MOVEUP_EXPR_NULL;
2311 : : }
2312 : 1093 : else if (can_substitute_through_p (through_insn,
2313 : : *rhs_dsp)
2314 : 1093 : && substitute_reg_in_expr (expr, through_insn, false))
2315 : : {
2316 : : /* ??? We cannot perform substitution AND speculation on the same
2317 : : insn. */
2318 : 79 : gcc_assert (!was_changed);
2319 : 79 : was_changed = true;
2320 : 79 : if (ptrans_type)
2321 : 79 : *ptrans_type = TRANS_SUBSTITUTION;
2322 : 79 : EXPR_WAS_SUBSTITUTED (expr) = true;
2323 : : }
2324 : : else
2325 : 1014 : return MOVEUP_EXPR_NULL;
2326 : : }
2327 : :
2328 : : /* Don't move trapping insns through jumps.
2329 : : This check should be at the end to give a chance to control speculation
2330 : : to perform its duties. */
2331 : 1143 : if (CANT_MOVE_TRAPPING (expr, through_insn))
2332 : : return MOVEUP_EXPR_NULL;
2333 : :
2334 : 1141 : return (was_changed
2335 : 1141 : ? MOVEUP_EXPR_CHANGED
2336 : : : (as_rhs
2337 : 1062 : ? MOVEUP_EXPR_AS_RHS
2338 : : : MOVEUP_EXPR_SAME));
2339 : : }
2340 : :
2341 : : /* Try to look at bitmap caches for EXPR and INSN pair, return true
2342 : : if successful. When INSIDE_INSN_GROUP, also try ignore dependencies
2343 : : that can exist within a parallel group. Write to RES the resulting
2344 : : code for moveup_expr. */
2345 : : static bool
2346 : 152267 : try_bitmap_cache (expr_t expr, insn_t insn,
2347 : : bool inside_insn_group,
2348 : : enum MOVEUP_EXPR_CODE *res)
2349 : : {
2350 : 152267 : int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2351 : :
2352 : : /* First check whether we've analyzed this situation already. */
2353 : 152267 : if (bitmap_bit_p (INSN_ANALYZED_DEPS (insn), expr_uid))
2354 : : {
2355 : 121527 : if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2356 : : {
2357 : 34810 : if (sched_verbose >= 6)
2358 : 0 : sel_print ("removed (cached)\n");
2359 : 34810 : *res = MOVEUP_EXPR_NULL;
2360 : 34810 : return true;
2361 : : }
2362 : : else
2363 : : {
2364 : 86717 : if (sched_verbose >= 6)
2365 : 0 : sel_print ("unchanged (cached)\n");
2366 : 86717 : *res = MOVEUP_EXPR_SAME;
2367 : 86717 : return true;
2368 : : }
2369 : : }
2370 : 30740 : else if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2371 : : {
2372 : 15539 : if (inside_insn_group)
2373 : : {
2374 : 0 : if (sched_verbose >= 6)
2375 : 0 : sel_print ("unchanged (as RHS, cached, inside insn group)\n");
2376 : 0 : *res = MOVEUP_EXPR_SAME;
2377 : 0 : return true;
2378 : :
2379 : : }
2380 : : else
2381 : 15539 : EXPR_TARGET_AVAILABLE (expr) = false;
2382 : :
2383 : : /* This is the only case when propagation result can change over time,
2384 : : as we can dynamically switch off scheduling as RHS. In this case,
2385 : : just check the flag to reach the correct decision. */
2386 : 15539 : if (enable_schedule_as_rhs_p)
2387 : : {
2388 : 15420 : if (sched_verbose >= 6)
2389 : 0 : sel_print ("unchanged (as RHS, cached)\n");
2390 : 15420 : *res = MOVEUP_EXPR_AS_RHS;
2391 : 15420 : return true;
2392 : : }
2393 : : else
2394 : : {
2395 : 119 : if (sched_verbose >= 6)
2396 : 0 : sel_print ("removed (cached as RHS, but renaming"
2397 : : " is now disabled)\n");
2398 : 119 : *res = MOVEUP_EXPR_NULL;
2399 : 119 : return true;
2400 : : }
2401 : : }
2402 : :
2403 : : return false;
2404 : : }
2405 : :
2406 : : /* Try to look at bitmap caches for EXPR and INSN pair, return true
2407 : : if successful. Write to RES the resulting code for moveup_expr. */
2408 : : static bool
2409 : 15201 : try_transformation_cache (expr_t expr, insn_t insn,
2410 : : enum MOVEUP_EXPR_CODE *res)
2411 : : {
2412 : 15201 : struct transformed_insns *pti
2413 : : = (struct transformed_insns *)
2414 : 15201 : htab_find_with_hash (INSN_TRANSFORMED_INSNS (insn),
2415 : 15201 : &EXPR_VINSN (expr),
2416 : 15201 : VINSN_HASH_RTX (EXPR_VINSN (expr)));
2417 : 15201 : if (pti)
2418 : : {
2419 : : /* This EXPR was already moved through this insn and was
2420 : : changed as a result. Fetch the proper data from
2421 : : the hashtable. */
2422 : 916 : insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2423 : 458 : INSN_UID (insn), pti->type,
2424 : : pti->vinsn_old, pti->vinsn_new,
2425 : : EXPR_SPEC_DONE_DS (expr));
2426 : :
2427 : 458 : if (INSN_IN_STREAM_P (VINSN_INSN_RTX (pti->vinsn_new)))
2428 : 2 : pti->vinsn_new = vinsn_copy (pti->vinsn_new, true);
2429 : 458 : change_vinsn_in_expr (expr, pti->vinsn_new);
2430 : 458 : if (pti->was_target_conflict)
2431 : 89 : EXPR_TARGET_AVAILABLE (expr) = false;
2432 : 458 : if (pti->type == TRANS_SPECULATION)
2433 : : {
2434 : 0 : EXPR_SPEC_DONE_DS (expr) = pti->ds;
2435 : 0 : EXPR_NEEDS_SPEC_CHECK_P (expr) |= pti->needs_check;
2436 : : }
2437 : :
2438 : 458 : if (sched_verbose >= 6)
2439 : : {
2440 : 0 : sel_print ("changed (cached): ");
2441 : 0 : dump_expr (expr);
2442 : 0 : sel_print ("\n");
2443 : : }
2444 : :
2445 : 458 : *res = MOVEUP_EXPR_CHANGED;
2446 : 458 : return true;
2447 : : }
2448 : :
2449 : : return false;
2450 : : }
2451 : :
2452 : : /* Update bitmap caches on INSN with result RES of propagating EXPR. */
2453 : : static void
2454 : 14718 : update_bitmap_cache (expr_t expr, insn_t insn, bool inside_insn_group,
2455 : : enum MOVEUP_EXPR_CODE res)
2456 : : {
2457 : 14718 : int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2458 : :
2459 : : /* Do not cache result of propagating jumps through an insn group,
2460 : : as it is always true, which is not useful outside the group. */
2461 : 14718 : if (inside_insn_group)
2462 : : return;
2463 : :
2464 : 14718 : if (res == MOVEUP_EXPR_NULL)
2465 : : {
2466 : 3851 : bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2467 : 3851 : bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2468 : : }
2469 : 10867 : else if (res == MOVEUP_EXPR_SAME)
2470 : : {
2471 : 9805 : bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2472 : 9805 : bitmap_clear_bit (INSN_FOUND_DEPS (insn), expr_uid);
2473 : : }
2474 : 1062 : else if (res == MOVEUP_EXPR_AS_RHS)
2475 : : {
2476 : 1062 : bitmap_clear_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2477 : 1062 : bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2478 : : }
2479 : : else
2480 : 0 : gcc_unreachable ();
2481 : : }
2482 : :
2483 : : /* Update hashtable on INSN with changed EXPR, old EXPR_OLD_VINSN
2484 : : and transformation type TRANS_TYPE. */
2485 : : static void
2486 : 79 : update_transformation_cache (expr_t expr, insn_t insn,
2487 : : bool inside_insn_group,
2488 : : enum local_trans_type trans_type,
2489 : : vinsn_t expr_old_vinsn)
2490 : : {
2491 : 79 : struct transformed_insns *pti;
2492 : :
2493 : 79 : if (inside_insn_group)
2494 : : return;
2495 : :
2496 : 79 : pti = XNEW (struct transformed_insns);
2497 : 79 : pti->vinsn_old = expr_old_vinsn;
2498 : 79 : pti->vinsn_new = EXPR_VINSN (expr);
2499 : 79 : pti->type = trans_type;
2500 : 79 : pti->was_target_conflict = was_target_conflict;
2501 : 79 : pti->ds = EXPR_SPEC_DONE_DS (expr);
2502 : 79 : pti->needs_check = EXPR_NEEDS_SPEC_CHECK_P (expr);
2503 : 79 : vinsn_attach (pti->vinsn_old);
2504 : 79 : vinsn_attach (pti->vinsn_new);
2505 : 79 : *((struct transformed_insns **)
2506 : 79 : htab_find_slot_with_hash (INSN_TRANSFORMED_INSNS (insn),
2507 : : pti, VINSN_HASH_RTX (expr_old_vinsn),
2508 : 79 : INSERT)) = pti;
2509 : : }
2510 : :
2511 : : /* Same as moveup_expr, but first looks up the result of
2512 : : transformation in caches. */
2513 : : static enum MOVEUP_EXPR_CODE
2514 : 152321 : moveup_expr_cached (expr_t expr, insn_t insn, bool inside_insn_group)
2515 : : {
2516 : 152321 : enum MOVEUP_EXPR_CODE res;
2517 : 152321 : bool got_answer = false;
2518 : :
2519 : 152321 : if (sched_verbose >= 6)
2520 : : {
2521 : 0 : sel_print ("Moving ");
2522 : 0 : dump_expr (expr);
2523 : 0 : sel_print (" through %d: ", INSN_UID (insn));
2524 : : }
2525 : :
2526 : 152321 : if (DEBUG_INSN_P (EXPR_INSN_RTX (expr))
2527 : 296 : && BLOCK_FOR_INSN (EXPR_INSN_RTX (expr))
2528 : 152578 : && (sel_bb_head (BLOCK_FOR_INSN (EXPR_INSN_RTX (expr)))
2529 : 257 : == EXPR_INSN_RTX (expr)))
2530 : : /* Don't use cached information for debug insns that are heads of
2531 : : basic blocks. */;
2532 : 152267 : else if (try_bitmap_cache (expr, insn, inside_insn_group, &res))
2533 : : /* When inside insn group, we do not want remove stores conflicting
2534 : : with previosly issued loads. */
2535 : 137066 : got_answer = ! inside_insn_group || res != MOVEUP_EXPR_NULL;
2536 : 15201 : else if (try_transformation_cache (expr, insn, &res))
2537 : : got_answer = true;
2538 : :
2539 : : if (! got_answer)
2540 : : {
2541 : : /* Invoke moveup_expr and record the results. */
2542 : 14797 : vinsn_t expr_old_vinsn = EXPR_VINSN (expr);
2543 : 14797 : ds_t expr_old_spec_ds = EXPR_SPEC_DONE_DS (expr);
2544 : 14797 : int expr_uid = INSN_UID (VINSN_INSN_RTX (expr_old_vinsn));
2545 : 14797 : bool unique_p = VINSN_UNIQUE_P (expr_old_vinsn);
2546 : 14797 : enum local_trans_type trans_type = TRANS_SUBSTITUTION;
2547 : :
2548 : : /* ??? Invent something better than this. We can't allow old_vinsn
2549 : : to go, we need it for the history vector. */
2550 : 14797 : vinsn_attach (expr_old_vinsn);
2551 : :
2552 : 14797 : res = moveup_expr (expr, insn, inside_insn_group,
2553 : : &trans_type);
2554 : 14797 : switch (res)
2555 : : {
2556 : 3851 : case MOVEUP_EXPR_NULL:
2557 : 3851 : update_bitmap_cache (expr, insn, inside_insn_group, res);
2558 : 3851 : if (sched_verbose >= 6)
2559 : 0 : sel_print ("removed\n");
2560 : : break;
2561 : :
2562 : 9805 : case MOVEUP_EXPR_SAME:
2563 : 9805 : update_bitmap_cache (expr, insn, inside_insn_group, res);
2564 : 9805 : if (sched_verbose >= 6)
2565 : 0 : sel_print ("unchanged\n");
2566 : : break;
2567 : :
2568 : 1062 : case MOVEUP_EXPR_AS_RHS:
2569 : 1062 : gcc_assert (!unique_p || inside_insn_group);
2570 : 1062 : update_bitmap_cache (expr, insn, inside_insn_group, res);
2571 : 1062 : if (sched_verbose >= 6)
2572 : 0 : sel_print ("unchanged (as RHS)\n");
2573 : : break;
2574 : :
2575 : 79 : case MOVEUP_EXPR_CHANGED:
2576 : 79 : gcc_assert (INSN_UID (EXPR_INSN_RTX (expr)) != expr_uid
2577 : : || EXPR_SPEC_DONE_DS (expr) != expr_old_spec_ds);
2578 : 158 : insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2579 : 79 : INSN_UID (insn), trans_type,
2580 : : expr_old_vinsn, EXPR_VINSN (expr),
2581 : : expr_old_spec_ds);
2582 : 79 : update_transformation_cache (expr, insn, inside_insn_group,
2583 : : trans_type, expr_old_vinsn);
2584 : 79 : if (sched_verbose >= 6)
2585 : : {
2586 : 0 : sel_print ("changed: ");
2587 : 0 : dump_expr (expr);
2588 : 0 : sel_print ("\n");
2589 : : }
2590 : : break;
2591 : : default:
2592 : : gcc_unreachable ();
2593 : : }
2594 : :
2595 : 14797 : vinsn_detach (expr_old_vinsn);
2596 : : }
2597 : :
2598 : 152321 : return res;
2599 : : }
2600 : :
2601 : : /* Moves an av set AVP up through INSN, performing necessary
2602 : : transformations. */
2603 : : static void
2604 : 49001 : moveup_set_expr (av_set_t *avp, insn_t insn, bool inside_insn_group)
2605 : : {
2606 : 49001 : av_set_iterator i;
2607 : 49001 : expr_t expr;
2608 : :
2609 : 345437 : FOR_EACH_EXPR_1 (expr, i, avp)
2610 : : {
2611 : :
2612 : 148218 : switch (moveup_expr_cached (expr, insn, inside_insn_group))
2613 : : {
2614 : : case MOVEUP_EXPR_SAME:
2615 : : case MOVEUP_EXPR_AS_RHS:
2616 : : break;
2617 : :
2618 : 38780 : case MOVEUP_EXPR_NULL:
2619 : 38780 : av_set_iter_remove (&i);
2620 : 38780 : break;
2621 : :
2622 : 531 : case MOVEUP_EXPR_CHANGED:
2623 : 531 : expr = merge_with_other_exprs (avp, &i, expr);
2624 : 531 : break;
2625 : :
2626 : 0 : default:
2627 : 0 : gcc_unreachable ();
2628 : : }
2629 : : }
2630 : 49001 : }
2631 : :
2632 : : /* Moves AVP set along PATH. */
2633 : : static void
2634 : 5171 : moveup_set_inside_insn_group (av_set_t *avp, ilist_t path)
2635 : : {
2636 : 5171 : int last_cycle;
2637 : :
2638 : 5171 : if (sched_verbose >= 6)
2639 : 0 : sel_print ("Moving expressions up in the insn group...\n");
2640 : 5171 : if (! path)
2641 : : return;
2642 : 0 : last_cycle = INSN_SCHED_CYCLE (ILIST_INSN (path));
2643 : 0 : while (path
2644 : 0 : && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2645 : : {
2646 : 0 : moveup_set_expr (avp, ILIST_INSN (path), true);
2647 : 0 : path = ILIST_NEXT (path);
2648 : : }
2649 : : }
2650 : :
2651 : : /* Returns true if after moving EXPR along PATH it equals to EXPR_VLIW. */
2652 : : static bool
2653 : 17468 : equal_after_moveup_path_p (expr_t expr, ilist_t path, expr_t expr_vliw)
2654 : : {
2655 : 17468 : expr_def _tmp, *tmp = &_tmp;
2656 : 17468 : int last_cycle;
2657 : 17468 : bool res = true;
2658 : :
2659 : 17468 : copy_expr_onside (tmp, expr);
2660 : 17468 : last_cycle = path ? INSN_SCHED_CYCLE (ILIST_INSN (path)) : 0;
2661 : 17468 : while (path
2662 : 17468 : && res
2663 : 17468 : && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2664 : : {
2665 : 0 : res = (moveup_expr_cached (tmp, ILIST_INSN (path), true)
2666 : : != MOVEUP_EXPR_NULL);
2667 : 0 : path = ILIST_NEXT (path);
2668 : : }
2669 : :
2670 : 17468 : if (res)
2671 : : {
2672 : 17468 : vinsn_t tmp_vinsn = EXPR_VINSN (tmp);
2673 : 17468 : vinsn_t expr_vliw_vinsn = EXPR_VINSN (expr_vliw);
2674 : :
2675 : 17468 : if (tmp_vinsn != expr_vliw_vinsn)
2676 : 9803 : res = vinsn_equal_p (tmp_vinsn, expr_vliw_vinsn);
2677 : : }
2678 : :
2679 : 17468 : clear_expr (tmp);
2680 : 17468 : return res;
2681 : : }
2682 : :
2683 : :
2684 : : /* Functions that compute av and lv sets. */
2685 : :
2686 : : /* Returns true if INSN is not a downward continuation of the given path P in
2687 : : the current stage. */
2688 : : static bool
2689 : 67659 : is_ineligible_successor (insn_t insn, ilist_t p)
2690 : : {
2691 : 67659 : insn_t prev_insn;
2692 : :
2693 : : /* Check if insn is not deleted. */
2694 : 67659 : if (PREV_INSN (insn) && NEXT_INSN (PREV_INSN (insn)) != insn)
2695 : 0 : gcc_unreachable ();
2696 : 67659 : else if (NEXT_INSN (insn) && PREV_INSN (NEXT_INSN (insn)) != insn)
2697 : 0 : gcc_unreachable ();
2698 : :
2699 : : /* If it's the first insn visited, then the successor is ok. */
2700 : 67659 : if (!p)
2701 : : return false;
2702 : :
2703 : 54216 : prev_insn = ILIST_INSN (p);
2704 : :
2705 : 54216 : if (/* a backward edge. */
2706 : 54216 : INSN_SEQNO (insn) < INSN_SEQNO (prev_insn)
2707 : : /* is already visited. */
2708 : 53349 : || (INSN_SEQNO (insn) == INSN_SEQNO (prev_insn)
2709 : 2404 : && (ilist_is_in_p (p, insn)
2710 : : /* We can reach another fence here and still seqno of insn
2711 : : would be equal to seqno of prev_insn. This is possible
2712 : : when prev_insn is a previously created bookkeeping copy.
2713 : : In that case it'd get a seqno of insn. Thus, check here
2714 : : whether insn is in current fence too. */
2715 : 2404 : || IN_CURRENT_FENCE_P (insn)))
2716 : : /* Was already scheduled on this round. */
2717 : 53346 : || (INSN_SEQNO (insn) > INSN_SEQNO (prev_insn)
2718 : 50945 : && IN_CURRENT_FENCE_P (insn))
2719 : : /* An insn from another fence could also be
2720 : : scheduled earlier even if this insn is not in
2721 : : a fence list right now. Check INSN_SCHED_CYCLE instead. */
2722 : 107560 : || (!pipelining_p
2723 : 21876 : && INSN_SCHED_TIMES (insn) > 0))
2724 : 928 : return true;
2725 : : else
2726 : : return false;
2727 : : }
2728 : :
2729 : : /* Computes the av_set below the last bb insn INSN, doing all the 'dirty work'
2730 : : of handling multiple successors and properly merging its av_sets. P is
2731 : : the current path traversed. WS is the size of lookahead window.
2732 : : Return the av set computed. */
2733 : : static av_set_t
2734 : 7952 : compute_av_set_at_bb_end (insn_t insn, ilist_t p, int ws)
2735 : : {
2736 : 7952 : struct succs_info *sinfo;
2737 : 7952 : av_set_t expr_in_all_succ_branches = NULL;
2738 : 7952 : int is;
2739 : 7952 : insn_t succ, zero_succ = NULL;
2740 : 7952 : av_set_t av1 = NULL;
2741 : :
2742 : 7952 : gcc_assert (sel_bb_end_p (insn));
2743 : :
2744 : : /* Find different kind of successors needed for correct computing of
2745 : : SPEC and TARGET_AVAILABLE attributes. */
2746 : 7952 : sinfo = compute_succs_info (insn, SUCCS_NORMAL);
2747 : :
2748 : : /* Debug output. */
2749 : 7952 : if (sched_verbose >= 6)
2750 : : {
2751 : 0 : sel_print ("successors of bb end (%d): ", INSN_UID (insn));
2752 : 0 : dump_insn_vector (sinfo->succs_ok);
2753 : 0 : sel_print ("\n");
2754 : 0 : if (sinfo->succs_ok_n != sinfo->all_succs_n)
2755 : 0 : sel_print ("real successors num: %d\n", sinfo->all_succs_n);
2756 : : }
2757 : :
2758 : : /* Add insn to the tail of current path. */
2759 : 7952 : ilist_add (&p, insn);
2760 : :
2761 : 13587 : FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ)
2762 : : {
2763 : 5635 : av_set_t succ_set;
2764 : :
2765 : : /* We will edit SUCC_SET and EXPR_SPEC field of its elements. */
2766 : 5635 : succ_set = compute_av_set_inside_bb (succ, p, ws, true);
2767 : :
2768 : 11270 : av_set_split_usefulness (succ_set,
2769 : 5635 : sinfo->probs_ok[is],
2770 : : sinfo->all_prob);
2771 : :
2772 : 5635 : if (sinfo->all_succs_n > 1)
2773 : : {
2774 : : /* Find EXPR'es that came from *all* successors and save them
2775 : : into expr_in_all_succ_branches. This set will be used later
2776 : : for calculating speculation attributes of EXPR'es. */
2777 : 4184 : if (is == 0)
2778 : : {
2779 : 3247 : expr_in_all_succ_branches = av_set_copy (succ_set);
2780 : :
2781 : : /* Remember the first successor for later. */
2782 : 3247 : zero_succ = succ;
2783 : : }
2784 : : else
2785 : : {
2786 : 937 : av_set_iterator i;
2787 : 937 : expr_t expr;
2788 : :
2789 : 6451 : FOR_EACH_EXPR_1 (expr, i, &expr_in_all_succ_branches)
2790 : 2757 : if (!av_set_is_in_p (succ_set, EXPR_VINSN (expr)))
2791 : 1366 : av_set_iter_remove (&i);
2792 : : }
2793 : : }
2794 : :
2795 : : /* Union the av_sets. Check liveness restrictions on target registers
2796 : : in special case of two successors. */
2797 : 5635 : if (sinfo->succs_ok_n == 2 && is == 1)
2798 : : {
2799 : 937 : basic_block bb0 = BLOCK_FOR_INSN (zero_succ);
2800 : 937 : basic_block bb1 = BLOCK_FOR_INSN (succ);
2801 : :
2802 : 937 : gcc_assert (BB_LV_SET_VALID_P (bb0) && BB_LV_SET_VALID_P (bb1));
2803 : 937 : av_set_union_and_live (&av1, &succ_set,
2804 : 937 : BB_LV_SET (bb0),
2805 : 937 : BB_LV_SET (bb1),
2806 : : insn);
2807 : 937 : }
2808 : : else
2809 : 4698 : av_set_union_and_clear (&av1, &succ_set, insn);
2810 : : }
2811 : :
2812 : : /* Check liveness restrictions via hard way when there are more than
2813 : : two successors. */
2814 : 7952 : if (sinfo->succs_ok_n > 2)
2815 : 0 : FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ)
2816 : : {
2817 : 0 : basic_block succ_bb = BLOCK_FOR_INSN (succ);
2818 : 0 : av_set_t av_succ = (is_ineligible_successor (succ, p)
2819 : 0 : ? NULL
2820 : 0 : : BB_AV_SET (succ_bb));
2821 : :
2822 : 0 : gcc_assert (BB_LV_SET_VALID_P (succ_bb));
2823 : 0 : mark_unavailable_targets (av1, av_succ, BB_LV_SET (succ_bb));
2824 : : }
2825 : :
2826 : : /* Finally, check liveness restrictions on paths leaving the region. */
2827 : 7952 : if (sinfo->all_succs_n > sinfo->succs_ok_n)
2828 : 12600 : FOR_EACH_VEC_ELT (sinfo->succs_other, is, succ)
2829 : 7065 : mark_unavailable_targets
2830 : 7065 : (av1, NULL, BB_LV_SET (BLOCK_FOR_INSN (succ)));
2831 : :
2832 : 7952 : if (sinfo->all_succs_n > 1)
2833 : : {
2834 : : av_set_iterator i;
2835 : : expr_t expr;
2836 : :
2837 : : /* Increase the spec attribute of all EXPR'es that didn't come
2838 : : from all successors. */
2839 : 16363 : FOR_EACH_EXPR (expr, i, av1)
2840 : 11607 : if (!av_set_is_in_p (expr_in_all_succ_branches, EXPR_VINSN (expr)))
2841 : 2566 : EXPR_SPEC (expr)++;
2842 : :
2843 : 4756 : av_set_clear (&expr_in_all_succ_branches);
2844 : :
2845 : : /* Do not move conditional branches through other
2846 : : conditional branches. So, remove all conditional
2847 : : branches from av_set if current operator is a conditional
2848 : : branch. */
2849 : 4756 : av_set_substract_cond_branches (&av1);
2850 : : }
2851 : :
2852 : 7952 : ilist_remove (&p);
2853 : 7952 : free_succs_info (sinfo);
2854 : :
2855 : 7952 : if (sched_verbose >= 6)
2856 : : {
2857 : 0 : sel_print ("av_succs (%d): ", INSN_UID (insn));
2858 : 0 : dump_av_set (av1);
2859 : 0 : sel_print ("\n");
2860 : : }
2861 : :
2862 : 7952 : return av1;
2863 : : }
2864 : :
2865 : : /* This function computes av_set for the FIRST_INSN by dragging valid
2866 : : av_set through all basic block insns either from the end of basic block
2867 : : (computed using compute_av_set_at_bb_end) or from the insn on which
2868 : : MAX_WS was exceeded. It uses compute_av_set_at_bb_end to compute av_set
2869 : : below the basic block and handling conditional branches.
2870 : : FIRST_INSN - the basic block head, P - path consisting of the insns
2871 : : traversed on the way to the FIRST_INSN (the path is sparse, only bb heads
2872 : : and bb ends are added to the path), WS - current window size,
2873 : : NEED_COPY_P - true if we'll make a copy of av_set before returning it. */
2874 : : static av_set_t
2875 : 11374 : compute_av_set_inside_bb (insn_t first_insn, ilist_t p, int ws,
2876 : : bool need_copy_p)
2877 : : {
2878 : 11374 : insn_t cur_insn;
2879 : 11374 : int end_ws = ws;
2880 : 11374 : insn_t bb_end = sel_bb_end (BLOCK_FOR_INSN (first_insn));
2881 : 11374 : insn_t after_bb_end = NEXT_INSN (bb_end);
2882 : 11374 : insn_t last_insn;
2883 : 11374 : av_set_t av = NULL;
2884 : 11374 : basic_block cur_bb = BLOCK_FOR_INSN (first_insn);
2885 : :
2886 : : /* Return NULL if insn is not on the legitimate downward path. */
2887 : 11374 : if (is_ineligible_successor (first_insn, p))
2888 : : {
2889 : 505 : if (sched_verbose >= 6)
2890 : 0 : sel_print ("Insn %d is ineligible_successor\n", INSN_UID (first_insn));
2891 : :
2892 : 505 : return NULL;
2893 : : }
2894 : :
2895 : : /* If insn already has valid av(insn) computed, just return it. */
2896 : 10869 : if (AV_SET_VALID_P (first_insn))
2897 : : {
2898 : 2031 : av_set_t av_set;
2899 : :
2900 : 2031 : if (sel_bb_head_p (first_insn))
2901 : 1989 : av_set = BB_AV_SET (BLOCK_FOR_INSN (first_insn));
2902 : : else
2903 : : av_set = NULL;
2904 : :
2905 : 2031 : if (sched_verbose >= 6)
2906 : : {
2907 : 0 : sel_print ("Insn %d has a valid av set: ", INSN_UID (first_insn));
2908 : 0 : dump_av_set (av_set);
2909 : 0 : sel_print ("\n");
2910 : : }
2911 : :
2912 : 2031 : return need_copy_p ? av_set_copy (av_set) : av_set;
2913 : : }
2914 : :
2915 : 8838 : ilist_add (&p, first_insn);
2916 : :
2917 : : /* As the result after this loop have completed, in LAST_INSN we'll
2918 : : have the insn which has valid av_set to start backward computation
2919 : : from: it either will be NULL because on it the window size was exceeded
2920 : : or other valid av_set as returned by compute_av_set for the last insn
2921 : : of the basic block. */
2922 : 57883 : for (last_insn = first_insn; last_insn != after_bb_end;
2923 : 49045 : last_insn = NEXT_INSN (last_insn))
2924 : : {
2925 : : /* We may encounter valid av_set not only on bb_head, but also on
2926 : : those insns on which previously MAX_WS was exceeded. */
2927 : 49931 : if (AV_SET_VALID_P (last_insn))
2928 : : {
2929 : 279 : if (sched_verbose >= 6)
2930 : 0 : sel_print ("Insn %d has a valid empty av set\n", INSN_UID (last_insn));
2931 : : break;
2932 : : }
2933 : :
2934 : : /* The special case: the last insn of the BB may be an
2935 : : ineligible_successor due to its SEQ_NO that was set on
2936 : : it as a bookkeeping. */
2937 : 49652 : if (last_insn != first_insn
2938 : 49652 : && is_ineligible_successor (last_insn, p))
2939 : : {
2940 : 415 : if (sched_verbose >= 6)
2941 : 0 : sel_print ("Insn %d is ineligible_successor\n", INSN_UID (last_insn));
2942 : : break;
2943 : : }
2944 : :
2945 : 49237 : if (DEBUG_INSN_P (last_insn))
2946 : 131 : continue;
2947 : :
2948 : 49106 : if (end_ws > max_ws)
2949 : : {
2950 : : /* We can reach max lookahead size at bb_header, so clean av_set
2951 : : first. */
2952 : 192 : INSN_WS_LEVEL (last_insn) = global_level;
2953 : :
2954 : 192 : if (sched_verbose >= 6)
2955 : 0 : sel_print ("Insn %d is beyond the software lookahead window size\n",
2956 : 0 : INSN_UID (last_insn));
2957 : : break;
2958 : : }
2959 : :
2960 : 48914 : end_ws++;
2961 : : }
2962 : :
2963 : : /* Get the valid av_set into AV above the LAST_INSN to start backward
2964 : : computation from. It either will be empty av_set or av_set computed from
2965 : : the successors on the last insn of the current bb. */
2966 : 8838 : if (last_insn != after_bb_end)
2967 : : {
2968 : 886 : av = NULL;
2969 : :
2970 : : /* This is needed only to obtain av_sets that are identical to
2971 : : those computed by the old compute_av_set version. */
2972 : 886 : if (last_insn == first_insn && !INSN_NOP_P (last_insn))
2973 : 21 : av_set_add (&av, INSN_EXPR (last_insn));
2974 : : }
2975 : : else
2976 : : /* END_WS is always already increased by 1 if LAST_INSN == AFTER_BB_END. */
2977 : 7952 : av = compute_av_set_at_bb_end (bb_end, p, end_ws);
2978 : :
2979 : : /* Compute av_set in AV starting from below the LAST_INSN up to
2980 : : location above the FIRST_INSN. */
2981 : 57883 : for (cur_insn = PREV_INSN (last_insn); cur_insn != PREV_INSN (first_insn);
2982 : 49045 : cur_insn = PREV_INSN (cur_insn))
2983 : 49045 : if (!INSN_NOP_P (cur_insn))
2984 : : {
2985 : 49001 : expr_t expr;
2986 : :
2987 : 49001 : moveup_set_expr (&av, cur_insn, false);
2988 : :
2989 : : /* If the expression for CUR_INSN is already in the set,
2990 : : replace it by the new one. */
2991 : 49001 : expr = av_set_lookup (av, INSN_VINSN (cur_insn));
2992 : 49001 : if (expr != NULL)
2993 : : {
2994 : 345 : clear_expr (expr);
2995 : 345 : copy_expr (expr, INSN_EXPR (cur_insn));
2996 : : }
2997 : : else
2998 : 48656 : av_set_add (&av, INSN_EXPR (cur_insn));
2999 : : }
3000 : :
3001 : : /* Clear stale bb_av_set. */
3002 : 8838 : if (sel_bb_head_p (first_insn))
3003 : : {
3004 : 4784 : av_set_clear (&BB_AV_SET (cur_bb));
3005 : 4784 : BB_AV_SET (cur_bb) = need_copy_p ? av_set_copy (av) : av;
3006 : 4784 : BB_AV_LEVEL (cur_bb) = global_level;
3007 : : }
3008 : :
3009 : 8838 : if (sched_verbose >= 6)
3010 : : {
3011 : 0 : sel_print ("Computed av set for insn %d: ", INSN_UID (first_insn));
3012 : 0 : dump_av_set (av);
3013 : 0 : sel_print ("\n");
3014 : : }
3015 : :
3016 : 8838 : ilist_remove (&p);
3017 : 8838 : return av;
3018 : : }
3019 : :
3020 : : /* Compute av set before INSN.
3021 : : INSN - the current operation (actual rtx INSN)
3022 : : P - the current path, which is list of insns visited so far
3023 : : WS - software lookahead window size.
3024 : : UNIQUE_P - TRUE, if returned av_set will be changed, hence
3025 : : if we want to save computed av_set in s_i_d, we should make a copy of it.
3026 : :
3027 : : In the resulting set we will have only expressions that don't have delay
3028 : : stalls and nonsubstitutable dependences. */
3029 : : static av_set_t
3030 : 5739 : compute_av_set (insn_t insn, ilist_t p, int ws, bool unique_p)
3031 : : {
3032 : 0 : return compute_av_set_inside_bb (insn, p, ws, unique_p);
3033 : : }
3034 : :
3035 : : /* Propagate a liveness set LV through INSN. */
3036 : : static void
3037 : 10773 : propagate_lv_set (regset lv, insn_t insn)
3038 : : {
3039 : 10773 : gcc_assert (INSN_P (insn));
3040 : :
3041 : 10773 : if (INSN_NOP_P (insn))
3042 : : return;
3043 : :
3044 : 10729 : df_simulate_one_insn_backwards (BLOCK_FOR_INSN (insn), insn, lv);
3045 : : }
3046 : :
3047 : : /* Return livness set at the end of BB. */
3048 : : static regset
3049 : 1075 : compute_live_after_bb (basic_block bb)
3050 : : {
3051 : 1075 : edge e;
3052 : 1075 : edge_iterator ei;
3053 : 1075 : regset lv = get_clear_regset_from_pool ();
3054 : :
3055 : 1075 : gcc_assert (!ignore_first);
3056 : :
3057 : 2680 : FOR_EACH_EDGE (e, ei, bb->succs)
3058 : 1605 : if (sel_bb_empty_p (e->dest))
3059 : : {
3060 : 10 : if (! BB_LV_SET_VALID_P (e->dest))
3061 : : {
3062 : 0 : gcc_unreachable ();
3063 : : gcc_assert (BB_LV_SET (e->dest) == NULL);
3064 : : BB_LV_SET (e->dest) = compute_live_after_bb (e->dest);
3065 : : BB_LV_SET_VALID_P (e->dest) = true;
3066 : : }
3067 : 10 : IOR_REG_SET (lv, BB_LV_SET (e->dest));
3068 : : }
3069 : : else
3070 : 1595 : IOR_REG_SET (lv, compute_live (sel_bb_head (e->dest)));
3071 : :
3072 : 1075 : return lv;
3073 : : }
3074 : :
3075 : : /* Compute the set of all live registers at the point before INSN and save
3076 : : it at INSN if INSN is bb header. */
3077 : : regset
3078 : 13410 : compute_live (insn_t insn)
3079 : : {
3080 : 13410 : basic_block bb = BLOCK_FOR_INSN (insn);
3081 : 13410 : insn_t final, temp;
3082 : 13410 : regset lv;
3083 : :
3084 : : /* Return the valid set if we're already on it. */
3085 : 13410 : if (!ignore_first)
3086 : : {
3087 : 8550 : regset src = NULL;
3088 : :
3089 : 8550 : if (sel_bb_head_p (insn) && BB_LV_SET_VALID_P (bb))
3090 : 4947 : src = BB_LV_SET (bb);
3091 : : else
3092 : : {
3093 : 3603 : gcc_assert (in_current_region_p (bb));
3094 : 3603 : if (INSN_LIVE_VALID_P (insn))
3095 : 3481 : src = INSN_LIVE (insn);
3096 : : }
3097 : :
3098 : 8428 : if (src)
3099 : : {
3100 : 8428 : lv = get_regset_from_pool ();
3101 : 8428 : COPY_REG_SET (lv, src);
3102 : :
3103 : 8428 : if (sel_bb_head_p (insn) && ! BB_LV_SET_VALID_P (bb))
3104 : : {
3105 : 0 : COPY_REG_SET (BB_LV_SET (bb), lv);
3106 : 0 : BB_LV_SET_VALID_P (bb) = true;
3107 : : }
3108 : :
3109 : 8428 : return_regset_to_pool (lv);
3110 : 8428 : return lv;
3111 : : }
3112 : : }
3113 : :
3114 : : /* We've skipped the wrong lv_set. Don't skip the right one. */
3115 : 4982 : ignore_first = false;
3116 : 4982 : gcc_assert (in_current_region_p (bb));
3117 : :
3118 : : /* Find a valid LV set in this block or below, if needed.
3119 : : Start searching from the next insn: either ignore_first is true, or
3120 : : INSN doesn't have a correct live set. */
3121 : 4982 : temp = NEXT_INSN (insn);
3122 : 4982 : final = NEXT_INSN (BB_END (bb));
3123 : 11848 : while (temp != final && ! INSN_LIVE_VALID_P (temp))
3124 : 1884 : temp = NEXT_INSN (temp);
3125 : 4982 : if (temp == final)
3126 : : {
3127 : 1075 : lv = compute_live_after_bb (bb);
3128 : 1075 : temp = PREV_INSN (temp);
3129 : : }
3130 : : else
3131 : : {
3132 : 3907 : lv = get_regset_from_pool ();
3133 : 3907 : COPY_REG_SET (lv, INSN_LIVE (temp));
3134 : : }
3135 : :
3136 : : /* Put correct lv sets on the insns which have bad sets. */
3137 : 4982 : final = PREV_INSN (insn);
3138 : 20737 : while (temp != final)
3139 : : {
3140 : 10773 : propagate_lv_set (lv, temp);
3141 : 10773 : COPY_REG_SET (INSN_LIVE (temp), lv);
3142 : 10773 : INSN_LIVE_VALID_P (temp) = true;
3143 : 10773 : temp = PREV_INSN (temp);
3144 : : }
3145 : :
3146 : : /* Also put it in a BB. */
3147 : 4982 : if (sel_bb_head_p (insn))
3148 : : {
3149 : 961 : basic_block bb = BLOCK_FOR_INSN (insn);
3150 : :
3151 : 961 : COPY_REG_SET (BB_LV_SET (bb), lv);
3152 : 961 : BB_LV_SET_VALID_P (bb) = true;
3153 : : }
3154 : :
3155 : : /* We return LV to the pool, but will not clear it there. Thus we can
3156 : : legimatelly use LV till the next use of regset_pool_get (). */
3157 : 4982 : return_regset_to_pool (lv);
3158 : 4982 : return lv;
3159 : : }
3160 : :
3161 : : /* Update liveness sets for INSN. */
3162 : : static inline void
3163 : 4860 : update_liveness_on_insn (rtx_insn *insn)
3164 : : {
3165 : 4860 : ignore_first = true;
3166 : 4860 : compute_live (insn);
3167 : 23 : }
3168 : :
3169 : : /* Compute liveness below INSN and write it into REGS. */
3170 : : static inline void
3171 : 3745 : compute_live_below_insn (rtx_insn *insn, regset regs)
3172 : : {
3173 : 3745 : rtx_insn *succ;
3174 : 3745 : succ_iterator si;
3175 : :
3176 : 7490 : FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_ALL)
3177 : 3745 : IOR_REG_SET (regs, compute_live (succ));
3178 : 3745 : }
3179 : :
3180 : : /* Update the data gathered in av and lv sets starting from INSN. */
3181 : : static void
3182 : 568 : update_data_sets (rtx_insn *insn)
3183 : : {
3184 : 568 : update_liveness_on_insn (insn);
3185 : 568 : if (sel_bb_head_p (insn))
3186 : : {
3187 : 568 : gcc_assert (AV_LEVEL (insn) != 0);
3188 : 568 : BB_AV_LEVEL (BLOCK_FOR_INSN (insn)) = -1;
3189 : 568 : compute_av_set (insn, NULL, 0, 0);
3190 : : }
3191 : 568 : }
3192 : :
3193 : :
3194 : : /* Helper for move_op () and find_used_regs ().
3195 : : Return speculation type for which a check should be created on the place
3196 : : of INSN. EXPR is one of the original ops we are searching for. */
3197 : : static ds_t
3198 : 4950 : get_spec_check_type_for_insn (insn_t insn, expr_t expr)
3199 : : {
3200 : 4950 : ds_t to_check_ds;
3201 : 4950 : ds_t already_checked_ds = EXPR_SPEC_DONE_DS (INSN_EXPR (insn));
3202 : :
3203 : 4950 : to_check_ds = EXPR_SPEC_TO_CHECK_DS (expr);
3204 : :
3205 : 4950 : if (targetm.sched.get_insn_checked_ds)
3206 : 0 : already_checked_ds |= targetm.sched.get_insn_checked_ds (insn);
3207 : :
3208 : 4950 : if (spec_info != NULL
3209 : 0 : && (spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL))
3210 : 0 : already_checked_ds |= BEGIN_CONTROL;
3211 : :
3212 : 4950 : already_checked_ds = ds_get_speculation_types (already_checked_ds);
3213 : :
3214 : 4950 : to_check_ds &= ~already_checked_ds;
3215 : :
3216 : 4950 : return to_check_ds;
3217 : : }
3218 : :
3219 : : /* Find the set of registers that are unavailable for storing expres
3220 : : while moving ORIG_OPS up on the path starting from INSN due to
3221 : : liveness (USED_REGS) or hardware restrictions (REG_RENAME_P).
3222 : :
3223 : : All the original operations found during the traversal are saved in the
3224 : : ORIGINAL_INSNS list.
3225 : :
3226 : : REG_RENAME_P denotes the set of hardware registers that
3227 : : cannot be used with renaming due to the register class restrictions,
3228 : : mode restrictions and other (the register we'll choose should be
3229 : : compatible class with the original uses, shouldn't be in call_used_regs,
3230 : : should be HARD_REGNO_RENAME_OK etc).
3231 : :
3232 : : Returns TRUE if we've found all original insns, FALSE otherwise.
3233 : :
3234 : : This function utilizes code_motion_path_driver (formerly find_used_regs_1)
3235 : : to traverse the code motion paths. This helper function finds registers
3236 : : that are not available for storing expres while moving ORIG_OPS up on the
3237 : : path starting from INSN. A register considered as used on the moving path,
3238 : : if one of the following conditions is not satisfied:
3239 : :
3240 : : (1) a register not set or read on any path from xi to an instance of
3241 : : the original operation,
3242 : : (2) not among the live registers of the point immediately following the
3243 : : first original operation on a given downward path, except for the
3244 : : original target register of the operation,
3245 : : (3) not live on the other path of any conditional branch that is passed
3246 : : by the operation, in case original operations are not present on
3247 : : both paths of the conditional branch.
3248 : :
3249 : : All the original operations found during the traversal are saved in the
3250 : : ORIGINAL_INSNS list.
3251 : :
3252 : : REG_RENAME_P->CROSSED_CALL_ABIS is true, if there is a call insn on the path
3253 : : from INSN to original insn. In this case CALL_USED_REG_SET will be added
3254 : : to unavailable hard regs at the point original operation is found. */
3255 : :
3256 : : static bool
3257 : 2821 : find_used_regs (insn_t insn, av_set_t orig_ops, regset used_regs,
3258 : : struct reg_rename *reg_rename_p, def_list_t *original_insns)
3259 : : {
3260 : 2821 : def_list_iterator i;
3261 : 2821 : def_t def;
3262 : 2821 : int res;
3263 : 2821 : bool needs_spec_check_p = false;
3264 : 2821 : expr_t expr;
3265 : 2821 : av_set_iterator expr_iter;
3266 : 2821 : struct fur_static_params sparams;
3267 : 2821 : struct cmpd_local_params lparams;
3268 : :
3269 : : /* We haven't visited any blocks yet. */
3270 : 2821 : bitmap_clear (code_motion_visited_blocks);
3271 : :
3272 : : /* Init parameters for code_motion_path_driver. */
3273 : 2821 : sparams.crossed_call_abis = 0;
3274 : 2821 : sparams.original_insns = original_insns;
3275 : 2821 : sparams.used_regs = used_regs;
3276 : :
3277 : : /* Set the appropriate hooks and data. */
3278 : 2821 : code_motion_path_driver_info = &fur_hooks;
3279 : :
3280 : 2821 : res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
3281 : :
3282 : 2821 : reg_rename_p->crossed_call_abis |= sparams.crossed_call_abis;
3283 : :
3284 : 2821 : gcc_assert (res == 1);
3285 : 2821 : gcc_assert (original_insns && *original_insns);
3286 : :
3287 : : /* ??? We calculate whether an expression needs a check when computing
3288 : : av sets. This information is not as precise as it could be due to
3289 : : merging this bit in merge_expr. We can do better in find_used_regs,
3290 : : but we want to avoid multiple traversals of the same code motion
3291 : : paths. */
3292 : 9387 : FOR_EACH_EXPR (expr, expr_iter, orig_ops)
3293 : 2821 : needs_spec_check_p |= EXPR_NEEDS_SPEC_CHECK_P (expr);
3294 : :
3295 : : /* Mark hardware regs in REG_RENAME_P that are not suitable
3296 : : for renaming expr in INSN due to hardware restrictions (register class,
3297 : : modes compatibility etc). */
3298 : 6566 : FOR_EACH_DEF (def, i, *original_insns)
3299 : : {
3300 : 3745 : vinsn_t vinsn = INSN_VINSN (def->orig_insn);
3301 : :
3302 : 3745 : if (VINSN_SEPARABLE_P (vinsn))
3303 : 3361 : mark_unavailable_hard_regs (def, reg_rename_p, used_regs);
3304 : :
3305 : : /* Do not allow clobbering of ld.[sa] address in case some of the
3306 : : original operations need a check. */
3307 : 3745 : if (needs_spec_check_p)
3308 : 0 : IOR_REG_SET (used_regs, VINSN_REG_USES (vinsn));
3309 : : }
3310 : :
3311 : 2821 : return true;
3312 : : }
3313 : :
3314 : :
3315 : : /* Functions to choose the best insn from available ones. */
3316 : :
3317 : : /* Adjusts the priority for EXPR using the backend *_adjust_priority hook. */
3318 : : static int
3319 : 17842 : sel_target_adjust_priority (expr_t expr)
3320 : : {
3321 : 17842 : int priority = EXPR_PRIORITY (expr);
3322 : 17842 : int new_priority;
3323 : :
3324 : 17842 : if (targetm.sched.adjust_priority)
3325 : 17842 : new_priority = targetm.sched.adjust_priority (EXPR_INSN_RTX (expr), priority);
3326 : : else
3327 : : new_priority = priority;
3328 : :
3329 : : /* If the priority has changed, adjust EXPR_PRIORITY_ADJ accordingly. */
3330 : 17842 : EXPR_PRIORITY_ADJ (expr) = new_priority - EXPR_PRIORITY (expr);
3331 : :
3332 : 17842 : if (sched_verbose >= 4)
3333 : 0 : sel_print ("sel_target_adjust_priority: insn %d, %d+%d = %d.\n",
3334 : 0 : INSN_UID (EXPR_INSN_RTX (expr)), EXPR_PRIORITY (expr),
3335 : : EXPR_PRIORITY_ADJ (expr), new_priority);
3336 : :
3337 : 17842 : return new_priority;
3338 : : }
3339 : :
3340 : : /* Rank two available exprs for schedule. Never return 0 here. */
3341 : : static int
3342 : 119120 : sel_rank_for_schedule (const void *x, const void *y)
3343 : : {
3344 : 119120 : expr_t tmp = *(const expr_t *) y;
3345 : 119120 : expr_t tmp2 = *(const expr_t *) x;
3346 : 119120 : insn_t tmp_insn, tmp2_insn;
3347 : 119120 : vinsn_t tmp_vinsn, tmp2_vinsn;
3348 : 119120 : int val;
3349 : :
3350 : 119120 : tmp_vinsn = EXPR_VINSN (tmp);
3351 : 119120 : tmp2_vinsn = EXPR_VINSN (tmp2);
3352 : 119120 : tmp_insn = EXPR_INSN_RTX (tmp);
3353 : 119120 : tmp2_insn = EXPR_INSN_RTX (tmp2);
3354 : :
3355 : : /* Schedule debug insns as early as possible. */
3356 : 119120 : if (DEBUG_INSN_P (tmp_insn) && !DEBUG_INSN_P (tmp2_insn))
3357 : : return -1;
3358 : 119120 : else if (DEBUG_INSN_P (tmp2_insn))
3359 : : return 1;
3360 : :
3361 : : /* Prefer SCHED_GROUP_P insns to any others. */
3362 : 119120 : if (SCHED_GROUP_P (tmp_insn) != SCHED_GROUP_P (tmp2_insn))
3363 : : {
3364 : 1597 : if (VINSN_UNIQUE_P (tmp_vinsn) && VINSN_UNIQUE_P (tmp2_vinsn))
3365 : 0 : return SCHED_GROUP_P (tmp2_insn) ? 1 : -1;
3366 : :
3367 : : /* Now uniqueness means SCHED_GROUP_P is set, because schedule groups
3368 : : cannot be cloned. */
3369 : 1597 : if (VINSN_UNIQUE_P (tmp2_vinsn))
3370 : : return 1;
3371 : : return -1;
3372 : : }
3373 : :
3374 : : /* Discourage scheduling of speculative checks. */
3375 : 117523 : val = (sel_insn_is_speculation_check (tmp_insn)
3376 : 117523 : - sel_insn_is_speculation_check (tmp2_insn));
3377 : 117523 : if (val)
3378 : : return val;
3379 : :
3380 : : /* Prefer not scheduled insn over scheduled one. */
3381 : 117523 : if (EXPR_SCHED_TIMES (tmp) > 0 || EXPR_SCHED_TIMES (tmp2) > 0)
3382 : : {
3383 : 10786 : val = EXPR_SCHED_TIMES (tmp) - EXPR_SCHED_TIMES (tmp2);
3384 : 10786 : if (val)
3385 : : return val;
3386 : : }
3387 : :
3388 : : /* Prefer jump over non-jump instruction. */
3389 : 111343 : if (control_flow_insn_p (tmp_insn) && !control_flow_insn_p (tmp2_insn))
3390 : : return -1;
3391 : 110715 : else if (control_flow_insn_p (tmp2_insn) && !control_flow_insn_p (tmp_insn))
3392 : : return 1;
3393 : :
3394 : : /* Prefer an expr with non-zero usefulness. */
3395 : 110388 : int u1 = EXPR_USEFULNESS (tmp), u2 = EXPR_USEFULNESS (tmp2);
3396 : :
3397 : 110388 : if (u1 == 0)
3398 : : {
3399 : 393 : if (u2 == 0)
3400 : : u1 = u2 = 1;
3401 : : else
3402 : : return 1;
3403 : : }
3404 : 109995 : else if (u2 == 0)
3405 : : return -1;
3406 : :
3407 : : /* Prefer an expr with greater priority. */
3408 : 110061 : val = (u2 * (EXPR_PRIORITY (tmp2) + EXPR_PRIORITY_ADJ (tmp2))
3409 : 110061 : - u1 * (EXPR_PRIORITY (tmp) + EXPR_PRIORITY_ADJ (tmp)));
3410 : 110061 : if (val)
3411 : : return val;
3412 : :
3413 : 36478 : if (spec_info != NULL && spec_info->mask != 0)
3414 : : /* This code was taken from haifa-sched.cc: rank_for_schedule (). */
3415 : : {
3416 : 0 : ds_t ds1, ds2;
3417 : 0 : dw_t dw1, dw2;
3418 : 0 : int dw;
3419 : :
3420 : 0 : ds1 = EXPR_SPEC_DONE_DS (tmp);
3421 : 0 : if (ds1)
3422 : 0 : dw1 = ds_weak (ds1);
3423 : : else
3424 : : dw1 = NO_DEP_WEAK;
3425 : :
3426 : 0 : ds2 = EXPR_SPEC_DONE_DS (tmp2);
3427 : 0 : if (ds2)
3428 : 0 : dw2 = ds_weak (ds2);
3429 : : else
3430 : : dw2 = NO_DEP_WEAK;
3431 : :
3432 : 0 : dw = dw2 - dw1;
3433 : 0 : if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
3434 : : return dw;
3435 : : }
3436 : :
3437 : : /* Prefer an old insn to a bookkeeping insn. */
3438 : 36478 : if (INSN_UID (tmp_insn) < first_emitted_uid
3439 : 36478 : && INSN_UID (tmp2_insn) >= first_emitted_uid)
3440 : : return -1;
3441 : 36047 : if (INSN_UID (tmp_insn) >= first_emitted_uid
3442 : 36047 : && INSN_UID (tmp2_insn) < first_emitted_uid)
3443 : : return 1;
3444 : :
3445 : : /* Prefer an insn with smaller UID, as a last resort.
3446 : : We can't safely use INSN_LUID as it is defined only for those insns
3447 : : that are in the stream. */
3448 : 35778 : return INSN_UID (tmp_insn) - INSN_UID (tmp2_insn);
3449 : : }
3450 : :
3451 : : /* Filter out expressions from av set pointed to by AV_PTR
3452 : : that are pipelined too many times. */
3453 : : static void
3454 : 7718 : process_pipelined_exprs (av_set_t *av_ptr)
3455 : : {
3456 : 7718 : expr_t expr;
3457 : 7718 : av_set_iterator si;
3458 : :
3459 : : /* Don't pipeline already pipelined code as that would increase
3460 : : number of unnecessary register moves. */
3461 : 44032 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3462 : : {
3463 : 18157 : if (EXPR_SCHED_TIMES (expr)
3464 : 18157 : >= param_selsched_max_sched_times)
3465 : 72 : av_set_iter_remove (&si);
3466 : : }
3467 : 7718 : }
3468 : :
3469 : : /* Filter speculative insns from AV_PTR if we don't want them. */
3470 : : static void
3471 : 7718 : process_spec_exprs (av_set_t *av_ptr)
3472 : : {
3473 : 7718 : expr_t expr;
3474 : 7718 : av_set_iterator si;
3475 : :
3476 : 7718 : if (spec_info == NULL)
3477 : 7718 : return;
3478 : :
3479 : : /* Scan *AV_PTR to find out if we want to consider speculative
3480 : : instructions for scheduling. */
3481 : 0 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3482 : : {
3483 : 0 : ds_t ds;
3484 : :
3485 : 0 : ds = EXPR_SPEC_DONE_DS (expr);
3486 : :
3487 : : /* The probability of a success is too low - don't speculate. */
3488 : 0 : if ((ds & SPECULATIVE)
3489 : 0 : && (ds_weak (ds) < spec_info->data_weakness_cutoff
3490 : 0 : || EXPR_USEFULNESS (expr) < spec_info->control_weakness_cutoff
3491 : 0 : || (pipelining_p && false
3492 : : && (ds & DATA_SPEC)
3493 : : && (ds & CONTROL_SPEC))))
3494 : : {
3495 : 0 : av_set_iter_remove (&si);
3496 : 0 : continue;
3497 : : }
3498 : : }
3499 : : }
3500 : :
3501 : : /* Search for any use-like insns in AV_PTR and decide on scheduling
3502 : : them. Return one when found, and NULL otherwise.
3503 : : Note that we check here whether a USE could be scheduled to avoid
3504 : : an infinite loop later. */
3505 : : static expr_t
3506 : 7718 : process_use_exprs (av_set_t *av_ptr)
3507 : : {
3508 : 7718 : expr_t expr;
3509 : 7718 : av_set_iterator si;
3510 : 7718 : bool uses_present_p = false;
3511 : 7718 : bool try_uses_p = true;
3512 : :
3513 : 43510 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3514 : : {
3515 : : /* This will also initialize INSN_CODE for later use. */
3516 : 18005 : if (recog_memoized (EXPR_INSN_RTX (expr)) < 0)
3517 : : {
3518 : : /* If we have a USE in *AV_PTR that was not scheduled yet,
3519 : : do so because it will do good only. */
3520 : 130 : if (EXPR_SCHED_TIMES (expr) <= 0)
3521 : : {
3522 : 127 : if (EXPR_TARGET_AVAILABLE (expr) == 1)
3523 : : return expr;
3524 : :
3525 : 18 : av_set_iter_remove (&si);
3526 : : }
3527 : : else
3528 : : {
3529 : 3 : gcc_assert (pipelining_p);
3530 : :
3531 : : uses_present_p = true;
3532 : : }
3533 : : }
3534 : : else
3535 : : try_uses_p = false;
3536 : : }
3537 : :
3538 : 7609 : if (uses_present_p)
3539 : : {
3540 : : /* If we don't want to schedule any USEs right now and we have some
3541 : : in *AV_PTR, remove them, else just return the first one found. */
3542 : 3 : if (!try_uses_p)
3543 : : {
3544 : 31 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3545 : 14 : if (INSN_CODE (EXPR_INSN_RTX (expr)) < 0)
3546 : 3 : av_set_iter_remove (&si);
3547 : : }
3548 : : else
3549 : : {
3550 : 0 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3551 : : {
3552 : 0 : gcc_assert (INSN_CODE (EXPR_INSN_RTX (expr)) < 0);
3553 : :
3554 : 0 : if (EXPR_TARGET_AVAILABLE (expr) == 1)
3555 : : return expr;
3556 : :
3557 : 0 : av_set_iter_remove (&si);
3558 : : }
3559 : : }
3560 : : }
3561 : :
3562 : : return NULL;
3563 : : }
3564 : :
3565 : : /* Lookup EXPR in VINSN_VEC and return TRUE if found. Also check patterns from
3566 : : EXPR's history of changes. */
3567 : : static bool
3568 : 35684 : vinsn_vec_has_expr_p (vinsn_vec_t vinsn_vec, expr_t expr)
3569 : : {
3570 : 35684 : vinsn_t vinsn, expr_vinsn;
3571 : 35684 : int n;
3572 : 35684 : unsigned i;
3573 : :
3574 : : /* Start with checking expr itself and then proceed with all the old forms
3575 : : of expr taken from its history vector. */
3576 : 35684 : for (i = 0, expr_vinsn = EXPR_VINSN (expr);
3577 : 72069 : expr_vinsn;
3578 : 72069 : expr_vinsn = (i < EXPR_HISTORY_OF_CHANGES (expr).length ()
3579 : 1464 : ? EXPR_HISTORY_OF_CHANGES (expr)[i++].old_expr_vinsn
3580 : : : NULL))
3581 : 41547 : FOR_EACH_VEC_ELT (vinsn_vec, n, vinsn)
3582 : 2555 : if (VINSN_SEPARABLE_P (vinsn))
3583 : : {
3584 : 2403 : if (vinsn_equal_p (vinsn, expr_vinsn))
3585 : : return true;
3586 : : }
3587 : : else
3588 : : {
3589 : : /* For non-separable instructions, the blocking insn can have
3590 : : another pattern due to substitution, and we can't choose
3591 : : different register as in the above case. Check all registers
3592 : : being written instead. */
3593 : 152 : if (bitmap_intersect_p (VINSN_REG_SETS (vinsn),
3594 : 152 : VINSN_REG_SETS (expr_vinsn)))
3595 : : return true;
3596 : : }
3597 : :
3598 : : return false;
3599 : : }
3600 : :
3601 : : /* Return true if either of expressions from ORIG_OPS can be blocked
3602 : : by previously created bookkeeping code. STATIC_PARAMS points to static
3603 : : parameters of move_op. */
3604 : : static bool
3605 : 0 : av_set_could_be_blocked_by_bookkeeping_p (av_set_t orig_ops, void *static_params)
3606 : : {
3607 : 0 : expr_t expr;
3608 : 0 : av_set_iterator iter;
3609 : 0 : moveop_static_params_p sparams;
3610 : :
3611 : : /* This checks that expressions in ORIG_OPS are not blocked by bookkeeping
3612 : : created while scheduling on another fence. */
3613 : 0 : FOR_EACH_EXPR (expr, iter, orig_ops)
3614 : 0 : if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr))
3615 : : return true;
3616 : :
3617 : 0 : gcc_assert (code_motion_path_driver_info == &move_op_hooks);
3618 : 0 : sparams = (moveop_static_params_p) static_params;
3619 : :
3620 : : /* Expressions can be also blocked by bookkeeping created during current
3621 : : move_op. */
3622 : 0 : if (bitmap_bit_p (current_copies, INSN_UID (sparams->failed_insn)))
3623 : 0 : FOR_EACH_EXPR (expr, iter, orig_ops)
3624 : 0 : if (moveup_expr_cached (expr, sparams->failed_insn, false) != MOVEUP_EXPR_NULL)
3625 : : return true;
3626 : :
3627 : : /* Expressions in ORIG_OPS may have wrong destination register due to
3628 : : renaming. Check with the right register instead. */
3629 : 0 : if (sparams->dest && REG_P (sparams->dest))
3630 : : {
3631 : 0 : rtx reg = sparams->dest;
3632 : 0 : vinsn_t failed_vinsn = INSN_VINSN (sparams->failed_insn);
3633 : :
3634 : 0 : if (register_unavailable_p (VINSN_REG_SETS (failed_vinsn), reg)
3635 : 0 : || register_unavailable_p (VINSN_REG_USES (failed_vinsn), reg)
3636 : 0 : || register_unavailable_p (VINSN_REG_CLOBBERS (failed_vinsn), reg))
3637 : 0 : return true;
3638 : : }
3639 : :
3640 : : return false;
3641 : : }
3642 : :
3643 : : /* Clear VINSN_VEC and detach vinsns. */
3644 : : static void
3645 : 3502 : vinsn_vec_clear (vinsn_vec_t *vinsn_vec)
3646 : : {
3647 : 3502 : unsigned len = vinsn_vec->length ();
3648 : 198 : if (len > 0)
3649 : : {
3650 : : vinsn_t vinsn;
3651 : : int n;
3652 : :
3653 : 86 : FOR_EACH_VEC_ELT (*vinsn_vec, n, vinsn)
3654 : 60 : vinsn_detach (vinsn);
3655 : 26 : vinsn_vec->block_remove (0, len);
3656 : : }
3657 : 3502 : }
3658 : :
3659 : : /* Add the vinsn of EXPR to the VINSN_VEC. */
3660 : : static void
3661 : 60 : vinsn_vec_add (vinsn_vec_t *vinsn_vec, expr_t expr)
3662 : : {
3663 : 60 : vinsn_attach (EXPR_VINSN (expr));
3664 : 60 : vinsn_vec->safe_push (EXPR_VINSN (expr));
3665 : 60 : }
3666 : :
3667 : : /* Free the vector representing blocked expressions. */
3668 : : static void
3669 : 1564 : vinsn_vec_free (vinsn_vec_t &vinsn_vec)
3670 : : {
3671 : 0 : vinsn_vec.release ();
3672 : 0 : }
3673 : :
3674 : : /* Increase EXPR_PRIORITY_ADJ for INSN by AMOUNT. */
3675 : :
3676 : 0 : void sel_add_to_insn_priority (rtx insn, int amount)
3677 : : {
3678 : 0 : EXPR_PRIORITY_ADJ (INSN_EXPR (insn)) += amount;
3679 : :
3680 : 0 : if (sched_verbose >= 2)
3681 : 0 : sel_print ("sel_add_to_insn_priority: insn %d, by %d (now %d+%d).\n",
3682 : 0 : INSN_UID (insn), amount, EXPR_PRIORITY (INSN_EXPR (insn)),
3683 : 0 : EXPR_PRIORITY_ADJ (INSN_EXPR (insn)));
3684 : 0 : }
3685 : :
3686 : : /* Turn AV into a vector, filter inappropriate insns and sort it. Return
3687 : : true if there is something to schedule. BNDS and FENCE are current
3688 : : boundaries and fence, respectively. If we need to stall for some cycles
3689 : : before an expr from AV would become available, write this number to
3690 : : *PNEED_STALL. */
3691 : : static bool
3692 : 7609 : fill_vec_av_set (av_set_t av, blist_t bnds, fence_t fence,
3693 : : int *pneed_stall)
3694 : : {
3695 : 7609 : av_set_iterator si;
3696 : 7609 : expr_t expr;
3697 : 7609 : int sched_next_worked = 0, stalled, n;
3698 : 7609 : static int av_max_prio, est_ticks_till_branch;
3699 : 7609 : int min_need_stall = -1;
3700 : 7609 : deps_t dc = BND_DC (BLIST_BND (bnds));
3701 : :
3702 : : /* Bail out early when the ready list contained only USEs/CLOBBERs that are
3703 : : already scheduled. */
3704 : 7609 : if (av == NULL)
3705 : : return false;
3706 : :
3707 : : /* Empty vector from the previous stuff. */
3708 : 7609 : if (vec_av_set.length () > 0)
3709 : 5222 : vec_av_set.block_remove (0, vec_av_set.length ());
3710 : :
3711 : : /* Turn the set into a vector for sorting and call sel_target_adjust_priority
3712 : : for each insn. */
3713 : 33060 : gcc_assert (vec_av_set.is_empty ());
3714 : 25451 : FOR_EACH_EXPR (expr, si, av)
3715 : : {
3716 : 17842 : vec_av_set.safe_push (expr);
3717 : :
3718 : 17842 : gcc_assert (EXPR_PRIORITY_ADJ (expr) == 0 || *pneed_stall);
3719 : :
3720 : : /* Adjust priority using target backend hook. */
3721 : 17842 : sel_target_adjust_priority (expr);
3722 : : }
3723 : :
3724 : : /* Sort the vector. */
3725 : 7609 : vec_av_set.qsort (sel_rank_for_schedule);
3726 : :
3727 : : /* We record maximal priority of insns in av set for current instruction
3728 : : group. */
3729 : 7609 : if (FENCE_STARTS_CYCLE_P (fence))
3730 : 3732 : av_max_prio = est_ticks_till_branch = INT_MIN;
3731 : :
3732 : : /* Filter out inappropriate expressions. Loop's direction is reversed to
3733 : : visit "best" instructions first. We assume that vec::unordered_remove
3734 : : moves last element in place of one being deleted. */
3735 : 33060 : for (n = vec_av_set.length () - 1, stalled = 0; n >= 0; n--)
3736 : : {
3737 : 17842 : expr_t expr = vec_av_set[n];
3738 : 17842 : insn_t insn = EXPR_INSN_RTX (expr);
3739 : 17842 : signed char target_available;
3740 : 17842 : bool is_orig_reg_p = true;
3741 : 17842 : int need_cycles, new_prio;
3742 : 17842 : bool fence_insn_p = INSN_UID (insn) == INSN_UID (FENCE_INSN (fence));
3743 : :
3744 : : /* Don't allow any insns other than from SCHED_GROUP if we have one. */
3745 : 17842 : if (FENCE_SCHED_NEXT (fence) && insn != FENCE_SCHED_NEXT (fence))
3746 : : {
3747 : 0 : vec_av_set.unordered_remove (n);
3748 : 7933 : continue;
3749 : : }
3750 : :
3751 : : /* Set number of sched_next insns (just in case there
3752 : : could be several). */
3753 : 17842 : if (FENCE_SCHED_NEXT (fence))
3754 : 0 : sched_next_worked++;
3755 : :
3756 : : /* Check all liveness requirements and try renaming.
3757 : : FIXME: try to minimize calls to this. */
3758 : 17842 : target_available = EXPR_TARGET_AVAILABLE (expr);
3759 : :
3760 : : /* If insn was already scheduled on the current fence,
3761 : : set TARGET_AVAILABLE to -1 no matter what expr's attribute says. */
3762 : 17842 : if (vinsn_vec_has_expr_p (vec_target_unavailable_vinsns, expr)
3763 : 17842 : && !fence_insn_p)
3764 : 17842 : target_available = -1;
3765 : :
3766 : : /* If the availability of the EXPR is invalidated by the insertion of
3767 : : bookkeeping earlier, make sure that we won't choose this expr for
3768 : : scheduling if it's not separable, and if it is separable, then
3769 : : we have to recompute the set of available registers for it. */
3770 : 17842 : if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr))
3771 : : {
3772 : 14 : vec_av_set.unordered_remove (n);
3773 : 14 : if (sched_verbose >= 4)
3774 : 0 : sel_print ("Expr %d is blocked by bookkeeping inserted earlier\n",
3775 : 0 : INSN_UID (insn));
3776 : 14 : continue;
3777 : : }
3778 : :
3779 : 17828 : if (target_available == true)
3780 : : {
3781 : : /* Do nothing -- we can use an existing register. */
3782 : 13339 : is_orig_reg_p = EXPR_SEPARABLE_P (expr);
3783 : : }
3784 : 8805 : else if (/* Non-separable instruction will never
3785 : : get another register. */
3786 : : (target_available == false
3787 : 4259 : && !EXPR_SEPARABLE_P (expr))
3788 : : /* Don't try to find a register for low-priority expression. */
3789 : 4135 : || (int) vec_av_set.length () - 1 - n >= max_insns_to_rename
3790 : : /* ??? FIXME: Don't try to rename data speculation. */
3791 : 2821 : || (EXPR_SPEC_DONE_DS (expr) & BEGIN_DATA)
3792 : 7310 : || ! find_best_reg_for_expr (expr, bnds, &is_orig_reg_p))
3793 : : {
3794 : 4316 : vec_av_set.unordered_remove (n);
3795 : 4316 : if (sched_verbose >= 4)
3796 : 0 : sel_print ("Expr %d has no suitable target register\n",
3797 : 0 : INSN_UID (insn));
3798 : :
3799 : : /* A fence insn should not get here. */
3800 : 4316 : gcc_assert (!fence_insn_p);
3801 : 4316 : continue;
3802 : : }
3803 : :
3804 : : /* At this point a fence insn should always be available. */
3805 : 13512 : gcc_assert (!fence_insn_p
3806 : : || INSN_UID (FENCE_INSN (fence)) == INSN_UID (EXPR_INSN_RTX (expr)));
3807 : :
3808 : : /* Filter expressions that need to be renamed or speculated when
3809 : : pipelining, because compensating register copies or speculation
3810 : : checks are likely to be placed near the beginning of the loop,
3811 : : causing a stall. */
3812 : 13512 : if (pipelining_p && EXPR_ORIG_SCHED_CYCLE (expr) > 0
3813 : 315 : && (!is_orig_reg_p || EXPR_SPEC_DONE_DS (expr) != 0))
3814 : : {
3815 : : /* Estimation of number of cycles until loop branch for
3816 : : renaming/speculation to be successful. */
3817 : 160 : int need_n_ticks_till_branch = sel_vinsn_cost (EXPR_VINSN (expr));
3818 : :
3819 : 160 : if ((int) current_loop_nest->ninsns < 9)
3820 : : {
3821 : 13 : vec_av_set.unordered_remove (n);
3822 : 13 : if (sched_verbose >= 4)
3823 : 0 : sel_print ("Pipelining expr %d will likely cause stall\n",
3824 : 0 : INSN_UID (insn));
3825 : 13 : continue;
3826 : : }
3827 : :
3828 : 147 : if ((int) current_loop_nest->ninsns - num_insns_scheduled
3829 : 147 : < need_n_ticks_till_branch * issue_rate / 2
3830 : 26 : && est_ticks_till_branch < need_n_ticks_till_branch)
3831 : : {
3832 : 13 : vec_av_set.unordered_remove (n);
3833 : 13 : if (sched_verbose >= 4)
3834 : 0 : sel_print ("Pipelining expr %d will likely cause stall\n",
3835 : 0 : INSN_UID (insn));
3836 : 13 : continue;
3837 : : }
3838 : : }
3839 : :
3840 : : /* We want to schedule speculation checks as late as possible. Discard
3841 : : them from av set if there are instructions with higher priority. */
3842 : 13486 : if (sel_insn_is_speculation_check (insn)
3843 : 13486 : && EXPR_PRIORITY (expr) < av_max_prio)
3844 : : {
3845 : 0 : stalled++;
3846 : 0 : min_need_stall = min_need_stall < 0 ? 1 : MIN (min_need_stall, 1);
3847 : 0 : vec_av_set.unordered_remove (n);
3848 : 0 : if (sched_verbose >= 4)
3849 : 0 : sel_print ("Delaying speculation check %d until its first use\n",
3850 : 0 : INSN_UID (insn));
3851 : 0 : continue;
3852 : : }
3853 : :
3854 : : /* Ignore EXPRs available from pipelining to update AV_MAX_PRIO. */
3855 : 13486 : if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3856 : 11985 : av_max_prio = MAX (av_max_prio, EXPR_PRIORITY (expr));
3857 : :
3858 : : /* Don't allow any insns whose data is not yet ready.
3859 : : Check first whether we've already tried them and failed. */
3860 : 13486 : if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
3861 : : {
3862 : 13222 : need_cycles = (FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3863 : 13222 : - FENCE_CYCLE (fence));
3864 : 13222 : if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3865 : 11738 : est_ticks_till_branch = MAX (est_ticks_till_branch,
3866 : : EXPR_PRIORITY (expr) + need_cycles);
3867 : :
3868 : 13222 : if (need_cycles > 0)
3869 : : {
3870 : 1636 : stalled++;
3871 : 3272 : min_need_stall = (min_need_stall < 0
3872 : 1636 : ? need_cycles
3873 : 797 : : MIN (min_need_stall, need_cycles));
3874 : 1636 : vec_av_set.unordered_remove (n);
3875 : :
3876 : 1636 : if (sched_verbose >= 4)
3877 : 0 : sel_print ("Expr %d is not ready until cycle %d (cached)\n",
3878 : : INSN_UID (insn),
3879 : 0 : FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3880 : 1636 : continue;
3881 : : }
3882 : : }
3883 : :
3884 : : /* Now resort to dependence analysis to find whether EXPR might be
3885 : : stalled due to dependencies from FENCE's context. */
3886 : 11850 : need_cycles = tick_check_p (expr, dc, fence);
3887 : 11850 : new_prio = EXPR_PRIORITY (expr) + EXPR_PRIORITY_ADJ (expr) + need_cycles;
3888 : :
3889 : 11850 : if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3890 : 10442 : est_ticks_till_branch = MAX (est_ticks_till_branch,
3891 : : new_prio);
3892 : :
3893 : 11850 : if (need_cycles > 0)
3894 : : {
3895 : 1941 : if (INSN_UID (insn) >= FENCE_READY_TICKS_SIZE (fence))
3896 : : {
3897 : 71 : int new_size = INSN_UID (insn) * 3 / 2;
3898 : :
3899 : 71 : FENCE_READY_TICKS (fence)
3900 : 71 : = (int *) xrecalloc (FENCE_READY_TICKS (fence),
3901 : : new_size, FENCE_READY_TICKS_SIZE (fence),
3902 : : sizeof (int));
3903 : : }
3904 : 3882 : FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3905 : 1941 : = FENCE_CYCLE (fence) + need_cycles;
3906 : :
3907 : 1941 : stalled++;
3908 : 3882 : min_need_stall = (min_need_stall < 0
3909 : 1941 : ? need_cycles
3910 : 533 : : MIN (min_need_stall, need_cycles));
3911 : :
3912 : 1941 : vec_av_set.unordered_remove (n);
3913 : :
3914 : 1941 : if (sched_verbose >= 4)
3915 : 0 : sel_print ("Expr %d is not ready yet until cycle %d\n",
3916 : : INSN_UID (insn),
3917 : 0 : FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3918 : 1941 : continue;
3919 : 1941 : }
3920 : :
3921 : 9909 : if (sched_verbose >= 4)
3922 : 0 : sel_print ("Expr %d is ok\n", INSN_UID (insn));
3923 : 9909 : min_need_stall = 0;
3924 : : }
3925 : :
3926 : : /* Clear SCHED_NEXT. */
3927 : 7609 : if (FENCE_SCHED_NEXT (fence))
3928 : : {
3929 : 0 : gcc_assert (sched_next_worked == 1);
3930 : 0 : FENCE_SCHED_NEXT (fence) = NULL;
3931 : : }
3932 : :
3933 : : /* No need to stall if this variable was not initialized. */
3934 : 7609 : if (min_need_stall < 0)
3935 : : min_need_stall = 0;
3936 : :
3937 : 7609 : if (vec_av_set.is_empty ())
3938 : : {
3939 : : /* We need to set *pneed_stall here, because later we skip this code
3940 : : when ready list is empty. */
3941 : 1605 : *pneed_stall = min_need_stall;
3942 : 1605 : return false;
3943 : : }
3944 : : else
3945 : 6004 : gcc_assert (min_need_stall == 0);
3946 : :
3947 : : /* Sort the vector. */
3948 : 6004 : vec_av_set.qsort (sel_rank_for_schedule);
3949 : :
3950 : 6004 : if (sched_verbose >= 4)
3951 : : {
3952 : 0 : sel_print ("Total ready exprs: %d, stalled: %d\n",
3953 : : vec_av_set.length (), stalled);
3954 : 0 : sel_print ("Sorted av set (%d): ", vec_av_set.length ());
3955 : 0 : FOR_EACH_VEC_ELT (vec_av_set, n, expr)
3956 : 0 : dump_expr (expr);
3957 : 0 : sel_print ("\n");
3958 : : }
3959 : :
3960 : 6004 : *pneed_stall = 0;
3961 : 6004 : return true;
3962 : : }
3963 : :
3964 : : /* Convert a vectored and sorted av set to the ready list that
3965 : : the rest of the backend wants to see. */
3966 : : static void
3967 : 6004 : convert_vec_av_set_to_ready (void)
3968 : : {
3969 : 6004 : int n;
3970 : 6004 : expr_t expr;
3971 : :
3972 : : /* Allocate and fill the ready list from the sorted vector. */
3973 : 6004 : ready.n_ready = vec_av_set.length ();
3974 : 6004 : ready.first = ready.n_ready - 1;
3975 : :
3976 : 6004 : gcc_assert (ready.n_ready > 0);
3977 : :
3978 : 6004 : if (ready.n_ready > max_issue_size)
3979 : : {
3980 : 889 : max_issue_size = ready.n_ready;
3981 : 889 : sched_extend_ready_list (ready.n_ready);
3982 : : }
3983 : :
3984 : 15913 : FOR_EACH_VEC_ELT (vec_av_set, n, expr)
3985 : : {
3986 : 9909 : vinsn_t vi = EXPR_VINSN (expr);
3987 : 9909 : insn_t insn = VINSN_INSN_RTX (vi);
3988 : :
3989 : 9909 : ready_try[n] = 0;
3990 : 9909 : ready.vec[n] = insn;
3991 : : }
3992 : 6004 : }
3993 : :
3994 : : /* Initialize ready list from *AV_PTR for the max_issue () call.
3995 : : If any unrecognizable insn found in *AV_PTR, return it (and skip
3996 : : max_issue). BND and FENCE are current boundary and fence,
3997 : : respectively. If we need to stall for some cycles before an expr
3998 : : from *AV_PTR would become available, write this number to *PNEED_STALL. */
3999 : : static expr_t
4000 : 7718 : fill_ready_list (av_set_t *av_ptr, blist_t bnds, fence_t fence,
4001 : : int *pneed_stall)
4002 : : {
4003 : 7718 : expr_t expr;
4004 : :
4005 : : /* We do not support multiple boundaries per fence. */
4006 : 7718 : gcc_assert (BLIST_NEXT (bnds) == NULL);
4007 : :
4008 : : /* Process expressions required special handling, i.e. pipelined,
4009 : : speculative and recog() < 0 expressions first. */
4010 : 7718 : process_pipelined_exprs (av_ptr);
4011 : 7718 : process_spec_exprs (av_ptr);
4012 : :
4013 : : /* A USE could be scheduled immediately. */
4014 : 7718 : expr = process_use_exprs (av_ptr);
4015 : 7718 : if (expr)
4016 : : {
4017 : 109 : *pneed_stall = 0;
4018 : 109 : return expr;
4019 : : }
4020 : :
4021 : : /* Turn the av set to a vector for sorting. */
4022 : 7609 : if (! fill_vec_av_set (*av_ptr, bnds, fence, pneed_stall))
4023 : : {
4024 : 1605 : ready.n_ready = 0;
4025 : 1605 : return NULL;
4026 : : }
4027 : :
4028 : : /* Build the final ready list. */
4029 : 6004 : convert_vec_av_set_to_ready ();
4030 : 6004 : return NULL;
4031 : : }
4032 : :
4033 : : /* Wrapper for dfa_new_cycle (). Returns TRUE if cycle was advanced. */
4034 : : static bool
4035 : 4883 : sel_dfa_new_cycle (insn_t insn, fence_t fence)
4036 : : {
4037 : 4883 : int last_scheduled_cycle = FENCE_LAST_SCHEDULED_INSN (fence)
4038 : 4883 : ? INSN_SCHED_CYCLE (FENCE_LAST_SCHEDULED_INSN (fence))
4039 : 902 : : FENCE_CYCLE (fence) - 1;
4040 : 4883 : bool res = false;
4041 : 4883 : int sort_p = 0;
4042 : :
4043 : 4883 : if (!targetm.sched.dfa_new_cycle)
4044 : : return false;
4045 : :
4046 : 0 : memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4047 : :
4048 : 0 : while (!sort_p && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
4049 : : insn, last_scheduled_cycle,
4050 : : FENCE_CYCLE (fence), &sort_p))
4051 : : {
4052 : 0 : memcpy (FENCE_STATE (fence), curr_state, dfa_state_size);
4053 : 0 : advance_one_cycle (fence);
4054 : 0 : memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4055 : 0 : res = true;
4056 : : }
4057 : :
4058 : : return res;
4059 : : }
4060 : :
4061 : : /* Invoke reorder* target hooks on the ready list. Return the number of insns
4062 : : we can issue. FENCE is the current fence. */
4063 : : static int
4064 : 6004 : invoke_reorder_hooks (fence_t fence)
4065 : : {
4066 : 6004 : int issue_more;
4067 : 6004 : bool ran_hook = false;
4068 : :
4069 : : /* Call the reorder hook at the beginning of the cycle, and call
4070 : : the reorder2 hook in the middle of the cycle. */
4071 : 6004 : if (FENCE_ISSUED_INSNS (fence) == 0)
4072 : : {
4073 : 3514 : if (targetm.sched.reorder
4074 : 3514 : && !SCHED_GROUP_P (ready_element (&ready, 0))
4075 : 6669 : && ready.n_ready > 1)
4076 : : {
4077 : : /* Don't give reorder the most prioritized insn as it can break
4078 : : pipelining. */
4079 : 878 : if (pipelining_p)
4080 : 282 : --ready.n_ready;
4081 : :
4082 : 878 : issue_more
4083 : 878 : = targetm.sched.reorder (sched_dump, sched_verbose,
4084 : : ready_lastpos (&ready),
4085 : : &ready.n_ready, FENCE_CYCLE (fence));
4086 : :
4087 : 878 : if (pipelining_p)
4088 : 282 : ++ready.n_ready;
4089 : :
4090 : : ran_hook = true;
4091 : : }
4092 : : else
4093 : : /* Initialize can_issue_more for variable_issue. */
4094 : 2636 : issue_more = issue_rate;
4095 : : }
4096 : 2490 : else if (targetm.sched.reorder2
4097 : 2490 : && !SCHED_GROUP_P (ready_element (&ready, 0)))
4098 : : {
4099 : 0 : if (ready.n_ready == 1)
4100 : 0 : issue_more =
4101 : 0 : targetm.sched.reorder2 (sched_dump, sched_verbose,
4102 : : ready_lastpos (&ready),
4103 : : &ready.n_ready, FENCE_CYCLE (fence));
4104 : : else
4105 : : {
4106 : 0 : if (pipelining_p)
4107 : 0 : --ready.n_ready;
4108 : :
4109 : 0 : issue_more =
4110 : 0 : targetm.sched.reorder2 (sched_dump, sched_verbose,
4111 : 0 : ready.n_ready
4112 : 0 : ? ready_lastpos (&ready) : NULL,
4113 : : &ready.n_ready, FENCE_CYCLE (fence));
4114 : :
4115 : 0 : if (pipelining_p)
4116 : 0 : ++ready.n_ready;
4117 : : }
4118 : :
4119 : : ran_hook = true;
4120 : : }
4121 : : else
4122 : 2490 : issue_more = FENCE_ISSUE_MORE (fence);
4123 : :
4124 : : /* Ensure that ready list and vec_av_set are in line with each other,
4125 : : i.e. vec_av_set[i] == ready_element (&ready, i). */
4126 : 6004 : if (issue_more && ran_hook)
4127 : : {
4128 : 878 : int i, j, n;
4129 : 878 : rtx_insn **arr = ready.vec;
4130 : 878 : expr_t *vec = vec_av_set.address ();
4131 : :
4132 : 3496 : for (i = 0, n = ready.n_ready; i < n; i++)
4133 : 2618 : if (EXPR_INSN_RTX (vec[i]) != arr[i])
4134 : : {
4135 : 0 : for (j = i; j < n; j++)
4136 : 0 : if (EXPR_INSN_RTX (vec[j]) == arr[i])
4137 : : break;
4138 : 0 : gcc_assert (j < n);
4139 : :
4140 : 0 : std::swap (vec[i], vec[j]);
4141 : : }
4142 : : }
4143 : :
4144 : 6004 : return issue_more;
4145 : : }
4146 : :
4147 : : /* Return an EXPR corresponding to INDEX element of ready list, if
4148 : : FOLLOW_READY_ELEMENT is true (i.e., an expr of
4149 : : ready_element (&ready, INDEX) will be returned), and to INDEX element of
4150 : : ready.vec otherwise. */
4151 : : static inline expr_t
4152 : 27532 : find_expr_for_ready (int index, bool follow_ready_element)
4153 : : {
4154 : 27532 : expr_t expr;
4155 : 27532 : int real_index;
4156 : :
4157 : 27532 : real_index = follow_ready_element ? ready.first - index : index;
4158 : :
4159 : 27532 : expr = vec_av_set[real_index];
4160 : 27532 : gcc_assert (ready.vec[real_index] == EXPR_INSN_RTX (expr));
4161 : :
4162 : 27532 : return expr;
4163 : : }
4164 : :
4165 : : /* Calculate insns worth trying via lookahead_guard hook. Return a number
4166 : : of such insns found. */
4167 : : static int
4168 : 4903 : invoke_dfa_lookahead_guard (void)
4169 : : {
4170 : 4903 : int i, n;
4171 : 4903 : bool have_hook
4172 : 4903 : = targetm.sched.first_cycle_multipass_dfa_lookahead_guard != NULL;
4173 : :
4174 : 4903 : if (sched_verbose >= 2)
4175 : 0 : sel_print ("ready after reorder: ");
4176 : :
4177 : 13113 : for (i = 0, n = 0; i < ready.n_ready; i++)
4178 : : {
4179 : 8210 : expr_t expr;
4180 : 8210 : insn_t insn;
4181 : 8210 : int r;
4182 : :
4183 : : /* In this loop insn is Ith element of the ready list given by
4184 : : ready_element, not Ith element of ready.vec. */
4185 : 8210 : insn = ready_element (&ready, i);
4186 : :
4187 : 8210 : if (! have_hook || i == 0)
4188 : : r = 0;
4189 : : else
4190 : 0 : r = targetm.sched.first_cycle_multipass_dfa_lookahead_guard (insn, i);
4191 : :
4192 : 8210 : gcc_assert (INSN_CODE (insn) >= 0);
4193 : :
4194 : : /* Only insns with ready_try = 0 can get here
4195 : : from fill_ready_list. */
4196 : 8210 : gcc_assert (ready_try [i] == 0);
4197 : 8210 : ready_try[i] = r;
4198 : 8210 : if (!r)
4199 : 8210 : n++;
4200 : :
4201 : 8210 : expr = find_expr_for_ready (i, true);
4202 : :
4203 : 8210 : if (sched_verbose >= 2)
4204 : : {
4205 : 0 : dump_vinsn (EXPR_VINSN (expr));
4206 : 0 : sel_print (":%d; ", ready_try[i]);
4207 : : }
4208 : : }
4209 : :
4210 : 4903 : if (sched_verbose >= 2)
4211 : 0 : sel_print ("\n");
4212 : 4903 : return n;
4213 : : }
4214 : :
4215 : : /* Calculate the number of privileged insns and return it. */
4216 : : static int
4217 : 4903 : calculate_privileged_insns (void)
4218 : : {
4219 : 4903 : expr_t cur_expr, min_spec_expr = NULL;
4220 : 4903 : int privileged_n = 0, i;
4221 : :
4222 : 13014 : for (i = 0; i < ready.n_ready; i++)
4223 : : {
4224 : 8171 : if (ready_try[i])
4225 : 0 : continue;
4226 : :
4227 : 8171 : if (! min_spec_expr)
4228 : 4903 : min_spec_expr = find_expr_for_ready (i, true);
4229 : :
4230 : 8171 : cur_expr = find_expr_for_ready (i, true);
4231 : :
4232 : 8171 : if (EXPR_SPEC (cur_expr) > EXPR_SPEC (min_spec_expr))
4233 : : break;
4234 : :
4235 : 8111 : ++privileged_n;
4236 : : }
4237 : :
4238 : 4903 : if (i == ready.n_ready)
4239 : 4843 : privileged_n = 0;
4240 : :
4241 : 4903 : if (sched_verbose >= 2)
4242 : 0 : sel_print ("privileged_n: %d insns with SPEC %d\n",
4243 : : privileged_n, privileged_n ? EXPR_SPEC (min_spec_expr) : -1);
4244 : 4903 : return privileged_n;
4245 : : }
4246 : :
4247 : : /* Call the rest of the hooks after the choice was made. Return
4248 : : the number of insns that still can be issued given that the current
4249 : : number is ISSUE_MORE. FENCE and BEST_INSN are the current fence
4250 : : and the insn chosen for scheduling, respectively. */
4251 : : static int
4252 : 4883 : invoke_aftermath_hooks (fence_t fence, rtx_insn *best_insn, int issue_more)
4253 : : {
4254 : 4883 : gcc_assert (INSN_P (best_insn));
4255 : :
4256 : : /* First, call dfa_new_cycle, and then variable_issue, if available. */
4257 : 4883 : sel_dfa_new_cycle (best_insn, fence);
4258 : :
4259 : 4883 : if (targetm.sched.variable_issue)
4260 : : {
4261 : 0 : memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4262 : 0 : issue_more =
4263 : 0 : targetm.sched.variable_issue (sched_dump, sched_verbose, best_insn,
4264 : : issue_more);
4265 : 0 : memcpy (FENCE_STATE (fence), curr_state, dfa_state_size);
4266 : : }
4267 : 4883 : else if (!DEBUG_INSN_P (best_insn)
4268 : 4840 : && GET_CODE (PATTERN (best_insn)) != USE
4269 : 9678 : && GET_CODE (PATTERN (best_insn)) != CLOBBER)
4270 : 4784 : issue_more--;
4271 : :
4272 : 4883 : return issue_more;
4273 : : }
4274 : :
4275 : : /* Estimate the cost of issuing INSN on DFA state STATE. */
4276 : : static int
4277 : 3097 : estimate_insn_cost (rtx_insn *insn, state_t state)
4278 : : {
4279 : 3097 : static state_t temp = NULL;
4280 : 3097 : int cost;
4281 : :
4282 : 3097 : if (!temp)
4283 : 53 : temp = xmalloc (dfa_state_size);
4284 : :
4285 : 3097 : memcpy (temp, state, dfa_state_size);
4286 : 3097 : cost = state_transition (temp, insn);
4287 : :
4288 : 3097 : if (cost < 0)
4289 : : return 0;
4290 : 447 : else if (cost == 0)
4291 : 0 : return 1;
4292 : : return cost;
4293 : : }
4294 : :
4295 : : /* Return the cost of issuing EXPR on the FENCE as estimated by DFA.
4296 : : This function properly handles ASMs, USEs etc. */
4297 : : static int
4298 : 1474 : get_expr_cost (expr_t expr, fence_t fence)
4299 : : {
4300 : 1474 : rtx_insn *insn = EXPR_INSN_RTX (expr);
4301 : :
4302 : 1474 : if (recog_memoized (insn) < 0)
4303 : : {
4304 : 0 : if (!FENCE_STARTS_CYCLE_P (fence)
4305 : 0 : && INSN_ASM_P (insn))
4306 : : /* This is asm insn which is tryed to be issued on the
4307 : : cycle not first. Issue it on the next cycle. */
4308 : : return 1;
4309 : : else
4310 : : /* A USE insn, or something else we don't need to
4311 : : understand. We can't pass these directly to
4312 : : state_transition because it will trigger a
4313 : : fatal error for unrecognizable insns. */
4314 : 0 : return 0;
4315 : : }
4316 : : else
4317 : 1474 : return estimate_insn_cost (insn, FENCE_STATE (fence));
4318 : : }
4319 : :
4320 : : /* Find the best insn for scheduling, either via max_issue or just take
4321 : : the most prioritized available. */
4322 : : static int
4323 : 4903 : choose_best_insn (fence_t fence, int privileged_n, int *index)
4324 : : {
4325 : 4903 : int can_issue = 0;
4326 : :
4327 : 4903 : if (dfa_lookahead > 0)
4328 : : {
4329 : 3589 : cycle_issued_insns = FENCE_ISSUED_INSNS (fence);
4330 : : /* TODO: pass equivalent of first_cycle_insn_p to max_issue (). */
4331 : 3589 : can_issue = max_issue (&ready, privileged_n,
4332 : : FENCE_STATE (fence), true, index);
4333 : 3589 : if (sched_verbose >= 2)
4334 : 0 : sel_print ("max_issue: we can issue %d insns, already did %d insns\n",
4335 : : can_issue, FENCE_ISSUED_INSNS (fence));
4336 : : }
4337 : : else
4338 : : {
4339 : : /* We can't use max_issue; just return the first available element. */
4340 : : int i;
4341 : :
4342 : 1565 : for (i = 0; i < ready.n_ready; i++)
4343 : : {
4344 : 1474 : expr_t expr = find_expr_for_ready (i, true);
4345 : :
4346 : 1474 : if (get_expr_cost (expr, fence) < 1)
4347 : : {
4348 : 1223 : can_issue = can_issue_more;
4349 : 1223 : *index = i;
4350 : :
4351 : 1223 : if (sched_verbose >= 2)
4352 : 0 : sel_print ("using %dth insn from the ready list\n", i + 1);
4353 : :
4354 : : break;
4355 : : }
4356 : : }
4357 : :
4358 : 1314 : if (i == ready.n_ready)
4359 : : {
4360 : 91 : can_issue = 0;
4361 : 91 : *index = -1;
4362 : : }
4363 : : }
4364 : :
4365 : 4903 : return can_issue;
4366 : : }
4367 : :
4368 : : /* Choose the best expr from *AV_VLIW_PTR and a suitable register for it.
4369 : : BNDS and FENCE are current boundaries and scheduling fence respectively.
4370 : : Return the expr found and NULL if nothing can be issued atm.
4371 : : Write to PNEED_STALL the number of cycles to stall if no expr was found. */
4372 : : static expr_t
4373 : 7718 : find_best_expr (av_set_t *av_vliw_ptr, blist_t bnds, fence_t fence,
4374 : : int *pneed_stall)
4375 : : {
4376 : 7718 : expr_t best;
4377 : :
4378 : : /* Choose the best insn for scheduling via:
4379 : : 1) sorting the ready list based on priority;
4380 : : 2) calling the reorder hook;
4381 : : 3) calling max_issue. */
4382 : 7718 : best = fill_ready_list (av_vliw_ptr, bnds, fence, pneed_stall);
4383 : 7718 : if (best == NULL && ready.n_ready > 0)
4384 : : {
4385 : 6004 : int privileged_n, index;
4386 : :
4387 : 6004 : can_issue_more = invoke_reorder_hooks (fence);
4388 : 6004 : if (can_issue_more > 0)
4389 : : {
4390 : : /* Try choosing the best insn until we find one that is could be
4391 : : scheduled due to liveness restrictions on its destination register.
4392 : : In the future, we'd like to choose once and then just probe insns
4393 : : in the order of their priority. */
4394 : 4903 : invoke_dfa_lookahead_guard ();
4395 : 4903 : privileged_n = calculate_privileged_insns ();
4396 : 4903 : can_issue_more = choose_best_insn (fence, privileged_n, &index);
4397 : 4903 : if (can_issue_more)
4398 : 4774 : best = find_expr_for_ready (index, true);
4399 : : }
4400 : : /* We had some available insns, so if we can't issue them,
4401 : : we have a stall. */
4402 : 6004 : if (can_issue_more == 0)
4403 : : {
4404 : 1230 : best = NULL;
4405 : 1230 : *pneed_stall = 1;
4406 : : }
4407 : : }
4408 : :
4409 : 7609 : if (best != NULL)
4410 : : {
4411 : 4883 : can_issue_more = invoke_aftermath_hooks (fence, EXPR_INSN_RTX (best),
4412 : : can_issue_more);
4413 : 4883 : if (targetm.sched.variable_issue
4414 : 0 : && can_issue_more == 0)
4415 : 0 : *pneed_stall = 1;
4416 : : }
4417 : :
4418 : 7718 : if (sched_verbose >= 2)
4419 : : {
4420 : 0 : if (best != NULL)
4421 : : {
4422 : 0 : sel_print ("Best expression (vliw form): ");
4423 : 0 : dump_expr (best);
4424 : 0 : sel_print ("; cycle %d\n", FENCE_CYCLE (fence));
4425 : : }
4426 : : else
4427 : 0 : sel_print ("No best expr found!\n");
4428 : : }
4429 : :
4430 : 7718 : return best;
4431 : : }
4432 : :
4433 : :
4434 : : /* Functions that implement the core of the scheduler. */
4435 : :
4436 : :
4437 : : /* Emit an instruction from EXPR with SEQNO and VINSN after
4438 : : PLACE_TO_INSERT. */
4439 : : static insn_t
4440 : 199 : emit_insn_from_expr_after (expr_t expr, vinsn_t vinsn, int seqno,
4441 : : insn_t place_to_insert)
4442 : : {
4443 : : /* This assert fails when we have identical instructions
4444 : : one of which dominates the other. In this case move_op ()
4445 : : finds the first instruction and doesn't search for second one.
4446 : : The solution would be to compute av_set after the first found
4447 : : insn and, if insn present in that set, continue searching.
4448 : : For now we workaround this issue in move_op. */
4449 : 199 : gcc_assert (!INSN_IN_STREAM_P (EXPR_INSN_RTX (expr)));
4450 : :
4451 : 199 : if (EXPR_WAS_RENAMED (expr))
4452 : : {
4453 : 47 : unsigned regno = expr_dest_regno (expr);
4454 : :
4455 : 47 : if (HARD_REGISTER_NUM_P (regno))
4456 : : {
4457 : 25 : df_set_regs_ever_live (regno, true);
4458 : 25 : reg_rename_tick[regno] = ++reg_rename_this_tick;
4459 : : }
4460 : : }
4461 : :
4462 : 199 : return sel_gen_insn_from_expr_after (expr, vinsn, seqno,
4463 : 199 : place_to_insert);
4464 : : }
4465 : :
4466 : : /* Return TRUE if BB can hold bookkeeping code. */
4467 : : static bool
4468 : 177 : block_valid_for_bookkeeping_p (basic_block bb)
4469 : : {
4470 : 177 : insn_t bb_end = BB_END (bb);
4471 : :
4472 : 177 : if (!in_current_region_p (bb) || EDGE_COUNT (bb->succs) > 1)
4473 : : return false;
4474 : :
4475 : 128 : if (INSN_P (bb_end))
4476 : : {
4477 : 120 : if (INSN_SCHED_TIMES (bb_end) > 0)
4478 : 3 : return false;
4479 : : }
4480 : : else
4481 : 8 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (bb_end));
4482 : :
4483 : : return true;
4484 : : }
4485 : :
4486 : : /* Attempt to find a block that can hold bookkeeping code for path(s) incoming
4487 : : into E2->dest, except from E1->src (there may be a sequence of empty basic
4488 : : blocks between E1->src and E2->dest). Return found block, or NULL if new
4489 : : one must be created. If LAX holds, don't assume there is a simple path
4490 : : from E1->src to E2->dest. */
4491 : : static basic_block
4492 : 211 : find_block_for_bookkeeping (edge e1, edge e2, bool lax)
4493 : : {
4494 : 211 : basic_block candidate_block = NULL;
4495 : 211 : edge e;
4496 : :
4497 : : /* Loop over edges from E1 to E2, inclusive. */
4498 : 221 : for (e = e1; !lax || e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun); e =
4499 : 10 : EDGE_SUCC (e->dest, 0))
4500 : : {
4501 : 221 : if (EDGE_COUNT (e->dest->preds) == 2)
4502 : : {
4503 : 192 : if (candidate_block == NULL)
4504 : 192 : candidate_block = (EDGE_PRED (e->dest, 0) == e
4505 : 192 : ? EDGE_PRED (e->dest, 1)->src
4506 : : : EDGE_PRED (e->dest, 0)->src);
4507 : : else
4508 : : /* Found additional edge leading to path from e1 to e2
4509 : : from aside. */
4510 : : return NULL;
4511 : : }
4512 : 29 : else if (EDGE_COUNT (e->dest->preds) > 2)
4513 : : /* Several edges leading to path from e1 to e2 from aside. */
4514 : : return NULL;
4515 : :
4516 : 208 : if (e == e2)
4517 : 177 : return ((!lax || candidate_block)
4518 : 177 : && block_valid_for_bookkeeping_p (candidate_block)
4519 : 177 : ? candidate_block
4520 : : : NULL);
4521 : :
4522 : 96 : if (lax && EDGE_COUNT (e->dest->succs) != 1)
4523 : : return NULL;
4524 : : }
4525 : :
4526 : : if (lax)
4527 : : return NULL;
4528 : :
4529 : : gcc_unreachable ();
4530 : : }
4531 : :
4532 : : /* Create new basic block for bookkeeping code for path(s) incoming into
4533 : : E2->dest, except from E1->src. Return created block. */
4534 : : static basic_block
4535 : 51 : create_block_for_bookkeeping (edge e1, edge e2)
4536 : : {
4537 : 51 : basic_block new_bb, bb = e2->dest;
4538 : :
4539 : : /* Check that we don't spoil the loop structure. */
4540 : 51 : if (current_loop_nest)
4541 : : {
4542 : 41 : basic_block latch = current_loop_nest->latch;
4543 : :
4544 : : /* We do not split header. */
4545 : 41 : gcc_assert (e2->dest != current_loop_nest->header);
4546 : :
4547 : : /* We do not redirect the only edge to the latch block. */
4548 : 59 : gcc_assert (e1->dest != latch
4549 : : || !single_pred_p (latch)
4550 : : || e1 != single_pred_edge (latch));
4551 : : }
4552 : :
4553 : : /* Split BB to insert BOOK_INSN there. */
4554 : 51 : new_bb = sched_split_block (bb, NULL);
4555 : :
4556 : : /* Move note_list from the upper bb. */
4557 : 51 : gcc_assert (BB_NOTE_LIST (new_bb) == NULL_RTX);
4558 : 51 : BB_NOTE_LIST (new_bb) = BB_NOTE_LIST (bb);
4559 : 51 : BB_NOTE_LIST (bb) = NULL;
4560 : :
4561 : 51 : gcc_assert (e2->dest == bb);
4562 : :
4563 : : /* Skip block for bookkeeping copy when leaving E1->src. */
4564 : 51 : if (e1->flags & EDGE_FALLTHRU)
4565 : 26 : sel_redirect_edge_and_branch_force (e1, new_bb);
4566 : : else
4567 : 25 : sel_redirect_edge_and_branch (e1, new_bb);
4568 : :
4569 : 51 : gcc_assert (e1->dest == new_bb);
4570 : 51 : gcc_assert (sel_bb_empty_p (bb));
4571 : :
4572 : : /* To keep basic block numbers in sync between debug and non-debug
4573 : : compilations, we have to rotate blocks here. Consider that we
4574 : : started from (a,b)->d, (c,d)->e, and d contained only debug
4575 : : insns. It would have been removed before if the debug insns
4576 : : weren't there, so we'd have split e rather than d. So what we do
4577 : : now is to swap the block numbers of new_bb and
4578 : : single_succ(new_bb) == e, so that the insns that were in e before
4579 : : get the new block number. */
4580 : :
4581 : 51 : if (MAY_HAVE_DEBUG_INSNS)
4582 : : {
4583 : 0 : basic_block succ;
4584 : 0 : insn_t insn = sel_bb_head (new_bb);
4585 : 0 : insn_t last;
4586 : :
4587 : 0 : if (DEBUG_INSN_P (insn)
4588 : 0 : && single_succ_p (new_bb)
4589 : 0 : && (succ = single_succ (new_bb))
4590 : 0 : && succ != EXIT_BLOCK_PTR_FOR_FN (cfun)
4591 : 0 : && DEBUG_INSN_P ((last = sel_bb_end (new_bb))))
4592 : : {
4593 : 0 : while (insn != last && (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4594 : 0 : insn = NEXT_INSN (insn);
4595 : :
4596 : 0 : if (insn == last)
4597 : : {
4598 : 0 : sel_global_bb_info_def gbi;
4599 : 0 : sel_region_bb_info_def rbi;
4600 : :
4601 : 0 : if (sched_verbose >= 2)
4602 : 0 : sel_print ("Swapping block ids %i and %i\n",
4603 : : new_bb->index, succ->index);
4604 : :
4605 : 0 : std::swap (new_bb->index, succ->index);
4606 : :
4607 : 0 : SET_BASIC_BLOCK_FOR_FN (cfun, new_bb->index, new_bb);
4608 : 0 : SET_BASIC_BLOCK_FOR_FN (cfun, succ->index, succ);
4609 : :
4610 : 0 : memcpy (&gbi, SEL_GLOBAL_BB_INFO (new_bb), sizeof (gbi));
4611 : 0 : memcpy (SEL_GLOBAL_BB_INFO (new_bb), SEL_GLOBAL_BB_INFO (succ),
4612 : : sizeof (gbi));
4613 : 0 : memcpy (SEL_GLOBAL_BB_INFO (succ), &gbi, sizeof (gbi));
4614 : :
4615 : 0 : memcpy (&rbi, SEL_REGION_BB_INFO (new_bb), sizeof (rbi));
4616 : 0 : memcpy (SEL_REGION_BB_INFO (new_bb), SEL_REGION_BB_INFO (succ),
4617 : : sizeof (rbi));
4618 : 0 : memcpy (SEL_REGION_BB_INFO (succ), &rbi, sizeof (rbi));
4619 : :
4620 : 0 : std::swap (BLOCK_TO_BB (new_bb->index),
4621 : 0 : BLOCK_TO_BB (succ->index));
4622 : :
4623 : 0 : std::swap (CONTAINING_RGN (new_bb->index),
4624 : 0 : CONTAINING_RGN (succ->index));
4625 : :
4626 : 0 : for (int i = 0; i < current_nr_blocks; i++)
4627 : 0 : if (BB_TO_BLOCK (i) == succ->index)
4628 : 0 : BB_TO_BLOCK (i) = new_bb->index;
4629 : 0 : else if (BB_TO_BLOCK (i) == new_bb->index)
4630 : 0 : BB_TO_BLOCK (i) = succ->index;
4631 : :
4632 : 0 : FOR_BB_INSNS (new_bb, insn)
4633 : 0 : if (INSN_P (insn))
4634 : 0 : EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = new_bb->index;
4635 : :
4636 : 0 : FOR_BB_INSNS (succ, insn)
4637 : 0 : if (INSN_P (insn))
4638 : 0 : EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = succ->index;
4639 : :
4640 : 0 : if (bitmap_clear_bit (code_motion_visited_blocks, new_bb->index))
4641 : 0 : bitmap_set_bit (code_motion_visited_blocks, succ->index);
4642 : :
4643 : 0 : gcc_assert (LABEL_P (BB_HEAD (new_bb))
4644 : : && LABEL_P (BB_HEAD (succ)));
4645 : :
4646 : 0 : if (sched_verbose >= 4)
4647 : 0 : sel_print ("Swapping code labels %i and %i\n",
4648 : : CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4649 : : CODE_LABEL_NUMBER (BB_HEAD (succ)));
4650 : :
4651 : 0 : std::swap (CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4652 : 0 : CODE_LABEL_NUMBER (BB_HEAD (succ)));
4653 : : }
4654 : : }
4655 : : }
4656 : :
4657 : 51 : return bb;
4658 : : }
4659 : :
4660 : : /* Return insn after which we must insert bookkeeping code for path(s) incoming
4661 : : into E2->dest, except from E1->src. If the returned insn immediately
4662 : : precedes a fence, assign that fence to *FENCE_TO_REWIND. */
4663 : : static insn_t
4664 : 166 : find_place_for_bookkeeping (edge e1, edge e2, fence_t *fence_to_rewind)
4665 : : {
4666 : 166 : insn_t place_to_insert;
4667 : : /* Find a basic block that can hold bookkeeping. If it can be found, do not
4668 : : create new basic block, but insert bookkeeping there. */
4669 : 166 : basic_block book_block = find_block_for_bookkeeping (e1, e2, false);
4670 : :
4671 : 166 : if (book_block)
4672 : : {
4673 : 115 : place_to_insert = BB_END (book_block);
4674 : :
4675 : : /* Don't use a block containing only debug insns for
4676 : : bookkeeping, this causes scheduling differences between debug
4677 : : and non-debug compilations, for the block would have been
4678 : : removed already. */
4679 : 115 : if (DEBUG_INSN_P (place_to_insert))
4680 : : {
4681 : 2 : rtx_insn *insn = sel_bb_head (book_block);
4682 : :
4683 : 4 : while (insn != place_to_insert &&
4684 : 2 : (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4685 : 0 : insn = NEXT_INSN (insn);
4686 : :
4687 : 2 : if (insn == place_to_insert)
4688 : : book_block = NULL;
4689 : : }
4690 : : }
4691 : :
4692 : : if (!book_block)
4693 : : {
4694 : 51 : book_block = create_block_for_bookkeeping (e1, e2);
4695 : 51 : place_to_insert = BB_END (book_block);
4696 : 51 : if (sched_verbose >= 9)
4697 : 0 : sel_print ("New block is %i, split from bookkeeping block %i\n",
4698 : 0 : EDGE_SUCC (book_block, 0)->dest->index, book_block->index);
4699 : : }
4700 : : else
4701 : : {
4702 : 115 : if (sched_verbose >= 9)
4703 : 0 : sel_print ("Pre-existing bookkeeping block is %i\n", book_block->index);
4704 : : }
4705 : :
4706 : 166 : *fence_to_rewind = NULL;
4707 : : /* If basic block ends with a jump, insert bookkeeping code right before it.
4708 : : Notice if we are crossing a fence when taking PREV_INSN. */
4709 : 166 : if (INSN_P (place_to_insert) && control_flow_insn_p (place_to_insert))
4710 : : {
4711 : 8 : *fence_to_rewind = flist_lookup (fences, place_to_insert);
4712 : 8 : place_to_insert = PREV_INSN (place_to_insert);
4713 : : }
4714 : :
4715 : 166 : return place_to_insert;
4716 : : }
4717 : :
4718 : : /* Find a proper seqno for bookkeeing insn inserted at PLACE_TO_INSERT
4719 : : for JOIN_POINT. */
4720 : : static int
4721 : 166 : find_seqno_for_bookkeeping (insn_t place_to_insert, insn_t join_point)
4722 : : {
4723 : 166 : int seqno;
4724 : :
4725 : : /* Check if we are about to insert bookkeeping copy before a jump, and use
4726 : : jump's seqno for the copy; otherwise, use JOIN_POINT's seqno. */
4727 : 166 : rtx_insn *next = NEXT_INSN (place_to_insert);
4728 : 166 : if (INSN_P (next)
4729 : : && JUMP_P (next)
4730 : 166 : && BLOCK_FOR_INSN (next) == BLOCK_FOR_INSN (place_to_insert))
4731 : : {
4732 : 8 : gcc_assert (INSN_SCHED_TIMES (next) == 0);
4733 : 8 : seqno = INSN_SEQNO (next);
4734 : : }
4735 : 158 : else if (INSN_SEQNO (join_point) > 0)
4736 : : seqno = INSN_SEQNO (join_point);
4737 : : else
4738 : : {
4739 : 0 : seqno = get_seqno_by_preds (place_to_insert);
4740 : :
4741 : : /* Sometimes the fences can move in such a way that there will be
4742 : : no instructions with positive seqno around this bookkeeping.
4743 : : This means that there will be no way to get to it by a regular
4744 : : fence movement. Never mind because we pick up such pieces for
4745 : : rescheduling anyways, so any positive value will do for now. */
4746 : 0 : if (seqno < 0)
4747 : : {
4748 : 0 : gcc_assert (pipelining_p);
4749 : : seqno = 1;
4750 : : }
4751 : : }
4752 : :
4753 : 166 : gcc_assert (seqno > 0);
4754 : 166 : return seqno;
4755 : : }
4756 : :
4757 : : /* Insert bookkeeping copy of C_EXPS's insn after PLACE_TO_INSERT, assigning
4758 : : NEW_SEQNO to it. Return created insn. */
4759 : : static insn_t
4760 : 166 : emit_bookkeeping_insn (insn_t place_to_insert, expr_t c_expr, int new_seqno)
4761 : : {
4762 : 166 : rtx_insn *new_insn_rtx = create_copy_of_insn_rtx (EXPR_INSN_RTX (c_expr));
4763 : :
4764 : 166 : vinsn_t new_vinsn
4765 : 332 : = create_vinsn_from_insn_rtx (new_insn_rtx,
4766 : 166 : VINSN_UNIQUE_P (EXPR_VINSN (c_expr)));
4767 : :
4768 : 166 : insn_t new_insn = emit_insn_from_expr_after (c_expr, new_vinsn, new_seqno,
4769 : : place_to_insert);
4770 : :
4771 : 166 : INSN_SCHED_TIMES (new_insn) = 0;
4772 : 166 : bitmap_set_bit (current_copies, INSN_UID (new_insn));
4773 : :
4774 : 166 : return new_insn;
4775 : : }
4776 : :
4777 : : /* Generate a bookkeeping copy of C_EXPR's insn for path(s) incoming into to
4778 : : E2->dest, except from E1->src (there may be a sequence of empty blocks
4779 : : between E1->src and E2->dest). Return block containing the copy.
4780 : : All scheduler data is initialized for the newly created insn. */
4781 : : static basic_block
4782 : 166 : generate_bookkeeping_insn (expr_t c_expr, edge e1, edge e2)
4783 : : {
4784 : 166 : insn_t join_point, place_to_insert, new_insn;
4785 : 166 : int new_seqno;
4786 : 166 : bool need_to_exchange_data_sets;
4787 : 166 : fence_t fence_to_rewind;
4788 : :
4789 : 166 : if (sched_verbose >= 4)
4790 : 0 : sel_print ("Generating bookkeeping insn (%d->%d)\n", e1->src->index,
4791 : 0 : e2->dest->index);
4792 : :
4793 : 166 : join_point = sel_bb_head (e2->dest);
4794 : 166 : place_to_insert = find_place_for_bookkeeping (e1, e2, &fence_to_rewind);
4795 : 166 : new_seqno = find_seqno_for_bookkeeping (place_to_insert, join_point);
4796 : 166 : need_to_exchange_data_sets
4797 : 166 : = sel_bb_empty_p (BLOCK_FOR_INSN (place_to_insert));
4798 : :
4799 : 166 : new_insn = emit_bookkeeping_insn (place_to_insert, c_expr, new_seqno);
4800 : :
4801 : 166 : if (fence_to_rewind)
4802 : 0 : FENCE_INSN (fence_to_rewind) = new_insn;
4803 : :
4804 : : /* When inserting bookkeeping insn in new block, av sets should be
4805 : : following: old basic block (that now holds bookkeeping) data sets are
4806 : : the same as was before generation of bookkeeping, and new basic block
4807 : : (that now hold all other insns of old basic block) data sets are
4808 : : invalid. So exchange data sets for these basic blocks as sel_split_block
4809 : : mistakenly exchanges them in this case. Cannot do it earlier because
4810 : : when single instruction is added to new basic block it should hold NULL
4811 : : lv_set. */
4812 : 166 : if (need_to_exchange_data_sets)
4813 : 59 : exchange_data_sets (BLOCK_FOR_INSN (new_insn),
4814 : 59 : BLOCK_FOR_INSN (join_point));
4815 : :
4816 : 166 : stat_bookkeeping_copies++;
4817 : 166 : return BLOCK_FOR_INSN (new_insn);
4818 : : }
4819 : :
4820 : : /* Remove from AV_PTR all insns that may need bookkeeping when scheduling
4821 : : on FENCE, but we are unable to copy them. */
4822 : : static void
4823 : 5171 : remove_insns_that_need_bookkeeping (fence_t fence, av_set_t *av_ptr)
4824 : : {
4825 : 5171 : expr_t expr;
4826 : 5171 : av_set_iterator i;
4827 : :
4828 : : /* An expression does not need bookkeeping if it is available on all paths
4829 : : from current block to original block and current block dominates
4830 : : original block. We check availability on all paths by examining
4831 : : EXPR_SPEC; this is not equivalent, because it may be positive even
4832 : : if expr is available on all paths (but if expr is not available on
4833 : : any path, EXPR_SPEC will be positive). */
4834 : :
4835 : 30239 : FOR_EACH_EXPR_1 (expr, i, av_ptr)
4836 : : {
4837 : 12534 : if (!control_flow_insn_p (EXPR_INSN_RTX (expr))
4838 : 11624 : && (!bookkeeping_p || VINSN_UNIQUE_P (EXPR_VINSN (expr)))
4839 : 14441 : && (EXPR_SPEC (expr)
4840 : 1870 : || !EXPR_ORIG_BB_INDEX (expr)
4841 : 1870 : || !dominated_by_p (CDI_DOMINATORS,
4842 : 1870 : BASIC_BLOCK_FOR_FN (cfun,
4843 : : EXPR_ORIG_BB_INDEX (expr)),
4844 : 1870 : BLOCK_FOR_INSN (FENCE_INSN (fence)))))
4845 : : {
4846 : 50 : if (sched_verbose >= 4)
4847 : 0 : sel_print ("Expr %d removed because it would need bookkeeping, which "
4848 : 0 : "cannot be created\n", INSN_UID (EXPR_INSN_RTX (expr)));
4849 : 50 : av_set_iter_remove (&i);
4850 : : }
4851 : : }
4852 : 5171 : }
4853 : :
4854 : : /* Moving conditional jump through some instructions.
4855 : :
4856 : : Consider example:
4857 : :
4858 : : ... <- current scheduling point
4859 : : NOTE BASIC BLOCK: <- bb header
4860 : : (p8) add r14=r14+0x9;;
4861 : : (p8) mov [r14]=r23
4862 : : (!p8) jump L1;;
4863 : : NOTE BASIC BLOCK:
4864 : : ...
4865 : :
4866 : : We can schedule jump one cycle earlier, than mov, because they cannot be
4867 : : executed together as their predicates are mutually exclusive.
4868 : :
4869 : : This is done in this way: first, new fallthrough basic block is created
4870 : : after jump (it is always can be done, because there already should be a
4871 : : fallthrough block, where control flow goes in case of predicate being true -
4872 : : in our example; otherwise there should be a dependence between those
4873 : : instructions and jump and we cannot schedule jump right now);
4874 : : next, all instructions between jump and current scheduling point are moved
4875 : : to this new block. And the result is this:
4876 : :
4877 : : NOTE BASIC BLOCK:
4878 : : (!p8) jump L1 <- current scheduling point
4879 : : NOTE BASIC BLOCK: <- bb header
4880 : : (p8) add r14=r14+0x9;;
4881 : : (p8) mov [r14]=r23
4882 : : NOTE BASIC BLOCK:
4883 : : ...
4884 : : */
4885 : : static void
4886 : 0 : move_cond_jump (rtx_insn *insn, bnd_t bnd)
4887 : : {
4888 : 0 : edge ft_edge;
4889 : 0 : basic_block block_from, block_next, block_new, block_bnd, bb;
4890 : 0 : rtx_insn *next, *prev, *link, *head;
4891 : :
4892 : 0 : block_from = BLOCK_FOR_INSN (insn);
4893 : 0 : block_bnd = BLOCK_FOR_INSN (BND_TO (bnd));
4894 : 0 : prev = BND_TO (bnd);
4895 : :
4896 : : /* Moving of jump should not cross any other jumps or beginnings of new
4897 : : basic blocks. The only exception is when we move a jump through
4898 : : mutually exclusive insns along fallthru edges. */
4899 : 0 : if (flag_checking && block_from != block_bnd)
4900 : : {
4901 : 0 : bb = block_from;
4902 : 0 : for (link = PREV_INSN (insn); link != PREV_INSN (prev);
4903 : 0 : link = PREV_INSN (link))
4904 : : {
4905 : 0 : if (INSN_P (link))
4906 : 0 : gcc_assert (sched_insns_conditions_mutex_p (insn, link));
4907 : 0 : if (BLOCK_FOR_INSN (link) && BLOCK_FOR_INSN (link) != bb)
4908 : : {
4909 : 0 : gcc_assert (single_pred (bb) == BLOCK_FOR_INSN (link));
4910 : : bb = BLOCK_FOR_INSN (link);
4911 : : }
4912 : : }
4913 : : }
4914 : :
4915 : : /* Jump is moved to the boundary. */
4916 : 0 : next = PREV_INSN (insn);
4917 : 0 : BND_TO (bnd) = insn;
4918 : :
4919 : 0 : ft_edge = find_fallthru_edge_from (block_from);
4920 : 0 : block_next = ft_edge->dest;
4921 : : /* There must be a fallthrough block (or where should go
4922 : : control flow in case of false jump predicate otherwise?). */
4923 : 0 : gcc_assert (block_next);
4924 : :
4925 : : /* Create new empty basic block after source block. */
4926 : 0 : block_new = sel_split_edge (ft_edge);
4927 : 0 : gcc_assert (block_new->next_bb == block_next
4928 : : && block_from->next_bb == block_new);
4929 : :
4930 : : /* Move all instructions except INSN to BLOCK_NEW. */
4931 : 0 : bb = block_bnd;
4932 : 0 : head = BB_HEAD (block_new);
4933 : 0 : while (bb != block_from->next_bb)
4934 : : {
4935 : 0 : rtx_insn *from, *to;
4936 : 0 : from = bb == block_bnd ? prev : sel_bb_head (bb);
4937 : 0 : to = bb == block_from ? next : sel_bb_end (bb);
4938 : :
4939 : : /* The jump being moved can be the first insn in the block.
4940 : : In this case we don't have to move anything in this block. */
4941 : 0 : if (NEXT_INSN (to) != from)
4942 : : {
4943 : 0 : reorder_insns (from, to, head);
4944 : :
4945 : 0 : for (link = to; link != head; link = PREV_INSN (link))
4946 : 0 : EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index;
4947 : : head = to;
4948 : : }
4949 : :
4950 : : /* Cleanup possibly empty blocks left. */
4951 : 0 : block_next = bb->next_bb;
4952 : 0 : if (bb != block_from)
4953 : 0 : tidy_control_flow (bb, false);
4954 : : bb = block_next;
4955 : : }
4956 : :
4957 : : /* Assert there is no jump to BLOCK_NEW, only fallthrough edge. */
4958 : 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (BB_HEAD (block_new)));
4959 : :
4960 : 0 : gcc_assert (!sel_bb_empty_p (block_from)
4961 : : && !sel_bb_empty_p (block_new));
4962 : :
4963 : : /* Update data sets for BLOCK_NEW to represent that INSN and
4964 : : instructions from the other branch of INSN is no longer
4965 : : available at BLOCK_NEW. */
4966 : 0 : BB_AV_LEVEL (block_new) = global_level;
4967 : 0 : gcc_assert (BB_LV_SET (block_new) == NULL);
4968 : 0 : BB_LV_SET (block_new) = get_clear_regset_from_pool ();
4969 : 0 : update_data_sets (sel_bb_head (block_new));
4970 : :
4971 : : /* INSN is a new basic block header - so prepare its data
4972 : : structures and update availability and liveness sets. */
4973 : 0 : update_data_sets (insn);
4974 : :
4975 : 0 : if (sched_verbose >= 4)
4976 : 0 : sel_print ("Moving jump %d\n", INSN_UID (insn));
4977 : 0 : }
4978 : :
4979 : : /* Remove nops generated during move_op for preventing removal of empty
4980 : : basic blocks. */
4981 : : static void
4982 : 4883 : remove_temp_moveop_nops (bool full_tidying)
4983 : : {
4984 : 4883 : int i;
4985 : 4883 : insn_t insn;
4986 : :
4987 : 5240 : FOR_EACH_VEC_ELT (vec_temp_moveop_nops, i, insn)
4988 : : {
4989 : 357 : gcc_assert (INSN_NOP_P (insn));
4990 : 357 : return_nop_to_pool (insn, full_tidying);
4991 : : }
4992 : :
4993 : : /* Empty the vector. */
4994 : 4883 : if (vec_temp_moveop_nops.length () > 0)
4995 : 348 : vec_temp_moveop_nops.block_remove (0, vec_temp_moveop_nops.length ());
4996 : 4883 : }
4997 : :
4998 : : /* Records the maximal UID before moving up an instruction. Used for
4999 : : distinguishing between bookkeeping copies and original insns. */
5000 : : static int max_uid_before_move_op = 0;
5001 : :
5002 : : /* When true, we're always scheduling next insn on the already scheduled code
5003 : : to get the right insn data for the following bundling or other passes. */
5004 : : static int force_next_insn = 0;
5005 : :
5006 : : /* Remove from AV_VLIW_P all instructions but next when debug counter
5007 : : tells us so. Next instruction is fetched from BNDS. */
5008 : : static void
5009 : 5171 : remove_insns_for_debug (blist_t bnds, av_set_t *av_vliw_p)
5010 : : {
5011 : 5171 : if (! dbg_cnt (sel_sched_insn_cnt) || force_next_insn)
5012 : : /* Leave only the next insn in av_vliw. */
5013 : : {
5014 : 0 : av_set_iterator av_it;
5015 : 0 : expr_t expr;
5016 : 0 : bnd_t bnd = BLIST_BND (bnds);
5017 : 0 : insn_t next = BND_TO (bnd);
5018 : :
5019 : 0 : gcc_assert (BLIST_NEXT (bnds) == NULL);
5020 : :
5021 : 0 : FOR_EACH_EXPR_1 (expr, av_it, av_vliw_p)
5022 : 0 : if (EXPR_INSN_RTX (expr) != next)
5023 : 0 : av_set_iter_remove (&av_it);
5024 : : }
5025 : 5171 : }
5026 : :
5027 : : /* Compute available instructions on BNDS. FENCE is the current fence. Write
5028 : : the computed set to *AV_VLIW_P. */
5029 : : static void
5030 : 5171 : compute_av_set_on_boundaries (fence_t fence, blist_t bnds, av_set_t *av_vliw_p)
5031 : : {
5032 : 5171 : if (sched_verbose >= 2)
5033 : : {
5034 : 0 : sel_print ("Boundaries: ");
5035 : 0 : dump_blist (bnds);
5036 : 0 : sel_print ("\n");
5037 : : }
5038 : :
5039 : 10342 : for (; bnds; bnds = BLIST_NEXT (bnds))
5040 : : {
5041 : 5171 : bnd_t bnd = BLIST_BND (bnds);
5042 : 5171 : av_set_t av1_copy;
5043 : 5171 : insn_t bnd_to = BND_TO (bnd);
5044 : :
5045 : : /* Rewind BND->TO to the basic block header in case some bookkeeping
5046 : : instructions were inserted before BND->TO and it needs to be
5047 : : adjusted. */
5048 : 5171 : if (sel_bb_head_p (bnd_to))
5049 : 1075 : gcc_assert (INSN_SCHED_TIMES (bnd_to) == 0);
5050 : : else
5051 : 4096 : while (INSN_SCHED_TIMES (PREV_INSN (bnd_to)) == 0)
5052 : : {
5053 : 0 : bnd_to = PREV_INSN (bnd_to);
5054 : 0 : if (sel_bb_head_p (bnd_to))
5055 : : break;
5056 : : }
5057 : :
5058 : 5171 : if (BND_TO (bnd) != bnd_to)
5059 : : {
5060 : 0 : gcc_assert (FENCE_INSN (fence) == BND_TO (bnd));
5061 : 0 : FENCE_INSN (fence) = bnd_to;
5062 : 0 : BND_TO (bnd) = bnd_to;
5063 : : }
5064 : :
5065 : 5171 : av_set_clear (&BND_AV (bnd));
5066 : 5171 : BND_AV (bnd) = compute_av_set (BND_TO (bnd), NULL, 0, true);
5067 : :
5068 : 5171 : av_set_clear (&BND_AV1 (bnd));
5069 : 5171 : BND_AV1 (bnd) = av_set_copy (BND_AV (bnd));
5070 : :
5071 : 5171 : moveup_set_inside_insn_group (&BND_AV1 (bnd), NULL);
5072 : :
5073 : 5171 : av1_copy = av_set_copy (BND_AV1 (bnd));
5074 : 5171 : av_set_union_and_clear (av_vliw_p, &av1_copy, NULL);
5075 : : }
5076 : :
5077 : 5171 : if (sched_verbose >= 2)
5078 : : {
5079 : 0 : sel_print ("Available exprs (vliw form): ");
5080 : 0 : dump_av_set (*av_vliw_p);
5081 : 0 : sel_print ("\n");
5082 : : }
5083 : 5171 : }
5084 : :
5085 : : /* Calculate the sequential av set on BND corresponding to the EXPR_VLIW
5086 : : expression. When FOR_MOVEOP is true, also replace the register of
5087 : : expressions found with the register from EXPR_VLIW. */
5088 : : static av_set_t
5089 : 7704 : find_sequential_best_exprs (bnd_t bnd, expr_t expr_vliw, bool for_moveop)
5090 : : {
5091 : 7704 : av_set_t expr_seq = NULL;
5092 : 7704 : expr_t expr;
5093 : 7704 : av_set_iterator i;
5094 : :
5095 : 17468 : FOR_EACH_EXPR (expr, i, BND_AV (bnd))
5096 : : {
5097 : 17468 : if (equal_after_moveup_path_p (expr, NULL, expr_vliw))
5098 : : {
5099 : 7704 : if (for_moveop)
5100 : : {
5101 : : /* The sequential expression has the right form to pass
5102 : : to move_op except when renaming happened. Put the
5103 : : correct register in EXPR then. */
5104 : 4883 : if (EXPR_SEPARABLE_P (expr) && REG_P (EXPR_LHS (expr)))
5105 : : {
5106 : 1280 : if (expr_dest_regno (expr) != expr_dest_regno (expr_vliw))
5107 : : {
5108 : 28 : replace_dest_with_reg_in_expr (expr, EXPR_LHS (expr_vliw));
5109 : 28 : stat_renamed_scheduled++;
5110 : : }
5111 : : /* Also put the correct TARGET_AVAILABLE bit on the expr.
5112 : : This is needed when renaming came up with original
5113 : : register. */
5114 : 1252 : else if (EXPR_TARGET_AVAILABLE (expr)
5115 : 1252 : != EXPR_TARGET_AVAILABLE (expr_vliw))
5116 : : {
5117 : 2 : gcc_assert (EXPR_TARGET_AVAILABLE (expr_vliw) == 1);
5118 : 2 : EXPR_TARGET_AVAILABLE (expr) = 1;
5119 : : }
5120 : : }
5121 : 4883 : if (EXPR_WAS_SUBSTITUTED (expr))
5122 : 1 : stat_substitutions_total++;
5123 : : }
5124 : :
5125 : 7704 : av_set_add (&expr_seq, expr);
5126 : :
5127 : : /* With substitution inside insn group, it is possible
5128 : : that more than one expression in expr_seq will correspond
5129 : : to expr_vliw. In this case, choose one as the attempt to
5130 : : move both leads to miscompiles. */
5131 : 7704 : break;
5132 : : }
5133 : : }
5134 : :
5135 : 7704 : if (for_moveop && sched_verbose >= 2)
5136 : : {
5137 : 0 : sel_print ("Best expression(s) (sequential form): ");
5138 : 0 : dump_av_set (expr_seq);
5139 : 0 : sel_print ("\n");
5140 : : }
5141 : :
5142 : 7704 : return expr_seq;
5143 : : }
5144 : :
5145 : :
5146 : : /* Move nop to previous block. */
5147 : : static void ATTRIBUTE_UNUSED
5148 : 0 : move_nop_to_previous_block (insn_t nop, basic_block prev_bb)
5149 : : {
5150 : 0 : insn_t prev_insn, next_insn;
5151 : :
5152 : 0 : gcc_assert (sel_bb_head_p (nop)
5153 : : && prev_bb == BLOCK_FOR_INSN (nop)->prev_bb);
5154 : 0 : rtx_note *note = bb_note (BLOCK_FOR_INSN (nop));
5155 : 0 : prev_insn = sel_bb_end (prev_bb);
5156 : 0 : next_insn = NEXT_INSN (nop);
5157 : 0 : gcc_assert (prev_insn != NULL_RTX
5158 : : && PREV_INSN (note) == prev_insn);
5159 : :
5160 : 0 : SET_NEXT_INSN (prev_insn) = nop;
5161 : 0 : SET_PREV_INSN (nop) = prev_insn;
5162 : :
5163 : 0 : SET_PREV_INSN (note) = nop;
5164 : 0 : SET_NEXT_INSN (note) = next_insn;
5165 : :
5166 : 0 : SET_NEXT_INSN (nop) = note;
5167 : 0 : SET_PREV_INSN (next_insn) = note;
5168 : :
5169 : 0 : BB_END (prev_bb) = nop;
5170 : 0 : BLOCK_FOR_INSN (nop) = prev_bb;
5171 : 0 : }
5172 : :
5173 : : /* Prepare a place to insert the chosen expression on BND. */
5174 : : static insn_t
5175 : 4883 : prepare_place_to_insert (bnd_t bnd)
5176 : : {
5177 : 4883 : insn_t place_to_insert;
5178 : :
5179 : : /* Init place_to_insert before calling move_op, as the later
5180 : : can possibly remove BND_TO (bnd). */
5181 : 4883 : if (/* If this is not the first insn scheduled. */
5182 : 4883 : BND_PTR (bnd))
5183 : : {
5184 : : /* Add it after last scheduled. */
5185 : 3186 : place_to_insert = ILIST_INSN (BND_PTR (bnd));
5186 : 3186 : if (DEBUG_INSN_P (place_to_insert))
5187 : : {
5188 : : ilist_t l = BND_PTR (bnd);
5189 : 160 : while ((l = ILIST_NEXT (l)) &&
5190 : 42 : DEBUG_INSN_P (ILIST_INSN (l)))
5191 : : ;
5192 : 43 : if (!l)
5193 : : place_to_insert = NULL;
5194 : : }
5195 : : }
5196 : : else
5197 : : place_to_insert = NULL;
5198 : :
5199 : : if (!place_to_insert)
5200 : : {
5201 : : /* Add it before BND_TO. The difference is in the
5202 : : basic block, where INSN will be added. */
5203 : 1735 : place_to_insert = get_nop_from_pool (BND_TO (bnd));
5204 : 1735 : gcc_assert (BLOCK_FOR_INSN (place_to_insert)
5205 : : == BLOCK_FOR_INSN (BND_TO (bnd)));
5206 : : }
5207 : :
5208 : 4883 : return place_to_insert;
5209 : : }
5210 : :
5211 : : /* Find original instructions for EXPR_SEQ and move it to BND boundary.
5212 : : Return the expression to emit in C_EXPR. */
5213 : : static bool
5214 : 4883 : move_exprs_to_boundary (bnd_t bnd, expr_t expr_vliw,
5215 : : av_set_t expr_seq, expr_t c_expr)
5216 : : {
5217 : 4883 : bool b, should_move;
5218 : 4883 : unsigned book_uid;
5219 : 4883 : bitmap_iterator bi;
5220 : 4883 : int n_bookkeeping_copies_before_moveop;
5221 : :
5222 : : /* Make a move. This call will remove the original operation,
5223 : : insert all necessary bookkeeping instructions and update the
5224 : : data sets. After that all we have to do is add the operation
5225 : : at before BND_TO (BND). */
5226 : 4883 : n_bookkeeping_copies_before_moveop = stat_bookkeeping_copies;
5227 : 4883 : max_uid_before_move_op = get_max_uid ();
5228 : 4883 : bitmap_clear (current_copies);
5229 : 4883 : bitmap_clear (current_originators);
5230 : :
5231 : 4883 : b = move_op (BND_TO (bnd), expr_seq, expr_vliw,
5232 : : get_dest_from_orig_ops (expr_seq), c_expr, &should_move);
5233 : :
5234 : : /* We should be able to find the expression we've chosen for
5235 : : scheduling. */
5236 : 4883 : gcc_assert (b);
5237 : :
5238 : 4883 : if (stat_bookkeeping_copies > n_bookkeeping_copies_before_moveop)
5239 : 99 : stat_insns_needed_bookkeeping++;
5240 : :
5241 : 5009 : EXECUTE_IF_SET_IN_BITMAP (current_copies, 0, book_uid, bi)
5242 : : {
5243 : 126 : unsigned uid;
5244 : 126 : bitmap_iterator bi;
5245 : :
5246 : : /* We allocate these bitmaps lazily. */
5247 : 126 : if (! INSN_ORIGINATORS_BY_UID (book_uid))
5248 : 126 : INSN_ORIGINATORS_BY_UID (book_uid) = BITMAP_ALLOC (NULL);
5249 : :
5250 : 126 : bitmap_copy (INSN_ORIGINATORS_BY_UID (book_uid),
5251 : : current_originators);
5252 : :
5253 : : /* Transitively add all originators' originators. */
5254 : 356 : EXECUTE_IF_SET_IN_BITMAP (current_originators, 0, uid, bi)
5255 : 230 : if (INSN_ORIGINATORS_BY_UID (uid))
5256 : 12 : bitmap_ior_into (INSN_ORIGINATORS_BY_UID (book_uid),
5257 : 12 : INSN_ORIGINATORS_BY_UID (uid));
5258 : : }
5259 : :
5260 : 4883 : return should_move;
5261 : : }
5262 : :
5263 : :
5264 : : /* Debug a DFA state as an array of bytes. */
5265 : : static void
5266 : 0 : debug_state (state_t state)
5267 : : {
5268 : 0 : unsigned char *p;
5269 : 0 : unsigned int i, size = dfa_state_size;
5270 : :
5271 : 0 : sel_print ("state (%u):", size);
5272 : 0 : for (i = 0, p = (unsigned char *) state; i < size; i++)
5273 : 0 : sel_print (" %d", p[i]);
5274 : 0 : sel_print ("\n");
5275 : 0 : }
5276 : :
5277 : : /* Advance state on FENCE with INSN. Return true if INSN is
5278 : : an ASM, and we should advance state once more. */
5279 : : static bool
5280 : 4883 : advance_state_on_fence (fence_t fence, insn_t insn)
5281 : : {
5282 : 4883 : bool asm_p;
5283 : :
5284 : 4883 : if (recog_memoized (insn) >= 0)
5285 : : {
5286 : 4774 : int res;
5287 : 4774 : state_t temp_state = alloca (dfa_state_size);
5288 : :
5289 : 4774 : gcc_assert (!INSN_ASM_P (insn));
5290 : 4774 : asm_p = false;
5291 : :
5292 : 4774 : memcpy (temp_state, FENCE_STATE (fence), dfa_state_size);
5293 : 4774 : res = state_transition (FENCE_STATE (fence), insn);
5294 : 4774 : gcc_assert (res < 0);
5295 : :
5296 : 4774 : if (memcmp (temp_state, FENCE_STATE (fence), dfa_state_size))
5297 : : {
5298 : 4707 : FENCE_ISSUED_INSNS (fence)++;
5299 : :
5300 : : /* We should never issue more than issue_rate insns. */
5301 : 4707 : if (FENCE_ISSUED_INSNS (fence) > issue_rate)
5302 : 0 : gcc_unreachable ();
5303 : : }
5304 : : }
5305 : : else
5306 : : {
5307 : : /* This could be an ASM insn which we'd like to schedule
5308 : : on the next cycle. */
5309 : 109 : asm_p = INSN_ASM_P (insn);
5310 : 109 : if (!FENCE_STARTS_CYCLE_P (fence) && asm_p)
5311 : 9 : advance_one_cycle (fence);
5312 : : }
5313 : :
5314 : 4883 : if (sched_verbose >= 2)
5315 : 0 : debug_state (FENCE_STATE (fence));
5316 : 4883 : if (!DEBUG_INSN_P (insn))
5317 : 4840 : FENCE_STARTS_CYCLE_P (fence) = 0;
5318 : 4883 : FENCE_ISSUE_MORE (fence) = can_issue_more;
5319 : 4883 : return asm_p;
5320 : : }
5321 : :
5322 : : /* Update FENCE on which INSN was scheduled and this INSN, too. NEED_STALL
5323 : : is nonzero if we need to stall after issuing INSN. */
5324 : : static void
5325 : 4883 : update_fence_and_insn (fence_t fence, insn_t insn, int need_stall)
5326 : : {
5327 : 4883 : bool asm_p;
5328 : :
5329 : : /* First, reflect that something is scheduled on this fence. */
5330 : 4883 : asm_p = advance_state_on_fence (fence, insn);
5331 : 4883 : FENCE_LAST_SCHEDULED_INSN (fence) = insn;
5332 : 4883 : vec_safe_push (FENCE_EXECUTING_INSNS (fence), insn);
5333 : 4883 : if (SCHED_GROUP_P (insn))
5334 : : {
5335 : 363 : FENCE_SCHED_NEXT (fence) = INSN_SCHED_NEXT (insn);
5336 : 363 : SCHED_GROUP_P (insn) = 0;
5337 : : }
5338 : : else
5339 : 4520 : FENCE_SCHED_NEXT (fence) = NULL;
5340 : 4883 : if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
5341 : 4787 : FENCE_READY_TICKS (fence) [INSN_UID (insn)] = 0;
5342 : :
5343 : : /* Set instruction scheduling info. This will be used in bundling,
5344 : : pipelining, tick computations etc. */
5345 : 4883 : ++INSN_SCHED_TIMES (insn);
5346 : 4883 : EXPR_TARGET_AVAILABLE (INSN_EXPR (insn)) = true;
5347 : 4883 : EXPR_ORIG_SCHED_CYCLE (INSN_EXPR (insn)) = FENCE_CYCLE (fence);
5348 : 4883 : INSN_AFTER_STALL_P (insn) = FENCE_AFTER_STALL_P (fence);
5349 : 4883 : INSN_SCHED_CYCLE (insn) = FENCE_CYCLE (fence);
5350 : :
5351 : : /* This does not account for adjust_cost hooks, just add the biggest
5352 : : constant the hook may add to the latency. TODO: make this
5353 : : a target dependent constant. */
5354 : 4883 : INSN_READY_CYCLE (insn)
5355 : 9766 : = INSN_SCHED_CYCLE (insn) + (INSN_CODE (insn) < 0
5356 : 9657 : ? 1
5357 : 4774 : : maximal_insn_latency (insn) + 1);
5358 : :
5359 : : /* Change these fields last, as they're used above. */
5360 : 4883 : FENCE_AFTER_STALL_P (fence) = 0;
5361 : 4883 : if (asm_p || need_stall)
5362 : 10 : advance_one_cycle (fence);
5363 : :
5364 : : /* Indicate that we've scheduled something on this fence. */
5365 : 4883 : FENCE_SCHEDULED_P (fence) = true;
5366 : 4883 : scheduled_something_on_previous_fence = true;
5367 : :
5368 : : /* Print debug information when insn's fields are updated. */
5369 : 4883 : if (sched_verbose >= 2)
5370 : : {
5371 : 0 : sel_print ("Scheduling insn: ");
5372 : 0 : dump_insn_1 (insn, 1);
5373 : 0 : sel_print ("\n");
5374 : : }
5375 : 4883 : }
5376 : :
5377 : : /* Update boundary BND (and, if needed, FENCE) with INSN, remove the
5378 : : old boundary from BNDSP, add new boundaries to BNDS_TAIL_P and
5379 : : return it. */
5380 : : static blist_t *
5381 : 4883 : update_boundaries (fence_t fence, bnd_t bnd, insn_t insn, blist_t *bndsp,
5382 : : blist_t *bnds_tailp)
5383 : : {
5384 : 4883 : succ_iterator si;
5385 : 4883 : insn_t succ;
5386 : :
5387 : 4883 : advance_deps_context (BND_DC (bnd), insn);
5388 : 9110 : FOR_EACH_SUCC_1 (succ, si, insn,
5389 : : SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
5390 : : {
5391 : 4227 : ilist_t ptr = ilist_copy (BND_PTR (bnd));
5392 : :
5393 : 4227 : ilist_add (&ptr, insn);
5394 : :
5395 : 43 : if (DEBUG_INSN_P (insn) && sel_bb_end_p (insn)
5396 : 4227 : && is_ineligible_successor (succ, ptr))
5397 : : {
5398 : 0 : ilist_clear (&ptr);
5399 : 0 : continue;
5400 : : }
5401 : :
5402 : 4227 : if (FENCE_INSN (fence) == insn && !sel_bb_end_p (insn))
5403 : : {
5404 : 2820 : if (sched_verbose >= 9)
5405 : 0 : sel_print ("Updating fence insn from %i to %i\n",
5406 : 0 : INSN_UID (insn), INSN_UID (succ));
5407 : 2820 : FENCE_INSN (fence) = succ;
5408 : : }
5409 : 4227 : blist_add (bnds_tailp, succ, ptr, BND_DC (bnd));
5410 : 4227 : bnds_tailp = &BLIST_NEXT (*bnds_tailp);
5411 : : }
5412 : :
5413 : 4883 : blist_remove (bndsp);
5414 : 4883 : return bnds_tailp;
5415 : : }
5416 : :
5417 : : /* Schedule EXPR_VLIW on BND. Return the insn emitted. */
5418 : : static insn_t
5419 : 4883 : schedule_expr_on_boundary (bnd_t bnd, expr_t expr_vliw, int seqno)
5420 : : {
5421 : 4883 : av_set_t expr_seq;
5422 : 4883 : expr_t c_expr = XALLOCA (expr_def);
5423 : 4883 : insn_t place_to_insert;
5424 : 4883 : insn_t insn;
5425 : 4883 : bool should_move;
5426 : :
5427 : 4883 : expr_seq = find_sequential_best_exprs (bnd, expr_vliw, true);
5428 : :
5429 : : /* In case of scheduling a jump skipping some other instructions,
5430 : : prepare CFG. After this, jump is at the boundary and can be
5431 : : scheduled as usual insn by MOVE_OP. */
5432 : 4883 : if (vinsn_cond_branch_p (EXPR_VINSN (expr_vliw)))
5433 : : {
5434 : 844 : insn = EXPR_INSN_RTX (expr_vliw);
5435 : :
5436 : : /* Speculative jumps are not handled. */
5437 : 844 : if (insn != BND_TO (bnd)
5438 : 844 : && !sel_insn_is_speculation_check (insn))
5439 : 0 : move_cond_jump (insn, bnd);
5440 : : }
5441 : :
5442 : : /* Find a place for C_EXPR to schedule. */
5443 : 4883 : place_to_insert = prepare_place_to_insert (bnd);
5444 : 4883 : should_move = move_exprs_to_boundary (bnd, expr_vliw, expr_seq, c_expr);
5445 : 4883 : clear_expr (c_expr);
5446 : :
5447 : : /* Add the instruction. The corner case to care about is when
5448 : : the expr_seq set has more than one expr, and we chose the one that
5449 : : is not equal to expr_vliw. Then expr_vliw may be insn in stream, and
5450 : : we can't use it. Generate the new vinsn. */
5451 : 4883 : if (INSN_IN_STREAM_P (EXPR_INSN_RTX (expr_vliw)))
5452 : : {
5453 : 0 : vinsn_t vinsn_new;
5454 : :
5455 : 0 : vinsn_new = vinsn_copy (EXPR_VINSN (expr_vliw), false);
5456 : 0 : change_vinsn_in_expr (expr_vliw, vinsn_new);
5457 : 0 : should_move = false;
5458 : : }
5459 : 4883 : if (should_move)
5460 : 4850 : insn = sel_move_insn (expr_vliw, seqno, place_to_insert);
5461 : : else
5462 : 33 : insn = emit_insn_from_expr_after (expr_vliw, NULL, seqno,
5463 : : place_to_insert);
5464 : :
5465 : : /* Return the nops generated for preserving of data sets back
5466 : : into pool. */
5467 : 4883 : if (INSN_NOP_P (place_to_insert))
5468 : 1735 : return_nop_to_pool (place_to_insert, !DEBUG_INSN_P (insn));
5469 : 4883 : remove_temp_moveop_nops (!DEBUG_INSN_P (insn));
5470 : :
5471 : 4883 : av_set_clear (&expr_seq);
5472 : :
5473 : : /* Save the expression scheduled so to reset target availability if we'll
5474 : : meet it later on the same fence. */
5475 : 4883 : if (EXPR_WAS_RENAMED (expr_vliw))
5476 : 30 : vinsn_vec_add (&vec_target_unavailable_vinsns, INSN_EXPR (insn));
5477 : :
5478 : : /* Check that the recent movement didn't destroyed loop
5479 : : structure. */
5480 : 4883 : gcc_assert (!pipelining_p
5481 : : || current_loop_nest == NULL
5482 : : || loop_latch_edge (current_loop_nest));
5483 : 4883 : return insn;
5484 : : }
5485 : :
5486 : : /* Stall for N cycles on FENCE. */
5487 : : static void
5488 : 2835 : stall_for_cycles (fence_t fence, int n)
5489 : : {
5490 : 2835 : int could_more;
5491 : :
5492 : 2835 : could_more = n > 1 || FENCE_ISSUED_INSNS (fence) < issue_rate;
5493 : 6157 : while (n--)
5494 : 3322 : advance_one_cycle (fence);
5495 : 2835 : if (could_more)
5496 : 2679 : FENCE_AFTER_STALL_P (fence) = 1;
5497 : 2835 : }
5498 : :
5499 : : /* Gather a parallel group of insns at FENCE and assign their seqno
5500 : : to SEQNO. All scheduled insns are gathered in SCHEDULED_INSNS_TAILPP
5501 : : list for later recalculation of seqnos. */
5502 : : static void
5503 : 1886 : fill_insns (fence_t fence, int seqno, ilist_t **scheduled_insns_tailpp)
5504 : : {
5505 : 1886 : blist_t bnds = NULL, *bnds_tailp;
5506 : 1886 : av_set_t av_vliw = NULL;
5507 : 1886 : insn_t insn = FENCE_INSN (fence);
5508 : :
5509 : 1886 : if (sched_verbose >= 2)
5510 : 0 : sel_print ("Starting fill_insns for insn %d, cycle %d\n",
5511 : 0 : INSN_UID (insn), FENCE_CYCLE (fence));
5512 : :
5513 : 1886 : blist_add (&bnds, insn, NULL, FENCE_DC (fence));
5514 : 1886 : bnds_tailp = &BLIST_NEXT (bnds);
5515 : 1886 : set_target_context (FENCE_TC (fence));
5516 : 1886 : can_issue_more = FENCE_ISSUE_MORE (fence);
5517 : 1886 : target_bb = INSN_BB (insn);
5518 : :
5519 : : /* Do while we can add any operation to the current group. */
5520 : 5171 : do
5521 : : {
5522 : 5171 : blist_t *bnds_tailp1, *bndsp;
5523 : 5171 : expr_t expr_vliw;
5524 : 5171 : int need_stall = false;
5525 : 5171 : int was_stall = 0, scheduled_insns = 0;
5526 : 5171 : int max_insns = pipelining_p ? issue_rate : 2 * issue_rate;
5527 : 5171 : int max_stall = pipelining_p ? 1 : 3;
5528 : 5171 : bool last_insn_was_debug = false;
5529 : 5171 : bool was_debug_bb_end_p = false;
5530 : :
5531 : 5171 : compute_av_set_on_boundaries (fence, bnds, &av_vliw);
5532 : 5171 : remove_insns_that_need_bookkeeping (fence, &av_vliw);
5533 : 5171 : remove_insns_for_debug (bnds, &av_vliw);
5534 : :
5535 : : /* Return early if we have nothing to schedule. */
5536 : 5171 : if (av_vliw == NULL)
5537 : : break;
5538 : :
5539 : : /* Choose the best expression and, if needed, destination register
5540 : : for it. */
5541 : 7718 : do
5542 : : {
5543 : 7718 : expr_vliw = find_best_expr (&av_vliw, bnds, fence, &need_stall);
5544 : 7718 : if (! expr_vliw && need_stall)
5545 : : {
5546 : : /* All expressions required a stall. Do not recompute av sets
5547 : : as we'll get the same answer (modulo the insns between
5548 : : the fence and its boundary, which will not be available for
5549 : : pipelining).
5550 : : If we are going to stall for too long, break to recompute av
5551 : : sets and bring more insns for pipelining. */
5552 : 2835 : was_stall++;
5553 : 2835 : if (need_stall <= 3)
5554 : 2589 : stall_for_cycles (fence, need_stall);
5555 : : else
5556 : : {
5557 : 246 : stall_for_cycles (fence, 1);
5558 : 246 : break;
5559 : : }
5560 : : }
5561 : : }
5562 : 2589 : while (! expr_vliw && need_stall);
5563 : :
5564 : : /* Now either we've selected expr_vliw or we have nothing to schedule. */
5565 : 5129 : if (!expr_vliw)
5566 : : {
5567 : 246 : av_set_clear (&av_vliw);
5568 : 246 : break;
5569 : : }
5570 : :
5571 : : bndsp = &bnds;
5572 : : bnds_tailp1 = bnds_tailp;
5573 : :
5574 : 9110 : do
5575 : : /* This code will be executed only once until we'd have several
5576 : : boundaries per fence. */
5577 : : {
5578 : 9110 : bnd_t bnd = BLIST_BND (*bndsp);
5579 : :
5580 : 9110 : if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr_vliw)))
5581 : : {
5582 : 4227 : bndsp = &BLIST_NEXT (*bndsp);
5583 : 4227 : continue;
5584 : : }
5585 : :
5586 : 4883 : insn = schedule_expr_on_boundary (bnd, expr_vliw, seqno);
5587 : 4883 : last_insn_was_debug = DEBUG_INSN_P (insn);
5588 : 4883 : if (last_insn_was_debug)
5589 : 43 : was_debug_bb_end_p = (insn == BND_TO (bnd) && sel_bb_end_p (insn));
5590 : 4883 : update_fence_and_insn (fence, insn, need_stall);
5591 : 4883 : bnds_tailp = update_boundaries (fence, bnd, insn, bndsp, bnds_tailp);
5592 : :
5593 : : /* Add insn to the list of scheduled on this cycle instructions. */
5594 : 4883 : ilist_add (*scheduled_insns_tailpp, insn);
5595 : 4883 : *scheduled_insns_tailpp = &ILIST_NEXT (**scheduled_insns_tailpp);
5596 : : }
5597 : 9110 : while (*bndsp != *bnds_tailp1);
5598 : :
5599 : 4883 : av_set_clear (&av_vliw);
5600 : 4883 : if (!last_insn_was_debug)
5601 : 4840 : scheduled_insns++;
5602 : :
5603 : : /* We currently support information about candidate blocks only for
5604 : : one 'target_bb' block. Hence we can't schedule after jump insn,
5605 : : as this will bring two boundaries and, hence, necessity to handle
5606 : : information for two or more blocks concurrently. */
5607 : 4840 : if ((last_insn_was_debug ? was_debug_bb_end_p : sel_bb_end_p (insn))
5608 : 3809 : || (was_stall
5609 : 1768 : && (was_stall >= max_stall
5610 : 1768 : || scheduled_insns >= max_insns)))
5611 : : break;
5612 : : }
5613 : 3285 : while (bnds);
5614 : :
5615 : 1886 : gcc_assert (!FENCE_BNDS (fence));
5616 : :
5617 : : /* Update boundaries of the FENCE. */
5618 : 3116 : while (bnds)
5619 : : {
5620 : 1230 : ilist_t ptr = BND_PTR (BLIST_BND (bnds));
5621 : :
5622 : 1230 : if (ptr)
5623 : : {
5624 : 1041 : insn = ILIST_INSN (ptr);
5625 : :
5626 : 2082 : if (!ilist_is_in_p (FENCE_BNDS (fence), insn))
5627 : 957 : ilist_add (&FENCE_BNDS (fence), insn);
5628 : : }
5629 : :
5630 : 1230 : blist_remove (&bnds);
5631 : : }
5632 : :
5633 : : /* Update target context on the fence. */
5634 : 1886 : reset_target_context (FENCE_TC (fence), false);
5635 : 1886 : }
5636 : :
5637 : : /* All exprs in ORIG_OPS must have the same destination register or memory.
5638 : : Return that destination. */
5639 : : static rtx
5640 : 4883 : get_dest_from_orig_ops (av_set_t orig_ops)
5641 : : {
5642 : 4883 : rtx dest = NULL_RTX;
5643 : 4883 : av_set_iterator av_it;
5644 : 4883 : expr_t expr;
5645 : 4883 : bool first_p = true;
5646 : :
5647 : 9766 : FOR_EACH_EXPR (expr, av_it, orig_ops)
5648 : : {
5649 : 4883 : rtx x = EXPR_LHS (expr);
5650 : :
5651 : 4883 : if (first_p)
5652 : : {
5653 : : first_p = false;
5654 : : dest = x;
5655 : : }
5656 : : else
5657 : 0 : gcc_assert (dest == x
5658 : : || (dest != NULL_RTX && x != NULL_RTX
5659 : : && rtx_equal_p (dest, x)));
5660 : : }
5661 : :
5662 : 4883 : return dest;
5663 : : }
5664 : :
5665 : : /* Update data sets for the bookkeeping block and record those expressions
5666 : : which become no longer available after inserting this bookkeeping. */
5667 : : static void
5668 : 166 : update_and_record_unavailable_insns (basic_block book_block)
5669 : : {
5670 : 166 : av_set_iterator i;
5671 : 166 : av_set_t old_av_set = NULL;
5672 : 166 : expr_t cur_expr;
5673 : 166 : rtx_insn *bb_end = sel_bb_end (book_block);
5674 : :
5675 : : /* First, get correct liveness in the bookkeeping block. The problem is
5676 : : the range between the bookeeping insn and the end of block. */
5677 : 166 : update_liveness_on_insn (bb_end);
5678 : 166 : if (control_flow_insn_p (bb_end))
5679 : 8 : update_liveness_on_insn (PREV_INSN (bb_end));
5680 : :
5681 : : /* If there's valid av_set on BOOK_BLOCK, then there might exist another
5682 : : fence above, where we may choose to schedule an insn which is
5683 : : actually blocked from moving up with the bookkeeping we create here. */
5684 : 166 : if (AV_SET_VALID_P (sel_bb_head (book_block)))
5685 : : {
5686 : 123 : old_av_set = av_set_copy (BB_AV_SET (book_block));
5687 : 123 : update_data_sets (sel_bb_head (book_block));
5688 : :
5689 : : /* Traverse all the expressions in the old av_set and check whether
5690 : : CUR_EXPR is in new AV_SET. */
5691 : 614 : FOR_EACH_EXPR (cur_expr, i, old_av_set)
5692 : : {
5693 : 368 : expr_t new_expr = av_set_lookup (BB_AV_SET (book_block),
5694 : : EXPR_VINSN (cur_expr));
5695 : :
5696 : 368 : if (! new_expr
5697 : : /* In this case, we can just turn off the E_T_A bit, but we can't
5698 : : represent this information with the current vector. */
5699 : 346 : || EXPR_TARGET_AVAILABLE (new_expr)
5700 : 346 : != EXPR_TARGET_AVAILABLE (cur_expr))
5701 : : /* Unfortunately, the below code could be also fired up on
5702 : : separable insns, e.g. when moving insns through the new
5703 : : speculation check as in PR 53701. */
5704 : 30 : vinsn_vec_add (&vec_bookkeeping_blocked_vinsns, cur_expr);
5705 : : }
5706 : :
5707 : 123 : av_set_clear (&old_av_set);
5708 : : }
5709 : 166 : }
5710 : :
5711 : : /* The main effect of this function is that sparams->c_expr is merged
5712 : : with (or copied to) lparams->c_expr_merged. If there's only one successor,
5713 : : we avoid merging anything by copying sparams->c_expr to lparams->c_expr_merged.
5714 : : lparams->c_expr_merged is copied back to sparams->c_expr after all
5715 : : successors has been traversed. lparams->c_expr_local is an expr allocated
5716 : : on stack in the caller function, and is used if there is more than one
5717 : : successor.
5718 : :
5719 : : SUCC is one of the SUCCS_NORMAL successors of INSN,
5720 : : MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ,
5721 : : LPARAMS and STATIC_PARAMS contain the parameters described above. */
5722 : : static void
5723 : 511 : move_op_merge_succs (insn_t insn ATTRIBUTE_UNUSED,
5724 : : insn_t succ ATTRIBUTE_UNUSED,
5725 : : int moveop_drv_call_res,
5726 : : cmpd_local_params_p lparams, void *static_params)
5727 : : {
5728 : 511 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
5729 : :
5730 : : /* Nothing to do, if original expr wasn't found below. */
5731 : 511 : if (moveop_drv_call_res != 1)
5732 : : return;
5733 : :
5734 : : /* If this is a first successor. */
5735 : 445 : if (!lparams->c_expr_merged)
5736 : : {
5737 : 378 : lparams->c_expr_merged = sparams->c_expr;
5738 : 378 : sparams->c_expr = lparams->c_expr_local;
5739 : : }
5740 : : else
5741 : : {
5742 : : /* We must merge all found expressions to get reasonable
5743 : : EXPR_SPEC_DONE_DS for the resulting insn. If we don't
5744 : : do so then we can first find the expr with epsilon
5745 : : speculation success probability and only then with the
5746 : : good probability. As a result the insn will get epsilon
5747 : : probability and will never be scheduled because of
5748 : : weakness_cutoff in find_best_expr.
5749 : :
5750 : : We call merge_expr_data here instead of merge_expr
5751 : : because due to speculation C_EXPR and X may have the
5752 : : same insns with different speculation types. And as of
5753 : : now such insns are considered non-equal.
5754 : :
5755 : : However, EXPR_SCHED_TIMES is different -- we must get
5756 : : SCHED_TIMES from a real insn, not a bookkeeping copy.
5757 : : We force this here. Instead, we may consider merging
5758 : : SCHED_TIMES to the maximum instead of minimum in the
5759 : : below function. */
5760 : 67 : int old_times = EXPR_SCHED_TIMES (lparams->c_expr_merged);
5761 : :
5762 : 67 : merge_expr_data (lparams->c_expr_merged, sparams->c_expr, NULL);
5763 : 67 : if (EXPR_SCHED_TIMES (sparams->c_expr) == 0)
5764 : 66 : EXPR_SCHED_TIMES (lparams->c_expr_merged) = old_times;
5765 : :
5766 : 67 : clear_expr (sparams->c_expr);
5767 : : }
5768 : : }
5769 : :
5770 : : /* Add used regs for the successor SUCC into SPARAMS->USED_REGS.
5771 : :
5772 : : SUCC is one of the SUCCS_NORMAL successors of INSN,
5773 : : MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ or 0,
5774 : : if SUCC is one of SUCCS_BACK or SUCCS_OUT.
5775 : : STATIC_PARAMS contain USED_REGS set. */
5776 : : static void
5777 : 9685 : fur_merge_succs (insn_t insn ATTRIBUTE_UNUSED, insn_t succ,
5778 : : int moveop_drv_call_res,
5779 : : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5780 : : void *static_params)
5781 : : {
5782 : 9685 : regset succ_live;
5783 : 9685 : fur_static_params_p sparams = (fur_static_params_p) static_params;
5784 : :
5785 : : /* Here we compute live regsets only for branches that do not lie
5786 : : on the code motion paths. These branches correspond to value
5787 : : MOVEOP_DRV_CALL_RES==0 and include SUCCS_BACK and SUCCS_OUT, though
5788 : : for such branches code_motion_path_driver is not called. */
5789 : 9685 : if (moveop_drv_call_res != 0)
5790 : : return;
5791 : :
5792 : : /* Mark all registers that do not meet the following condition:
5793 : : (3) not live on the other path of any conditional branch
5794 : : that is passed by the operation, in case original
5795 : : operations are not present on both paths of the
5796 : : conditional branch. */
5797 : 3208 : succ_live = compute_live (succ);
5798 : 3208 : IOR_REG_SET (sparams->used_regs, succ_live);
5799 : : }
5800 : :
5801 : : /* This function is called after the last successor. Copies LP->C_EXPR_MERGED
5802 : : into SP->CEXPR. */
5803 : : static void
5804 : 378 : move_op_after_merge_succs (cmpd_local_params_p lp, void *sparams)
5805 : : {
5806 : 378 : moveop_static_params_p sp = (moveop_static_params_p) sparams;
5807 : :
5808 : 378 : sp->c_expr = lp->c_expr_merged;
5809 : 378 : }
5810 : :
5811 : : /* Track bookkeeping copies created, insns scheduled, and blocks for
5812 : : rescheduling when INSN is found by move_op. */
5813 : : static void
5814 : 4950 : track_scheduled_insns_and_blocks (rtx_insn *insn)
5815 : : {
5816 : : /* Even if this insn can be a copy that will be removed during current move_op,
5817 : : we still need to count it as an originator. */
5818 : 4950 : bitmap_set_bit (current_originators, INSN_UID (insn));
5819 : :
5820 : 4950 : if (!bitmap_clear_bit (current_copies, INSN_UID (insn)))
5821 : : {
5822 : : /* Note that original block needs to be rescheduled, as we pulled an
5823 : : instruction out of it. */
5824 : 4910 : if (INSN_SCHED_TIMES (insn) > 0)
5825 : 47 : bitmap_set_bit (blocks_to_reschedule, BLOCK_FOR_INSN (insn)->index);
5826 : 4863 : else if (INSN_UID (insn) < first_emitted_uid && !DEBUG_INSN_P (insn))
5827 : 4695 : num_insns_scheduled++;
5828 : : }
5829 : :
5830 : : /* For instructions we must immediately remove insn from the
5831 : : stream, so subsequent update_data_sets () won't include this
5832 : : insn into av_set.
5833 : : For expr we must make insn look like "INSN_REG (insn) := c_expr". */
5834 : 4950 : if (INSN_UID (insn) > max_uid_before_move_op)
5835 : 38 : stat_bookkeeping_copies--;
5836 : 4950 : }
5837 : :
5838 : : /* Emit a register-register copy for INSN if needed. Return true if
5839 : : emitted one. PARAMS is the move_op static parameters. */
5840 : : static bool
5841 : 4950 : maybe_emit_renaming_copy (rtx_insn *insn,
5842 : : moveop_static_params_p params)
5843 : : {
5844 : 4950 : bool insn_emitted = false;
5845 : 4950 : rtx cur_reg;
5846 : :
5847 : : /* Bail out early when expression cannot be renamed at all. */
5848 : 4950 : if (!EXPR_SEPARABLE_P (params->c_expr))
5849 : : return false;
5850 : :
5851 : 1324 : cur_reg = expr_dest_reg (params->c_expr);
5852 : 1324 : gcc_assert (cur_reg && params->dest && REG_P (params->dest));
5853 : :
5854 : : /* If original operation has expr and the register chosen for
5855 : : that expr is not original operation's dest reg, substitute
5856 : : operation's right hand side with the register chosen. */
5857 : 1324 : if (REGNO (params->dest) != REGNO (cur_reg))
5858 : : {
5859 : 31 : insn_t reg_move_insn, reg_move_insn_rtx;
5860 : :
5861 : 31 : reg_move_insn_rtx = create_insn_rtx_with_rhs (INSN_VINSN (insn),
5862 : : params->dest);
5863 : 93 : reg_move_insn = sel_gen_insn_from_rtx_after (reg_move_insn_rtx,
5864 : 31 : INSN_EXPR (insn),
5865 : 31 : INSN_SEQNO (insn),
5866 : : insn);
5867 : 31 : EXPR_SPEC_DONE_DS (INSN_EXPR (reg_move_insn)) = 0;
5868 : 31 : replace_dest_with_reg_in_expr (params->c_expr, params->dest);
5869 : :
5870 : 31 : insn_emitted = true;
5871 : 31 : params->was_renamed = true;
5872 : : }
5873 : :
5874 : : return insn_emitted;
5875 : : }
5876 : :
5877 : : /* Emit a speculative check for INSN speculated as EXPR if needed.
5878 : : Return true if we've emitted one. PARAMS is the move_op static
5879 : : parameters. */
5880 : : static bool
5881 : 4950 : maybe_emit_speculative_check (rtx_insn *insn, expr_t expr,
5882 : : moveop_static_params_p params)
5883 : : {
5884 : 4950 : bool insn_emitted = false;
5885 : 4950 : insn_t x;
5886 : 4950 : ds_t check_ds;
5887 : :
5888 : 4950 : check_ds = get_spec_check_type_for_insn (insn, expr);
5889 : 4950 : if (check_ds != 0)
5890 : : {
5891 : : /* A speculation check should be inserted. */
5892 : 0 : x = create_speculation_check (params->c_expr, check_ds, insn);
5893 : 0 : insn_emitted = true;
5894 : : }
5895 : : else
5896 : : {
5897 : 4950 : EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
5898 : 4950 : x = insn;
5899 : : }
5900 : :
5901 : 4950 : gcc_assert (EXPR_SPEC_DONE_DS (INSN_EXPR (x)) == 0
5902 : : && EXPR_SPEC_TO_CHECK_DS (INSN_EXPR (x)) == 0);
5903 : 4950 : return insn_emitted;
5904 : : }
5905 : :
5906 : : /* Handle transformations that leave an insn in place of original
5907 : : insn such as renaming/speculation. Return true if one of such
5908 : : transformations actually happened, and we have emitted this insn. */
5909 : : static bool
5910 : 4950 : handle_emitting_transformations (rtx_insn *insn, expr_t expr,
5911 : : moveop_static_params_p params)
5912 : : {
5913 : 4950 : bool insn_emitted = false;
5914 : :
5915 : 4950 : insn_emitted = maybe_emit_renaming_copy (insn, params);
5916 : 4950 : insn_emitted |= maybe_emit_speculative_check (insn, expr, params);
5917 : :
5918 : 4950 : return insn_emitted;
5919 : : }
5920 : :
5921 : : /* If INSN is the only insn in the basic block (not counting JUMP,
5922 : : which may be a jump to next insn, and DEBUG_INSNs), we want to
5923 : : leave a NOP there till the return to fill_insns. */
5924 : :
5925 : : static bool
5926 : 4950 : need_nop_to_preserve_insn_bb (rtx_insn *insn)
5927 : : {
5928 : 4950 : insn_t bb_head, bb_end, bb_next, in_next;
5929 : 4950 : basic_block bb = BLOCK_FOR_INSN (insn);
5930 : :
5931 : 4950 : bb_head = sel_bb_head (bb);
5932 : 4950 : bb_end = sel_bb_end (bb);
5933 : :
5934 : 4950 : if (bb_head == bb_end)
5935 : : return true;
5936 : :
5937 : 5043 : while (bb_head != bb_end && DEBUG_INSN_P (bb_head))
5938 : 135 : bb_head = NEXT_INSN (bb_head);
5939 : :
5940 : 4908 : if (bb_head == bb_end)
5941 : : return true;
5942 : :
5943 : 4916 : while (bb_head != bb_end && DEBUG_INSN_P (bb_end))
5944 : 8 : bb_end = PREV_INSN (bb_end);
5945 : :
5946 : 4908 : if (bb_head == bb_end)
5947 : : return true;
5948 : :
5949 : 4908 : bb_next = NEXT_INSN (bb_head);
5950 : 9851 : while (bb_next != bb_end && DEBUG_INSN_P (bb_next))
5951 : 35 : bb_next = NEXT_INSN (bb_next);
5952 : :
5953 : 4908 : if (bb_next == bb_end && JUMP_P (bb_end))
5954 : : return true;
5955 : :
5956 : 4593 : in_next = NEXT_INSN (insn);
5957 : 9204 : while (DEBUG_INSN_P (in_next))
5958 : 18 : in_next = NEXT_INSN (in_next);
5959 : :
5960 : 4593 : if (IN_CURRENT_FENCE_P (in_next))
5961 : : return true;
5962 : :
5963 : : return false;
5964 : : }
5965 : :
5966 : : /* Remove INSN from stream. When ONLY_DISCONNECT is true, its data
5967 : : is not removed but reused when INSN is re-emitted. */
5968 : : static void
5969 : 4950 : remove_insn_from_stream (rtx_insn *insn, bool only_disconnect)
5970 : : {
5971 : : /* If there's only one insn in the BB, make sure that a nop is
5972 : : inserted into it, so the basic block won't disappear when we'll
5973 : : delete INSN below with sel_remove_insn. It should also survive
5974 : : till the return to fill_insns. */
5975 : 4950 : if (need_nop_to_preserve_insn_bb (insn))
5976 : : {
5977 : 357 : insn_t nop = get_nop_from_pool (insn);
5978 : 357 : gcc_assert (INSN_NOP_P (nop));
5979 : 357 : vec_temp_moveop_nops.safe_push (nop);
5980 : : }
5981 : :
5982 : 4950 : sel_remove_insn (insn, only_disconnect, false);
5983 : 4950 : }
5984 : :
5985 : : /* This function is called when original expr is found.
5986 : : INSN - current insn traversed, EXPR - the corresponding expr found.
5987 : : LPARAMS is the local parameters of code modion driver, STATIC_PARAMS
5988 : : is static parameters of move_op. */
5989 : : static void
5990 : 4950 : move_op_orig_expr_found (insn_t insn, expr_t expr,
5991 : : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5992 : : void *static_params)
5993 : : {
5994 : 4950 : bool only_disconnect;
5995 : 4950 : moveop_static_params_p params = (moveop_static_params_p) static_params;
5996 : :
5997 : 4950 : copy_expr_onside (params->c_expr, INSN_EXPR (insn));
5998 : 4950 : track_scheduled_insns_and_blocks (insn);
5999 : 4950 : handle_emitting_transformations (insn, expr, params);
6000 : 4950 : only_disconnect = params->uid == INSN_UID (insn);
6001 : :
6002 : : /* Mark that we've disconnected an insn. */
6003 : 4950 : if (only_disconnect)
6004 : 4850 : params->uid = -1;
6005 : 4950 : remove_insn_from_stream (insn, only_disconnect);
6006 : 4950 : }
6007 : :
6008 : : /* The function is called when original expr is found.
6009 : : INSN - current insn traversed, EXPR - the corresponding expr found,
6010 : : crossed_call_abis and original_insns in STATIC_PARAMS are updated. */
6011 : : static void
6012 : 3745 : fur_orig_expr_found (insn_t insn, expr_t expr ATTRIBUTE_UNUSED,
6013 : : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6014 : : void *static_params)
6015 : : {
6016 : 3745 : fur_static_params_p params = (fur_static_params_p) static_params;
6017 : 3745 : regset tmp;
6018 : :
6019 : 3745 : if (CALL_P (insn))
6020 : 0 : params->crossed_call_abis |= 1 << insn_callee_abi (insn).id ();
6021 : :
6022 : 3745 : def_list_add (params->original_insns, insn, params->crossed_call_abis);
6023 : :
6024 : : /* Mark the registers that do not meet the following condition:
6025 : : (2) not among the live registers of the point
6026 : : immediately following the first original operation on
6027 : : a given downward path, except for the original target
6028 : : register of the operation. */
6029 : 3745 : tmp = get_clear_regset_from_pool ();
6030 : 3745 : compute_live_below_insn (insn, tmp);
6031 : 3745 : AND_COMPL_REG_SET (tmp, INSN_REG_SETS (insn));
6032 : 3745 : AND_COMPL_REG_SET (tmp, INSN_REG_CLOBBERS (insn));
6033 : 3745 : IOR_REG_SET (params->used_regs, tmp);
6034 : 3745 : return_regset_to_pool (tmp);
6035 : :
6036 : : /* (*1) We need to add to USED_REGS registers that are read by
6037 : : INSN's lhs. This may lead to choosing wrong src register.
6038 : : E.g. (scheduling const expr enabled):
6039 : :
6040 : : 429: ax=0x0 <- Can't use AX for this expr (0x0)
6041 : : 433: dx=[bp-0x18]
6042 : : 427: [ax+dx+0x1]=ax
6043 : : REG_DEAD: ax
6044 : : 168: di=dx
6045 : : REG_DEAD: dx
6046 : : */
6047 : : /* FIXME: see comment above and enable MEM_P
6048 : : in vinsn_separable_p. */
6049 : 3745 : gcc_assert (!VINSN_SEPARABLE_P (INSN_VINSN (insn))
6050 : : || !MEM_P (INSN_LHS (insn)));
6051 : 3745 : }
6052 : :
6053 : : /* This function is called on the ascending pass, before returning from
6054 : : current basic block. */
6055 : : static void
6056 : 5328 : move_op_at_first_insn (insn_t insn, cmpd_local_params_p lparams,
6057 : : void *static_params)
6058 : : {
6059 : 5328 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6060 : 5328 : basic_block book_block = NULL;
6061 : :
6062 : : /* When we have removed the boundary insn for scheduling, which also
6063 : : happened to be the end insn in its bb, we don't need to update sets. */
6064 : 5328 : if (!lparams->removed_last_insn
6065 : 4565 : && lparams->e1
6066 : 5773 : && sel_bb_head_p (insn))
6067 : : {
6068 : : /* We should generate bookkeeping code only if we are not at the
6069 : : top level of the move_op. */
6070 : 445 : if (sel_num_cfg_preds_gt_1 (insn))
6071 : 166 : book_block = generate_bookkeeping_insn (sparams->c_expr,
6072 : : lparams->e1, lparams->e2);
6073 : : /* Update data sets for the current insn. */
6074 : 445 : update_data_sets (insn);
6075 : : }
6076 : :
6077 : : /* If bookkeeping code was inserted, we need to update av sets of basic
6078 : : block that received bookkeeping. After generation of bookkeeping insn,
6079 : : bookkeeping block does not contain valid av set because we are not following
6080 : : the original algorithm in every detail with regards to e.g. renaming
6081 : : simple reg-reg copies. Consider example:
6082 : :
6083 : : bookkeeping block scheduling fence
6084 : : \ /
6085 : : \ join /
6086 : : ----------
6087 : : | |
6088 : : ----------
6089 : : / \
6090 : : / \
6091 : : r1 := r2 r1 := r3
6092 : :
6093 : : We try to schedule insn "r1 := r3" on the current
6094 : : scheduling fence. Also, note that av set of bookkeeping block
6095 : : contain both insns "r1 := r2" and "r1 := r3". When the insn has
6096 : : been scheduled, the CFG is as follows:
6097 : :
6098 : : r1 := r3 r1 := r3
6099 : : bookkeeping block scheduling fence
6100 : : \ /
6101 : : \ join /
6102 : : ----------
6103 : : | |
6104 : : ----------
6105 : : / \
6106 : : / \
6107 : : r1 := r2
6108 : :
6109 : : Here, insn "r1 := r3" was scheduled at the current scheduling point
6110 : : and bookkeeping code was generated at the bookeeping block. This
6111 : : way insn "r1 := r2" is no longer available as a whole instruction
6112 : : (but only as expr) ahead of insn "r1 := r3" in bookkeeping block.
6113 : : This situation is handled by calling update_data_sets.
6114 : :
6115 : : Since update_data_sets is called only on the bookkeeping block, and
6116 : : it also may have predecessors with av_sets, containing instructions that
6117 : : are no longer available, we save all such expressions that become
6118 : : unavailable during data sets update on the bookkeeping block in
6119 : : VEC_BOOKKEEPING_BLOCKED_VINSNS. Later we avoid selecting such
6120 : : expressions for scheduling. This allows us to avoid recomputation of
6121 : : av_sets outside the code motion path. */
6122 : :
6123 : 445 : if (book_block)
6124 : 166 : update_and_record_unavailable_insns (book_block);
6125 : :
6126 : : /* If INSN was previously marked for deletion, it's time to do it. */
6127 : 5328 : if (lparams->removed_last_insn)
6128 : 763 : insn = PREV_INSN (insn);
6129 : :
6130 : : /* Do not tidy control flow at the topmost moveop, as we can erroneously
6131 : : kill a block with a single nop in which the insn should be emitted. */
6132 : 5328 : if (lparams->e1)
6133 : 445 : tidy_control_flow (BLOCK_FOR_INSN (insn), true);
6134 : 5328 : }
6135 : :
6136 : : /* This function is called on the ascending pass, before returning from the
6137 : : current basic block. */
6138 : : static void
6139 : 8964 : fur_at_first_insn (insn_t insn,
6140 : : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6141 : : void *static_params ATTRIBUTE_UNUSED)
6142 : : {
6143 : 8964 : gcc_assert (!sel_bb_head_p (insn) || AV_SET_VALID_P (insn)
6144 : : || AV_LEVEL (insn) == -1);
6145 : 8964 : }
6146 : :
6147 : : /* Called on the backward stage of recursion to call moveup_expr for insn
6148 : : and sparams->c_expr. */
6149 : : static void
6150 : 4103 : move_op_ascend (insn_t insn, void *static_params)
6151 : : {
6152 : 4103 : enum MOVEUP_EXPR_CODE res;
6153 : 4103 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6154 : :
6155 : 4103 : if (! INSN_NOP_P (insn))
6156 : : {
6157 : 4103 : res = moveup_expr_cached (sparams->c_expr, insn, false);
6158 : 4103 : gcc_assert (res != MOVEUP_EXPR_NULL);
6159 : : }
6160 : :
6161 : : /* Update liveness for this insn as it was invalidated. */
6162 : 4103 : update_liveness_on_insn (insn);
6163 : 4103 : }
6164 : :
6165 : : /* This function is called on enter to the basic block.
6166 : : Returns TRUE if this block already have been visited and
6167 : : code_motion_path_driver should return 1, FALSE otherwise. */
6168 : : static int
6169 : 10070 : fur_on_enter (insn_t insn ATTRIBUTE_UNUSED, cmpd_local_params_p local_params,
6170 : : void *static_params, bool visited_p)
6171 : : {
6172 : 10070 : fur_static_params_p sparams = (fur_static_params_p) static_params;
6173 : :
6174 : 10070 : if (visited_p)
6175 : : {
6176 : : /* If we have found something below this block, there should be at
6177 : : least one insn in ORIGINAL_INSNS. */
6178 : 334 : gcc_assert (*sparams->original_insns);
6179 : :
6180 : : /* Adjust CROSSED_CALL_ABIS, since we may have come to this block along
6181 : : different path. */
6182 : 334 : DEF_LIST_DEF (*sparams->original_insns)->crossed_call_abis
6183 : 334 : |= sparams->crossed_call_abis;
6184 : : }
6185 : : else
6186 : 9736 : local_params->old_original_insns = *sparams->original_insns;
6187 : :
6188 : 10070 : return 1;
6189 : : }
6190 : :
6191 : : /* Same as above but for move_op. */
6192 : : static int
6193 : 5393 : move_op_on_enter (insn_t insn ATTRIBUTE_UNUSED,
6194 : : cmpd_local_params_p local_params ATTRIBUTE_UNUSED,
6195 : : void *static_params ATTRIBUTE_UNUSED, bool visited_p)
6196 : : {
6197 : 5393 : if (visited_p)
6198 : 0 : return -1;
6199 : : return 1;
6200 : : }
6201 : :
6202 : : /* This function is called while descending current basic block if current
6203 : : insn is not the original EXPR we're searching for.
6204 : :
6205 : : Return value: FALSE, if code_motion_path_driver should perform a local
6206 : : cleanup and return 0 itself;
6207 : : TRUE, if code_motion_path_driver should continue. */
6208 : : static bool
6209 : 4103 : move_op_orig_expr_not_found (insn_t insn, av_set_t orig_ops ATTRIBUTE_UNUSED,
6210 : : void *static_params)
6211 : : {
6212 : 4103 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6213 : :
6214 : 4103 : sparams->failed_insn = insn;
6215 : :
6216 : : /* If we're scheduling separate expr, in order to generate correct code
6217 : : we need to stop the search at bookkeeping code generated with the
6218 : : same destination register or memory. */
6219 : 4103 : if (lhs_of_insn_equals_to_dest_p (insn, sparams->dest))
6220 : 0 : return false;
6221 : : return true;
6222 : : }
6223 : :
6224 : : /* This function is called while descending current basic block if current
6225 : : insn is not the original EXPR we're searching for.
6226 : :
6227 : : Return value: TRUE (code_motion_path_driver should continue). */
6228 : : static bool
6229 : 35782 : fur_orig_expr_not_found (insn_t insn, av_set_t orig_ops, void *static_params)
6230 : : {
6231 : 35782 : bool mutexed;
6232 : 35782 : expr_t r;
6233 : 35782 : av_set_iterator avi;
6234 : 35782 : fur_static_params_p sparams = (fur_static_params_p) static_params;
6235 : :
6236 : 35782 : if (CALL_P (insn))
6237 : 1264 : sparams->crossed_call_abis |= 1 << insn_callee_abi (insn).id ();
6238 : 34518 : else if (DEBUG_INSN_P (insn))
6239 : : return true;
6240 : :
6241 : : /* If current insn we are looking at cannot be executed together
6242 : : with original insn, then we can skip it safely.
6243 : :
6244 : : Example: ORIG_OPS = { (p6) r14 = sign_extend (r15); }
6245 : : INSN = (!p6) r14 = r14 + 1;
6246 : :
6247 : : Here we can schedule ORIG_OP with lhs = r14, though only
6248 : : looking at the set of used and set registers of INSN we must
6249 : : forbid it. So, add set/used in INSN registers to the
6250 : : untouchable set only if there is an insn in ORIG_OPS that can
6251 : : affect INSN. */
6252 : 35779 : mutexed = true;
6253 : 35779 : FOR_EACH_EXPR (r, avi, orig_ops)
6254 : 35779 : if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (r)))
6255 : : {
6256 : : mutexed = false;
6257 : : break;
6258 : : }
6259 : :
6260 : : /* Mark all registers that do not meet the following condition:
6261 : : (1) Not set or read on any path from xi to an instance of the
6262 : : original operation. */
6263 : 35779 : if (!mutexed)
6264 : : {
6265 : 35779 : IOR_REG_SET (sparams->used_regs, INSN_REG_SETS (insn));
6266 : 35779 : IOR_REG_SET (sparams->used_regs, INSN_REG_USES (insn));
6267 : 35779 : IOR_REG_SET (sparams->used_regs, INSN_REG_CLOBBERS (insn));
6268 : : }
6269 : :
6270 : : return true;
6271 : : }
6272 : :
6273 : : /* Hooks and data to perform move_op operations with code_motion_path_driver. */
6274 : : struct code_motion_path_driver_info_def move_op_hooks = {
6275 : : move_op_on_enter,
6276 : : move_op_orig_expr_found,
6277 : : move_op_orig_expr_not_found,
6278 : : move_op_merge_succs,
6279 : : move_op_after_merge_succs,
6280 : : move_op_ascend,
6281 : : move_op_at_first_insn,
6282 : : SUCCS_NORMAL,
6283 : : "move_op"
6284 : : };
6285 : :
6286 : : /* Hooks and data to perform find_used_regs operations
6287 : : with code_motion_path_driver. */
6288 : : struct code_motion_path_driver_info_def fur_hooks = {
6289 : : fur_on_enter,
6290 : : fur_orig_expr_found,
6291 : : fur_orig_expr_not_found,
6292 : : fur_merge_succs,
6293 : : NULL, /* fur_after_merge_succs */
6294 : : NULL, /* fur_ascend */
6295 : : fur_at_first_insn,
6296 : : SUCCS_ALL,
6297 : : "find_used_regs"
6298 : : };
6299 : :
6300 : : /* Traverse all successors of INSN. For each successor that is SUCCS_NORMAL
6301 : : code_motion_path_driver is called recursively. Original operation
6302 : : was found at least on one path that is starting with one of INSN's
6303 : : successors (this fact is asserted). ORIG_OPS is expressions we're looking
6304 : : for, PATH is the path we've traversed, STATIC_PARAMS is the parameters
6305 : : of either move_op or find_used_regs depending on the caller.
6306 : :
6307 : : Return 0 if we haven't found expression, 1 if we found it, -1 if we don't
6308 : : know for sure at this point. */
6309 : : static int
6310 : 5597 : code_motion_process_successors (insn_t insn, av_set_t orig_ops,
6311 : : ilist_t path, void *static_params)
6312 : : {
6313 : 5597 : int res = 0;
6314 : 5597 : succ_iterator succ_i;
6315 : 5597 : insn_t succ;
6316 : 5597 : basic_block bb;
6317 : 5597 : int old_index;
6318 : 5597 : unsigned old_succs;
6319 : :
6320 : 5597 : struct cmpd_local_params lparams;
6321 : 5597 : expr_def _x;
6322 : :
6323 : 5597 : lparams.c_expr_local = &_x;
6324 : 5597 : lparams.c_expr_merged = NULL;
6325 : :
6326 : : /* We need to process only NORMAL succs for move_op, and collect live
6327 : : registers from ALL branches (including those leading out of the
6328 : : region) for find_used_regs.
6329 : :
6330 : : In move_op, there can be a case when insn's bb number has changed
6331 : : due to created bookkeeping. This happens very rare, as we need to
6332 : : move expression from the beginning to the end of the same block.
6333 : : Rescan successors in this case. */
6334 : :
6335 : 5597 : rescan:
6336 : 5597 : bb = BLOCK_FOR_INSN (insn);
6337 : 5597 : old_index = bb->index;
6338 : 5597 : old_succs = EDGE_COUNT (bb->succs);
6339 : :
6340 : 15793 : FOR_EACH_SUCC_1 (succ, succ_i, insn, code_motion_path_driver_info->succ_flags)
6341 : : {
6342 : 10196 : int b;
6343 : :
6344 : 10196 : lparams.e1 = succ_i.e1;
6345 : 10196 : lparams.e2 = succ_i.e2;
6346 : :
6347 : : /* Go deep into recursion only for NORMAL edges (non-backedges within the
6348 : : current region). */
6349 : 10196 : if (succ_i.current_flags == SUCCS_NORMAL)
6350 : 7767 : b = code_motion_path_driver (succ, orig_ops, path, &lparams,
6351 : : static_params);
6352 : : else
6353 : : b = 0;
6354 : :
6355 : : /* Merge c_expres found or unify live register sets from different
6356 : : successors. */
6357 : 10196 : code_motion_path_driver_info->merge_succs (insn, succ, b, &lparams,
6358 : : static_params);
6359 : 10196 : if (b == 1)
6360 : : res = b;
6361 : 3274 : else if (b == -1 && res != 1)
6362 : 0 : res = b;
6363 : :
6364 : : /* We have simplified the control flow below this point. In this case,
6365 : : the iterator becomes invalid. We need to try again.
6366 : : If we have removed the insn itself, it could be only an
6367 : : unconditional jump. Thus, do not rescan but break immediately --
6368 : : we have already visited the only successor block. */
6369 : 10196 : if (!BLOCK_FOR_INSN (insn))
6370 : : {
6371 : 0 : if (sched_verbose >= 6)
6372 : 0 : sel_print ("Not doing rescan: already visited the only successor"
6373 : : " of block %d\n", old_index);
6374 : : break;
6375 : : }
6376 : 10196 : if (BLOCK_FOR_INSN (insn)->index != old_index
6377 : 20392 : || EDGE_COUNT (bb->succs) != old_succs)
6378 : : {
6379 : 0 : if (sched_verbose >= 6)
6380 : 0 : sel_print ("Rescan: CFG was simplified below insn %d, block %d\n",
6381 : 0 : INSN_UID (insn), BLOCK_FOR_INSN (insn)->index);
6382 : 0 : insn = sel_bb_end (BLOCK_FOR_INSN (insn));
6383 : 0 : goto rescan;
6384 : : }
6385 : : }
6386 : :
6387 : : /* Here, RES==1 if original expr was found at least for one of the
6388 : : successors. After the loop, RES may happen to have zero value
6389 : : only if at some point the expr searched is present in av_set, but is
6390 : : not found below. In most cases, this situation is an error.
6391 : : The exception is when the original operation is blocked by
6392 : : bookkeeping generated for another fence or for another path in current
6393 : : move_op. */
6394 : 5597 : gcc_checking_assert (res == 1
6395 : : || (res == 0
6396 : : && av_set_could_be_blocked_by_bookkeeping_p (orig_ops, static_params))
6397 : : || res == -1);
6398 : :
6399 : : /* Merge data, clean up, etc. */
6400 : 5597 : if (res != -1 && code_motion_path_driver_info->after_merge_succs)
6401 : 378 : code_motion_path_driver_info->after_merge_succs (&lparams, static_params);
6402 : :
6403 : 5597 : return res;
6404 : : }
6405 : :
6406 : :
6407 : : /* Perform a cleanup when the driver is about to terminate. ORIG_OPS_P
6408 : : is the pointer to the av set with expressions we were looking for,
6409 : : PATH_P is the pointer to the traversed path. */
6410 : : static inline void
6411 : 0 : code_motion_path_driver_cleanup (av_set_t *orig_ops_p, ilist_t *path_p)
6412 : : {
6413 : 0 : ilist_remove (path_p);
6414 : 0 : av_set_clear (orig_ops_p);
6415 : 0 : }
6416 : :
6417 : : /* The driver function that implements move_op or find_used_regs
6418 : : functionality dependent whether code_motion_path_driver_INFO is set to
6419 : : &MOVE_OP_HOOKS or &FUR_HOOKS. This function implements the common parts
6420 : : of code (CFG traversal etc) that are shared among both functions. INSN
6421 : : is the insn we're starting the search from, ORIG_OPS are the expressions
6422 : : we're searching for, PATH is traversed path, LOCAL_PARAMS_IN are local
6423 : : parameters of the driver, and STATIC_PARAMS are static parameters of
6424 : : the caller.
6425 : :
6426 : : Returns whether original instructions were found. Note that top-level
6427 : : code_motion_path_driver always returns true. */
6428 : : static int
6429 : 15471 : code_motion_path_driver (insn_t insn, av_set_t orig_ops, ilist_t path,
6430 : : cmpd_local_params_p local_params_in,
6431 : : void *static_params)
6432 : : {
6433 : 15471 : expr_t expr = NULL;
6434 : 15471 : basic_block bb = BLOCK_FOR_INSN (insn);
6435 : 15471 : insn_t first_insn, original_insn, bb_tail, before_first;
6436 : 15471 : bool removed_last_insn = false;
6437 : :
6438 : 15471 : if (sched_verbose >= 6)
6439 : : {
6440 : 0 : sel_print ("%s (", code_motion_path_driver_info->routine_name);
6441 : 0 : dump_insn (insn);
6442 : 0 : sel_print (",");
6443 : 0 : dump_av_set (orig_ops);
6444 : 0 : sel_print (")\n");
6445 : : }
6446 : :
6447 : 15471 : gcc_assert (orig_ops);
6448 : :
6449 : : /* If no original operations exist below this insn, return immediately. */
6450 : 15471 : if (is_ineligible_successor (insn, path))
6451 : : {
6452 : 8 : if (sched_verbose >= 6)
6453 : 0 : sel_print ("Insn %d is ineligible successor\n", INSN_UID (insn));
6454 : 8 : return false;
6455 : : }
6456 : :
6457 : : /* The block can have invalid av set, in which case it was created earlier
6458 : : during move_op. Return immediately. */
6459 : 15463 : if (sel_bb_head_p (insn))
6460 : : {
6461 : 7980 : if (! AV_SET_VALID_P (insn))
6462 : : {
6463 : 0 : if (sched_verbose >= 6)
6464 : 0 : sel_print ("Returned from block %d as it had invalid av set\n",
6465 : : bb->index);
6466 : 0 : return false;
6467 : : }
6468 : :
6469 : 7980 : if (bitmap_bit_p (code_motion_visited_blocks, bb->index))
6470 : : {
6471 : : /* We have already found an original operation on this branch, do not
6472 : : go any further and just return TRUE here. If we don't stop here,
6473 : : function can have exponential behavior even on the small code
6474 : : with many different paths (e.g. with data speculation and
6475 : : recovery blocks). */
6476 : 334 : if (sched_verbose >= 6)
6477 : 0 : sel_print ("Block %d already visited in this traversal\n", bb->index);
6478 : 334 : if (code_motion_path_driver_info->on_enter)
6479 : 334 : return code_motion_path_driver_info->on_enter (insn,
6480 : : local_params_in,
6481 : : static_params,
6482 : 334 : true);
6483 : : }
6484 : : }
6485 : :
6486 : 15129 : if (code_motion_path_driver_info->on_enter)
6487 : 15129 : code_motion_path_driver_info->on_enter (insn, local_params_in,
6488 : : static_params, false);
6489 : 15129 : orig_ops = av_set_copy (orig_ops);
6490 : :
6491 : : /* Filter the orig_ops set. */
6492 : 15129 : if (AV_SET_VALID_P (insn))
6493 : 7646 : av_set_code_motion_filter (&orig_ops, AV_SET (insn));
6494 : :
6495 : : /* If no more original ops, return immediately. */
6496 : 15129 : if (!orig_ops)
6497 : : {
6498 : 837 : if (sched_verbose >= 6)
6499 : 0 : sel_print ("No intersection with av set of block %d\n", bb->index);
6500 : 837 : return false;
6501 : : }
6502 : :
6503 : : /* For non-speculative insns we have to leave only one form of the
6504 : : original operation, because if we don't, we may end up with
6505 : : different C_EXPRes and, consequently, with bookkeepings for different
6506 : : expression forms along the same code motion path. That may lead to
6507 : : generation of incorrect code. So for each code motion we stick to
6508 : : the single form of the instruction, except for speculative insns
6509 : : which we need to keep in different forms with all speculation
6510 : : types. */
6511 : 14292 : av_set_leave_one_nonspec (&orig_ops);
6512 : :
6513 : : /* It is not possible that all ORIG_OPS are filtered out. */
6514 : 14292 : gcc_assert (orig_ops);
6515 : :
6516 : : /* It is enough to place only heads and tails of visited basic blocks into
6517 : : the PATH. */
6518 : 14292 : ilist_add (&path, insn);
6519 : 14292 : first_insn = original_insn = insn;
6520 : 14292 : bb_tail = sel_bb_end (bb);
6521 : :
6522 : : /* Descend the basic block in search of the original expr; this part
6523 : : corresponds to the part of the original move_op procedure executed
6524 : : before the recursive call. */
6525 : 34288 : for (;;)
6526 : : {
6527 : : /* Look at the insn and decide if it could be an ancestor of currently
6528 : : scheduling operation. If it is so, then the insn "dest = op" could
6529 : : either be replaced with "dest = reg", because REG now holds the result
6530 : : of OP, or just removed, if we've scheduled the insn as a whole.
6531 : :
6532 : : If this insn doesn't contain currently scheduling OP, then proceed
6533 : : with searching and look at its successors. Operations we're searching
6534 : : for could have changed when moving up through this insn via
6535 : : substituting. In this case, perform unsubstitution on them first.
6536 : :
6537 : : When traversing the DAG below this insn is finished, insert
6538 : : bookkeeping code, if the insn is a joint point, and remove
6539 : : leftovers. */
6540 : :
6541 : 48580 : expr = av_set_lookup (orig_ops, INSN_VINSN (insn));
6542 : 48580 : if (expr)
6543 : : {
6544 : 8695 : insn_t last_insn = PREV_INSN (insn);
6545 : :
6546 : : /* We have found the original operation. */
6547 : 8695 : if (sched_verbose >= 6)
6548 : 0 : sel_print ("Found original operation at insn %d\n", INSN_UID (insn));
6549 : :
6550 : 8695 : code_motion_path_driver_info->orig_expr_found
6551 : 8695 : (insn, expr, local_params_in, static_params);
6552 : :
6553 : : /* Step back, so on the way back we'll start traversing from the
6554 : : previous insn (or we'll see that it's bb_note and skip that
6555 : : loop). */
6556 : 8695 : if (insn == first_insn)
6557 : : {
6558 : 4825 : first_insn = NEXT_INSN (last_insn);
6559 : 4825 : removed_last_insn = sel_bb_end_p (last_insn);
6560 : : }
6561 : : insn = last_insn;
6562 : : break;
6563 : : }
6564 : : else
6565 : : {
6566 : : /* We haven't found the original expr, continue descending the basic
6567 : : block. */
6568 : 39885 : if (code_motion_path_driver_info->orig_expr_not_found
6569 : 39885 : (insn, orig_ops, static_params))
6570 : : {
6571 : : /* Av set ops could have been changed when moving through this
6572 : : insn. To find them below it, we have to un-substitute them. */
6573 : 39885 : undo_transformations (&orig_ops, insn);
6574 : : }
6575 : : else
6576 : : {
6577 : : /* Clean up and return, if the hook tells us to do so. It may
6578 : : happen if we've encountered the previously created
6579 : : bookkeeping. */
6580 : 0 : code_motion_path_driver_cleanup (&orig_ops, &path);
6581 : 0 : return -1;
6582 : : }
6583 : :
6584 : 39885 : gcc_assert (orig_ops);
6585 : : }
6586 : :
6587 : : /* Stop at insn if we got to the end of BB. */
6588 : 39885 : if (insn == bb_tail)
6589 : : break;
6590 : :
6591 : 34288 : insn = NEXT_INSN (insn);
6592 : 34288 : }
6593 : :
6594 : : /* Here INSN either points to the insn before the original insn (may be
6595 : : bb_note, if original insn was a bb_head) or to the bb_end. */
6596 : 14292 : if (!expr)
6597 : : {
6598 : 5597 : int res;
6599 : 5597 : rtx_insn *last_insn = PREV_INSN (insn);
6600 : 5597 : bool added_to_path;
6601 : :
6602 : 5597 : gcc_assert (insn == sel_bb_end (bb));
6603 : :
6604 : : /* Add bb tail to PATH (but it doesn't make any sense if it's a bb_head -
6605 : : it's already in PATH then). */
6606 : 5597 : if (insn != first_insn)
6607 : : {
6608 : 4397 : ilist_add (&path, insn);
6609 : 4397 : added_to_path = true;
6610 : : }
6611 : : else
6612 : : added_to_path = false;
6613 : :
6614 : : /* Process_successors should be able to find at least one
6615 : : successor for which code_motion_path_driver returns TRUE. */
6616 : 5597 : res = code_motion_process_successors (insn, orig_ops,
6617 : : path, static_params);
6618 : :
6619 : : /* Jump in the end of basic block could have been removed or replaced
6620 : : during code_motion_process_successors, so recompute insn as the
6621 : : last insn in bb. */
6622 : 5597 : if (NEXT_INSN (last_insn) != insn)
6623 : : {
6624 : 0 : insn = sel_bb_end (bb);
6625 : 0 : first_insn = sel_bb_head (bb);
6626 : 0 : if (first_insn != original_insn)
6627 : : first_insn = original_insn;
6628 : : }
6629 : :
6630 : : /* Remove bb tail from path. */
6631 : 5597 : if (added_to_path)
6632 : 4397 : ilist_remove (&path);
6633 : :
6634 : 5597 : if (res != 1)
6635 : : {
6636 : : /* This is the case when one of the original expr is no longer available
6637 : : due to bookkeeping created on this branch with the same register.
6638 : : In the original algorithm, which doesn't have update_data_sets call
6639 : : on a bookkeeping block, it would simply result in returning
6640 : : FALSE when we've encountered a previously generated bookkeeping
6641 : : insn in moveop_orig_expr_not_found. */
6642 : 0 : code_motion_path_driver_cleanup (&orig_ops, &path);
6643 : 0 : return res;
6644 : : }
6645 : : }
6646 : :
6647 : : /* Don't need it any more. */
6648 : 14292 : av_set_clear (&orig_ops);
6649 : :
6650 : : /* Backward pass: now, when we have C_EXPR computed, we'll drag it to
6651 : : the beginning of the basic block. */
6652 : 14292 : before_first = PREV_INSN (first_insn);
6653 : 68469 : while (insn != before_first)
6654 : : {
6655 : 39885 : if (code_motion_path_driver_info->ascend)
6656 : 4103 : code_motion_path_driver_info->ascend (insn, static_params);
6657 : :
6658 : 39885 : insn = PREV_INSN (insn);
6659 : : }
6660 : :
6661 : : /* Now we're at the bb head. */
6662 : 14292 : insn = first_insn;
6663 : 14292 : ilist_remove (&path);
6664 : 14292 : local_params_in->removed_last_insn = removed_last_insn;
6665 : 14292 : code_motion_path_driver_info->at_first_insn (insn, local_params_in, static_params);
6666 : :
6667 : : /* This should be the very last operation as at bb head we could change
6668 : : the numbering by creating bookkeeping blocks. */
6669 : 14292 : if (removed_last_insn)
6670 : 763 : insn = PREV_INSN (insn);
6671 : :
6672 : : /* If we have simplified the control flow and removed the first jump insn,
6673 : : there's no point in marking this block in the visited blocks bitmap. */
6674 : 14292 : if (BLOCK_FOR_INSN (insn))
6675 : 14292 : bitmap_set_bit (code_motion_visited_blocks, BLOCK_FOR_INSN (insn)->index);
6676 : : return true;
6677 : : }
6678 : :
6679 : : /* Move up the operations from ORIG_OPS set traversing the dag starting
6680 : : from INSN. PATH represents the edges traversed so far.
6681 : : DEST is the register chosen for scheduling the current expr. Insert
6682 : : bookkeeping code in the join points. EXPR_VLIW is the chosen expression,
6683 : : C_EXPR is how it looks like at the given cfg point.
6684 : : Set *SHOULD_MOVE to indicate whether we have only disconnected
6685 : : one of the insns found.
6686 : :
6687 : : Returns whether original instructions were found, which is asserted
6688 : : to be true in the caller. */
6689 : : static bool
6690 : 4883 : move_op (insn_t insn, av_set_t orig_ops, expr_t expr_vliw,
6691 : : rtx dest, expr_t c_expr, bool *should_move)
6692 : : {
6693 : 4883 : struct moveop_static_params sparams;
6694 : 4883 : struct cmpd_local_params lparams;
6695 : 4883 : int res;
6696 : :
6697 : : /* Init params for code_motion_path_driver. */
6698 : 4883 : sparams.dest = dest;
6699 : 4883 : sparams.c_expr = c_expr;
6700 : 4883 : sparams.uid = INSN_UID (EXPR_INSN_RTX (expr_vliw));
6701 : 4883 : sparams.failed_insn = NULL;
6702 : 4883 : sparams.was_renamed = false;
6703 : 4883 : lparams.e1 = NULL;
6704 : :
6705 : : /* We haven't visited any blocks yet. */
6706 : 4883 : bitmap_clear (code_motion_visited_blocks);
6707 : :
6708 : : /* Set appropriate hooks and data. */
6709 : 4883 : code_motion_path_driver_info = &move_op_hooks;
6710 : 4883 : res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
6711 : :
6712 : 4883 : gcc_assert (res != -1);
6713 : :
6714 : 4883 : if (sparams.was_renamed)
6715 : 30 : EXPR_WAS_RENAMED (expr_vliw) = true;
6716 : :
6717 : 4883 : *should_move = (sparams.uid == -1);
6718 : :
6719 : 4883 : return res;
6720 : : }
6721 : :
6722 : :
6723 : : /* Functions that work with regions. */
6724 : :
6725 : : /* Current number of seqno used in init_seqno and init_seqno_1. */
6726 : : static int cur_seqno;
6727 : :
6728 : : /* A helper for init_seqno. Traverse the region starting from BB and
6729 : : compute seqnos for visited insns, marking visited bbs in VISITED_BBS.
6730 : : Clear visited blocks from BLOCKS_TO_RESCHEDULE. */
6731 : : static void
6732 : 1122 : init_seqno_1 (basic_block bb, sbitmap visited_bbs, bitmap blocks_to_reschedule)
6733 : : {
6734 : 1122 : int bbi = BLOCK_TO_BB (bb->index);
6735 : 1122 : insn_t insn;
6736 : 1122 : insn_t succ_insn;
6737 : 1122 : succ_iterator si;
6738 : :
6739 : 1122 : rtx_note *note = bb_note (bb);
6740 : 1122 : bitmap_set_bit (visited_bbs, bbi);
6741 : 1122 : if (blocks_to_reschedule)
6742 : 55 : bitmap_clear_bit (blocks_to_reschedule, bb->index);
6743 : :
6744 : 1588 : FOR_EACH_SUCC_1 (succ_insn, si, BB_END (bb),
6745 : : SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
6746 : : {
6747 : 466 : basic_block succ = BLOCK_FOR_INSN (succ_insn);
6748 : 466 : int succ_bbi = BLOCK_TO_BB (succ->index);
6749 : :
6750 : 466 : gcc_assert (in_current_region_p (succ));
6751 : :
6752 : 466 : if (!bitmap_bit_p (visited_bbs, succ_bbi))
6753 : : {
6754 : 319 : gcc_assert (succ_bbi > bbi);
6755 : :
6756 : 319 : init_seqno_1 (succ, visited_bbs, blocks_to_reschedule);
6757 : : }
6758 : 147 : else if (blocks_to_reschedule)
6759 : 23 : bitmap_set_bit (forced_ebb_heads, succ->index);
6760 : : }
6761 : :
6762 : 5998 : for (insn = BB_END (bb); insn != note; insn = PREV_INSN (insn))
6763 : 4876 : INSN_SEQNO (insn) = cur_seqno--;
6764 : 1122 : }
6765 : :
6766 : : /* Initialize seqnos for the current region. BLOCKS_TO_RESCHEDULE contains
6767 : : blocks on which we're rescheduling when pipelining, FROM is the block where
6768 : : traversing region begins (it may not be the head of the region when
6769 : : pipelining, but the head of the loop instead).
6770 : :
6771 : : Returns the maximal seqno found. */
6772 : : static int
6773 : 803 : init_seqno (bitmap blocks_to_reschedule, basic_block from)
6774 : : {
6775 : 803 : bitmap_iterator bi;
6776 : 803 : unsigned bbi;
6777 : :
6778 : 803 : auto_sbitmap visited_bbs (current_nr_blocks);
6779 : :
6780 : 803 : if (blocks_to_reschedule)
6781 : : {
6782 : 21 : bitmap_ones (visited_bbs);
6783 : 99 : EXECUTE_IF_SET_IN_BITMAP (blocks_to_reschedule, 0, bbi, bi)
6784 : : {
6785 : 78 : gcc_assert (BLOCK_TO_BB (bbi) < current_nr_blocks);
6786 : 78 : bitmap_clear_bit (visited_bbs, BLOCK_TO_BB (bbi));
6787 : : }
6788 : : }
6789 : : else
6790 : : {
6791 : 782 : bitmap_clear (visited_bbs);
6792 : 782 : from = EBB_FIRST_BB (0);
6793 : : }
6794 : :
6795 : 803 : cur_seqno = sched_max_luid - 1;
6796 : 803 : init_seqno_1 (from, visited_bbs, blocks_to_reschedule);
6797 : :
6798 : : /* cur_seqno may be positive if the number of instructions is less than
6799 : : sched_max_luid - 1 (when rescheduling or if some instructions have been
6800 : : removed by the call to purge_empty_blocks in sel_sched_region_1). */
6801 : 803 : gcc_assert (cur_seqno >= 0);
6802 : :
6803 : 803 : return sched_max_luid - 1;
6804 : 803 : }
6805 : :
6806 : : /* Initialize scheduling parameters for current region. */
6807 : : static void
6808 : 782 : sel_setup_region_sched_flags (void)
6809 : : {
6810 : 782 : enable_schedule_as_rhs_p = 1;
6811 : 782 : bookkeeping_p = 1;
6812 : 1564 : pipelining_p = (bookkeeping_p
6813 : 782 : && (flag_sel_sched_pipelining != 0)
6814 : 384 : && current_loop_nest != NULL
6815 : 55 : && loop_has_exit_edges (current_loop_nest));
6816 : 782 : max_insns_to_rename = param_selsched_insns_to_rename;
6817 : 782 : max_ws = MAX_WS;
6818 : 782 : }
6819 : :
6820 : : /* Return true if all basic blocks of current region are empty. */
6821 : : static bool
6822 : 802 : current_region_empty_p (void)
6823 : : {
6824 : 802 : int i;
6825 : 847 : for (i = 0; i < current_nr_blocks; i++)
6826 : 827 : if (! sel_bb_empty_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i))))
6827 : : return false;
6828 : :
6829 : : return true;
6830 : : }
6831 : :
6832 : : /* Prepare and verify loop nest for pipelining. */
6833 : : static void
6834 : 384 : setup_current_loop_nest (int rgn, bb_vec_t *bbs)
6835 : : {
6836 : 384 : current_loop_nest = get_loop_nest_for_rgn (rgn);
6837 : :
6838 : 384 : if (!current_loop_nest)
6839 : : return;
6840 : :
6841 : : /* If this loop has any saved loop preheaders from nested loops,
6842 : : add these basic blocks to the current region. */
6843 : 55 : sel_add_loop_preheaders (bbs);
6844 : :
6845 : : /* Check that we're starting with a valid information. */
6846 : 55 : gcc_assert (loop_latch_edge (current_loop_nest));
6847 : 55 : gcc_assert (LOOP_MARKED_FOR_PIPELINING_P (current_loop_nest));
6848 : : }
6849 : :
6850 : : /* Compute instruction priorities for current region. */
6851 : : static void
6852 : 782 : sel_compute_priorities (int rgn)
6853 : : {
6854 : 782 : sched_rgn_compute_dependencies (rgn);
6855 : :
6856 : : /* Compute insn priorities in haifa style. Then free haifa style
6857 : : dependencies that we've calculated for this. */
6858 : 782 : compute_priorities ();
6859 : :
6860 : 782 : if (sched_verbose >= 5)
6861 : 0 : debug_rgn_dependencies (0);
6862 : :
6863 : 782 : free_rgn_deps ();
6864 : 782 : }
6865 : :
6866 : : /* Init scheduling data for RGN. Returns true when this region should not
6867 : : be scheduled. */
6868 : : static bool
6869 : 802 : sel_region_init (int rgn)
6870 : : {
6871 : 802 : int i;
6872 : 802 : bb_vec_t bbs;
6873 : :
6874 : 802 : rgn_setup_region (rgn);
6875 : :
6876 : : /* Even if sched_is_disabled_for_current_region_p() is true, we still
6877 : : do region initialization here so the region can be bundled correctly,
6878 : : but we'll skip the scheduling in sel_sched_region (). */
6879 : 802 : if (current_region_empty_p ())
6880 : : return true;
6881 : :
6882 : 782 : bbs.create (current_nr_blocks);
6883 : :
6884 : 2631 : for (i = 0; i < current_nr_blocks; i++)
6885 : 1067 : bbs.quick_push (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i)));
6886 : :
6887 : 782 : sel_init_bbs (bbs);
6888 : :
6889 : 782 : if (flag_sel_sched_pipelining)
6890 : 384 : setup_current_loop_nest (rgn, &bbs);
6891 : :
6892 : 782 : sel_setup_region_sched_flags ();
6893 : :
6894 : : /* Initialize luids and dependence analysis which both sel-sched and haifa
6895 : : need. */
6896 : 782 : sched_init_luids (bbs);
6897 : 782 : sched_deps_init (false);
6898 : :
6899 : : /* Initialize haifa data. */
6900 : 782 : rgn_setup_sched_infos ();
6901 : 782 : sel_set_sched_flags ();
6902 : 782 : haifa_init_h_i_d (bbs);
6903 : :
6904 : 782 : sel_compute_priorities (rgn);
6905 : 782 : init_deps_global ();
6906 : :
6907 : : /* Main initialization. */
6908 : 782 : sel_setup_sched_infos ();
6909 : 782 : sel_init_global_and_expr (bbs);
6910 : :
6911 : 782 : bbs.release ();
6912 : :
6913 : 782 : blocks_to_reschedule = BITMAP_ALLOC (NULL);
6914 : :
6915 : : /* Init correct liveness sets on each instruction of a single-block loop.
6916 : : This is the only situation when we can't update liveness when calling
6917 : : compute_live for the first insn of the loop. */
6918 : 782 : if (current_loop_nest)
6919 : : {
6920 : 55 : int header =
6921 : 55 : (sel_is_loop_preheader_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (0)))
6922 : 55 : ? 1
6923 : 55 : : 0);
6924 : :
6925 : 55 : if (current_nr_blocks == header + 1)
6926 : 15 : update_liveness_on_insn
6927 : 15 : (sel_bb_head (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (header))));
6928 : : }
6929 : :
6930 : : /* Set hooks so that no newly generated insn will go out unnoticed. */
6931 : 782 : sel_register_cfg_hooks ();
6932 : :
6933 : : /* !!! We call target.sched.init () for the whole region, but we invoke
6934 : : targetm.sched.finish () for every ebb. */
6935 : 782 : if (targetm.sched.init)
6936 : : /* None of the arguments are actually used in any target. */
6937 : 0 : targetm.sched.init (sched_dump, sched_verbose, -1);
6938 : :
6939 : 782 : first_emitted_uid = get_max_uid () + 1;
6940 : 782 : preheader_removed = false;
6941 : :
6942 : : /* Reset register allocation ticks array. */
6943 : 782 : memset (reg_rename_tick, 0, sizeof reg_rename_tick);
6944 : 782 : reg_rename_this_tick = 0;
6945 : :
6946 : 782 : forced_ebb_heads = BITMAP_ALLOC (NULL);
6947 : :
6948 : 782 : setup_nop_vinsn ();
6949 : 782 : current_copies = BITMAP_ALLOC (NULL);
6950 : 782 : current_originators = BITMAP_ALLOC (NULL);
6951 : 782 : code_motion_visited_blocks = BITMAP_ALLOC (NULL);
6952 : :
6953 : 782 : return false;
6954 : : }
6955 : :
6956 : : /* Simplify insns after the scheduling. */
6957 : : static void
6958 : 782 : simplify_changed_insns (void)
6959 : : {
6960 : 782 : int i;
6961 : :
6962 : 1861 : for (i = 0; i < current_nr_blocks; i++)
6963 : : {
6964 : 1079 : basic_block bb = BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i));
6965 : 1079 : rtx_insn *insn;
6966 : :
6967 : 7254 : FOR_BB_INSNS (bb, insn)
6968 : 6175 : if (INSN_P (insn))
6969 : : {
6970 : 4585 : expr_t expr = INSN_EXPR (insn);
6971 : :
6972 : 4585 : if (EXPR_WAS_SUBSTITUTED (expr))
6973 : 0 : validate_simplify_insn (insn);
6974 : : }
6975 : : }
6976 : 782 : }
6977 : :
6978 : : /* Find boundaries of the EBB starting from basic block BB, marking blocks of
6979 : : this EBB in SCHEDULED_BLOCKS and appropriately filling in HEAD, TAIL,
6980 : : PREV_HEAD, and NEXT_TAIL fields of CURRENT_SCHED_INFO structure. */
6981 : : static void
6982 : 625 : find_ebb_boundaries (basic_block bb, bitmap scheduled_blocks)
6983 : : {
6984 : 625 : rtx_insn *head, *tail;
6985 : 625 : basic_block bb1 = bb;
6986 : 625 : if (sched_verbose >= 2)
6987 : 0 : sel_print ("Finishing schedule in bbs: ");
6988 : :
6989 : 749 : do
6990 : : {
6991 : 749 : bitmap_set_bit (scheduled_blocks, BLOCK_TO_BB (bb1->index));
6992 : :
6993 : 749 : if (sched_verbose >= 2)
6994 : 0 : sel_print ("%d; ", bb1->index);
6995 : : }
6996 : 749 : while (!bb_ends_ebb_p (bb1) && (bb1 = bb_next_bb (bb1)));
6997 : :
6998 : 625 : if (sched_verbose >= 2)
6999 : 0 : sel_print ("\n");
7000 : :
7001 : 625 : get_ebb_head_tail (bb, bb1, &head, &tail);
7002 : :
7003 : 625 : current_sched_info->head = head;
7004 : 625 : current_sched_info->tail = tail;
7005 : 625 : current_sched_info->prev_head = PREV_INSN (head);
7006 : 625 : current_sched_info->next_tail = NEXT_INSN (tail);
7007 : 625 : }
7008 : :
7009 : : /* Regenerate INSN_SCHED_CYCLEs for insns of current EBB. */
7010 : : static void
7011 : 104 : reset_sched_cycles_in_current_ebb (void)
7012 : : {
7013 : 104 : int last_clock = 0;
7014 : 104 : int haifa_last_clock = -1;
7015 : 104 : int haifa_clock = 0;
7016 : 104 : int issued_insns = 0;
7017 : 104 : insn_t insn;
7018 : :
7019 : 104 : if (targetm.sched.init)
7020 : : {
7021 : : /* None of the arguments are actually used in any target.
7022 : : NB: We should have md_reset () hook for cases like this. */
7023 : 0 : targetm.sched.init (sched_dump, sched_verbose, -1);
7024 : : }
7025 : :
7026 : 104 : state_reset (curr_state);
7027 : 104 : advance_state (curr_state);
7028 : :
7029 : 1267 : for (insn = current_sched_info->head;
7030 : 1267 : insn != current_sched_info->next_tail;
7031 : 1163 : insn = NEXT_INSN (insn))
7032 : : {
7033 : 1163 : int cost, haifa_cost;
7034 : 1163 : int sort_p;
7035 : 1163 : bool asm_p, real_insn, after_stall, all_issued;
7036 : 1163 : int clock;
7037 : :
7038 : 1163 : if (!INSN_P (insn))
7039 : 97 : continue;
7040 : :
7041 : 1066 : asm_p = false;
7042 : 1066 : real_insn = recog_memoized (insn) >= 0;
7043 : 1066 : clock = INSN_SCHED_CYCLE (insn);
7044 : :
7045 : 1066 : cost = clock - last_clock;
7046 : :
7047 : : /* Initialize HAIFA_COST. */
7048 : 1066 : if (! real_insn)
7049 : : {
7050 : 5 : asm_p = INSN_ASM_P (insn);
7051 : :
7052 : 5 : if (asm_p)
7053 : : /* This is asm insn which *had* to be scheduled first
7054 : : on the cycle. */
7055 : : haifa_cost = 1;
7056 : : else
7057 : : /* This is a use/clobber insn. It should not change
7058 : : cost. */
7059 : 2 : haifa_cost = 0;
7060 : : }
7061 : : else
7062 : 1061 : haifa_cost = estimate_insn_cost (insn, curr_state);
7063 : :
7064 : : /* Stall for whatever cycles we've stalled before. */
7065 : 1066 : after_stall = 0;
7066 : 1066 : if (INSN_AFTER_STALL_P (insn) && cost > haifa_cost)
7067 : : {
7068 : : haifa_cost = cost;
7069 : : after_stall = 1;
7070 : : }
7071 : 1066 : all_issued = issued_insns == issue_rate;
7072 : 1066 : if (haifa_cost == 0 && all_issued)
7073 : 0 : haifa_cost = 1;
7074 : 1066 : if (haifa_cost > 0)
7075 : : {
7076 : : int i = 0;
7077 : :
7078 : 1515 : while (haifa_cost--)
7079 : : {
7080 : 841 : advance_state (curr_state);
7081 : 841 : issued_insns = 0;
7082 : 841 : i++;
7083 : :
7084 : 841 : if (sched_verbose >= 2)
7085 : : {
7086 : 0 : sel_print ("advance_state (state_transition)\n");
7087 : 0 : debug_state (curr_state);
7088 : : }
7089 : :
7090 : : /* The DFA may report that e.g. insn requires 2 cycles to be
7091 : : issued, but on the next cycle it says that insn is ready
7092 : : to go. Check this here. */
7093 : 841 : if (!after_stall
7094 : 841 : && real_insn
7095 : 182 : && haifa_cost > 0
7096 : 853 : && estimate_insn_cost (insn, curr_state) == 0)
7097 : : break;
7098 : :
7099 : : /* When the data dependency stall is longer than the DFA stall,
7100 : : and when we have issued exactly issue_rate insns and stalled,
7101 : : it could be that after this longer stall the insn will again
7102 : : become unavailable to the DFA restrictions. Looks strange
7103 : : but happens e.g. on x86-64. So recheck DFA on the last
7104 : : iteration. */
7105 : 841 : if ((after_stall || all_issued)
7106 : : && real_insn
7107 : 705 : && haifa_cost == 0)
7108 : 550 : haifa_cost = estimate_insn_cost (insn, curr_state);
7109 : : }
7110 : :
7111 : 674 : haifa_clock += i;
7112 : 674 : if (sched_verbose >= 2)
7113 : 0 : sel_print ("haifa clock: %d\n", haifa_clock);
7114 : : }
7115 : : else
7116 : : gcc_assert (haifa_cost == 0);
7117 : :
7118 : 1066 : if (sched_verbose >= 2)
7119 : 0 : sel_print ("Haifa cost for insn %d: %d\n", INSN_UID (insn), haifa_cost);
7120 : :
7121 : 1066 : if (targetm.sched.dfa_new_cycle)
7122 : 0 : while (targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, insn,
7123 : : haifa_last_clock, haifa_clock,
7124 : : &sort_p))
7125 : : {
7126 : 0 : advance_state (curr_state);
7127 : 0 : issued_insns = 0;
7128 : 0 : haifa_clock++;
7129 : 0 : if (sched_verbose >= 2)
7130 : : {
7131 : 0 : sel_print ("advance_state (dfa_new_cycle)\n");
7132 : 0 : debug_state (curr_state);
7133 : 0 : sel_print ("haifa clock: %d\n", haifa_clock + 1);
7134 : : }
7135 : : }
7136 : :
7137 : 1066 : if (real_insn)
7138 : : {
7139 : 1061 : static state_t temp = NULL;
7140 : :
7141 : 1061 : if (!temp)
7142 : 26 : temp = xmalloc (dfa_state_size);
7143 : 1061 : memcpy (temp, curr_state, dfa_state_size);
7144 : :
7145 : 1061 : cost = state_transition (curr_state, insn);
7146 : 1061 : if (memcmp (temp, curr_state, dfa_state_size))
7147 : 1058 : issued_insns++;
7148 : :
7149 : 1061 : if (sched_verbose >= 2)
7150 : : {
7151 : 0 : sel_print ("scheduled insn %d, clock %d\n", INSN_UID (insn),
7152 : : haifa_clock + 1);
7153 : 0 : debug_state (curr_state);
7154 : : }
7155 : 1061 : gcc_assert (cost < 0);
7156 : : }
7157 : :
7158 : 1066 : if (targetm.sched.variable_issue)
7159 : 0 : targetm.sched.variable_issue (sched_dump, sched_verbose, insn, 0);
7160 : :
7161 : 1066 : INSN_SCHED_CYCLE (insn) = haifa_clock;
7162 : :
7163 : 1066 : last_clock = clock;
7164 : 1066 : haifa_last_clock = haifa_clock;
7165 : : }
7166 : 104 : }
7167 : :
7168 : : /* Put TImode markers on insns starting a new issue group. */
7169 : : static void
7170 : 625 : put_TImodes (void)
7171 : : {
7172 : 625 : int last_clock = -1;
7173 : 625 : insn_t insn;
7174 : :
7175 : 3970 : for (insn = current_sched_info->head; insn != current_sched_info->next_tail;
7176 : 3345 : insn = NEXT_INSN (insn))
7177 : : {
7178 : 3345 : int cost, clock;
7179 : :
7180 : 3345 : if (!INSN_P (insn))
7181 : 126 : continue;
7182 : :
7183 : 3219 : clock = INSN_SCHED_CYCLE (insn);
7184 : 3219 : cost = (last_clock == -1) ? 1 : clock - last_clock;
7185 : :
7186 : 2581 : gcc_assert (cost >= 0);
7187 : :
7188 : 3219 : if (issue_rate > 1
7189 : 3219 : && GET_CODE (PATTERN (insn)) != USE
7190 : 6412 : && GET_CODE (PATTERN (insn)) != CLOBBER)
7191 : : {
7192 : 3187 : if (reload_completed && cost > 0)
7193 : 2472 : PUT_MODE (insn, TImode);
7194 : :
7195 : : last_clock = clock;
7196 : : }
7197 : :
7198 : 3219 : if (sched_verbose >= 2)
7199 : 0 : sel_print ("Cost for insn %d is %d\n", INSN_UID (insn), cost);
7200 : : }
7201 : 625 : }
7202 : :
7203 : : /* Perform MD_FINISH on EBBs comprising current region. When
7204 : : RESET_SCHED_CYCLES_P is true, run a pass emulating the scheduler
7205 : : to produce correct sched cycles on insns. */
7206 : : static void
7207 : 540 : sel_region_target_finish (bool reset_sched_cycles_p)
7208 : : {
7209 : 540 : int i;
7210 : 540 : bitmap scheduled_blocks = BITMAP_ALLOC (NULL);
7211 : :
7212 : 1879 : for (i = 0; i < current_nr_blocks; i++)
7213 : : {
7214 : 799 : if (bitmap_bit_p (scheduled_blocks, i))
7215 : 124 : continue;
7216 : :
7217 : : /* While pipelining outer loops, skip bundling for loop
7218 : : preheaders. Those will be rescheduled in the outer loop. */
7219 : 675 : if (sel_is_loop_preheader_p (EBB_FIRST_BB (i)))
7220 : 50 : continue;
7221 : :
7222 : 625 : find_ebb_boundaries (EBB_FIRST_BB (i), scheduled_blocks);
7223 : :
7224 : 625 : if (no_real_insns_p (current_sched_info->head, current_sched_info->tail))
7225 : 0 : continue;
7226 : :
7227 : 625 : if (reset_sched_cycles_p)
7228 : 104 : reset_sched_cycles_in_current_ebb ();
7229 : :
7230 : 625 : if (targetm.sched.init)
7231 : 0 : targetm.sched.init (sched_dump, sched_verbose, -1);
7232 : :
7233 : 625 : put_TImodes ();
7234 : :
7235 : 625 : if (targetm.sched.finish)
7236 : : {
7237 : 0 : targetm.sched.finish (sched_dump, sched_verbose);
7238 : :
7239 : : /* Extend luids so that insns generated by the target will
7240 : : get zero luid. */
7241 : 0 : sched_extend_luids ();
7242 : : }
7243 : : }
7244 : :
7245 : 540 : BITMAP_FREE (scheduled_blocks);
7246 : 540 : }
7247 : :
7248 : : /* Free the scheduling data for the current region. When RESET_SCHED_CYCLES_P
7249 : : is true, make an additional pass emulating scheduler to get correct insn
7250 : : cycles for md_finish calls. */
7251 : : static void
7252 : 782 : sel_region_finish (bool reset_sched_cycles_p)
7253 : : {
7254 : 782 : simplify_changed_insns ();
7255 : 782 : sched_finish_ready_list ();
7256 : 782 : free_nop_pool ();
7257 : :
7258 : : /* Free the vectors. */
7259 : 782 : vec_av_set.release ();
7260 : 782 : BITMAP_FREE (current_copies);
7261 : 782 : BITMAP_FREE (current_originators);
7262 : 782 : BITMAP_FREE (code_motion_visited_blocks);
7263 : 782 : vinsn_vec_free (vec_bookkeeping_blocked_vinsns);
7264 : 782 : vinsn_vec_free (vec_target_unavailable_vinsns);
7265 : :
7266 : : /* If LV_SET of the region head should be updated, do it now because
7267 : : there will be no other chance. */
7268 : 782 : {
7269 : 782 : succ_iterator si;
7270 : 782 : insn_t insn;
7271 : :
7272 : 1564 : FOR_EACH_SUCC_1 (insn, si, bb_note (EBB_FIRST_BB (0)),
7273 : : SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
7274 : : {
7275 : 782 : basic_block bb = BLOCK_FOR_INSN (insn);
7276 : :
7277 : 782 : if (!BB_LV_SET_VALID_P (bb))
7278 : 0 : compute_live (insn);
7279 : : }
7280 : : }
7281 : :
7282 : : /* Emulate the Haifa scheduler for bundling. */
7283 : 782 : if (reload_completed)
7284 : 540 : sel_region_target_finish (reset_sched_cycles_p);
7285 : :
7286 : 782 : sel_finish_global_and_expr ();
7287 : :
7288 : 782 : BITMAP_FREE (forced_ebb_heads);
7289 : :
7290 : 782 : free_nop_vinsn ();
7291 : :
7292 : 782 : finish_deps_global ();
7293 : 782 : sched_finish_luids ();
7294 : 782 : h_d_i_d.release ();
7295 : :
7296 : 782 : sel_finish_bbs ();
7297 : 782 : BITMAP_FREE (blocks_to_reschedule);
7298 : :
7299 : 782 : sel_unregister_cfg_hooks ();
7300 : :
7301 : 782 : max_issue_size = 0;
7302 : 782 : }
7303 : :
7304 : :
7305 : : /* Functions that implement the scheduler driver. */
7306 : :
7307 : : /* Schedule a parallel instruction group on each of FENCES. MAX_SEQNO
7308 : : is the current maximum seqno. SCHEDULED_INSNS_TAILPP is the list
7309 : : of insns scheduled -- these would be postprocessed later. */
7310 : : static void
7311 : 1751 : schedule_on_fences (flist_t fences, int max_seqno,
7312 : : ilist_t **scheduled_insns_tailpp)
7313 : : {
7314 : 1751 : flist_t old_fences = fences;
7315 : :
7316 : 1751 : if (sched_verbose >= 1)
7317 : : {
7318 : 0 : sel_print ("\nScheduling on fences: ");
7319 : 0 : dump_flist (fences);
7320 : 0 : sel_print ("\n");
7321 : : }
7322 : :
7323 : 1751 : scheduled_something_on_previous_fence = false;
7324 : 3637 : for (; fences; fences = FLIST_NEXT (fences))
7325 : : {
7326 : : fence_t fence = NULL;
7327 : : int seqno = 0;
7328 : : flist_t fences2;
7329 : : bool first_p = true;
7330 : :
7331 : : /* Choose the next fence group to schedule.
7332 : : The fact that insn can be scheduled only once
7333 : : on the cycle is guaranteed by two properties:
7334 : : 1. seqnos of parallel groups decrease with each iteration.
7335 : : 2. If is_ineligible_successor () sees the larger seqno, it
7336 : : checks if candidate insn is_in_current_fence_p (). */
7337 : 4074 : for (fences2 = old_fences; fences2; fences2 = FLIST_NEXT (fences2))
7338 : : {
7339 : 2188 : fence_t f = FLIST_FENCE (fences2);
7340 : :
7341 : 2188 : if (!FENCE_PROCESSED_P (f))
7342 : : {
7343 : 2037 : int i = INSN_SEQNO (FENCE_INSN (f));
7344 : :
7345 : 2037 : if (first_p || i > seqno)
7346 : : {
7347 : 1957 : seqno = i;
7348 : 1957 : fence = f;
7349 : 1957 : first_p = false;
7350 : : }
7351 : : else
7352 : : /* ??? Seqnos of different groups should be different. */
7353 : : gcc_assert (1 || i != seqno);
7354 : : }
7355 : : }
7356 : :
7357 : 1886 : gcc_assert (fence);
7358 : :
7359 : : /* As FENCE is nonnull, SEQNO is initialized. */
7360 : 1886 : seqno -= max_seqno + 1;
7361 : 1886 : fill_insns (fence, seqno, scheduled_insns_tailpp);
7362 : 1886 : FENCE_PROCESSED_P (fence) = true;
7363 : : }
7364 : :
7365 : : /* All av_sets are invalidated by GLOBAL_LEVEL increase, thus we
7366 : : don't need to keep bookkeeping-invalidated and target-unavailable
7367 : : vinsns any more. */
7368 : 1751 : vinsn_vec_clear (&vec_bookkeeping_blocked_vinsns);
7369 : 1751 : vinsn_vec_clear (&vec_target_unavailable_vinsns);
7370 : 1751 : }
7371 : :
7372 : : /* Calculate MIN_SEQNO and MAX_SEQNO. */
7373 : : static void
7374 : 1751 : find_min_max_seqno (flist_t fences, int *min_seqno, int *max_seqno)
7375 : : {
7376 : 1751 : *min_seqno = *max_seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7377 : :
7378 : : /* The first element is already processed. */
7379 : 1886 : while ((fences = FLIST_NEXT (fences)))
7380 : : {
7381 : 135 : int seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7382 : :
7383 : 135 : if (*min_seqno > seqno)
7384 : 64 : *min_seqno = seqno;
7385 : 71 : else if (*max_seqno < seqno)
7386 : 64 : *max_seqno = seqno;
7387 : : }
7388 : 1751 : }
7389 : :
7390 : : /* Calculate new fences from FENCES. Write the current time to PTIME. */
7391 : : static flist_t
7392 : 1751 : calculate_new_fences (flist_t fences, int orig_max_seqno, int *ptime)
7393 : : {
7394 : 1751 : flist_t old_fences = fences;
7395 : 1751 : struct flist_tail_def _new_fences, *new_fences = &_new_fences;
7396 : 1751 : int max_time = 0;
7397 : :
7398 : 1751 : flist_tail_init (new_fences);
7399 : 3637 : for (; fences; fences = FLIST_NEXT (fences))
7400 : : {
7401 : 1886 : fence_t fence = FLIST_FENCE (fences);
7402 : 1886 : insn_t insn;
7403 : :
7404 : 1886 : if (!FENCE_BNDS (fence))
7405 : : {
7406 : : /* This fence doesn't have any successors. */
7407 : 929 : if (!FENCE_SCHEDULED_P (fence))
7408 : : {
7409 : : /* Nothing was scheduled on this fence. */
7410 : 189 : int seqno;
7411 : :
7412 : 189 : insn = FENCE_INSN (fence);
7413 : 189 : seqno = INSN_SEQNO (insn);
7414 : 189 : gcc_assert (seqno > 0 && seqno <= orig_max_seqno);
7415 : :
7416 : 189 : if (sched_verbose >= 1)
7417 : 0 : sel_print ("Fence %d[%d] has not changed\n",
7418 : : INSN_UID (insn),
7419 : 0 : BLOCK_NUM (insn));
7420 : 189 : move_fence_to_fences (fences, new_fences);
7421 : : }
7422 : : }
7423 : : else
7424 : 957 : extract_new_fences_from (fences, new_fences, orig_max_seqno);
7425 : 1886 : max_time = MAX (max_time, FENCE_CYCLE (fence));
7426 : : }
7427 : :
7428 : 1751 : flist_clear (&old_fences);
7429 : 1751 : *ptime = max_time;
7430 : 1751 : return FLIST_TAIL_HEAD (new_fences);
7431 : : }
7432 : :
7433 : : /* Update seqnos of insns given by PSCHEDULED_INSNS. MIN_SEQNO and MAX_SEQNO
7434 : : are the miminum and maximum seqnos of the group, HIGHEST_SEQNO_IN_USE is
7435 : : the highest seqno used in a region. Return the updated highest seqno. */
7436 : : static int
7437 : 1751 : update_seqnos_and_stage (int min_seqno, int max_seqno,
7438 : : int highest_seqno_in_use,
7439 : : ilist_t *pscheduled_insns)
7440 : : {
7441 : 1751 : int new_hs;
7442 : 1751 : ilist_iterator ii;
7443 : 1751 : insn_t insn;
7444 : :
7445 : : /* Actually, new_hs is the seqno of the instruction, that was
7446 : : scheduled first (i.e. it is the first one in SCHEDULED_INSNS). */
7447 : 1751 : if (*pscheduled_insns)
7448 : : {
7449 : 1565 : new_hs = (INSN_SEQNO (ILIST_INSN (*pscheduled_insns))
7450 : 1565 : + highest_seqno_in_use + max_seqno - min_seqno + 2);
7451 : 1565 : gcc_assert (new_hs > highest_seqno_in_use);
7452 : : }
7453 : : else
7454 : : new_hs = highest_seqno_in_use;
7455 : :
7456 : 6634 : FOR_EACH_INSN (insn, ii, *pscheduled_insns)
7457 : : {
7458 : 4883 : gcc_assert (INSN_SEQNO (insn) < 0);
7459 : 4883 : INSN_SEQNO (insn) += highest_seqno_in_use + max_seqno - min_seqno + 2;
7460 : 4883 : gcc_assert (INSN_SEQNO (insn) <= new_hs);
7461 : :
7462 : : /* When not pipelining, purge unneeded insn info on the scheduled insns.
7463 : : For example, having reg_last array of INSN_DEPS_CONTEXT in memory may
7464 : : require > 1GB of memory e.g. on limit-fnargs.c. */
7465 : 4883 : if (! pipelining_p)
7466 : 3671 : free_data_for_scheduled_insn (insn);
7467 : : }
7468 : :
7469 : 1751 : ilist_clear (pscheduled_insns);
7470 : 1751 : global_level++;
7471 : :
7472 : 1751 : return new_hs;
7473 : : }
7474 : :
7475 : : /* The main driver for scheduling a region. This function is responsible
7476 : : for correct propagation of fences (i.e. scheduling points) and creating
7477 : : a group of parallel insns at each of them. It also supports
7478 : : pipelining. ORIG_MAX_SEQNO is the maximal seqno before this pass
7479 : : of scheduling. */
7480 : : static void
7481 : 803 : sel_sched_region_2 (int orig_max_seqno)
7482 : : {
7483 : 803 : int highest_seqno_in_use = orig_max_seqno;
7484 : 803 : int max_time = 0;
7485 : :
7486 : 803 : stat_bookkeeping_copies = 0;
7487 : 803 : stat_insns_needed_bookkeeping = 0;
7488 : 803 : stat_renamed_scheduled = 0;
7489 : 803 : stat_substitutions_total = 0;
7490 : 803 : num_insns_scheduled = 0;
7491 : :
7492 : 2554 : while (fences)
7493 : : {
7494 : 1751 : int min_seqno, max_seqno;
7495 : 1751 : ilist_t scheduled_insns = NULL;
7496 : 1751 : ilist_t *scheduled_insns_tailp = &scheduled_insns;
7497 : :
7498 : 1751 : find_min_max_seqno (fences, &min_seqno, &max_seqno);
7499 : 1751 : schedule_on_fences (fences, max_seqno, &scheduled_insns_tailp);
7500 : 1751 : fences = calculate_new_fences (fences, orig_max_seqno, &max_time);
7501 : 1751 : highest_seqno_in_use = update_seqnos_and_stage (min_seqno, max_seqno,
7502 : : highest_seqno_in_use,
7503 : : &scheduled_insns);
7504 : : }
7505 : :
7506 : 803 : if (sched_verbose >= 1)
7507 : : {
7508 : 0 : sel_print ("Total scheduling time: %d cycles\n", max_time);
7509 : 0 : sel_print ("Scheduled %d bookkeeping copies, %d insns needed "
7510 : : "bookkeeping, %d insns renamed, %d insns substituted\n",
7511 : : stat_bookkeeping_copies,
7512 : : stat_insns_needed_bookkeeping,
7513 : : stat_renamed_scheduled,
7514 : : stat_substitutions_total);
7515 : : }
7516 : 803 : }
7517 : :
7518 : : /* Schedule a region. When pipelining, search for possibly never scheduled
7519 : : bookkeeping code and schedule it. Reschedule pipelined code without
7520 : : pipelining after. */
7521 : : static void
7522 : 782 : sel_sched_region_1 (void)
7523 : : {
7524 : 782 : int orig_max_seqno;
7525 : :
7526 : : /* Remove empty blocks that might be in the region from the beginning. */
7527 : 782 : purge_empty_blocks ();
7528 : :
7529 : 782 : orig_max_seqno = init_seqno (NULL, NULL);
7530 : 782 : gcc_assert (orig_max_seqno >= 1);
7531 : :
7532 : : /* When pipelining outer loops, create fences on the loop header,
7533 : : not preheader. */
7534 : 782 : fences = NULL;
7535 : 782 : if (current_loop_nest)
7536 : 55 : init_fences (BB_END (EBB_FIRST_BB (0)));
7537 : : else
7538 : 727 : init_fences (bb_note (EBB_FIRST_BB (0)));
7539 : 782 : global_level = 1;
7540 : :
7541 : 782 : sel_sched_region_2 (orig_max_seqno);
7542 : :
7543 : 782 : gcc_assert (fences == NULL);
7544 : :
7545 : 782 : if (pipelining_p)
7546 : : {
7547 : 53 : int i;
7548 : 53 : basic_block bb;
7549 : 53 : struct flist_tail_def _new_fences;
7550 : 53 : flist_tail_t new_fences = &_new_fences;
7551 : 53 : bool do_p = true;
7552 : :
7553 : 53 : pipelining_p = false;
7554 : 53 : max_ws = MIN (max_ws, issue_rate * 3 / 2);
7555 : 53 : bookkeeping_p = false;
7556 : 53 : enable_schedule_as_rhs_p = false;
7557 : :
7558 : : /* Schedule newly created code, that has not been scheduled yet. */
7559 : 53 : do_p = true;
7560 : :
7561 : 180 : while (do_p)
7562 : : {
7563 : 526 : do_p = false;
7564 : :
7565 : 526 : for (i = 0; i < current_nr_blocks; i++)
7566 : : {
7567 : 452 : basic_block bb = EBB_FIRST_BB (i);
7568 : :
7569 : 452 : if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7570 : : {
7571 : 78 : if (! bb_ends_ebb_p (bb))
7572 : 28 : bitmap_set_bit (blocks_to_reschedule, bb_next_bb (bb)->index);
7573 : 78 : if (sel_bb_empty_p (bb))
7574 : : {
7575 : 0 : bitmap_clear_bit (blocks_to_reschedule, bb->index);
7576 : 0 : continue;
7577 : : }
7578 : 78 : clear_outdated_rtx_info (bb);
7579 : 78 : if (sel_insn_is_speculation_check (BB_END (bb))
7580 : 78 : && JUMP_P (BB_END (bb)))
7581 : 0 : bitmap_set_bit (blocks_to_reschedule,
7582 : 0 : BRANCH_EDGE (bb)->dest->index);
7583 : : }
7584 : 374 : else if (! sel_bb_empty_p (bb)
7585 : 374 : && INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0)
7586 : 38 : bitmap_set_bit (blocks_to_reschedule, bb->index);
7587 : : }
7588 : :
7589 : 405 : for (i = 0; i < current_nr_blocks; i++)
7590 : : {
7591 : 352 : bb = EBB_FIRST_BB (i);
7592 : :
7593 : : /* While pipelining outer loops, skip bundling for loop
7594 : : preheaders. Those will be rescheduled in the outer
7595 : : loop. */
7596 : 352 : if (sel_is_loop_preheader_p (bb))
7597 : : {
7598 : 74 : clear_outdated_rtx_info (bb);
7599 : 74 : continue;
7600 : : }
7601 : :
7602 : 278 : if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7603 : : {
7604 : 21 : flist_tail_init (new_fences);
7605 : :
7606 : 21 : orig_max_seqno = init_seqno (blocks_to_reschedule, bb);
7607 : :
7608 : : /* Mark BB as head of the new ebb. */
7609 : 21 : bitmap_set_bit (forced_ebb_heads, bb->index);
7610 : :
7611 : 21 : gcc_assert (fences == NULL);
7612 : :
7613 : 21 : init_fences (bb_note (bb));
7614 : :
7615 : 21 : sel_sched_region_2 (orig_max_seqno);
7616 : :
7617 : 21 : do_p = true;
7618 : 21 : break;
7619 : : }
7620 : : }
7621 : : }
7622 : : }
7623 : 782 : }
7624 : :
7625 : : /* Schedule the RGN region. */
7626 : : void
7627 : 802 : sel_sched_region (int rgn)
7628 : : {
7629 : 802 : bool schedule_p;
7630 : 802 : bool reset_sched_cycles_p;
7631 : :
7632 : 802 : if (sel_region_init (rgn))
7633 : : return;
7634 : :
7635 : 782 : if (sched_verbose >= 1)
7636 : 0 : sel_print ("Scheduling region %d\n", rgn);
7637 : :
7638 : 782 : schedule_p = (!sched_is_disabled_for_current_region_p ()
7639 : 782 : && dbg_cnt (sel_sched_region_cnt));
7640 : 782 : reset_sched_cycles_p = pipelining_p;
7641 : 782 : if (schedule_p)
7642 : 782 : sel_sched_region_1 ();
7643 : : else
7644 : : {
7645 : : /* Schedule always selecting the next insn to make the correct data
7646 : : for bundling or other later passes. */
7647 : 0 : pipelining_p = false;
7648 : 0 : reset_sched_cycles_p = false;
7649 : 0 : force_next_insn = 1;
7650 : 0 : sel_sched_region_1 ();
7651 : 0 : force_next_insn = 0;
7652 : : }
7653 : 782 : sel_region_finish (reset_sched_cycles_p);
7654 : : }
7655 : :
7656 : : /* Perform global init for the scheduler. */
7657 : : static void
7658 : 131 : sel_global_init (void)
7659 : : {
7660 : : /* Remove empty blocks: their presence can break assumptions elsewhere,
7661 : : e.g. the logic to invoke update_liveness_on_insn in sel_region_init. */
7662 : 131 : cleanup_cfg (0);
7663 : :
7664 : 131 : calculate_dominance_info (CDI_DOMINATORS);
7665 : 131 : alloc_sched_pools ();
7666 : :
7667 : : /* Setup the infos for sched_init. */
7668 : 131 : sel_setup_sched_infos ();
7669 : 131 : setup_sched_dump ();
7670 : :
7671 : 131 : sched_rgn_init (false);
7672 : 131 : sched_init ();
7673 : :
7674 : 131 : sched_init_bbs ();
7675 : : /* Reset AFTER_RECOVERY if it has been set by the 1st scheduler pass. */
7676 : 131 : after_recovery = 0;
7677 : 131 : can_issue_more = issue_rate;
7678 : :
7679 : 131 : sched_extend_target ();
7680 : 131 : sched_deps_init (true);
7681 : 131 : setup_nop_and_exit_insns ();
7682 : 131 : sel_extend_global_bb_info ();
7683 : 131 : init_lv_sets ();
7684 : 131 : init_hard_regs_data ();
7685 : 131 : }
7686 : :
7687 : : /* Free the global data of the scheduler. */
7688 : : static void
7689 : 131 : sel_global_finish (void)
7690 : : {
7691 : 131 : free_bb_note_pool ();
7692 : 131 : free_lv_sets ();
7693 : 131 : sel_finish_global_bb_info ();
7694 : :
7695 : 131 : free_regset_pool ();
7696 : 131 : free_nop_and_exit_insns ();
7697 : :
7698 : 131 : sched_rgn_finish ();
7699 : 131 : sched_deps_finish ();
7700 : 131 : sched_finish ();
7701 : :
7702 : 131 : if (current_loops)
7703 : 43 : sel_finish_pipelining ();
7704 : :
7705 : 131 : free_sched_pools ();
7706 : 131 : free_dominance_info (CDI_DOMINATORS);
7707 : 131 : }
7708 : :
7709 : : /* Return true when we need to skip selective scheduling. Used for debugging. */
7710 : : bool
7711 : 131 : maybe_skip_selective_scheduling (void)
7712 : : {
7713 : 131 : return ! dbg_cnt (sel_sched_cnt);
7714 : : }
7715 : :
7716 : : /* The entry point. */
7717 : : void
7718 : 131 : run_selective_scheduling (void)
7719 : : {
7720 : 131 : int rgn;
7721 : :
7722 : 131 : if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
7723 : : return;
7724 : :
7725 : 131 : sel_global_init ();
7726 : :
7727 : 1064 : for (rgn = 0; rgn < nr_regions; rgn++)
7728 : 802 : sel_sched_region (rgn);
7729 : :
7730 : 131 : sel_global_finish ();
7731 : : }
7732 : :
7733 : : #endif
|
