Line data Source code
1 : /* Instruction scheduling pass. Selective scheduler and pipeliner.
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "tree.h"
25 : #include "rtl.h"
26 : #include "df.h"
27 : #include "memmodel.h"
28 : #include "tm_p.h"
29 : #include "regs.h"
30 : #include "cfgbuild.h"
31 : #include "cfgcleanup.h"
32 : #include "insn-config.h"
33 : #include "insn-attr.h"
34 : #include "target.h"
35 : #include "sched-int.h"
36 : #include "rtlhooks-def.h"
37 : #include "ira.h"
38 : #include "ira-int.h"
39 : #include "rtl-iter.h"
40 :
41 : #ifdef INSN_SCHEDULING
42 : #include "regset.h"
43 : #include "cfgloop.h"
44 : #include "sel-sched-ir.h"
45 : #include "sel-sched-dump.h"
46 : #include "sel-sched.h"
47 : #include "dbgcnt.h"
48 : #include "function-abi.h"
49 :
50 : /* Implementation of selective scheduling approach.
51 : The below implementation follows the original approach with the following
52 : changes:
53 :
54 : o the scheduler works after register allocation (but can be also tuned
55 : to work before RA);
56 : o some instructions are not copied or register renamed;
57 : o conditional jumps are not moved with code duplication;
58 : o several jumps in one parallel group are not supported;
59 : o when pipelining outer loops, code motion through inner loops
60 : is not supported;
61 : o control and data speculation are supported;
62 : o some improvements for better compile time/performance were made.
63 :
64 : Terminology
65 : ===========
66 :
67 : A vinsn, or virtual insn, is an insn with additional data characterizing
68 : insn pattern, such as LHS, RHS, register sets used/set/clobbered, etc.
69 : Vinsns also act as smart pointers to save memory by reusing them in
70 : different expressions. A vinsn is described by vinsn_t type.
71 :
72 : An expression is a vinsn with additional data characterizing its properties
73 : at some point in the control flow graph. The data may be its usefulness,
74 : priority, speculative status, whether it was renamed/subsituted, etc.
75 : An expression is described by expr_t type.
76 :
77 : Availability set (av_set) is a set of expressions at a given control flow
78 : point. It is represented as av_set_t. The expressions in av sets are kept
79 : sorted in the terms of expr_greater_p function. It allows to truncate
80 : the set while leaving the best expressions.
81 :
82 : A fence is a point through which code motion is prohibited. On each step,
83 : we gather a parallel group of insns at a fence. It is possible to have
84 : multiple fences. A fence is represented via fence_t.
85 :
86 : A boundary is the border between the fence group and the rest of the code.
87 : Currently, we never have more than one boundary per fence, as we finalize
88 : the fence group when a jump is scheduled. A boundary is represented
89 : via bnd_t.
90 :
91 : High-level overview
92 : ===================
93 :
94 : The scheduler finds regions to schedule, schedules each one, and finalizes.
95 : The regions are formed starting from innermost loops, so that when the inner
96 : loop is pipelined, its prologue can be scheduled together with yet unprocessed
97 : outer loop. The rest of acyclic regions are found using extend_rgns:
98 : the blocks that are not yet allocated to any regions are traversed in top-down
99 : order, and a block is added to a region to which all its predecessors belong;
100 : otherwise, the block starts its own region.
101 :
102 : The main scheduling loop (sel_sched_region_2) consists of just
103 : scheduling on each fence and updating fences. For each fence,
104 : we fill a parallel group of insns (fill_insns) until some insns can be added.
105 : First, we compute available exprs (av-set) at the boundary of the current
106 : group. Second, we choose the best expression from it. If the stall is
107 : required to schedule any of the expressions, we advance the current cycle
108 : appropriately. So, the final group does not exactly correspond to a VLIW
109 : word. Third, we move the chosen expression to the boundary (move_op)
110 : and update the intermediate av sets and liveness sets. We quit fill_insns
111 : when either no insns left for scheduling or we have scheduled enough insns
112 : so we feel like advancing a scheduling point.
113 :
114 : Computing available expressions
115 : ===============================
116 :
117 : The computation (compute_av_set) is a bottom-up traversal. At each insn,
118 : we're moving the union of its successors' sets through it via
119 : moveup_expr_set. The dependent expressions are removed. Local
120 : transformations (substitution, speculation) are applied to move more
121 : exprs. Then the expr corresponding to the current insn is added.
122 : The result is saved on each basic block header.
123 :
124 : When traversing the CFG, we're moving down for no more than max_ws insns.
125 : Also, we do not move down to ineligible successors (is_ineligible_successor),
126 : which include moving along a back-edge, moving to already scheduled code,
127 : and moving to another fence. The first two restrictions are lifted during
128 : pipelining, which allows us to move insns along a back-edge. We always have
129 : an acyclic region for scheduling because we forbid motion through fences.
130 :
131 : Choosing the best expression
132 : ============================
133 :
134 : We sort the final availability set via sel_rank_for_schedule, then we remove
135 : expressions which are not yet ready (tick_check_p) or which dest registers
136 : cannot be used. For some of them, we choose another register via
137 : find_best_reg. To do this, we run find_used_regs to calculate the set of
138 : registers which cannot be used. The find_used_regs function performs
139 : a traversal of code motion paths for an expr. We consider for renaming
140 : only registers which are from the same regclass as the original one and
141 : using which does not interfere with any live ranges. Finally, we convert
142 : the resulting set to the ready list format and use max_issue and reorder*
143 : hooks similarly to the Haifa scheduler.
144 :
145 : Scheduling the best expression
146 : ==============================
147 :
148 : We run the move_op routine to perform the same type of code motion paths
149 : traversal as in find_used_regs. (These are working via the same driver,
150 : code_motion_path_driver.) When moving down the CFG, we look for original
151 : instruction that gave birth to a chosen expression. We undo
152 : the transformations performed on an expression via the history saved in it.
153 : When found, we remove the instruction or leave a reg-reg copy/speculation
154 : check if needed. On a way up, we insert bookkeeping copies at each join
155 : point. If a copy is not needed, it will be removed later during this
156 : traversal. We update the saved av sets and liveness sets on the way up, too.
157 :
158 : Finalizing the schedule
159 : =======================
160 :
161 : When pipelining, we reschedule the blocks from which insns were pipelined
162 : to get a tighter schedule. On Itanium, we also perform bundling via
163 : the same routine from ia64.cc.
164 :
165 : Dependence analysis changes
166 : ===========================
167 :
168 : We augmented the sched-deps.cc with hooks that get called when a particular
169 : dependence is found in a particular part of an insn. Using these hooks, we
170 : can do several actions such as: determine whether an insn can be moved through
171 : another (has_dependence_p, moveup_expr); find out whether an insn can be
172 : scheduled on the current cycle (tick_check_p); find out registers that
173 : are set/used/clobbered by an insn and find out all the strange stuff that
174 : restrict its movement, like SCHED_GROUP_P or CANT_MOVE (done in
175 : init_global_and_expr_for_insn).
176 :
177 : Initialization changes
178 : ======================
179 :
180 : There are parts of haifa-sched.cc, sched-deps.cc, and sched-rgn.cc that are
181 : reused in all of the schedulers. We have split up the initialization of data
182 : of such parts into different functions prefixed with scheduler type and
183 : postfixed with the type of data initialized: {,sel_,haifa_}sched_{init,finish},
184 : sched_rgn_init/finish, sched_deps_init/finish, sched_init_{luids/bbs}, etc.
185 : The same splitting is done with current_sched_info structure:
186 : dependence-related parts are in sched_deps_info, common part is in
187 : common_sched_info, and haifa/sel/etc part is in current_sched_info.
188 :
189 : Target contexts
190 : ===============
191 :
192 : As we now have multiple-point scheduling, this would not work with backends
193 : which save some of the scheduler state to use it in the target hooks.
194 : For this purpose, we introduce a concept of target contexts, which
195 : encapsulate such information. The backend should implement simple routines
196 : of allocating/freeing/setting such a context. The scheduler calls these
197 : as target hooks and handles the target context as an opaque pointer (similar
198 : to the DFA state type, state_t).
199 :
200 : Various speedups
201 : ================
202 :
203 : As the correct data dependence graph is not supported during scheduling (which
204 : is to be changed in mid-term), we cache as much of the dependence analysis
205 : results as possible to avoid reanalyzing. This includes: bitmap caches on
206 : each insn in stream of the region saying yes/no for a query with a pair of
207 : UIDs; hashtables with the previously done transformations on each insn in
208 : stream; a vector keeping a history of transformations on each expr.
209 :
210 : Also, we try to minimize the dependence context used on each fence to check
211 : whether the given expression is ready for scheduling by removing from it
212 : insns that are definitely completed the execution. The results of
213 : tick_check_p checks are also cached in a vector on each fence.
214 :
215 : We keep a valid liveness set on each insn in a region to avoid the high
216 : cost of recomputation on large basic blocks.
217 :
218 : Finally, we try to minimize the number of needed updates to the availability
219 : sets. The updates happen in two cases: when fill_insns terminates,
220 : we advance all fences and increase the stage number to show that the region
221 : has changed and the sets are to be recomputed; and when the next iteration
222 : of a loop in fill_insns happens (but this one reuses the saved av sets
223 : on bb headers.) Thus, we try to break the fill_insns loop only when
224 : "significant" number of insns from the current scheduling window was
225 : scheduled. This should be made a target param.
226 :
227 :
228 : TODO: correctly support the data dependence graph at all stages and get rid
229 : of all caches. This should speed up the scheduler.
230 : TODO: implement moving cond jumps with bookkeeping copies on both targets.
231 : TODO: tune the scheduler before RA so it does not create too much pseudos.
232 :
233 :
234 : References:
235 : S.-M. Moon and K. Ebcioglu. Parallelizing nonnumerical code with
236 : selective scheduling and software pipelining.
237 : ACM TOPLAS, Vol 19, No. 6, pages 853--898, Nov. 1997.
238 :
239 : Andrey Belevantsev, Maxim Kuvyrkov, Vladimir Makarov, Dmitry Melnik,
240 : and Dmitry Zhurikhin. An interblock VLIW-targeted instruction scheduler
241 : for GCC. In Proceedings of GCC Developers' Summit 2006.
242 :
243 : Arutyun Avetisyan, Andrey Belevantsev, and Dmitry Melnik. GCC Instruction
244 : Scheduler and Software Pipeliner on the Itanium Platform. EPIC-7 Workshop.
245 : http://rogue.colorado.edu/EPIC7/.
246 :
247 : */
248 :
249 : /* True when pipelining is enabled. */
250 : bool pipelining_p;
251 :
252 : /* True if bookkeeping is enabled. */
253 : bool bookkeeping_p;
254 :
255 : /* Maximum number of insns that are eligible for renaming. */
256 : int max_insns_to_rename;
257 :
258 :
259 : /* Definitions of local types and macros. */
260 :
261 : /* Represents possible outcomes of moving an expression through an insn. */
262 : enum MOVEUP_EXPR_CODE
263 : {
264 : /* The expression is not changed. */
265 : MOVEUP_EXPR_SAME,
266 :
267 : /* Not changed, but requires a new destination register. */
268 : MOVEUP_EXPR_AS_RHS,
269 :
270 : /* Cannot be moved. */
271 : MOVEUP_EXPR_NULL,
272 :
273 : /* Changed (substituted or speculated). */
274 : MOVEUP_EXPR_CHANGED
275 : };
276 :
277 : /* The container to be passed into rtx search & replace functions. */
278 : struct rtx_search_arg
279 : {
280 : /* What we are searching for. */
281 : rtx x;
282 :
283 : /* The occurrence counter. */
284 : int n;
285 : };
286 :
287 : typedef struct rtx_search_arg *rtx_search_arg_p;
288 :
289 : /* This struct contains precomputed hard reg sets that are needed when
290 : computing registers available for renaming. */
291 : struct hard_regs_data
292 : {
293 : /* For every mode, this stores registers available for use with
294 : that mode. */
295 : HARD_REG_SET regs_for_mode[NUM_MACHINE_MODES];
296 :
297 : /* True when regs_for_mode[mode] is initialized. */
298 : bool regs_for_mode_ok[NUM_MACHINE_MODES];
299 :
300 : /* For every register, it has regs that are ok to rename into it.
301 : The register in question is always set. If not, this means
302 : that the whole set is not computed yet. */
303 : HARD_REG_SET regs_for_rename[FIRST_PSEUDO_REGISTER];
304 :
305 : /* All registers that are used or call used. */
306 : HARD_REG_SET regs_ever_used;
307 :
308 : #ifdef STACK_REGS
309 : /* Stack registers. */
310 : HARD_REG_SET stack_regs;
311 : #endif
312 : };
313 :
314 : /* Holds the results of computation of available for renaming and
315 : unavailable hard registers. */
316 : struct reg_rename
317 : {
318 : /* These are unavailable due to calls crossing, globalness, etc. */
319 : HARD_REG_SET unavailable_hard_regs;
320 :
321 : /* These are *available* for renaming. */
322 : HARD_REG_SET available_for_renaming;
323 :
324 : /* The set of ABIs used by calls that the code motion path crosses. */
325 : unsigned int crossed_call_abis : NUM_ABI_IDS;
326 : };
327 :
328 : /* A global structure that contains the needed information about harg
329 : regs. */
330 : static struct hard_regs_data sel_hrd;
331 :
332 :
333 : /* This structure holds local data used in code_motion_path_driver hooks on
334 : the same or adjacent levels of recursion. Here we keep those parameters
335 : that are not used in code_motion_path_driver routine itself, but only in
336 : its hooks. Moreover, all parameters that can be modified in hooks are
337 : in this structure, so all other parameters passed explicitly to hooks are
338 : read-only. */
339 : struct cmpd_local_params
340 : {
341 : /* Local params used in move_op_* functions. */
342 :
343 : /* Edges for bookkeeping generation. */
344 : edge e1, e2;
345 :
346 : /* C_EXPR merged from all successors and locally allocated temporary C_EXPR. */
347 : expr_t c_expr_merged, c_expr_local;
348 :
349 : /* Local params used in fur_* functions. */
350 : /* Copy of the ORIGINAL_INSN list, stores the original insns already
351 : found before entering the current level of code_motion_path_driver. */
352 : def_list_t old_original_insns;
353 :
354 : /* Local params used in move_op_* functions. */
355 : /* True when we have removed last insn in the block which was
356 : also a boundary. Do not update anything or create bookkeeping copies. */
357 : BOOL_BITFIELD removed_last_insn : 1;
358 : };
359 :
360 : /* Stores the static parameters for move_op_* calls. */
361 : struct moveop_static_params
362 : {
363 : /* Destination register. */
364 : rtx dest;
365 :
366 : /* Current C_EXPR. */
367 : expr_t c_expr;
368 :
369 : /* An UID of expr_vliw which is to be moved up. If we find other exprs,
370 : they are to be removed. */
371 : int uid;
372 :
373 : /* This is initialized to the insn on which the driver stopped its traversal. */
374 : insn_t failed_insn;
375 :
376 : /* True if we scheduled an insn with different register. */
377 : bool was_renamed;
378 : };
379 :
380 : /* Stores the static parameters for fur_* calls. */
381 : struct fur_static_params
382 : {
383 : /* Set of registers unavailable on the code motion path. */
384 : regset used_regs;
385 :
386 : /* Pointer to the list of original insns definitions. */
387 : def_list_t *original_insns;
388 :
389 : /* The set of ABIs used by calls that the code motion path crosses. */
390 : unsigned int crossed_call_abis : NUM_ABI_IDS;
391 : };
392 :
393 : typedef struct fur_static_params *fur_static_params_p;
394 : typedef struct cmpd_local_params *cmpd_local_params_p;
395 : typedef struct moveop_static_params *moveop_static_params_p;
396 :
397 : /* Set of hooks and parameters that determine behavior specific to
398 : move_op or find_used_regs functions. */
399 : struct code_motion_path_driver_info_def
400 : {
401 : /* Called on enter to the basic block. */
402 : int (*on_enter) (insn_t, cmpd_local_params_p, void *, bool);
403 :
404 : /* Called when original expr is found. */
405 : void (*orig_expr_found) (insn_t, expr_t, cmpd_local_params_p, void *);
406 :
407 : /* Called while descending current basic block if current insn is not
408 : the original EXPR we're searching for. */
409 : bool (*orig_expr_not_found) (insn_t, av_set_t, void *);
410 :
411 : /* Function to merge C_EXPRes from different successors. */
412 : void (*merge_succs) (insn_t, insn_t, int, cmpd_local_params_p, void *);
413 :
414 : /* Function to finalize merge from different successors and possibly
415 : deallocate temporary data structures used for merging. */
416 : void (*after_merge_succs) (cmpd_local_params_p, void *);
417 :
418 : /* Called on the backward stage of recursion to do moveup_expr.
419 : Used only with move_op_*. */
420 : void (*ascend) (insn_t, void *);
421 :
422 : /* Called on the ascending pass, before returning from the current basic
423 : block or from the whole traversal. */
424 : void (*at_first_insn) (insn_t, cmpd_local_params_p, void *);
425 :
426 : /* When processing successors in move_op we need only descend into
427 : SUCCS_NORMAL successors, while in find_used_regs we need SUCCS_ALL. */
428 : int succ_flags;
429 :
430 : /* The routine name to print in dumps ("move_op" of "find_used_regs"). */
431 : const char *routine_name;
432 : };
433 :
434 : /* Global pointer to current hooks, either points to MOVE_OP_HOOKS or
435 : FUR_HOOKS. */
436 : struct code_motion_path_driver_info_def *code_motion_path_driver_info;
437 :
438 : /* Set of hooks for performing move_op and find_used_regs routines with
439 : code_motion_path_driver. */
440 : extern struct code_motion_path_driver_info_def move_op_hooks, fur_hooks;
441 :
442 : /* True if/when we want to emulate Haifa scheduler in the common code.
443 : This is used in sched_rgn_local_init and in various places in
444 : sched-deps.cc. */
445 : int sched_emulate_haifa_p;
446 :
447 : /* GLOBAL_LEVEL is used to discard information stored in basic block headers
448 : av_sets. Av_set of bb header is valid if its (bb header's) level is equal
449 : to GLOBAL_LEVEL. And invalid if lesser. This is primarily used to advance
450 : scheduling window. */
451 : int global_level;
452 :
453 : /* Current fences. */
454 : flist_t fences;
455 :
456 : /* True when separable insns should be scheduled as RHSes. */
457 : static bool enable_schedule_as_rhs_p;
458 :
459 : /* Used in verify_target_availability to assert that target reg is reported
460 : unavailabile by both TARGET_UNAVAILABLE and find_used_regs only if
461 : we haven't scheduled anything on the previous fence.
462 : if scheduled_something_on_previous_fence is true, TARGET_UNAVAILABLE can
463 : have more conservative value than the one returned by the
464 : find_used_regs, thus we shouldn't assert that these values are equal. */
465 : static bool scheduled_something_on_previous_fence;
466 :
467 : /* All newly emitted insns will have their uids greater than this value. */
468 : static int first_emitted_uid;
469 :
470 : /* Set of basic blocks that are forced to start new ebbs. This is a subset
471 : of all the ebb heads. */
472 : bitmap forced_ebb_heads;
473 :
474 : /* Blocks that need to be rescheduled after pipelining. */
475 : bitmap blocks_to_reschedule = NULL;
476 :
477 : /* True when the first lv set should be ignored when updating liveness. */
478 : static bool ignore_first = false;
479 :
480 : /* Number of insns max_issue has initialized data structures for. */
481 : static int max_issue_size = 0;
482 :
483 : /* Whether we can issue more instructions. */
484 : static int can_issue_more;
485 :
486 : /* Maximum software lookahead window size, reduced when rescheduling after
487 : pipelining. */
488 : static int max_ws;
489 :
490 : /* Number of insns scheduled in current region. */
491 : static int num_insns_scheduled;
492 :
493 : /* A vector of expressions is used to be able to sort them. */
494 : static vec<expr_t> vec_av_set;
495 :
496 : /* A vector of vinsns is used to hold temporary lists of vinsns. */
497 : typedef vec<vinsn_t> vinsn_vec_t;
498 :
499 : /* This vector has the exprs which may still present in av_sets, but actually
500 : can't be moved up due to bookkeeping created during code motion to another
501 : fence. See comment near the call to update_and_record_unavailable_insns
502 : for the detailed explanations. */
503 : static vinsn_vec_t vec_bookkeeping_blocked_vinsns = vinsn_vec_t ();
504 :
505 : /* This vector has vinsns which are scheduled with renaming on the first fence
506 : and then seen on the second. For expressions with such vinsns, target
507 : availability information may be wrong. */
508 : static vinsn_vec_t vec_target_unavailable_vinsns = vinsn_vec_t ();
509 :
510 : /* Vector to store temporary nops inserted in move_op to prevent removal
511 : of empty bbs. */
512 : static vec<insn_t> vec_temp_moveop_nops;
513 :
514 : /* These bitmaps record original instructions scheduled on the current
515 : iteration and bookkeeping copies created by them. */
516 : static bitmap current_originators = NULL;
517 : static bitmap current_copies = NULL;
518 :
519 : /* This bitmap marks the blocks visited by code_motion_path_driver so we don't
520 : visit them afterwards. */
521 : static bitmap code_motion_visited_blocks = NULL;
522 :
523 : /* Variables to accumulate different statistics. */
524 :
525 : /* The number of bookkeeping copies created. */
526 : static int stat_bookkeeping_copies;
527 :
528 : /* The number of insns that required bookkeeiping for their scheduling. */
529 : static int stat_insns_needed_bookkeeping;
530 :
531 : /* The number of insns that got renamed. */
532 : static int stat_renamed_scheduled;
533 :
534 : /* The number of substitutions made during scheduling. */
535 : static int stat_substitutions_total;
536 :
537 :
538 : /* Forward declarations of static functions. */
539 : static bool rtx_ok_for_substitution_p (rtx, rtx);
540 : static int sel_rank_for_schedule (const void *, const void *);
541 : static av_set_t find_sequential_best_exprs (bnd_t, expr_t, bool);
542 : static basic_block find_block_for_bookkeeping (edge e1, edge e2, bool lax);
543 :
544 : static rtx get_dest_from_orig_ops (av_set_t);
545 : static basic_block generate_bookkeeping_insn (expr_t, edge, edge);
546 : static bool find_used_regs (insn_t, av_set_t, regset, struct reg_rename *,
547 : def_list_t *);
548 : static bool move_op (insn_t, av_set_t, expr_t, rtx, expr_t, bool*);
549 : static int code_motion_path_driver (insn_t, av_set_t, ilist_t,
550 : cmpd_local_params_p, void *);
551 : static void sel_sched_region_1 (void);
552 : static void sel_sched_region_2 (int);
553 : static av_set_t compute_av_set_inside_bb (insn_t, ilist_t, int, bool);
554 :
555 : static void debug_state (state_t);
556 :
557 :
558 : /* Functions that work with fences. */
559 :
560 : /* Advance one cycle on FENCE. */
561 : static void
562 3348 : advance_one_cycle (fence_t fence)
563 : {
564 3348 : unsigned i;
565 3348 : int cycle;
566 3348 : rtx_insn *insn;
567 :
568 3348 : advance_state (FENCE_STATE (fence));
569 3348 : cycle = ++FENCE_CYCLE (fence);
570 3348 : FENCE_ISSUED_INSNS (fence) = 0;
571 3348 : FENCE_STARTS_CYCLE_P (fence) = 1;
572 3348 : can_issue_more = issue_rate;
573 3348 : FENCE_ISSUE_MORE (fence) = can_issue_more;
574 :
575 14373 : for (i = 0; vec_safe_iterate (FENCE_EXECUTING_INSNS (fence), i, &insn); )
576 : {
577 11025 : if (INSN_READY_CYCLE (insn) < cycle)
578 : {
579 2355 : remove_from_deps (FENCE_DC (fence), insn);
580 2355 : FENCE_EXECUTING_INSNS (fence)->unordered_remove (i);
581 2355 : continue;
582 : }
583 8670 : i++;
584 : }
585 3348 : if (sched_verbose >= 2)
586 : {
587 0 : sel_print ("Finished a cycle. Current cycle = %d\n", FENCE_CYCLE (fence));
588 0 : debug_state (FENCE_STATE (fence));
589 : }
590 3348 : }
591 :
592 : /* Returns true when SUCC in a fallthru bb of INSN, possibly
593 : skipping empty basic blocks. */
594 : static bool
595 108 : in_fallthru_bb_p (rtx_insn *insn, rtx succ)
596 : {
597 108 : basic_block bb = BLOCK_FOR_INSN (insn);
598 108 : edge e;
599 :
600 108 : if (bb == BLOCK_FOR_INSN (succ))
601 : return true;
602 :
603 108 : e = find_fallthru_edge_from (bb);
604 108 : if (e)
605 96 : bb = e->dest;
606 : else
607 : return false;
608 :
609 96 : while (sel_bb_empty_p (bb))
610 0 : bb = bb->next_bb;
611 :
612 96 : return bb == BLOCK_FOR_INSN (succ);
613 : }
614 :
615 : /* Construct successor fences from OLD_FENCEs and put them in NEW_FENCES.
616 : When a successor will continue a ebb, transfer all parameters of a fence
617 : to the new fence. ORIG_MAX_SEQNO is the maximal seqno before this round
618 : of scheduling helping to distinguish between the old and the new code. */
619 : static void
620 958 : extract_new_fences_from (flist_t old_fences, flist_tail_t new_fences,
621 : int orig_max_seqno)
622 : {
623 958 : bool was_here_p = false;
624 958 : insn_t insn = NULL;
625 958 : insn_t succ;
626 958 : succ_iterator si;
627 958 : ilist_iterator ii;
628 958 : fence_t fence = FLIST_FENCE (old_fences);
629 958 : basic_block bb;
630 :
631 : /* Get the only element of FENCE_BNDS (fence). */
632 1916 : FOR_EACH_INSN (insn, ii, FENCE_BNDS (fence))
633 : {
634 958 : gcc_assert (!was_here_p);
635 958 : was_here_p = true;
636 : }
637 958 : gcc_assert (was_here_p && insn != NULL_RTX);
638 :
639 : /* When in the "middle" of the block, just move this fence
640 : to the new list. */
641 958 : bb = BLOCK_FOR_INSN (insn);
642 958 : if (! sel_bb_end_p (insn)
643 1280 : || (single_succ_p (bb)
644 322 : && single_pred_p (single_succ (bb))))
645 : {
646 640 : insn_t succ;
647 :
648 640 : succ = (sel_bb_end_p (insn)
649 644 : ? sel_bb_head (single_succ (bb))
650 636 : : NEXT_INSN (insn));
651 :
652 640 : if (INSN_SEQNO (succ) > 0
653 640 : && INSN_SEQNO (succ) <= orig_max_seqno
654 1280 : && INSN_SCHED_TIMES (succ) <= 0)
655 : {
656 639 : FENCE_INSN (fence) = succ;
657 639 : move_fence_to_fences (old_fences, new_fences);
658 :
659 639 : if (sched_verbose >= 1)
660 0 : sel_print ("Fence %d continues as %d[%d] (state continue)\n",
661 0 : INSN_UID (insn), INSN_UID (succ), BLOCK_NUM (succ));
662 : }
663 640 : return;
664 : }
665 :
666 : /* Otherwise copy fence's structures to (possibly) multiple successors. */
667 711 : FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
668 : {
669 393 : int seqno = INSN_SEQNO (succ);
670 :
671 393 : if (seqno > 0 && seqno <= orig_max_seqno
672 393 : && (pipelining_p || INSN_SCHED_TIMES (succ) <= 0))
673 : {
674 269 : bool b = (in_same_ebb_p (insn, succ)
675 269 : || in_fallthru_bb_p (insn, succ));
676 :
677 269 : if (sched_verbose >= 1)
678 0 : sel_print ("Fence %d continues as %d[%d] (state %s)\n",
679 0 : INSN_UID (insn), INSN_UID (succ),
680 0 : BLOCK_NUM (succ), b ? "continue" : "reset");
681 :
682 269 : if (b)
683 178 : add_dirty_fence_to_fences (new_fences, succ, fence);
684 : else
685 : {
686 : /* Mark block of the SUCC as head of the new ebb. */
687 91 : bitmap_set_bit (forced_ebb_heads, BLOCK_NUM (succ));
688 91 : add_clean_fence_to_fences (new_fences, succ, fence);
689 : }
690 : }
691 : }
692 : }
693 :
694 :
695 : /* Functions to support substitution. */
696 :
697 : /* Returns whether INSN with dependence status DS is eligible for
698 : substitution, i.e. it's a copy operation x := y, and RHS that is
699 : moved up through this insn should be substituted. */
700 : static bool
701 1067 : can_substitute_through_p (insn_t insn, ds_t ds)
702 : {
703 : /* We can substitute only true dependencies. */
704 1067 : if ((ds & DEP_OUTPUT)
705 1066 : || (ds & DEP_ANTI)
706 941 : || ! INSN_RHS (insn)
707 1780 : || ! INSN_LHS (insn))
708 : return false;
709 :
710 : /* Now we just need to make sure the INSN_RHS consists of only one
711 : simple REG rtx. */
712 713 : if (REG_P (INSN_LHS (insn))
713 713 : && REG_P (INSN_RHS (insn)))
714 101 : return true;
715 : return false;
716 : }
717 :
718 : /* Substitute all occurrences of INSN's destination in EXPR' vinsn with INSN's
719 : source (if INSN is eligible for substitution). Returns TRUE if
720 : substitution was actually performed, FALSE otherwise. Substitution might
721 : be not performed because it's either EXPR' vinsn doesn't contain INSN's
722 : destination or the resulting insn is invalid for the target machine.
723 : When UNDO is true, perform unsubstitution instead (the difference is in
724 : the part of rtx on which validate_replace_rtx is called). */
725 : static bool
726 101 : substitute_reg_in_expr (expr_t expr, insn_t insn, bool undo)
727 : {
728 101 : rtx *where;
729 101 : bool new_insn_valid;
730 101 : vinsn_t *vi = &EXPR_VINSN (expr);
731 101 : bool has_rhs = VINSN_RHS (*vi) != NULL;
732 101 : rtx old, new_rtx;
733 :
734 : /* Do not try to replace in SET_DEST. Although we'll choose new
735 : register for the RHS, we don't want to change RHS' original reg.
736 : If the insn is not SET, we may still be able to substitute something
737 : in it, and if we're here (don't have deps), it doesn't write INSN's
738 : dest. */
739 101 : where = (has_rhs
740 101 : ? &VINSN_RHS (*vi)
741 0 : : &PATTERN (VINSN_INSN_RTX (*vi)));
742 101 : old = undo ? INSN_RHS (insn) : INSN_LHS (insn);
743 :
744 : /* Substitute if INSN has a form of x:=y and LHS(INSN) occurs in *VI. */
745 101 : if (rtx_ok_for_substitution_p (old, *where))
746 : {
747 88 : rtx_insn *new_insn;
748 88 : rtx *where_replace;
749 :
750 : /* We should copy these rtxes before substitution. */
751 88 : new_rtx = copy_rtx (undo ? INSN_LHS (insn) : INSN_RHS (insn));
752 88 : new_insn = create_copy_of_insn_rtx (VINSN_INSN_RTX (*vi));
753 :
754 : /* Where we'll replace.
755 : WHERE_REPLACE should point inside NEW_INSN, so INSN_RHS couldn't be
756 : used instead of SET_SRC. */
757 176 : where_replace = (has_rhs
758 88 : ? &SET_SRC (PATTERN (new_insn))
759 0 : : &PATTERN (new_insn));
760 :
761 88 : new_insn_valid
762 88 : = validate_replace_rtx_part_nosimplify (old, new_rtx, where_replace,
763 : new_insn);
764 :
765 : /* ??? Actually, constrain_operands result depends upon choice of
766 : destination register. E.g. if we allow single register to be an rhs,
767 : and if we try to move dx=ax(as rhs) through ax=dx, we'll result
768 : in invalid insn dx=dx, so we'll loose this rhs here.
769 : Just can't come up with significant testcase for this, so just
770 : leaving it for now. */
771 88 : if (new_insn_valid)
772 : {
773 64 : change_vinsn_in_expr (expr,
774 : create_vinsn_from_insn_rtx (new_insn, false));
775 :
776 : /* Do not allow clobbering the address register of speculative
777 : insns. */
778 64 : if ((EXPR_SPEC_DONE_DS (expr) & SPECULATIVE)
779 64 : && register_unavailable_p (VINSN_REG_USES (EXPR_VINSN (expr)),
780 : expr_dest_reg (expr)))
781 0 : EXPR_TARGET_AVAILABLE (expr) = false;
782 :
783 64 : return true;
784 : }
785 : else
786 : return false;
787 : }
788 : else
789 : return false;
790 : }
791 :
792 : /* Return the number of places WHAT appears within WHERE.
793 : Bail out when we found a reference occupying several hard registers. */
794 : static int
795 101 : count_occurrences_equiv (const_rtx what, const_rtx where)
796 : {
797 101 : int count = 0;
798 101 : subrtx_iterator::array_type array;
799 393 : FOR_EACH_SUBRTX (iter, array, where, NONCONST)
800 : {
801 305 : const_rtx x = *iter;
802 305 : if (REG_P (x) && REGNO (x) == REGNO (what))
803 : {
804 : /* Bail out if mode is different or more than one register is
805 : used. */
806 100 : if (GET_MODE (x) != GET_MODE (what) || REG_NREGS (x) > 1)
807 13 : return 0;
808 88 : count += 1;
809 : }
810 205 : else if (GET_CODE (x) == SUBREG
811 205 : && (!REG_P (SUBREG_REG (x))
812 2 : || REGNO (SUBREG_REG (x)) == REGNO (what)))
813 : /* ??? Do not support substituting regs inside subregs. In that case,
814 : simplify_subreg will be called by validate_replace_rtx, and
815 : unsubstitution will fail later. */
816 : return 0;
817 : }
818 88 : return count;
819 101 : }
820 :
821 : /* Returns TRUE if WHAT is found in WHERE rtx tree. */
822 : static bool
823 101 : rtx_ok_for_substitution_p (rtx what, rtx where)
824 : {
825 101 : return (count_occurrences_equiv (what, where) > 0);
826 : }
827 :
828 :
829 : /* Functions to support register renaming. */
830 :
831 : /* Substitute VI's set source with REGNO. Returns newly created pattern
832 : that has REGNO as its source. */
833 : static rtx_insn *
834 29 : create_insn_rtx_with_rhs (vinsn_t vi, rtx rhs_rtx)
835 : {
836 29 : rtx lhs_rtx;
837 29 : rtx pattern;
838 29 : rtx_insn *insn_rtx;
839 :
840 29 : lhs_rtx = copy_rtx (VINSN_LHS (vi));
841 :
842 29 : pattern = gen_rtx_SET (lhs_rtx, rhs_rtx);
843 29 : insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
844 :
845 29 : return insn_rtx;
846 : }
847 :
848 : /* Returns whether INSN's src can be replaced with register number
849 : NEW_SRC_REG. E.g. the following insn is valid for i386:
850 :
851 : (insn:HI 2205 6585 2207 727 ../../gcc/libiberty/regex.c:3337
852 : (set (mem/s:QI (plus:SI (plus:SI (reg/f:SI 7 sp)
853 : (reg:SI 0 ax [orig:770 c1 ] [770]))
854 : (const_int 288 [0x120])) [0 str S1 A8])
855 : (const_int 0 [0x0])) 43 {*movqi_1} (nil)
856 : (nil))
857 :
858 : But if we change (const_int 0 [0x0]) to (reg:QI 4 si), it will be invalid
859 : because of operand constraints:
860 :
861 : (define_insn "*movqi_1"
862 : [(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m")
863 : (match_operand:QI 1 "general_operand" " q,qn,qm,q,rn,qm,qn")
864 : )]
865 :
866 : So do constrain_operands here, before choosing NEW_SRC_REG as best
867 : reg for rhs. */
868 :
869 : static bool
870 166 : replace_src_with_reg_ok_p (insn_t insn, rtx new_src_reg)
871 : {
872 166 : vinsn_t vi = INSN_VINSN (insn);
873 166 : machine_mode mode;
874 166 : rtx dst_loc;
875 166 : bool res;
876 :
877 166 : gcc_assert (VINSN_SEPARABLE_P (vi));
878 :
879 166 : get_dest_and_mode (insn, &dst_loc, &mode);
880 166 : gcc_assert (mode == GET_MODE (new_src_reg));
881 :
882 166 : if (REG_P (dst_loc) && REGNO (new_src_reg) == REGNO (dst_loc))
883 : return true;
884 :
885 : /* See whether SET_SRC can be replaced with this register. */
886 166 : validate_change (insn, &SET_SRC (PATTERN (insn)), new_src_reg, 1);
887 166 : res = verify_changes (0);
888 166 : cancel_changes (0);
889 :
890 166 : return res;
891 : }
892 :
893 : /* Returns whether INSN still be valid after replacing it's DEST with
894 : register NEW_REG. */
895 : static bool
896 166 : replace_dest_with_reg_ok_p (insn_t insn, rtx new_reg)
897 : {
898 166 : vinsn_t vi = INSN_VINSN (insn);
899 166 : bool res;
900 :
901 : /* We should deal here only with separable insns. */
902 166 : gcc_assert (VINSN_SEPARABLE_P (vi));
903 166 : gcc_assert (GET_MODE (VINSN_LHS (vi)) == GET_MODE (new_reg));
904 :
905 : /* See whether SET_DEST can be replaced with this register. */
906 166 : validate_change (insn, &SET_DEST (PATTERN (insn)), new_reg, 1);
907 166 : res = verify_changes (0);
908 166 : cancel_changes (0);
909 :
910 166 : return res;
911 : }
912 :
913 : /* Create a pattern with rhs of VI and lhs of LHS_RTX. */
914 : static rtx_insn *
915 210 : create_insn_rtx_with_lhs (vinsn_t vi, rtx lhs_rtx)
916 : {
917 210 : rtx rhs_rtx;
918 210 : rtx pattern;
919 210 : rtx_insn *insn_rtx;
920 :
921 210 : rhs_rtx = copy_rtx (VINSN_RHS (vi));
922 :
923 210 : pattern = gen_rtx_SET (lhs_rtx, rhs_rtx);
924 210 : insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
925 :
926 210 : return insn_rtx;
927 : }
928 :
929 : /* Substitute lhs in the given expression EXPR for the register with number
930 : NEW_REGNO. SET_DEST may be arbitrary rtx, not only register. */
931 : static void
932 210 : replace_dest_with_reg_in_expr (expr_t expr, rtx new_reg)
933 : {
934 210 : rtx_insn *insn_rtx;
935 210 : vinsn_t vinsn;
936 :
937 210 : insn_rtx = create_insn_rtx_with_lhs (EXPR_VINSN (expr), new_reg);
938 210 : vinsn = create_vinsn_from_insn_rtx (insn_rtx, false);
939 :
940 210 : change_vinsn_in_expr (expr, vinsn);
941 210 : EXPR_WAS_RENAMED (expr) = 1;
942 210 : EXPR_TARGET_AVAILABLE (expr) = 1;
943 210 : }
944 :
945 : /* Returns whether VI writes either one of the USED_REGS registers or,
946 : if a register is a hard one, one of the UNAVAILABLE_HARD_REGS registers. */
947 : static bool
948 161 : vinsn_writes_one_of_regs_p (vinsn_t vi, regset used_regs,
949 : HARD_REG_SET unavailable_hard_regs)
950 : {
951 161 : unsigned regno;
952 161 : reg_set_iterator rsi;
953 :
954 221 : EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_SETS (vi), 0, regno, rsi)
955 : {
956 161 : if (REGNO_REG_SET_P (used_regs, regno))
957 : return true;
958 60 : if (HARD_REGISTER_NUM_P (regno)
959 60 : && TEST_HARD_REG_BIT (unavailable_hard_regs, regno))
960 : return true;
961 : }
962 :
963 60 : EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_CLOBBERS (vi), 0, regno, rsi)
964 : {
965 0 : if (REGNO_REG_SET_P (used_regs, regno))
966 : return true;
967 0 : if (HARD_REGISTER_NUM_P (regno)
968 0 : && TEST_HARD_REG_BIT (unavailable_hard_regs, regno))
969 : return true;
970 : }
971 :
972 : return false;
973 : }
974 :
975 : /* Returns register class of the output register in INSN.
976 : Returns NO_REGS for call insns because some targets have constraints on
977 : destination register of a call insn.
978 :
979 : Code adopted from regrename.cc::build_def_use. */
980 : static enum reg_class
981 2362 : get_reg_class (rtx_insn *insn)
982 : {
983 2362 : int i, n_ops;
984 :
985 2362 : extract_constrain_insn (insn);
986 2362 : preprocess_constraints (insn);
987 2362 : n_ops = recog_data.n_operands;
988 :
989 2362 : const operand_alternative *op_alt = which_op_alt ();
990 2362 : if (asm_noperands (PATTERN (insn)) > 0)
991 : {
992 0 : for (i = 0; i < n_ops; i++)
993 0 : if (recog_data.operand_type[i] == OP_OUT)
994 : {
995 0 : rtx *loc = recog_data.operand_loc[i];
996 0 : rtx op = *loc;
997 0 : enum reg_class cl = alternative_class (op_alt, i);
998 :
999 0 : if (REG_P (op)
1000 0 : && REGNO (op) == ORIGINAL_REGNO (op))
1001 0 : continue;
1002 :
1003 : return cl;
1004 : }
1005 : }
1006 2362 : else if (!CALL_P (insn))
1007 : {
1008 3430 : for (i = 0; i < n_ops + recog_data.n_dups; i++)
1009 : {
1010 2896 : int opn = i < n_ops ? i : recog_data.dup_num[i - n_ops];
1011 2896 : enum reg_class cl = alternative_class (op_alt, opn);
1012 :
1013 2896 : if (recog_data.operand_type[opn] == OP_OUT ||
1014 : recog_data.operand_type[opn] == OP_INOUT)
1015 : return cl;
1016 : }
1017 : }
1018 :
1019 : /* Insns like
1020 : (insn (set (reg:CCZ 17 flags) (compare:CCZ ...)))
1021 : may result in returning NO_REGS, cause flags is written implicitly through
1022 : CMP insn, which has no OP_OUT | OP_INOUT operands. */
1023 : return NO_REGS;
1024 : }
1025 :
1026 : /* Calculate HARD_REGNO_RENAME_OK data for REGNO. */
1027 : static void
1028 74 : init_hard_regno_rename (int regno)
1029 : {
1030 74 : int cur_reg;
1031 :
1032 74 : SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], regno);
1033 :
1034 6882 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1035 : {
1036 : /* We are not interested in renaming in other regs. */
1037 6808 : if (!TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg))
1038 391 : continue;
1039 :
1040 6417 : if (HARD_REGNO_RENAME_OK (regno, cur_reg))
1041 5233 : SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], cur_reg);
1042 : }
1043 74 : }
1044 :
1045 : /* A wrapper around HARD_REGNO_RENAME_OK that will look into the hard regs
1046 : data first. */
1047 : static inline bool
1048 17327 : sel_hard_regno_rename_ok (int from ATTRIBUTE_UNUSED, int to ATTRIBUTE_UNUSED)
1049 : {
1050 : /* Check whether this is all calculated. */
1051 17327 : if (TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], from))
1052 17253 : return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to);
1053 :
1054 74 : init_hard_regno_rename (from);
1055 :
1056 74 : return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to);
1057 : }
1058 :
1059 : /* Calculate set of registers that are capable of holding MODE. */
1060 : static void
1061 42 : init_regs_for_mode (machine_mode mode)
1062 : {
1063 42 : int cur_reg;
1064 :
1065 42 : CLEAR_HARD_REG_SET (sel_hrd.regs_for_mode[mode]);
1066 :
1067 3906 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1068 : {
1069 3864 : int nregs;
1070 3864 : int i;
1071 :
1072 : /* See whether it accepts all modes that occur in
1073 : original insns. */
1074 3864 : if (!targetm.hard_regno_mode_ok (cur_reg, mode))
1075 1402 : continue;
1076 :
1077 2462 : nregs = hard_regno_nregs (cur_reg, mode);
1078 :
1079 3538 : for (i = nregs - 1; i >= 0; --i)
1080 2462 : if (fixed_regs[cur_reg + i]
1081 1622 : || global_regs[cur_reg + i]
1082 : /* Can't use regs which aren't saved by
1083 : the prologue. */
1084 1622 : || !TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg + i)
1085 : /* Can't use regs with non-null REG_BASE_VALUE, because adjusting
1086 : it affects aliasing globally and invalidates all AV sets. */
1087 3936 : || get_reg_base_value (cur_reg + i)
1088 : #ifdef LEAF_REGISTERS
1089 : /* We can't use a non-leaf register if we're in a
1090 : leaf function. */
1091 : || (crtl->is_leaf
1092 : && !LEAF_REGISTERS[cur_reg + i])
1093 : #endif
1094 : )
1095 : break;
1096 :
1097 2462 : if (i >= 0)
1098 1386 : continue;
1099 :
1100 : /* If the CUR_REG passed all the checks above,
1101 : then it's ok. */
1102 1076 : SET_HARD_REG_BIT (sel_hrd.regs_for_mode[mode], cur_reg);
1103 : }
1104 :
1105 42 : sel_hrd.regs_for_mode_ok[mode] = true;
1106 42 : }
1107 :
1108 : /* Init all register sets gathered in HRD. */
1109 : static void
1110 131 : init_hard_regs_data (void)
1111 : {
1112 131 : int cur_reg = 0;
1113 131 : int cur_mode = 0;
1114 :
1115 131 : CLEAR_HARD_REG_SET (sel_hrd.regs_ever_used);
1116 12183 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1117 12052 : if (df_regs_ever_live_p (cur_reg)
1118 12052 : || crtl->abi->clobbers_full_reg_p (cur_reg))
1119 11048 : SET_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg);
1120 :
1121 : /* Initialize registers that are valid based on mode when this is
1122 : really needed. */
1123 16375 : for (cur_mode = 0; cur_mode < NUM_MACHINE_MODES; cur_mode++)
1124 16244 : sel_hrd.regs_for_mode_ok[cur_mode] = false;
1125 :
1126 : /* Mark that all HARD_REGNO_RENAME_OK is not calculated. */
1127 12183 : for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1128 12052 : CLEAR_HARD_REG_SET (sel_hrd.regs_for_rename[cur_reg]);
1129 :
1130 : #ifdef STACK_REGS
1131 1179 : CLEAR_HARD_REG_SET (sel_hrd.stack_regs);
1132 :
1133 1179 : for (cur_reg = FIRST_STACK_REG; cur_reg <= LAST_STACK_REG; cur_reg++)
1134 1048 : SET_HARD_REG_BIT (sel_hrd.stack_regs, cur_reg);
1135 : #endif
1136 131 : }
1137 :
1138 : /* Mark hardware regs in REG_RENAME_P that are not suitable
1139 : for renaming rhs in INSN due to hardware restrictions (register class,
1140 : modes compatibility etc). This doesn't affect original insn's dest reg,
1141 : if it isn't in USED_REGS. DEF is a definition insn of rhs for which the
1142 : destination register is sought. LHS (DEF->ORIG_INSN) may be REG or MEM.
1143 : Registers that are in used_regs are always marked in
1144 : unavailable_hard_regs as well. */
1145 :
1146 : static void
1147 3147 : mark_unavailable_hard_regs (def_t def, struct reg_rename *reg_rename_p,
1148 : regset used_regs ATTRIBUTE_UNUSED)
1149 : {
1150 3147 : machine_mode mode;
1151 3147 : enum reg_class cl = NO_REGS;
1152 3147 : rtx orig_dest;
1153 3147 : unsigned cur_reg, regno;
1154 3147 : hard_reg_set_iterator hrsi;
1155 :
1156 3147 : gcc_assert (GET_CODE (PATTERN (def->orig_insn)) == SET);
1157 3147 : gcc_assert (reg_rename_p);
1158 :
1159 3147 : orig_dest = SET_DEST (PATTERN (def->orig_insn));
1160 :
1161 : /* We have decided not to rename 'mem = something;' insns, as 'something'
1162 : is usually a register. */
1163 3147 : if (!REG_P (orig_dest))
1164 1319 : return;
1165 :
1166 3147 : regno = REGNO (orig_dest);
1167 :
1168 : /* If before reload, don't try to work with pseudos. */
1169 3147 : if (!reload_completed && !HARD_REGISTER_NUM_P (regno))
1170 : return;
1171 :
1172 2362 : if (reload_completed)
1173 2362 : cl = get_reg_class (def->orig_insn);
1174 :
1175 : /* Stop if the original register is one of the fixed_regs, global_regs or
1176 : frame pointer, or we could not discover its class. */
1177 2362 : if (fixed_regs[regno]
1178 1828 : || global_regs[regno]
1179 1828 : || (!HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed
1180 39 : && regno == HARD_FRAME_POINTER_REGNUM)
1181 : || (HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed
1182 : && regno == FRAME_POINTER_REGNUM)
1183 1828 : || (reload_completed && cl == NO_REGS))
1184 : {
1185 534 : SET_HARD_REG_SET (reg_rename_p->unavailable_hard_regs);
1186 :
1187 : /* Give a chance for original register, if it isn't in used_regs. */
1188 534 : if (!def->crossed_call_abis)
1189 534 : CLEAR_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno);
1190 :
1191 534 : return;
1192 : }
1193 :
1194 : /* If something allocated on stack in this function, mark frame pointer
1195 : register unavailable, considering also modes.
1196 : FIXME: it is enough to do this once per all original defs. */
1197 1828 : if (frame_pointer_needed)
1198 : {
1199 39 : add_to_hard_reg_set (®_rename_p->unavailable_hard_regs,
1200 39 : Pmode, FRAME_POINTER_REGNUM);
1201 :
1202 39 : if (!HARD_FRAME_POINTER_IS_FRAME_POINTER)
1203 39 : add_to_hard_reg_set (®_rename_p->unavailable_hard_regs,
1204 : Pmode, HARD_FRAME_POINTER_REGNUM);
1205 : }
1206 :
1207 : #ifdef STACK_REGS
1208 : /* For the stack registers the presence of FIRST_STACK_REG in USED_REGS
1209 : is equivalent to as if all stack regs were in this set.
1210 : I.e. no stack register can be renamed, and even if it's an original
1211 : register here we make sure it won't be lifted over it's previous def
1212 : (it's previous def will appear as if it's a FIRST_STACK_REG def.
1213 : The HARD_REGNO_RENAME_OK covers other cases in condition below. */
1214 1828 : if (IN_RANGE (REGNO (orig_dest), FIRST_STACK_REG, LAST_STACK_REG)
1215 1828 : && REGNO_REG_SET_P (used_regs, FIRST_STACK_REG))
1216 1828 : reg_rename_p->unavailable_hard_regs |= sel_hrd.stack_regs;
1217 : #endif
1218 :
1219 1828 : mode = GET_MODE (orig_dest);
1220 :
1221 : /* If there's a call on this path, make regs from full_reg_clobbers
1222 : unavailable.
1223 :
1224 : ??? It would be better to track the set of clobbered registers
1225 : directly, but that would be quite expensive in a def_t. */
1226 1828 : if (def->crossed_call_abis)
1227 455 : reg_rename_p->unavailable_hard_regs
1228 910 : |= call_clobbers_in_region (def->crossed_call_abis,
1229 910 : reg_class_contents[ALL_REGS], mode);
1230 :
1231 : /* Stop here before reload: we need FRAME_REGS, STACK_REGS, and
1232 : crossed_call_abis, but not register classes. */
1233 1828 : if (!reload_completed)
1234 : return;
1235 :
1236 : /* Leave regs as 'available' only from the current
1237 : register class. */
1238 1828 : reg_rename_p->available_for_renaming = reg_class_contents[cl];
1239 :
1240 : /* Leave only registers available for this mode. */
1241 1828 : if (!sel_hrd.regs_for_mode_ok[mode])
1242 42 : init_regs_for_mode (mode);
1243 1828 : reg_rename_p->available_for_renaming &= sel_hrd.regs_for_mode[mode];
1244 :
1245 : /* Leave only those that are ok to rename. */
1246 19155 : EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1247 : 0, cur_reg, hrsi)
1248 : {
1249 17327 : int nregs;
1250 17327 : int i;
1251 :
1252 17327 : nregs = hard_regno_nregs (cur_reg, mode);
1253 17327 : gcc_assert (nregs > 0);
1254 :
1255 34654 : for (i = nregs - 1; i >= 0; --i)
1256 17327 : if (! sel_hard_regno_rename_ok (regno + i, cur_reg + i))
1257 : break;
1258 :
1259 17327 : if (i >= 0)
1260 0 : CLEAR_HARD_REG_BIT (reg_rename_p->available_for_renaming,
1261 : cur_reg);
1262 : }
1263 :
1264 1828 : reg_rename_p->available_for_renaming &= ~reg_rename_p->unavailable_hard_regs;
1265 :
1266 : /* Regno is always ok from the renaming part of view, but it really
1267 : could be in *unavailable_hard_regs already, so set it here instead
1268 : of there. */
1269 1828 : SET_HARD_REG_BIT (reg_rename_p->available_for_renaming, regno);
1270 : }
1271 :
1272 : /* reg_rename_tick[REG1] > reg_rename_tick[REG2] if REG1 was chosen as the
1273 : best register more recently than REG2. */
1274 : static int reg_rename_tick[FIRST_PSEUDO_REGISTER];
1275 :
1276 : /* Indicates the number of times renaming happened before the current one. */
1277 : static int reg_rename_this_tick;
1278 :
1279 : /* Choose the register among free, that is suitable for storing
1280 : the rhs value.
1281 :
1282 : ORIGINAL_INSNS is the list of insns where the operation (rhs)
1283 : originally appears. There could be multiple original operations
1284 : for single rhs since we moving it up and merging along different
1285 : paths.
1286 :
1287 : Some code is adapted from regrename.cc (regrename_optimize).
1288 : If original register is available, function returns it.
1289 : Otherwise it performs the checks, so the new register should
1290 : comply with the following:
1291 : - it should not violate any live ranges (such registers are in
1292 : REG_RENAME_P->available_for_renaming set);
1293 : - it should not be in the HARD_REGS_USED regset;
1294 : - it should be in the class compatible with original uses;
1295 : - it should not be clobbered through reference with different mode;
1296 : - if we're in the leaf function, then the new register should
1297 : not be in the LEAF_REGISTERS;
1298 : - etc.
1299 :
1300 : If several registers meet the conditions, the register with smallest
1301 : tick is returned to achieve more even register allocation.
1302 :
1303 : If original register seems to be ok, we set *IS_ORIG_REG_P_PTR to true.
1304 :
1305 : If no register satisfies the above conditions, NULL_RTX is returned. */
1306 : static rtx
1307 2340 : choose_best_reg_1 (HARD_REG_SET hard_regs_used,
1308 : struct reg_rename *reg_rename_p,
1309 : def_list_t original_insns, bool *is_orig_reg_p_ptr)
1310 : {
1311 2340 : int best_new_reg;
1312 2340 : unsigned cur_reg;
1313 2340 : machine_mode mode = VOIDmode;
1314 2340 : unsigned regno, i, n;
1315 2340 : hard_reg_set_iterator hrsi;
1316 2340 : def_list_iterator di;
1317 2340 : def_t def;
1318 :
1319 : /* If original register is available, return it. */
1320 2340 : *is_orig_reg_p_ptr = true;
1321 :
1322 4684 : FOR_EACH_DEF (def, di, original_insns)
1323 : {
1324 2350 : rtx orig_dest = SET_DEST (PATTERN (def->orig_insn));
1325 :
1326 2350 : gcc_assert (REG_P (orig_dest));
1327 :
1328 : /* Check that all original operations have the same mode.
1329 : This is done for the next loop; if we'd return from this
1330 : loop, we'd check only part of them, but in this case
1331 : it doesn't matter. */
1332 2350 : if (mode == VOIDmode)
1333 2340 : mode = GET_MODE (orig_dest);
1334 2350 : gcc_assert (mode == GET_MODE (orig_dest));
1335 :
1336 2350 : regno = REGNO (orig_dest);
1337 2356 : for (i = 0, n = REG_NREGS (orig_dest); i < n; i++)
1338 2350 : if (TEST_HARD_REG_BIT (hard_regs_used, regno + i))
1339 : break;
1340 :
1341 : /* All hard registers are available. */
1342 2350 : if (i == n)
1343 : {
1344 6 : gcc_assert (mode != VOIDmode);
1345 :
1346 : /* Hard registers should not be shared. */
1347 6 : return gen_rtx_REG (mode, regno);
1348 : }
1349 : }
1350 :
1351 2334 : *is_orig_reg_p_ptr = false;
1352 2334 : best_new_reg = -1;
1353 :
1354 : /* Among all available regs choose the register that was
1355 : allocated earliest. */
1356 19918 : EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1357 : 0, cur_reg, hrsi)
1358 18115 : if (! TEST_HARD_REG_BIT (hard_regs_used, cur_reg))
1359 : {
1360 : /* Check that all hard regs for mode are available. */
1361 547 : for (i = 1, n = hard_regno_nregs (cur_reg, mode); i < n; i++)
1362 0 : if (TEST_HARD_REG_BIT (hard_regs_used, cur_reg + i)
1363 0 : || !TEST_HARD_REG_BIT (reg_rename_p->available_for_renaming,
1364 : cur_reg + i))
1365 : break;
1366 :
1367 547 : if (i < n)
1368 0 : continue;
1369 :
1370 : /* All hard registers are available. */
1371 547 : if (best_new_reg < 0
1372 1 : || reg_rename_tick[cur_reg] < reg_rename_tick[best_new_reg])
1373 : {
1374 547 : best_new_reg = cur_reg;
1375 :
1376 : /* Return immediately when we know there's no better reg. */
1377 547 : if (! reg_rename_tick[best_new_reg])
1378 : break;
1379 : }
1380 : }
1381 :
1382 2334 : if (best_new_reg >= 0)
1383 : {
1384 : /* Use the check from the above loop. */
1385 546 : gcc_assert (mode != VOIDmode);
1386 546 : return gen_rtx_REG (mode, best_new_reg);
1387 : }
1388 :
1389 : return NULL_RTX;
1390 : }
1391 :
1392 : /* A wrapper around choose_best_reg_1 () to verify that we make correct
1393 : assumptions about available registers in the function. */
1394 : static rtx
1395 2340 : choose_best_reg (HARD_REG_SET hard_regs_used, struct reg_rename *reg_rename_p,
1396 : def_list_t original_insns, bool *is_orig_reg_p_ptr)
1397 : {
1398 2340 : rtx best_reg = choose_best_reg_1 (hard_regs_used, reg_rename_p,
1399 : original_insns, is_orig_reg_p_ptr);
1400 :
1401 : /* FIXME loop over hard_regno_nregs here. */
1402 2340 : gcc_assert (best_reg == NULL_RTX
1403 : || TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, REGNO (best_reg)));
1404 :
1405 2340 : return best_reg;
1406 : }
1407 :
1408 : /* Choose the pseudo register for storing rhs value. As this is supposed
1409 : to work before reload, we return either the original register or make
1410 : the new one. The parameters are the same that in choose_nest_reg_1
1411 : functions, except that USED_REGS may contain pseudos.
1412 : If we work with hard regs, check also REG_RENAME_P->UNAVAILABLE_HARD_REGS.
1413 :
1414 : TODO: take into account register pressure while doing this. Up to this
1415 : moment, this function would never return NULL for pseudos, but we should
1416 : not rely on this. */
1417 : static rtx
1418 289 : choose_best_pseudo_reg (regset used_regs,
1419 : struct reg_rename *reg_rename_p,
1420 : def_list_t original_insns, bool *is_orig_reg_p_ptr)
1421 : {
1422 289 : def_list_iterator i;
1423 289 : def_t def;
1424 289 : machine_mode mode = VOIDmode;
1425 289 : bool bad_hard_regs = false;
1426 :
1427 : /* We should not use this after reload. */
1428 289 : gcc_assert (!reload_completed);
1429 :
1430 : /* If original register is available, return it. */
1431 289 : *is_orig_reg_p_ptr = true;
1432 :
1433 1074 : FOR_EACH_DEF (def, i, original_insns)
1434 : {
1435 785 : rtx dest = SET_DEST (PATTERN (def->orig_insn));
1436 785 : int orig_regno;
1437 :
1438 785 : gcc_assert (REG_P (dest));
1439 :
1440 : /* Check that all original operations have the same mode. */
1441 785 : if (mode == VOIDmode)
1442 289 : mode = GET_MODE (dest);
1443 : else
1444 496 : gcc_assert (mode == GET_MODE (dest));
1445 785 : orig_regno = REGNO (dest);
1446 :
1447 : /* Check that nothing in used_regs intersects with orig_regno. When
1448 : we have a hard reg here, still loop over hard_regno_nregs. */
1449 785 : if (HARD_REGISTER_NUM_P (orig_regno))
1450 : {
1451 0 : int j, n;
1452 0 : for (j = 0, n = REG_NREGS (dest); j < n; j++)
1453 0 : if (REGNO_REG_SET_P (used_regs, orig_regno + j))
1454 : break;
1455 0 : if (j < n)
1456 0 : continue;
1457 : }
1458 : else
1459 : {
1460 785 : if (REGNO_REG_SET_P (used_regs, orig_regno))
1461 785 : continue;
1462 : }
1463 0 : if (HARD_REGISTER_NUM_P (orig_regno))
1464 : {
1465 0 : gcc_assert (df_regs_ever_live_p (orig_regno));
1466 :
1467 : /* For hard registers, we have to check hardware imposed
1468 : limitations (frame/stack registers, calls crossed). */
1469 0 : if (!TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs,
1470 : orig_regno))
1471 : {
1472 : /* Don't let register cross a call if it doesn't already
1473 : cross one. This condition is written in accordance with
1474 : that in sched-deps.cc sched_analyze_reg(). */
1475 0 : if (!reg_rename_p->crossed_call_abis
1476 0 : || REG_N_CALLS_CROSSED (orig_regno) > 0)
1477 0 : return gen_rtx_REG (mode, orig_regno);
1478 : }
1479 :
1480 : bad_hard_regs = true;
1481 : }
1482 : else
1483 : return dest;
1484 : }
1485 :
1486 289 : *is_orig_reg_p_ptr = false;
1487 :
1488 : /* We had some original hard registers that couldn't be used.
1489 : Those were likely special. Don't try to create a pseudo. */
1490 289 : if (bad_hard_regs)
1491 : return NULL_RTX;
1492 :
1493 : /* We haven't found a register from original operations. Get a new one.
1494 : FIXME: control register pressure somehow. */
1495 289 : {
1496 289 : rtx new_reg = gen_reg_rtx (mode);
1497 :
1498 289 : gcc_assert (mode != VOIDmode);
1499 :
1500 289 : max_regno = max_reg_num ();
1501 289 : maybe_extend_reg_info_p ();
1502 289 : REG_N_CALLS_CROSSED (REGNO (new_reg))
1503 289 : = reg_rename_p->crossed_call_abis ? 1 : 0;
1504 :
1505 289 : return new_reg;
1506 : }
1507 : }
1508 :
1509 : /* True when target of EXPR is available due to EXPR_TARGET_AVAILABLE,
1510 : USED_REGS and REG_RENAME_P->UNAVAILABLE_HARD_REGS. */
1511 : static void
1512 2629 : verify_target_availability (expr_t expr, regset used_regs,
1513 : struct reg_rename *reg_rename_p)
1514 : {
1515 2629 : unsigned n, i, regno;
1516 2629 : machine_mode mode;
1517 2629 : bool target_available, live_available, hard_available;
1518 :
1519 2629 : if (!REG_P (EXPR_LHS (expr)) || EXPR_TARGET_AVAILABLE (expr) < 0)
1520 : return;
1521 :
1522 2613 : regno = expr_dest_regno (expr);
1523 2613 : mode = GET_MODE (EXPR_LHS (expr));
1524 2613 : target_available = EXPR_TARGET_AVAILABLE (expr) == 1;
1525 2613 : n = HARD_REGISTER_NUM_P (regno) ? hard_regno_nregs (regno, mode) : 1;
1526 :
1527 2613 : live_available = hard_available = true;
1528 5226 : for (i = 0; i < n; i++)
1529 : {
1530 2613 : if (bitmap_bit_p (used_regs, regno + i))
1531 2527 : live_available = false;
1532 2613 : if (TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno + i))
1533 325 : hard_available = false;
1534 : }
1535 :
1536 : /* When target is not available, it may be due to hard register
1537 : restrictions, e.g. crosses calls, so we check hard_available too. */
1538 2613 : if (target_available)
1539 11 : gcc_assert (live_available);
1540 : else
1541 : /* Check only if we haven't scheduled something on the previous fence,
1542 : cause due to MAX_SOFTWARE_LOOKAHEAD_WINDOW_SIZE issues
1543 : and having more than one fence, we may end having targ_un in a block
1544 : in which successors target register is actually available.
1545 :
1546 : The last condition handles the case when a dependence from a call insn
1547 : was created in sched-deps.cc for insns with destination registers that
1548 : never crossed a call before, but do cross one after our code motion.
1549 :
1550 : FIXME: in the latter case, we just uselessly called find_used_regs,
1551 : because we can't move this expression with any other register
1552 : as well. */
1553 2602 : gcc_assert (scheduled_something_on_previous_fence || !live_available
1554 : || !hard_available
1555 : || (!reload_completed
1556 : && reg_rename_p->crossed_call_abis
1557 : && REG_N_CALLS_CROSSED (regno) == 0));
1558 : }
1559 :
1560 : /* Collect unavailable registers due to liveness for EXPR from BNDS
1561 : into USED_REGS. Save additional information about available
1562 : registers and unavailable due to hardware restriction registers
1563 : into REG_RENAME_P structure. Save original insns into ORIGINAL_INSNS
1564 : list. */
1565 : static void
1566 2790 : collect_unavailable_regs_from_bnds (expr_t expr, blist_t bnds, regset used_regs,
1567 : struct reg_rename *reg_rename_p,
1568 : def_list_t *original_insns)
1569 : {
1570 5580 : for (; bnds; bnds = BLIST_NEXT (bnds))
1571 : {
1572 2790 : bool res;
1573 2790 : av_set_t orig_ops = NULL;
1574 2790 : bnd_t bnd = BLIST_BND (bnds);
1575 :
1576 : /* If the chosen best expr doesn't belong to current boundary,
1577 : skip it. */
1578 2790 : if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr)))
1579 0 : continue;
1580 :
1581 : /* Put in ORIG_OPS all exprs from this boundary that became
1582 : RES on top. */
1583 2790 : orig_ops = find_sequential_best_exprs (bnd, expr, false);
1584 :
1585 : /* Compute used regs and OR it into the USED_REGS. */
1586 2790 : res = find_used_regs (BND_TO (bnd), orig_ops, used_regs,
1587 : reg_rename_p, original_insns);
1588 :
1589 : /* FIXME: the assert is true until we'd have several boundaries. */
1590 2790 : gcc_assert (res);
1591 2790 : av_set_clear (&orig_ops);
1592 : }
1593 2790 : }
1594 :
1595 : /* Return TRUE if it is possible to replace LHSes of ORIG_INSNS with BEST_REG.
1596 : If BEST_REG is valid, replace LHS of EXPR with it. */
1597 : static bool
1598 161 : try_replace_dest_reg (ilist_t orig_insns, rtx best_reg, expr_t expr)
1599 : {
1600 : /* Try whether we'll be able to generate the insn
1601 : 'dest := best_reg' at the place of the original operation. */
1602 339 : for (; orig_insns; orig_insns = ILIST_NEXT (orig_insns))
1603 : {
1604 178 : insn_t orig_insn = DEF_LIST_DEF (orig_insns)->orig_insn;
1605 :
1606 178 : gcc_assert (EXPR_SEPARABLE_P (INSN_EXPR (orig_insn)));
1607 :
1608 178 : if (REGNO (best_reg) != REGNO (INSN_LHS (orig_insn))
1609 178 : && (! replace_src_with_reg_ok_p (orig_insn, best_reg)
1610 166 : || ! replace_dest_with_reg_ok_p (orig_insn, best_reg)))
1611 0 : return false;
1612 : }
1613 :
1614 : /* Make sure that EXPR has the right destination
1615 : register. */
1616 161 : if (expr_dest_regno (expr) != REGNO (best_reg))
1617 : {
1618 155 : rtx_insn *vinsn = EXPR_INSN_RTX (expr);
1619 155 : validate_change (vinsn, &SET_DEST (PATTERN (vinsn)), best_reg, 1);
1620 155 : bool res = verify_changes (0);
1621 155 : cancel_changes (0);
1622 155 : if (!res)
1623 : return false;
1624 155 : replace_dest_with_reg_in_expr (expr, best_reg);
1625 : }
1626 : else
1627 6 : EXPR_TARGET_AVAILABLE (expr) = 1;
1628 :
1629 : return true;
1630 : }
1631 :
1632 : /* Select and assign best register to EXPR searching from BNDS.
1633 : Set *IS_ORIG_REG_P to TRUE if original register was selected.
1634 : Return FALSE if no register can be chosen, which could happen when:
1635 : * EXPR_SEPARABLE_P is true but we were unable to find suitable register;
1636 : * EXPR_SEPARABLE_P is false but the insn sets/clobbers one of the registers
1637 : that are used on the moving path. */
1638 : static bool
1639 2790 : find_best_reg_for_expr (expr_t expr, blist_t bnds, bool *is_orig_reg_p)
1640 : {
1641 2790 : static struct reg_rename reg_rename_data;
1642 :
1643 2790 : regset used_regs;
1644 2790 : def_list_t original_insns = NULL;
1645 2790 : bool reg_ok;
1646 :
1647 2790 : *is_orig_reg_p = false;
1648 :
1649 : /* Don't bother to do anything if this insn doesn't set any registers. */
1650 2790 : if (bitmap_empty_p (VINSN_REG_SETS (EXPR_VINSN (expr)))
1651 2790 : && bitmap_empty_p (VINSN_REG_CLOBBERS (EXPR_VINSN (expr))))
1652 : return true;
1653 :
1654 2790 : used_regs = get_clear_regset_from_pool ();
1655 2790 : CLEAR_HARD_REG_SET (reg_rename_data.unavailable_hard_regs);
1656 :
1657 2790 : collect_unavailable_regs_from_bnds (expr, bnds, used_regs, ®_rename_data,
1658 : &original_insns);
1659 :
1660 : /* If after reload, make sure we're working with hard regs here. */
1661 2790 : if (flag_checking && reload_completed)
1662 : {
1663 2350 : reg_set_iterator rsi;
1664 2350 : unsigned i;
1665 :
1666 2350 : EXECUTE_IF_SET_IN_REG_SET (used_regs, FIRST_PSEUDO_REGISTER, i, rsi)
1667 0 : gcc_unreachable ();
1668 : }
1669 :
1670 2790 : if (EXPR_SEPARABLE_P (expr))
1671 : {
1672 2629 : rtx best_reg = NULL_RTX;
1673 : /* Check that we have computed availability of a target register
1674 : correctly. */
1675 2629 : verify_target_availability (expr, used_regs, ®_rename_data);
1676 :
1677 : /* Turn everything in hard regs after reload. */
1678 2629 : if (reload_completed)
1679 : {
1680 : HARD_REG_SET hard_regs_used;
1681 2340 : REG_SET_TO_HARD_REG_SET (hard_regs_used, used_regs);
1682 :
1683 : /* Join hard registers unavailable due to register class
1684 : restrictions and live range intersection. */
1685 2340 : hard_regs_used |= reg_rename_data.unavailable_hard_regs;
1686 :
1687 2340 : best_reg = choose_best_reg (hard_regs_used, ®_rename_data,
1688 : original_insns, is_orig_reg_p);
1689 : }
1690 : else
1691 289 : best_reg = choose_best_pseudo_reg (used_regs, ®_rename_data,
1692 : original_insns, is_orig_reg_p);
1693 :
1694 2629 : if (!best_reg)
1695 : reg_ok = false;
1696 841 : else if (*is_orig_reg_p)
1697 : {
1698 : /* In case of unification BEST_REG may be different from EXPR's LHS
1699 : when EXPR's LHS is unavailable, and there is another LHS among
1700 : ORIGINAL_INSNS. */
1701 6 : reg_ok = try_replace_dest_reg (original_insns, best_reg, expr);
1702 : }
1703 : else
1704 : {
1705 : /* Forbid renaming of low-cost insns. */
1706 835 : if (sel_vinsn_cost (EXPR_VINSN (expr)) < 2)
1707 : reg_ok = false;
1708 : else
1709 155 : reg_ok = try_replace_dest_reg (original_insns, best_reg, expr);
1710 : }
1711 : }
1712 : else
1713 : {
1714 : /* If !EXPR_SCHEDULE_AS_RHS (EXPR), just make sure INSN doesn't set
1715 : any of the HARD_REGS_USED set. */
1716 161 : if (vinsn_writes_one_of_regs_p (EXPR_VINSN (expr), used_regs,
1717 : reg_rename_data.unavailable_hard_regs))
1718 : {
1719 101 : reg_ok = false;
1720 101 : gcc_assert (EXPR_TARGET_AVAILABLE (expr) <= 0);
1721 : }
1722 : else
1723 : {
1724 60 : reg_ok = true;
1725 60 : gcc_assert (EXPR_TARGET_AVAILABLE (expr) != 0);
1726 : }
1727 : }
1728 :
1729 2790 : ilist_clear (&original_insns);
1730 2790 : return_regset_to_pool (used_regs);
1731 :
1732 2790 : return reg_ok;
1733 : }
1734 :
1735 :
1736 : /* Return true if dependence described by DS can be overcomed. */
1737 : static bool
1738 4646 : can_speculate_dep_p (ds_t ds)
1739 : {
1740 4646 : if (spec_info == NULL)
1741 : return false;
1742 :
1743 : /* Leave only speculative data. */
1744 0 : ds &= SPECULATIVE;
1745 :
1746 0 : if (ds == 0)
1747 : return false;
1748 :
1749 0 : {
1750 : /* FIXME: make sched-deps.cc produce only those non-hard dependencies,
1751 : that we can overcome. */
1752 0 : ds_t spec_mask = spec_info->mask;
1753 :
1754 0 : if ((ds & spec_mask) != ds)
1755 : return false;
1756 : }
1757 :
1758 0 : if (ds_weak (ds) < spec_info->data_weakness_cutoff)
1759 : return false;
1760 :
1761 : return true;
1762 : }
1763 :
1764 : /* Get a speculation check instruction.
1765 : C_EXPR is a speculative expression,
1766 : CHECK_DS describes speculations that should be checked,
1767 : ORIG_INSN is the original non-speculative insn in the stream. */
1768 : static insn_t
1769 0 : create_speculation_check (expr_t c_expr, ds_t check_ds, insn_t orig_insn)
1770 : {
1771 0 : rtx check_pattern;
1772 0 : rtx_insn *insn_rtx;
1773 0 : insn_t insn;
1774 0 : basic_block recovery_block;
1775 0 : rtx_insn *label;
1776 :
1777 : /* Create a recovery block if target is going to emit branchy check, or if
1778 : ORIG_INSN was speculative already. */
1779 0 : if (targetm.sched.needs_block_p (check_ds)
1780 0 : || EXPR_SPEC_DONE_DS (INSN_EXPR (orig_insn)) != 0)
1781 : {
1782 0 : recovery_block = sel_create_recovery_block (orig_insn);
1783 0 : label = BB_HEAD (recovery_block);
1784 : }
1785 : else
1786 : {
1787 : recovery_block = NULL;
1788 : label = NULL;
1789 : }
1790 :
1791 : /* Get pattern of the check. */
1792 0 : check_pattern = targetm.sched.gen_spec_check (EXPR_INSN_RTX (c_expr), label,
1793 : check_ds);
1794 :
1795 0 : gcc_assert (check_pattern != NULL);
1796 :
1797 : /* Emit check. */
1798 0 : insn_rtx = create_insn_rtx_from_pattern (check_pattern, label);
1799 :
1800 0 : insn = sel_gen_insn_from_rtx_after (insn_rtx, INSN_EXPR (orig_insn),
1801 0 : INSN_SEQNO (orig_insn), orig_insn);
1802 :
1803 : /* Make check to be non-speculative. */
1804 0 : EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
1805 0 : INSN_SPEC_CHECKED_DS (insn) = check_ds;
1806 :
1807 : /* Decrease priority of check by difference of load/check instruction
1808 : latencies. */
1809 0 : EXPR_PRIORITY (INSN_EXPR (insn)) -= (sel_vinsn_cost (INSN_VINSN (orig_insn))
1810 0 : - sel_vinsn_cost (INSN_VINSN (insn)));
1811 :
1812 : /* Emit copy of original insn (though with replaced target register,
1813 : if needed) to the recovery block. */
1814 0 : if (recovery_block != NULL)
1815 : {
1816 0 : rtx twin_rtx;
1817 :
1818 0 : twin_rtx = copy_rtx (PATTERN (EXPR_INSN_RTX (c_expr)));
1819 0 : twin_rtx = create_insn_rtx_from_pattern (twin_rtx, NULL_RTX);
1820 0 : sel_gen_recovery_insn_from_rtx_after (twin_rtx,
1821 0 : INSN_EXPR (orig_insn),
1822 0 : INSN_SEQNO (insn),
1823 0 : bb_note (recovery_block));
1824 : }
1825 :
1826 : /* If we've generated a data speculation check, make sure
1827 : that all the bookkeeping instruction we'll create during
1828 : this move_op () will allocate an ALAT entry so that the
1829 : check won't fail.
1830 : In case of control speculation we must convert C_EXPR to control
1831 : speculative mode, because failing to do so will bring us an exception
1832 : thrown by the non-control-speculative load. */
1833 0 : check_ds = ds_get_max_dep_weak (check_ds);
1834 0 : speculate_expr (c_expr, check_ds);
1835 :
1836 0 : return insn;
1837 : }
1838 :
1839 : /* True when INSN is a "regN = regN" copy. */
1840 : static bool
1841 28 : identical_copy_p (rtx_insn *insn)
1842 : {
1843 28 : rtx lhs, rhs, pat;
1844 :
1845 28 : pat = PATTERN (insn);
1846 :
1847 28 : if (GET_CODE (pat) != SET)
1848 : return false;
1849 :
1850 25 : lhs = SET_DEST (pat);
1851 25 : if (!REG_P (lhs))
1852 : return false;
1853 :
1854 25 : rhs = SET_SRC (pat);
1855 25 : if (!REG_P (rhs))
1856 : return false;
1857 :
1858 25 : return REGNO (lhs) == REGNO (rhs);
1859 : }
1860 :
1861 : /* Undo all transformations on *AV_PTR that were done when
1862 : moving through INSN. */
1863 : static void
1864 48771 : undo_transformations (av_set_t *av_ptr, rtx_insn *insn)
1865 : {
1866 48771 : av_set_iterator av_iter;
1867 48771 : expr_t expr;
1868 48771 : av_set_t new_set = NULL;
1869 :
1870 : /* First, kill any EXPR that uses registers set by an insn. This is
1871 : required for correctness. */
1872 146943 : FOR_EACH_EXPR_1 (expr, av_iter, av_ptr)
1873 49086 : if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (expr))
1874 49086 : && bitmap_intersect_p (INSN_REG_SETS (insn),
1875 49086 : VINSN_REG_USES (EXPR_VINSN (expr)))
1876 : /* When an insn looks like 'r1 = r1', we could substitute through
1877 : it, but the above condition will still hold. This happened with
1878 : gcc.c-torture/execute/961125-1.c. */
1879 49114 : && !identical_copy_p (insn))
1880 : {
1881 28 : if (sched_verbose >= 6)
1882 0 : sel_print ("Expr %d removed due to use/set conflict\n",
1883 0 : INSN_UID (EXPR_INSN_RTX (expr)));
1884 28 : av_set_iter_remove (&av_iter);
1885 : }
1886 :
1887 : /* Undo transformations looking at the history vector. */
1888 146887 : FOR_EACH_EXPR (expr, av_iter, *av_ptr)
1889 : {
1890 49058 : int index = find_in_history_vect (EXPR_HISTORY_OF_CHANGES (expr),
1891 : insn, EXPR_VINSN (expr), true);
1892 :
1893 49058 : if (index >= 0)
1894 : {
1895 166 : expr_history_def *phist;
1896 :
1897 166 : phist = &EXPR_HISTORY_OF_CHANGES (expr)[index];
1898 :
1899 166 : switch (phist->type)
1900 : {
1901 0 : case TRANS_SPECULATION:
1902 0 : {
1903 0 : ds_t old_ds, new_ds;
1904 :
1905 : /* Compute the difference between old and new speculative
1906 : statuses: that's what we need to check.
1907 : Earlier we used to assert that the status will really
1908 : change. This no longer works because only the probability
1909 : bits in the status may have changed during compute_av_set,
1910 : and in the case of merging different probabilities of the
1911 : same speculative status along different paths we do not
1912 : record this in the history vector. */
1913 0 : old_ds = phist->spec_ds;
1914 0 : new_ds = EXPR_SPEC_DONE_DS (expr);
1915 :
1916 0 : old_ds &= SPECULATIVE;
1917 0 : new_ds &= SPECULATIVE;
1918 0 : new_ds &= ~old_ds;
1919 :
1920 0 : EXPR_SPEC_TO_CHECK_DS (expr) |= new_ds;
1921 0 : break;
1922 : }
1923 166 : case TRANS_SUBSTITUTION:
1924 166 : {
1925 166 : expr_def _tmp_expr, *tmp_expr = &_tmp_expr;
1926 166 : vinsn_t new_vi;
1927 166 : bool add = true;
1928 :
1929 166 : new_vi = phist->old_expr_vinsn;
1930 :
1931 166 : gcc_assert (VINSN_SEPARABLE_P (new_vi)
1932 : == EXPR_SEPARABLE_P (expr));
1933 166 : copy_expr (tmp_expr, expr);
1934 :
1935 166 : if (vinsn_equal_p (phist->new_expr_vinsn,
1936 : EXPR_VINSN (tmp_expr)))
1937 166 : change_vinsn_in_expr (tmp_expr, new_vi);
1938 : else
1939 : /* This happens when we're unsubstituting on a bookkeeping
1940 : copy, which was in turn substituted. The history is wrong
1941 : in this case. Do it the hard way. */
1942 0 : add = substitute_reg_in_expr (tmp_expr, insn, true);
1943 166 : if (add)
1944 166 : av_set_add (&new_set, tmp_expr);
1945 166 : clear_expr (tmp_expr);
1946 166 : break;
1947 : }
1948 0 : default:
1949 0 : gcc_unreachable ();
1950 : }
1951 : }
1952 :
1953 : }
1954 :
1955 48771 : av_set_union_and_clear (av_ptr, &new_set, NULL);
1956 48771 : }
1957 :
1958 :
1959 : /* Moveup_* helpers for code motion and computing av sets. */
1960 :
1961 : /* Propagates EXPR inside an insn group through THROUGH_INSN.
1962 : The difference from the below function is that only substitution is
1963 : performed. */
1964 : static enum MOVEUP_EXPR_CODE
1965 0 : moveup_expr_inside_insn_group (expr_t expr, insn_t through_insn)
1966 : {
1967 0 : vinsn_t vi = EXPR_VINSN (expr);
1968 0 : ds_t *has_dep_p;
1969 0 : ds_t full_ds;
1970 :
1971 : /* Do this only inside insn group. */
1972 0 : gcc_assert (INSN_SCHED_CYCLE (through_insn) > 0);
1973 :
1974 0 : full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
1975 0 : if (full_ds == 0)
1976 : return MOVEUP_EXPR_SAME;
1977 :
1978 : /* Substitution is the possible choice in this case. */
1979 0 : if (has_dep_p[DEPS_IN_RHS])
1980 : {
1981 : /* Can't substitute UNIQUE VINSNs. */
1982 0 : gcc_assert (!VINSN_UNIQUE_P (vi));
1983 :
1984 0 : if (can_substitute_through_p (through_insn,
1985 : has_dep_p[DEPS_IN_RHS])
1986 0 : && substitute_reg_in_expr (expr, through_insn, false))
1987 : {
1988 0 : EXPR_WAS_SUBSTITUTED (expr) = true;
1989 0 : return MOVEUP_EXPR_CHANGED;
1990 : }
1991 :
1992 : /* Don't care about this, as even true dependencies may be allowed
1993 : in an insn group. */
1994 0 : return MOVEUP_EXPR_SAME;
1995 : }
1996 :
1997 : /* This can catch output dependencies in COND_EXECs. */
1998 0 : if (has_dep_p[DEPS_IN_INSN])
1999 : return MOVEUP_EXPR_NULL;
2000 :
2001 : /* This is either an output or an anti dependence, which usually have
2002 : a zero latency. Allow this here, if we'd be wrong, tick_check_p
2003 : will fix this. */
2004 0 : gcc_assert (has_dep_p[DEPS_IN_LHS]);
2005 : return MOVEUP_EXPR_AS_RHS;
2006 : }
2007 :
2008 : /* True when a trapping EXPR cannot be moved through THROUGH_INSN. */
2009 : #define CANT_MOVE_TRAPPING(expr, through_insn) \
2010 : (VINSN_MAY_TRAP_P (EXPR_VINSN (expr)) \
2011 : && !sel_insn_has_single_succ_p ((through_insn), SUCCS_ALL) \
2012 : && !sel_insn_is_speculation_check (through_insn))
2013 :
2014 : /* True when a conflict on a target register was found during moveup_expr. */
2015 : static bool was_target_conflict = false;
2016 :
2017 : /* Return true when moving a debug INSN across THROUGH_INSN will
2018 : create a bookkeeping block. We don't want to create such blocks,
2019 : for they would cause codegen differences between compilations with
2020 : and without debug info. */
2021 :
2022 : static bool
2023 79 : moving_insn_creates_bookkeeping_block_p (insn_t insn,
2024 : insn_t through_insn)
2025 : {
2026 79 : basic_block bbi, bbt;
2027 79 : edge e1, e2;
2028 79 : edge_iterator ei1, ei2;
2029 :
2030 79 : if (!bookkeeping_can_be_created_if_moved_through_p (through_insn))
2031 : {
2032 59 : if (sched_verbose >= 9)
2033 0 : sel_print ("no bookkeeping required: ");
2034 59 : return false;
2035 : }
2036 :
2037 20 : bbi = BLOCK_FOR_INSN (insn);
2038 :
2039 20 : if (EDGE_COUNT (bbi->preds) == 1)
2040 : {
2041 1 : if (sched_verbose >= 9)
2042 0 : sel_print ("only one pred edge: ");
2043 1 : return true;
2044 : }
2045 :
2046 19 : bbt = BLOCK_FOR_INSN (through_insn);
2047 :
2048 36 : FOR_EACH_EDGE (e1, ei1, bbt->succs)
2049 : {
2050 62 : FOR_EACH_EDGE (e2, ei2, bbi->preds)
2051 : {
2052 45 : if (find_block_for_bookkeeping (e1, e2, true))
2053 : {
2054 10 : if (sched_verbose >= 9)
2055 0 : sel_print ("found existing block: ");
2056 10 : return false;
2057 : }
2058 : }
2059 : }
2060 :
2061 9 : if (sched_verbose >= 9)
2062 0 : sel_print ("would create bookkeeping block: ");
2063 :
2064 : return true;
2065 : }
2066 :
2067 : /* Return true when the conflict with newly created implicit clobbers
2068 : between EXPR and THROUGH_INSN is found because of renaming. */
2069 : static bool
2070 37 : implicit_clobber_conflict_p (insn_t through_insn, expr_t expr)
2071 : {
2072 37 : HARD_REG_SET temp;
2073 37 : rtx_insn *insn;
2074 37 : rtx reg, rhs, pat;
2075 37 : hard_reg_set_iterator hrsi;
2076 37 : unsigned regno;
2077 37 : bool valid;
2078 :
2079 : /* Make a new pseudo register. */
2080 37 : reg = gen_reg_rtx (GET_MODE (EXPR_LHS (expr)));
2081 37 : max_regno = max_reg_num ();
2082 37 : maybe_extend_reg_info_p ();
2083 :
2084 : /* Validate a change and bail out early. */
2085 37 : insn = EXPR_INSN_RTX (expr);
2086 37 : validate_change (insn, &SET_DEST (PATTERN (insn)), reg, true);
2087 37 : valid = verify_changes (0);
2088 37 : cancel_changes (0);
2089 37 : if (!valid)
2090 : {
2091 37 : if (sched_verbose >= 6)
2092 0 : sel_print ("implicit clobbers failed validation, ");
2093 37 : return true;
2094 : }
2095 :
2096 : /* Make a new insn with it. */
2097 0 : rhs = copy_rtx (VINSN_RHS (EXPR_VINSN (expr)));
2098 0 : pat = gen_rtx_SET (reg, rhs);
2099 0 : start_sequence ();
2100 0 : insn = emit_insn (pat);
2101 0 : end_sequence ();
2102 :
2103 : /* Calculate implicit clobbers. */
2104 0 : extract_insn (insn);
2105 0 : preprocess_constraints (insn);
2106 0 : alternative_mask prefrred = get_preferred_alternatives (insn);
2107 0 : ira_implicitly_set_insn_hard_regs (&temp, prefrred);
2108 0 : temp &= ~ira_no_alloc_regs;
2109 :
2110 : /* If any implicit clobber registers intersect with regular ones in
2111 : through_insn, we have a dependency and thus bail out. */
2112 0 : EXECUTE_IF_SET_IN_HARD_REG_SET (temp, 0, regno, hrsi)
2113 : {
2114 0 : vinsn_t vi = INSN_VINSN (through_insn);
2115 0 : if (bitmap_bit_p (VINSN_REG_SETS (vi), regno)
2116 0 : || bitmap_bit_p (VINSN_REG_CLOBBERS (vi), regno)
2117 0 : || bitmap_bit_p (VINSN_REG_USES (vi), regno))
2118 0 : return true;
2119 : }
2120 :
2121 : return false;
2122 : }
2123 :
2124 : /* Modifies EXPR so it can be moved through the THROUGH_INSN,
2125 : performing necessary transformations. Record the type of transformation
2126 : made in PTRANS_TYPE, when it is not NULL. When INSIDE_INSN_GROUP,
2127 : permit all dependencies except true ones, and try to remove those
2128 : too via forward substitution. All cases when a non-eliminable
2129 : non-zero cost dependency exists inside an insn group will be fixed
2130 : in tick_check_p instead. */
2131 : static enum MOVEUP_EXPR_CODE
2132 19305 : moveup_expr (expr_t expr, insn_t through_insn, bool inside_insn_group,
2133 : enum local_trans_type *ptrans_type)
2134 : {
2135 19305 : vinsn_t vi = EXPR_VINSN (expr);
2136 19305 : insn_t insn = VINSN_INSN_RTX (vi);
2137 19305 : bool was_changed = false;
2138 19305 : bool as_rhs = false;
2139 19305 : ds_t *has_dep_p;
2140 19305 : ds_t full_ds;
2141 :
2142 : /* ??? We use dependencies of non-debug insns on debug insns to
2143 : indicate that the debug insns need to be reset if the non-debug
2144 : insn is pulled ahead of it. It's hard to figure out how to
2145 : introduce such a notion in sel-sched, but it already fails to
2146 : support debug insns in other ways, so we just go ahead and
2147 : let the deug insns go corrupt for now. */
2148 19305 : if (DEBUG_INSN_P (through_insn) && !DEBUG_INSN_P (insn))
2149 : return MOVEUP_EXPR_SAME;
2150 :
2151 : /* When inside_insn_group, delegate to the helper. */
2152 19233 : if (inside_insn_group)
2153 0 : return moveup_expr_inside_insn_group (expr, through_insn);
2154 :
2155 : /* Deal with unique insns and control dependencies. */
2156 19233 : if (VINSN_UNIQUE_P (vi))
2157 : {
2158 : /* We can move jumps without side-effects or jumps that are
2159 : mutually exclusive with instruction THROUGH_INSN (all in cases
2160 : dependencies allow to do so and jump is not speculative). */
2161 1950 : if (control_flow_insn_p (insn))
2162 : {
2163 825 : basic_block fallthru_bb;
2164 :
2165 : /* Do not move checks and do not move jumps through other
2166 : jumps. */
2167 825 : if (control_flow_insn_p (through_insn)
2168 825 : || sel_insn_is_speculation_check (insn))
2169 2 : return MOVEUP_EXPR_NULL;
2170 :
2171 : /* Don't move jumps through CFG joins. */
2172 823 : if (bookkeeping_can_be_created_if_moved_through_p (through_insn))
2173 : return MOVEUP_EXPR_NULL;
2174 :
2175 : /* The jump should have a clear fallthru block, and
2176 : this block should be in the current region. */
2177 817 : if ((fallthru_bb = fallthru_bb_of_jump (insn)) == NULL
2178 817 : || ! in_current_region_p (fallthru_bb))
2179 646 : return MOVEUP_EXPR_NULL;
2180 :
2181 : /* And it should be mutually exclusive with through_insn. */
2182 171 : if (! sched_insns_conditions_mutex_p (insn, through_insn)
2183 171 : && ! DEBUG_INSN_P (through_insn))
2184 : return MOVEUP_EXPR_NULL;
2185 : }
2186 :
2187 : /* Don't move what we can't move. */
2188 1125 : if (EXPR_CANT_MOVE (expr)
2189 1125 : && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn))
2190 : return MOVEUP_EXPR_NULL;
2191 :
2192 : /* Don't move SCHED_GROUP instruction through anything.
2193 : If we don't force this, then it will be possible to start
2194 : scheduling a sched_group before all its dependencies are
2195 : resolved.
2196 : ??? Haifa deals with this issue by delaying the SCHED_GROUP
2197 : as late as possible through rank_for_schedule. */
2198 992 : if (SCHED_GROUP_P (insn))
2199 : return MOVEUP_EXPR_NULL;
2200 : }
2201 : else
2202 17283 : gcc_assert (!control_flow_insn_p (insn));
2203 :
2204 : /* Don't move debug insns if this would require bookkeeping. */
2205 18245 : if (DEBUG_INSN_P (insn)
2206 120 : && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn)
2207 18324 : && moving_insn_creates_bookkeeping_block_p (insn, through_insn))
2208 : return MOVEUP_EXPR_NULL;
2209 :
2210 : /* Deal with data dependencies. */
2211 18235 : was_target_conflict = false;
2212 18235 : full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
2213 18235 : if (full_ds == 0)
2214 : {
2215 14034 : if (!CANT_MOVE_TRAPPING (expr, through_insn))
2216 14033 : return MOVEUP_EXPR_SAME;
2217 : }
2218 : else
2219 : {
2220 : /* We can move UNIQUE insn up only as a whole and unchanged,
2221 : so it shouldn't have any dependencies. */
2222 4201 : if (VINSN_UNIQUE_P (vi))
2223 : return MOVEUP_EXPR_NULL;
2224 : }
2225 :
2226 3580 : if (full_ds != 0 && can_speculate_dep_p (full_ds))
2227 : {
2228 0 : int res;
2229 :
2230 0 : res = speculate_expr (expr, full_ds);
2231 0 : if (res >= 0)
2232 : {
2233 : /* Speculation was successful. */
2234 0 : full_ds = 0;
2235 0 : was_changed = (res > 0);
2236 0 : if (res == 2)
2237 0 : was_target_conflict = true;
2238 0 : if (ptrans_type)
2239 0 : *ptrans_type = TRANS_SPECULATION;
2240 0 : sel_clear_has_dependence ();
2241 : }
2242 : }
2243 :
2244 3580 : if (has_dep_p[DEPS_IN_INSN])
2245 : /* We have some dependency that cannot be discarded. */
2246 : return MOVEUP_EXPR_NULL;
2247 :
2248 2803 : if (has_dep_p[DEPS_IN_LHS])
2249 : {
2250 : /* Only separable insns can be moved up with the new register.
2251 : Anyways, we should mark that the original register is
2252 : unavailable. */
2253 1970 : if (!enable_schedule_as_rhs_p || !EXPR_SEPARABLE_P (expr))
2254 : return MOVEUP_EXPR_NULL;
2255 :
2256 : /* When renaming a hard register to a pseudo before reload, extra
2257 : dependencies can occur from the implicit clobbers of the insn.
2258 : Filter out such cases here. */
2259 205 : if (!reload_completed && REG_P (EXPR_LHS (expr))
2260 205 : && HARD_REGISTER_P (EXPR_LHS (expr))
2261 1340 : && implicit_clobber_conflict_p (through_insn, expr))
2262 : {
2263 37 : if (sched_verbose >= 6)
2264 0 : sel_print ("implicit clobbers conflict detected, ");
2265 37 : return MOVEUP_EXPR_NULL;
2266 : }
2267 1266 : EXPR_TARGET_AVAILABLE (expr) = false;
2268 1266 : was_target_conflict = true;
2269 1266 : as_rhs = true;
2270 : }
2271 :
2272 : /* At this point we have either separable insns, that will be lifted
2273 : up only as RHSes, or non-separable insns with no dependency in lhs.
2274 : If dependency is in RHS, then try to perform substitution and move up
2275 : substituted RHS:
2276 :
2277 : Ex. 1: Ex.2
2278 : y = x; y = x;
2279 : z = y*2; y = y*2;
2280 :
2281 : In Ex.1 y*2 can be substituted for x*2 and the whole operation can be
2282 : moved above y=x assignment as z=x*2.
2283 :
2284 : In Ex.2 y*2 also can be substituted for x*2, but only the right hand
2285 : side can be moved because of the output dependency. The operation was
2286 : cropped to its rhs above. */
2287 2099 : if (has_dep_p[DEPS_IN_RHS])
2288 : {
2289 1067 : ds_t *rhs_dsp = &has_dep_p[DEPS_IN_RHS];
2290 :
2291 : /* Can't substitute UNIQUE VINSNs. */
2292 1067 : gcc_assert (!VINSN_UNIQUE_P (vi));
2293 :
2294 1067 : if (can_speculate_dep_p (*rhs_dsp))
2295 : {
2296 0 : int res;
2297 :
2298 0 : res = speculate_expr (expr, *rhs_dsp);
2299 0 : if (res >= 0)
2300 : {
2301 : /* Speculation was successful. */
2302 0 : *rhs_dsp = 0;
2303 0 : was_changed = (res > 0);
2304 0 : if (res == 2)
2305 0 : was_target_conflict = true;
2306 0 : if (ptrans_type)
2307 0 : *ptrans_type = TRANS_SPECULATION;
2308 : }
2309 : else
2310 : return MOVEUP_EXPR_NULL;
2311 : }
2312 1067 : else if (can_substitute_through_p (through_insn,
2313 : *rhs_dsp)
2314 1067 : && substitute_reg_in_expr (expr, through_insn, false))
2315 : {
2316 : /* ??? We cannot perform substitution AND speculation on the same
2317 : insn. */
2318 64 : gcc_assert (!was_changed);
2319 64 : was_changed = true;
2320 64 : if (ptrans_type)
2321 64 : *ptrans_type = TRANS_SUBSTITUTION;
2322 64 : EXPR_WAS_SUBSTITUTED (expr) = true;
2323 : }
2324 : else
2325 1003 : return MOVEUP_EXPR_NULL;
2326 : }
2327 :
2328 : /* Don't move trapping insns through jumps.
2329 : This check should be at the end to give a chance to control speculation
2330 : to perform its duties. */
2331 1096 : if (CANT_MOVE_TRAPPING (expr, through_insn))
2332 : return MOVEUP_EXPR_NULL;
2333 :
2334 1094 : return (was_changed
2335 1094 : ? MOVEUP_EXPR_CHANGED
2336 : : (as_rhs
2337 1030 : ? MOVEUP_EXPR_AS_RHS
2338 : : MOVEUP_EXPR_SAME));
2339 : }
2340 :
2341 : /* Try to look at bitmap caches for EXPR and INSN pair, return true
2342 : if successful. When INSIDE_INSN_GROUP, also try ignore dependencies
2343 : that can exist within a parallel group. Write to RES the resulting
2344 : code for moveup_expr. */
2345 : static bool
2346 179038 : try_bitmap_cache (expr_t expr, insn_t insn,
2347 : bool inside_insn_group,
2348 : enum MOVEUP_EXPR_CODE *res)
2349 : {
2350 179038 : int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2351 :
2352 : /* First check whether we've analyzed this situation already. */
2353 179038 : if (bitmap_bit_p (INSN_ANALYZED_DEPS (insn), expr_uid))
2354 : {
2355 143386 : if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2356 : {
2357 39984 : if (sched_verbose >= 6)
2358 0 : sel_print ("removed (cached)\n");
2359 39984 : *res = MOVEUP_EXPR_NULL;
2360 39984 : return true;
2361 : }
2362 : else
2363 : {
2364 103402 : if (sched_verbose >= 6)
2365 0 : sel_print ("unchanged (cached)\n");
2366 103402 : *res = MOVEUP_EXPR_SAME;
2367 103402 : return true;
2368 : }
2369 : }
2370 35652 : else if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2371 : {
2372 16039 : if (inside_insn_group)
2373 : {
2374 0 : if (sched_verbose >= 6)
2375 0 : sel_print ("unchanged (as RHS, cached, inside insn group)\n");
2376 0 : *res = MOVEUP_EXPR_SAME;
2377 0 : return true;
2378 :
2379 : }
2380 : else
2381 16039 : EXPR_TARGET_AVAILABLE (expr) = false;
2382 :
2383 : /* This is the only case when propagation result can change over time,
2384 : as we can dynamically switch off scheduling as RHS. In this case,
2385 : just check the flag to reach the correct decision. */
2386 16039 : if (enable_schedule_as_rhs_p)
2387 : {
2388 15930 : if (sched_verbose >= 6)
2389 0 : sel_print ("unchanged (as RHS, cached)\n");
2390 15930 : *res = MOVEUP_EXPR_AS_RHS;
2391 15930 : return true;
2392 : }
2393 : else
2394 : {
2395 109 : if (sched_verbose >= 6)
2396 0 : sel_print ("removed (cached as RHS, but renaming"
2397 : " is now disabled)\n");
2398 109 : *res = MOVEUP_EXPR_NULL;
2399 109 : return true;
2400 : }
2401 : }
2402 :
2403 : return false;
2404 : }
2405 :
2406 : /* Try to look at bitmap caches for EXPR and INSN pair, return true
2407 : if successful. Write to RES the resulting code for moveup_expr. */
2408 : static bool
2409 19613 : try_transformation_cache (expr_t expr, insn_t insn,
2410 : enum MOVEUP_EXPR_CODE *res)
2411 : {
2412 19613 : struct transformed_insns *pti
2413 : = (struct transformed_insns *)
2414 19613 : htab_find_with_hash (INSN_TRANSFORMED_INSNS (insn),
2415 19613 : &EXPR_VINSN (expr),
2416 19613 : VINSN_HASH_RTX (EXPR_VINSN (expr)));
2417 19613 : if (pti)
2418 : {
2419 : /* This EXPR was already moved through this insn and was
2420 : changed as a result. Fetch the proper data from
2421 : the hashtable. */
2422 724 : insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2423 362 : INSN_UID (insn), pti->type,
2424 : pti->vinsn_old, pti->vinsn_new,
2425 : EXPR_SPEC_DONE_DS (expr));
2426 :
2427 362 : if (INSN_IN_STREAM_P (VINSN_INSN_RTX (pti->vinsn_new)))
2428 2 : pti->vinsn_new = vinsn_copy (pti->vinsn_new, true);
2429 362 : change_vinsn_in_expr (expr, pti->vinsn_new);
2430 362 : if (pti->was_target_conflict)
2431 35 : EXPR_TARGET_AVAILABLE (expr) = false;
2432 362 : if (pti->type == TRANS_SPECULATION)
2433 : {
2434 0 : EXPR_SPEC_DONE_DS (expr) = pti->ds;
2435 0 : EXPR_NEEDS_SPEC_CHECK_P (expr) |= pti->needs_check;
2436 : }
2437 :
2438 362 : if (sched_verbose >= 6)
2439 : {
2440 0 : sel_print ("changed (cached): ");
2441 0 : dump_expr (expr);
2442 0 : sel_print ("\n");
2443 : }
2444 :
2445 362 : *res = MOVEUP_EXPR_CHANGED;
2446 362 : return true;
2447 : }
2448 :
2449 : return false;
2450 : }
2451 :
2452 : /* Update bitmap caches on INSN with result RES of propagating EXPR. */
2453 : static void
2454 19241 : update_bitmap_cache (expr_t expr, insn_t insn, bool inside_insn_group,
2455 : enum MOVEUP_EXPR_CODE res)
2456 : {
2457 19241 : int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2458 :
2459 : /* Do not cache result of propagating jumps through an insn group,
2460 : as it is always true, which is not useful outside the group. */
2461 19241 : if (inside_insn_group)
2462 : return;
2463 :
2464 19241 : if (res == MOVEUP_EXPR_NULL)
2465 : {
2466 4106 : bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2467 4106 : bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2468 : }
2469 15135 : else if (res == MOVEUP_EXPR_SAME)
2470 : {
2471 14105 : bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2472 14105 : bitmap_clear_bit (INSN_FOUND_DEPS (insn), expr_uid);
2473 : }
2474 1030 : else if (res == MOVEUP_EXPR_AS_RHS)
2475 : {
2476 1030 : bitmap_clear_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2477 1030 : bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2478 : }
2479 : else
2480 0 : gcc_unreachable ();
2481 : }
2482 :
2483 : /* Update hashtable on INSN with changed EXPR, old EXPR_OLD_VINSN
2484 : and transformation type TRANS_TYPE. */
2485 : static void
2486 64 : update_transformation_cache (expr_t expr, insn_t insn,
2487 : bool inside_insn_group,
2488 : enum local_trans_type trans_type,
2489 : vinsn_t expr_old_vinsn)
2490 : {
2491 64 : struct transformed_insns *pti;
2492 :
2493 64 : if (inside_insn_group)
2494 : return;
2495 :
2496 64 : pti = XNEW (struct transformed_insns);
2497 64 : pti->vinsn_old = expr_old_vinsn;
2498 64 : pti->vinsn_new = EXPR_VINSN (expr);
2499 64 : pti->type = trans_type;
2500 64 : pti->was_target_conflict = was_target_conflict;
2501 64 : pti->ds = EXPR_SPEC_DONE_DS (expr);
2502 64 : pti->needs_check = EXPR_NEEDS_SPEC_CHECK_P (expr);
2503 64 : vinsn_attach (pti->vinsn_old);
2504 64 : vinsn_attach (pti->vinsn_new);
2505 64 : *((struct transformed_insns **)
2506 64 : htab_find_slot_with_hash (INSN_TRANSFORMED_INSNS (insn),
2507 64 : pti, VINSN_HASH_RTX (expr_old_vinsn),
2508 64 : INSERT)) = pti;
2509 : }
2510 :
2511 : /* Same as moveup_expr, but first looks up the result of
2512 : transformation in caches. */
2513 : static enum MOVEUP_EXPR_CODE
2514 179092 : moveup_expr_cached (expr_t expr, insn_t insn, bool inside_insn_group)
2515 : {
2516 179092 : enum MOVEUP_EXPR_CODE res;
2517 179092 : bool got_answer = false;
2518 :
2519 179092 : if (sched_verbose >= 6)
2520 : {
2521 0 : sel_print ("Moving ");
2522 0 : dump_expr (expr);
2523 0 : sel_print (" through %d: ", INSN_UID (insn));
2524 : }
2525 :
2526 179092 : if (DEBUG_INSN_P (EXPR_INSN_RTX (expr))
2527 296 : && BLOCK_FOR_INSN (EXPR_INSN_RTX (expr))
2528 179349 : && (sel_bb_head (BLOCK_FOR_INSN (EXPR_INSN_RTX (expr)))
2529 257 : == EXPR_INSN_RTX (expr)))
2530 : /* Don't use cached information for debug insns that are heads of
2531 : basic blocks. */;
2532 179038 : else if (try_bitmap_cache (expr, insn, inside_insn_group, &res))
2533 : /* When inside insn group, we do not want remove stores conflicting
2534 : with previosly issued loads. */
2535 159425 : got_answer = ! inside_insn_group || res != MOVEUP_EXPR_NULL;
2536 19613 : else if (try_transformation_cache (expr, insn, &res))
2537 : got_answer = true;
2538 :
2539 : if (! got_answer)
2540 : {
2541 : /* Invoke moveup_expr and record the results. */
2542 19305 : vinsn_t expr_old_vinsn = EXPR_VINSN (expr);
2543 19305 : ds_t expr_old_spec_ds = EXPR_SPEC_DONE_DS (expr);
2544 19305 : int expr_uid = INSN_UID (VINSN_INSN_RTX (expr_old_vinsn));
2545 19305 : bool unique_p = VINSN_UNIQUE_P (expr_old_vinsn);
2546 19305 : enum local_trans_type trans_type = TRANS_SUBSTITUTION;
2547 :
2548 : /* ??? Invent something better than this. We can't allow old_vinsn
2549 : to go, we need it for the history vector. */
2550 19305 : vinsn_attach (expr_old_vinsn);
2551 :
2552 19305 : res = moveup_expr (expr, insn, inside_insn_group,
2553 : &trans_type);
2554 19305 : switch (res)
2555 : {
2556 4106 : case MOVEUP_EXPR_NULL:
2557 4106 : update_bitmap_cache (expr, insn, inside_insn_group, res);
2558 4106 : if (sched_verbose >= 6)
2559 0 : sel_print ("removed\n");
2560 : break;
2561 :
2562 14105 : case MOVEUP_EXPR_SAME:
2563 14105 : update_bitmap_cache (expr, insn, inside_insn_group, res);
2564 14105 : if (sched_verbose >= 6)
2565 0 : sel_print ("unchanged\n");
2566 : break;
2567 :
2568 1030 : case MOVEUP_EXPR_AS_RHS:
2569 1030 : gcc_assert (!unique_p || inside_insn_group);
2570 1030 : update_bitmap_cache (expr, insn, inside_insn_group, res);
2571 1030 : if (sched_verbose >= 6)
2572 0 : sel_print ("unchanged (as RHS)\n");
2573 : break;
2574 :
2575 64 : case MOVEUP_EXPR_CHANGED:
2576 64 : gcc_assert (INSN_UID (EXPR_INSN_RTX (expr)) != expr_uid
2577 : || EXPR_SPEC_DONE_DS (expr) != expr_old_spec_ds);
2578 128 : insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2579 64 : INSN_UID (insn), trans_type,
2580 : expr_old_vinsn, EXPR_VINSN (expr),
2581 : expr_old_spec_ds);
2582 64 : update_transformation_cache (expr, insn, inside_insn_group,
2583 : trans_type, expr_old_vinsn);
2584 64 : if (sched_verbose >= 6)
2585 : {
2586 0 : sel_print ("changed: ");
2587 0 : dump_expr (expr);
2588 0 : sel_print ("\n");
2589 : }
2590 : break;
2591 : default:
2592 : gcc_unreachable ();
2593 : }
2594 :
2595 19305 : vinsn_detach (expr_old_vinsn);
2596 : }
2597 :
2598 179092 : return res;
2599 : }
2600 :
2601 : /* Moves an av set AVP up through INSN, performing necessary
2602 : transformations. */
2603 : static void
2604 54166 : moveup_set_expr (av_set_t *avp, insn_t insn, bool inside_insn_group)
2605 : {
2606 54166 : av_set_iterator i;
2607 54166 : expr_t expr;
2608 :
2609 398916 : FOR_EACH_EXPR_1 (expr, i, avp)
2610 : {
2611 :
2612 172375 : switch (moveup_expr_cached (expr, insn, inside_insn_group))
2613 : {
2614 : case MOVEUP_EXPR_SAME:
2615 : case MOVEUP_EXPR_AS_RHS:
2616 : break;
2617 :
2618 44199 : case MOVEUP_EXPR_NULL:
2619 44199 : av_set_iter_remove (&i);
2620 44199 : break;
2621 :
2622 422 : case MOVEUP_EXPR_CHANGED:
2623 422 : expr = merge_with_other_exprs (avp, &i, expr);
2624 422 : break;
2625 :
2626 0 : default:
2627 0 : gcc_unreachable ();
2628 : }
2629 : }
2630 54166 : }
2631 :
2632 : /* Moves AVP set along PATH. */
2633 : static void
2634 5189 : moveup_set_inside_insn_group (av_set_t *avp, ilist_t path)
2635 : {
2636 5189 : int last_cycle;
2637 :
2638 5189 : if (sched_verbose >= 6)
2639 0 : sel_print ("Moving expressions up in the insn group...\n");
2640 5189 : if (! path)
2641 : return;
2642 0 : last_cycle = INSN_SCHED_CYCLE (ILIST_INSN (path));
2643 0 : while (path
2644 0 : && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2645 : {
2646 0 : moveup_set_expr (avp, ILIST_INSN (path), true);
2647 0 : path = ILIST_NEXT (path);
2648 : }
2649 : }
2650 :
2651 : /* Returns true if after moving EXPR along PATH it equals to EXPR_VLIW. */
2652 : static bool
2653 17841 : equal_after_moveup_path_p (expr_t expr, ilist_t path, expr_t expr_vliw)
2654 : {
2655 17841 : expr_def _tmp, *tmp = &_tmp;
2656 17841 : int last_cycle;
2657 17841 : bool res = true;
2658 :
2659 17841 : copy_expr_onside (tmp, expr);
2660 17841 : last_cycle = path ? INSN_SCHED_CYCLE (ILIST_INSN (path)) : 0;
2661 17841 : while (path
2662 17841 : && res
2663 17841 : && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2664 : {
2665 0 : res = (moveup_expr_cached (tmp, ILIST_INSN (path), true)
2666 : != MOVEUP_EXPR_NULL);
2667 0 : path = ILIST_NEXT (path);
2668 : }
2669 :
2670 17841 : if (res)
2671 : {
2672 17841 : vinsn_t tmp_vinsn = EXPR_VINSN (tmp);
2673 17841 : vinsn_t expr_vliw_vinsn = EXPR_VINSN (expr_vliw);
2674 :
2675 17841 : if (tmp_vinsn != expr_vliw_vinsn)
2676 10187 : res = vinsn_equal_p (tmp_vinsn, expr_vliw_vinsn);
2677 : }
2678 :
2679 17841 : clear_expr (tmp);
2680 17841 : return res;
2681 : }
2682 :
2683 :
2684 : /* Functions that compute av and lv sets. */
2685 :
2686 : /* Returns true if INSN is not a downward continuation of the given path P in
2687 : the current stage. */
2688 : static bool
2689 74580 : is_ineligible_successor (insn_t insn, ilist_t p)
2690 : {
2691 74580 : insn_t prev_insn;
2692 :
2693 : /* Check if insn is not deleted. */
2694 74580 : if (PREV_INSN (insn) && NEXT_INSN (PREV_INSN (insn)) != insn)
2695 0 : gcc_unreachable ();
2696 74580 : else if (NEXT_INSN (insn) && PREV_INSN (NEXT_INSN (insn)) != insn)
2697 0 : gcc_unreachable ();
2698 :
2699 : /* If it's the first insn visited, then the successor is ok. */
2700 74580 : if (!p)
2701 : return false;
2702 :
2703 60475 : prev_insn = ILIST_INSN (p);
2704 :
2705 60475 : if (/* a backward edge. */
2706 60475 : INSN_SEQNO (insn) < INSN_SEQNO (prev_insn)
2707 : /* is already visited. */
2708 59554 : || (INSN_SEQNO (insn) == INSN_SEQNO (prev_insn)
2709 3131 : && (ilist_is_in_p (p, insn)
2710 : /* We can reach another fence here and still seqno of insn
2711 : would be equal to seqno of prev_insn. This is possible
2712 : when prev_insn is a previously created bookkeeping copy.
2713 : In that case it'd get a seqno of insn. Thus, check here
2714 : whether insn is in current fence too. */
2715 3131 : || IN_CURRENT_FENCE_P (insn)))
2716 : /* Was already scheduled on this round. */
2717 59553 : || (INSN_SEQNO (insn) > INSN_SEQNO (prev_insn)
2718 56423 : && IN_CURRENT_FENCE_P (insn))
2719 : /* An insn from another fence could also be
2720 : scheduled earlier even if this insn is not in
2721 : a fence list right now. Check INSN_SCHED_CYCLE instead. */
2722 120023 : || (!pipelining_p
2723 20616 : && INSN_SCHED_TIMES (insn) > 0))
2724 950 : return true;
2725 : else
2726 : return false;
2727 : }
2728 :
2729 : /* Computes the av_set below the last bb insn INSN, doing all the 'dirty work'
2730 : of handling multiple successors and properly merging its av_sets. P is
2731 : the current path traversed. WS is the size of lookahead window.
2732 : Return the av set computed. */
2733 : static av_set_t
2734 8544 : compute_av_set_at_bb_end (insn_t insn, ilist_t p, int ws)
2735 : {
2736 8544 : struct succs_info *sinfo;
2737 8544 : av_set_t expr_in_all_succ_branches = NULL;
2738 8544 : int is;
2739 8544 : insn_t succ, zero_succ = NULL;
2740 8544 : av_set_t av1 = NULL;
2741 :
2742 8544 : gcc_assert (sel_bb_end_p (insn));
2743 :
2744 : /* Find different kind of successors needed for correct computing of
2745 : SPEC and TARGET_AVAILABLE attributes. */
2746 8544 : sinfo = compute_succs_info (insn, SUCCS_NORMAL);
2747 :
2748 : /* Debug output. */
2749 8544 : if (sched_verbose >= 6)
2750 : {
2751 0 : sel_print ("successors of bb end (%d): ", INSN_UID (insn));
2752 0 : dump_insn_vector (sinfo->succs_ok);
2753 0 : sel_print ("\n");
2754 0 : if (sinfo->succs_ok_n != sinfo->all_succs_n)
2755 0 : sel_print ("real successors num: %d\n", sinfo->all_succs_n);
2756 : }
2757 :
2758 : /* Add insn to the tail of current path. */
2759 8544 : ilist_add (&p, insn);
2760 :
2761 14835 : FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ)
2762 : {
2763 6291 : av_set_t succ_set;
2764 :
2765 : /* We will edit SUCC_SET and EXPR_SPEC field of its elements. */
2766 6291 : succ_set = compute_av_set_inside_bb (succ, p, ws, true);
2767 :
2768 12582 : av_set_split_usefulness (succ_set,
2769 6291 : sinfo->probs_ok[is],
2770 : sinfo->all_prob);
2771 :
2772 6291 : if (sinfo->all_succs_n > 1)
2773 : {
2774 : /* Find EXPR'es that came from *all* successors and save them
2775 : into expr_in_all_succ_branches. This set will be used later
2776 : for calculating speculation attributes of EXPR'es. */
2777 4506 : if (is == 0)
2778 : {
2779 3410 : expr_in_all_succ_branches = av_set_copy (succ_set);
2780 :
2781 : /* Remember the first successor for later. */
2782 3410 : zero_succ = succ;
2783 : }
2784 : else
2785 : {
2786 1096 : av_set_iterator i;
2787 1096 : expr_t expr;
2788 :
2789 8620 : FOR_EACH_EXPR_1 (expr, i, &expr_in_all_succ_branches)
2790 3762 : if (!av_set_is_in_p (succ_set, EXPR_VINSN (expr)))
2791 1845 : av_set_iter_remove (&i);
2792 : }
2793 : }
2794 :
2795 : /* Union the av_sets. Check liveness restrictions on target registers
2796 : in special case of two successors. */
2797 6291 : if (sinfo->succs_ok_n == 2 && is == 1)
2798 : {
2799 1096 : basic_block bb0 = BLOCK_FOR_INSN (zero_succ);
2800 1096 : basic_block bb1 = BLOCK_FOR_INSN (succ);
2801 :
2802 1096 : gcc_assert (BB_LV_SET_VALID_P (bb0) && BB_LV_SET_VALID_P (bb1));
2803 1096 : av_set_union_and_live (&av1, &succ_set,
2804 1096 : BB_LV_SET (bb0),
2805 1096 : BB_LV_SET (bb1),
2806 : insn);
2807 1096 : }
2808 : else
2809 5195 : av_set_union_and_clear (&av1, &succ_set, insn);
2810 : }
2811 :
2812 : /* Check liveness restrictions via hard way when there are more than
2813 : two successors. */
2814 8544 : if (sinfo->succs_ok_n > 2)
2815 0 : FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ)
2816 : {
2817 0 : basic_block succ_bb = BLOCK_FOR_INSN (succ);
2818 0 : av_set_t av_succ = (is_ineligible_successor (succ, p)
2819 0 : ? NULL
2820 0 : : BB_AV_SET (succ_bb));
2821 :
2822 0 : gcc_assert (BB_LV_SET_VALID_P (succ_bb));
2823 0 : mark_unavailable_targets (av1, av_succ, BB_LV_SET (succ_bb));
2824 : }
2825 :
2826 : /* Finally, check liveness restrictions on paths leaving the region. */
2827 8544 : if (sinfo->all_succs_n > sinfo->succs_ok_n)
2828 12916 : FOR_EACH_VEC_ELT (sinfo->succs_other, is, succ)
2829 7290 : mark_unavailable_targets
2830 7290 : (av1, NULL, BB_LV_SET (BLOCK_FOR_INSN (succ)));
2831 :
2832 8544 : if (sinfo->all_succs_n > 1)
2833 : {
2834 : av_set_iterator i;
2835 : expr_t expr;
2836 :
2837 : /* Increase the spec attribute of all EXPR'es that didn't come
2838 : from all successors. */
2839 17729 : FOR_EACH_EXPR (expr, i, av1)
2840 12837 : if (!av_set_is_in_p (expr_in_all_succ_branches, EXPR_VINSN (expr)))
2841 3318 : EXPR_SPEC (expr)++;
2842 :
2843 4892 : av_set_clear (&expr_in_all_succ_branches);
2844 :
2845 : /* Do not move conditional branches through other
2846 : conditional branches. So, remove all conditional
2847 : branches from av_set if current operator is a conditional
2848 : branch. */
2849 4892 : av_set_substract_cond_branches (&av1);
2850 : }
2851 :
2852 8544 : ilist_remove (&p);
2853 8544 : free_succs_info (sinfo);
2854 :
2855 8544 : if (sched_verbose >= 6)
2856 : {
2857 0 : sel_print ("av_succs (%d): ", INSN_UID (insn));
2858 0 : dump_av_set (av1);
2859 0 : sel_print ("\n");
2860 : }
2861 :
2862 8544 : return av1;
2863 : }
2864 :
2865 : /* This function computes av_set for the FIRST_INSN by dragging valid
2866 : av_set through all basic block insns either from the end of basic block
2867 : (computed using compute_av_set_at_bb_end) or from the insn on which
2868 : MAX_WS was exceeded. It uses compute_av_set_at_bb_end to compute av_set
2869 : below the basic block and handling conditional branches.
2870 : FIRST_INSN - the basic block head, P - path consisting of the insns
2871 : traversed on the way to the FIRST_INSN (the path is sparse, only bb heads
2872 : and bb ends are added to the path), WS - current window size,
2873 : NEED_COPY_P - true if we'll make a copy of av_set before returning it. */
2874 : static av_set_t
2875 12705 : compute_av_set_inside_bb (insn_t first_insn, ilist_t p, int ws,
2876 : bool need_copy_p)
2877 : {
2878 12705 : insn_t cur_insn;
2879 12705 : int end_ws = ws;
2880 12705 : insn_t bb_end = sel_bb_end (BLOCK_FOR_INSN (first_insn));
2881 12705 : insn_t after_bb_end = NEXT_INSN (bb_end);
2882 12705 : insn_t last_insn;
2883 12705 : av_set_t av = NULL;
2884 12705 : basic_block cur_bb = BLOCK_FOR_INSN (first_insn);
2885 :
2886 : /* Return NULL if insn is not on the legitimate downward path. */
2887 12705 : if (is_ineligible_successor (first_insn, p))
2888 : {
2889 488 : if (sched_verbose >= 6)
2890 0 : sel_print ("Insn %d is ineligible_successor\n", INSN_UID (first_insn));
2891 :
2892 488 : return NULL;
2893 : }
2894 :
2895 : /* If insn already has valid av(insn) computed, just return it. */
2896 12217 : if (AV_SET_VALID_P (first_insn))
2897 : {
2898 2796 : av_set_t av_set;
2899 :
2900 2796 : if (sel_bb_head_p (first_insn))
2901 2763 : av_set = BB_AV_SET (BLOCK_FOR_INSN (first_insn));
2902 : else
2903 : av_set = NULL;
2904 :
2905 2796 : if (sched_verbose >= 6)
2906 : {
2907 0 : sel_print ("Insn %d has a valid av set: ", INSN_UID (first_insn));
2908 0 : dump_av_set (av_set);
2909 0 : sel_print ("\n");
2910 : }
2911 :
2912 2796 : return need_copy_p ? av_set_copy (av_set) : av_set;
2913 : }
2914 :
2915 9421 : ilist_add (&p, first_insn);
2916 :
2917 : /* As the result after this loop have completed, in LAST_INSN we'll
2918 : have the insn which has valid av_set to start backward computation
2919 : from: it either will be NULL because on it the window size was exceeded
2920 : or other valid av_set as returned by compute_av_set for the last insn
2921 : of the basic block. */
2922 63646 : for (last_insn = first_insn; last_insn != after_bb_end;
2923 54225 : last_insn = NEXT_INSN (last_insn))
2924 : {
2925 : /* We may encounter valid av_set not only on bb_head, but also on
2926 : those insns on which previously MAX_WS was exceeded. */
2927 55102 : if (AV_SET_VALID_P (last_insn))
2928 : {
2929 247 : if (sched_verbose >= 6)
2930 0 : sel_print ("Insn %d has a valid empty av set\n", INSN_UID (last_insn));
2931 : break;
2932 : }
2933 :
2934 : /* The special case: the last insn of the BB may be an
2935 : ineligible_successor due to its SEQ_NO that was set on
2936 : it as a bookkeeping. */
2937 54855 : if (last_insn != first_insn
2938 54855 : && is_ineligible_successor (last_insn, p))
2939 : {
2940 459 : if (sched_verbose >= 6)
2941 0 : sel_print ("Insn %d is ineligible_successor\n", INSN_UID (last_insn));
2942 : break;
2943 : }
2944 :
2945 54396 : if (DEBUG_INSN_P (last_insn))
2946 132 : continue;
2947 :
2948 54264 : if (end_ws > max_ws)
2949 : {
2950 : /* We can reach max lookahead size at bb_header, so clean av_set
2951 : first. */
2952 171 : INSN_WS_LEVEL (last_insn) = global_level;
2953 :
2954 171 : if (sched_verbose >= 6)
2955 0 : sel_print ("Insn %d is beyond the software lookahead window size\n",
2956 0 : INSN_UID (last_insn));
2957 : break;
2958 : }
2959 :
2960 54093 : end_ws++;
2961 : }
2962 :
2963 : /* Get the valid av_set into AV above the LAST_INSN to start backward
2964 : computation from. It either will be empty av_set or av_set computed from
2965 : the successors on the last insn of the current bb. */
2966 9421 : if (last_insn != after_bb_end)
2967 : {
2968 877 : av = NULL;
2969 :
2970 : /* This is needed only to obtain av_sets that are identical to
2971 : those computed by the old compute_av_set version. */
2972 877 : if (last_insn == first_insn && !INSN_NOP_P (last_insn))
2973 17 : av_set_add (&av, INSN_EXPR (last_insn));
2974 : }
2975 : else
2976 : /* END_WS is always already increased by 1 if LAST_INSN == AFTER_BB_END. */
2977 8544 : av = compute_av_set_at_bb_end (bb_end, p, end_ws);
2978 :
2979 : /* Compute av_set in AV starting from below the LAST_INSN up to
2980 : location above the FIRST_INSN. */
2981 63646 : for (cur_insn = PREV_INSN (last_insn); cur_insn != PREV_INSN (first_insn);
2982 54225 : cur_insn = PREV_INSN (cur_insn))
2983 54225 : if (!INSN_NOP_P (cur_insn))
2984 : {
2985 54166 : expr_t expr;
2986 :
2987 54166 : moveup_set_expr (&av, cur_insn, false);
2988 :
2989 : /* If the expression for CUR_INSN is already in the set,
2990 : replace it by the new one. */
2991 54166 : expr = av_set_lookup (av, INSN_VINSN (cur_insn));
2992 54166 : if (expr != NULL)
2993 : {
2994 638 : clear_expr (expr);
2995 638 : copy_expr (expr, INSN_EXPR (cur_insn));
2996 : }
2997 : else
2998 53528 : av_set_add (&av, INSN_EXPR (cur_insn));
2999 : }
3000 :
3001 : /* Clear stale bb_av_set. */
3002 9421 : if (sel_bb_head_p (first_insn))
3003 : {
3004 5316 : av_set_clear (&BB_AV_SET (cur_bb));
3005 5316 : BB_AV_SET (cur_bb) = need_copy_p ? av_set_copy (av) : av;
3006 5316 : BB_AV_LEVEL (cur_bb) = global_level;
3007 : }
3008 :
3009 9421 : if (sched_verbose >= 6)
3010 : {
3011 0 : sel_print ("Computed av set for insn %d: ", INSN_UID (first_insn));
3012 0 : dump_av_set (av);
3013 0 : sel_print ("\n");
3014 : }
3015 :
3016 9421 : ilist_remove (&p);
3017 9421 : return av;
3018 : }
3019 :
3020 : /* Compute av set before INSN.
3021 : INSN - the current operation (actual rtx INSN)
3022 : P - the current path, which is list of insns visited so far
3023 : WS - software lookahead window size.
3024 : UNIQUE_P - TRUE, if returned av_set will be changed, hence
3025 : if we want to save computed av_set in s_i_d, we should make a copy of it.
3026 :
3027 : In the resulting set we will have only expressions that don't have delay
3028 : stalls and nonsubstitutable dependences. */
3029 : static av_set_t
3030 6414 : compute_av_set (insn_t insn, ilist_t p, int ws, bool unique_p)
3031 : {
3032 0 : return compute_av_set_inside_bb (insn, p, ws, unique_p);
3033 : }
3034 :
3035 : /* Propagate a liveness set LV through INSN. */
3036 : static void
3037 17121 : propagate_lv_set (regset lv, insn_t insn)
3038 : {
3039 17121 : gcc_assert (INSN_P (insn));
3040 :
3041 17121 : if (INSN_NOP_P (insn))
3042 : return;
3043 :
3044 17060 : df_simulate_one_insn_backwards (BLOCK_FOR_INSN (insn), insn, lv);
3045 : }
3046 :
3047 : /* Return livness set at the end of BB. */
3048 : static regset
3049 1741 : compute_live_after_bb (basic_block bb)
3050 : {
3051 1741 : edge e;
3052 1741 : edge_iterator ei;
3053 1741 : regset lv = get_clear_regset_from_pool ();
3054 :
3055 1741 : gcc_assert (!ignore_first);
3056 :
3057 4266 : FOR_EACH_EDGE (e, ei, bb->succs)
3058 2525 : if (sel_bb_empty_p (e->dest))
3059 : {
3060 15 : if (! BB_LV_SET_VALID_P (e->dest))
3061 : {
3062 0 : gcc_unreachable ();
3063 : gcc_assert (BB_LV_SET (e->dest) == NULL);
3064 : BB_LV_SET (e->dest) = compute_live_after_bb (e->dest);
3065 : BB_LV_SET_VALID_P (e->dest) = true;
3066 : }
3067 15 : IOR_REG_SET (lv, BB_LV_SET (e->dest));
3068 : }
3069 : else
3070 2510 : IOR_REG_SET (lv, compute_live (sel_bb_head (e->dest)));
3071 :
3072 1741 : return lv;
3073 : }
3074 :
3075 : /* Compute the set of all live registers at the point before INSN and save
3076 : it at INSN if INSN is bb header. */
3077 : regset
3078 17694 : compute_live (insn_t insn)
3079 : {
3080 17694 : basic_block bb = BLOCK_FOR_INSN (insn);
3081 17694 : insn_t final, temp;
3082 17694 : regset lv;
3083 :
3084 : /* Return the valid set if we're already on it. */
3085 17694 : if (!ignore_first)
3086 : {
3087 9339 : regset src = NULL;
3088 :
3089 9339 : if (sel_bb_head_p (insn) && BB_LV_SET_VALID_P (bb))
3090 5719 : src = BB_LV_SET (bb);
3091 : else
3092 : {
3093 3620 : gcc_assert (in_current_region_p (bb));
3094 3620 : if (INSN_LIVE_VALID_P (insn))
3095 3503 : src = INSN_LIVE (insn);
3096 : }
3097 :
3098 9222 : if (src)
3099 : {
3100 9222 : lv = get_regset_from_pool ();
3101 9222 : COPY_REG_SET (lv, src);
3102 :
3103 9222 : if (sel_bb_head_p (insn) && ! BB_LV_SET_VALID_P (bb))
3104 : {
3105 0 : COPY_REG_SET (BB_LV_SET (bb), lv);
3106 0 : BB_LV_SET_VALID_P (bb) = true;
3107 : }
3108 :
3109 9222 : return_regset_to_pool (lv);
3110 9222 : return lv;
3111 : }
3112 : }
3113 :
3114 : /* We've skipped the wrong lv_set. Don't skip the right one. */
3115 8472 : ignore_first = false;
3116 8472 : gcc_assert (in_current_region_p (bb));
3117 :
3118 : /* Find a valid LV set in this block or below, if needed.
3119 : Start searching from the next insn: either ignore_first is true, or
3120 : INSN doesn't have a correct live set. */
3121 8472 : temp = NEXT_INSN (insn);
3122 8472 : final = NEXT_INSN (BB_END (bb));
3123 18862 : while (temp != final && ! INSN_LIVE_VALID_P (temp))
3124 1918 : temp = NEXT_INSN (temp);
3125 8472 : if (temp == final)
3126 : {
3127 1741 : lv = compute_live_after_bb (bb);
3128 1741 : temp = PREV_INSN (temp);
3129 : }
3130 : else
3131 : {
3132 6731 : lv = get_regset_from_pool ();
3133 6731 : COPY_REG_SET (lv, INSN_LIVE (temp));
3134 : }
3135 :
3136 : /* Put correct lv sets on the insns which have bad sets. */
3137 8472 : final = PREV_INSN (insn);
3138 34065 : while (temp != final)
3139 : {
3140 17121 : propagate_lv_set (lv, temp);
3141 17121 : COPY_REG_SET (INSN_LIVE (temp), lv);
3142 17121 : INSN_LIVE_VALID_P (temp) = true;
3143 17121 : temp = PREV_INSN (temp);
3144 : }
3145 :
3146 : /* Also put it in a BB. */
3147 8472 : if (sel_bb_head_p (insn))
3148 : {
3149 2082 : basic_block bb = BLOCK_FOR_INSN (insn);
3150 :
3151 2082 : COPY_REG_SET (BB_LV_SET (bb), lv);
3152 2082 : BB_LV_SET_VALID_P (bb) = true;
3153 : }
3154 :
3155 : /* We return LV to the pool, but will not clear it there. Thus we can
3156 : legimatelly use LV till the next use of regset_pool_get (). */
3157 8472 : return_regset_to_pool (lv);
3158 8472 : return lv;
3159 : }
3160 :
3161 : /* Update liveness sets for INSN. */
3162 : static inline void
3163 8355 : update_liveness_on_insn (rtx_insn *insn)
3164 : {
3165 8355 : ignore_first = true;
3166 8355 : compute_live (insn);
3167 24 : }
3168 :
3169 : /* Compute liveness below INSN and write it into REGS. */
3170 : static inline void
3171 3894 : compute_live_below_insn (rtx_insn *insn, regset regs)
3172 : {
3173 3894 : rtx_insn *succ;
3174 3894 : succ_iterator si;
3175 :
3176 7788 : FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_ALL)
3177 3894 : IOR_REG_SET (regs, compute_live (succ));
3178 3894 : }
3179 :
3180 : /* Update the data gathered in av and lv sets starting from INSN. */
3181 : static void
3182 1225 : update_data_sets (rtx_insn *insn)
3183 : {
3184 1225 : update_liveness_on_insn (insn);
3185 1225 : if (sel_bb_head_p (insn))
3186 : {
3187 1225 : gcc_assert (AV_LEVEL (insn) != 0);
3188 1225 : BB_AV_LEVEL (BLOCK_FOR_INSN (insn)) = -1;
3189 1225 : compute_av_set (insn, NULL, 0, 0);
3190 : }
3191 1225 : }
3192 :
3193 :
3194 : /* Helper for move_op () and find_used_regs ().
3195 : Return speculation type for which a check should be created on the place
3196 : of INSN. EXPR is one of the original ops we are searching for. */
3197 : static ds_t
3198 5155 : get_spec_check_type_for_insn (insn_t insn, expr_t expr)
3199 : {
3200 5155 : ds_t to_check_ds;
3201 5155 : ds_t already_checked_ds = EXPR_SPEC_DONE_DS (INSN_EXPR (insn));
3202 :
3203 5155 : to_check_ds = EXPR_SPEC_TO_CHECK_DS (expr);
3204 :
3205 5155 : if (targetm.sched.get_insn_checked_ds)
3206 0 : already_checked_ds |= targetm.sched.get_insn_checked_ds (insn);
3207 :
3208 5155 : if (spec_info != NULL
3209 0 : && (spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL))
3210 0 : already_checked_ds |= BEGIN_CONTROL;
3211 :
3212 5155 : already_checked_ds = ds_get_speculation_types (already_checked_ds);
3213 :
3214 5155 : to_check_ds &= ~already_checked_ds;
3215 :
3216 5155 : return to_check_ds;
3217 : }
3218 :
3219 : /* Find the set of registers that are unavailable for storing expres
3220 : while moving ORIG_OPS up on the path starting from INSN due to
3221 : liveness (USED_REGS) or hardware restrictions (REG_RENAME_P).
3222 :
3223 : All the original operations found during the traversal are saved in the
3224 : ORIGINAL_INSNS list.
3225 :
3226 : REG_RENAME_P denotes the set of hardware registers that
3227 : cannot be used with renaming due to the register class restrictions,
3228 : mode restrictions and other (the register we'll choose should be
3229 : compatible class with the original uses, shouldn't be in call_used_regs,
3230 : should be HARD_REGNO_RENAME_OK etc).
3231 :
3232 : Returns TRUE if we've found all original insns, FALSE otherwise.
3233 :
3234 : This function utilizes code_motion_path_driver (formerly find_used_regs_1)
3235 : to traverse the code motion paths. This helper function finds registers
3236 : that are not available for storing expres while moving ORIG_OPS up on the
3237 : path starting from INSN. A register considered as used on the moving path,
3238 : if one of the following conditions is not satisfied:
3239 :
3240 : (1) a register not set or read on any path from xi to an instance of
3241 : the original operation,
3242 : (2) not among the live registers of the point immediately following the
3243 : first original operation on a given downward path, except for the
3244 : original target register of the operation,
3245 : (3) not live on the other path of any conditional branch that is passed
3246 : by the operation, in case original operations are not present on
3247 : both paths of the conditional branch.
3248 :
3249 : All the original operations found during the traversal are saved in the
3250 : ORIGINAL_INSNS list.
3251 :
3252 : REG_RENAME_P->CROSSED_CALL_ABIS is true, if there is a call insn on the path
3253 : from INSN to original insn. In this case CALL_USED_REG_SET will be added
3254 : to unavailable hard regs at the point original operation is found. */
3255 :
3256 : static bool
3257 2790 : find_used_regs (insn_t insn, av_set_t orig_ops, regset used_regs,
3258 : struct reg_rename *reg_rename_p, def_list_t *original_insns)
3259 : {
3260 2790 : def_list_iterator i;
3261 2790 : def_t def;
3262 2790 : int res;
3263 2790 : bool needs_spec_check_p = false;
3264 2790 : expr_t expr;
3265 2790 : av_set_iterator expr_iter;
3266 2790 : struct fur_static_params sparams;
3267 2790 : struct cmpd_local_params lparams;
3268 :
3269 : /* We haven't visited any blocks yet. */
3270 2790 : bitmap_clear (code_motion_visited_blocks);
3271 :
3272 : /* Init parameters for code_motion_path_driver. */
3273 2790 : sparams.crossed_call_abis = 0;
3274 2790 : sparams.original_insns = original_insns;
3275 2790 : sparams.used_regs = used_regs;
3276 :
3277 : /* Set the appropriate hooks and data. */
3278 2790 : code_motion_path_driver_info = &fur_hooks;
3279 :
3280 2790 : res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
3281 :
3282 2790 : reg_rename_p->crossed_call_abis |= sparams.crossed_call_abis;
3283 :
3284 2790 : gcc_assert (res == 1);
3285 2790 : gcc_assert (original_insns && *original_insns);
3286 :
3287 : /* ??? We calculate whether an expression needs a check when computing
3288 : av sets. This information is not as precise as it could be due to
3289 : merging this bit in merge_expr. We can do better in find_used_regs,
3290 : but we want to avoid multiple traversals of the same code motion
3291 : paths. */
3292 9474 : FOR_EACH_EXPR (expr, expr_iter, orig_ops)
3293 2790 : needs_spec_check_p |= EXPR_NEEDS_SPEC_CHECK_P (expr);
3294 :
3295 : /* Mark hardware regs in REG_RENAME_P that are not suitable
3296 : for renaming expr in INSN due to hardware restrictions (register class,
3297 : modes compatibility etc). */
3298 6684 : FOR_EACH_DEF (def, i, *original_insns)
3299 : {
3300 3894 : vinsn_t vinsn = INSN_VINSN (def->orig_insn);
3301 :
3302 3894 : if (VINSN_SEPARABLE_P (vinsn))
3303 3147 : mark_unavailable_hard_regs (def, reg_rename_p, used_regs);
3304 :
3305 : /* Do not allow clobbering of ld.[sa] address in case some of the
3306 : original operations need a check. */
3307 3894 : if (needs_spec_check_p)
3308 0 : IOR_REG_SET (used_regs, VINSN_REG_USES (vinsn));
3309 : }
3310 :
3311 2790 : return true;
3312 : }
3313 :
3314 :
3315 : /* Functions to choose the best insn from available ones. */
3316 :
3317 : /* Adjusts the priority for EXPR using the backend *_adjust_priority hook. */
3318 : static int
3319 17883 : sel_target_adjust_priority (expr_t expr)
3320 : {
3321 17883 : int priority = EXPR_PRIORITY (expr);
3322 17883 : int new_priority;
3323 :
3324 17883 : if (targetm.sched.adjust_priority)
3325 17883 : new_priority = targetm.sched.adjust_priority (EXPR_INSN_RTX (expr), priority);
3326 : else
3327 : new_priority = priority;
3328 :
3329 : /* If the priority has changed, adjust EXPR_PRIORITY_ADJ accordingly. */
3330 17883 : EXPR_PRIORITY_ADJ (expr) = new_priority - EXPR_PRIORITY (expr);
3331 :
3332 17883 : if (sched_verbose >= 4)
3333 0 : sel_print ("sel_target_adjust_priority: insn %d, %d+%d = %d.\n",
3334 0 : INSN_UID (EXPR_INSN_RTX (expr)), EXPR_PRIORITY (expr),
3335 : EXPR_PRIORITY_ADJ (expr), new_priority);
3336 :
3337 17883 : return new_priority;
3338 : }
3339 :
3340 : /* Rank two available exprs for schedule. Never return 0 here. */
3341 : static int
3342 121309 : sel_rank_for_schedule (const void *x, const void *y)
3343 : {
3344 121309 : expr_t tmp = *(const expr_t *) y;
3345 121309 : expr_t tmp2 = *(const expr_t *) x;
3346 121309 : insn_t tmp_insn, tmp2_insn;
3347 121309 : vinsn_t tmp_vinsn, tmp2_vinsn;
3348 121309 : int val;
3349 :
3350 121309 : tmp_vinsn = EXPR_VINSN (tmp);
3351 121309 : tmp2_vinsn = EXPR_VINSN (tmp2);
3352 121309 : tmp_insn = EXPR_INSN_RTX (tmp);
3353 121309 : tmp2_insn = EXPR_INSN_RTX (tmp2);
3354 :
3355 : /* Schedule debug insns as early as possible. */
3356 121309 : if (DEBUG_INSN_P (tmp_insn) && !DEBUG_INSN_P (tmp2_insn))
3357 : return -1;
3358 121309 : else if (DEBUG_INSN_P (tmp2_insn))
3359 : return 1;
3360 :
3361 : /* Prefer SCHED_GROUP_P insns to any others. */
3362 121309 : if (SCHED_GROUP_P (tmp_insn) != SCHED_GROUP_P (tmp2_insn))
3363 : {
3364 1551 : if (VINSN_UNIQUE_P (tmp_vinsn) && VINSN_UNIQUE_P (tmp2_vinsn))
3365 0 : return SCHED_GROUP_P (tmp2_insn) ? 1 : -1;
3366 :
3367 : /* Now uniqueness means SCHED_GROUP_P is set, because schedule groups
3368 : cannot be cloned. */
3369 1551 : if (VINSN_UNIQUE_P (tmp2_vinsn))
3370 : return 1;
3371 : return -1;
3372 : }
3373 :
3374 : /* Discourage scheduling of speculative checks. */
3375 119758 : val = (sel_insn_is_speculation_check (tmp_insn)
3376 119758 : - sel_insn_is_speculation_check (tmp2_insn));
3377 119758 : if (val)
3378 : return val;
3379 :
3380 : /* Prefer not scheduled insn over scheduled one. */
3381 119758 : if (EXPR_SCHED_TIMES (tmp) > 0 || EXPR_SCHED_TIMES (tmp2) > 0)
3382 : {
3383 13315 : val = EXPR_SCHED_TIMES (tmp) - EXPR_SCHED_TIMES (tmp2);
3384 13315 : if (val)
3385 : return val;
3386 : }
3387 :
3388 : /* Prefer jump over non-jump instruction. */
3389 112346 : if (control_flow_insn_p (tmp_insn) && !control_flow_insn_p (tmp2_insn))
3390 : return -1;
3391 111863 : else if (control_flow_insn_p (tmp2_insn) && !control_flow_insn_p (tmp_insn))
3392 : return 1;
3393 :
3394 : /* Prefer an expr with non-zero usefulness. */
3395 111595 : int u1 = EXPR_USEFULNESS (tmp), u2 = EXPR_USEFULNESS (tmp2);
3396 :
3397 111595 : if (u1 == 0)
3398 : {
3399 226 : if (u2 == 0)
3400 : u1 = u2 = 1;
3401 : else
3402 : return 1;
3403 : }
3404 111369 : else if (u2 == 0)
3405 : return -1;
3406 :
3407 : /* Prefer an expr with greater priority. */
3408 111302 : val = (u2 * (EXPR_PRIORITY (tmp2) + EXPR_PRIORITY_ADJ (tmp2))
3409 111302 : - u1 * (EXPR_PRIORITY (tmp) + EXPR_PRIORITY_ADJ (tmp)));
3410 111302 : if (val)
3411 : return val;
3412 :
3413 38188 : if (spec_info != NULL && spec_info->mask != 0)
3414 : /* This code was taken from haifa-sched.cc: rank_for_schedule (). */
3415 : {
3416 0 : ds_t ds1, ds2;
3417 0 : dw_t dw1, dw2;
3418 0 : int dw;
3419 :
3420 0 : ds1 = EXPR_SPEC_DONE_DS (tmp);
3421 0 : if (ds1)
3422 0 : dw1 = ds_weak (ds1);
3423 : else
3424 : dw1 = NO_DEP_WEAK;
3425 :
3426 0 : ds2 = EXPR_SPEC_DONE_DS (tmp2);
3427 0 : if (ds2)
3428 0 : dw2 = ds_weak (ds2);
3429 : else
3430 : dw2 = NO_DEP_WEAK;
3431 :
3432 0 : dw = dw2 - dw1;
3433 0 : if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
3434 : return dw;
3435 : }
3436 :
3437 : /* Prefer an old insn to a bookkeeping insn. */
3438 38188 : if (INSN_UID (tmp_insn) < first_emitted_uid
3439 38188 : && INSN_UID (tmp2_insn) >= first_emitted_uid)
3440 : return -1;
3441 37754 : if (INSN_UID (tmp_insn) >= first_emitted_uid
3442 37754 : && INSN_UID (tmp2_insn) < first_emitted_uid)
3443 : return 1;
3444 :
3445 : /* Prefer an insn with smaller UID, as a last resort.
3446 : We can't safely use INSN_LUID as it is defined only for those insns
3447 : that are in the stream. */
3448 37466 : return INSN_UID (tmp_insn) - INSN_UID (tmp2_insn);
3449 : }
3450 :
3451 : /* Filter out expressions from av set pointed to by AV_PTR
3452 : that are pipelined too many times. */
3453 : static void
3454 7728 : process_pipelined_exprs (av_set_t *av_ptr)
3455 : {
3456 7728 : expr_t expr;
3457 7728 : av_set_iterator si;
3458 :
3459 : /* Don't pipeline already pipelined code as that would increase
3460 : number of unnecessary register moves. */
3461 44198 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3462 : {
3463 18235 : if (EXPR_SCHED_TIMES (expr)
3464 18235 : >= param_selsched_max_sched_times)
3465 114 : av_set_iter_remove (&si);
3466 : }
3467 7728 : }
3468 :
3469 : /* Filter speculative insns from AV_PTR if we don't want them. */
3470 : static void
3471 7728 : process_spec_exprs (av_set_t *av_ptr)
3472 : {
3473 7728 : expr_t expr;
3474 7728 : av_set_iterator si;
3475 :
3476 7728 : if (spec_info == NULL)
3477 7728 : return;
3478 :
3479 : /* Scan *AV_PTR to find out if we want to consider speculative
3480 : instructions for scheduling. */
3481 0 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3482 : {
3483 0 : ds_t ds;
3484 :
3485 0 : ds = EXPR_SPEC_DONE_DS (expr);
3486 :
3487 : /* The probability of a success is too low - don't speculate. */
3488 0 : if ((ds & SPECULATIVE)
3489 0 : && (ds_weak (ds) < spec_info->data_weakness_cutoff
3490 0 : || EXPR_USEFULNESS (expr) < spec_info->control_weakness_cutoff
3491 0 : || (pipelining_p && false
3492 : && (ds & DATA_SPEC)
3493 : && (ds & CONTROL_SPEC))))
3494 : {
3495 0 : av_set_iter_remove (&si);
3496 0 : continue;
3497 : }
3498 : }
3499 : }
3500 :
3501 : /* Search for any use-like insns in AV_PTR and decide on scheduling
3502 : them. Return one when found, and NULL otherwise.
3503 : Note that we check here whether a USE could be scheduled to avoid
3504 : an infinite loop later. */
3505 : static expr_t
3506 7728 : process_use_exprs (av_set_t *av_ptr)
3507 : {
3508 7728 : expr_t expr;
3509 7728 : av_set_iterator si;
3510 7728 : bool uses_present_p = false;
3511 7728 : bool try_uses_p = true;
3512 :
3513 43602 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3514 : {
3515 : /* This will also initialize INSN_CODE for later use. */
3516 18046 : if (recog_memoized (EXPR_INSN_RTX (expr)) < 0)
3517 : {
3518 : /* If we have a USE in *AV_PTR that was not scheduled yet,
3519 : do so because it will do good only. */
3520 130 : if (EXPR_SCHED_TIMES (expr) <= 0)
3521 : {
3522 127 : if (EXPR_TARGET_AVAILABLE (expr) == 1)
3523 : return expr;
3524 :
3525 18 : av_set_iter_remove (&si);
3526 : }
3527 : else
3528 : {
3529 3 : gcc_assert (pipelining_p);
3530 :
3531 : uses_present_p = true;
3532 : }
3533 : }
3534 : else
3535 : try_uses_p = false;
3536 : }
3537 :
3538 7619 : if (uses_present_p)
3539 : {
3540 : /* If we don't want to schedule any USEs right now and we have some
3541 : in *AV_PTR, remove them, else just return the first one found. */
3542 3 : if (!try_uses_p)
3543 : {
3544 29 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3545 13 : if (INSN_CODE (EXPR_INSN_RTX (expr)) < 0)
3546 3 : av_set_iter_remove (&si);
3547 : }
3548 : else
3549 : {
3550 0 : FOR_EACH_EXPR_1 (expr, si, av_ptr)
3551 : {
3552 0 : gcc_assert (INSN_CODE (EXPR_INSN_RTX (expr)) < 0);
3553 :
3554 0 : if (EXPR_TARGET_AVAILABLE (expr) == 1)
3555 : return expr;
3556 :
3557 0 : av_set_iter_remove (&si);
3558 : }
3559 : }
3560 : }
3561 :
3562 : return NULL;
3563 : }
3564 :
3565 : /* Lookup EXPR in VINSN_VEC and return TRUE if found. Also check patterns from
3566 : EXPR's history of changes. */
3567 : static bool
3568 35766 : vinsn_vec_has_expr_p (vinsn_vec_t vinsn_vec, expr_t expr)
3569 : {
3570 35766 : vinsn_t vinsn, expr_vinsn;
3571 35766 : int n;
3572 35766 : unsigned i;
3573 :
3574 : /* Start with checking expr itself and then proceed with all the old forms
3575 : of expr taken from its history vector. */
3576 35766 : for (i = 0, expr_vinsn = EXPR_VINSN (expr);
3577 36444 : expr_vinsn;
3578 72708 : expr_vinsn = (i < EXPR_HISTORY_OF_CHANGES (expr).length ()
3579 36354 : ? EXPR_HISTORY_OF_CHANGES (expr)[i++].old_expr_vinsn
3580 : : NULL))
3581 41842 : FOR_EACH_VEC_ELT (vinsn_vec, n, vinsn)
3582 2406 : if (VINSN_SEPARABLE_P (vinsn))
3583 : {
3584 2291 : if (vinsn_equal_p (vinsn, expr_vinsn))
3585 : return true;
3586 : }
3587 : else
3588 : {
3589 : /* For non-separable instructions, the blocking insn can have
3590 : another pattern due to substitution, and we can't choose
3591 : different register as in the above case. Check all registers
3592 : being written instead. */
3593 115 : if (bitmap_intersect_p (VINSN_REG_SETS (vinsn),
3594 115 : VINSN_REG_SETS (expr_vinsn)))
3595 : return true;
3596 : }
3597 :
3598 : return false;
3599 : }
3600 :
3601 : /* Return true if either of expressions from ORIG_OPS can be blocked
3602 : by previously created bookkeeping code. STATIC_PARAMS points to static
3603 : parameters of move_op. */
3604 : static bool
3605 0 : av_set_could_be_blocked_by_bookkeeping_p (av_set_t orig_ops, void *static_params)
3606 : {
3607 0 : expr_t expr;
3608 0 : av_set_iterator iter;
3609 0 : moveop_static_params_p sparams;
3610 :
3611 : /* This checks that expressions in ORIG_OPS are not blocked by bookkeeping
3612 : created while scheduling on another fence. */
3613 0 : FOR_EACH_EXPR (expr, iter, orig_ops)
3614 0 : if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr))
3615 : return true;
3616 :
3617 0 : gcc_assert (code_motion_path_driver_info == &move_op_hooks);
3618 0 : sparams = (moveop_static_params_p) static_params;
3619 :
3620 : /* Expressions can be also blocked by bookkeeping created during current
3621 : move_op. */
3622 0 : if (bitmap_bit_p (current_copies, INSN_UID (sparams->failed_insn)))
3623 0 : FOR_EACH_EXPR (expr, iter, orig_ops)
3624 0 : if (moveup_expr_cached (expr, sparams->failed_insn, false) != MOVEUP_EXPR_NULL)
3625 : return true;
3626 :
3627 : /* Expressions in ORIG_OPS may have wrong destination register due to
3628 : renaming. Check with the right register instead. */
3629 0 : if (sparams->dest && REG_P (sparams->dest))
3630 : {
3631 0 : rtx reg = sparams->dest;
3632 0 : vinsn_t failed_vinsn = INSN_VINSN (sparams->failed_insn);
3633 :
3634 0 : if (register_unavailable_p (VINSN_REG_SETS (failed_vinsn), reg)
3635 0 : || register_unavailable_p (VINSN_REG_USES (failed_vinsn), reg)
3636 0 : || register_unavailable_p (VINSN_REG_CLOBBERS (failed_vinsn), reg))
3637 0 : return true;
3638 : }
3639 :
3640 : return false;
3641 : }
3642 :
3643 : /* Clear VINSN_VEC and detach vinsns. */
3644 : static void
3645 3472 : vinsn_vec_clear (vinsn_vec_t *vinsn_vec)
3646 : {
3647 3472 : unsigned len = vinsn_vec->length ();
3648 223 : if (len > 0)
3649 : {
3650 : vinsn_t vinsn;
3651 : int n;
3652 :
3653 75 : FOR_EACH_VEC_ELT (*vinsn_vec, n, vinsn)
3654 50 : vinsn_detach (vinsn);
3655 25 : vinsn_vec->block_remove (0, len);
3656 : }
3657 3472 : }
3658 :
3659 : /* Add the vinsn of EXPR to the VINSN_VEC. */
3660 : static void
3661 50 : vinsn_vec_add (vinsn_vec_t *vinsn_vec, expr_t expr)
3662 : {
3663 50 : vinsn_attach (EXPR_VINSN (expr));
3664 50 : vinsn_vec->safe_push (EXPR_VINSN (expr));
3665 50 : }
3666 :
3667 : /* Free the vector representing blocked expressions. */
3668 : static void
3669 1540 : vinsn_vec_free (vinsn_vec_t &vinsn_vec)
3670 : {
3671 0 : vinsn_vec.release ();
3672 0 : }
3673 :
3674 : /* Increase EXPR_PRIORITY_ADJ for INSN by AMOUNT. */
3675 :
3676 0 : void sel_add_to_insn_priority (rtx insn, int amount)
3677 : {
3678 0 : EXPR_PRIORITY_ADJ (INSN_EXPR (insn)) += amount;
3679 :
3680 0 : if (sched_verbose >= 2)
3681 0 : sel_print ("sel_add_to_insn_priority: insn %d, by %d (now %d+%d).\n",
3682 0 : INSN_UID (insn), amount, EXPR_PRIORITY (INSN_EXPR (insn)),
3683 0 : EXPR_PRIORITY_ADJ (INSN_EXPR (insn)));
3684 0 : }
3685 :
3686 : /* Turn AV into a vector, filter inappropriate insns and sort it. Return
3687 : true if there is something to schedule. BNDS and FENCE are current
3688 : boundaries and fence, respectively. If we need to stall for some cycles
3689 : before an expr from AV would become available, write this number to
3690 : *PNEED_STALL. */
3691 : static bool
3692 7619 : fill_vec_av_set (av_set_t av, blist_t bnds, fence_t fence,
3693 : int *pneed_stall)
3694 : {
3695 7619 : av_set_iterator si;
3696 7619 : expr_t expr;
3697 7619 : int sched_next_worked = 0, stalled, n;
3698 7619 : static int av_max_prio, est_ticks_till_branch;
3699 7619 : int min_need_stall = -1;
3700 7619 : deps_t dc = BND_DC (BLIST_BND (bnds));
3701 :
3702 : /* Bail out early when the ready list contained only USEs/CLOBBERs that are
3703 : already scheduled. */
3704 7619 : if (av == NULL)
3705 : return false;
3706 :
3707 : /* Empty vector from the previous stuff. */
3708 7619 : if (vec_av_set.length () > 0)
3709 5224 : vec_av_set.block_remove (0, vec_av_set.length ());
3710 :
3711 : /* Turn the set into a vector for sorting and call sel_target_adjust_priority
3712 : for each insn. */
3713 33121 : gcc_assert (vec_av_set.is_empty ());
3714 25502 : FOR_EACH_EXPR (expr, si, av)
3715 : {
3716 17883 : vec_av_set.safe_push (expr);
3717 :
3718 17883 : gcc_assert (EXPR_PRIORITY_ADJ (expr) == 0 || *pneed_stall);
3719 :
3720 : /* Adjust priority using target backend hook. */
3721 17883 : sel_target_adjust_priority (expr);
3722 : }
3723 :
3724 : /* Sort the vector. */
3725 7619 : vec_av_set.qsort (sel_rank_for_schedule);
3726 :
3727 : /* We record maximal priority of insns in av set for current instruction
3728 : group. */
3729 7619 : if (FENCE_STARTS_CYCLE_P (fence))
3730 3700 : av_max_prio = est_ticks_till_branch = INT_MIN;
3731 :
3732 : /* Filter out inappropriate expressions. Loop's direction is reversed to
3733 : visit "best" instructions first. We assume that vec::unordered_remove
3734 : moves last element in place of one being deleted. */
3735 33121 : for (n = vec_av_set.length () - 1, stalled = 0; n >= 0; n--)
3736 : {
3737 17883 : expr_t expr = vec_av_set[n];
3738 17883 : insn_t insn = EXPR_INSN_RTX (expr);
3739 17883 : signed char target_available;
3740 17883 : bool is_orig_reg_p = true;
3741 17883 : int need_cycles, new_prio;
3742 17883 : bool fence_insn_p = INSN_UID (insn) == INSN_UID (FENCE_INSN (fence));
3743 :
3744 : /* Don't allow any insns other than from SCHED_GROUP if we have one. */
3745 17883 : if (FENCE_SCHED_NEXT (fence) && insn != FENCE_SCHED_NEXT (fence))
3746 : {
3747 0 : vec_av_set.unordered_remove (n);
3748 7853 : continue;
3749 : }
3750 :
3751 : /* Set number of sched_next insns (just in case there
3752 : could be several). */
3753 17883 : if (FENCE_SCHED_NEXT (fence))
3754 0 : sched_next_worked++;
3755 :
3756 : /* Check all liveness requirements and try renaming.
3757 : FIXME: try to minimize calls to this. */
3758 17883 : target_available = EXPR_TARGET_AVAILABLE (expr);
3759 :
3760 : /* If insn was already scheduled on the current fence,
3761 : set TARGET_AVAILABLE to -1 no matter what expr's attribute says. */
3762 17883 : if (vinsn_vec_has_expr_p (vec_target_unavailable_vinsns, expr)
3763 17883 : && !fence_insn_p)
3764 17883 : target_available = -1;
3765 :
3766 : /* If the availability of the EXPR is invalidated by the insertion of
3767 : bookkeeping earlier, make sure that we won't choose this expr for
3768 : scheduling if it's not separable, and if it is separable, then
3769 : we have to recompute the set of available registers for it. */
3770 17883 : if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr))
3771 : {
3772 17 : vec_av_set.unordered_remove (n);
3773 17 : if (sched_verbose >= 4)
3774 0 : sel_print ("Expr %d is blocked by bookkeeping inserted earlier\n",
3775 0 : INSN_UID (insn));
3776 17 : continue;
3777 : }
3778 :
3779 17866 : if (target_available == true)
3780 : {
3781 : /* Do nothing -- we can use an existing register. */
3782 13535 : is_orig_reg_p = EXPR_SEPARABLE_P (expr);
3783 : }
3784 8441 : else if (/* Non-separable instruction will never
3785 : get another register. */
3786 : (target_available == false
3787 4069 : && !EXPR_SEPARABLE_P (expr))
3788 : /* Don't try to find a register for low-priority expression. */
3789 4069 : || (int) vec_av_set.length () - 1 - n >= max_insns_to_rename
3790 : /* ??? FIXME: Don't try to rename data speculation. */
3791 2790 : || (EXPR_SPEC_DONE_DS (expr) & BEGIN_DATA)
3792 7121 : || ! find_best_reg_for_expr (expr, bnds, &is_orig_reg_p))
3793 : {
3794 4110 : vec_av_set.unordered_remove (n);
3795 4110 : if (sched_verbose >= 4)
3796 0 : sel_print ("Expr %d has no suitable target register\n",
3797 0 : INSN_UID (insn));
3798 :
3799 : /* A fence insn should not get here. */
3800 4110 : gcc_assert (!fence_insn_p);
3801 4110 : continue;
3802 : }
3803 :
3804 : /* At this point a fence insn should always be available. */
3805 13756 : gcc_assert (!fence_insn_p
3806 : || INSN_UID (FENCE_INSN (fence)) == INSN_UID (EXPR_INSN_RTX (expr)));
3807 :
3808 : /* Filter expressions that need to be renamed or speculated when
3809 : pipelining, because compensating register copies or speculation
3810 : checks are likely to be placed near the beginning of the loop,
3811 : causing a stall. */
3812 13756 : if (pipelining_p && EXPR_ORIG_SCHED_CYCLE (expr) > 0
3813 363 : && (!is_orig_reg_p || EXPR_SPEC_DONE_DS (expr) != 0))
3814 : {
3815 : /* Estimation of number of cycles until loop branch for
3816 : renaming/speculation to be successful. */
3817 162 : int need_n_ticks_till_branch = sel_vinsn_cost (EXPR_VINSN (expr));
3818 :
3819 162 : if ((int) current_loop_nest->ninsns < 9)
3820 : {
3821 11 : vec_av_set.unordered_remove (n);
3822 11 : if (sched_verbose >= 4)
3823 0 : sel_print ("Pipelining expr %d will likely cause stall\n",
3824 0 : INSN_UID (insn));
3825 11 : continue;
3826 : }
3827 :
3828 151 : if ((int) current_loop_nest->ninsns - num_insns_scheduled
3829 151 : < need_n_ticks_till_branch * issue_rate / 2
3830 30 : && est_ticks_till_branch < need_n_ticks_till_branch)
3831 : {
3832 16 : vec_av_set.unordered_remove (n);
3833 16 : if (sched_verbose >= 4)
3834 0 : sel_print ("Pipelining expr %d will likely cause stall\n",
3835 0 : INSN_UID (insn));
3836 16 : continue;
3837 : }
3838 : }
3839 :
3840 : /* We want to schedule speculation checks as late as possible. Discard
3841 : them from av set if there are instructions with higher priority. */
3842 13729 : if (sel_insn_is_speculation_check (insn)
3843 13729 : && EXPR_PRIORITY (expr) < av_max_prio)
3844 : {
3845 0 : stalled++;
3846 0 : min_need_stall = min_need_stall < 0 ? 1 : MIN (min_need_stall, 1);
3847 0 : vec_av_set.unordered_remove (n);
3848 0 : if (sched_verbose >= 4)
3849 0 : sel_print ("Delaying speculation check %d until its first use\n",
3850 0 : INSN_UID (insn));
3851 0 : continue;
3852 : }
3853 :
3854 : /* Ignore EXPRs available from pipelining to update AV_MAX_PRIO. */
3855 13729 : if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3856 12326 : av_max_prio = MAX (av_max_prio, EXPR_PRIORITY (expr));
3857 :
3858 : /* Don't allow any insns whose data is not yet ready.
3859 : Check first whether we've already tried them and failed. */
3860 13729 : if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
3861 : {
3862 13379 : need_cycles = (FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3863 13379 : - FENCE_CYCLE (fence));
3864 13379 : if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3865 11992 : est_ticks_till_branch = MAX (est_ticks_till_branch,
3866 : EXPR_PRIORITY (expr) + need_cycles);
3867 :
3868 13379 : if (need_cycles > 0)
3869 : {
3870 1721 : stalled++;
3871 3442 : min_need_stall = (min_need_stall < 0
3872 1721 : ? need_cycles
3873 818 : : MIN (min_need_stall, need_cycles));
3874 1721 : vec_av_set.unordered_remove (n);
3875 :
3876 1721 : if (sched_verbose >= 4)
3877 0 : sel_print ("Expr %d is not ready until cycle %d (cached)\n",
3878 : INSN_UID (insn),
3879 0 : FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3880 1721 : continue;
3881 : }
3882 : }
3883 :
3884 : /* Now resort to dependence analysis to find whether EXPR might be
3885 : stalled due to dependencies from FENCE's context. */
3886 12008 : need_cycles = tick_check_p (expr, dc, fence);
3887 12008 : new_prio = EXPR_PRIORITY (expr) + EXPR_PRIORITY_ADJ (expr) + need_cycles;
3888 :
3889 12008 : if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3890 10707 : est_ticks_till_branch = MAX (est_ticks_till_branch,
3891 : new_prio);
3892 :
3893 12008 : if (need_cycles > 0)
3894 : {
3895 1978 : if (INSN_UID (insn) >= FENCE_READY_TICKS_SIZE (fence))
3896 : {
3897 92 : int new_size = INSN_UID (insn) * 3 / 2;
3898 :
3899 92 : FENCE_READY_TICKS (fence)
3900 92 : = (int *) xrecalloc (FENCE_READY_TICKS (fence),
3901 : new_size, FENCE_READY_TICKS_SIZE (fence),
3902 : sizeof (int));
3903 : }
3904 3956 : FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3905 1978 : = FENCE_CYCLE (fence) + need_cycles;
3906 :
3907 1978 : stalled++;
3908 3956 : min_need_stall = (min_need_stall < 0
3909 1978 : ? need_cycles
3910 553 : : MIN (min_need_stall, need_cycles));
3911 :
3912 1978 : vec_av_set.unordered_remove (n);
3913 :
3914 1978 : if (sched_verbose >= 4)
3915 0 : sel_print ("Expr %d is not ready yet until cycle %d\n",
3916 : INSN_UID (insn),
3917 0 : FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3918 1978 : continue;
3919 1978 : }
3920 :
3921 10030 : if (sched_verbose >= 4)
3922 0 : sel_print ("Expr %d is ok\n", INSN_UID (insn));
3923 10030 : min_need_stall = 0;
3924 : }
3925 :
3926 : /* Clear SCHED_NEXT. */
3927 7619 : if (FENCE_SCHED_NEXT (fence))
3928 : {
3929 0 : gcc_assert (sched_next_worked == 1);
3930 0 : FENCE_SCHED_NEXT (fence) = NULL;
3931 : }
3932 :
3933 : /* No need to stall if this variable was not initialized. */
3934 7619 : if (min_need_stall < 0)
3935 : min_need_stall = 0;
3936 :
3937 7619 : if (vec_av_set.is_empty ())
3938 : {
3939 : /* We need to set *pneed_stall here, because later we skip this code
3940 : when ready list is empty. */
3941 1625 : *pneed_stall = min_need_stall;
3942 1625 : return false;
3943 : }
3944 : else
3945 5994 : gcc_assert (min_need_stall == 0);
3946 :
3947 : /* Sort the vector. */
3948 5994 : vec_av_set.qsort (sel_rank_for_schedule);
3949 :
3950 5994 : if (sched_verbose >= 4)
3951 : {
3952 0 : sel_print ("Total ready exprs: %d, stalled: %d\n",
3953 : vec_av_set.length (), stalled);
3954 0 : sel_print ("Sorted av set (%d): ", vec_av_set.length ());
3955 0 : FOR_EACH_VEC_ELT (vec_av_set, n, expr)
3956 0 : dump_expr (expr);
3957 0 : sel_print ("\n");
3958 : }
3959 :
3960 5994 : *pneed_stall = 0;
3961 5994 : return true;
3962 : }
3963 :
3964 : /* Convert a vectored and sorted av set to the ready list that
3965 : the rest of the backend wants to see. */
3966 : static void
3967 5994 : convert_vec_av_set_to_ready (void)
3968 : {
3969 5994 : int n;
3970 5994 : expr_t expr;
3971 :
3972 : /* Allocate and fill the ready list from the sorted vector. */
3973 5994 : ready.n_ready = vec_av_set.length ();
3974 5994 : ready.first = ready.n_ready - 1;
3975 :
3976 5994 : gcc_assert (ready.n_ready > 0);
3977 :
3978 5994 : if (ready.n_ready > max_issue_size)
3979 : {
3980 883 : max_issue_size = ready.n_ready;
3981 883 : sched_extend_ready_list (ready.n_ready);
3982 : }
3983 :
3984 16024 : FOR_EACH_VEC_ELT (vec_av_set, n, expr)
3985 : {
3986 10030 : vinsn_t vi = EXPR_VINSN (expr);
3987 10030 : insn_t insn = VINSN_INSN_RTX (vi);
3988 :
3989 10030 : ready_try[n] = 0;
3990 10030 : ready.vec[n] = insn;
3991 : }
3992 5994 : }
3993 :
3994 : /* Initialize ready list from *AV_PTR for the max_issue () call.
3995 : If any unrecognizable insn found in *AV_PTR, return it (and skip
3996 : max_issue). BND and FENCE are current boundary and fence,
3997 : respectively. If we need to stall for some cycles before an expr
3998 : from *AV_PTR would become available, write this number to *PNEED_STALL. */
3999 : static expr_t
4000 7728 : fill_ready_list (av_set_t *av_ptr, blist_t bnds, fence_t fence,
4001 : int *pneed_stall)
4002 : {
4003 7728 : expr_t expr;
4004 :
4005 : /* We do not support multiple boundaries per fence. */
4006 7728 : gcc_assert (BLIST_NEXT (bnds) == NULL);
4007 :
4008 : /* Process expressions required special handling, i.e. pipelined,
4009 : speculative and recog() < 0 expressions first. */
4010 7728 : process_pipelined_exprs (av_ptr);
4011 7728 : process_spec_exprs (av_ptr);
4012 :
4013 : /* A USE could be scheduled immediately. */
4014 7728 : expr = process_use_exprs (av_ptr);
4015 7728 : if (expr)
4016 : {
4017 109 : *pneed_stall = 0;
4018 109 : return expr;
4019 : }
4020 :
4021 : /* Turn the av set to a vector for sorting. */
4022 7619 : if (! fill_vec_av_set (*av_ptr, bnds, fence, pneed_stall))
4023 : {
4024 1625 : ready.n_ready = 0;
4025 1625 : return NULL;
4026 : }
4027 :
4028 : /* Build the final ready list. */
4029 5994 : convert_vec_av_set_to_ready ();
4030 5994 : return NULL;
4031 : }
4032 :
4033 : /* Wrapper for dfa_new_cycle (). Returns TRUE if cycle was advanced. */
4034 : static bool
4035 4901 : sel_dfa_new_cycle (insn_t insn, fence_t fence)
4036 : {
4037 4901 : int last_scheduled_cycle = FENCE_LAST_SCHEDULED_INSN (fence)
4038 4901 : ? INSN_SCHED_CYCLE (FENCE_LAST_SCHEDULED_INSN (fence))
4039 4901 : : FENCE_CYCLE (fence) - 1;
4040 4901 : bool res = false;
4041 4901 : int sort_p = 0;
4042 :
4043 4901 : if (!targetm.sched.dfa_new_cycle)
4044 : return false;
4045 :
4046 0 : memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4047 :
4048 0 : while (!sort_p && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
4049 : insn, last_scheduled_cycle,
4050 : FENCE_CYCLE (fence), &sort_p))
4051 : {
4052 0 : memcpy (FENCE_STATE (fence), curr_state, dfa_state_size);
4053 0 : advance_one_cycle (fence);
4054 0 : memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4055 0 : res = true;
4056 : }
4057 :
4058 : return res;
4059 : }
4060 :
4061 : /* Invoke reorder* target hooks on the ready list. Return the number of insns
4062 : we can issue. FENCE is the current fence. */
4063 : static int
4064 5994 : invoke_reorder_hooks (fence_t fence)
4065 : {
4066 5994 : int issue_more;
4067 5994 : bool ran_hook = false;
4068 :
4069 : /* Call the reorder hook at the beginning of the cycle, and call
4070 : the reorder2 hook in the middle of the cycle. */
4071 5994 : if (FENCE_ISSUED_INSNS (fence) == 0)
4072 : {
4073 3475 : if (targetm.sched.reorder
4074 3475 : && !SCHED_GROUP_P (ready_element (&ready, 0))
4075 6593 : && ready.n_ready > 1)
4076 : {
4077 : /* Don't give reorder the most prioritized insn as it can break
4078 : pipelining. */
4079 903 : if (pipelining_p)
4080 290 : --ready.n_ready;
4081 :
4082 903 : issue_more
4083 903 : = targetm.sched.reorder (sched_dump, sched_verbose,
4084 : ready_lastpos (&ready),
4085 : &ready.n_ready, FENCE_CYCLE (fence));
4086 :
4087 903 : if (pipelining_p)
4088 290 : ++ready.n_ready;
4089 :
4090 : ran_hook = true;
4091 : }
4092 : else
4093 : /* Initialize can_issue_more for variable_issue. */
4094 2572 : issue_more = issue_rate;
4095 : }
4096 2519 : else if (targetm.sched.reorder2
4097 2519 : && !SCHED_GROUP_P (ready_element (&ready, 0)))
4098 : {
4099 0 : if (ready.n_ready == 1)
4100 0 : issue_more =
4101 0 : targetm.sched.reorder2 (sched_dump, sched_verbose,
4102 : ready_lastpos (&ready),
4103 : &ready.n_ready, FENCE_CYCLE (fence));
4104 : else
4105 : {
4106 0 : if (pipelining_p)
4107 0 : --ready.n_ready;
4108 :
4109 0 : issue_more =
4110 0 : targetm.sched.reorder2 (sched_dump, sched_verbose,
4111 0 : ready.n_ready
4112 0 : ? ready_lastpos (&ready) : NULL,
4113 : &ready.n_ready, FENCE_CYCLE (fence));
4114 :
4115 0 : if (pipelining_p)
4116 0 : ++ready.n_ready;
4117 : }
4118 :
4119 : ran_hook = true;
4120 : }
4121 : else
4122 2519 : issue_more = FENCE_ISSUE_MORE (fence);
4123 :
4124 : /* Ensure that ready list and vec_av_set are in line with each other,
4125 : i.e. vec_av_set[i] == ready_element (&ready, i). */
4126 5994 : if (issue_more && ran_hook)
4127 : {
4128 903 : int i, j, n;
4129 903 : rtx_insn **arr = ready.vec;
4130 903 : expr_t *vec = vec_av_set.address ();
4131 :
4132 3584 : for (i = 0, n = ready.n_ready; i < n; i++)
4133 2681 : if (EXPR_INSN_RTX (vec[i]) != arr[i])
4134 : {
4135 0 : for (j = i; j < n; j++)
4136 0 : if (EXPR_INSN_RTX (vec[j]) == arr[i])
4137 : break;
4138 0 : gcc_assert (j < n);
4139 :
4140 0 : std::swap (vec[i], vec[j]);
4141 : }
4142 : }
4143 :
4144 5994 : return issue_more;
4145 : }
4146 :
4147 : /* Return an EXPR corresponding to INDEX element of ready list, if
4148 : FOLLOW_READY_ELEMENT is true (i.e., an expr of
4149 : ready_element (&ready, INDEX) will be returned), and to INDEX element of
4150 : ready.vec otherwise. */
4151 : static inline expr_t
4152 28013 : find_expr_for_ready (int index, bool follow_ready_element)
4153 : {
4154 28013 : expr_t expr;
4155 28013 : int real_index;
4156 :
4157 28013 : real_index = follow_ready_element ? ready.first - index : index;
4158 :
4159 28013 : expr = vec_av_set[real_index];
4160 28013 : gcc_assert (ready.vec[real_index] == EXPR_INSN_RTX (expr));
4161 :
4162 28013 : return expr;
4163 : }
4164 :
4165 : /* Calculate insns worth trying via lookahead_guard hook. Return a number
4166 : of such insns found. */
4167 : static int
4168 4928 : invoke_dfa_lookahead_guard (void)
4169 : {
4170 4928 : int i, n;
4171 4928 : bool have_hook
4172 4928 : = targetm.sched.first_cycle_multipass_dfa_lookahead_guard != NULL;
4173 :
4174 4928 : if (sched_verbose >= 2)
4175 0 : sel_print ("ready after reorder: ");
4176 :
4177 13298 : for (i = 0, n = 0; i < ready.n_ready; i++)
4178 : {
4179 8370 : expr_t expr;
4180 8370 : insn_t insn;
4181 8370 : int r;
4182 :
4183 : /* In this loop insn is Ith element of the ready list given by
4184 : ready_element, not Ith element of ready.vec. */
4185 8370 : insn = ready_element (&ready, i);
4186 :
4187 8370 : if (! have_hook || i == 0)
4188 : r = 0;
4189 : else
4190 0 : r = targetm.sched.first_cycle_multipass_dfa_lookahead_guard (insn, i);
4191 :
4192 8370 : gcc_assert (INSN_CODE (insn) >= 0);
4193 :
4194 : /* Only insns with ready_try = 0 can get here
4195 : from fill_ready_list. */
4196 8370 : gcc_assert (ready_try [i] == 0);
4197 8370 : ready_try[i] = r;
4198 8370 : if (!r)
4199 8370 : n++;
4200 :
4201 8370 : expr = find_expr_for_ready (i, true);
4202 :
4203 8370 : if (sched_verbose >= 2)
4204 : {
4205 0 : dump_vinsn (EXPR_VINSN (expr));
4206 0 : sel_print (":%d; ", ready_try[i]);
4207 : }
4208 : }
4209 :
4210 4928 : if (sched_verbose >= 2)
4211 0 : sel_print ("\n");
4212 4928 : return n;
4213 : }
4214 :
4215 : /* Calculate the number of privileged insns and return it. */
4216 : static int
4217 4928 : calculate_privileged_insns (void)
4218 : {
4219 4928 : expr_t cur_expr, min_spec_expr = NULL;
4220 4928 : int privileged_n = 0, i;
4221 :
4222 13212 : for (i = 0; i < ready.n_ready; i++)
4223 : {
4224 8343 : if (ready_try[i])
4225 0 : continue;
4226 :
4227 8343 : if (! min_spec_expr)
4228 4928 : min_spec_expr = find_expr_for_ready (i, true);
4229 :
4230 8343 : cur_expr = find_expr_for_ready (i, true);
4231 :
4232 8343 : if (EXPR_SPEC (cur_expr) > EXPR_SPEC (min_spec_expr))
4233 : break;
4234 :
4235 8284 : ++privileged_n;
4236 : }
4237 :
4238 4928 : if (i == ready.n_ready)
4239 4869 : privileged_n = 0;
4240 :
4241 4928 : if (sched_verbose >= 2)
4242 0 : sel_print ("privileged_n: %d insns with SPEC %d\n",
4243 : privileged_n, privileged_n ? EXPR_SPEC (min_spec_expr) : -1);
4244 4928 : return privileged_n;
4245 : }
4246 :
4247 : /* Call the rest of the hooks after the choice was made. Return
4248 : the number of insns that still can be issued given that the current
4249 : number is ISSUE_MORE. FENCE and BEST_INSN are the current fence
4250 : and the insn chosen for scheduling, respectively. */
4251 : static int
4252 4901 : invoke_aftermath_hooks (fence_t fence, rtx_insn *best_insn, int issue_more)
4253 : {
4254 4901 : gcc_assert (INSN_P (best_insn));
4255 :
4256 : /* First, call dfa_new_cycle, and then variable_issue, if available. */
4257 4901 : sel_dfa_new_cycle (best_insn, fence);
4258 :
4259 4901 : if (targetm.sched.variable_issue)
4260 : {
4261 0 : memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4262 0 : issue_more =
4263 0 : targetm.sched.variable_issue (sched_dump, sched_verbose, best_insn,
4264 : issue_more);
4265 0 : memcpy (FENCE_STATE (fence), curr_state, dfa_state_size);
4266 : }
4267 4901 : else if (!DEBUG_INSN_P (best_insn)
4268 4857 : && GET_CODE (PATTERN (best_insn)) != USE
4269 9713 : && GET_CODE (PATTERN (best_insn)) != CLOBBER)
4270 4802 : issue_more--;
4271 :
4272 4901 : return issue_more;
4273 : }
4274 :
4275 : /* Estimate the cost of issuing INSN on DFA state STATE. */
4276 : static int
4277 3192 : estimate_insn_cost (rtx_insn *insn, state_t state)
4278 : {
4279 3192 : static state_t temp = NULL;
4280 3192 : int cost;
4281 :
4282 3192 : if (!temp)
4283 54 : temp = xmalloc (dfa_state_size);
4284 :
4285 3192 : memcpy (temp, state, dfa_state_size);
4286 3192 : cost = state_transition (temp, insn);
4287 :
4288 3192 : if (cost < 0)
4289 : return 0;
4290 461 : else if (cost == 0)
4291 0 : return 1;
4292 : return cost;
4293 : }
4294 :
4295 : /* Return the cost of issuing EXPR on the FENCE as estimated by DFA.
4296 : This function properly handles ASMs, USEs etc. */
4297 : static int
4298 1580 : get_expr_cost (expr_t expr, fence_t fence)
4299 : {
4300 1580 : rtx_insn *insn = EXPR_INSN_RTX (expr);
4301 :
4302 1580 : if (recog_memoized (insn) < 0)
4303 : {
4304 0 : if (!FENCE_STARTS_CYCLE_P (fence)
4305 0 : && INSN_ASM_P (insn))
4306 : /* This is asm insn which is tryed to be issued on the
4307 : cycle not first. Issue it on the next cycle. */
4308 : return 1;
4309 : else
4310 : /* A USE insn, or something else we don't need to
4311 : understand. We can't pass these directly to
4312 : state_transition because it will trigger a
4313 : fatal error for unrecognizable insns. */
4314 0 : return 0;
4315 : }
4316 : else
4317 1580 : return estimate_insn_cost (insn, FENCE_STATE (fence));
4318 : }
4319 :
4320 : /* Find the best insn for scheduling, either via max_issue or just take
4321 : the most prioritized available. */
4322 : static int
4323 4928 : choose_best_insn (fence_t fence, int privileged_n, int *index)
4324 : {
4325 4928 : int can_issue = 0;
4326 :
4327 4928 : if (dfa_lookahead > 0)
4328 : {
4329 3517 : cycle_issued_insns = FENCE_ISSUED_INSNS (fence);
4330 : /* TODO: pass equivalent of first_cycle_insn_p to max_issue (). */
4331 3517 : can_issue = max_issue (&ready, privileged_n,
4332 : FENCE_STATE (fence), true, index);
4333 3517 : if (sched_verbose >= 2)
4334 0 : sel_print ("max_issue: we can issue %d insns, already did %d insns\n",
4335 : can_issue, FENCE_ISSUED_INSNS (fence));
4336 : }
4337 : else
4338 : {
4339 : /* We can't use max_issue; just return the first available element. */
4340 : int i;
4341 :
4342 1672 : for (i = 0; i < ready.n_ready; i++)
4343 : {
4344 1580 : expr_t expr = find_expr_for_ready (i, true);
4345 :
4346 1580 : if (get_expr_cost (expr, fence) < 1)
4347 : {
4348 1319 : can_issue = can_issue_more;
4349 1319 : *index = i;
4350 :
4351 1319 : if (sched_verbose >= 2)
4352 0 : sel_print ("using %dth insn from the ready list\n", i + 1);
4353 :
4354 : break;
4355 : }
4356 : }
4357 :
4358 1411 : if (i == ready.n_ready)
4359 : {
4360 92 : can_issue = 0;
4361 92 : *index = -1;
4362 : }
4363 : }
4364 :
4365 4928 : return can_issue;
4366 : }
4367 :
4368 : /* Choose the best expr from *AV_VLIW_PTR and a suitable register for it.
4369 : BNDS and FENCE are current boundaries and scheduling fence respectively.
4370 : Return the expr found and NULL if nothing can be issued atm.
4371 : Write to PNEED_STALL the number of cycles to stall if no expr was found. */
4372 : static expr_t
4373 7728 : find_best_expr (av_set_t *av_vliw_ptr, blist_t bnds, fence_t fence,
4374 : int *pneed_stall)
4375 : {
4376 7728 : expr_t best;
4377 :
4378 : /* Choose the best insn for scheduling via:
4379 : 1) sorting the ready list based on priority;
4380 : 2) calling the reorder hook;
4381 : 3) calling max_issue. */
4382 7728 : best = fill_ready_list (av_vliw_ptr, bnds, fence, pneed_stall);
4383 7728 : if (best == NULL && ready.n_ready > 0)
4384 : {
4385 5994 : int privileged_n, index;
4386 :
4387 5994 : can_issue_more = invoke_reorder_hooks (fence);
4388 5994 : if (can_issue_more > 0)
4389 : {
4390 : /* Try choosing the best insn until we find one that is could be
4391 : scheduled due to liveness restrictions on its destination register.
4392 : In the future, we'd like to choose once and then just probe insns
4393 : in the order of their priority. */
4394 4928 : invoke_dfa_lookahead_guard ();
4395 4928 : privileged_n = calculate_privileged_insns ();
4396 4928 : can_issue_more = choose_best_insn (fence, privileged_n, &index);
4397 4928 : if (can_issue_more)
4398 4792 : best = find_expr_for_ready (index, true);
4399 : }
4400 : /* We had some available insns, so if we can't issue them,
4401 : we have a stall. */
4402 5994 : if (can_issue_more == 0)
4403 : {
4404 1202 : best = NULL;
4405 1202 : *pneed_stall = 1;
4406 : }
4407 : }
4408 :
4409 7619 : if (best != NULL)
4410 : {
4411 4901 : can_issue_more = invoke_aftermath_hooks (fence, EXPR_INSN_RTX (best),
4412 : can_issue_more);
4413 4901 : if (targetm.sched.variable_issue
4414 0 : && can_issue_more == 0)
4415 0 : *pneed_stall = 1;
4416 : }
4417 :
4418 7728 : if (sched_verbose >= 2)
4419 : {
4420 0 : if (best != NULL)
4421 : {
4422 0 : sel_print ("Best expression (vliw form): ");
4423 0 : dump_expr (best);
4424 0 : sel_print ("; cycle %d\n", FENCE_CYCLE (fence));
4425 : }
4426 : else
4427 0 : sel_print ("No best expr found!\n");
4428 : }
4429 :
4430 7728 : return best;
4431 : }
4432 :
4433 :
4434 : /* Functions that implement the core of the scheduler. */
4435 :
4436 :
4437 : /* Emit an instruction from EXPR with SEQNO and VINSN after
4438 : PLACE_TO_INSERT. */
4439 : static insn_t
4440 419 : emit_insn_from_expr_after (expr_t expr, vinsn_t vinsn, int seqno,
4441 : insn_t place_to_insert)
4442 : {
4443 : /* This assert fails when we have identical instructions
4444 : one of which dominates the other. In this case move_op ()
4445 : finds the first instruction and doesn't search for second one.
4446 : The solution would be to compute av_set after the first found
4447 : insn and, if insn present in that set, continue searching.
4448 : For now we workaround this issue in move_op. */
4449 419 : gcc_assert (!INSN_IN_STREAM_P (EXPR_INSN_RTX (expr)));
4450 :
4451 419 : if (EXPR_WAS_RENAMED (expr))
4452 : {
4453 32 : unsigned regno = expr_dest_regno (expr);
4454 :
4455 32 : if (HARD_REGISTER_NUM_P (regno))
4456 : {
4457 18 : df_set_regs_ever_live (regno, true);
4458 18 : reg_rename_tick[regno] = ++reg_rename_this_tick;
4459 : }
4460 : }
4461 :
4462 419 : return sel_gen_insn_from_expr_after (expr, vinsn, seqno,
4463 419 : place_to_insert);
4464 : }
4465 :
4466 : /* Return TRUE if BB can hold bookkeeping code. */
4467 : static bool
4468 400 : block_valid_for_bookkeeping_p (basic_block bb)
4469 : {
4470 400 : insn_t bb_end = BB_END (bb);
4471 :
4472 400 : if (!in_current_region_p (bb) || EDGE_COUNT (bb->succs) > 1)
4473 : return false;
4474 :
4475 362 : if (INSN_P (bb_end))
4476 : {
4477 353 : if (INSN_SCHED_TIMES (bb_end) > 0)
4478 30 : return false;
4479 : }
4480 : else
4481 9 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (bb_end));
4482 :
4483 : return true;
4484 : }
4485 :
4486 : /* Attempt to find a block that can hold bookkeeping code for path(s) incoming
4487 : into E2->dest, except from E1->src (there may be a sequence of empty basic
4488 : blocks between E1->src and E2->dest). Return found block, or NULL if new
4489 : one must be created. If LAX holds, don't assume there is a simple path
4490 : from E1->src to E2->dest. */
4491 : static basic_block
4492 434 : find_block_for_bookkeeping (edge e1, edge e2, bool lax)
4493 : {
4494 434 : basic_block candidate_block = NULL;
4495 434 : edge e;
4496 :
4497 : /* Loop over edges from E1 to E2, inclusive. */
4498 444 : for (e = e1; !lax || e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun); e =
4499 10 : EDGE_SUCC (e->dest, 0))
4500 : {
4501 444 : if (EDGE_COUNT (e->dest->preds) == 2)
4502 : {
4503 415 : if (candidate_block == NULL)
4504 592 : candidate_block = (EDGE_PRED (e->dest, 0) == e
4505 415 : ? EDGE_PRED (e->dest, 1)->src
4506 : : EDGE_PRED (e->dest, 0)->src);
4507 : else
4508 : /* Found additional edge leading to path from e1 to e2
4509 : from aside. */
4510 : return NULL;
4511 : }
4512 29 : else if (EDGE_COUNT (e->dest->preds) > 2)
4513 : /* Several edges leading to path from e1 to e2 from aside. */
4514 : return NULL;
4515 :
4516 431 : if (e == e2)
4517 400 : return ((!lax || candidate_block)
4518 400 : && block_valid_for_bookkeeping_p (candidate_block)
4519 400 : ? candidate_block
4520 : : NULL);
4521 :
4522 112 : if (lax && EDGE_COUNT (e->dest->succs) != 1)
4523 : return NULL;
4524 : }
4525 :
4526 : if (lax)
4527 : return NULL;
4528 :
4529 : gcc_unreachable ();
4530 : }
4531 :
4532 : /* Create new basic block for bookkeeping code for path(s) incoming into
4533 : E2->dest, except from E1->src. Return created block. */
4534 : static basic_block
4535 67 : create_block_for_bookkeeping (edge e1, edge e2)
4536 : {
4537 67 : basic_block new_bb, bb = e2->dest;
4538 :
4539 : /* Check that we don't spoil the loop structure. */
4540 67 : if (current_loop_nest)
4541 : {
4542 60 : basic_block latch = current_loop_nest->latch;
4543 :
4544 : /* We do not split header. */
4545 60 : gcc_assert (e2->dest != current_loop_nest->header);
4546 :
4547 : /* We do not redirect the only edge to the latch block. */
4548 74 : gcc_assert (e1->dest != latch
4549 : || !single_pred_p (latch)
4550 : || e1 != single_pred_edge (latch));
4551 : }
4552 :
4553 : /* Split BB to insert BOOK_INSN there. */
4554 67 : new_bb = sched_split_block (bb, NULL);
4555 :
4556 : /* Move note_list from the upper bb. */
4557 67 : gcc_assert (BB_NOTE_LIST (new_bb) == NULL_RTX);
4558 67 : BB_NOTE_LIST (new_bb) = BB_NOTE_LIST (bb);
4559 67 : BB_NOTE_LIST (bb) = NULL;
4560 :
4561 67 : gcc_assert (e2->dest == bb);
4562 :
4563 : /* Skip block for bookkeeping copy when leaving E1->src. */
4564 67 : if (e1->flags & EDGE_FALLTHRU)
4565 16 : sel_redirect_edge_and_branch_force (e1, new_bb);
4566 : else
4567 51 : sel_redirect_edge_and_branch (e1, new_bb);
4568 :
4569 67 : gcc_assert (e1->dest == new_bb);
4570 67 : gcc_assert (sel_bb_empty_p (bb));
4571 :
4572 : /* To keep basic block numbers in sync between debug and non-debug
4573 : compilations, we have to rotate blocks here. Consider that we
4574 : started from (a,b)->d, (c,d)->e, and d contained only debug
4575 : insns. It would have been removed before if the debug insns
4576 : weren't there, so we'd have split e rather than d. So what we do
4577 : now is to swap the block numbers of new_bb and
4578 : single_succ(new_bb) == e, so that the insns that were in e before
4579 : get the new block number. */
4580 :
4581 67 : if (MAY_HAVE_DEBUG_INSNS)
4582 : {
4583 0 : basic_block succ;
4584 0 : insn_t insn = sel_bb_head (new_bb);
4585 0 : insn_t last;
4586 :
4587 0 : if (DEBUG_INSN_P (insn)
4588 0 : && single_succ_p (new_bb)
4589 0 : && (succ = single_succ (new_bb))
4590 0 : && succ != EXIT_BLOCK_PTR_FOR_FN (cfun)
4591 0 : && DEBUG_INSN_P ((last = sel_bb_end (new_bb))))
4592 : {
4593 0 : while (insn != last && (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4594 0 : insn = NEXT_INSN (insn);
4595 :
4596 0 : if (insn == last)
4597 : {
4598 0 : sel_global_bb_info_def gbi;
4599 0 : sel_region_bb_info_def rbi;
4600 :
4601 0 : if (sched_verbose >= 2)
4602 0 : sel_print ("Swapping block ids %i and %i\n",
4603 : new_bb->index, succ->index);
4604 :
4605 0 : std::swap (new_bb->index, succ->index);
4606 :
4607 0 : SET_BASIC_BLOCK_FOR_FN (cfun, new_bb->index, new_bb);
4608 0 : SET_BASIC_BLOCK_FOR_FN (cfun, succ->index, succ);
4609 :
4610 0 : memcpy (&gbi, SEL_GLOBAL_BB_INFO (new_bb), sizeof (gbi));
4611 0 : memcpy (SEL_GLOBAL_BB_INFO (new_bb), SEL_GLOBAL_BB_INFO (succ),
4612 : sizeof (gbi));
4613 0 : memcpy (SEL_GLOBAL_BB_INFO (succ), &gbi, sizeof (gbi));
4614 :
4615 0 : memcpy (&rbi, SEL_REGION_BB_INFO (new_bb), sizeof (rbi));
4616 0 : memcpy (SEL_REGION_BB_INFO (new_bb), SEL_REGION_BB_INFO (succ),
4617 : sizeof (rbi));
4618 0 : memcpy (SEL_REGION_BB_INFO (succ), &rbi, sizeof (rbi));
4619 :
4620 0 : std::swap (BLOCK_TO_BB (new_bb->index),
4621 0 : BLOCK_TO_BB (succ->index));
4622 :
4623 0 : std::swap (CONTAINING_RGN (new_bb->index),
4624 0 : CONTAINING_RGN (succ->index));
4625 :
4626 0 : for (int i = 0; i < current_nr_blocks; i++)
4627 0 : if (BB_TO_BLOCK (i) == succ->index)
4628 0 : BB_TO_BLOCK (i) = new_bb->index;
4629 0 : else if (BB_TO_BLOCK (i) == new_bb->index)
4630 0 : BB_TO_BLOCK (i) = succ->index;
4631 :
4632 0 : FOR_BB_INSNS (new_bb, insn)
4633 0 : if (INSN_P (insn))
4634 0 : EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = new_bb->index;
4635 :
4636 0 : FOR_BB_INSNS (succ, insn)
4637 0 : if (INSN_P (insn))
4638 0 : EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = succ->index;
4639 :
4640 0 : if (bitmap_clear_bit (code_motion_visited_blocks, new_bb->index))
4641 0 : bitmap_set_bit (code_motion_visited_blocks, succ->index);
4642 :
4643 0 : gcc_assert (LABEL_P (BB_HEAD (new_bb))
4644 : && LABEL_P (BB_HEAD (succ)));
4645 :
4646 0 : if (sched_verbose >= 4)
4647 0 : sel_print ("Swapping code labels %i and %i\n",
4648 : CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4649 : CODE_LABEL_NUMBER (BB_HEAD (succ)));
4650 :
4651 0 : std::swap (CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4652 0 : CODE_LABEL_NUMBER (BB_HEAD (succ)));
4653 : }
4654 : }
4655 : }
4656 :
4657 67 : return bb;
4658 : }
4659 :
4660 : /* Return insn after which we must insert bookkeeping code for path(s) incoming
4661 : into E2->dest, except from E1->src. If the returned insn immediately
4662 : precedes a fence, assign that fence to *FENCE_TO_REWIND. */
4663 : static insn_t
4664 389 : find_place_for_bookkeeping (edge e1, edge e2, fence_t *fence_to_rewind)
4665 : {
4666 389 : insn_t place_to_insert;
4667 : /* Find a basic block that can hold bookkeeping. If it can be found, do not
4668 : create new basic block, but insert bookkeeping there. */
4669 389 : basic_block book_block = find_block_for_bookkeeping (e1, e2, false);
4670 :
4671 389 : if (book_block)
4672 : {
4673 322 : place_to_insert = BB_END (book_block);
4674 :
4675 : /* Don't use a block containing only debug insns for
4676 : bookkeeping, this causes scheduling differences between debug
4677 : and non-debug compilations, for the block would have been
4678 : removed already. */
4679 322 : if (DEBUG_INSN_P (place_to_insert))
4680 : {
4681 2 : rtx_insn *insn = sel_bb_head (book_block);
4682 :
4683 4 : while (insn != place_to_insert &&
4684 2 : (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4685 0 : insn = NEXT_INSN (insn);
4686 :
4687 2 : if (insn == place_to_insert)
4688 : book_block = NULL;
4689 : }
4690 : }
4691 :
4692 : if (!book_block)
4693 : {
4694 67 : book_block = create_block_for_bookkeeping (e1, e2);
4695 67 : place_to_insert = BB_END (book_block);
4696 67 : if (sched_verbose >= 9)
4697 0 : sel_print ("New block is %i, split from bookkeeping block %i\n",
4698 0 : EDGE_SUCC (book_block, 0)->dest->index, book_block->index);
4699 : }
4700 : else
4701 : {
4702 322 : if (sched_verbose >= 9)
4703 0 : sel_print ("Pre-existing bookkeeping block is %i\n", book_block->index);
4704 : }
4705 :
4706 389 : *fence_to_rewind = NULL;
4707 : /* If basic block ends with a jump, insert bookkeeping code right before it.
4708 : Notice if we are crossing a fence when taking PREV_INSN. */
4709 389 : if (INSN_P (place_to_insert) && control_flow_insn_p (place_to_insert))
4710 : {
4711 8 : *fence_to_rewind = flist_lookup (fences, place_to_insert);
4712 8 : place_to_insert = PREV_INSN (place_to_insert);
4713 : }
4714 :
4715 389 : return place_to_insert;
4716 : }
4717 :
4718 : /* Find a proper seqno for bookkeeing insn inserted at PLACE_TO_INSERT
4719 : for JOIN_POINT. */
4720 : static int
4721 389 : find_seqno_for_bookkeeping (insn_t place_to_insert, insn_t join_point)
4722 : {
4723 389 : int seqno;
4724 :
4725 : /* Check if we are about to insert bookkeeping copy before a jump, and use
4726 : jump's seqno for the copy; otherwise, use JOIN_POINT's seqno. */
4727 389 : rtx_insn *next = NEXT_INSN (place_to_insert);
4728 389 : if (INSN_P (next)
4729 : && JUMP_P (next)
4730 389 : && BLOCK_FOR_INSN (next) == BLOCK_FOR_INSN (place_to_insert))
4731 : {
4732 8 : gcc_assert (INSN_SCHED_TIMES (next) == 0);
4733 8 : seqno = INSN_SEQNO (next);
4734 : }
4735 381 : else if (INSN_SEQNO (join_point) > 0)
4736 : seqno = INSN_SEQNO (join_point);
4737 : else
4738 : {
4739 0 : seqno = get_seqno_by_preds (place_to_insert);
4740 :
4741 : /* Sometimes the fences can move in such a way that there will be
4742 : no instructions with positive seqno around this bookkeeping.
4743 : This means that there will be no way to get to it by a regular
4744 : fence movement. Never mind because we pick up such pieces for
4745 : rescheduling anyways, so any positive value will do for now. */
4746 0 : if (seqno < 0)
4747 : {
4748 0 : gcc_assert (pipelining_p);
4749 : seqno = 1;
4750 : }
4751 : }
4752 :
4753 389 : gcc_assert (seqno > 0);
4754 389 : return seqno;
4755 : }
4756 :
4757 : /* Insert bookkeeping copy of C_EXPS's insn after PLACE_TO_INSERT, assigning
4758 : NEW_SEQNO to it. Return created insn. */
4759 : static insn_t
4760 389 : emit_bookkeeping_insn (insn_t place_to_insert, expr_t c_expr, int new_seqno)
4761 : {
4762 389 : rtx_insn *new_insn_rtx = create_copy_of_insn_rtx (EXPR_INSN_RTX (c_expr));
4763 :
4764 389 : vinsn_t new_vinsn
4765 778 : = create_vinsn_from_insn_rtx (new_insn_rtx,
4766 389 : VINSN_UNIQUE_P (EXPR_VINSN (c_expr)));
4767 :
4768 389 : insn_t new_insn = emit_insn_from_expr_after (c_expr, new_vinsn, new_seqno,
4769 : place_to_insert);
4770 :
4771 389 : INSN_SCHED_TIMES (new_insn) = 0;
4772 389 : bitmap_set_bit (current_copies, INSN_UID (new_insn));
4773 :
4774 389 : return new_insn;
4775 : }
4776 :
4777 : /* Generate a bookkeeping copy of C_EXPR's insn for path(s) incoming into to
4778 : E2->dest, except from E1->src (there may be a sequence of empty blocks
4779 : between E1->src and E2->dest). Return block containing the copy.
4780 : All scheduler data is initialized for the newly created insn. */
4781 : static basic_block
4782 389 : generate_bookkeeping_insn (expr_t c_expr, edge e1, edge e2)
4783 : {
4784 389 : insn_t join_point, place_to_insert, new_insn;
4785 389 : int new_seqno;
4786 389 : bool need_to_exchange_data_sets;
4787 389 : fence_t fence_to_rewind;
4788 :
4789 389 : if (sched_verbose >= 4)
4790 0 : sel_print ("Generating bookkeeping insn (%d->%d)\n", e1->src->index,
4791 0 : e2->dest->index);
4792 :
4793 389 : join_point = sel_bb_head (e2->dest);
4794 389 : place_to_insert = find_place_for_bookkeeping (e1, e2, &fence_to_rewind);
4795 389 : new_seqno = find_seqno_for_bookkeeping (place_to_insert, join_point);
4796 389 : need_to_exchange_data_sets
4797 389 : = sel_bb_empty_p (BLOCK_FOR_INSN (place_to_insert));
4798 :
4799 389 : new_insn = emit_bookkeeping_insn (place_to_insert, c_expr, new_seqno);
4800 :
4801 389 : if (fence_to_rewind)
4802 0 : FENCE_INSN (fence_to_rewind) = new_insn;
4803 :
4804 : /* When inserting bookkeeping insn in new block, av sets should be
4805 : following: old basic block (that now holds bookkeeping) data sets are
4806 : the same as was before generation of bookkeeping, and new basic block
4807 : (that now hold all other insns of old basic block) data sets are
4808 : invalid. So exchange data sets for these basic blocks as sel_split_block
4809 : mistakenly exchanges them in this case. Cannot do it earlier because
4810 : when single instruction is added to new basic block it should hold NULL
4811 : lv_set. */
4812 389 : if (need_to_exchange_data_sets)
4813 76 : exchange_data_sets (BLOCK_FOR_INSN (new_insn),
4814 76 : BLOCK_FOR_INSN (join_point));
4815 :
4816 389 : stat_bookkeeping_copies++;
4817 389 : return BLOCK_FOR_INSN (new_insn);
4818 : }
4819 :
4820 : /* Remove from AV_PTR all insns that may need bookkeeping when scheduling
4821 : on FENCE, but we are unable to copy them. */
4822 : static void
4823 5189 : remove_insns_that_need_bookkeeping (fence_t fence, av_set_t *av_ptr)
4824 : {
4825 5189 : expr_t expr;
4826 5189 : av_set_iterator i;
4827 :
4828 : /* An expression does not need bookkeeping if it is available on all paths
4829 : from current block to original block and current block dominates
4830 : original block. We check availability on all paths by examining
4831 : EXPR_SPEC; this is not equivalent, because it may be positive even
4832 : if expr is available on all paths (but if expr is not available on
4833 : any path, EXPR_SPEC will be positive). */
4834 :
4835 30569 : FOR_EACH_EXPR_1 (expr, i, av_ptr)
4836 : {
4837 12690 : if (!control_flow_insn_p (EXPR_INSN_RTX (expr))
4838 11806 : && (!bookkeeping_p || VINSN_UNIQUE_P (EXPR_VINSN (expr)))
4839 14555 : && (EXPR_SPEC (expr)
4840 1848 : || !EXPR_ORIG_BB_INDEX (expr)
4841 1848 : || !dominated_by_p (CDI_DOMINATORS,
4842 1848 : BASIC_BLOCK_FOR_FN (cfun,
4843 : EXPR_ORIG_BB_INDEX (expr)),
4844 1848 : BLOCK_FOR_INSN (FENCE_INSN (fence)))))
4845 : {
4846 25 : if (sched_verbose >= 4)
4847 0 : sel_print ("Expr %d removed because it would need bookkeeping, which "
4848 0 : "cannot be created\n", INSN_UID (EXPR_INSN_RTX (expr)));
4849 25 : av_set_iter_remove (&i);
4850 : }
4851 : }
4852 5189 : }
4853 :
4854 : /* Moving conditional jump through some instructions.
4855 :
4856 : Consider example:
4857 :
4858 : ... <- current scheduling point
4859 : NOTE BASIC BLOCK: <- bb header
4860 : (p8) add r14=r14+0x9;;
4861 : (p8) mov [r14]=r23
4862 : (!p8) jump L1;;
4863 : NOTE BASIC BLOCK:
4864 : ...
4865 :
4866 : We can schedule jump one cycle earlier, than mov, because they cannot be
4867 : executed together as their predicates are mutually exclusive.
4868 :
4869 : This is done in this way: first, new fallthrough basic block is created
4870 : after jump (it is always can be done, because there already should be a
4871 : fallthrough block, where control flow goes in case of predicate being true -
4872 : in our example; otherwise there should be a dependence between those
4873 : instructions and jump and we cannot schedule jump right now);
4874 : next, all instructions between jump and current scheduling point are moved
4875 : to this new block. And the result is this:
4876 :
4877 : NOTE BASIC BLOCK:
4878 : (!p8) jump L1 <- current scheduling point
4879 : NOTE BASIC BLOCK: <- bb header
4880 : (p8) add r14=r14+0x9;;
4881 : (p8) mov [r14]=r23
4882 : NOTE BASIC BLOCK:
4883 : ...
4884 : */
4885 : static void
4886 0 : move_cond_jump (rtx_insn *insn, bnd_t bnd)
4887 : {
4888 0 : edge ft_edge;
4889 0 : basic_block block_from, block_next, block_new, block_bnd, bb;
4890 0 : rtx_insn *next, *prev, *link, *head;
4891 :
4892 0 : block_from = BLOCK_FOR_INSN (insn);
4893 0 : block_bnd = BLOCK_FOR_INSN (BND_TO (bnd));
4894 0 : prev = BND_TO (bnd);
4895 :
4896 : /* Moving of jump should not cross any other jumps or beginnings of new
4897 : basic blocks. The only exception is when we move a jump through
4898 : mutually exclusive insns along fallthru edges. */
4899 0 : if (flag_checking && block_from != block_bnd)
4900 : {
4901 0 : bb = block_from;
4902 0 : for (link = PREV_INSN (insn); link != PREV_INSN (prev);
4903 0 : link = PREV_INSN (link))
4904 : {
4905 0 : if (INSN_P (link))
4906 0 : gcc_assert (sched_insns_conditions_mutex_p (insn, link));
4907 0 : if (BLOCK_FOR_INSN (link) && BLOCK_FOR_INSN (link) != bb)
4908 : {
4909 0 : gcc_assert (single_pred (bb) == BLOCK_FOR_INSN (link));
4910 : bb = BLOCK_FOR_INSN (link);
4911 : }
4912 : }
4913 : }
4914 :
4915 : /* Jump is moved to the boundary. */
4916 0 : next = PREV_INSN (insn);
4917 0 : BND_TO (bnd) = insn;
4918 :
4919 0 : ft_edge = find_fallthru_edge_from (block_from);
4920 0 : block_next = ft_edge->dest;
4921 : /* There must be a fallthrough block (or where should go
4922 : control flow in case of false jump predicate otherwise?). */
4923 0 : gcc_assert (block_next);
4924 :
4925 : /* Create new empty basic block after source block. */
4926 0 : block_new = sel_split_edge (ft_edge);
4927 0 : gcc_assert (block_new->next_bb == block_next
4928 : && block_from->next_bb == block_new);
4929 :
4930 : /* Move all instructions except INSN to BLOCK_NEW. */
4931 0 : bb = block_bnd;
4932 0 : head = BB_HEAD (block_new);
4933 0 : while (bb != block_from->next_bb)
4934 : {
4935 0 : rtx_insn *from, *to;
4936 0 : from = bb == block_bnd ? prev : sel_bb_head (bb);
4937 0 : to = bb == block_from ? next : sel_bb_end (bb);
4938 :
4939 : /* The jump being moved can be the first insn in the block.
4940 : In this case we don't have to move anything in this block. */
4941 0 : if (NEXT_INSN (to) != from)
4942 : {
4943 0 : reorder_insns (from, to, head);
4944 :
4945 0 : for (link = to; link != head; link = PREV_INSN (link))
4946 0 : EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index;
4947 : head = to;
4948 : }
4949 :
4950 : /* Cleanup possibly empty blocks left. */
4951 0 : block_next = bb->next_bb;
4952 0 : if (bb != block_from)
4953 0 : tidy_control_flow (bb, false);
4954 : bb = block_next;
4955 : }
4956 :
4957 : /* Assert there is no jump to BLOCK_NEW, only fallthrough edge. */
4958 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (BB_HEAD (block_new)));
4959 :
4960 0 : gcc_assert (!sel_bb_empty_p (block_from)
4961 : && !sel_bb_empty_p (block_new));
4962 :
4963 : /* Update data sets for BLOCK_NEW to represent that INSN and
4964 : instructions from the other branch of INSN is no longer
4965 : available at BLOCK_NEW. */
4966 0 : BB_AV_LEVEL (block_new) = global_level;
4967 0 : gcc_assert (BB_LV_SET (block_new) == NULL);
4968 0 : BB_LV_SET (block_new) = get_clear_regset_from_pool ();
4969 0 : update_data_sets (sel_bb_head (block_new));
4970 :
4971 : /* INSN is a new basic block header - so prepare its data
4972 : structures and update availability and liveness sets. */
4973 0 : update_data_sets (insn);
4974 :
4975 0 : if (sched_verbose >= 4)
4976 0 : sel_print ("Moving jump %d\n", INSN_UID (insn));
4977 0 : }
4978 :
4979 : /* Remove nops generated during move_op for preventing removal of empty
4980 : basic blocks. */
4981 : static void
4982 4901 : remove_temp_moveop_nops (bool full_tidying)
4983 : {
4984 4901 : int i;
4985 4901 : insn_t insn;
4986 :
4987 5241 : FOR_EACH_VEC_ELT (vec_temp_moveop_nops, i, insn)
4988 : {
4989 340 : gcc_assert (INSN_NOP_P (insn));
4990 340 : return_nop_to_pool (insn, full_tidying);
4991 : }
4992 :
4993 : /* Empty the vector. */
4994 4901 : if (vec_temp_moveop_nops.length () > 0)
4995 332 : vec_temp_moveop_nops.block_remove (0, vec_temp_moveop_nops.length ());
4996 4901 : }
4997 :
4998 : /* Records the maximal UID before moving up an instruction. Used for
4999 : distinguishing between bookkeeping copies and original insns. */
5000 : static int max_uid_before_move_op = 0;
5001 :
5002 : /* When true, we're always scheduling next insn on the already scheduled code
5003 : to get the right insn data for the following bundling or other passes. */
5004 : static int force_next_insn = 0;
5005 :
5006 : /* Remove from AV_VLIW_P all instructions but next when debug counter
5007 : tells us so. Next instruction is fetched from BNDS. */
5008 : static void
5009 5189 : remove_insns_for_debug (blist_t bnds, av_set_t *av_vliw_p)
5010 : {
5011 5189 : if (! dbg_cnt (sel_sched_insn_cnt) || force_next_insn)
5012 : /* Leave only the next insn in av_vliw. */
5013 : {
5014 0 : av_set_iterator av_it;
5015 0 : expr_t expr;
5016 0 : bnd_t bnd = BLIST_BND (bnds);
5017 0 : insn_t next = BND_TO (bnd);
5018 :
5019 0 : gcc_assert (BLIST_NEXT (bnds) == NULL);
5020 :
5021 0 : FOR_EACH_EXPR_1 (expr, av_it, av_vliw_p)
5022 0 : if (EXPR_INSN_RTX (expr) != next)
5023 0 : av_set_iter_remove (&av_it);
5024 : }
5025 5189 : }
5026 :
5027 : /* Compute available instructions on BNDS. FENCE is the current fence. Write
5028 : the computed set to *AV_VLIW_P. */
5029 : static void
5030 5189 : compute_av_set_on_boundaries (fence_t fence, blist_t bnds, av_set_t *av_vliw_p)
5031 : {
5032 5189 : if (sched_verbose >= 2)
5033 : {
5034 0 : sel_print ("Boundaries: ");
5035 0 : dump_blist (bnds);
5036 0 : sel_print ("\n");
5037 : }
5038 :
5039 10378 : for (; bnds; bnds = BLIST_NEXT (bnds))
5040 : {
5041 5189 : bnd_t bnd = BLIST_BND (bnds);
5042 5189 : av_set_t av1_copy;
5043 5189 : insn_t bnd_to = BND_TO (bnd);
5044 :
5045 : /* Rewind BND->TO to the basic block header in case some bookkeeping
5046 : instructions were inserted before BND->TO and it needs to be
5047 : adjusted. */
5048 5189 : if (sel_bb_head_p (bnd_to))
5049 1051 : gcc_assert (INSN_SCHED_TIMES (bnd_to) == 0);
5050 : else
5051 4138 : while (INSN_SCHED_TIMES (PREV_INSN (bnd_to)) == 0)
5052 : {
5053 0 : bnd_to = PREV_INSN (bnd_to);
5054 0 : if (sel_bb_head_p (bnd_to))
5055 : break;
5056 : }
5057 :
5058 5189 : if (BND_TO (bnd) != bnd_to)
5059 : {
5060 0 : gcc_assert (FENCE_INSN (fence) == BND_TO (bnd));
5061 0 : FENCE_INSN (fence) = bnd_to;
5062 0 : BND_TO (bnd) = bnd_to;
5063 : }
5064 :
5065 5189 : av_set_clear (&BND_AV (bnd));
5066 5189 : BND_AV (bnd) = compute_av_set (BND_TO (bnd), NULL, 0, true);
5067 :
5068 5189 : av_set_clear (&BND_AV1 (bnd));
5069 5189 : BND_AV1 (bnd) = av_set_copy (BND_AV (bnd));
5070 :
5071 5189 : moveup_set_inside_insn_group (&BND_AV1 (bnd), NULL);
5072 :
5073 5189 : av1_copy = av_set_copy (BND_AV1 (bnd));
5074 5189 : av_set_union_and_clear (av_vliw_p, &av1_copy, NULL);
5075 : }
5076 :
5077 5189 : if (sched_verbose >= 2)
5078 : {
5079 0 : sel_print ("Available exprs (vliw form): ");
5080 0 : dump_av_set (*av_vliw_p);
5081 0 : sel_print ("\n");
5082 : }
5083 5189 : }
5084 :
5085 : /* Calculate the sequential av set on BND corresponding to the EXPR_VLIW
5086 : expression. When FOR_MOVEOP is true, also replace the register of
5087 : expressions found with the register from EXPR_VLIW. */
5088 : static av_set_t
5089 7691 : find_sequential_best_exprs (bnd_t bnd, expr_t expr_vliw, bool for_moveop)
5090 : {
5091 7691 : av_set_t expr_seq = NULL;
5092 7691 : expr_t expr;
5093 7691 : av_set_iterator i;
5094 :
5095 17841 : FOR_EACH_EXPR (expr, i, BND_AV (bnd))
5096 : {
5097 17841 : if (equal_after_moveup_path_p (expr, NULL, expr_vliw))
5098 : {
5099 7691 : if (for_moveop)
5100 : {
5101 : /* The sequential expression has the right form to pass
5102 : to move_op except when renaming happened. Put the
5103 : correct register in EXPR then. */
5104 4901 : if (EXPR_SEPARABLE_P (expr) && REG_P (EXPR_LHS (expr)))
5105 : {
5106 1260 : if (expr_dest_regno (expr) != expr_dest_regno (expr_vliw))
5107 : {
5108 26 : replace_dest_with_reg_in_expr (expr, EXPR_LHS (expr_vliw));
5109 26 : stat_renamed_scheduled++;
5110 : }
5111 : /* Also put the correct TARGET_AVAILABLE bit on the expr.
5112 : This is needed when renaming came up with original
5113 : register. */
5114 1234 : else if (EXPR_TARGET_AVAILABLE (expr)
5115 1234 : != EXPR_TARGET_AVAILABLE (expr_vliw))
5116 : {
5117 2 : gcc_assert (EXPR_TARGET_AVAILABLE (expr_vliw) == 1);
5118 2 : EXPR_TARGET_AVAILABLE (expr) = 1;
5119 : }
5120 : }
5121 4901 : if (EXPR_WAS_SUBSTITUTED (expr))
5122 0 : stat_substitutions_total++;
5123 : }
5124 :
5125 7691 : av_set_add (&expr_seq, expr);
5126 :
5127 : /* With substitution inside insn group, it is possible
5128 : that more than one expression in expr_seq will correspond
5129 : to expr_vliw. In this case, choose one as the attempt to
5130 : move both leads to miscompiles. */
5131 7691 : break;
5132 : }
5133 : }
5134 :
5135 7691 : if (for_moveop && sched_verbose >= 2)
5136 : {
5137 0 : sel_print ("Best expression(s) (sequential form): ");
5138 0 : dump_av_set (expr_seq);
5139 0 : sel_print ("\n");
5140 : }
5141 :
5142 7691 : return expr_seq;
5143 : }
5144 :
5145 :
5146 : /* Move nop to previous block. */
5147 : static void ATTRIBUTE_UNUSED
5148 0 : move_nop_to_previous_block (insn_t nop, basic_block prev_bb)
5149 : {
5150 0 : insn_t prev_insn, next_insn;
5151 :
5152 0 : gcc_assert (sel_bb_head_p (nop)
5153 : && prev_bb == BLOCK_FOR_INSN (nop)->prev_bb);
5154 0 : rtx_note *note = bb_note (BLOCK_FOR_INSN (nop));
5155 0 : prev_insn = sel_bb_end (prev_bb);
5156 0 : next_insn = NEXT_INSN (nop);
5157 0 : gcc_assert (prev_insn != NULL_RTX
5158 : && PREV_INSN (note) == prev_insn);
5159 :
5160 0 : SET_NEXT_INSN (prev_insn) = nop;
5161 0 : SET_PREV_INSN (nop) = prev_insn;
5162 :
5163 0 : SET_PREV_INSN (note) = nop;
5164 0 : SET_NEXT_INSN (note) = next_insn;
5165 :
5166 0 : SET_NEXT_INSN (nop) = note;
5167 0 : SET_PREV_INSN (next_insn) = note;
5168 :
5169 0 : BB_END (prev_bb) = nop;
5170 0 : BLOCK_FOR_INSN (nop) = prev_bb;
5171 0 : }
5172 :
5173 : /* Prepare a place to insert the chosen expression on BND. */
5174 : static insn_t
5175 4901 : prepare_place_to_insert (bnd_t bnd)
5176 : {
5177 4901 : insn_t place_to_insert;
5178 :
5179 : /* Init place_to_insert before calling move_op, as the later
5180 : can possibly remove BND_TO (bnd). */
5181 4901 : if (/* If this is not the first insn scheduled. */
5182 4901 : BND_PTR (bnd))
5183 : {
5184 : /* Add it after last scheduled. */
5185 3215 : place_to_insert = ILIST_INSN (BND_PTR (bnd));
5186 3215 : if (DEBUG_INSN_P (place_to_insert))
5187 : {
5188 : ilist_t l = BND_PTR (bnd);
5189 162 : while ((l = ILIST_NEXT (l)) &&
5190 42 : DEBUG_INSN_P (ILIST_INSN (l)))
5191 : ;
5192 44 : if (!l)
5193 : place_to_insert = NULL;
5194 : }
5195 : }
5196 : else
5197 : place_to_insert = NULL;
5198 :
5199 : if (!place_to_insert)
5200 : {
5201 : /* Add it before BND_TO. The difference is in the
5202 : basic block, where INSN will be added. */
5203 1725 : place_to_insert = get_nop_from_pool (BND_TO (bnd));
5204 1725 : gcc_assert (BLOCK_FOR_INSN (place_to_insert)
5205 : == BLOCK_FOR_INSN (BND_TO (bnd)));
5206 : }
5207 :
5208 4901 : return place_to_insert;
5209 : }
5210 :
5211 : /* Find original instructions for EXPR_SEQ and move it to BND boundary.
5212 : Return the expression to emit in C_EXPR. */
5213 : static bool
5214 4901 : move_exprs_to_boundary (bnd_t bnd, expr_t expr_vliw,
5215 : av_set_t expr_seq, expr_t c_expr)
5216 : {
5217 4901 : bool b, should_move;
5218 4901 : unsigned book_uid;
5219 4901 : bitmap_iterator bi;
5220 4901 : int n_bookkeeping_copies_before_moveop;
5221 :
5222 : /* Make a move. This call will remove the original operation,
5223 : insert all necessary bookkeeping instructions and update the
5224 : data sets. After that all we have to do is add the operation
5225 : at before BND_TO (BND). */
5226 4901 : n_bookkeeping_copies_before_moveop = stat_bookkeeping_copies;
5227 4901 : max_uid_before_move_op = get_max_uid ();
5228 4901 : bitmap_clear (current_copies);
5229 4901 : bitmap_clear (current_originators);
5230 :
5231 4901 : b = move_op (BND_TO (bnd), expr_seq, expr_vliw,
5232 : get_dest_from_orig_ops (expr_seq), c_expr, &should_move);
5233 :
5234 : /* We should be able to find the expression we've chosen for
5235 : scheduling. */
5236 4901 : gcc_assert (b);
5237 :
5238 4901 : if (stat_bookkeeping_copies > n_bookkeeping_copies_before_moveop)
5239 136 : stat_insns_needed_bookkeeping++;
5240 :
5241 5114 : EXECUTE_IF_SET_IN_BITMAP (current_copies, 0, book_uid, bi)
5242 : {
5243 213 : unsigned uid;
5244 213 : bitmap_iterator bi;
5245 :
5246 : /* We allocate these bitmaps lazily. */
5247 213 : if (! INSN_ORIGINATORS_BY_UID (book_uid))
5248 213 : INSN_ORIGINATORS_BY_UID (book_uid) = BITMAP_ALLOC (NULL);
5249 :
5250 213 : bitmap_copy (INSN_ORIGINATORS_BY_UID (book_uid),
5251 : current_originators);
5252 :
5253 : /* Transitively add all originators' originators. */
5254 998 : EXECUTE_IF_SET_IN_BITMAP (current_originators, 0, uid, bi)
5255 785 : if (INSN_ORIGINATORS_BY_UID (uid))
5256 265 : bitmap_ior_into (INSN_ORIGINATORS_BY_UID (book_uid),
5257 265 : INSN_ORIGINATORS_BY_UID (uid));
5258 : }
5259 :
5260 4901 : return should_move;
5261 : }
5262 :
5263 :
5264 : /* Debug a DFA state as an array of bytes. */
5265 : static void
5266 0 : debug_state (state_t state)
5267 : {
5268 0 : unsigned char *p;
5269 0 : unsigned int i, size = dfa_state_size;
5270 :
5271 0 : sel_print ("state (%u):", size);
5272 0 : for (i = 0, p = (unsigned char *) state; i < size; i++)
5273 0 : sel_print (" %d", p[i]);
5274 0 : sel_print ("\n");
5275 0 : }
5276 :
5277 : /* Advance state on FENCE with INSN. Return true if INSN is
5278 : an ASM, and we should advance state once more. */
5279 : static bool
5280 4901 : advance_state_on_fence (fence_t fence, insn_t insn)
5281 : {
5282 4901 : bool asm_p;
5283 :
5284 4901 : if (recog_memoized (insn) >= 0)
5285 : {
5286 4792 : int res;
5287 4792 : state_t temp_state = alloca (dfa_state_size);
5288 :
5289 4792 : gcc_assert (!INSN_ASM_P (insn));
5290 4792 : asm_p = false;
5291 :
5292 4792 : memcpy (temp_state, FENCE_STATE (fence), dfa_state_size);
5293 4792 : res = state_transition (FENCE_STATE (fence), insn);
5294 4792 : gcc_assert (res < 0);
5295 :
5296 4792 : if (memcmp (temp_state, FENCE_STATE (fence), dfa_state_size))
5297 : {
5298 4722 : FENCE_ISSUED_INSNS (fence)++;
5299 :
5300 : /* We should never issue more than issue_rate insns. */
5301 4722 : if (FENCE_ISSUED_INSNS (fence) > issue_rate)
5302 0 : gcc_unreachable ();
5303 : }
5304 : }
5305 : else
5306 : {
5307 : /* This could be an ASM insn which we'd like to schedule
5308 : on the next cycle. */
5309 109 : asm_p = INSN_ASM_P (insn);
5310 109 : if (!FENCE_STARTS_CYCLE_P (fence) && asm_p)
5311 9 : advance_one_cycle (fence);
5312 : }
5313 :
5314 4901 : if (sched_verbose >= 2)
5315 0 : debug_state (FENCE_STATE (fence));
5316 4901 : if (!DEBUG_INSN_P (insn))
5317 4857 : FENCE_STARTS_CYCLE_P (fence) = 0;
5318 4901 : FENCE_ISSUE_MORE (fence) = can_issue_more;
5319 4901 : return asm_p;
5320 : }
5321 :
5322 : /* Update FENCE on which INSN was scheduled and this INSN, too. NEED_STALL
5323 : is nonzero if we need to stall after issuing INSN. */
5324 : static void
5325 4901 : update_fence_and_insn (fence_t fence, insn_t insn, int need_stall)
5326 : {
5327 4901 : bool asm_p;
5328 :
5329 : /* First, reflect that something is scheduled on this fence. */
5330 4901 : asm_p = advance_state_on_fence (fence, insn);
5331 4901 : FENCE_LAST_SCHEDULED_INSN (fence) = insn;
5332 4901 : vec_safe_push (FENCE_EXECUTING_INSNS (fence), insn);
5333 4901 : if (SCHED_GROUP_P (insn))
5334 : {
5335 361 : FENCE_SCHED_NEXT (fence) = INSN_SCHED_NEXT (insn);
5336 361 : SCHED_GROUP_P (insn) = 0;
5337 : }
5338 : else
5339 4540 : FENCE_SCHED_NEXT (fence) = NULL;
5340 4901 : if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
5341 4771 : FENCE_READY_TICKS (fence) [INSN_UID (insn)] = 0;
5342 :
5343 : /* Set instruction scheduling info. This will be used in bundling,
5344 : pipelining, tick computations etc. */
5345 4901 : ++INSN_SCHED_TIMES (insn);
5346 4901 : EXPR_TARGET_AVAILABLE (INSN_EXPR (insn)) = true;
5347 4901 : EXPR_ORIG_SCHED_CYCLE (INSN_EXPR (insn)) = FENCE_CYCLE (fence);
5348 4901 : INSN_AFTER_STALL_P (insn) = FENCE_AFTER_STALL_P (fence);
5349 4901 : INSN_SCHED_CYCLE (insn) = FENCE_CYCLE (fence);
5350 :
5351 : /* This does not account for adjust_cost hooks, just add the biggest
5352 : constant the hook may add to the latency. TODO: make this
5353 : a target dependent constant. */
5354 4901 : INSN_READY_CYCLE (insn)
5355 9802 : = INSN_SCHED_CYCLE (insn) + (INSN_CODE (insn) < 0
5356 9693 : ? 1
5357 4792 : : maximal_insn_latency (insn) + 1);
5358 :
5359 : /* Change these fields last, as they're used above. */
5360 4901 : FENCE_AFTER_STALL_P (fence) = 0;
5361 4901 : if (asm_p || need_stall)
5362 10 : advance_one_cycle (fence);
5363 :
5364 : /* Indicate that we've scheduled something on this fence. */
5365 4901 : FENCE_SCHEDULED_P (fence) = true;
5366 4901 : scheduled_something_on_previous_fence = true;
5367 :
5368 : /* Print debug information when insn's fields are updated. */
5369 4901 : if (sched_verbose >= 2)
5370 : {
5371 0 : sel_print ("Scheduling insn: ");
5372 0 : dump_insn_1 (insn, 1);
5373 0 : sel_print ("\n");
5374 : }
5375 4901 : }
5376 :
5377 : /* Update boundary BND (and, if needed, FENCE) with INSN, remove the
5378 : old boundary from BNDSP, add new boundaries to BNDS_TAIL_P and
5379 : return it. */
5380 : static blist_t *
5381 4901 : update_boundaries (fence_t fence, bnd_t bnd, insn_t insn, blist_t *bndsp,
5382 : blist_t *bnds_tailp)
5383 : {
5384 4901 : succ_iterator si;
5385 4901 : insn_t succ;
5386 :
5387 4901 : advance_deps_context (BND_DC (bnd), insn);
5388 9149 : FOR_EACH_SUCC_1 (succ, si, insn,
5389 : SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
5390 : {
5391 4248 : ilist_t ptr = ilist_copy (BND_PTR (bnd));
5392 :
5393 4248 : ilist_add (&ptr, insn);
5394 :
5395 44 : if (DEBUG_INSN_P (insn) && sel_bb_end_p (insn)
5396 4248 : && is_ineligible_successor (succ, ptr))
5397 : {
5398 0 : ilist_clear (&ptr);
5399 0 : continue;
5400 : }
5401 :
5402 4248 : if (FENCE_INSN (fence) == insn && !sel_bb_end_p (insn))
5403 : {
5404 2806 : if (sched_verbose >= 9)
5405 0 : sel_print ("Updating fence insn from %i to %i\n",
5406 0 : INSN_UID (insn), INSN_UID (succ));
5407 2806 : FENCE_INSN (fence) = succ;
5408 : }
5409 4248 : blist_add (bnds_tailp, succ, ptr, BND_DC (bnd));
5410 4248 : bnds_tailp = &BLIST_NEXT (*bnds_tailp);
5411 : }
5412 :
5413 4901 : blist_remove (bndsp);
5414 4901 : return bnds_tailp;
5415 : }
5416 :
5417 : /* Schedule EXPR_VLIW on BND. Return the insn emitted. */
5418 : static insn_t
5419 4901 : schedule_expr_on_boundary (bnd_t bnd, expr_t expr_vliw, int seqno)
5420 : {
5421 4901 : av_set_t expr_seq;
5422 4901 : expr_t c_expr = XALLOCA (expr_def);
5423 4901 : insn_t place_to_insert;
5424 4901 : insn_t insn;
5425 4901 : bool should_move;
5426 :
5427 4901 : expr_seq = find_sequential_best_exprs (bnd, expr_vliw, true);
5428 :
5429 : /* In case of scheduling a jump skipping some other instructions,
5430 : prepare CFG. After this, jump is at the boundary and can be
5431 : scheduled as usual insn by MOVE_OP. */
5432 4901 : if (vinsn_cond_branch_p (EXPR_VINSN (expr_vliw)))
5433 : {
5434 821 : insn = EXPR_INSN_RTX (expr_vliw);
5435 :
5436 : /* Speculative jumps are not handled. */
5437 821 : if (insn != BND_TO (bnd)
5438 821 : && !sel_insn_is_speculation_check (insn))
5439 0 : move_cond_jump (insn, bnd);
5440 : }
5441 :
5442 : /* Find a place for C_EXPR to schedule. */
5443 4901 : place_to_insert = prepare_place_to_insert (bnd);
5444 4901 : should_move = move_exprs_to_boundary (bnd, expr_vliw, expr_seq, c_expr);
5445 4901 : clear_expr (c_expr);
5446 :
5447 : /* Add the instruction. The corner case to care about is when
5448 : the expr_seq set has more than one expr, and we chose the one that
5449 : is not equal to expr_vliw. Then expr_vliw may be insn in stream, and
5450 : we can't use it. Generate the new vinsn. */
5451 4901 : if (INSN_IN_STREAM_P (EXPR_INSN_RTX (expr_vliw)))
5452 : {
5453 0 : vinsn_t vinsn_new;
5454 :
5455 0 : vinsn_new = vinsn_copy (EXPR_VINSN (expr_vliw), false);
5456 0 : change_vinsn_in_expr (expr_vliw, vinsn_new);
5457 0 : should_move = false;
5458 : }
5459 4901 : if (should_move)
5460 4871 : insn = sel_move_insn (expr_vliw, seqno, place_to_insert);
5461 : else
5462 30 : insn = emit_insn_from_expr_after (expr_vliw, NULL, seqno,
5463 : place_to_insert);
5464 :
5465 : /* Return the nops generated for preserving of data sets back
5466 : into pool. */
5467 4901 : if (INSN_NOP_P (place_to_insert))
5468 1725 : return_nop_to_pool (place_to_insert, !DEBUG_INSN_P (insn));
5469 4901 : remove_temp_moveop_nops (!DEBUG_INSN_P (insn));
5470 :
5471 4901 : av_set_clear (&expr_seq);
5472 :
5473 : /* Save the expression scheduled so to reset target availability if we'll
5474 : meet it later on the same fence. */
5475 4901 : if (EXPR_WAS_RENAMED (expr_vliw))
5476 28 : vinsn_vec_add (&vec_target_unavailable_vinsns, INSN_EXPR (insn));
5477 :
5478 : /* Check that the recent movement didn't destroyed loop
5479 : structure. */
5480 4901 : gcc_assert (!pipelining_p
5481 : || current_loop_nest == NULL
5482 : || loop_latch_edge (current_loop_nest));
5483 4901 : return insn;
5484 : }
5485 :
5486 : /* Stall for N cycles on FENCE. */
5487 : static void
5488 2827 : stall_for_cycles (fence_t fence, int n)
5489 : {
5490 2827 : int could_more;
5491 :
5492 2827 : could_more = n > 1 || FENCE_ISSUED_INSNS (fence) < issue_rate;
5493 6156 : while (n--)
5494 3329 : advance_one_cycle (fence);
5495 2827 : if (could_more)
5496 2666 : FENCE_AFTER_STALL_P (fence) = 1;
5497 2827 : }
5498 :
5499 : /* Gather a parallel group of insns at FENCE and assign their seqno
5500 : to SEQNO. All scheduled insns are gathered in SCHEDULED_INSNS_TAILPP
5501 : list for later recalculation of seqnos. */
5502 : static void
5503 1878 : fill_insns (fence_t fence, int seqno, ilist_t **scheduled_insns_tailpp)
5504 : {
5505 1878 : blist_t bnds = NULL, *bnds_tailp;
5506 1878 : av_set_t av_vliw = NULL;
5507 1878 : insn_t insn = FENCE_INSN (fence);
5508 :
5509 1878 : if (sched_verbose >= 2)
5510 0 : sel_print ("Starting fill_insns for insn %d, cycle %d\n",
5511 0 : INSN_UID (insn), FENCE_CYCLE (fence));
5512 :
5513 1878 : blist_add (&bnds, insn, NULL, FENCE_DC (fence));
5514 1878 : bnds_tailp = &BLIST_NEXT (bnds);
5515 1878 : set_target_context (FENCE_TC (fence));
5516 1878 : can_issue_more = FENCE_ISSUE_MORE (fence);
5517 1878 : target_bb = INSN_BB (insn);
5518 :
5519 : /* Do while we can add any operation to the current group. */
5520 5189 : do
5521 : {
5522 5189 : blist_t *bnds_tailp1, *bndsp;
5523 5189 : expr_t expr_vliw;
5524 5189 : int need_stall = false;
5525 5189 : int was_stall = 0, scheduled_insns = 0;
5526 5189 : int max_insns = pipelining_p ? issue_rate : 2 * issue_rate;
5527 5189 : int max_stall = pipelining_p ? 1 : 3;
5528 5189 : bool last_insn_was_debug = false;
5529 5189 : bool was_debug_bb_end_p = false;
5530 :
5531 5189 : compute_av_set_on_boundaries (fence, bnds, &av_vliw);
5532 5189 : remove_insns_that_need_bookkeeping (fence, &av_vliw);
5533 5189 : remove_insns_for_debug (bnds, &av_vliw);
5534 :
5535 : /* Return early if we have nothing to schedule. */
5536 5189 : if (av_vliw == NULL)
5537 : break;
5538 :
5539 : /* Choose the best expression and, if needed, destination register
5540 : for it. */
5541 7728 : do
5542 : {
5543 7728 : expr_vliw = find_best_expr (&av_vliw, bnds, fence, &need_stall);
5544 7728 : if (! expr_vliw && need_stall)
5545 : {
5546 : /* All expressions required a stall. Do not recompute av sets
5547 : as we'll get the same answer (modulo the insns between
5548 : the fence and its boundary, which will not be available for
5549 : pipelining).
5550 : If we are going to stall for too long, break to recompute av
5551 : sets and bring more insns for pipelining. */
5552 2827 : was_stall++;
5553 2827 : if (need_stall <= 3)
5554 2572 : stall_for_cycles (fence, need_stall);
5555 : else
5556 : {
5557 255 : stall_for_cycles (fence, 1);
5558 255 : break;
5559 : }
5560 : }
5561 : }
5562 2572 : while (! expr_vliw && need_stall);
5563 :
5564 : /* Now either we've selected expr_vliw or we have nothing to schedule. */
5565 5156 : if (!expr_vliw)
5566 : {
5567 255 : av_set_clear (&av_vliw);
5568 255 : break;
5569 : }
5570 :
5571 : bndsp = &bnds;
5572 : bnds_tailp1 = bnds_tailp;
5573 :
5574 9149 : do
5575 : /* This code will be executed only once until we'd have several
5576 : boundaries per fence. */
5577 : {
5578 9149 : bnd_t bnd = BLIST_BND (*bndsp);
5579 :
5580 9149 : if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr_vliw)))
5581 : {
5582 4248 : bndsp = &BLIST_NEXT (*bndsp);
5583 4248 : continue;
5584 : }
5585 :
5586 4901 : insn = schedule_expr_on_boundary (bnd, expr_vliw, seqno);
5587 4901 : last_insn_was_debug = DEBUG_INSN_P (insn);
5588 4901 : if (last_insn_was_debug)
5589 44 : was_debug_bb_end_p = (insn == BND_TO (bnd) && sel_bb_end_p (insn));
5590 4901 : update_fence_and_insn (fence, insn, need_stall);
5591 4901 : bnds_tailp = update_boundaries (fence, bnd, insn, bndsp, bnds_tailp);
5592 :
5593 : /* Add insn to the list of scheduled on this cycle instructions. */
5594 4901 : ilist_add (*scheduled_insns_tailpp, insn);
5595 4901 : *scheduled_insns_tailpp = &ILIST_NEXT (**scheduled_insns_tailpp);
5596 : }
5597 9149 : while (*bndsp != *bnds_tailp1);
5598 :
5599 4901 : av_set_clear (&av_vliw);
5600 4901 : if (!last_insn_was_debug)
5601 4857 : scheduled_insns++;
5602 :
5603 : /* We currently support information about candidate blocks only for
5604 : one 'target_bb' block. Hence we can't schedule after jump insn,
5605 : as this will bring two boundaries and, hence, necessity to handle
5606 : information for two or more blocks concurrently. */
5607 4857 : if ((last_insn_was_debug ? was_debug_bb_end_p : sel_bb_end_p (insn))
5608 3851 : || (was_stall
5609 1762 : && (was_stall >= max_stall
5610 1762 : || scheduled_insns >= max_insns)))
5611 : break;
5612 : }
5613 3311 : while (bnds);
5614 :
5615 1878 : gcc_assert (!FENCE_BNDS (fence));
5616 :
5617 : /* Update boundaries of the FENCE. */
5618 3103 : while (bnds)
5619 : {
5620 1225 : ilist_t ptr = BND_PTR (BLIST_BND (bnds));
5621 :
5622 1225 : if (ptr)
5623 : {
5624 1033 : insn = ILIST_INSN (ptr);
5625 :
5626 2066 : if (!ilist_is_in_p (FENCE_BNDS (fence), insn))
5627 958 : ilist_add (&FENCE_BNDS (fence), insn);
5628 : }
5629 :
5630 1225 : blist_remove (&bnds);
5631 : }
5632 :
5633 : /* Update target context on the fence. */
5634 1878 : reset_target_context (FENCE_TC (fence), false);
5635 1878 : }
5636 :
5637 : /* All exprs in ORIG_OPS must have the same destination register or memory.
5638 : Return that destination. */
5639 : static rtx
5640 4901 : get_dest_from_orig_ops (av_set_t orig_ops)
5641 : {
5642 4901 : rtx dest = NULL_RTX;
5643 4901 : av_set_iterator av_it;
5644 4901 : expr_t expr;
5645 4901 : bool first_p = true;
5646 :
5647 9802 : FOR_EACH_EXPR (expr, av_it, orig_ops)
5648 : {
5649 4901 : rtx x = EXPR_LHS (expr);
5650 :
5651 4901 : if (first_p)
5652 : {
5653 : first_p = false;
5654 : dest = x;
5655 : }
5656 : else
5657 0 : gcc_assert (dest == x
5658 : || (dest != NULL_RTX && x != NULL_RTX
5659 : && rtx_equal_p (dest, x)));
5660 : }
5661 :
5662 4901 : return dest;
5663 : }
5664 :
5665 : /* Update data sets for the bookkeeping block and record those expressions
5666 : which become no longer available after inserting this bookkeeping. */
5667 : static void
5668 389 : update_and_record_unavailable_insns (basic_block book_block)
5669 : {
5670 389 : av_set_iterator i;
5671 389 : av_set_t old_av_set = NULL;
5672 389 : expr_t cur_expr;
5673 389 : rtx_insn *bb_end = sel_bb_end (book_block);
5674 :
5675 : /* First, get correct liveness in the bookkeeping block. The problem is
5676 : the range between the bookeeping insn and the end of block. */
5677 389 : update_liveness_on_insn (bb_end);
5678 389 : if (control_flow_insn_p (bb_end))
5679 8 : update_liveness_on_insn (PREV_INSN (bb_end));
5680 :
5681 : /* If there's valid av_set on BOOK_BLOCK, then there might exist another
5682 : fence above, where we may choose to schedule an insn which is
5683 : actually blocked from moving up with the bookkeeping we create here. */
5684 389 : if (AV_SET_VALID_P (sel_bb_head (book_block)))
5685 : {
5686 320 : old_av_set = av_set_copy (BB_AV_SET (book_block));
5687 320 : update_data_sets (sel_bb_head (book_block));
5688 :
5689 : /* Traverse all the expressions in the old av_set and check whether
5690 : CUR_EXPR is in new AV_SET. */
5691 1774 : FOR_EACH_EXPR (cur_expr, i, old_av_set)
5692 : {
5693 1134 : expr_t new_expr = av_set_lookup (BB_AV_SET (book_block),
5694 : EXPR_VINSN (cur_expr));
5695 :
5696 1134 : if (! new_expr
5697 : /* In this case, we can just turn off the E_T_A bit, but we can't
5698 : represent this information with the current vector. */
5699 1119 : || EXPR_TARGET_AVAILABLE (new_expr)
5700 1119 : != EXPR_TARGET_AVAILABLE (cur_expr))
5701 : /* Unfortunately, the below code could be also fired up on
5702 : separable insns, e.g. when moving insns through the new
5703 : speculation check as in PR 53701. */
5704 22 : vinsn_vec_add (&vec_bookkeeping_blocked_vinsns, cur_expr);
5705 : }
5706 :
5707 320 : av_set_clear (&old_av_set);
5708 : }
5709 389 : }
5710 :
5711 : /* The main effect of this function is that sparams->c_expr is merged
5712 : with (or copied to) lparams->c_expr_merged. If there's only one successor,
5713 : we avoid merging anything by copying sparams->c_expr to lparams->c_expr_merged.
5714 : lparams->c_expr_merged is copied back to sparams->c_expr after all
5715 : successors has been traversed. lparams->c_expr_local is an expr allocated
5716 : on stack in the caller function, and is used if there is more than one
5717 : successor.
5718 :
5719 : SUCC is one of the SUCCS_NORMAL successors of INSN,
5720 : MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ,
5721 : LPARAMS and STATIC_PARAMS contain the parameters described above. */
5722 : static void
5723 1002 : move_op_merge_succs (insn_t insn ATTRIBUTE_UNUSED,
5724 : insn_t succ ATTRIBUTE_UNUSED,
5725 : int moveop_drv_call_res,
5726 : cmpd_local_params_p lparams, void *static_params)
5727 : {
5728 1002 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
5729 :
5730 : /* Nothing to do, if original expr wasn't found below. */
5731 1002 : if (moveop_drv_call_res != 1)
5732 : return;
5733 :
5734 : /* If this is a first successor. */
5735 905 : if (!lparams->c_expr_merged)
5736 : {
5737 651 : lparams->c_expr_merged = sparams->c_expr;
5738 651 : sparams->c_expr = lparams->c_expr_local;
5739 : }
5740 : else
5741 : {
5742 : /* We must merge all found expressions to get reasonable
5743 : EXPR_SPEC_DONE_DS for the resulting insn. If we don't
5744 : do so then we can first find the expr with epsilon
5745 : speculation success probability and only then with the
5746 : good probability. As a result the insn will get epsilon
5747 : probability and will never be scheduled because of
5748 : weakness_cutoff in find_best_expr.
5749 :
5750 : We call merge_expr_data here instead of merge_expr
5751 : because due to speculation C_EXPR and X may have the
5752 : same insns with different speculation types. And as of
5753 : now such insns are considered non-equal.
5754 :
5755 : However, EXPR_SCHED_TIMES is different -- we must get
5756 : SCHED_TIMES from a real insn, not a bookkeeping copy.
5757 : We force this here. Instead, we may consider merging
5758 : SCHED_TIMES to the maximum instead of minimum in the
5759 : below function. */
5760 254 : int old_times = EXPR_SCHED_TIMES (lparams->c_expr_merged);
5761 :
5762 254 : merge_expr_data (lparams->c_expr_merged, sparams->c_expr, NULL);
5763 254 : if (EXPR_SCHED_TIMES (sparams->c_expr) == 0)
5764 254 : EXPR_SCHED_TIMES (lparams->c_expr_merged) = old_times;
5765 :
5766 254 : clear_expr (sparams->c_expr);
5767 : }
5768 : }
5769 :
5770 : /* Add used regs for the successor SUCC into SPARAMS->USED_REGS.
5771 :
5772 : SUCC is one of the SUCCS_NORMAL successors of INSN,
5773 : MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ or 0,
5774 : if SUCC is one of SUCCS_BACK or SUCCS_OUT.
5775 : STATIC_PARAMS contain USED_REGS set. */
5776 : static void
5777 9985 : fur_merge_succs (insn_t insn ATTRIBUTE_UNUSED, insn_t succ,
5778 : int moveop_drv_call_res,
5779 : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5780 : void *static_params)
5781 : {
5782 9985 : regset succ_live;
5783 9985 : fur_static_params_p sparams = (fur_static_params_p) static_params;
5784 :
5785 : /* Here we compute live regsets only for branches that do not lie
5786 : on the code motion paths. These branches correspond to value
5787 : MOVEOP_DRV_CALL_RES==0 and include SUCCS_BACK and SUCCS_OUT, though
5788 : for such branches code_motion_path_driver is not called. */
5789 9985 : if (moveop_drv_call_res != 0)
5790 : return;
5791 :
5792 : /* Mark all registers that do not meet the following condition:
5793 : (3) not live on the other path of any conditional branch
5794 : that is passed by the operation, in case original
5795 : operations are not present on both paths of the
5796 : conditional branch. */
5797 2934 : succ_live = compute_live (succ);
5798 2934 : IOR_REG_SET (sparams->used_regs, succ_live);
5799 : }
5800 :
5801 : /* This function is called after the last successor. Copies LP->C_EXPR_MERGED
5802 : into SP->CEXPR. */
5803 : static void
5804 651 : move_op_after_merge_succs (cmpd_local_params_p lp, void *sparams)
5805 : {
5806 651 : moveop_static_params_p sp = (moveop_static_params_p) sparams;
5807 :
5808 651 : sp->c_expr = lp->c_expr_merged;
5809 651 : }
5810 :
5811 : /* Track bookkeeping copies created, insns scheduled, and blocks for
5812 : rescheduling when INSN is found by move_op. */
5813 : static void
5814 5155 : track_scheduled_insns_and_blocks (rtx_insn *insn)
5815 : {
5816 : /* Even if this insn can be a copy that will be removed during current move_op,
5817 : we still need to count it as an originator. */
5818 5155 : bitmap_set_bit (current_originators, INSN_UID (insn));
5819 :
5820 5155 : if (!bitmap_clear_bit (current_copies, INSN_UID (insn)))
5821 : {
5822 : /* Note that original block needs to be rescheduled, as we pulled an
5823 : instruction out of it. */
5824 4979 : if (INSN_SCHED_TIMES (insn) > 0)
5825 80 : bitmap_set_bit (blocks_to_reschedule, BLOCK_FOR_INSN (insn)->index);
5826 4899 : else if (INSN_UID (insn) < first_emitted_uid && !DEBUG_INSN_P (insn))
5827 4681 : num_insns_scheduled++;
5828 : }
5829 :
5830 : /* For instructions we must immediately remove insn from the
5831 : stream, so subsequent update_data_sets () won't include this
5832 : insn into av_set.
5833 : For expr we must make insn look like "INSN_REG (insn) := c_expr". */
5834 5155 : if (INSN_UID (insn) > max_uid_before_move_op)
5835 174 : stat_bookkeeping_copies--;
5836 5155 : }
5837 :
5838 : /* Emit a register-register copy for INSN if needed. Return true if
5839 : emitted one. PARAMS is the move_op static parameters. */
5840 : static bool
5841 5155 : maybe_emit_renaming_copy (rtx_insn *insn,
5842 : moveop_static_params_p params)
5843 : {
5844 5155 : bool insn_emitted = false;
5845 5155 : rtx cur_reg;
5846 :
5847 : /* Bail out early when expression cannot be renamed at all. */
5848 5155 : if (!EXPR_SEPARABLE_P (params->c_expr))
5849 : return false;
5850 :
5851 1297 : cur_reg = expr_dest_reg (params->c_expr);
5852 1297 : gcc_assert (cur_reg && params->dest && REG_P (params->dest));
5853 :
5854 : /* If original operation has expr and the register chosen for
5855 : that expr is not original operation's dest reg, substitute
5856 : operation's right hand side with the register chosen. */
5857 1297 : if (REGNO (params->dest) != REGNO (cur_reg))
5858 : {
5859 29 : insn_t reg_move_insn, reg_move_insn_rtx;
5860 :
5861 29 : reg_move_insn_rtx = create_insn_rtx_with_rhs (INSN_VINSN (insn),
5862 : params->dest);
5863 87 : reg_move_insn = sel_gen_insn_from_rtx_after (reg_move_insn_rtx,
5864 29 : INSN_EXPR (insn),
5865 29 : INSN_SEQNO (insn),
5866 : insn);
5867 29 : EXPR_SPEC_DONE_DS (INSN_EXPR (reg_move_insn)) = 0;
5868 29 : replace_dest_with_reg_in_expr (params->c_expr, params->dest);
5869 :
5870 29 : insn_emitted = true;
5871 29 : params->was_renamed = true;
5872 : }
5873 :
5874 : return insn_emitted;
5875 : }
5876 :
5877 : /* Emit a speculative check for INSN speculated as EXPR if needed.
5878 : Return true if we've emitted one. PARAMS is the move_op static
5879 : parameters. */
5880 : static bool
5881 5155 : maybe_emit_speculative_check (rtx_insn *insn, expr_t expr,
5882 : moveop_static_params_p params)
5883 : {
5884 5155 : bool insn_emitted = false;
5885 5155 : insn_t x;
5886 5155 : ds_t check_ds;
5887 :
5888 5155 : check_ds = get_spec_check_type_for_insn (insn, expr);
5889 5155 : if (check_ds != 0)
5890 : {
5891 : /* A speculation check should be inserted. */
5892 0 : x = create_speculation_check (params->c_expr, check_ds, insn);
5893 0 : insn_emitted = true;
5894 : }
5895 : else
5896 : {
5897 5155 : EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
5898 5155 : x = insn;
5899 : }
5900 :
5901 5155 : gcc_assert (EXPR_SPEC_DONE_DS (INSN_EXPR (x)) == 0
5902 : && EXPR_SPEC_TO_CHECK_DS (INSN_EXPR (x)) == 0);
5903 5155 : return insn_emitted;
5904 : }
5905 :
5906 : /* Handle transformations that leave an insn in place of original
5907 : insn such as renaming/speculation. Return true if one of such
5908 : transformations actually happened, and we have emitted this insn. */
5909 : static bool
5910 5155 : handle_emitting_transformations (rtx_insn *insn, expr_t expr,
5911 : moveop_static_params_p params)
5912 : {
5913 5155 : bool insn_emitted = false;
5914 :
5915 5155 : insn_emitted = maybe_emit_renaming_copy (insn, params);
5916 5155 : insn_emitted |= maybe_emit_speculative_check (insn, expr, params);
5917 :
5918 5155 : return insn_emitted;
5919 : }
5920 :
5921 : /* If INSN is the only insn in the basic block (not counting JUMP,
5922 : which may be a jump to next insn, and DEBUG_INSNs), we want to
5923 : leave a NOP there till the return to fill_insns. */
5924 :
5925 : static bool
5926 5155 : need_nop_to_preserve_insn_bb (rtx_insn *insn)
5927 : {
5928 5155 : insn_t bb_head, bb_end, bb_next, in_next;
5929 5155 : basic_block bb = BLOCK_FOR_INSN (insn);
5930 :
5931 5155 : bb_head = sel_bb_head (bb);
5932 5155 : bb_end = sel_bb_end (bb);
5933 :
5934 5155 : if (bb_head == bb_end)
5935 : return true;
5936 :
5937 5230 : while (bb_head != bb_end && DEBUG_INSN_P (bb_head))
5938 132 : bb_head = NEXT_INSN (bb_head);
5939 :
5940 5098 : if (bb_head == bb_end)
5941 : return true;
5942 :
5943 5106 : while (bb_head != bb_end && DEBUG_INSN_P (bb_end))
5944 8 : bb_end = PREV_INSN (bb_end);
5945 :
5946 5098 : if (bb_head == bb_end)
5947 : return true;
5948 :
5949 5098 : bb_next = NEXT_INSN (bb_head);
5950 10232 : while (bb_next != bb_end && DEBUG_INSN_P (bb_next))
5951 36 : bb_next = NEXT_INSN (bb_next);
5952 :
5953 5098 : if (bb_next == bb_end && JUMP_P (bb_end))
5954 : return true;
5955 :
5956 4815 : in_next = NEXT_INSN (insn);
5957 9648 : while (DEBUG_INSN_P (in_next))
5958 18 : in_next = NEXT_INSN (in_next);
5959 :
5960 4815 : if (IN_CURRENT_FENCE_P (in_next))
5961 : return true;
5962 :
5963 : return false;
5964 : }
5965 :
5966 : /* Remove INSN from stream. When ONLY_DISCONNECT is true, its data
5967 : is not removed but reused when INSN is re-emitted. */
5968 : static void
5969 5155 : remove_insn_from_stream (rtx_insn *insn, bool only_disconnect)
5970 : {
5971 : /* If there's only one insn in the BB, make sure that a nop is
5972 : inserted into it, so the basic block won't disappear when we'll
5973 : delete INSN below with sel_remove_insn. It should also survive
5974 : till the return to fill_insns. */
5975 5155 : if (need_nop_to_preserve_insn_bb (insn))
5976 : {
5977 340 : insn_t nop = get_nop_from_pool (insn);
5978 340 : gcc_assert (INSN_NOP_P (nop));
5979 340 : vec_temp_moveop_nops.safe_push (nop);
5980 : }
5981 :
5982 5155 : sel_remove_insn (insn, only_disconnect, false);
5983 5155 : }
5984 :
5985 : /* This function is called when original expr is found.
5986 : INSN - current insn traversed, EXPR - the corresponding expr found.
5987 : LPARAMS is the local parameters of code modion driver, STATIC_PARAMS
5988 : is static parameters of move_op. */
5989 : static void
5990 5155 : move_op_orig_expr_found (insn_t insn, expr_t expr,
5991 : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5992 : void *static_params)
5993 : {
5994 5155 : bool only_disconnect;
5995 5155 : moveop_static_params_p params = (moveop_static_params_p) static_params;
5996 :
5997 5155 : copy_expr_onside (params->c_expr, INSN_EXPR (insn));
5998 5155 : track_scheduled_insns_and_blocks (insn);
5999 5155 : handle_emitting_transformations (insn, expr, params);
6000 5155 : only_disconnect = params->uid == INSN_UID (insn);
6001 :
6002 : /* Mark that we've disconnected an insn. */
6003 5155 : if (only_disconnect)
6004 4871 : params->uid = -1;
6005 5155 : remove_insn_from_stream (insn, only_disconnect);
6006 5155 : }
6007 :
6008 : /* The function is called when original expr is found.
6009 : INSN - current insn traversed, EXPR - the corresponding expr found,
6010 : crossed_call_abis and original_insns in STATIC_PARAMS are updated. */
6011 : static void
6012 3894 : fur_orig_expr_found (insn_t insn, expr_t expr ATTRIBUTE_UNUSED,
6013 : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6014 : void *static_params)
6015 : {
6016 3894 : fur_static_params_p params = (fur_static_params_p) static_params;
6017 3894 : regset tmp;
6018 :
6019 3894 : if (CALL_P (insn))
6020 0 : params->crossed_call_abis |= 1 << insn_callee_abi (insn).id ();
6021 :
6022 3894 : def_list_add (params->original_insns, insn, params->crossed_call_abis);
6023 :
6024 : /* Mark the registers that do not meet the following condition:
6025 : (2) not among the live registers of the point
6026 : immediately following the first original operation on
6027 : a given downward path, except for the original target
6028 : register of the operation. */
6029 3894 : tmp = get_clear_regset_from_pool ();
6030 3894 : compute_live_below_insn (insn, tmp);
6031 3894 : AND_COMPL_REG_SET (tmp, INSN_REG_SETS (insn));
6032 3894 : AND_COMPL_REG_SET (tmp, INSN_REG_CLOBBERS (insn));
6033 3894 : IOR_REG_SET (params->used_regs, tmp);
6034 3894 : return_regset_to_pool (tmp);
6035 :
6036 : /* (*1) We need to add to USED_REGS registers that are read by
6037 : INSN's lhs. This may lead to choosing wrong src register.
6038 : E.g. (scheduling const expr enabled):
6039 :
6040 : 429: ax=0x0 <- Can't use AX for this expr (0x0)
6041 : 433: dx=[bp-0x18]
6042 : 427: [ax+dx+0x1]=ax
6043 : REG_DEAD: ax
6044 : 168: di=dx
6045 : REG_DEAD: dx
6046 : */
6047 : /* FIXME: see comment above and enable MEM_P
6048 : in vinsn_separable_p. */
6049 3894 : gcc_assert (!VINSN_SEPARABLE_P (INSN_VINSN (insn))
6050 : || !MEM_P (INSN_LHS (insn)));
6051 3894 : }
6052 :
6053 : /* This function is called on the ascending pass, before returning from
6054 : current basic block. */
6055 : static void
6056 5806 : move_op_at_first_insn (insn_t insn, cmpd_local_params_p lparams,
6057 : void *static_params)
6058 : {
6059 5806 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6060 5806 : basic_block book_block = NULL;
6061 :
6062 : /* When we have removed the boundary insn for scheduling, which also
6063 : happened to be the end insn in its bb, we don't need to update sets. */
6064 5806 : if (!lparams->removed_last_insn
6065 5035 : && lparams->e1
6066 6711 : && sel_bb_head_p (insn))
6067 : {
6068 : /* We should generate bookkeeping code only if we are not at the
6069 : top level of the move_op. */
6070 905 : if (sel_num_cfg_preds_gt_1 (insn))
6071 389 : book_block = generate_bookkeeping_insn (sparams->c_expr,
6072 : lparams->e1, lparams->e2);
6073 : /* Update data sets for the current insn. */
6074 905 : update_data_sets (insn);
6075 : }
6076 :
6077 : /* If bookkeeping code was inserted, we need to update av sets of basic
6078 : block that received bookkeeping. After generation of bookkeeping insn,
6079 : bookkeeping block does not contain valid av set because we are not following
6080 : the original algorithm in every detail with regards to e.g. renaming
6081 : simple reg-reg copies. Consider example:
6082 :
6083 : bookkeeping block scheduling fence
6084 : \ /
6085 : \ join /
6086 : ----------
6087 : | |
6088 : ----------
6089 : / \
6090 : / \
6091 : r1 := r2 r1 := r3
6092 :
6093 : We try to schedule insn "r1 := r3" on the current
6094 : scheduling fence. Also, note that av set of bookkeeping block
6095 : contain both insns "r1 := r2" and "r1 := r3". When the insn has
6096 : been scheduled, the CFG is as follows:
6097 :
6098 : r1 := r3 r1 := r3
6099 : bookkeeping block scheduling fence
6100 : \ /
6101 : \ join /
6102 : ----------
6103 : | |
6104 : ----------
6105 : / \
6106 : / \
6107 : r1 := r2
6108 :
6109 : Here, insn "r1 := r3" was scheduled at the current scheduling point
6110 : and bookkeeping code was generated at the bookeeping block. This
6111 : way insn "r1 := r2" is no longer available as a whole instruction
6112 : (but only as expr) ahead of insn "r1 := r3" in bookkeeping block.
6113 : This situation is handled by calling update_data_sets.
6114 :
6115 : Since update_data_sets is called only on the bookkeeping block, and
6116 : it also may have predecessors with av_sets, containing instructions that
6117 : are no longer available, we save all such expressions that become
6118 : unavailable during data sets update on the bookkeeping block in
6119 : VEC_BOOKKEEPING_BLOCKED_VINSNS. Later we avoid selecting such
6120 : expressions for scheduling. This allows us to avoid recomputation of
6121 : av_sets outside the code motion path. */
6122 :
6123 905 : if (book_block)
6124 389 : update_and_record_unavailable_insns (book_block);
6125 :
6126 : /* If INSN was previously marked for deletion, it's time to do it. */
6127 5806 : if (lparams->removed_last_insn)
6128 771 : insn = PREV_INSN (insn);
6129 :
6130 : /* Do not tidy control flow at the topmost moveop, as we can erroneously
6131 : kill a block with a single nop in which the insn should be emitted. */
6132 5806 : if (lparams->e1)
6133 905 : tidy_control_flow (BLOCK_FOR_INSN (insn), true);
6134 5806 : }
6135 :
6136 : /* This function is called on the ascending pass, before returning from the
6137 : current basic block. */
6138 : static void
6139 9375 : fur_at_first_insn (insn_t insn,
6140 : cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6141 : void *static_params ATTRIBUTE_UNUSED)
6142 : {
6143 9375 : gcc_assert (!sel_bb_head_p (insn) || AV_SET_VALID_P (insn)
6144 : || AV_LEVEL (insn) == -1);
6145 9375 : }
6146 :
6147 : /* Called on the backward stage of recursion to call moveup_expr for insn
6148 : and sparams->c_expr. */
6149 : static void
6150 6717 : move_op_ascend (insn_t insn, void *static_params)
6151 : {
6152 6717 : enum MOVEUP_EXPR_CODE res;
6153 6717 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6154 :
6155 6717 : if (! INSN_NOP_P (insn))
6156 : {
6157 6717 : res = moveup_expr_cached (sparams->c_expr, insn, false);
6158 6717 : gcc_assert (res != MOVEUP_EXPR_NULL);
6159 : }
6160 :
6161 : /* Update liveness for this insn as it was invalidated. */
6162 6717 : update_liveness_on_insn (insn);
6163 6717 : }
6164 :
6165 : /* This function is called on enter to the basic block.
6166 : Returns TRUE if this block already have been visited and
6167 : code_motion_path_driver should return 1, FALSE otherwise. */
6168 : static int
6169 10535 : fur_on_enter (insn_t insn ATTRIBUTE_UNUSED, cmpd_local_params_p local_params,
6170 : void *static_params, bool visited_p)
6171 : {
6172 10535 : fur_static_params_p sparams = (fur_static_params_p) static_params;
6173 :
6174 10535 : if (visited_p)
6175 : {
6176 : /* If we have found something below this block, there should be at
6177 : least one insn in ORIGINAL_INSNS. */
6178 466 : gcc_assert (*sparams->original_insns);
6179 :
6180 : /* Adjust CROSSED_CALL_ABIS, since we may have come to this block along
6181 : different path. */
6182 466 : DEF_LIST_DEF (*sparams->original_insns)->crossed_call_abis
6183 466 : |= sparams->crossed_call_abis;
6184 : }
6185 : else
6186 10069 : local_params->old_original_insns = *sparams->original_insns;
6187 :
6188 10535 : return 1;
6189 : }
6190 :
6191 : /* Same as above but for move_op. */
6192 : static int
6193 5903 : move_op_on_enter (insn_t insn ATTRIBUTE_UNUSED,
6194 : cmpd_local_params_p local_params ATTRIBUTE_UNUSED,
6195 : void *static_params ATTRIBUTE_UNUSED, bool visited_p)
6196 : {
6197 5903 : if (visited_p)
6198 24 : return -1;
6199 : return 1;
6200 : }
6201 :
6202 : /* This function is called while descending current basic block if current
6203 : insn is not the original EXPR we're searching for.
6204 :
6205 : Return value: FALSE, if code_motion_path_driver should perform a local
6206 : cleanup and return 0 itself;
6207 : TRUE, if code_motion_path_driver should continue. */
6208 : static bool
6209 6717 : move_op_orig_expr_not_found (insn_t insn, av_set_t orig_ops ATTRIBUTE_UNUSED,
6210 : void *static_params)
6211 : {
6212 6717 : moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6213 :
6214 6717 : sparams->failed_insn = insn;
6215 :
6216 : /* If we're scheduling separate expr, in order to generate correct code
6217 : we need to stop the search at bookkeeping code generated with the
6218 : same destination register or memory. */
6219 6717 : if (lhs_of_insn_equals_to_dest_p (insn, sparams->dest))
6220 0 : return false;
6221 : return true;
6222 : }
6223 :
6224 : /* This function is called while descending current basic block if current
6225 : insn is not the original EXPR we're searching for.
6226 :
6227 : Return value: TRUE (code_motion_path_driver should continue). */
6228 : static bool
6229 42054 : fur_orig_expr_not_found (insn_t insn, av_set_t orig_ops, void *static_params)
6230 : {
6231 42054 : bool mutexed;
6232 42054 : expr_t r;
6233 42054 : av_set_iterator avi;
6234 42054 : fur_static_params_p sparams = (fur_static_params_p) static_params;
6235 :
6236 42054 : if (CALL_P (insn))
6237 1217 : sparams->crossed_call_abis |= 1 << insn_callee_abi (insn).id ();
6238 40837 : else if (DEBUG_INSN_P (insn))
6239 : return true;
6240 :
6241 : /* If current insn we are looking at cannot be executed together
6242 : with original insn, then we can skip it safely.
6243 :
6244 : Example: ORIG_OPS = { (p6) r14 = sign_extend (r15); }
6245 : INSN = (!p6) r14 = r14 + 1;
6246 :
6247 : Here we can schedule ORIG_OP with lhs = r14, though only
6248 : looking at the set of used and set registers of INSN we must
6249 : forbid it. So, add set/used in INSN registers to the
6250 : untouchable set only if there is an insn in ORIG_OPS that can
6251 : affect INSN. */
6252 42051 : mutexed = true;
6253 42051 : FOR_EACH_EXPR (r, avi, orig_ops)
6254 42051 : if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (r)))
6255 : {
6256 : mutexed = false;
6257 : break;
6258 : }
6259 :
6260 : /* Mark all registers that do not meet the following condition:
6261 : (1) Not set or read on any path from xi to an instance of the
6262 : original operation. */
6263 42051 : if (!mutexed)
6264 : {
6265 42051 : IOR_REG_SET (sparams->used_regs, INSN_REG_SETS (insn));
6266 42051 : IOR_REG_SET (sparams->used_regs, INSN_REG_USES (insn));
6267 42051 : IOR_REG_SET (sparams->used_regs, INSN_REG_CLOBBERS (insn));
6268 : }
6269 :
6270 : return true;
6271 : }
6272 :
6273 : /* Hooks and data to perform move_op operations with code_motion_path_driver. */
6274 : struct code_motion_path_driver_info_def move_op_hooks = {
6275 : move_op_on_enter,
6276 : move_op_orig_expr_found,
6277 : move_op_orig_expr_not_found,
6278 : move_op_merge_succs,
6279 : move_op_after_merge_succs,
6280 : move_op_ascend,
6281 : move_op_at_first_insn,
6282 : SUCCS_NORMAL,
6283 : "move_op"
6284 : };
6285 :
6286 : /* Hooks and data to perform find_used_regs operations
6287 : with code_motion_path_driver. */
6288 : struct code_motion_path_driver_info_def fur_hooks = {
6289 : fur_on_enter,
6290 : fur_orig_expr_found,
6291 : fur_orig_expr_not_found,
6292 : fur_merge_succs,
6293 : NULL, /* fur_after_merge_succs */
6294 : NULL, /* fur_ascend */
6295 : fur_at_first_insn,
6296 : SUCCS_ALL,
6297 : "find_used_regs"
6298 : };
6299 :
6300 : /* Traverse all successors of INSN. For each successor that is SUCCS_NORMAL
6301 : code_motion_path_driver is called recursively. Original operation
6302 : was found at least on one path that is starting with one of INSN's
6303 : successors (this fact is asserted). ORIG_OPS is expressions we're looking
6304 : for, PATH is the path we've traversed, STATIC_PARAMS is the parameters
6305 : of either move_op or find_used_regs depending on the caller.
6306 :
6307 : Return 0 if we haven't found expression, 1 if we found it, -1 if we don't
6308 : know for sure at this point. */
6309 : static int
6310 6132 : code_motion_process_successors (insn_t insn, av_set_t orig_ops,
6311 : ilist_t path, void *static_params)
6312 : {
6313 6132 : int res = 0;
6314 6132 : succ_iterator succ_i;
6315 6132 : insn_t succ;
6316 6132 : basic_block bb;
6317 6132 : int old_index;
6318 6132 : unsigned old_succs;
6319 :
6320 6132 : struct cmpd_local_params lparams;
6321 6132 : expr_def _x;
6322 :
6323 6132 : lparams.c_expr_local = &_x;
6324 6132 : lparams.c_expr_merged = NULL;
6325 :
6326 : /* We need to process only NORMAL succs for move_op, and collect live
6327 : registers from ALL branches (including those leading out of the
6328 : region) for find_used_regs.
6329 :
6330 : In move_op, there can be a case when insn's bb number has changed
6331 : due to created bookkeeping. This happens very rare, as we need to
6332 : move expression from the beginning to the end of the same block.
6333 : Rescan successors in this case. */
6334 :
6335 6156 : rescan:
6336 6156 : bb = BLOCK_FOR_INSN (insn);
6337 6156 : old_index = bb->index;
6338 6156 : old_succs = EDGE_COUNT (bb->succs);
6339 :
6340 17119 : FOR_EACH_SUCC_1 (succ, succ_i, insn, code_motion_path_driver_info->succ_flags)
6341 : {
6342 10987 : int b;
6343 :
6344 10987 : lparams.e1 = succ_i.e1;
6345 10987 : lparams.e2 = succ_i.e2;
6346 :
6347 : /* Go deep into recursion only for NORMAL edges (non-backedges within the
6348 : current region). */
6349 10987 : if (succ_i.current_flags == SUCCS_NORMAL)
6350 8750 : b = code_motion_path_driver (succ, orig_ops, path, &lparams,
6351 : static_params);
6352 : else
6353 : b = 0;
6354 :
6355 : /* Merge c_expres found or unify live register sets from different
6356 : successors. */
6357 10987 : code_motion_path_driver_info->merge_succs (insn, succ, b, &lparams,
6358 : static_params);
6359 10987 : if (b == 1)
6360 : res = b;
6361 3031 : else if (b == -1 && res != 1)
6362 0 : res = b;
6363 :
6364 : /* We have simplified the control flow below this point. In this case,
6365 : the iterator becomes invalid. We need to try again.
6366 : If we have removed the insn itself, it could be only an
6367 : unconditional jump. Thus, do not rescan but break immediately --
6368 : we have already visited the only successor block. */
6369 10987 : if (!BLOCK_FOR_INSN (insn))
6370 : {
6371 0 : if (sched_verbose >= 6)
6372 0 : sel_print ("Not doing rescan: already visited the only successor"
6373 : " of block %d\n", old_index);
6374 : break;
6375 : }
6376 10987 : if (BLOCK_FOR_INSN (insn)->index != old_index
6377 21950 : || EDGE_COUNT (bb->succs) != old_succs)
6378 : {
6379 24 : if (sched_verbose >= 6)
6380 0 : sel_print ("Rescan: CFG was simplified below insn %d, block %d\n",
6381 0 : INSN_UID (insn), BLOCK_FOR_INSN (insn)->index);
6382 24 : insn = sel_bb_end (BLOCK_FOR_INSN (insn));
6383 24 : goto rescan;
6384 : }
6385 : }
6386 :
6387 : /* Here, RES==1 if original expr was found at least for one of the
6388 : successors. After the loop, RES may happen to have zero value
6389 : only if at some point the expr searched is present in av_set, but is
6390 : not found below. In most cases, this situation is an error.
6391 : The exception is when the original operation is blocked by
6392 : bookkeeping generated for another fence or for another path in current
6393 : move_op. */
6394 6132 : gcc_checking_assert (res == 1
6395 : || (res == 0
6396 : && av_set_could_be_blocked_by_bookkeeping_p (orig_ops, static_params))
6397 : || res == -1);
6398 :
6399 : /* Merge data, clean up, etc. */
6400 6132 : if (res != -1 && code_motion_path_driver_info->after_merge_succs)
6401 651 : code_motion_path_driver_info->after_merge_succs (&lparams, static_params);
6402 :
6403 6132 : return res;
6404 : }
6405 :
6406 :
6407 : /* Perform a cleanup when the driver is about to terminate. ORIG_OPS_P
6408 : is the pointer to the av set with expressions we were looking for,
6409 : PATH_P is the pointer to the traversed path. */
6410 : static inline void
6411 0 : code_motion_path_driver_cleanup (av_set_t *orig_ops_p, ilist_t *path_p)
6412 : {
6413 0 : ilist_remove (path_p);
6414 0 : av_set_clear (orig_ops_p);
6415 0 : }
6416 :
6417 : /* The driver function that implements move_op or find_used_regs
6418 : functionality dependent whether code_motion_path_driver_INFO is set to
6419 : &MOVE_OP_HOOKS or &FUR_HOOKS. This function implements the common parts
6420 : of code (CFG traversal etc) that are shared among both functions. INSN
6421 : is the insn we're starting the search from, ORIG_OPS are the expressions
6422 : we're searching for, PATH is traversed path, LOCAL_PARAMS_IN are local
6423 : parameters of the driver, and STATIC_PARAMS are static parameters of
6424 : the caller.
6425 :
6426 : Returns whether original instructions were found. Note that top-level
6427 : code_motion_path_driver always returns true. */
6428 : static int
6429 16441 : code_motion_path_driver (insn_t insn, av_set_t orig_ops, ilist_t path,
6430 : cmpd_local_params_p local_params_in,
6431 : void *static_params)
6432 : {
6433 16441 : expr_t expr = NULL;
6434 16441 : basic_block bb = BLOCK_FOR_INSN (insn);
6435 16441 : insn_t first_insn, original_insn, bb_tail, before_first;
6436 16441 : bool removed_last_insn = false;
6437 :
6438 16441 : if (sched_verbose >= 6)
6439 : {
6440 0 : sel_print ("%s (", code_motion_path_driver_info->routine_name);
6441 0 : dump_insn (insn);
6442 0 : sel_print (",");
6443 0 : dump_av_set (orig_ops);
6444 0 : sel_print (")\n");
6445 : }
6446 :
6447 16441 : gcc_assert (orig_ops);
6448 :
6449 : /* If no original operations exist below this insn, return immediately. */
6450 16441 : if (is_ineligible_successor (insn, path))
6451 : {
6452 3 : if (sched_verbose >= 6)
6453 0 : sel_print ("Insn %d is ineligible successor\n", INSN_UID (insn));
6454 3 : return false;
6455 : }
6456 :
6457 : /* The block can have invalid av set, in which case it was created earlier
6458 : during move_op. Return immediately. */
6459 16438 : if (sel_bb_head_p (insn))
6460 : {
6461 8914 : if (! AV_SET_VALID_P (insn))
6462 : {
6463 0 : if (sched_verbose >= 6)
6464 0 : sel_print ("Returned from block %d as it had invalid av set\n",
6465 : bb->index);
6466 0 : return false;
6467 : }
6468 :
6469 8914 : if (bitmap_bit_p (code_motion_visited_blocks, bb->index))
6470 : {
6471 : /* We have already found an original operation on this branch, do not
6472 : go any further and just return TRUE here. If we don't stop here,
6473 : function can have exponential behavior even on the small code
6474 : with many different paths (e.g. with data speculation and
6475 : recovery blocks). */
6476 490 : if (sched_verbose >= 6)
6477 0 : sel_print ("Block %d already visited in this traversal\n", bb->index);
6478 490 : if (code_motion_path_driver_info->on_enter)
6479 490 : return code_motion_path_driver_info->on_enter (insn,
6480 : local_params_in,
6481 : static_params,
6482 490 : true);
6483 : }
6484 : }
6485 :
6486 15948 : if (code_motion_path_driver_info->on_enter)
6487 15948 : code_motion_path_driver_info->on_enter (insn, local_params_in,
6488 : static_params, false);
6489 15948 : orig_ops = av_set_copy (orig_ops);
6490 :
6491 : /* Filter the orig_ops set. */
6492 15948 : if (AV_SET_VALID_P (insn))
6493 8424 : av_set_code_motion_filter (&orig_ops, AV_SET (insn));
6494 :
6495 : /* If no more original ops, return immediately. */
6496 15948 : if (!orig_ops)
6497 : {
6498 767 : if (sched_verbose >= 6)
6499 0 : sel_print ("No intersection with av set of block %d\n", bb->index);
6500 767 : return false;
6501 : }
6502 :
6503 : /* For non-speculative insns we have to leave only one form of the
6504 : original operation, because if we don't, we may end up with
6505 : different C_EXPRes and, consequently, with bookkeepings for different
6506 : expression forms along the same code motion path. That may lead to
6507 : generation of incorrect code. So for each code motion we stick to
6508 : the single form of the instruction, except for speculative insns
6509 : which we need to keep in different forms with all speculation
6510 : types. */
6511 15181 : av_set_leave_one_nonspec (&orig_ops);
6512 :
6513 : /* It is not possible that all ORIG_OPS are filtered out. */
6514 15181 : gcc_assert (orig_ops);
6515 :
6516 : /* It is enough to place only heads and tails of visited basic blocks into
6517 : the PATH. */
6518 15181 : ilist_add (&path, insn);
6519 15181 : first_insn = original_insn = insn;
6520 15181 : bb_tail = sel_bb_end (bb);
6521 :
6522 : /* Descend the basic block in search of the original expr; this part
6523 : corresponds to the part of the original move_op procedure executed
6524 : before the recursive call. */
6525 42639 : for (;;)
6526 : {
6527 : /* Look at the insn and decide if it could be an ancestor of currently
6528 : scheduling operation. If it is so, then the insn "dest = op" could
6529 : either be replaced with "dest = reg", because REG now holds the result
6530 : of OP, or just removed, if we've scheduled the insn as a whole.
6531 :
6532 : If this insn doesn't contain currently scheduling OP, then proceed
6533 : with searching and look at its successors. Operations we're searching
6534 : for could have changed when moving up through this insn via
6535 : substituting. In this case, perform unsubstitution on them first.
6536 :
6537 : When traversing the DAG below this insn is finished, insert
6538 : bookkeeping code, if the insn is a joint point, and remove
6539 : leftovers. */
6540 :
6541 57820 : expr = av_set_lookup (orig_ops, INSN_VINSN (insn));
6542 57820 : if (expr)
6543 : {
6544 9049 : insn_t last_insn = PREV_INSN (insn);
6545 :
6546 : /* We have found the original operation. */
6547 9049 : if (sched_verbose >= 6)
6548 0 : sel_print ("Found original operation at insn %d\n", INSN_UID (insn));
6549 :
6550 9049 : code_motion_path_driver_info->orig_expr_found
6551 9049 : (insn, expr, local_params_in, static_params);
6552 :
6553 : /* Step back, so on the way back we'll start traversing from the
6554 : previous insn (or we'll see that it's bb_note and skip that
6555 : loop). */
6556 9049 : if (insn == first_insn)
6557 : {
6558 4476 : first_insn = NEXT_INSN (last_insn);
6559 4476 : removed_last_insn = sel_bb_end_p (last_insn);
6560 : }
6561 : insn = last_insn;
6562 : break;
6563 : }
6564 : else
6565 : {
6566 : /* We haven't found the original expr, continue descending the basic
6567 : block. */
6568 48771 : if (code_motion_path_driver_info->orig_expr_not_found
6569 48771 : (insn, orig_ops, static_params))
6570 : {
6571 : /* Av set ops could have been changed when moving through this
6572 : insn. To find them below it, we have to un-substitute them. */
6573 48771 : undo_transformations (&orig_ops, insn);
6574 : }
6575 : else
6576 : {
6577 : /* Clean up and return, if the hook tells us to do so. It may
6578 : happen if we've encountered the previously created
6579 : bookkeeping. */
6580 0 : code_motion_path_driver_cleanup (&orig_ops, &path);
6581 0 : return -1;
6582 : }
6583 :
6584 48771 : gcc_assert (orig_ops);
6585 : }
6586 :
6587 : /* Stop at insn if we got to the end of BB. */
6588 48771 : if (insn == bb_tail)
6589 : break;
6590 :
6591 42639 : insn = NEXT_INSN (insn);
6592 42639 : }
6593 :
6594 : /* Here INSN either points to the insn before the original insn (may be
6595 : bb_note, if original insn was a bb_head) or to the bb_end. */
6596 15181 : if (!expr)
6597 : {
6598 6132 : int res;
6599 6132 : rtx_insn *last_insn = PREV_INSN (insn);
6600 6132 : bool added_to_path;
6601 :
6602 6132 : gcc_assert (insn == sel_bb_end (bb));
6603 :
6604 : /* Add bb tail to PATH (but it doesn't make any sense if it's a bb_head -
6605 : it's already in PATH then). */
6606 6132 : if (insn != first_insn)
6607 : {
6608 4938 : ilist_add (&path, insn);
6609 4938 : added_to_path = true;
6610 : }
6611 : else
6612 : added_to_path = false;
6613 :
6614 : /* Process_successors should be able to find at least one
6615 : successor for which code_motion_path_driver returns TRUE. */
6616 6132 : res = code_motion_process_successors (insn, orig_ops,
6617 : path, static_params);
6618 :
6619 : /* Jump in the end of basic block could have been removed or replaced
6620 : during code_motion_process_successors, so recompute insn as the
6621 : last insn in bb. */
6622 6132 : if (NEXT_INSN (last_insn) != insn)
6623 : {
6624 0 : insn = sel_bb_end (bb);
6625 0 : first_insn = sel_bb_head (bb);
6626 0 : if (first_insn != original_insn)
6627 : first_insn = original_insn;
6628 : }
6629 :
6630 : /* Remove bb tail from path. */
6631 6132 : if (added_to_path)
6632 4938 : ilist_remove (&path);
6633 :
6634 6132 : if (res != 1)
6635 : {
6636 : /* This is the case when one of the original expr is no longer available
6637 : due to bookkeeping created on this branch with the same register.
6638 : In the original algorithm, which doesn't have update_data_sets call
6639 : on a bookkeeping block, it would simply result in returning
6640 : FALSE when we've encountered a previously generated bookkeeping
6641 : insn in moveop_orig_expr_not_found. */
6642 0 : code_motion_path_driver_cleanup (&orig_ops, &path);
6643 0 : return res;
6644 : }
6645 : }
6646 :
6647 : /* Don't need it any more. */
6648 15181 : av_set_clear (&orig_ops);
6649 :
6650 : /* Backward pass: now, when we have C_EXPR computed, we'll drag it to
6651 : the beginning of the basic block. */
6652 15181 : before_first = PREV_INSN (first_insn);
6653 79133 : while (insn != before_first)
6654 : {
6655 48771 : if (code_motion_path_driver_info->ascend)
6656 6717 : code_motion_path_driver_info->ascend (insn, static_params);
6657 :
6658 48771 : insn = PREV_INSN (insn);
6659 : }
6660 :
6661 : /* Now we're at the bb head. */
6662 15181 : insn = first_insn;
6663 15181 : ilist_remove (&path);
6664 15181 : local_params_in->removed_last_insn = removed_last_insn;
6665 15181 : code_motion_path_driver_info->at_first_insn (insn, local_params_in, static_params);
6666 :
6667 : /* This should be the very last operation as at bb head we could change
6668 : the numbering by creating bookkeeping blocks. */
6669 15181 : if (removed_last_insn)
6670 771 : insn = PREV_INSN (insn);
6671 :
6672 : /* If we have simplified the control flow and removed the first jump insn,
6673 : there's no point in marking this block in the visited blocks bitmap. */
6674 15181 : if (BLOCK_FOR_INSN (insn))
6675 15181 : bitmap_set_bit (code_motion_visited_blocks, BLOCK_FOR_INSN (insn)->index);
6676 : return true;
6677 : }
6678 :
6679 : /* Move up the operations from ORIG_OPS set traversing the dag starting
6680 : from INSN. PATH represents the edges traversed so far.
6681 : DEST is the register chosen for scheduling the current expr. Insert
6682 : bookkeeping code in the join points. EXPR_VLIW is the chosen expression,
6683 : C_EXPR is how it looks like at the given cfg point.
6684 : Set *SHOULD_MOVE to indicate whether we have only disconnected
6685 : one of the insns found.
6686 :
6687 : Returns whether original instructions were found, which is asserted
6688 : to be true in the caller. */
6689 : static bool
6690 4901 : move_op (insn_t insn, av_set_t orig_ops, expr_t expr_vliw,
6691 : rtx dest, expr_t c_expr, bool *should_move)
6692 : {
6693 4901 : struct moveop_static_params sparams;
6694 4901 : struct cmpd_local_params lparams;
6695 4901 : int res;
6696 :
6697 : /* Init params for code_motion_path_driver. */
6698 4901 : sparams.dest = dest;
6699 4901 : sparams.c_expr = c_expr;
6700 4901 : sparams.uid = INSN_UID (EXPR_INSN_RTX (expr_vliw));
6701 4901 : sparams.failed_insn = NULL;
6702 4901 : sparams.was_renamed = false;
6703 4901 : lparams.e1 = NULL;
6704 :
6705 : /* We haven't visited any blocks yet. */
6706 4901 : bitmap_clear (code_motion_visited_blocks);
6707 :
6708 : /* Set appropriate hooks and data. */
6709 4901 : code_motion_path_driver_info = &move_op_hooks;
6710 4901 : res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
6711 :
6712 4901 : gcc_assert (res != -1);
6713 :
6714 4901 : if (sparams.was_renamed)
6715 28 : EXPR_WAS_RENAMED (expr_vliw) = true;
6716 :
6717 4901 : *should_move = (sparams.uid == -1);
6718 :
6719 4901 : return res;
6720 : }
6721 :
6722 :
6723 : /* Functions that work with regions. */
6724 :
6725 : /* Current number of seqno used in init_seqno and init_seqno_1. */
6726 : static int cur_seqno;
6727 :
6728 : /* A helper for init_seqno. Traverse the region starting from BB and
6729 : compute seqnos for visited insns, marking visited bbs in VISITED_BBS.
6730 : Clear visited blocks from BLOCKS_TO_RESCHEDULE. */
6731 : static void
6732 1096 : init_seqno_1 (basic_block bb, sbitmap visited_bbs, bitmap blocks_to_reschedule)
6733 : {
6734 1096 : int bbi = BLOCK_TO_BB (bb->index);
6735 1096 : insn_t insn;
6736 1096 : insn_t succ_insn;
6737 1096 : succ_iterator si;
6738 :
6739 1096 : rtx_note *note = bb_note (bb);
6740 1096 : bitmap_set_bit (visited_bbs, bbi);
6741 1096 : if (blocks_to_reschedule)
6742 52 : bitmap_clear_bit (blocks_to_reschedule, bb->index);
6743 :
6744 1539 : FOR_EACH_SUCC_1 (succ_insn, si, BB_END (bb),
6745 : SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
6746 : {
6747 443 : basic_block succ = BLOCK_FOR_INSN (succ_insn);
6748 443 : int succ_bbi = BLOCK_TO_BB (succ->index);
6749 :
6750 443 : gcc_assert (in_current_region_p (succ));
6751 :
6752 443 : if (!bitmap_bit_p (visited_bbs, succ_bbi))
6753 : {
6754 306 : gcc_assert (succ_bbi > bbi);
6755 :
6756 306 : init_seqno_1 (succ, visited_bbs, blocks_to_reschedule);
6757 : }
6758 137 : else if (blocks_to_reschedule)
6759 19 : bitmap_set_bit (forced_ebb_heads, succ->index);
6760 : }
6761 :
6762 5954 : for (insn = BB_END (bb); insn != note; insn = PREV_INSN (insn))
6763 4858 : INSN_SEQNO (insn) = cur_seqno--;
6764 1096 : }
6765 :
6766 : /* Initialize seqnos for the current region. BLOCKS_TO_RESCHEDULE contains
6767 : blocks on which we're rescheduling when pipelining, FROM is the block where
6768 : traversing region begins (it may not be the head of the region when
6769 : pipelining, but the head of the loop instead).
6770 :
6771 : Returns the maximal seqno found. */
6772 : static int
6773 790 : init_seqno (bitmap blocks_to_reschedule, basic_block from)
6774 : {
6775 790 : bitmap_iterator bi;
6776 790 : unsigned bbi;
6777 :
6778 790 : auto_sbitmap visited_bbs (current_nr_blocks);
6779 :
6780 790 : if (blocks_to_reschedule)
6781 : {
6782 20 : bitmap_ones (visited_bbs);
6783 94 : EXECUTE_IF_SET_IN_BITMAP (blocks_to_reschedule, 0, bbi, bi)
6784 : {
6785 74 : gcc_assert (BLOCK_TO_BB (bbi) < current_nr_blocks);
6786 74 : bitmap_clear_bit (visited_bbs, BLOCK_TO_BB (bbi));
6787 : }
6788 : }
6789 : else
6790 : {
6791 770 : bitmap_clear (visited_bbs);
6792 770 : from = EBB_FIRST_BB (0);
6793 : }
6794 :
6795 790 : cur_seqno = sched_max_luid - 1;
6796 790 : init_seqno_1 (from, visited_bbs, blocks_to_reschedule);
6797 :
6798 : /* cur_seqno may be positive if the number of instructions is less than
6799 : sched_max_luid - 1 (when rescheduling or if some instructions have been
6800 : removed by the call to purge_empty_blocks in sel_sched_region_1). */
6801 790 : gcc_assert (cur_seqno >= 0);
6802 :
6803 790 : return sched_max_luid - 1;
6804 790 : }
6805 :
6806 : /* Initialize scheduling parameters for current region. */
6807 : static void
6808 770 : sel_setup_region_sched_flags (void)
6809 : {
6810 770 : enable_schedule_as_rhs_p = 1;
6811 770 : bookkeeping_p = 1;
6812 1540 : pipelining_p = (bookkeeping_p
6813 770 : && (flag_sel_sched_pipelining != 0)
6814 387 : && current_loop_nest != NULL
6815 56 : && loop_has_exit_edges (current_loop_nest));
6816 770 : max_insns_to_rename = param_selsched_insns_to_rename;
6817 770 : max_ws = MAX_WS;
6818 770 : }
6819 :
6820 : /* Return true if all basic blocks of current region are empty. */
6821 : static bool
6822 782 : current_region_empty_p (void)
6823 : {
6824 782 : int i;
6825 819 : for (i = 0; i < current_nr_blocks; i++)
6826 807 : if (! sel_bb_empty_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i))))
6827 : return false;
6828 :
6829 : return true;
6830 : }
6831 :
6832 : /* Prepare and verify loop nest for pipelining. */
6833 : static void
6834 387 : setup_current_loop_nest (int rgn, bb_vec_t *bbs)
6835 : {
6836 387 : current_loop_nest = get_loop_nest_for_rgn (rgn);
6837 :
6838 387 : if (!current_loop_nest)
6839 : return;
6840 :
6841 : /* If this loop has any saved loop preheaders from nested loops,
6842 : add these basic blocks to the current region. */
6843 56 : sel_add_loop_preheaders (bbs);
6844 :
6845 : /* Check that we're starting with a valid information. */
6846 56 : gcc_assert (loop_latch_edge (current_loop_nest));
6847 56 : gcc_assert (LOOP_MARKED_FOR_PIPELINING_P (current_loop_nest));
6848 : }
6849 :
6850 : /* Compute instruction priorities for current region. */
6851 : static void
6852 770 : sel_compute_priorities (int rgn)
6853 : {
6854 770 : sched_rgn_compute_dependencies (rgn);
6855 :
6856 : /* Compute insn priorities in haifa style. Then free haifa style
6857 : dependencies that we've calculated for this. */
6858 770 : compute_priorities ();
6859 :
6860 770 : if (sched_verbose >= 5)
6861 0 : debug_rgn_dependencies (0);
6862 :
6863 770 : free_rgn_deps ();
6864 770 : }
6865 :
6866 : /* Init scheduling data for RGN. Returns true when this region should not
6867 : be scheduled. */
6868 : static bool
6869 782 : sel_region_init (int rgn)
6870 : {
6871 782 : int i;
6872 782 : bb_vec_t bbs;
6873 :
6874 782 : rgn_setup_region (rgn);
6875 :
6876 : /* Even if sched_is_disabled_for_current_region_p() is true, we still
6877 : do region initialization here so the region can be bundled correctly,
6878 : but we'll skip the scheduling in sel_sched_region (). */
6879 782 : if (current_region_empty_p ())
6880 : return true;
6881 :
6882 770 : bbs.create (current_nr_blocks);
6883 :
6884 2583 : for (i = 0; i < current_nr_blocks; i++)
6885 1043 : bbs.quick_push (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i)));
6886 :
6887 770 : sel_init_bbs (bbs);
6888 :
6889 770 : if (flag_sel_sched_pipelining)
6890 387 : setup_current_loop_nest (rgn, &bbs);
6891 :
6892 770 : sel_setup_region_sched_flags ();
6893 :
6894 : /* Initialize luids and dependence analysis which both sel-sched and haifa
6895 : need. */
6896 770 : sched_init_luids (bbs);
6897 770 : sched_deps_init (false);
6898 :
6899 : /* Initialize haifa data. */
6900 770 : rgn_setup_sched_infos ();
6901 770 : sel_set_sched_flags ();
6902 770 : haifa_init_h_i_d (bbs);
6903 :
6904 770 : sel_compute_priorities (rgn);
6905 770 : init_deps_global ();
6906 :
6907 : /* Main initialization. */
6908 770 : sel_setup_sched_infos ();
6909 770 : sel_init_global_and_expr (bbs);
6910 :
6911 770 : bbs.release ();
6912 :
6913 770 : blocks_to_reschedule = BITMAP_ALLOC (NULL);
6914 :
6915 : /* Init correct liveness sets on each instruction of a single-block loop.
6916 : This is the only situation when we can't update liveness when calling
6917 : compute_live for the first insn of the loop. */
6918 770 : if (current_loop_nest)
6919 : {
6920 56 : int header =
6921 56 : (sel_is_loop_preheader_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (0)))
6922 56 : ? 1
6923 0 : : 0);
6924 :
6925 56 : if (current_nr_blocks == header + 1)
6926 16 : update_liveness_on_insn
6927 16 : (sel_bb_head (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (header))));
6928 : }
6929 :
6930 : /* Set hooks so that no newly generated insn will go out unnoticed. */
6931 770 : sel_register_cfg_hooks ();
6932 :
6933 : /* !!! We call target.sched.init () for the whole region, but we invoke
6934 : targetm.sched.finish () for every ebb. */
6935 770 : if (targetm.sched.init)
6936 : /* None of the arguments are actually used in any target. */
6937 0 : targetm.sched.init (sched_dump, sched_verbose, -1);
6938 :
6939 770 : first_emitted_uid = get_max_uid () + 1;
6940 770 : preheader_removed = false;
6941 :
6942 : /* Reset register allocation ticks array. */
6943 770 : memset (reg_rename_tick, 0, sizeof reg_rename_tick);
6944 770 : reg_rename_this_tick = 0;
6945 :
6946 770 : forced_ebb_heads = BITMAP_ALLOC (NULL);
6947 :
6948 770 : setup_nop_vinsn ();
6949 770 : current_copies = BITMAP_ALLOC (NULL);
6950 770 : current_originators = BITMAP_ALLOC (NULL);
6951 770 : code_motion_visited_blocks = BITMAP_ALLOC (NULL);
6952 :
6953 770 : return false;
6954 : }
6955 :
6956 : /* Simplify insns after the scheduling. */
6957 : static void
6958 770 : simplify_changed_insns (void)
6959 : {
6960 770 : int i;
6961 :
6962 1826 : for (i = 0; i < current_nr_blocks; i++)
6963 : {
6964 1056 : basic_block bb = BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i));
6965 1056 : rtx_insn *insn;
6966 :
6967 7274 : FOR_BB_INSNS (bb, insn)
6968 6218 : if (INSN_P (insn))
6969 : {
6970 4645 : expr_t expr = INSN_EXPR (insn);
6971 :
6972 4645 : if (EXPR_WAS_SUBSTITUTED (expr))
6973 0 : validate_simplify_insn (insn);
6974 : }
6975 : }
6976 770 : }
6977 :
6978 : /* Find boundaries of the EBB starting from basic block BB, marking blocks of
6979 : this EBB in SCHEDULED_BLOCKS and appropriately filling in HEAD, TAIL,
6980 : PREV_HEAD, and NEXT_TAIL fields of CURRENT_SCHED_INFO structure. */
6981 : static void
6982 610 : find_ebb_boundaries (basic_block bb, bitmap scheduled_blocks)
6983 : {
6984 610 : rtx_insn *head, *tail;
6985 610 : basic_block bb1 = bb;
6986 610 : if (sched_verbose >= 2)
6987 0 : sel_print ("Finishing schedule in bbs: ");
6988 :
6989 733 : do
6990 : {
6991 733 : bitmap_set_bit (scheduled_blocks, BLOCK_TO_BB (bb1->index));
6992 :
6993 733 : if (sched_verbose >= 2)
6994 0 : sel_print ("%d; ", bb1->index);
6995 : }
6996 733 : while (!bb_ends_ebb_p (bb1) && (bb1 = bb_next_bb (bb1)));
6997 :
6998 610 : if (sched_verbose >= 2)
6999 0 : sel_print ("\n");
7000 :
7001 610 : get_ebb_head_tail (bb, bb1, &head, &tail);
7002 :
7003 610 : current_sched_info->head = head;
7004 610 : current_sched_info->tail = tail;
7005 610 : current_sched_info->prev_head = PREV_INSN (head);
7006 610 : current_sched_info->next_tail = NEXT_INSN (tail);
7007 610 : }
7008 :
7009 : /* Regenerate INSN_SCHED_CYCLEs for insns of current EBB. */
7010 : static void
7011 104 : reset_sched_cycles_in_current_ebb (void)
7012 : {
7013 104 : int last_clock = 0;
7014 104 : int haifa_last_clock = -1;
7015 104 : int haifa_clock = 0;
7016 104 : int issued_insns = 0;
7017 104 : insn_t insn;
7018 :
7019 104 : if (targetm.sched.init)
7020 : {
7021 : /* None of the arguments are actually used in any target.
7022 : NB: We should have md_reset () hook for cases like this. */
7023 0 : targetm.sched.init (sched_dump, sched_verbose, -1);
7024 : }
7025 :
7026 104 : state_reset (curr_state);
7027 104 : advance_state (curr_state);
7028 :
7029 1260 : for (insn = current_sched_info->head;
7030 1260 : insn != current_sched_info->next_tail;
7031 1156 : insn = NEXT_INSN (insn))
7032 : {
7033 1156 : int cost, haifa_cost;
7034 1156 : int sort_p;
7035 1156 : bool asm_p, real_insn, after_stall, all_issued;
7036 1156 : int clock;
7037 :
7038 1156 : if (!INSN_P (insn))
7039 98 : continue;
7040 :
7041 1058 : asm_p = false;
7042 1058 : real_insn = recog_memoized (insn) >= 0;
7043 1058 : clock = INSN_SCHED_CYCLE (insn);
7044 :
7045 1058 : cost = clock - last_clock;
7046 :
7047 : /* Initialize HAIFA_COST. */
7048 1058 : if (! real_insn)
7049 : {
7050 5 : asm_p = INSN_ASM_P (insn);
7051 :
7052 5 : if (asm_p)
7053 : /* This is asm insn which *had* to be scheduled first
7054 : on the cycle. */
7055 : haifa_cost = 1;
7056 : else
7057 : /* This is a use/clobber insn. It should not change
7058 : cost. */
7059 2 : haifa_cost = 0;
7060 : }
7061 : else
7062 1053 : haifa_cost = estimate_insn_cost (insn, curr_state);
7063 :
7064 : /* Stall for whatever cycles we've stalled before. */
7065 1058 : after_stall = 0;
7066 1058 : if (INSN_AFTER_STALL_P (insn) && cost > haifa_cost)
7067 : {
7068 : haifa_cost = cost;
7069 : after_stall = 1;
7070 : }
7071 1058 : all_issued = issued_insns == issue_rate;
7072 1058 : if (haifa_cost == 0 && all_issued)
7073 0 : haifa_cost = 1;
7074 1058 : if (haifa_cost > 0)
7075 : {
7076 : int i = 0;
7077 :
7078 1498 : while (haifa_cost--)
7079 : {
7080 827 : advance_state (curr_state);
7081 827 : issued_insns = 0;
7082 827 : i++;
7083 :
7084 827 : if (sched_verbose >= 2)
7085 : {
7086 0 : sel_print ("advance_state (state_transition)\n");
7087 0 : debug_state (curr_state);
7088 : }
7089 :
7090 : /* The DFA may report that e.g. insn requires 2 cycles to be
7091 : issued, but on the next cycle it says that insn is ready
7092 : to go. Check this here. */
7093 827 : if (!after_stall
7094 827 : && real_insn
7095 186 : && haifa_cost > 0
7096 839 : && estimate_insn_cost (insn, curr_state) == 0)
7097 : break;
7098 :
7099 : /* When the data dependency stall is longer than the DFA stall,
7100 : and when we have issued exactly issue_rate insns and stalled,
7101 : it could be that after this longer stall the insn will again
7102 : become unavailable to the DFA restrictions. Looks strange
7103 : but happens e.g. on x86-64. So recheck DFA on the last
7104 : iteration. */
7105 827 : if ((after_stall || all_issued)
7106 : && real_insn
7107 691 : && haifa_cost == 0)
7108 547 : haifa_cost = estimate_insn_cost (insn, curr_state);
7109 : }
7110 :
7111 671 : haifa_clock += i;
7112 671 : if (sched_verbose >= 2)
7113 0 : sel_print ("haifa clock: %d\n", haifa_clock);
7114 : }
7115 : else
7116 : gcc_assert (haifa_cost == 0);
7117 :
7118 1058 : if (sched_verbose >= 2)
7119 0 : sel_print ("Haifa cost for insn %d: %d\n", INSN_UID (insn), haifa_cost);
7120 :
7121 1058 : if (targetm.sched.dfa_new_cycle)
7122 0 : while (targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, insn,
7123 : haifa_last_clock, haifa_clock,
7124 : &sort_p))
7125 : {
7126 0 : advance_state (curr_state);
7127 0 : issued_insns = 0;
7128 0 : haifa_clock++;
7129 0 : if (sched_verbose >= 2)
7130 : {
7131 0 : sel_print ("advance_state (dfa_new_cycle)\n");
7132 0 : debug_state (curr_state);
7133 0 : sel_print ("haifa clock: %d\n", haifa_clock + 1);
7134 : }
7135 : }
7136 :
7137 1058 : if (real_insn)
7138 : {
7139 1053 : static state_t temp = NULL;
7140 :
7141 1053 : if (!temp)
7142 26 : temp = xmalloc (dfa_state_size);
7143 1053 : memcpy (temp, curr_state, dfa_state_size);
7144 :
7145 1053 : cost = state_transition (curr_state, insn);
7146 1053 : if (memcmp (temp, curr_state, dfa_state_size))
7147 1050 : issued_insns++;
7148 :
7149 1053 : if (sched_verbose >= 2)
7150 : {
7151 0 : sel_print ("scheduled insn %d, clock %d\n", INSN_UID (insn),
7152 : haifa_clock + 1);
7153 0 : debug_state (curr_state);
7154 : }
7155 1053 : gcc_assert (cost < 0);
7156 : }
7157 :
7158 1058 : if (targetm.sched.variable_issue)
7159 0 : targetm.sched.variable_issue (sched_dump, sched_verbose, insn, 0);
7160 :
7161 1058 : INSN_SCHED_CYCLE (insn) = haifa_clock;
7162 :
7163 1058 : last_clock = clock;
7164 1058 : haifa_last_clock = haifa_clock;
7165 : }
7166 104 : }
7167 :
7168 : /* Put TImode markers on insns starting a new issue group. */
7169 : static void
7170 610 : put_TImodes (void)
7171 : {
7172 610 : int last_clock = -1;
7173 610 : insn_t insn;
7174 :
7175 3931 : for (insn = current_sched_info->head; insn != current_sched_info->next_tail;
7176 3321 : insn = NEXT_INSN (insn))
7177 : {
7178 3321 : int cost, clock;
7179 :
7180 3321 : if (!INSN_P (insn))
7181 126 : continue;
7182 :
7183 3195 : clock = INSN_SCHED_CYCLE (insn);
7184 3195 : cost = (last_clock == -1) ? 1 : clock - last_clock;
7185 :
7186 2571 : gcc_assert (cost >= 0);
7187 :
7188 3195 : if (issue_rate > 1
7189 3195 : && GET_CODE (PATTERN (insn)) != USE
7190 6364 : && GET_CODE (PATTERN (insn)) != CLOBBER)
7191 : {
7192 3163 : if (reload_completed && cost > 0)
7193 2422 : PUT_MODE (insn, TImode);
7194 :
7195 : last_clock = clock;
7196 : }
7197 :
7198 3195 : if (sched_verbose >= 2)
7199 0 : sel_print ("Cost for insn %d is %d\n", INSN_UID (insn), cost);
7200 : }
7201 610 : }
7202 :
7203 : /* Perform MD_FINISH on EBBs comprising current region. When
7204 : RESET_SCHED_CYCLES_P is true, run a pass emulating the scheduler
7205 : to produce correct sched cycles on insns. */
7206 : static void
7207 531 : sel_region_target_finish (bool reset_sched_cycles_p)
7208 : {
7209 531 : int i;
7210 531 : bitmap scheduled_blocks = BITMAP_ALLOC (NULL);
7211 :
7212 1846 : for (i = 0; i < current_nr_blocks; i++)
7213 : {
7214 784 : if (bitmap_bit_p (scheduled_blocks, i))
7215 123 : continue;
7216 :
7217 : /* While pipelining outer loops, skip bundling for loop
7218 : preheaders. Those will be rescheduled in the outer loop. */
7219 661 : if (sel_is_loop_preheader_p (EBB_FIRST_BB (i)))
7220 51 : continue;
7221 :
7222 610 : find_ebb_boundaries (EBB_FIRST_BB (i), scheduled_blocks);
7223 :
7224 610 : if (no_real_insns_p (current_sched_info->head, current_sched_info->tail))
7225 0 : continue;
7226 :
7227 610 : if (reset_sched_cycles_p)
7228 104 : reset_sched_cycles_in_current_ebb ();
7229 :
7230 610 : if (targetm.sched.init)
7231 0 : targetm.sched.init (sched_dump, sched_verbose, -1);
7232 :
7233 610 : put_TImodes ();
7234 :
7235 610 : if (targetm.sched.finish)
7236 : {
7237 0 : targetm.sched.finish (sched_dump, sched_verbose);
7238 :
7239 : /* Extend luids so that insns generated by the target will
7240 : get zero luid. */
7241 0 : sched_extend_luids ();
7242 : }
7243 : }
7244 :
7245 531 : BITMAP_FREE (scheduled_blocks);
7246 531 : }
7247 :
7248 : /* Free the scheduling data for the current region. When RESET_SCHED_CYCLES_P
7249 : is true, make an additional pass emulating scheduler to get correct insn
7250 : cycles for md_finish calls. */
7251 : static void
7252 770 : sel_region_finish (bool reset_sched_cycles_p)
7253 : {
7254 770 : simplify_changed_insns ();
7255 770 : sched_finish_ready_list ();
7256 770 : free_nop_pool ();
7257 :
7258 : /* Free the vectors. */
7259 770 : vec_av_set.release ();
7260 770 : BITMAP_FREE (current_copies);
7261 770 : BITMAP_FREE (current_originators);
7262 770 : BITMAP_FREE (code_motion_visited_blocks);
7263 770 : vinsn_vec_free (vec_bookkeeping_blocked_vinsns);
7264 770 : vinsn_vec_free (vec_target_unavailable_vinsns);
7265 :
7266 : /* If LV_SET of the region head should be updated, do it now because
7267 : there will be no other chance. */
7268 770 : {
7269 770 : succ_iterator si;
7270 770 : insn_t insn;
7271 :
7272 1540 : FOR_EACH_SUCC_1 (insn, si, bb_note (EBB_FIRST_BB (0)),
7273 : SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
7274 : {
7275 770 : basic_block bb = BLOCK_FOR_INSN (insn);
7276 :
7277 770 : if (!BB_LV_SET_VALID_P (bb))
7278 0 : compute_live (insn);
7279 : }
7280 : }
7281 :
7282 : /* Emulate the Haifa scheduler for bundling. */
7283 770 : if (reload_completed)
7284 531 : sel_region_target_finish (reset_sched_cycles_p);
7285 :
7286 770 : sel_finish_global_and_expr ();
7287 :
7288 770 : BITMAP_FREE (forced_ebb_heads);
7289 :
7290 770 : free_nop_vinsn ();
7291 :
7292 770 : finish_deps_global ();
7293 770 : sched_finish_luids ();
7294 770 : h_d_i_d.release ();
7295 :
7296 770 : sel_finish_bbs ();
7297 770 : BITMAP_FREE (blocks_to_reschedule);
7298 :
7299 770 : sel_unregister_cfg_hooks ();
7300 :
7301 770 : max_issue_size = 0;
7302 770 : }
7303 :
7304 :
7305 : /* Functions that implement the scheduler driver. */
7306 :
7307 : /* Schedule a parallel instruction group on each of FENCES. MAX_SEQNO
7308 : is the current maximum seqno. SCHEDULED_INSNS_TAILPP is the list
7309 : of insns scheduled -- these would be postprocessed later. */
7310 : static void
7311 1736 : schedule_on_fences (flist_t fences, int max_seqno,
7312 : ilist_t **scheduled_insns_tailpp)
7313 : {
7314 1736 : flist_t old_fences = fences;
7315 :
7316 1736 : if (sched_verbose >= 1)
7317 : {
7318 0 : sel_print ("\nScheduling on fences: ");
7319 0 : dump_flist (fences);
7320 0 : sel_print ("\n");
7321 : }
7322 :
7323 1736 : scheduled_something_on_previous_fence = false;
7324 3614 : for (; fences; fences = FLIST_NEXT (fences))
7325 : {
7326 : fence_t fence = NULL;
7327 : int seqno = 0;
7328 : flist_t fences2;
7329 : bool first_p = true;
7330 :
7331 : /* Choose the next fence group to schedule.
7332 : The fact that insn can be scheduled only once
7333 : on the cycle is guaranteed by two properties:
7334 : 1. seqnos of parallel groups decrease with each iteration.
7335 : 2. If is_ineligible_successor () sees the larger seqno, it
7336 : checks if candidate insn is_in_current_fence_p (). */
7337 4070 : for (fences2 = old_fences; fences2; fences2 = FLIST_NEXT (fences2))
7338 : {
7339 2192 : fence_t f = FLIST_FENCE (fences2);
7340 :
7341 2192 : if (!FENCE_PROCESSED_P (f))
7342 : {
7343 2035 : int i = INSN_SEQNO (FENCE_INSN (f));
7344 :
7345 2035 : if (first_p || i > seqno)
7346 : {
7347 1947 : seqno = i;
7348 1947 : fence = f;
7349 1947 : first_p = false;
7350 : }
7351 : else
7352 : /* ??? Seqnos of different groups should be different. */
7353 : gcc_assert (1 || i != seqno);
7354 : }
7355 : }
7356 :
7357 1878 : gcc_assert (fence);
7358 :
7359 : /* As FENCE is nonnull, SEQNO is initialized. */
7360 1878 : seqno -= max_seqno + 1;
7361 1878 : fill_insns (fence, seqno, scheduled_insns_tailpp);
7362 1878 : FENCE_PROCESSED_P (fence) = true;
7363 : }
7364 :
7365 : /* All av_sets are invalidated by GLOBAL_LEVEL increase, thus we
7366 : don't need to keep bookkeeping-invalidated and target-unavailable
7367 : vinsns any more. */
7368 1736 : vinsn_vec_clear (&vec_bookkeeping_blocked_vinsns);
7369 1736 : vinsn_vec_clear (&vec_target_unavailable_vinsns);
7370 1736 : }
7371 :
7372 : /* Calculate MIN_SEQNO and MAX_SEQNO. */
7373 : static void
7374 1736 : find_min_max_seqno (flist_t fences, int *min_seqno, int *max_seqno)
7375 : {
7376 1736 : *min_seqno = *max_seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7377 :
7378 : /* The first element is already processed. */
7379 1878 : while ((fences = FLIST_NEXT (fences)))
7380 : {
7381 142 : int seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7382 :
7383 142 : if (*min_seqno > seqno)
7384 73 : *min_seqno = seqno;
7385 69 : else if (*max_seqno < seqno)
7386 64 : *max_seqno = seqno;
7387 : }
7388 1736 : }
7389 :
7390 : /* Calculate new fences from FENCES. Write the current time to PTIME. */
7391 : static flist_t
7392 1736 : calculate_new_fences (flist_t fences, int orig_max_seqno, int *ptime)
7393 : {
7394 1736 : flist_t old_fences = fences;
7395 1736 : struct flist_tail_def _new_fences, *new_fences = &_new_fences;
7396 1736 : int max_time = 0;
7397 :
7398 1736 : flist_tail_init (new_fences);
7399 3614 : for (; fences; fences = FLIST_NEXT (fences))
7400 : {
7401 1878 : fence_t fence = FLIST_FENCE (fences);
7402 1878 : insn_t insn;
7403 :
7404 1878 : if (!FENCE_BNDS (fence))
7405 : {
7406 : /* This fence doesn't have any successors. */
7407 920 : if (!FENCE_SCHEDULED_P (fence))
7408 : {
7409 : /* Nothing was scheduled on this fence. */
7410 192 : int seqno;
7411 :
7412 192 : insn = FENCE_INSN (fence);
7413 192 : seqno = INSN_SEQNO (insn);
7414 192 : gcc_assert (seqno > 0 && seqno <= orig_max_seqno);
7415 :
7416 192 : if (sched_verbose >= 1)
7417 0 : sel_print ("Fence %d[%d] has not changed\n",
7418 : INSN_UID (insn),
7419 0 : BLOCK_NUM (insn));
7420 192 : move_fence_to_fences (fences, new_fences);
7421 : }
7422 : }
7423 : else
7424 958 : extract_new_fences_from (fences, new_fences, orig_max_seqno);
7425 1878 : max_time = MAX (max_time, FENCE_CYCLE (fence));
7426 : }
7427 :
7428 1736 : flist_clear (&old_fences);
7429 1736 : *ptime = max_time;
7430 1736 : return FLIST_TAIL_HEAD (new_fences);
7431 : }
7432 :
7433 : /* Update seqnos of insns given by PSCHEDULED_INSNS. MIN_SEQNO and MAX_SEQNO
7434 : are the miminum and maximum seqnos of the group, HIGHEST_SEQNO_IN_USE is
7435 : the highest seqno used in a region. Return the updated highest seqno. */
7436 : static int
7437 1736 : update_seqnos_and_stage (int min_seqno, int max_seqno,
7438 : int highest_seqno_in_use,
7439 : ilist_t *pscheduled_insns)
7440 : {
7441 1736 : int new_hs;
7442 1736 : ilist_iterator ii;
7443 1736 : insn_t insn;
7444 :
7445 : /* Actually, new_hs is the seqno of the instruction, that was
7446 : scheduled first (i.e. it is the first one in SCHEDULED_INSNS). */
7447 1736 : if (*pscheduled_insns)
7448 : {
7449 1555 : new_hs = (INSN_SEQNO (ILIST_INSN (*pscheduled_insns))
7450 1555 : + highest_seqno_in_use + max_seqno - min_seqno + 2);
7451 1555 : gcc_assert (new_hs > highest_seqno_in_use);
7452 : }
7453 : else
7454 : new_hs = highest_seqno_in_use;
7455 :
7456 6637 : FOR_EACH_INSN (insn, ii, *pscheduled_insns)
7457 : {
7458 4901 : gcc_assert (INSN_SEQNO (insn) < 0);
7459 4901 : INSN_SEQNO (insn) += highest_seqno_in_use + max_seqno - min_seqno + 2;
7460 4901 : gcc_assert (INSN_SEQNO (insn) <= new_hs);
7461 :
7462 : /* When not pipelining, purge unneeded insn info on the scheduled insns.
7463 : For example, having reg_last array of INSN_DEPS_CONTEXT in memory may
7464 : require > 1GB of memory e.g. on limit-fnargs.c. */
7465 4901 : if (! pipelining_p)
7466 3640 : free_data_for_scheduled_insn (insn);
7467 : }
7468 :
7469 1736 : ilist_clear (pscheduled_insns);
7470 1736 : global_level++;
7471 :
7472 1736 : return new_hs;
7473 : }
7474 :
7475 : /* The main driver for scheduling a region. This function is responsible
7476 : for correct propagation of fences (i.e. scheduling points) and creating
7477 : a group of parallel insns at each of them. It also supports
7478 : pipelining. ORIG_MAX_SEQNO is the maximal seqno before this pass
7479 : of scheduling. */
7480 : static void
7481 790 : sel_sched_region_2 (int orig_max_seqno)
7482 : {
7483 790 : int highest_seqno_in_use = orig_max_seqno;
7484 790 : int max_time = 0;
7485 :
7486 790 : stat_bookkeeping_copies = 0;
7487 790 : stat_insns_needed_bookkeeping = 0;
7488 790 : stat_renamed_scheduled = 0;
7489 790 : stat_substitutions_total = 0;
7490 790 : num_insns_scheduled = 0;
7491 :
7492 2526 : while (fences)
7493 : {
7494 1736 : int min_seqno, max_seqno;
7495 1736 : ilist_t scheduled_insns = NULL;
7496 1736 : ilist_t *scheduled_insns_tailp = &scheduled_insns;
7497 :
7498 1736 : find_min_max_seqno (fences, &min_seqno, &max_seqno);
7499 1736 : schedule_on_fences (fences, max_seqno, &scheduled_insns_tailp);
7500 1736 : fences = calculate_new_fences (fences, orig_max_seqno, &max_time);
7501 1736 : highest_seqno_in_use = update_seqnos_and_stage (min_seqno, max_seqno,
7502 : highest_seqno_in_use,
7503 : &scheduled_insns);
7504 : }
7505 :
7506 790 : if (sched_verbose >= 1)
7507 : {
7508 0 : sel_print ("Total scheduling time: %d cycles\n", max_time);
7509 0 : sel_print ("Scheduled %d bookkeeping copies, %d insns needed "
7510 : "bookkeeping, %d insns renamed, %d insns substituted\n",
7511 : stat_bookkeeping_copies,
7512 : stat_insns_needed_bookkeeping,
7513 : stat_renamed_scheduled,
7514 : stat_substitutions_total);
7515 : }
7516 790 : }
7517 :
7518 : /* Schedule a region. When pipelining, search for possibly never scheduled
7519 : bookkeeping code and schedule it. Reschedule pipelined code without
7520 : pipelining after. */
7521 : static void
7522 770 : sel_sched_region_1 (void)
7523 : {
7524 770 : int orig_max_seqno;
7525 :
7526 : /* Remove empty blocks that might be in the region from the beginning. */
7527 770 : purge_empty_blocks ();
7528 :
7529 770 : orig_max_seqno = init_seqno (NULL, NULL);
7530 770 : gcc_assert (orig_max_seqno >= 1);
7531 :
7532 : /* When pipelining outer loops, create fences on the loop header,
7533 : not preheader. */
7534 770 : fences = NULL;
7535 770 : if (current_loop_nest)
7536 56 : init_fences (BB_END (EBB_FIRST_BB (0)));
7537 : else
7538 714 : init_fences (bb_note (EBB_FIRST_BB (0)));
7539 770 : global_level = 1;
7540 :
7541 770 : sel_sched_region_2 (orig_max_seqno);
7542 :
7543 770 : gcc_assert (fences == NULL);
7544 :
7545 770 : if (pipelining_p)
7546 : {
7547 54 : int i;
7548 54 : basic_block bb;
7549 54 : struct flist_tail_def _new_fences;
7550 54 : flist_tail_t new_fences = &_new_fences;
7551 54 : bool do_p = true;
7552 :
7553 54 : pipelining_p = false;
7554 54 : max_ws = MIN (max_ws, issue_rate * 3 / 2);
7555 54 : bookkeeping_p = false;
7556 54 : enable_schedule_as_rhs_p = false;
7557 :
7558 : /* Schedule newly created code, that has not been scheduled yet. */
7559 54 : do_p = true;
7560 :
7561 182 : while (do_p)
7562 : {
7563 514 : do_p = false;
7564 :
7565 514 : for (i = 0; i < current_nr_blocks; i++)
7566 : {
7567 440 : basic_block bb = EBB_FIRST_BB (i);
7568 :
7569 440 : if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7570 : {
7571 74 : if (! bb_ends_ebb_p (bb))
7572 28 : bitmap_set_bit (blocks_to_reschedule, bb_next_bb (bb)->index);
7573 74 : if (sel_bb_empty_p (bb))
7574 : {
7575 0 : bitmap_clear_bit (blocks_to_reschedule, bb->index);
7576 0 : continue;
7577 : }
7578 74 : clear_outdated_rtx_info (bb);
7579 74 : if (sel_insn_is_speculation_check (BB_END (bb))
7580 74 : && JUMP_P (BB_END (bb)))
7581 0 : bitmap_set_bit (blocks_to_reschedule,
7582 0 : BRANCH_EDGE (bb)->dest->index);
7583 : }
7584 366 : else if (! sel_bb_empty_p (bb)
7585 366 : && INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0)
7586 40 : bitmap_set_bit (blocks_to_reschedule, bb->index);
7587 : }
7588 :
7589 400 : for (i = 0; i < current_nr_blocks; i++)
7590 : {
7591 346 : bb = EBB_FIRST_BB (i);
7592 :
7593 : /* While pipelining outer loops, skip bundling for loop
7594 : preheaders. Those will be rescheduled in the outer
7595 : loop. */
7596 346 : if (sel_is_loop_preheader_p (bb))
7597 : {
7598 74 : clear_outdated_rtx_info (bb);
7599 74 : continue;
7600 : }
7601 :
7602 272 : if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7603 : {
7604 20 : flist_tail_init (new_fences);
7605 :
7606 20 : orig_max_seqno = init_seqno (blocks_to_reschedule, bb);
7607 :
7608 : /* Mark BB as head of the new ebb. */
7609 20 : bitmap_set_bit (forced_ebb_heads, bb->index);
7610 :
7611 20 : gcc_assert (fences == NULL);
7612 :
7613 20 : init_fences (bb_note (bb));
7614 :
7615 20 : sel_sched_region_2 (orig_max_seqno);
7616 :
7617 20 : do_p = true;
7618 20 : break;
7619 : }
7620 : }
7621 : }
7622 : }
7623 770 : }
7624 :
7625 : /* Schedule the RGN region. */
7626 : void
7627 782 : sel_sched_region (int rgn)
7628 : {
7629 782 : bool schedule_p;
7630 782 : bool reset_sched_cycles_p;
7631 :
7632 782 : if (sel_region_init (rgn))
7633 : return;
7634 :
7635 770 : if (sched_verbose >= 1)
7636 0 : sel_print ("Scheduling region %d\n", rgn);
7637 :
7638 770 : schedule_p = (!sched_is_disabled_for_current_region_p ()
7639 770 : && dbg_cnt (sel_sched_region_cnt));
7640 770 : reset_sched_cycles_p = pipelining_p;
7641 770 : if (schedule_p)
7642 770 : sel_sched_region_1 ();
7643 : else
7644 : {
7645 : /* Schedule always selecting the next insn to make the correct data
7646 : for bundling or other later passes. */
7647 0 : pipelining_p = false;
7648 0 : reset_sched_cycles_p = false;
7649 0 : force_next_insn = 1;
7650 0 : sel_sched_region_1 ();
7651 0 : force_next_insn = 0;
7652 : }
7653 770 : sel_region_finish (reset_sched_cycles_p);
7654 : }
7655 :
7656 : /* Perform global init for the scheduler. */
7657 : static void
7658 131 : sel_global_init (void)
7659 : {
7660 : /* Remove empty blocks: their presence can break assumptions elsewhere,
7661 : e.g. the logic to invoke update_liveness_on_insn in sel_region_init. */
7662 131 : cleanup_cfg (0);
7663 :
7664 131 : calculate_dominance_info (CDI_DOMINATORS);
7665 131 : alloc_sched_pools ();
7666 :
7667 : /* Setup the infos for sched_init. */
7668 131 : sel_setup_sched_infos ();
7669 131 : setup_sched_dump ();
7670 :
7671 131 : sched_rgn_init (false);
7672 131 : sched_init ();
7673 :
7674 131 : sched_init_bbs ();
7675 : /* Reset AFTER_RECOVERY if it has been set by the 1st scheduler pass. */
7676 131 : after_recovery = 0;
7677 131 : can_issue_more = issue_rate;
7678 :
7679 131 : sched_extend_target ();
7680 131 : sched_deps_init (true);
7681 131 : setup_nop_and_exit_insns ();
7682 131 : sel_extend_global_bb_info ();
7683 131 : init_lv_sets ();
7684 131 : init_hard_regs_data ();
7685 131 : }
7686 :
7687 : /* Free the global data of the scheduler. */
7688 : static void
7689 131 : sel_global_finish (void)
7690 : {
7691 131 : free_bb_note_pool ();
7692 131 : free_lv_sets ();
7693 131 : sel_finish_global_bb_info ();
7694 :
7695 131 : free_regset_pool ();
7696 131 : free_nop_and_exit_insns ();
7697 :
7698 131 : sched_rgn_finish ();
7699 131 : sched_deps_finish ();
7700 131 : sched_finish ();
7701 :
7702 131 : if (current_loops)
7703 43 : sel_finish_pipelining ();
7704 :
7705 131 : free_sched_pools ();
7706 131 : free_dominance_info (CDI_DOMINATORS);
7707 131 : }
7708 :
7709 : /* Return true when we need to skip selective scheduling. Used for debugging. */
7710 : bool
7711 131 : maybe_skip_selective_scheduling (void)
7712 : {
7713 131 : return ! dbg_cnt (sel_sched_cnt);
7714 : }
7715 :
7716 : /* The entry point. */
7717 : void
7718 131 : run_selective_scheduling (void)
7719 : {
7720 131 : int rgn;
7721 :
7722 131 : if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
7723 : return;
7724 :
7725 131 : sel_global_init ();
7726 :
7727 1044 : for (rgn = 0; rgn < nr_regions; rgn++)
7728 782 : sel_sched_region (rgn);
7729 :
7730 131 : sel_global_finish ();
7731 : }
7732 :
7733 : #endif
|
