| Quellcodebibliothek Statistik Leitseite products/Sources/formale Sprachen/C/MariaDB/sql/ (MariaDB Server Version 8.1-8.4©) Datei vom 1.9.2026 mit Größe 1 MB |
|
Quellcode-Bibliothek sql_select.cc Sprache: C |
|||||||||||||||
|
/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. Copyright (c) 2009, 2024, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /** @file @brief mysql_select and join optimization @defgroup Query_Optimizer Query Optimizer @{ */ #include "mariadb.h" #include "sql_priv.h" #include "unireg.h" #include "sql_select.h" #include "sql_cache.h" // query_cache_* #include "sql_table.h" // primary_key_name #include "probes_mysql.h" #include "key.h" // key_copy, key_cmp, key_cmp_if_same #include "lock.h" // mysql_unlock_some_tables, // mysql_unlock_read_tables #include "sql_show.h" // append_identifier #include "sql_base.h" // setup_wild, setup_fields, fill_record #include "sql_parse.h" // check_stack_overrun #include "sql_partition.h" // make_used_partitions_str #include "sql_test.h" // print_where, print_keyuse_array, // print_sjm, print_plan, TEST_join #include "records.h" // init_read_record, end_read_record #include "filesort.h" // filesort_free_buffers #include "filesort_utils.h" // get_qsort_sort_cost #include "sql_union.h" // mysql_union #include "opt_subselect.h" #include "opt_context_store_replay.h" #include "sql_derived.h" #include "sql_statistics.h" #include "sql_cte.h" #include "sql_window.h" #include "tztime.h" #include "debug_sync.h" // DEBUG_SYNC #include <m_ctype.h> #include <my_bit.h> #include <hash.h> #include <ft_global.h> #include "sys_vars_shared.h" #include "sp_head.h" #include "sp_rcontext.h" #include "rowid_filter.h" #include "select_handler.h" #include "my_json_writer.h" #include "opt_trace.h" #include "derived_handler.h" #include "create_tmp_table.h" #include "optimizer_defaults.h" #include "derived_handler.h" #include "opt_hints.h" #include "opt_group_by_cardinality.h" /* A key part number that means we're using a fulltext scan. In order not to confuse it with regular equalities, we need to pick a number that's greater than MAX_REF_PARTS. Hash Join code stores field->field_index in KEYUSE::keypart, so the number needs to be bigger than MAX_FIELDS, also. CAUTION: sql_test.cc has its own definition of FT_KEYPART. */ #define FT_KEYPART (MAX_FIELDS+10) /* We assume that when we do hash join, only 10 % rows in the hash will match the current found row. */ #define HASH_FANOUT 0.1 /* The following is used to check that A <= B, but with some margin as the calculation is done slightly differently (mathematically correct, but double calculations are not exact). This is only used when comparing read rows and output rows, which means that we can assume that both values are >= 0 and B cannot be notable smaller than A. */ #define crash_if_first_double_is_bigger(A,B) DBUG_ASSERT(((A) == 0.0 && (B) == 0.0) || (A)/( #define double_to_rows(A) ((A) >= ((double)HA_ROWS_MAX) ? HA_ROWS_MAX : (ha_rows) (A)) #define double_to_ulonglong(A) ((A) >= ((double)ULONGLONG_MAX) ? ULONGLONG_MAX : (ulonglo /* Used to ensure that costs are calculate the same way */ inline bool compare_cost(double a, double b) { DBUG_ASSERT(a >= 0.0 && b >= 0.0); return (a >= b - b/10000000.0 && a <= b+b/10000000.0); } inline double safe_filtered(double a, double b) { return b != 0 ? a/b*100.0 : 0.0; } const char *join_type_str[]={ "UNKNOWN","system","const","eq_ref","ref", "MAYBE_REF","ALL","range","index","fulltext", "ref_or_null","unique_subquery","index_subquery", "index_merge", "hash_ALL", "hash_range", "hash_index", "hash_index_merge" }; static const Lex_ident_column group_key= "group_key"_Lex_ident_column; static const Lex_ident_column distinct_key= "distinct_key"_Lex_ident_column; struct st_sargable_param; static bool make_join_statistics(JOIN *join, List<TABLE_LIST> &leaves, DYNAMIC_ARRAY *keyuse); static bool update_ref_and_keys(THD *thd, DYNAMIC_ARRAY *keyuse, JOIN_TAB *join_tab, uint tables, COND *conds, table_map table_map, SELECT_LEX *select_lex, SARGABLE_PARAM **sargables); static int sort_keyuse(const void *a, const void *b); static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables); static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, bool allow_full_scan, table_map used_tables); static bool get_quick_record_count(THD *thd, SQL_SELECT *select, TABLE *table, const key_map *keys,ha_rows limit, ha_rows *quick_count); static void optimize_straight_join(JOIN *join, table_map join_tables); static bool greedy_search(JOIN *join, table_map remaining_tables, uint depth, uint use_cond_selectivity); enum enum_best_search { SEARCH_ABORT= -2, SEARCH_ERROR= -1, SEARCH_OK= 0, SEARCH_FOUND_EDGE=1 }; static enum_best_search best_extension_by_limited_search(JOIN *join, table_map remaining_tables, uint idx, double record_count, double read_time, uint depth, uint use_cond_selectivity, table_map *processed_eq_ref_tables); static uint determine_search_depth(JOIN* join); C_MODE_START static int join_tab_cmp(void *dummy, const void* ptr1, const void* ptr2); static int join_tab_cmp_straight(void *dummy, const void* ptr1, const void* ptr2); static int join_tab_cmp_embedded_first(void *emb, const void* ptr1, const void *ptr2); C_MODE_END static uint cache_record_length(JOIN *join,uint index); static store_key *get_store_key(THD *thd, KEYUSE *keyuse, table_map used_tables, KEY_PART_INFO *key_part, uchar *key_buff, uint maybe_null); static bool make_outerjoin_info(JOIN *join); static Item* make_cond_after_sjm(THD *thd, Item *root_cond, Item *cond, table_map tables, table_map sjm_tables, bool inside_or_clause); static bool make_join_select(JOIN *join,SQL_SELECT *select,COND *item); static void revise_cache_usage(JOIN_TAB *join_tab); static bool make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after); static bool only_eq_ref_tables(JOIN *join, ORDER *order, table_map tables); static void update_depend_map(JOIN *join); static void update_depend_map_for_order(JOIN *join, ORDER *order); static ORDER *remove_const(JOIN *join,ORDER *first_order,COND *cond, bool change_list, bool *simple_order); static int return_zero_rows(JOIN *join, select_result *res, List<TABLE_LIST> *tables, List<Item> *fields, bool send_row, ulonglong select_options, const char *info, Item *having, List<Item> *all_fields); static COND *build_equal_items(JOIN *join, COND *cond, COND_EQUAL *inherited, List<TABLE_LIST> *join_list, bool ignore_on_conds, COND_EQUAL **cond_equal_ref, bool link_equal_fields= FALSE); static COND* substitute_for_best_equal_field(THD *thd, JOIN_TAB *context_tab, COND *cond, COND_EQUAL *cond_equal, void *table_join_idx, bool do_substitution); static COND *simplify_joins(JOIN *join, List<TABLE_LIST> *join_list, COND *conds, bool top, bool in_sj); static bool check_interleaving_with_nj(JOIN_TAB *next); static void restore_prev_nj_state(JOIN_TAB *last); static uint reset_nj_counters(JOIN *join, List<TABLE_LIST> *join_list); static uint build_bitmap_for_nested_joins(List<TABLE_LIST> *join_list, uint first_unused); static COND *optimize_cond(JOIN *join, COND *conds, List<TABLE_LIST> *join_list, bool ignore_on_conds, Item::cond_result *cond_value, COND_EQUAL **cond_equal, int flags= 0); bool const_expression_in_where(COND *conds,Item *item, Item **comp_item); static int do_select(JOIN *join, Procedure *procedure); static enum_nested_loop_state evaluate_join_record(JOIN *, JOIN_TAB *, int); static enum_nested_loop_state evaluate_null_complemented_join_record(JOIN *join, JOIN_TAB *join_tab); static enum_nested_loop_state end_send(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); static enum_nested_loop_state end_write(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); static enum_nested_loop_state end_update(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); static enum_nested_loop_state end_unique_update(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); static int join_read_const_table(THD *thd, JOIN_TAB *tab, POSITION *pos); static int join_read_system(JOIN_TAB *tab); static int join_read_const(JOIN_TAB *tab); static int join_read_key(JOIN_TAB *tab); static void join_read_key_unlock_row(st_join_table *tab); static void join_const_unlock_row(JOIN_TAB *tab); static int join_read_always_key(JOIN_TAB *tab); static int join_read_last_key(JOIN_TAB *tab); static int join_no_more_records(READ_RECORD *info); static int join_read_next(READ_RECORD *info); static int join_hlindex_read_next(READ_RECORD *info); static int join_init_quick_read_record(JOIN_TAB *tab); static quick_select_return test_if_quick_select(JOIN_TAB *tab); static int test_if_use_dynamic_range_scan(JOIN_TAB *join_tab); static int join_read_first(JOIN_TAB *tab); static int join_read_next_same(READ_RECORD *info); static int join_read_last(JOIN_TAB *tab); static int join_read_prev_same(READ_RECORD *info); static int join_read_prev(READ_RECORD *info); static int join_ft_read_first(JOIN_TAB *tab); static int join_ft_read_next(READ_RECORD *info); int join_read_always_key_or_null(JOIN_TAB *tab); int join_read_next_same_or_null(READ_RECORD *info); COND *make_cond_for_table(THD *thd, Item *cond,table_map table, table_map used_table, int join_tab_idx_arg, bool exclude_expensive_cond, bool retain_ref_cond); COND *make_cond_for_table_from_pred(THD *thd, Item *root_cond, Item *cond, table_map tables, table_map used_table, int join_tab_idx_arg, bool exclude_expensive_cond, bool retain_ref_cond, bool is_top_and_level); static Item* part_of_refkey(TABLE *form,Field *field); static bool test_if_cheaper_ordering(bool in_join_optimizer, const JOIN_TAB *tab, ORDER *order, TABLE *table, key_map usable_keys, int key, ha_rows select_limit, int *new_key, int *new_key_direction, ha_rows *new_select_limit, double *new_read_time, uint *new_used_key_parts= NULL, uint *saved_best_key_parts= NULL); static int test_if_order_by_key(JOIN *, ORDER *, TABLE *, uint, uint *); static bool test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order, ha_rows select_limit, bool no_changes, const key_map *map, bool *fatal_error); static bool list_contains_unique_index(TABLE *table, bool (*find_func) (Field *, void *), void *data); static bool find_field_in_item_list (Field *field, void *data); static bool find_field_in_order_list (Field *field, void *data); int create_sort_index(THD *thd, JOIN *join, JOIN_TAB *tab, Filesort *fsort); static int remove_dup_with_compare(THD *thd, TABLE *entry, Field **field, SORT_FIELD *sortorder, ulong keylength, Item *having); static int remove_dup_with_hash_index(THD *thd,TABLE *table, uint field_count, Field **first_field, SORT_FIELD *sortorder, ulong key_length,Item *having); static bool cmp_buffer_with_ref(THD *thd, TABLE *table, TABLE_REF *tab_ref); static bool setup_new_fields(THD *thd, List<Item> &fields, List<Item> &all_fields, ORDER *new_order); static ORDER *create_distinct_group(THD *thd, Ref_ptr_array ref_pointer_array, ORDER *order, List<Item> &fields, List<Item> &all_fields, bool *all_order_by_fields_used); static bool test_if_subpart(ORDER *group_by, ORDER *order_by); static TABLE *get_sort_by_table(ORDER *a,ORDER *b,List<TABLE_LIST> &tables, table_map const_tables); static void calc_group_buffer(JOIN *join, ORDER *group); static bool make_group_fields(JOIN *main_join, JOIN *curr_join); static bool alloc_group_fields(JOIN *join, ORDER *group); static bool alloc_order_fields(JOIN *join, ORDER *group, uint max_number_of_elements); // Create list for using with tempory table static bool change_to_use_tmp_fields(THD *thd, Ref_ptr_array ref_pointer_array, List<Item> &new_list1, List<Item> &new_list2, uint elements, List<Item> &items); // Create list for using with tempory table static bool change_refs_to_tmp_fields(THD *thd, Ref_ptr_array ref_pointer_array, List<Item> &new_list1, List<Item> &new_list2, uint elements, List<Item> &items); static void init_tmptable_sum_functions(Item_sum **func); static void update_tmptable_sum_func(Item_sum **func,TABLE *tmp_table); static void copy_sum_funcs(Item_sum **func_ptr, Item_sum **end); static bool add_ref_to_table_cond(THD *thd, JOIN_TAB *join_tab); static bool setup_sum_funcs(THD *thd, Item_sum **func_ptr); static bool init_sum_functions(Item_sum **func, Item_sum **end); static bool update_sum_func(Item_sum **func); static void select_describe(JOIN *join, bool need_tmp_table,bool need_order, bool distinct, const char *message=NullS); static void add_group_and_distinct_keys(JOIN *join, JOIN_TAB *join_tab); static uint make_join_orderinfo(JOIN *join); static bool generate_derived_keys(DYNAMIC_ARRAY *keyuse_array); Item_equal *find_item_equal(COND_EQUAL *cond_equal, Field *field, bool *inherited_fl); JOIN_TAB *first_depth_first_tab(JOIN* join); JOIN_TAB *next_depth_first_tab(JOIN* join, JOIN_TAB* tab); static JOIN_TAB *next_breadth_first_tab(JOIN_TAB *first_top_tab, uint n_top_tabs_count, JOIN_TAB *tab); static bool find_order_in_list(THD *, Ref_ptr_array, TABLE_LIST *, ORDER *, List<Item> &, List<Item> &, bool, bool, bool); static double table_after_join_selectivity(JOIN *join, uint idx, JOIN_TAB *s, table_map rem_tables, double *records_out); void set_postjoin_aggr_write_func(JOIN_TAB *tab); static Item **get_sargable_cond(JOIN *join, TABLE *table); bool is_eq_cond_injected_for_split_opt(Item_func_eq *eq_item); void print_list_item(String *str, List_item *list, enum_query_type query_type); static bool build_notnull_conds_for_range_scans(JOIN *join, COND *cond, table_map allowed); static void build_notnull_conds_for_inner_nest_of_outer_join(JOIN *join, TABLE_LIST *nest_tbl); static void fix_items_after_optimize(THD *thd, SELECT_LEX *select_lex); static void optimize_rownum(THD *thd, SELECT_LEX_UNIT *unit, Item *cond); static bool process_direct_rownum_comparison(THD *thd, SELECT_LEX_UNIT *unit, Item *cond); static double prev_record_reads(const POSITION *positions, uint idx, table_map found_ref, double record_count, double *same_keys); static bool join_limit_shortcut_is_applicable(const JOIN *join); POSITION *join_limit_shortcut_finalize_plan(JOIN *join, double *cost); static bool find_indexes_matching_order(JOIN *, TABLE *, ORDER *, key_map *); static void init_join_plan_search_state(JOIN *join); #ifndef DBUG_OFF /* SHOW EXPLAIN testing: wait for, and serve n_calls APC requests. */ void dbug_serve_apcs(THD *thd, int n_calls) { const char *save_proc_info= thd->proc_info; /* Busy-wait for n_calls APC requests to arrive and be processed */ int n_apcs= thd->apc_target.n_calls_processed + n_calls; while (thd->apc_target.n_calls_processed < n_apcs) { /* This is so that mysqltest knows we're ready to serve requests: */ thd_proc_info(thd, "show_explain_trap"); my_sleep(30000); thd_proc_info(thd, save_proc_info); if (unlikely(thd->check_killed(1))) break; } } /* Debugging: check if @name=value, comparing as integer Intended usage: DBUG_EXECUTE_IF("show_explain_probe_2", if (dbug_user_var_equals_int(thd, "select_id", select_id)) dbug_serve_apcs(thd, 1); ); */ bool dbug_user_var_equals_int(THD *thd, const char *name, int value) { user_var_entry *var; LEX_CSTRING varname= { name, strlen(name)}; if ((var= get_variable(&thd->user_vars, &varname, FALSE))) { bool null_value; longlong var_value= var->val_int(&null_value); if (!null_value && var_value == value) return TRUE; } return FALSE; } /* Debugging : check if @name= value, comparing as string Intended usage : DBUG_EXECUTE_IF("log_slow_statement_end", if (dbug_user_var_equals_str(thd, "show_explain_probe_query", thd->query())) dbug_serve_apcs(thd, 1); ); */ bool dbug_user_var_equals_str(THD *thd, const char *name, const char* value) { user_var_entry *var; LEX_CSTRING varname= {name, strlen(name)}; if ((var= get_variable(&thd->user_vars, &varname, FALSE))) { bool null_value; String str; auto var_value= var->val_str(&null_value, &str, 10)->ptr(); if (!null_value && !strncmp(var_value, value, strlen(value))) return TRUE; } return FALSE; } #endif /* DBUG_OFF */ JOIN_TAB_RANGE *JOIN_TAB_RANGE::create(THD *thd, uint count) { JOIN_TAB *jt; JOIN_TAB_RANGE *jt_range; if (!(jt= thd->alloc<JOIN_TAB>(count)) || !(jt_range= new JOIN_TAB_RANGE)) return nullptr; jt_range->start= jt; jt_range->end= jt + count; return jt_range; } /* Intialize POSITION structure. */ POSITION::POSITION() { table= 0; records_read= cond_selectivity= read_time= records_out= records_init= 0.0; prefix_record_count= 0.0; key= 0; forced_index= 0; use_join_buffer= 0; firstmatch_with_join_buf= false; sj_strategy= SJ_OPT_NONE; n_sj_tables= 0; spl_plan= 0; range_rowid_filter_info= 0; ref_depend_map= dups_producing_tables= 0; inner_tables_handled_with_other_sjs= 0; type= JT_UNKNOWN; key_dependent= 0; dups_weedout_picker.set_empty(); firstmatch_picker.set_empty(); loosescan_picker.set_empty(); sjmat_picker.set_empty(); } void JOIN::init(THD *thd_arg, List<Item> &fields_arg, ulonglong select_options_arg, select_result *result_arg) { join_tab= 0; table= 0; table_count= 0; top_join_tab_count= 0; const_tables= 0; const_table_map= found_const_table_map= not_usable_rowid_map= 0; aggr_tables= 0; eliminated_tables= 0; join_list= 0; implicit_grouping= FALSE; sort_and_group= 0; first_record= 0; do_send_rows= 1; duplicate_rows= send_records= 0; found_records= accepted_rows= 0; fetch_limit= HA_POS_ERROR; thd= thd_arg; sum_funcs= sum_funcs2= 0; procedure= 0; having= tmp_having= having_history= 0; having_is_correlated= false; group_list_for_estimates= 0; select_options= select_options_arg; result= result_arg; lock= thd_arg->lock; select_lex= 0; //for safety select_distinct= MY_TEST(select_options & SELECT_DISTINCT); no_order= 0; simple_order= 0; simple_group= 0; ordered_index_usage= ordered_index_void; need_distinct= 0; skip_sort_order= 0; with_two_phase_optimization= 0; save_qep= 0; spl_opt_info= 0; ext_keyuses_for_splitting= 0; spl_opt_info= 0; need_tmp= 0; hidden_group_fields= 0; /*safety*/ error= 0; select= 0; return_tab= 0; ref_ptrs.reset(); items0.reset(); items1.reset(); items2.reset(); items3.reset(); zero_result_cause= 0; optimization_state= JOIN::NOT_OPTIMIZED; have_query_plan= QEP_NOT_PRESENT_YET; initialized= 0; cleaned= 0; cond_equal= 0; having_equal= 0; exec_const_cond= 0; group_optimized_away= 0; no_rows_in_result_called= 0; positions= best_positions= 0; pushdown_query= 0; original_join_tab= 0; explain= NULL; tmp_table_keep_current_rowid= 0; allowed_top_level_tables= 0; all_fields= fields_arg; if (&fields_list != &fields_arg) /* Avoid valgrind-warning */ fields_list= fields_arg; non_agg_fields.empty(); bzero((char*) &keyuse,sizeof(keyuse)); having_value= Item::COND_UNDEF; tmp_table_param.init(); tmp_table_param.end_write_records= HA_POS_ERROR; rollup.state= ROLLUP::STATE_NONE; no_const_tables= FALSE; first_select= sub_select; group_sent= 0; outer_ref_cond= pseudo_bits_cond= NULL; in_to_exists_where= NULL; in_to_exists_having= NULL; emb_sjm_nest= NULL; sjm_lookup_tables= 0; sjm_scan_tables= 0; is_orig_degenerated= false; with_ties_order_count= 0; prepared= false; sql_cmd_dml= NULL; }; static void trace_table_dependencies(THD *thd, JOIN_TAB *join_tabs, uint table_count) { DBUG_ASSERT(thd->trace_started()); Json_writer_object trace_wrapper(thd); Json_writer_array trace_dep(thd, "table_dependencies"); for (uint i= 0; i < table_count; i++) { TABLE_LIST *table_ref= join_tabs[i].tab_list; Json_writer_object trace_one_table(thd); trace_one_table. add_table_name(&join_tabs[i]). add("row_may_be_null", (bool)table_ref->table->maybe_null); const table_map map= table_ref->get_map(); DBUG_ASSERT(map < (1ULL << table_count)); for (uint j= 0; j < table_count; j++) { if (map & (1ULL << j)) { trace_one_table.add("map_bit", j); break; } } Json_writer_array depends_on(thd, "depends_on_map_bits"); Table_map_iterator it(join_tabs[i].dependent); uint dep_bit; while ((dep_bit= it++) != Table_map_iterator::BITMAP_END) depends_on.add(static_cast<longlong>(dep_bit)); } } /** This handles SELECT with and without UNION. */ bool handle_select(THD *thd, LEX *lex, select_result *result, ulonglong setup_tables_done_option) { bool res; SELECT_LEX *select_lex= lex->first_select_lex(); DBUG_ENTER("handle_select"); MYSQL_SELECT_START(thd->query()); if (select_lex->master_unit()->is_unit_op() || select_lex->master_unit()->fake_select_lex) res= mysql_union(thd, lex, result, &lex->unit, setup_tables_done_option); else { SELECT_LEX_UNIT *unit= &lex->unit; unit->set_limit(unit->global_parameters()); /* 'options' of mysql_select will be set in JOIN, as far as JOIN for every PS/SP execution new, we will not need reset this flag if setup_tables_done_option changed for next reexecution */ res= mysql_select(thd, select_lex->table_list.first, select_lex->item_list, select_lex->where, select_lex->order_list.elements + select_lex->group_list.elements, select_lex->order_list.first, select_lex->group_list.first, select_lex->having, lex->proc_list.first, select_lex->options | thd->variables.option_bits | setup_tables_done_option, result, unit, select_lex); } DBUG_PRINT("info",("res: %d is_error(): %d", res, thd->is_error())); res|= thd->is_error(); if (unlikely(res)) result->abort_result_set(); if (unlikely(thd->killed == ABORT_QUERY && !thd->no_errors)) { /* If LIMIT ROWS EXAMINED interrupted query execution continue with normal processing and produce an incomplete query result. A warning was generated by killed_for_exceeding_limit_rows(). */ thd->reset_killed(); } /* Deactivate LIMIT ROWS EXAMINED after query execution. */ thd->lex->deactivate_limit_rows_examined(); thd->killed_for_exceeding_limit_rows_warning_given= 0; MYSQL_SELECT_DONE((int) res, (ulong) thd->limit_found_rows); DBUG_RETURN(res); } /** Fix fields referenced from inner selects. @param thd Thread handle @param all_fields List of all fields used in select @param select Current select @param ref_pointer_array Array of references to Items used in current select @param group_list GROUP BY list (is NULL by default) @details The function serves 3 purposes - adds fields referenced from inner query blocks to the current select list - Decides which class to use to reference the items (Item_ref or Item_direct_ref) - fixes references (Item_ref objects) to these fields. If a field isn't already on the select list and the ref_pointer_array is provided then it is added to the all_fields list and the pointer to it is saved in the ref_pointer_array. The class to access the outer field is determined by the following rules: -#. If the outer field isn't used under an aggregate function then the Item_ref class should be used. -#. If the outer field is used under an aggregate function and this function is, in turn, aggregated in the query block where the outer field was resolved or some query nested therein, then the Item_direct_ref class should be used. Also it should be used if we are grouping by a subquery that references this outer field. The resolution is done here and not at the fix_fields() stage as it can be done only after aggregate functions are fixed and pulled up to selects where they are to be aggregated. When the class is chosen it substitutes the original field in the Item_outer_ref object. After this we proceed with fixing references (Item_outer_ref objects) to this field from inner subqueries. @return Status @retval true An error occurred. @retval false OK. */ bool fix_inner_refs(THD *thd, List<Item> &all_fields, SELECT_LEX *select, Ref_ptr_array ref_pointer_array) { Item_outer_ref *ref; /* Mark the references from the inner_refs_list that are occurred in the group by expressions. Those references will contain direct references to the referred fields. The markers are set in the found_in_group_by field of the references from the list. */ List_iterator_fast <Item_outer_ref> ref_it(select->inner_refs_list); for (ORDER *group= select->join->group_list; group; group= group->next) { (*group->item)->walk(&Item::check_inner_refs_processor, &ref_it, WALK_SUBQUERY); } while ((ref= ref_it++)) { bool direct_ref= false; Item *item= ref->outer_ref; Item **item_ref= ref->ref; Item_ref *new_ref; /* TODO: this field item already might be present in the select list. In this case instead of adding new field item we could use an existing one. The change will lead to less operations for copying fields, smaller temporary tables and less data passed through filesort. */ if (!ref_pointer_array.is_null() && !ref->found_in_select_list) { int el= all_fields.elements; ref_pointer_array[el]= item; /* Add the field item to the select list of the current select. */ all_fields.push_front(item, thd->mem_root); /* If it's needed reset each Item_ref item that refers this field with a new reference taken from ref_pointer_array. */ item_ref= &ref_pointer_array[el]; } if (ref->in_sum_func) { Item_sum *sum_func; if (ref->in_sum_func->nest_level > select->nest_level) direct_ref= TRUE; else { for (sum_func= ref->in_sum_func; sum_func && sum_func->aggr_level >= select->nest_level; sum_func= sum_func->in_sum_func) { if (sum_func->aggr_level == select->nest_level) { direct_ref= TRUE; break; } } } } else if (ref->found_in_group_by) direct_ref= TRUE; new_ref= direct_ref ? new (thd->mem_root) Item_direct_ref(thd, ref->context, item_ref, ref->table_name, ref->field_name, ref->alias_name_used) : new (thd->mem_root) Item_ref(thd, ref->context, item_ref, ref->table_name, ref->field_name, ref->alias_name_used); if (!new_ref) return TRUE; ref->outer_ref= new_ref; ref->ref= &ref->outer_ref; if (ref->fix_fields_if_needed(thd, 0)) return TRUE; thd->lex->used_tables|= item->used_tables(); thd->lex->current_select->select_list_tables|= item->used_tables(); } return false; } /** The following clauses are redundant for subqueries: DISTINCT GROUP BY if there are no aggregate functions and no HAVING clause Because redundant clauses are removed both from JOIN and select_lex, the removal is permanent. Thus, it only makes sense to call this function for normal queries and on first execution of SP/PS @param subq_select_lex select_lex that is part of a subquery predicate. This object and the associated join is modified. */ static void remove_redundant_subquery_clauses(st_select_lex *subq_select_lex) { DBUG_ENTER("remove_redundant_subquery_clauses"); Item_subselect *subq_predicate= subq_select_lex->master_unit()->item; /* The removal should happen for IN, ALL, ANY and EXISTS subqueries, which means all but single row subqueries. Example single row subqueries: a) SELECT * FROM t1 WHERE t1.a = (<single row subquery>) b) SELECT a, (<single row subquery) FROM t1 */ if (subq_predicate->substype() == Item_subselect::SINGLEROW_SUBS) DBUG_VOID_RETURN; /* A subquery that is not single row should be one of IN/ALL/ANY/EXISTS. */ DBUG_ASSERT (subq_predicate->substype() == Item_subselect::EXISTS_SUBS || subq_predicate->is_in_predicate()); if (subq_select_lex->options & SELECT_DISTINCT) { subq_select_lex->join->select_distinct= false; subq_select_lex->options&= ~SELECT_DISTINCT; DBUG_PRINT("info", ("DISTINCT removed")); } /* Remove GROUP BY if there are no aggregate functions and no HAVING clause */ if (subq_select_lex->group_list.elements && !subq_select_lex->with_sum_func && !subq_select_lex->join->having) { /* Temporary workaround for MDEV-28621: Do not remove GROUP BY expression if it has any subqueries in it. */ bool have_subquery= false; for (ORDER *ord= subq_select_lex->group_list.first; ord; ord= ord->next) { if ((*ord->item)->with_subquery()) { have_subquery= true; break; } } if (!have_subquery) { for (ORDER *ord= subq_select_lex->group_list.first; ord; ord= ord->next) { /* Do not remove the item if it is used in select list and then referred from GROUP BY clause by its name or number. Example: select (select ... ) as SUBQ ... group by SUBQ Here SUBQ cannot be removed. */ if (!ord->in_field_list) { /* Not necessary due to workaround for MDEV-28621: (*ord->item)->walk(&Item::eliminate_subselect_processor, FALSE, NULL); */ /* Remove from the JOIN::all_fields list any reference to the elements of the eliminated GROUP BY list unless it is 'in_field_list'. This is needed in order not to confuse JOIN::make_aggr_tables_info() when it constructs different structure for execution phase. */ List_iterator<Item> li(subq_select_lex->join->all_fields); Item *item; while ((item= li++)) { if (item == *ord->item) li.remove(); } } } subq_select_lex->join->group_list= NULL; subq_select_lex->group_list.empty(); DBUG_PRINT("info", ("GROUP BY removed")); } } /* TODO: This would prevent processing queries with ORDER BY ... LIMIT therefore we disable this optimization for now. Remove GROUP BY if there are no aggregate functions and no HAVING clause if (subq_select_lex->group_list.elements && !subq_select_lex->with_sum_func && !subq_select_lex->join->having) { subq_select_lex->join->group_list= NULL; subq_select_lex->group_list.empty(); } */ DBUG_VOID_RETURN; } /** Function to setup clauses without sum functions. */ static inline int setup_without_group(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, List<TABLE_LIST> &leaves, List<Item> &fields, List<Item> &all_fields, COND **conds, ORDER *order, ORDER *group, List<Window_spec> &win_specs, List<Item_window_func> &win_funcs, bool *hidden_group_fields) { int res; enum_parsing_place save_place; st_select_lex *const select= thd->lex->current_select; nesting_map save_allow_sum_func= thd->lex->allow_sum_func; /* Need to stave the value, so we can turn off only any new non_agg_field_used additions coming from the WHERE */ const bool saved_non_agg_field_used= select->non_agg_field_used(); DBUG_ENTER("setup_without_group"); thd->lex->allow_sum_func.clear_bit(select->nest_level); res= setup_conds(thd, tables, leaves, conds, &all_fields); /* it's not wrong to have non-aggregated columns in a WHERE */ select->set_non_agg_field_used(saved_non_agg_field_used); thd->lex->allow_sum_func.set_bit(select->nest_level); save_place= thd->lex->current_select->context_analysis_place; thd->lex->current_select->context_analysis_place= IN_ORDER_BY; res= res || setup_order(thd, ref_pointer_array, tables, fields, all_fields, order); thd->lex->allow_sum_func.clear_bit(select->nest_level); thd->lex->current_select->context_analysis_place= IN_GROUP_BY; res= res || setup_group(thd, ref_pointer_array, tables, fields, all_fields, group, hidden_group_fields); thd->lex->current_select->context_analysis_place= save_place; thd->lex->allow_sum_func.set_bit(select->nest_level); res= res || setup_windows(thd, ref_pointer_array, tables, fields, all_fields, win_specs, win_funcs); thd->lex->allow_sum_func= save_allow_sum_func; DBUG_RETURN(res); } bool vers_select_conds_t::init_from_sysvar(THD *thd) { vers_asof_timestamp_t &in= thd->variables.vers_asof_timestamp; type= (vers_system_time_t) in.type; delete_history= false; start.unit= VERS_TIMESTAMP; if (type != SYSTEM_TIME_UNSPECIFIED && type != SYSTEM_TIME_ALL) { DBUG_ASSERT(type == SYSTEM_TIME_AS_OF); Datetime dt(in.unix_time, in.second_part, thd->variables.time_zone); start.item= new (thd->mem_root) Item_datetime_literal(thd, &dt, TIME_SECOND_PART_DIGITS); if (!start.item) return true; } else start.item= NULL; end.empty(); return false; } void vers_select_conds_t::print(String *str, enum_query_type query_type) const { switch (orig_type) { case SYSTEM_TIME_UNSPECIFIED: break; case SYSTEM_TIME_AS_OF: start.print(str, query_type, STRING_WITH_LEN(" FOR SYSTEM_TIME AS OF ")); break; case SYSTEM_TIME_FROM_TO: start.print(str, query_type, STRING_WITH_LEN(" FOR SYSTEM_TIME FROM ")); end.print(str, query_type, STRING_WITH_LEN(" TO ")); break; case SYSTEM_TIME_BETWEEN: start.print(str, query_type, STRING_WITH_LEN(" FOR SYSTEM_TIME BETWEEN ")); end.print(str, query_type, STRING_WITH_LEN(" AND ")); break; case SYSTEM_TIME_BEFORE: start.print(str, query_type, STRING_WITH_LEN(" FOR SYSTEM_TIME BEFORE ")); break; case SYSTEM_TIME_HISTORY: // nothing to add break; case SYSTEM_TIME_ALL: str->append(STRING_WITH_LEN(" FOR SYSTEM_TIME ALL")); break; } } static Item* period_get_condition(THD *thd, TABLE_LIST *table, SELECT_LEX *select, vers_select_conds_t *conds, bool timestamp) { DBUG_ASSERT(table); DBUG_ASSERT(table->table); #define newx new (thd->mem_root) TABLE_SHARE *share= table->table->s; const TABLE_SHARE::period_info_t *period= conds->period; const LEX_CSTRING &fstart= period->start_field(share)->field_name; const LEX_CSTRING &fend= period->end_field(share)->field_name; conds->field_start= newx Item_field(thd, &select->context, table->db, table->alias, thd->strmake_lex_cstring(fstart)); conds->field_end= newx Item_field(thd, &select->context, table->db, table->alias, thd->strmake_lex_cstring(fend)); Item *cond1= NULL, *cond2= NULL, *cond3= NULL, *curr= NULL; if (timestamp) { MYSQL_TIME max_time; switch (conds->type) { case SYSTEM_TIME_UNSPECIFIED: case SYSTEM_TIME_HISTORY: { thd->variables.time_zone->gmt_sec_to_TIME(&max_time, TIMESTAMP_MAX_VALUE); max_time.second_part= TIME_MAX_SECOND_PART; Datetime dt(&max_time); curr= newx Item_datetime_literal(thd, &dt, TIME_SECOND_PART_DIGITS); if (conds->type == SYSTEM_TIME_UNSPECIFIED) cond1= newx Item_func_eq(thd, conds->field_end, curr); else cond1= newx Item_func_lt(thd, conds->field_end, curr); break; } case SYSTEM_TIME_AS_OF: cond1= newx Item_func_le(thd, conds->field_start, conds->start.item); cond2= newx Item_func_gt(thd, conds->field_end, conds->start.item); break; case SYSTEM_TIME_FROM_TO: cond1= newx Item_func_lt(thd, conds->field_start, conds->end.item); cond2= newx Item_func_gt(thd, conds->field_end, conds->start.item); cond3= newx Item_func_lt(thd, conds->start.item, conds->end.item); break; case SYSTEM_TIME_BETWEEN: cond1= newx Item_func_le(thd, conds->field_start, conds->end.item); cond2= newx Item_func_gt(thd, conds->field_end, conds->start.item); cond3= newx Item_func_le(thd, conds->start.item, conds->end.item); break; case SYSTEM_TIME_BEFORE: cond1= newx Item_func_history(thd, conds->field_end); cond2= newx Item_func_lt(thd, conds->field_end, conds->start.item); break; default: DBUG_ASSERT(0); } } else { DBUG_ASSERT(table->table->s && table->table->s->db_plugin); Item *trx_id0= conds->start.item; Item *trx_id1= conds->end.item; if (conds->start.item && conds->start.unit == VERS_TIMESTAMP) { bool backwards= conds->type != SYSTEM_TIME_AS_OF; trx_id0= newx Item_func_trt_id(thd, conds->start.item, TR_table::FLD_TRX_ID, backwards); } if (conds->end.item && conds->end.unit == VERS_TIMESTAMP) { trx_id1= newx Item_func_trt_id(thd, conds->end.item, TR_table::FLD_TRX_ID, false); } switch (conds->type) { case SYSTEM_TIME_UNSPECIFIED: case SYSTEM_TIME_HISTORY: curr= newx Item_int(thd, ULONGLONG_MAX); if (conds->type == SYSTEM_TIME_UNSPECIFIED) cond1= newx Item_func_eq(thd, conds->field_end, curr); else cond1= newx Item_func_lt(thd, conds->field_end, curr); break; DBUG_ASSERT(!conds->start.item); DBUG_ASSERT(!conds->end.item); break; case SYSTEM_TIME_AS_OF: cond1= newx Item_func_trt_trx_sees_eq(thd, trx_id0, conds->field_start); cond2= newx Item_func_trt_trx_sees(thd, conds->field_end, trx_id0); DBUG_ASSERT(!conds->end.item); break; case SYSTEM_TIME_FROM_TO: cond1= newx Item_func_trt_trx_sees(thd, trx_id1, conds->field_start); cond2= newx Item_func_trt_trx_sees_eq(thd, conds->field_end, trx_id0); cond3= newx Item_func_lt(thd, conds->start.item, conds->end.item); break; case SYSTEM_TIME_BETWEEN: cond1= newx Item_func_trt_trx_sees_eq(thd, trx_id1, conds->field_start); cond2= newx Item_func_trt_trx_sees_eq(thd, conds->field_end, trx_id0); cond3= newx Item_func_le(thd, conds->start.item, conds->end.item); break; case SYSTEM_TIME_BEFORE: cond1= newx Item_func_history(thd, conds->field_end); cond2= newx Item_func_trt_trx_sees(thd, trx_id0, conds->field_end); break; default: DBUG_ASSERT(0); } } if (cond1) { cond1= and_items(thd, cond2, cond1); cond1= and_items(thd, cond3, cond1); } return cond1; } static bool skip_setup_conds(THD *thd) { return (!thd->stmt_arena->is_conventional() && !thd->stmt_arena->is_stmt_prepare_or_first_sp_execute()) || thd->lex->is_view_context_analysis(); } int SELECT_LEX::period_setup_conds(THD *thd, TABLE_LIST *tables) { DBUG_ENTER("SELECT_LEX::period_setup_conds"); const bool update_conds= !skip_setup_conds(thd); Query_arena backup; Query_arena *arena= thd->activate_stmt_arena_if_needed(&backup); DBUG_ASSERT(!tables->next_local && tables->table); Item *result= NULL; for (TABLE_LIST *table= tables; table; table= table->next_local) { if (!table->table) continue; vers_select_conds_t &conds= table->period_conditions; if (!table->table->s->period.name.streq_safe(conds.name)) { my_error(ER_PERIOD_NOT_FOUND, MYF(0), conds.name.str); if (arena) thd->restore_active_arena(arena, &backup); DBUG_RETURN(-1); } if (update_conds) { conds.period= &table->table->s->period; result= and_items(thd, result, period_get_condition(thd, table, this, &conds, true)); } } if (update_conds) where= and_items(thd, where, result); if (arena) thd->restore_active_arena(arena, &backup); DBUG_RETURN(0); } int SELECT_LEX::vers_setup_conds(THD *thd, TABLE_LIST *tables) { DBUG_ENTER("SELECT_LEX::vers_setup_conds"); const bool update_conds= !skip_setup_conds(thd); if (!versioned_tables) { for (TABLE_LIST *table= tables; table; table= table->next_local) { if (table->table && table->table->versioned()) versioned_tables++; else if (table->vers_conditions.is_set() && (table->is_non_derived() || !table->vers_conditions.used)) { my_error(ER_VERS_NOT_VERSIONED, MYF(0), table->alias.str); DBUG_RETURN(-1); } } } if (versioned_tables == 0) DBUG_RETURN(0); /* For prepared statements we create items on statement arena, because they must outlive execution phase for multiple executions. */ Query_arena_stmt on_stmt_arena(thd); // find outer system_time SELECT_LEX *outer_slex= outer_select(); TABLE_LIST* outer_table= NULL; if (outer_slex) { TABLE_LIST* derived= master_unit()->derived; // inner SELECT may not be a derived table (derived == NULL) while (derived && outer_slex && !derived->vers_conditions.is_set()) { derived= outer_slex->master_unit()->derived; outer_slex= outer_slex->outer_select(); } if (derived && outer_slex) { DBUG_ASSERT(derived->vers_conditions.is_set()); outer_table= derived; } } bool is_select= false; bool use_sysvar= false; switch (thd->lex->sql_command) { case SQLCOM_SELECT: use_sysvar= true; /* fall through */ case SQLCOM_CREATE_TABLE: case SQLCOM_INSERT_SELECT: case SQLCOM_REPLACE_SELECT: case SQLCOM_DELETE_MULTI: case SQLCOM_UPDATE_MULTI: is_select= true; default: break; } for (TABLE_LIST *table= tables; table; table= table->next_local) { if (!table->table || table->is_view() || !table->table->versioned()) continue; vers_select_conds_t &vers_conditions= table->vers_conditions; #ifdef WITH_PARTITION_STORAGE_ENGINE /* if the history is stored in partitions, then partitions themselves are not versioned */ if (table->partition_names && table->table->part_info->vers_info) { /* If the history is stored in partitions, then partitions themselves are not versioned. */ if (vers_conditions.was_set()) { my_error(ER_VERS_QUERY_IN_PARTITION, MYF(0), table->alias.str); DBUG_RETURN(-1); } else if (!vers_conditions.is_set()) vers_conditions.set_all(); } #endif if (outer_table && !vers_conditions.is_set()) { // propagate system_time from nearest outer SELECT_LEX vers_conditions= outer_table->vers_conditions; outer_table->vers_conditions.used= true; } // propagate system_time from sysvar if (!vers_conditions.is_set() && use_sysvar) { if (vers_conditions.init_from_sysvar(thd)) DBUG_RETURN(-1); } if (vers_conditions.is_set()) { if (vers_conditions.was_set() && table->lock_type >= TL_FIRST_WRITE && !vers_conditions.delete_history) { my_error(ER_TABLE_NOT_LOCKED_FOR_WRITE, MYF(0), table->alias.str); DBUG_RETURN(-1); } if (vers_conditions.type == SYSTEM_TIME_ALL) continue; } bool timestamps_only= table->table->versioned(VERS_TIMESTAMP); bool update_this= update_conds; if (vers_conditions.is_set() && vers_conditions.type != SYSTEM_TIME_HISTORY) { thd->where= THD_WHERE::FOR_SYSTEM_TIME; /* TODO: do resolve fix_length_and_dec(), fix_fields(). This requires storing vers_conditions as Item and make some magic related to vers_system_time_t/VERS_TRX_ID at stage of fix_fields() (this is large refactoring). */ if (vers_conditions.check_units(thd)) DBUG_RETURN(-1); if (timestamps_only && (vers_conditions.start.unit == VERS_TRX_ID || vers_conditions.end.unit == VERS_TRX_ID)) { my_error(ER_VERS_ENGINE_UNSUPPORTED, MYF(0), table->table_name.str); DBUG_RETURN(-1); } if (vers_conditions.has_param) { /* PS parameter in history expression requires processing at execution stage when parameters has values substituted. So at prepare continue the loop, but at execution enter update_this. The second execution is skipped on vers_conditions.type == SYSTEM_TIME_ALL condition. */ if (thd->stmt_arena->is_stmt_prepare()) continue; DBUG_ASSERT(thd->stmt_arena->is_stmt_execute()); update_this= true; } } if (update_this) { vers_conditions.period = &table->table->s->vers; Item *cond= period_get_condition(thd, table, this, &vers_conditions, timestamps_only); if (is_select) table->on_expr= and_items(thd, table->on_expr, cond); else { if (join) { where= and_items(thd, join->conds, cond); join->conds= where; } else where= and_items(thd, where, cond); table->where= and_items(thd, table->where, cond); if (where && vers_conditions.has_param && vers_conditions.delete_history) prep_where= where->copy_andor_structure(thd); } table->vers_conditions.set_all(); } } // for (table= tables; ...) DBUG_RETURN(0); } /***************************************************************************** Check fields, find best join, do the select and output fields. mysql_select assumes that all tables are already opened *****************************************************************************/ /* Check if we have a field reference. If yes, we have to use mixed_implicit_grouping. */ static bool check_list_for_field(List<Item> *items) { List_iterator_fast <Item> select_it(*items); Item *select_el; while ((select_el= select_it++)) { if (select_el->with_field()) return true; } return false; } static bool check_list_for_field(ORDER *order) { for (; order; order= order->next) { if (order->item[0]->with_field()) return true; } return false; } /** Prepare of whole select (including sub queries in future). @todo Add check of calculation of GROUP functions and fields: SELECT COUNT(*)+table.col1 from table1; @retval -1 on error @retval 0 on success */ int JOIN::prepare(TABLE_LIST *tables_init, COND *conds_init, uint og_num, ORDER *order_init, bool skip_order_by, ORDER *group_init, Item *having_init, ORDER *proc_param_init, SELECT_LEX *select_lex_arg, SELECT_LEX_UNIT *unit_arg) { DBUG_ENTER("JOIN::prepare"); // to prevent double initialization on EXPLAIN if (optimization_state != JOIN::NOT_OPTIMIZED) DBUG_RETURN(0); conds= conds_init; order= order_init; group_list= group_init; having= having_init; proc_param= proc_param_init; tables_list= tables_init; select_lex= select_lex_arg; DBUG_PRINT("info", ("select %p (%u) = JOIN %p", select_lex, select_lex->select_number, this)); select_lex->join= this; join_list= &select_lex->top_join_list; union_part= unit_arg->is_unit_op(); Json_writer_object trace_wrapper(thd); Json_writer_object trace_prepare(thd, "join_preparation"); trace_prepare.add_select_number(select_lex->select_number); Json_writer_array trace_steps(thd, "steps"); // simple check that we got usable conds dbug_print_item(conds); /* Fix items that requires the join structure to exist */ fix_items_after_optimize(thd, select_lex); /* It is hack which force creating EXPLAIN object always on runt-time arena (because very top JOIN::prepare executes always with runtime arena, but constant subquery like (SELECT 'x') can be called with statement arena during prepare phase of top SELECT). */ if (!(thd->lex->context_analysis_only & CONTEXT_ANALYSIS_ONLY_PREPARE)) create_explain_query_if_not_exists(thd->lex, thd->mem_root); if (select_lex->handle_derived(thd->lex, DT_PREPARE)) DBUG_RETURN(-1); thd->lex->current_select->context_analysis_place= NO_MATTER; thd->lex->current_select->is_item_list_lookup= 1; /* If we have already executed SELECT, then it have not sense to prevent its table from update (see unique_table()) Affects only materialized derived tables. */ /* Check that all tables, fields, conds and order are ok */ if (!(select_options & OPTION_SETUP_TABLES_DONE) && setup_tables_and_check_access(thd, &select_lex->context, join_list, tables_list, select_lex->leaf_tables, false, SELECT_ACL, SELECT_ACL, false)) DBUG_RETURN(-1); if (thd->lex->opt_hints_global && select_lex->select_number == 1) { thd->lex->opt_hints_global->fix_hint(thd); /* There's no need to call opt_hints_global->check_unresolved(), this is done for each query block individually */ } if (select_lex->opt_hints_qb) select_lex->opt_hints_qb->check_unfixed(thd); /* System Versioning: handle FOR SYSTEM_TIME clause. */ if (select_lex->vers_setup_conds(thd, tables_list) < 0) DBUG_RETURN(-1); /* mixed_implicit_grouping will be set to TRUE if the SELECT list mixes elements with and without grouping, and there is no GROUP BY clause. Mixing non-aggregated fields with aggregate functions in the SELECT list or HAVING is a MySQL extension that is allowed only if the ONLY_FULL_GROUP_BY sql mode is not set. */ mixed_implicit_grouping= false; if ((~thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY) && select_lex->with_sum_func && !group_list) { if (check_list_for_field(&fields_list) || check_list_for_field(order)) { List_iterator_fast<TABLE_LIST> li(select_lex->leaf_tables); mixed_implicit_grouping= true; // mark for future while (TABLE_LIST *tbl= li++) { /* If the query uses implicit grouping where the select list contains both aggregate functions and non-aggregate fields, any non-aggregated field may produce a NULL value. Set all fields of each table as nullable before semantic analysis to take into account this change of nullability. Note: this loop doesn't touch tables inside merged semi-joins, because subquery-to-semijoin conversion has not been done yet. This is intended. */ if (tbl->table) tbl->table->maybe_null= 1; } } } table_count= select_lex->leaf_tables.elements; uint real_og_num= og_num; if (skip_order_by && select_lex != select_lex->master_unit()->global_parameters()) real_og_num+= select_lex->order_list.elements; DBUG_ASSERT(select_lex->hidden_bit_fields == 0); if (setup_wild(thd, tables_list, fields_list, &all_fields, select_lex, false)) DBUG_RETURN(-1); /* If the select_lex is immediately contained within a derived table AND this derived table is a CTE WITH supplied column names AND we have the correct number of elements in both lists (mismatches found in mysql_derived_prepare/rename_columns_of_derived_unit) THEN NOW is the time to take a copy of these item_names for later restoration if required. */ TABLE_LIST *derived= select_lex->master_unit()->derived; if (derived && derived->with && derived->with->column_list.elements && (derived->with->column_list.elements == select_lex->item_list.elements)) { if (select_lex->save_item_list_names(thd)) DBUG_RETURN(-1); } if (thd->lex->current_select->first_cond_optimization) { if ( conds && ! thd->lex->current_select->merged_into) select_lex->select_n_reserved= conds->exists2in_reserved_items(); else select_lex->select_n_reserved= 0; } if (select_lex->setup_ref_array(thd, real_og_num)) DBUG_RETURN(-1); ref_ptrs= ref_ptr_array_slice(0); enum_parsing_place save_place= thd->lex->current_select->context_analysis_place; thd->lex->current_select->context_analysis_place= SELECT_LIST; { List_iterator_fast<TABLE_LIST> it(select_lex->leaf_tables); while (TABLE_LIST *tbl= it++) { if (tbl->table_function && tbl->table_function->setup(thd, tbl, select_lex_arg)) DBUG_RETURN(-1); } } if (setup_fields(thd, ref_ptrs, fields_list, select_lex->item_list_usage, &all_fields, &select_lex->pre_fix, 1)) DBUG_RETURN(-1); thd->lex->current_select->context_analysis_place= save_place; if (setup_without_group(thd, ref_ptrs, tables_list, select_lex->leaf_tables, fields_list, all_fields, &conds, order, group_list, select_lex->window_specs, select_lex->window_funcs, &hidden_group_fields)) DBUG_RETURN(-1); /* Permanently remove redundant parts from the query if 1) This is a subquery 2) This is the first time this query is optimized (since the transformation is permanent 3) Not normalizing a view. Removal should take place when a query involving a view is optimized, not when the view is created */ if (select_lex->master_unit()->item && // 1) select_lex->first_cond_optimization && // 2) !thd->lex->is_view_context_analysis()) // 3) { remove_redundant_subquery_clauses(select_lex); } /* Resolve the ORDER BY that was skipped, then remove it. */ if (skip_order_by && select_lex != select_lex->master_unit()->global_parameters()) { nesting_map save_allow_sum_func= thd->lex->allow_sum_func; thd->lex->allow_sum_func.set_bit(select_lex->nest_level); thd->where= THD_WHERE::ORDER_CLAUSE; for (ORDER *order= select_lex->order_list.first; order; order= order->next) { /* Don't add the order items to all fields. Just resolve them to ensure the query is valid, we'll drop them immediately after. */ if (find_order_in_list(thd, ref_ptrs, tables_list, order, fields_list, all_fields, false, false, false)) DBUG_RETURN(-1); } thd->lex->allow_sum_func= save_allow_sum_func; select_lex->order_list.empty(); } if (having) { nesting_map save_allow_sum_func= thd->lex->allow_sum_func; thd->where= THD_WHERE::HAVING_CLAUSE; thd->lex->allow_sum_func.set_bit(select_lex_arg->nest_level); select_lex->having_fix_field= 1; /* Wrap alone field in HAVING clause in case it will be outer field of subquery which need persistent pointer on it, but having could be changed by optimizer */ if (having->type() == Item::REF_ITEM && ((Item_ref *)having)->ref_type() == Item_ref::REF) wrap_ident(thd, &having); bool having_fix_rc= having->fix_fields_if_needed_for_bool(thd, &having); select_lex->having_fix_field= 0; if (unlikely(having_fix_rc || thd->is_error())) DBUG_RETURN(-1); /* purecov: inspected */ thd->lex->allow_sum_func= save_allow_sum_func; if (having->with_window_func()) { my_error(ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION, MYF(0)); DBUG_RETURN(-1); } } /* After setting up window functions, we may have discovered additional used tables from the PARTITION BY and ORDER BY list. Update all items that contain window functions. */ if (select_lex->have_window_funcs()) { List_iterator_fast<Item> it(select_lex->item_list); Item *item; while ((item= it++)) { if (item->with_window_func()) item->update_used_tables(); } } With_element *with_elem= select_lex->get_with_element(); if (with_elem && select_lex->check_unrestricted_recursive( thd->variables.only_standard_compliant_cte)) DBUG_RETURN(-1); if (!(select_lex->changed_elements & TOUCHED_SEL_COND)) select_lex->check_subqueries_with_recursive_references(); int res= check_and_do_in_subquery_rewrites(this); select_lex->fix_prepare_information(thd, &conds, &having); if (res) DBUG_RETURN(res); if (order) { bool requires_sorting= FALSE; /* WITH TIES forces the results to be sorted, even if it's not sanely sortable. */ if (select_lex->limit_params.with_ties) requires_sorting= true; /* Go through each ORDER BY item and perform the following: 1. Detect if none of the items contain meaningful data, which means we can drop the sorting altogether. 2. Split any columns with aggregation functions or window functions into their base components and store them as separate fields. (see split_sum_func) for more details. */ for (ORDER *ord= order; ord; ord= ord->next) { Item *item= *ord->item; /* Disregard sort order if there's only zero length NOT NULL fields (e.g. {VAR}CHAR(0) NOT NULL") or zero length NOT NULL string functions there. Such tuples don't contain any data to sort. */ if (!requires_sorting && /* Not a zero length NOT NULL field */ ((item->type() != Item::FIELD_ITEM || ((Item_field *) item)->field->maybe_null() || ((Item_field *) item)->field->sort_length()) && /* AND not a zero length NOT NULL string function. */ (item->type() != Item::FUNC_ITEM || item->maybe_null() || item->result_type() != STRING_RESULT || item->max_length))) requires_sorting= TRUE; if ((item->with_sum_func() && item->type() != Item::SUM_FUNC_ITEM) || item->with_window_func()) item->split_sum_func(thd, ref_ptrs, all_fields, SPLIT_SUM_SELECT); } /* Drop the ORDER BY clause if none of the columns contain any data that can produce a meaningful sorted set. */ if (!requires_sorting) order= NULL; } else { /* The current select does not have an ORDER BY */ if (select_lex->limit_params.with_ties) { my_error(ER_WITH_TIES_NEEDS_ORDER, MYF(0)); DBUG_RETURN(-1); } } if (having && (having->with_sum_func() || having->with_rownum_func())) having->split_sum_func2(thd, ref_ptrs, all_fields, &having, SPLIT_SUM_SKIP_REGISTERED); if (select_lex->inner_sum_func_list) { Item_sum *end=select_lex->inner_sum_func_list; Item_sum *item_sum= end; do { item_sum= item_sum->next; item_sum->split_sum_func2(thd, ref_ptrs, all_fields, item_sum->ref_by, 0); } while (item_sum != end); } if (select_lex->inner_refs_list.elements && fix_inner_refs(thd, all_fields, select_lex, ref_ptrs)) DBUG_RETURN(-1); if (group_list) { /* Because HEAP tables can't index BIT fields we need to use an additional hidden field for grouping because later it will be converted to a LONG field. Original field will remain of the BIT type and will be returned to a client. */ for (ORDER *ord= group_list; ord; ord= ord->next) { if ((*ord->item)->type() == Item::FIELD_ITEM && (*ord->item)->field_type() == MYSQL_TYPE_BIT) { Item_field *field= new (thd->mem_root) Item_field(thd, *(Item_field**)ord->item); if (!field) DBUG_RETURN(-1); int el= all_fields.elements; ref_ptrs[el]= field; all_fields.push_front(field, thd->mem_root); ord->item= &ref_ptrs[el]; } } } /* Check if there are references to un-aggregated columns when computing aggregate functions with implicit grouping (there is no GROUP BY). */ if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY && !group_list && !(select_lex->master_unit()->item && select_lex->master_unit()->item->is_in_predicate() && select_lex->master_unit()->item->get_IN_subquery()-> test_set_strategy(SUBS_MAXMIN_INJECTED)) && select_lex->non_agg_field_used() && select_lex->agg_func_used()) { my_message(ER_MIX_OF_GROUP_FUNC_AND_FIELDS, ER_THD(thd, ER_MIX_OF_GROUP_FUNC_AND_FIELDS), MYF(0)); DBUG_RETURN(-1); } { /* Caclulate the number of groups */ send_group_parts= 0; for (ORDER *group_tmp= group_list ; group_tmp ; group_tmp= group_tmp->next) send_group_parts++; } procedure= setup_procedure(thd, proc_param, result, fields_list, &error); if (unlikely(error)) goto err; /* purecov: inspected */ if (procedure) { if (setup_new_fields(thd, fields_list, all_fields, procedure->param_fields)) goto err; /* purecov: inspected */ if (procedure->group) { if (!test_if_subpart(procedure->group,group_list)) { /* purecov: inspected */ my_message(ER_DIFF_GROUPS_PROC, ER_THD(thd, ER_DIFF_GROUPS_PROC), MYF(0)); /* purecov: inspected */ goto err; /* purecov: inspected */ } } if (order && (procedure->flags & PROC_NO_SORT)) { /* purecov: inspected */ my_message(ER_ORDER_WITH_PROC, ER_THD(thd, ER_ORDER_WITH_PROC), MYF(0)); /* purecov: inspected */ goto err; /* purecov: inspected */ } if (thd->lex->derived_tables) { /* Queries with derived tables and PROCEDURE are not allowed. Many of such queries are disallowed grammatically, but there are still some complex cases: SELECT 1 FROM (SELECT 1) a PROCEDURE ANALYSE() */ my_error(ER_WRONG_USAGE, MYF(0), "PROCEDURE", thd->lex->derived_tables & DERIVED_VIEW ? "view" : "subquery"); goto err; } if (thd->lex->sql_command != SQLCOM_SELECT) { // EXPLAIN SELECT * FROM t1 PROCEDURE ANALYSE() my_error(ER_WRONG_USAGE, MYF(0), "PROCEDURE", "non-SELECT"); goto err; } } if (unlikely(thd->trace_started())) { Json_writer_object trace_wrapper(thd); opt_trace_print_expanded_query(thd, select_lex, &trace_wrapper); } if (!procedure && result && result->prepare(fields_list, unit_arg)) goto err; /* purecov: inspected */ select_lex->where_cond_after_prepare= conds; unit= unit_arg; if (prepare_stage2()) goto err; prepared= true; DBUG_RETURN(0); // All OK err: delete procedure; /* purecov: inspected */ procedure= 0; DBUG_RETURN(-1); /* purecov: inspected */ } /** Second phase of prepare where we collect some statistic. @details We made this part separate to be able recalculate some statistic after transforming subquery on optimization phase. */ bool JOIN::prepare_stage2() { bool res= TRUE; DBUG_ENTER("JOIN::prepare_stage2"); /* Init join struct */ count_field_types(select_lex, &tmp_table_param, all_fields, 0); this->group= group_list != 0; if (tmp_table_param.sum_func_count && !group_list) { implicit_grouping= TRUE; // Result will contain zero or one row - ordering is meaningless order= NULL; } #ifdef RESTRICTED_GROUP if (implicit_grouping) { my_message(ER_WRONG_SUM_SELECT,ER_THD(thd, ER_WRONG_SUM_SELECT),MYF(0)); goto err; } #endif if (select_lex->olap == ROLLUP_TYPE && rollup_init()) goto err; if (alloc_func_list() || make_sum_func_list(all_fields, fields_list, false)) goto err; res= FALSE; err: DBUG_RETURN(res); /* purecov: inspected */ } bool JOIN::build_explain() { DBUG_ENTER("JOIN::build_explain"); have_query_plan= QEP_AVAILABLE; /* explain data must be created on the Explain_query::mem_root. Because it's just a memroot, not an arena, explain data must not contain any Items */ MEM_ROOT *old_mem_root= thd->mem_root; Item *old_free_list __attribute__((unused))= thd->free_list; thd->mem_root= thd->lex->explain->mem_root; bool res= save_explain_data(thd->lex->explain, false /* can overwrite */, need_tmp, !skip_sort_order && !no_order && (order || group_list), select_distinct); thd->mem_root= old_mem_root; DBUG_ASSERT(thd->free_list == old_free_list); // no Items were created if (res) DBUG_RETURN(1); uint select_nr= select_lex->select_number; JOIN_TAB *curr_tab= join_tab + exec_join_tab_cnt(); for (uint i= 0; i < aggr_tables; i++, curr_tab++) { if (select_nr == FAKE_SELECT_LEX_ID) { /* this is a fake_select_lex of a union */ select_nr= select_lex->master_unit()->first_select()->select_number; curr_tab->tracker= thd->lex->explain->get_union(select_nr)-> get_tmptable_read_tracker(); } else if (select_nr < INT_MAX) { Explain_select *tmp= thd->lex->explain->get_select(select_nr); if (tmp) curr_tab->tracker= tmp->get_using_temporary_read_tracker(); } } if (is_in_subquery()) { Item_in_subselect *subq= unit->item->get_IN_subquery(); subq->init_subq_materialization_tracker(thd); } DBUG_RETURN(0); } int JOIN::optimize() { int res= 0; if (select_lex->pushdown_select) { if (optimization_state == JOIN::OPTIMIZATION_DONE) return 0; DBUG_ASSERT(optimization_state == JOIN::NOT_OPTIMIZED); // Do same as JOIN::optimize_inner does: fields= &select_lex->item_list; if (!(select_options & SELECT_DESCRIBE)) { /* Prepare to execute the query pushed into a foreign engine */ res= select_lex->pushdown_select->prepare(); } with_two_phase_optimization= false; } else { /* This function may be invoked multiple times. Do nothing if the optimization (either full or stage1) are already done. */ if (optimization_state != JOIN::NOT_OPTIMIZED) return FALSE; optimization_state= JOIN::OPTIMIZATION_IN_PROGRESS; res= optimize_inner(); } if (!with_two_phase_optimization) { if (!res && have_query_plan != QEP_DELETED) res= build_explain(); optimization_state= JOIN::OPTIMIZATION_DONE; } /* Store the cost of this query into a user variable TODO: calculate a correct cost for a query with subqueries and UNIONs. */ if (select_lex->select_number == 1) thd->status_var.last_query_cost= best_read; return res; } /* @brief Call optimize_stage2() and save the query plan. */ int JOIN::optimize_stage2_and_finish() { int res= 0; DBUG_ASSERT(with_two_phase_optimization); DBUG_ASSERT(optimization_state == OPTIMIZATION_PHASE_1_DONE); if (optimize_stage2()) res= 1; else { if (have_query_plan != JOIN::QEP_DELETED) res= build_explain(); optimization_state= JOIN::OPTIMIZATION_DONE; } return res; } /** @brief Create range filters objects needed in execution for all join tables @details For each join table from the chosen execution plan such that a range filter is used when joining this table the function creates a Rowid_filter object for this range filter. In order to do this the function first constructs a quick select to scan the range for this range filter. Then it creates a container for the range filter and finally constructs a Range_rowid_filter object a pointer to which is set in the field JOIN_TAB::rowid_filter of the joined table. @retval false Ok @retval true Error, query should abort */ bool JOIN::make_range_rowid_filters() { DBUG_ENTER("make_range_rowid_filters"); /* Do not build range filters with detected impossible WHERE. Anyway conditions cannot be used anymore to extract ranges for filters. */ if (const_table_map != found_const_table_map) DBUG_RETURN(0); JOIN_TAB *tab; for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { if (!tab->range_rowid_filter_info) continue; DBUG_ASSERT(!(tab->ref.key >= 0 && tab->ref.key == (int) tab->range_rowid_filter_info->get_key_no())); DBUG_ASSERT(!(tab->ref.key == -1 && tab->quick && tab->quick->index == tab->range_rowid_filter_info->get_key_no())); int err; SQL_SELECT *sel= NULL; Rowid_filter_container *filter_container= NULL; Item **sargable_cond= get_sargable_cond(this, tab->table); sel= make_select(tab->table, const_table_map, const_table_map, *sargable_cond, (SORT_INFO*) 0, 1, &err); if (!sel) continue; key_map filter_map; filter_map.clear_all(); filter_map.set_bit(tab->range_rowid_filter_info->get_key_no()); filter_map.merge(tab->table->with_impossible_ranges); quick_select_return rc; /* EQ_FUNC and EQUAL_FUNC already sent unusable key notes (if any) during update_ref_and_keys(). Have only other functions raise notes from can_optimize_scalar_range(). */ rc= sel->test_quick_select(thd, filter_map, (table_map) 0, (ha_rows) HA_POS_ERROR, true, false, true, true, Item_func::BITMAP_EXCEPT_ANY_EQUALITY); if (rc == SQL_SELECT::ERROR || thd->is_error() || thd->check_killed()) { delete sel; DBUG_RETURN(true); /* Fatal error */ } /* If SUBS_IN_TO_EXISTS strategy is chosen for the subquery then additional conditions are injected into WHERE/ON/HAVING and it may happen that the call of test_quick_select() discovers impossible range. */ if (rc == SQL_SELECT::IMPOSSIBLE_RANGE) { const_table_map|= tab->table->map; goto no_filter; } // Hints may cause test_quick_select not to find the best table read plan. if (!sel->quick) goto no_filter; filter_container= tab->range_rowid_filter_info->create_container(); if (filter_container) { tab->rowid_filter= new (thd->mem_root) Range_rowid_filter(tab->table, tab->range_rowid_filter_info, filter_container, sel); if (tab->rowid_filter) { tab->need_to_build_rowid_filter= true; continue; } } no_filter: delete sel; } DBUG_RETURN(0); } /** @brief Allocate memory the rowid containers of the used the range filters @details For each join table from the chosen execution plan such that a range filter is used when joining this table the function allocate memory for the rowid container employed by the filter. On success it lets the table engine know that what rowid filter will be used when accessing the table rows. @retval false OK true Error, query should abort */ bool JOIN::init_range_rowid_filters() { JOIN_TAB *tab; DBUG_ENTER("init_range_rowid_filters"); for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { tab->need_to_build_rowid_filter= false; // Safety if (!tab->rowid_filter) continue; if (tab->rowid_filter->get_container()->alloc()) { tab->clear_range_rowid_filter(); continue; } tab->table->file->rowid_filter_push(tab->rowid_filter); tab->need_to_build_rowid_filter= true; } DBUG_RETURN(0); } /** global select optimisation. @note error code saved in field 'error' @retval 0 success @retval 1 error */ int JOIN::optimize_inner() { DBUG_ENTER("JOIN::optimize_inner"); subq_exit_fl= false; best_read= 0.0; DEBUG_SYNC(thd, "before_join_optimize"); THD_STAGE_INFO(thd, stage_optimizing); #ifndef DBUG_OFF dbug_join_tab_array_size= 0; #endif // rownum used somewhere in query, no limits and it is derived if (unlikely(thd->lex->with_rownum && select_lex->first_cond_optimization && select_lex->master_unit()->derived)) optimize_upper_rownum_func(); do_send_rows = (unit->lim.get_select_limit()) ? 1 : 0; set_allowed_join_cache_types(); need_distinct= TRUE; Json_writer_object trace_wrapper(thd); Json_writer_object trace_prepare(thd, "join_optimization"); trace_prepare.add_select_number(select_lex->select_number); Json_writer_array trace_steps(thd, "steps"); if (select_lex->opt_hints_qb) select_lex->opt_hints_qb->trace_hints(thd); /* Needed in case optimizer short-cuts, set properly in make_aggr_tables_info() */ fields= &select_lex->item_list; if (select_lex->first_cond_optimization) { //Do it only for the first execution /* Merge all mergeable derived tables/views in this SELECT. */ if (select_lex->handle_derived(thd->lex, DT_MERGE)) DBUG_RETURN(TRUE); table_count= select_lex->leaf_tables.elements; } if (select_lex->first_cond_optimization && transform_in_predicates_into_in_subq(thd)) DBUG_RETURN(1); /* Update used tables after all handling derived table procedures After this call, select_lex->select_list_tables contains the table bits of all items in the select list (but not bits from WHERE clause or other items). */ select_lex->update_used_tables(); /* In fact we transform underlying subqueries after their 'prepare' phase and before 'optimize' from upper query 'optimize' to allow semijoin conversion happened (which done in the same way. */ if (select_lex->first_cond_optimization && conds && conds->walk(&Item::exists2in_processor, thd, 0)) DBUG_RETURN(1); /* TODO make view to decide if it is possible to write to WHERE directly or make Semi-Joins able to proces for (TABLE_LIST *tbl= tables_list; tbl; tbl= tbl->next_local) { if (tbl->on_expr && tbl->on_expr->walk(&Item::exists2in_processor, 0, thd)) DBUG_RETURN(1); } */ if (transform_max_min_subquery()) DBUG_RETURN(1); /* purecov: inspected */ if (select_lex->first_cond_optimization) { /* dump_TABLE_LIST_graph(select_lex, select_lex->leaf_tables); */ if (convert_join_subqueries_to_semijoins(this)) DBUG_RETURN(1); /* purecov: inspected */ /* dump_TABLE_LIST_graph(select_lex, select_lex->leaf_tables); */ select_lex->update_used_tables(); } eval_select_list_used_tables(); table_count= select_lex->leaf_tables.elements; if (select_lex->options & OPTION_SCHEMA_TABLE && optimize_schema_tables_memory_usage(select_lex->leaf_tables)) DBUG_RETURN(1); if (setup_ftfuncs(select_lex)) /* should be after having->fix_fields */ DBUG_RETURN(-1); row_limit= ((select_distinct || order || group_list) ? HA_POS_ERROR : unit->lim.get_select_limit()); /* select_limit is used to decide if we are likely to scan the whole table */ select_limit= unit->lim.get_select_limit(); if (having || (select_options & OPTION_FOUND_ROWS)) select_limit= HA_POS_ERROR; #ifdef HAVE_REF_TO_FIELDS // Not done yet /* Add HAVING to WHERE if possible */ if (having && !group_list && !sum_func_count) { if (!conds) { conds= having; having= 0; } else if ((conds=new (thd->mem_root) Item_cond_and(conds,having))) { /* Item_cond_and can't be fixed after creation, so we do not check conds->fixed() */ conds->fix_fields(thd, &conds); conds->change_ref_to_fields(thd, tables_list); conds->top_level_item(); having= 0; } } #endif SELECT_LEX *sel= select_lex; if (sel->first_cond_optimization) { bool error= false; /* The following code will allocate the new items in a permanent MEMROOT for prepared statements and stored procedures. But first we need to ensure that thd->lex->explain is allocated in the execution arena */ create_explain_query_if_not_exists(thd->lex, thd->mem_root); Query_arena *arena, backup; arena= thd->activate_stmt_arena_if_needed(&backup); sel->first_cond_optimization= 0; /* Convert all outer joins to inner joins if possible */ conds= simplify_joins(this, join_list, conds, TRUE, FALSE); add_table_function_dependencies(join_list, table_map(-1), &error); if (thd->is_error() || (!select_lex->leaf_tables_saved && select_lex->save_leaf_tables(thd))) { /* If there was an error above, the data structures may have been left in some undefined state. If this is a PS/SP statement, it might not be safe to run it again. Note that it needs to be re-prepared. */ thd->lex->needs_reprepare= true; if (arena) thd->restore_active_arena(arena, &backup); DBUG_RETURN(1); } select_lex->leaf_tables_saved= true; build_bitmap_for_nested_joins(join_list, 0); sel->prep_where= conds ? conds->copy_andor_structure(thd) : 0; sel->where= conds; select_lex->update_used_tables(); if (arena) thd->restore_active_arena(arena, &backup); } if (!allowed_top_level_tables) calc_allowed_top_level_tables(select_lex); if (select_lex->optimize_constant_subqueries()) DBUG_RETURN(1); if (conds && conds->with_subquery()) (void) conds->walk(&Item::cleanup_is_expensive_cache_processor, 0, 0); if (having && having->with_subquery()) (void) having->walk(&Item::cleanup_is_expensive_cache_processor, 0, 0); List<Item> eq_list; if (setup_degenerate_jtbm_semi_joins(this, join_list, eq_list)) DBUG_RETURN(1); if (eq_list.elements != 0) { Item *new_cond; if (eq_list.elements == 1) new_cond= eq_list.pop(); else new_cond= new (thd->mem_root) Item_cond_and(thd, eq_list); if (new_cond && ((new_cond->fix_fields(thd, &new_cond) || !(conds= and_items(thd, conds, new_cond)) || conds->fix_fields(thd, &conds)))) DBUG_RETURN(TRUE); } eq_list.empty(); if (select_lex->cond_pushed_into_where) { conds= and_conds(thd, conds, select_lex->cond_pushed_into_where); if (conds && conds->fix_fields(thd, &conds)) DBUG_RETURN(1); } if (select_lex->cond_pushed_into_having) { having= and_conds(thd, having, select_lex->cond_pushed_into_having); if (having) { select_lex->having_fix_field= 1; select_lex->having_fix_field_for_pushed_cond= 1; if (having->fix_fields(thd, &having)) DBUG_RETURN(1); select_lex->having_fix_field= 0; select_lex->having_fix_field_for_pushed_cond= 0; } } bool ignore_on_expr= false; /* PS/SP note: on_expr of versioned table can not be reallocated (see build_equal_items() below) because it can be not rebuilt at second invocation. */ if (!thd->stmt_arena->is_conventional() && thd->mem_root != thd->stmt_arena->mem_root) for (TABLE_LIST *tbl= tables_list; tbl; tbl= tbl->next_local) if (tbl->table && tbl->on_expr && tbl->table->versioned()) { ignore_on_expr= true; break; } transform_in_predicates_into_equalities(thd); if (thd->lex->are_date_funcs_used()) transform_date_conds_into_sargable(); if (optimizer_flag(thd, OPTIMIZER_SWITCH_SARGABLE_CASEFOLD)) { transform_all_conds_and_on_exprs( thd, &Item::varchar_upper_cmp_transformer); } if (substitute_indexed_vcols_for_join(this)) { error= 1; DBUG_RETURN(1); } conds= optimize_cond(this, conds, join_list, ignore_on_expr, &cond_value, &cond_equal, OPT_LINK_EQUAL_FIELDS); if (thd->is_error()) { error= 1; DBUG_PRINT("error",("Error from optimize_cond")); DBUG_RETURN(1); } if (select_lex->with_rownum && ! order && ! group_list && !select_distinct && conds && select_lex == unit->global_parameters() && select_lex->first_rownum_optimization) { optimize_rownum(thd, unit, conds); select_lex->first_rownum_optimization= false; } having= optimize_cond(this, having, join_list, TRUE, &having_value, &having_equal); if (thd->is_error()) { error= 1; DBUG_PRINT("error",("Error from optimize_cond")); DBUG_RETURN(1); } /* Do not push into WHERE from HAVING if cond_value == Item::COND_FALSE */ if (thd->lex->sql_command == SQLCOM_SELECT && optimizer_flag(thd, OPTIMIZER_SWITCH_COND_PUSHDOWN_FROM_HAVING) && cond_value != Item::COND_FALSE) { having= select_lex->pushdown_from_having_into_where(thd, having); if (select_lex->attach_to_conds.elements != 0) { conds= and_new_conditions_to_optimized_cond(thd, conds, &cond_equal, select_lex->attach_to_conds, &cond_value); sel->attach_to_conds.empty(); Json_writer_object wrapper(thd); Json_writer_object pushd(thd, "condition_pushdown_from_having"); pushd.add("conds", conds); pushd.add("having", having); } } if (optimizer_flag(thd, OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_SUBQUERY)) { TABLE_LIST *tbl; List_iterator_fast<TABLE_LIST> li(select_lex->leaf_tables); while ((tbl= li++)) if (tbl->jtbm_subselect) { if (tbl->jtbm_subselect->pushdown_cond_for_in_subquery(thd, conds)) DBUG_RETURN(1); } } if (setup_jtbm_semi_joins(this, join_list, eq_list)) DBUG_RETURN(1); if (eq_list.elements != 0) { conds= and_new_conditions_to_optimized_cond(thd, conds, &cond_equal, eq_list, &cond_value); if (!conds && cond_value != Item::COND_FALSE && cond_value != Item::COND_TRUE) DBUG_RETURN(TRUE); } TABLE_LIST *tbl; List_iterator_fast<TABLE_LIST> li(select_lex->leaf_tables); while ((tbl= li++)) { const bool is_derived_pushdown_allowed= hint_table_state( thd, tbl->table, DERIVED_CONDITION_PUSHDOWN_HINT_ENUM, optimizer_flag(thd, OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_DERIVED)); if (!is_derived_pushdown_allowed) { /* Run optimize phase on this derived table/view. */ if (tbl->is_view_or_derived() && tbl->handle_derived(thd->lex, DT_OPTIMIZE)) DBUG_RETURN(1); continue; } if (tbl->is_materialized_derived()) { JOIN *join= tbl->get_unit()->first_select()->join; if (join && join->optimization_state == JOIN::OPTIMIZATION_PHASE_1_DONE && join->with_two_phase_optimization) continue; /* Do not push conditions from where into materialized inner tables of outer joins: this is not valid. */ if (!tbl->is_inner_table_of_outer_join()) { if (pushdown_cond_for_derived(thd, conds, tbl)) DBUG_RETURN(1); } if (mysql_handle_single_derived(thd->lex, tbl, DT_OPTIMIZE)) { error= 1; DBUG_RETURN(1); } } } { if (select_lex->where) { select_lex->cond_value= cond_value; if (sel->where != conds && cond_value == Item::COND_OK) thd->change_item_tree(&sel->where, conds); } if (select_lex->having) { select_lex->having_value= having_value; if (sel->having != having && having_value == Item::COND_OK) thd->change_item_tree(&sel->having, having); } if (cond_value == Item::COND_FALSE || having_value == Item::COND_FALSE || (!unit->lim.get_select_limit() && !(select_options & OPTION_FOUND_ROWS))) { /* Impossible cond */ if (unit->lim.get_select_limit()) { DBUG_PRINT("info", (having_value == Item::COND_FALSE ? "Impossible HAVING" : "Impossible WHERE")); zero_result_cause= having_value == Item::COND_FALSE ? "Impossible HAVING" : "Impossible WHERE"; } else { DBUG_PRINT("info", ("Zero limit")); zero_result_cause= "Zero limit"; } table_count= top_join_tab_count= 0; handle_implicit_grouping_with_window_funcs(); error= 0; subq_exit_fl= true; goto setup_subq_exit; } } #ifdef WITH_PARTITION_STORAGE_ENGINE { TABLE_LIST *tbl; List_iterator_fast<TABLE_LIST> li(select_lex->leaf_tables); while ((tbl= li++)) { Item **prune_cond= get_sargable_cond(this, tbl->table); tbl->table->all_partitions_pruned_away= prune_partitions(thd, tbl->table, *prune_cond); } } #endif /* Try to optimize count(*), MY_MIN() and MY_MAX() to const fields if there is implicit grouping (aggregate functions but no group_list). In this case, the result set shall only contain one row. */ if (tables_list && implicit_grouping) { int res; /* opt_sum_query() returns HA_ERR_KEY_NOT_FOUND if no rows match to the WHERE conditions, or 1 if all items were resolved (optimized away), or 0, or an error number HA_ERR_... If all items were resolved by opt_sum_query, there is no need to open any tables. */ /* The following resetting and restoring of sum_funcs is needed to go around a bug in spider where it assumes that make_sum_func_list() has not been called yet and do logical choices based on this if special handling of min/max functions should be done. We disable this special handling while we are trying to find out if we can replace MIN/MAX values with constants. */ Item_sum **save_func_sums= sum_funcs, *tmp_sum_funcs= 0; sum_funcs= &tmp_sum_funcs; res= opt_sum_query(thd, select_lex->leaf_tables, all_fields, conds); sum_funcs= save_func_sums; if (res) { DBUG_ASSERT(res >= 0); if (res == HA_ERR_KEY_NOT_FOUND) { DBUG_PRINT("info",("No matching min/max row")); zero_result_cause= "No matching min/max row"; table_count= top_join_tab_count= 0; error=0; subq_exit_fl= true; handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } if (res > 1) { error= res; DBUG_PRINT("error",("Error from opt_sum_query")); DBUG_RETURN(1); } DBUG_PRINT("info",("Select tables optimized away")); if (!select_lex->have_window_funcs()) zero_result_cause= "Select tables optimized away"; tables_list= 0; // All tables resolved select_lex->min_max_opt_list.empty(); const_tables= top_join_tab_count= table_count; handle_implicit_grouping_with_window_funcs(); /* Extract all table-independent conditions and replace the WHERE clause with them. All other conditions were computed by opt_sum_query and the MIN/MAX/COUNT function(s) have been replaced by constants, so there is no need to compute the whole WHERE clause again. Notice that make_cond_for_table() will always succeed to remove all computed conditions, because opt_sum_query() is applicable only to conjunctions. Preserve conditions for EXPLAIN. */ if (conds && !(thd->lex->describe & DESCRIBE_EXTENDED)) { COND *table_independent_conds= make_cond_for_table(thd, conds, PSEUDO_TABLE_BITS, 0, -1, FALSE, FALSE); if (!table_independent_conds && thd->is_error()) DBUG_RETURN(1); DBUG_EXECUTE("where", print_where(table_independent_conds, "where after opt_sum_query()", QT_ORDINARY);); conds= table_independent_conds; } } } if (!tables_list) { DBUG_PRINT("info",("No tables")); error= 0; subq_exit_fl= true; goto setup_subq_exit; } error= -1; // Error is sent to client /* get_sort_by_table() call used to be here: */ MEM_UNDEFINED(&sort_by_table, sizeof(sort_by_table)); /* We have to remove constants and duplicates from group_list before calling make_join_statistics() as this may call get_best_group_min_max() which needs a simplified group_list. */ if (group_list && table_count == 1) { group_list= remove_const(this, group_list, conds, rollup.state == ROLLUP::STATE_NONE, &simple_group); if (unlikely(thd->is_error())) { error= 1; DBUG_RETURN(1); } if (!group_list) { /* The output has only one row */ order=0; simple_order=1; group_optimized_away= 1; select_distinct=0; } } /* Calculate how to do the join */ THD_STAGE_INFO(thd, stage_statistics); result->prepare_to_read_rows(); if (unlikely(make_join_statistics(this, select_lex->leaf_tables, &keyuse)) || unlikely(thd->is_error())) { DBUG_PRINT("error",("Error: make_join_statistics() failed")); DBUG_RETURN(1); } /* If a splittable materialized derived/view dt_i is embedded into into another splittable materialized derived/view dt_o then splitting plans for dt_i and dt_o are evaluated independently. First the optimizer looks for the best splitting plan sp_i for dt_i. It happens when non-splitting plans for dt_o are evaluated. The cost of sp_i is considered as the cost of materialization of dt_i when evaluating any splitting plan for dt_o. */ if (fix_all_splittings_in_plan()) DBUG_RETURN(1); setup_subq_exit: with_two_phase_optimization= check_two_phase_optimization(thd); if (with_two_phase_optimization) optimization_state= JOIN::OPTIMIZATION_PHASE_1_DONE; else { if (optimize_stage2()) DBUG_RETURN(1); } DBUG_RETURN(0); } /* @brief In the Stage 1 we've picked the join order. Now, refine the query plan and sort out all the details. The choice how to handle GROUP/ORDER BY is also made here. @detail The main reason this is a separate function is Split-Materialized optimization. There, we first consider doing non-split Materialization for a SELECT. After that, the parent SELECT will attempt doing Splitting in multiple ways and make the final choice. */ int JOIN::optimize_stage2() { ulonglong select_opts_for_readinfo; uint no_jbuf_after; JOIN_TAB *tab; DBUG_ENTER("JOIN::optimize_stage2"); if (subq_exit_fl) goto setup_subq_exit; if (unlikely(thd->check_killed())) DBUG_RETURN(1); /* Generate an execution plan from the found optimal join order. */ if (get_best_combination()) DBUG_RETURN(1); if (make_range_rowid_filters()) DBUG_RETURN(1); if (select_lex->handle_derived(thd->lex, DT_OPTIMIZE_STAGE2)) DBUG_RETURN(1); /* We have to call drop_unused_derived_keys() even if we don't have any generated keys (enabled with OPTIMIZER_SWITCH_DERIVED_WITH_KEYS) as we may still have unique constraints we have to get rid of. */ drop_unused_derived_keys(); if (rollup.state != ROLLUP::STATE_NONE) { if (rollup_process_const_fields()) { DBUG_PRINT("error", ("Error: rollup_process_fields() failed")); DBUG_RETURN(1); } } else { /* Remove distinct if only const tables */ select_distinct= select_distinct && (const_tables != table_count); } THD_STAGE_INFO(thd, stage_preparing); if (result->initialize_tables(this)) { DBUG_PRINT("error",("Error: initialize_tables() failed")); DBUG_RETURN(1); // error == -1 } if (const_table_map != found_const_table_map && !(select_options & SELECT_DESCRIBE)) { // There is at least one empty const table zero_result_cause= "no matching row in const table"; DBUG_PRINT("error",("Error: %s", zero_result_cause)); error= 0; handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } if (!(thd->variables.option_bits & OPTION_BIG_SELECTS) && join_record_count > (double) thd->variables.max_join_size && !(select_options & SELECT_DESCRIBE)) { /* purecov: inspected */ my_message(ER_TOO_BIG_SELECT, ER_THD(thd, ER_TOO_BIG_SELECT), MYF(0)); error= -1; DBUG_RETURN(1); } if (const_tables && !thd->locked_tables_mode && !(select_options & SELECT_NO_UNLOCK)) { /* Unlock all tables, except sequences, as accessing these may still require table updates. It's safe to ignore result code as all tables where opened for read only. */ (void) mysql_unlock_some_tables(thd, table, const_tables, GET_LOCK_SKIP_SEQUENCES); } if (!conds && outer_join) { /* Handle the case where we have an OUTER JOIN without a WHERE */ conds= (Item*) Item_true; } if (impossible_where) { zero_result_cause= "Impossible WHERE noticed after reading const tables"; select_lex->mark_const_derived(zero_result_cause); handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } select= make_select(*table, const_table_map, const_table_map, conds, (SORT_INFO*) 0, 1, &error); if (unlikely(error)) { /* purecov: inspected */ error= -1; /* purecov: inspected */ DBUG_PRINT("error",("Error: make_select() failed")); DBUG_RETURN(1); } reset_nj_counters(this, join_list); if (make_outerjoin_info(this)) { DBUG_RETURN(1); } /* Among the equal fields belonging to the same multiple equality choose the one that is to be retrieved first and substitute all references to these in where condition for a reference for the selected field. */ if (conds) { conds= substitute_for_best_equal_field(thd, NO_PARTICULAR_TAB, conds, cond_equal, map2table, true); if (unlikely(thd->is_error())) { error= 1; DBUG_PRINT("error",("Error from substitute_for_best_equal")); DBUG_RETURN(1); } conds->update_used_tables(); if (unlikely(thd->trace_started())) trace_condition(thd, "WHERE", "substitute_best_equal", conds); DBUG_EXECUTE("where", print_where(conds, "after substitute_best_equal", QT_ORDINARY);); } if (having) { having= substitute_for_best_equal_field(thd, NO_PARTICULAR_TAB, having, having_equal, map2table, false); if (thd->is_error()) { error= 1; DBUG_PRINT("error",("Error from substitute_for_best_equal")); DBUG_RETURN(1); } if (having) { having->update_used_tables(); if (unlikely(thd->trace_started())) trace_condition(thd, "HAVING", "substitute_best_equal", having); } DBUG_EXECUTE("having", print_where(having, "after substitute_best_equal", QT_ORDINARY);); } /* Perform the optimization on fields evaluation mentioned above for all on expressions. */ for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { if (*tab->on_expr_ref) { *tab->on_expr_ref= substitute_for_best_equal_field(thd, NO_PARTICULAR_TAB, *tab->on_expr_ref, tab->cond_equal, map2table, true); if (unlikely(thd->is_error())) { error= 1; DBUG_PRINT("error",("Error from substitute_for_best_equal")); DBUG_RETURN(1); } (*tab->on_expr_ref)->update_used_tables(); if (unlikely(thd->trace_started())) { trace_condition(thd, "ON expr", "substitute_best_equal", (*tab->on_expr_ref), tab->table->alias.c_ptr()); } } } /* Perform the optimization on fields evaluation mentioned above for all used ref items. */ for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { uint key_copy_index=0; for (uint i=0; i < tab->ref.key_parts; i++) { Item **ref_item_ptr= tab->ref.items+i; Item *ref_item= *ref_item_ptr; if (!ref_item->used_tables() && !(select_options & SELECT_DESCRIBE)) continue; COND_EQUAL *equals= cond_equal; JOIN_TAB *first_inner= tab->first_inner; while (equals) { ref_item= substitute_for_best_equal_field(thd, tab, ref_item, equals, map2table, true); if (unlikely(thd->is_error())) DBUG_RETURN(1); if (first_inner) { equals= first_inner->cond_equal; first_inner= first_inner->first_upper; } else equals= 0; } ref_item->update_used_tables(); if (*ref_item_ptr != ref_item) { *ref_item_ptr= ref_item; Item *item= ref_item->real_item(); store_key *key_copy= tab->ref.key_copy[key_copy_index]; if (key_copy->type() == store_key::FIELD_STORE_KEY) { if (item->basic_const_item()) { /* It is constant propagated here */ tab->ref.key_copy[key_copy_index]= new store_key_const_item(*tab->ref.key_copy[key_copy_index], item); } else if (item->const_item()) { tab->ref.key_copy[key_copy_index]= new store_key_item(*tab->ref.key_copy[key_copy_index], item, TRUE); } else { store_key_field *field_copy= ((store_key_field *)key_copy); DBUG_ASSERT(item->type() == Item::FIELD_ITEM); field_copy->change_source_field((Item_field *) item); } } } key_copy_index++; } } if (conds && const_table_map != found_const_table_map && (select_options & SELECT_DESCRIBE)) conds= (Item*) Item_false; /* Cache constant expressions in WHERE, HAVING, ON clauses. */ cache_const_exprs(); if (setup_semijoin_loosescan(this)) DBUG_RETURN(1); if (make_join_select(this, select, conds)) { if (thd->is_error()) DBUG_RETURN(1); zero_result_cause= "Impossible WHERE noticed after reading const tables"; select_lex->mark_const_derived(zero_result_cause); handle_implicit_grouping_with_window_funcs(); goto setup_subq_exit; } error= -1; /* if goto err */ /* Optimize distinct away if possible */ { ORDER *org_order= order; order=remove_const(this, order,conds,1, &simple_order); if (unlikely(thd->is_error())) { error= 1; DBUG_RETURN(1); } /* If we are using ORDER BY NULL or ORDER BY const_expression, return result in any order (even if we are using a GROUP BY) */ if (!order && org_order) skip_sort_order= 1; } /* For FETCH ... WITH TIES save how many items order by had, after we've removed constant items that have no relevance on the final sorting. */ if (unit->lim.is_with_ties()) { DBUG_ASSERT(with_ties_order_count == 0); for (ORDER *it= order; it; it= it->next) with_ties_order_count+= 1; } /* Check if we can optimize away GROUP BY/DISTINCT. We can do that if there are no aggregate functions, the fields in DISTINCT clause (if present) and/or columns in GROUP BY (if present) contain direct references to all key parts of an unique index (in whatever order) and if the key parts of the unique index cannot contain NULLs. Note that the unique keys for DISTINCT and GROUP BY should not be the same (as long as they are unique). The FROM clause must contain a single non-constant table. */ if (table_count - const_tables == 1 && (group || select_distinct) && !tmp_table_param.sum_func_count && (!join_tab[const_tables].select || !join_tab[const_tables].select->quick || join_tab[const_tables].select->quick->get_type() != QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) && !select_lex->have_window_funcs()) { if (group && rollup.state == ROLLUP::STATE_NONE && list_contains_unique_index(join_tab[const_tables].table, find_field_in_order_list, (void *) group_list)) { /* We have found that grouping can be removed since groups correspond to only one row anyway, but we still have to guarantee correct result order. The line below effectively rewrites the query from GROUP BY <fields> to ORDER BY <fields>. There are three exceptions: - if skip_sort_order is set (see above), then we can simply skip GROUP BY; - if we are in a subquery, we don't have to maintain order unless there is a limit clause in the subquery. - we can only rewrite ORDER BY if the ORDER BY fields are 'compatible' with the GROUP BY ones, i.e. either one is a prefix of another. We only check if the ORDER BY is a prefix of GROUP BY. In this case test_if_subpart() copies the ASC/DESC attributes from the original ORDER BY fields. If GROUP BY is a prefix of ORDER BY, then it is safe to leave 'order' as is. */ if (!order || test_if_subpart(group_list, order)) { if (skip_sort_order || (select_lex->master_unit()->item && select_limit == HA_POS_ERROR)) // This is a subquery order= NULL; else order= group_list; } /* If we have an IGNORE INDEX FOR GROUP BY(fields) clause, this must be rewritten to IGNORE INDEX FOR ORDER BY(fields). */ join_tab->table->keys_in_use_for_order_by= join_tab->table->keys_in_use_for_group_by; group_list= 0; group= 0; } if (select_distinct && list_contains_unique_index(join_tab[const_tables].table, find_field_in_item_list, (void *) &fields_list)) { select_distinct= 0; } } if (group || tmp_table_param.sum_func_count) { if (! hidden_group_fields && rollup.state == ROLLUP::STATE_NONE && !select_lex->have_window_funcs()) select_distinct=0; } else if (select_distinct && table_count - const_tables == 1 && rollup.state == ROLLUP::STATE_NONE && !select_lex->have_window_funcs()) { /* We are only using one table. In this case we change DISTINCT to a GROUP BY query if: - The GROUP BY can be done through indexes (no sort) and the ORDER BY only uses selected fields. (In this case we can later optimize away GROUP BY and ORDER BY) - We are scanning the whole table without LIMIT This can happen if: - We are using CALC_FOUND_ROWS - We are using an ORDER BY that can't be optimized away. We don't want to use this optimization when we are using LIMIT because in this case we can just create a temporary table that holds LIMIT rows and stop when this table is full. */ bool all_order_fields_used; tab= &join_tab[const_tables]; if (order) { bool fatal_err; skip_sort_order= test_if_skip_sort_order(tab, order, select_limit, true, // no_changes &tab->table->keys_in_use_for_order_by, &fatal_err); if (fatal_err) DBUG_RETURN(1); } if ((group_list=create_distinct_group(thd, select_lex->ref_pointer_array, order, fields_list, all_fields, &all_order_fields_used))) { bool fatal_err= 0; const bool skip_group= skip_sort_order && test_if_skip_sort_order(tab, group_list, select_limit, true, // no_changes &tab->table->keys_in_use_for_group_by, &fatal_err); if (fatal_err) DBUG_RETURN(1); count_field_types(select_lex, &tmp_table_param, all_fields, 0); if ((skip_group && all_order_fields_used) || select_limit == HA_POS_ERROR || (order && !skip_sort_order)) { /* Change DISTINCT to GROUP BY */ select_distinct= 0; no_order= !order; if (all_order_fields_used) { if (order && skip_sort_order) { /* Force MySQL to read the table in sorted order to get result in ORDER BY order. */ tmp_table_param.quick_group=0; } order=0; } group=1; // For end_write_group } else group_list= 0; } else if (thd->is_error()) // End of memory DBUG_RETURN(1); } simple_group= rollup.state == ROLLUP::STATE_NONE; if (group) { /* Update simple_group and group_list as we now have more information, like which tables or columns are constant. */ group_list= remove_const(this, group_list, conds, rollup.state == ROLLUP::STATE_NONE, &simple_group); if (unlikely(thd->is_error())) { error= 1; DBUG_RETURN(1); } if (!group_list) { /* The output has only one row */ order=0; simple_order=1; select_distinct= 0; group_optimized_away= 1; } } calc_group_buffer(this, group_list); send_group_parts= tmp_table_param.group_parts; /* Save org parts */ if (procedure && procedure->group) { group_list= procedure->group= remove_const(this, procedure->group, conds, 1, &simple_group); if (unlikely(thd->is_error())) { error= 1; DBUG_RETURN(1); } calc_group_buffer(this, group_list); } /* We can ignore ORDER BY if it's a prefix of the GROUP BY list (as MariaDB is by default sorting on GROUP BY) or if there is no GROUP BY and aggregate functions are used (as the result will only contain one row). (1) - Do not remove ORDER BY if we have WITH TIES and are using QUICK_GROUP_MIN_MAX_SELECT to handle GROUP BY. See the comment for using_with_ties_and_group_min_max() for details. */ if (order && (test_if_subpart(group_list, order) || (!group_list && tmp_table_param.sum_func_count)) && !using_with_ties_and_group_min_max(this)) // (1) order=0; // Can't use sort on head table if using join buffering if (full_join || hash_join) { TABLE *stable= (sort_by_table == (TABLE *) 1 ? join_tab[const_tables].table : sort_by_table); /* FORCE INDEX FOR ORDER BY can be used to prevent join buffering when sorting on the first table. */ if (!stable || (!stable->force_index_order && !map2table[stable->tablenr]->keep_current_rowid)) { if (group_list) simple_group= 0; if (order) simple_order= 0; } } need_tmp= test_if_need_tmp_table(); /* If window functions are present then we can't have simple_order set to TRUE as the window function needs a temp table for computation. ORDER BY is computed after the window function computation is done, so the sort will be done on the temp table. */ if (select_lex->have_window_funcs()) simple_order= FALSE; /* If the hint FORCE INDEX FOR ORDER BY/GROUP BY is used for the table whose columns are required to be returned in a sorted order, then the proper value for no_jbuf_after should be yielded by a call to the make_join_orderinfo function. Yet the current implementation of FORCE INDEX hints does not allow us to do it in a clean manner. */ no_jbuf_after= 1 ? table_count : make_join_orderinfo(this); // Don't use join buffering when we use MATCH select_opts_for_readinfo= (select_options & (SELECT_DESCRIBE | SELECT_NO_JOIN_CACHE)) | (select_lex->ftfunc_list->elements ? SELECT_NO_JOIN_CACHE : 0); if (select_lex->options & OPTION_SCHEMA_TABLE && optimize_schema_tables_reads(this)) DBUG_RETURN(1); if (make_join_readinfo(this, select_opts_for_readinfo, no_jbuf_after)) DBUG_RETURN(1); /* Perform FULLTEXT search before all regular searches */ if (!(select_options & SELECT_DESCRIBE)) if (init_ftfuncs(thd, select_lex, MY_TEST(order))) DBUG_RETURN(1); /* It's necessary to check const part of HAVING cond as there is a chance that some cond parts may become const items after make_join_statistics(for example when Item is a reference to cost table field from outer join). This check is performed only for those conditions which do not use aggregate functions. In such case temporary table may not be used and const condition elements may be lost during further having condition transformation. */ if (having && const_table_map && !having->with_sum_func()) { having->update_used_tables(); if (having->const_item() && !having->is_expensive()) { if (!having->val_bool()) { having= Item_false; zero_result_cause= "Impossible HAVING noticed after reading const tables"; error= 0; select_lex->mark_const_derived(zero_result_cause); goto setup_subq_exit; } having= Item_true; } } if (optimize_unflattened_subqueries()) DBUG_RETURN(1); int res; if ((res= rewrite_to_index_subquery_engine(this)) != -1) DBUG_RETURN(res); if (setup_subquery_caches()) DBUG_RETURN(-1); /* Need to tell handlers that to play it safe, it should fetch all columns of the primary key of the tables: this is because MySQL may build row pointers for the rows, and for all columns of the primary key the read set has not necessarily been set by the server code. */ if (need_tmp || select_distinct || group_list || order) { for (uint i= 0; i < table_count; i++) { if (!(table[i]->map & const_table_map)) table[i]->prepare_for_position(); } } DBUG_EXECUTE("info",TEST_join(this);); if (!only_const_tables()) { JOIN_TAB *tab= &join_tab[const_tables]; if (order && !need_tmp) { /* Force using of tmp table if sorting by a SP or UDF function due to their expensive and probably non-deterministic nature. */ for (ORDER *tmp_order= order; tmp_order ; tmp_order=tmp_order->next) { Item *item= *tmp_order->item; if (item->is_expensive()) { /* Force tmp table without sort */ need_tmp=1; simple_order=simple_group=0; break; } } } /* Because filesort always does a full table scan or a quick range scan we must add the removed reference to the select for the table. We only need to do this when we have a simple_order or simple_group as in other cases the join is done before the sort. */ if ((order || group_list) && tab->type != JT_ALL && tab->type != JT_RANGE && tab->type != JT_NEXT && tab->type != JT_FT && tab->type != JT_REF_OR_NULL && ((order && simple_order) || (group_list && simple_group))) { if (add_ref_to_table_cond(thd,tab)) { DBUG_RETURN(1); } } /* Investigate whether we may use an ordered index as part of either DISTINCT, GROUP BY or ORDER BY execution. An ordered index may be used for only the first of any of these terms to be executed. This is reflected in the order which we check for test_if_skip_sort_order() below. However we do not check for DISTINCT here, as it would have been transformed to a GROUP BY at this stage if it is a candidate for ordered index optimization. If a decision was made to use an ordered index, the availability of such an access path is stored in 'ordered_index_usage' for later use by 'execute' or 'explain' */ DBUG_ASSERT(ordered_index_usage == ordered_index_void); if (group_list) // GROUP BY honoured first // (DISTINCT was rewritten to GROUP BY if skippable) { /* When there is SQL_BIG_RESULT do not sort using index for GROUP BY, and thus force sorting on disk unless a group min-max optimization is going to be used as it is applied now only for one table queries with covering indexes. */ if (!(select_options & SELECT_BIG_RESULT) || (tab->select && tab->select->quick && tab->select->quick->get_type() == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX)) { if (simple_group && // GROUP BY is possibly skippable !select_distinct) // .. if not preceded by a DISTINCT { /* Calculate a possible 'limit' of table rows for 'GROUP BY': A specified 'LIMIT' is relative to the final resultset. 'need_tmp' implies that there will be more postprocessing so the specified 'limit' should not be enforced yet. */ bool fatal_err; const ha_rows limit = need_tmp ? HA_POS_ERROR : select_limit; if (test_if_skip_sort_order(tab, group_list, limit, false, &tab->table->keys_in_use_for_group_by, &fatal_err)) { ordered_index_usage= ordered_index_group_by; } if (fatal_err) DBUG_RETURN(1); } /* If we are going to use semi-join LooseScan, it will depend on the selected index scan to be used. If index is not used for the GROUP BY, we risk that sorting is put on the LooseScan table. In order to avoid this, force use of temporary table. TODO: Explain the quick_group part of the test below. */ if ((ordered_index_usage != ordered_index_group_by) && ((tmp_table_param.quick_group && !procedure) || (tab->emb_sj_nest && best_positions[const_tables].sj_strategy == SJ_OPT_LOOSE_SCAN))) { need_tmp=1; simple_order= simple_group= false; // Force tmp table without sort } } } else if (order && // ORDER BY wo/ preceding GROUP BY (simple_order || skip_sort_order)) // which is possibly skippable { bool fatal_err; if (test_if_skip_sort_order(tab, order, select_limit, false, &tab->table->keys_in_use_for_order_by, &fatal_err)) { ordered_index_usage= ordered_index_order_by; } if (fatal_err) DBUG_RETURN(1); } } if (having) having_is_correlated= MY_TEST(having->used_tables() & OUTER_REF_TABLE_BIT); tmp_having= having; if (unlikely(thd->is_error())) DBUG_RETURN(TRUE); /* The loose index scan access method guarantees that all grouping or duplicate row elimination (for distinct) is already performed during data retrieval, and that all MIN/MAX functions are already computed for each group. Thus all MIN/MAX functions should be treated as regular functions, and there is no need to perform grouping in the main execution loop. Notice that currently loose index scan is applicable only for single table queries, thus it is sufficient to test only the first join_tab element of the plan for its access method. */ if (join_tab->is_using_loose_index_scan()) { tmp_table_param.precomputed_group_by= TRUE; if (join_tab->is_using_agg_loose_index_scan()) { need_distinct= FALSE; tmp_table_param.precomputed_group_by= FALSE; } } if (make_aggr_tables_info()) DBUG_RETURN(1); init_join_cache_and_keyread(); if (init_range_rowid_filters()) DBUG_RETURN(1); error= 0; if (select_options & SELECT_DESCRIBE) goto derived_exit; DBUG_RETURN(0); setup_subq_exit: /* Choose an execution strategy for this JOIN. */ if (!tables_list || !table_count) { choose_tableless_subquery_plan(); /* The output has atmost one row */ if (group_list) { group_list= NULL; group_optimized_away= 1; rollup.state= ROLLUP::STATE_NONE; } order= NULL; simple_order= TRUE; select_distinct= FALSE; if (select_lex->have_window_funcs()) { if (!(join_tab= thd->alloc<JOIN_TAB>(1))) DBUG_RETURN(1); #ifndef DBUG_OFF dbug_join_tab_array_size= 1; #endif need_tmp= 1; } if (make_aggr_tables_info()) DBUG_RETURN(1); /* It could be that we've only done optimization stage 1 for some of the derived tables, and never did stage 2. Do it now, otherwise Explain data structure will not be complete. */ if (select_lex->handle_derived(thd->lex, DT_OPTIMIZE_STAGE2)) DBUG_RETURN(1); } /* Even with zero matching rows, subqueries in the HAVING clause may need to be evaluated if there are aggregate functions in the query. */ if (optimize_unflattened_subqueries()) DBUG_RETURN(1); error= 0; derived_exit: select_lex->mark_const_derived(zero_result_cause); DBUG_RETURN(0); } /** Add having condition as a where clause condition of the given temp table. @param tab Table to which having condition is added. @returns false if success, true if error. */ bool JOIN::add_having_as_table_cond(JOIN_TAB *tab) { tmp_having->update_used_tables(); table_map used_tables= tab->table->map | OUTER_REF_TABLE_BIT; /* If tmp table is not used then consider conditions of const table also */ if (!need_tmp) used_tables|= const_table_map; DBUG_ENTER("JOIN::add_having_as_table_cond"); Item* sort_table_cond= make_cond_for_table(thd, tmp_having, used_tables, (table_map) 0, 0, false, false); if (sort_table_cond) { if (!tab->select) { if (!(tab->select= new SQL_SELECT)) DBUG_RETURN(true); tab->select->head= tab->table; } if (!tab->select->cond) tab->select->cond= sort_table_cond; else { if (!(tab->select->cond= new (thd->mem_root) Item_cond_and(thd, tab->select->cond, sort_table_cond))) DBUG_RETURN(true); } if (tab->pre_idx_push_select_cond) { if (sort_table_cond->type() == Item::COND_ITEM) sort_table_cond= sort_table_cond->copy_andor_structure(thd); if (!(tab->pre_idx_push_select_cond= new (thd->mem_root) Item_cond_and(thd, tab->pre_idx_push_select_cond, sort_table_cond))) DBUG_RETURN(true); } if (tab->select->cond) tab->select->cond->fix_fields_if_needed(thd, 0); if (tab->pre_idx_push_select_cond) tab->pre_idx_push_select_cond->fix_fields_if_needed(thd, 0); tab->select->pre_idx_push_select_cond= tab->pre_idx_push_select_cond; tab->set_select_cond(tab->select->cond, __LINE__); tab->select_cond->top_level_item(); DBUG_EXECUTE("where",print_where(tab->select->cond, "select and having", QT_ORDINARY);); having= make_cond_for_table(thd, tmp_having, ~ (table_map) 0, ~used_tables, 0, false, false); if (!having && thd->is_error()) DBUG_RETURN(true); DBUG_EXECUTE("where", print_where(having, "having after sort", QT_ORDINARY);); } else if (thd->is_error()) DBUG_RETURN(true); DBUG_RETURN(false); } bool JOIN::add_fields_for_current_rowid(JOIN_TAB *cur, List<Item> *table_fields) { /* this will not walk into semi-join materialization nests but this is ok because we will never need to save current rowids for those. */ for (JOIN_TAB *tab=join_tab; tab < cur; tab++) { if (!tab->keep_current_rowid) continue; Item *item= new (thd->mem_root) Item_temptable_rowid(tab->table); item->fix_fields(thd, 0); /* table_fields points to JOIN::all_fields or JOIN::tmp_all_fields_*. These lists start with "added" fields and then their suffix is shared with JOIN::fields_list or JOIN::tmp_fields_list*. Because of that, new elements can only be added to the front of the list, not to the back. */ table_fields->push_front(item, thd->mem_root); cur->tmp_table_param->func_count++; } return 0; } /** Set info for aggregation tables @details This function finalizes execution plan by taking following actions: .) aggregation temporary tables are created, but not instantiated (this is done during execution). JOIN_TABs for aggregation tables are set appropriately (see JOIN::create_postjoin_aggr_table). .) prepare fields lists (fields, all_fields, ref_pointer_array slices) for each required stage of execution. These fields lists are set for working tables' tabs and for the tab of last table in the join. .) info for sorting/grouping/dups removal is prepared and saved in appropriate tabs. Here is an example: @returns false - Ok true - Error */ bool JOIN::make_aggr_tables_info() { List<Item> *curr_all_fields= &all_fields; List<Item> *curr_fields_list= &fields_list; // Avoid UB (applying .. offset to nullptr) when join_tab is nullptr JOIN_TAB *curr_tab= join_tab ? join_tab + const_tables : nullptr; TABLE *exec_tmp_table= NULL; bool distinct= false; const bool has_group_by= this->group; bool keep_row_order= thd->lex->with_rownum && (group_list || order); bool is_having_added_as_table_cond= false; DBUG_ENTER("JOIN::make_aggr_tables_info"); DBUG_ASSERT(current_ref_ptrs == items0); sort_and_group_aggr_tab= NULL; if (group_optimized_away) implicit_grouping= true; bool implicit_grouping_with_window_funcs= implicit_grouping && select_lex->have_window_funcs(); bool implicit_grouping_without_tables= implicit_grouping && !tables_list; /* Setup last table to provide fields and all_fields lists to the next node in the plan. */ if (join_tab && top_join_tab_count && tables_list) { join_tab[top_join_tab_count - 1].fields= &fields_list; join_tab[top_join_tab_count - 1].all_fields= &all_fields; } /* All optimization is done. Check if we can use the storage engines group by handler to evaluate the group by. Some storage engines, like spider can also do joins, group by and distinct in the engine, so we do this for all queries, not only GROUP BY queries. */ if (tables_list && top_join_tab_count && !only_const_tables() && !procedure) { /* At the moment we only support push down for queries where all tables are in the same storage engine */ TABLE_LIST *tbl= tables_list; handlerton *ht= tbl && tbl->table ? tbl->table->file->partition_ht() : 0; for (tbl= tbl->next_local; ht && tbl; tbl= tbl->next_local) { if (!tbl->table || tbl->table->file->partition_ht() != ht) ht= 0; } if (ht && ht->create_group_by) { /* Check if the storage engine can intercept the query JOIN::optimize_stage2() might convert DISTINCT into GROUP BY and then optimize away GROUP BY (group_list). In such a case, we need to notify a storage engine supporting a group by handler of the existence of the original DISTINCT. Thus, we set select_distinct || group_optimized_away to Query::distinct. */ Query query= {&all_fields, (int) all_fields.elements - (int) fields_list.elements, select_distinct || group_optimized_away, tables_list, conds, group_list, order ? order : group_list, having, &select_lex->master_unit()->lim}; group_by_handler *gbh= ht->create_group_by(thd, &query); if (gbh) { if (!(pushdown_query= new (thd->mem_root) Pushdown_query(select_lex, gbh))) DBUG_RETURN(1); /* We must store rows in the tmp table if we need to do an ORDER BY or DISTINCT and the storage handler can't handle it. */ need_tmp= query.order_by || query.group_by || query.distinct; distinct= query.distinct; keep_row_order= query.order_by || query.group_by; order= query.order_by; aggr_tables++; curr_tab= join_tab + exec_join_tab_cnt(); bzero((void*)curr_tab, sizeof(JOIN_TAB)); curr_tab->ref.key= -1; curr_tab->join= this; if (!(curr_tab->tmp_table_param= new TMP_TABLE_PARAM(tmp_table_param))) DBUG_RETURN(1); curr_tab->tmp_table_param->func_count= all_fields.elements; TABLE* table= create_tmp_table(thd, curr_tab->tmp_table_param, all_fields, NULL, distinct, TRUE, select_options, HA_ROWS_MAX, &empty_clex_str, !need_tmp, keep_row_order); if (!table) DBUG_RETURN(1); if (!(curr_tab->aggr= new (thd->mem_root) AGGR_OP(curr_tab))) DBUG_RETURN(1); curr_tab->aggr->set_write_func(::end_send); curr_tab->table= table; /* Setup reference fields, used by summary functions and group by fields, to point to the temporary table. The actual switching to the temporary tables fields for HAVING and ORDER BY is done in do_select() by calling set_items_ref_array(items1). */ init_items_ref_array(); items1= ref_ptr_array_slice(2); if (change_to_use_tmp_fields(thd, items1, tmp_fields_list1, tmp_all_fields1, fields_list.elements, all_fields)) DBUG_RETURN(1); /* Give storage engine access to temporary table */ gbh->table= table; pushdown_query->store_data_in_temp_table= need_tmp; pushdown_query->having= having; /* Group by and having is calculated by the group_by handler. Reset the group by and having */ DBUG_ASSERT(query.group_by == NULL); group= 0; group_list= 0; having= tmp_having= 0; /* Select distinct is handled by handler or by creating an unique index over all fields in the temporary table */ select_distinct= 0; order= query.order_by; tmp_table_param.field_count+= tmp_table_param.sum_func_count; tmp_table_param.sum_func_count= 0; fields= curr_fields_list; //todo: new: curr_tab->ref_array= &items1; curr_tab->all_fields= &tmp_all_fields1; curr_tab->fields= &tmp_fields_list1; DBUG_RETURN(thd->is_error()); } } } /* The loose index scan access method guarantees that all grouping or duplicate row elimination (for distinct) is already performed during data retrieval, and that all MIN/MAX functions are already computed for each group. Thus all MIN/MAX functions should be treated as regular functions, and there is no need to perform grouping in the main execution loop. Notice that currently loose index scan is applicable only for single table queries, thus it is sufficient to test only the first join_tab element of the plan for its access method. */ if (join_tab && top_join_tab_count && tables_list && join_tab->is_using_loose_index_scan()) tmp_table_param.precomputed_group_by= !join_tab->is_using_agg_loose_index_scan(); group_list_for_estimates= group_list; /* Create a tmp table if distinct or if the sort is too complicated */ if (need_tmp) { aggr_tables++; curr_tab= join_tab + exec_join_tab_cnt(); DBUG_ASSERT(curr_tab - join_tab < dbug_join_tab_array_size); bzero((void*)curr_tab, sizeof(JOIN_TAB)); curr_tab->ref.key= -1; if (only_const_tables()) first_select= sub_select_postjoin_aggr; /* Create temporary table on first execution of this join. (Will be reused if this is a subquery that is executed several times.) */ init_items_ref_array(); ORDER *tmp_group= (ORDER *) 0; if (!simple_group && !procedure && !(test_flags & TEST_NO_KEY_GROUP)) tmp_group= group_list; tmp_table_param.hidden_field_count= all_fields.elements - fields_list.elements; distinct= select_distinct && !group_list && !select_lex->have_window_funcs(); keep_row_order= thd->lex->with_rownum && (group_list || order); bool save_sum_fields= (group_list && simple_group) || implicit_grouping_with_window_funcs; if (create_postjoin_aggr_table(curr_tab, &all_fields, tmp_group, save_sum_fields, distinct, keep_row_order)) DBUG_RETURN(true); exec_tmp_table= curr_tab->table; if (exec_tmp_table->distinct) optimize_distinct(); /* Change sum_fields reference to calculated fields in tmp_table */ items1= ref_ptr_array_slice(2); if ((sort_and_group || curr_tab->table->group || tmp_table_param.precomputed_group_by) && !implicit_grouping_without_tables) { if (change_to_use_tmp_fields(thd, items1, tmp_fields_list1, tmp_all_fields1, fields_list.elements, all_fields)) DBUG_RETURN(true); } else { if (change_refs_to_tmp_fields(thd, items1, tmp_fields_list1, tmp_all_fields1, fields_list.elements, all_fields)) DBUG_RETURN(true); } curr_all_fields= &tmp_all_fields1; curr_fields_list= &tmp_fields_list1; // Need to set them now for correct group_fields setup, reset at the end. set_items_ref_array(items1); curr_tab->ref_array= &items1; curr_tab->all_fields= &tmp_all_fields1; curr_tab->fields= &tmp_fields_list1; set_postjoin_aggr_write_func(curr_tab); /* If having is not handled here, it will be checked before the row is sent to the client. */ if (tmp_having && (sort_and_group || (exec_tmp_table->distinct && !group_list) || select_lex->have_window_funcs())) { /* If there is no select distinct and there are no window functions then move the having to table conds of tmp table. NOTE : We cannot apply having after distinct or window functions If columns of having are not part of select distinct, then distinct may remove rows which can satisfy having. In the case of window functions we *must* make sure to not store any rows which don't match HAVING within the temp table, as rows will end up being used during their computation. */ if (!select_distinct && !select_lex->have_window_funcs() && add_having_as_table_cond(curr_tab)) DBUG_RETURN(true); is_having_added_as_table_cond= tmp_having != having; /* Having condition which we are not able to add as tmp table conds are kept as before. And, this will be applied before storing the rows in tmp table. */ curr_tab->having= having; having= NULL; // Already done } tmp_table_param.func_count= 0; tmp_table_param.field_count+= tmp_table_param.func_count; if (sort_and_group || curr_tab->table->group) { tmp_table_param.field_count+= tmp_table_param.sum_func_count; tmp_table_param.sum_func_count= 0; } if (exec_tmp_table->group) { // Already grouped if (!order && !no_order && !skip_sort_order) order= group_list; /* order by group */ group_list= NULL; } /* If we have different sort & group then we must sort the data by group and copy it to another tmp table. This code is also used if we are using distinct something we haven't been able to store in the temporary table yet like SEC_TO_TIME(SUM(...)). 3. Also, this is used when - the query has Window functions, - the GROUP BY operation is done with OrderedGroupBy algorithm. In this case, the first temptable will contain pre-GROUP-BY data. Force the creation of the second temporary table. Post-GROUP-BY dataset will be written there, and then Window Function processing code will be able to process it. */ if ((group_list && (!test_if_subpart(group_list, order) || select_distinct)) || (select_distinct && tmp_table_param.using_outer_summary_function) || (group_list && !tmp_table_param.quick_group && // (3) select_lex->have_window_funcs())) // (3) { /* Must copy to another table */ DBUG_PRINT("info",("Creating group table")); calc_group_buffer(this, group_list); count_field_types(select_lex, &tmp_table_param, tmp_all_fields1, select_distinct && !group_list); tmp_table_param.hidden_field_count= tmp_all_fields1.elements - tmp_fields_list1.elements; curr_tab++; aggr_tables++; DBUG_ASSERT(curr_tab - join_tab < dbug_join_tab_array_size); bzero((void*)curr_tab, sizeof(JOIN_TAB)); curr_tab->ref.key= -1; /* group data to new table */ /* If the access method is loose index scan then all MIN/MAX functions are precomputed, and should be treated as regular functions. See extended comment above. */ if (join_tab->is_using_loose_index_scan()) tmp_table_param.precomputed_group_by= TRUE; tmp_table_param.hidden_field_count= curr_all_fields->elements - curr_fields_list->elements; ORDER *dummy= NULL; //TODO can use table->group here also if (create_postjoin_aggr_table(curr_tab, curr_all_fields, dummy, true, distinct, keep_row_order)) DBUG_RETURN(true); if (group_list) { if (!only_const_tables()) // No need to sort a single row { if (add_sorting_to_table(curr_tab - 1, group_list)) DBUG_RETURN(true); } if (make_group_fields(this, this)) DBUG_RETURN(true); } // Setup sum funcs only when necessary, otherwise we might break info // for the first table if (group_list || tmp_table_param.sum_func_count) { if (make_sum_func_list(*curr_all_fields, *curr_fields_list, true)) DBUG_RETURN(true); if (prepare_sum_aggregators(thd, sum_funcs, !join_tab->is_using_agg_loose_index_scan())) DBUG_RETURN(true); group_list= NULL; if (setup_sum_funcs(thd, sum_funcs)) DBUG_RETURN(true); } // No sum funcs anymore DBUG_ASSERT(items2.is_null()); items2= ref_ptr_array_slice(3); if (change_to_use_tmp_fields(thd, items2, tmp_fields_list2, tmp_all_fields2, fields_list.elements, tmp_all_fields1)) DBUG_RETURN(true); curr_fields_list= &tmp_fields_list2; curr_all_fields= &tmp_all_fields2; set_items_ref_array(items2); curr_tab->ref_array= &items2; curr_tab->all_fields= &tmp_all_fields2; curr_tab->fields= &tmp_fields_list2; set_postjoin_aggr_write_func(curr_tab); tmp_table_param.field_count+= tmp_table_param.sum_func_count; tmp_table_param.sum_func_count= 0; } if (curr_tab->table->distinct) select_distinct= false; /* Each row is unique */ if (select_distinct && !group_list) { if (having) { curr_tab->having= having; having->update_used_tables(); } /* We only need DISTINCT operation if the join is not degenerate. If it is, we must not request DISTINCT processing, because remove_duplicates() assumes there is a preceding computation step (and in the degenerate join, there's none) */ if (top_join_tab_count && tables_list) curr_tab->distinct= true; having= NULL; select_distinct= false; } /* Clean tmp_table_param for the next tmp table. */ tmp_table_param.field_count= tmp_table_param.sum_func_count= tmp_table_param.func_count= 0; tmp_table_param.copy_field= tmp_table_param.copy_field_end=0; first_record= sort_and_group=0; if (!group_optimized_away || implicit_grouping_with_window_funcs) { group= false; } else { /* If grouping has been optimized away, a temporary table is normally not needed unless we're explicitly requested to create one (e.g. due to a SQL_BUFFER_RESULT hint or INSERT ... SELECT). In this case (grouping was optimized away), temp_table was created without a grouping expression and JOIN::exec() will not perform the necessary grouping (by the use of end_send_group() or end_write_group()) if JOIN::group is set to false. */ // the temporary table was explicitly requested DBUG_ASSERT(select_options & OPTION_BUFFER_RESULT); // the temporary table does not have a grouping expression DBUG_ASSERT(!curr_tab->table->group); } calc_group_buffer(this, group_list); count_field_types(select_lex, &tmp_table_param, *curr_all_fields, false); } if (group || (implicit_grouping && !implicit_grouping_with_window_funcs) || tmp_table_param.sum_func_count) { if (make_group_fields(this, this)) DBUG_RETURN(true); DBUG_ASSERT(items3.is_null()); if (items0.is_null()) init_items_ref_array(); items3= ref_ptr_array_slice(4); setup_copy_fields(thd, &tmp_table_param, items3, tmp_fields_list3, tmp_all_fields3, curr_fields_list->elements, *curr_all_fields); curr_fields_list= &tmp_fields_list3; curr_all_fields= &tmp_all_fields3; set_items_ref_array(items3); if (join_tab) { JOIN_TAB *last_tab= join_tab + top_join_tab_count + aggr_tables - 1; // Set grouped fields on the last table last_tab->ref_array= &items3; last_tab->all_fields= &tmp_all_fields3; last_tab->fields= &tmp_fields_list3; } if (make_sum_func_list(*curr_all_fields, *curr_fields_list, true)) DBUG_RETURN(true); if (prepare_sum_aggregators(thd, sum_funcs, !join_tab || !join_tab-> is_using_agg_loose_index_scan())) DBUG_RETURN(true); if (unlikely(setup_sum_funcs(thd, sum_funcs) || thd->is_error())) DBUG_RETURN(true); } if (group_list || order) { DBUG_PRINT("info",("Sorting for send_result_set_metadata")); THD_STAGE_INFO(thd, stage_sorting_result); /* If we have already done the group, add HAVING to sorted table */ if (tmp_having && !is_having_added_as_table_cond && !group_list && !sort_and_group) { if (add_having_as_table_cond(curr_tab)) DBUG_RETURN(true); } if (group) select_limit= HA_POS_ERROR; else if (!need_tmp) { /* We can abort sorting after thd->select_limit rows if there are no filter conditions for any tables after the sorted one. Filter conditions come in several forms: 1. as a condition item attached to the join_tab, or 2. as a keyuse attached to the join_tab (ref access). */ for (uint i= const_tables + 1; i < top_join_tab_count; i++) { JOIN_TAB *const tab= join_tab + i; if (tab->select_cond || // 1 (tab->keyuse && !tab->first_inner)) // 2 { /* We have to sort all rows */ select_limit= HA_POS_ERROR; break; } } } /* Here we add sorting stage for ORDER BY/GROUP BY clause, if the optimiser chose FILESORT to be faster than INDEX SCAN or there is no suitable index present. OPTION_FOUND_ROWS supersedes LIMIT and is taken into account. */ DBUG_PRINT("info",("Sorting for order by/group by")); ORDER *order_arg= group_list ? group_list : order; if (top_join_tab_count + aggr_tables > const_tables && ordered_index_usage != (group_list ? ordered_index_group_by : ordered_index_order_by) && curr_tab->type != JT_CONST && curr_tab->type != JT_EQ_REF) // Don't sort 1 row { // Sort either first non-const table or the last tmp table JOIN_TAB *sort_tab= curr_tab; if (add_sorting_to_table(sort_tab, order_arg)) DBUG_RETURN(true); /* filesort_limit: Return only this many rows from filesort(). We can use select_limit_cnt only if we have no group_by and 1 table. This allows us to use Bounded_queue for queries like: "select SQL_CALC_FOUND_ROWS * from t1 order by b desc limit 1;" m_select_limit == HA_POS_ERROR (we need a full table scan) unit->select_limit_cnt == 1 (we only need one row in the result set) */ sort_tab->filesort->limit= (has_group_by || (join_tab + top_join_tab_count > curr_tab + 1)) ? select_limit : unit->lim.get_select_limit(); if (unit->lim.is_with_ties()) sort_tab->filesort->limit= HA_POS_ERROR; } if (!only_const_tables() && !join_tab[const_tables].filesort && !(select_options & SELECT_DESCRIBE)) { /* If no IO cache exists for the first table then we are using an INDEX SCAN and no filesort. Thus we should not remove the sorted attribute on the INDEX SCAN. */ skip_sort_order= true; } } /* Window functions computation step should be attached to the last join_tab that's doing aggregation. The last join_tab reads the data from the temp. table. It also may do - sorting - duplicate value removal Both of these operations are done after window function computation step. */ if (select_lex->window_funcs.elements) { curr_tab= join_tab + total_join_tab_cnt(); if (!(curr_tab->window_funcs_step= new Window_funcs_computation)) DBUG_RETURN(true); if (curr_tab->window_funcs_step->setup(thd, &select_lex->window_funcs, curr_tab)) DBUG_RETURN(true); /* Count that we're using window functions. */ status_var_increment(thd->status_var.feature_window_functions); } if (select_lex->custom_agg_func_used()) status_var_increment(thd->status_var.feature_custom_aggregate_functions); /* Allocate Cached_items of ORDER BY for FETCH FIRST .. WITH TIES. The order list might have been modified prior to this, but we are only interested in the initial order by columns, after all const elements are removed. */ if (unit->lim.is_with_ties()) { /* When ORDER BY is eliminated, we make use of the GROUP BY list. We've already counted how many elements from ORDER BY must be evaluated as part of WITH TIES so we use that. */ ORDER *order_src = order ? order : group_list; if (alloc_order_fields(this, order_src, with_ties_order_count)) DBUG_RETURN(true); } fields= curr_fields_list; // Reset before execution set_items_ref_array(items0); if (join_tab) join_tab[exec_join_tab_cnt() + aggr_tables - 1].next_select= setup_end_select_func(this); group= has_group_by; DBUG_RETURN(false); } bool JOIN::create_postjoin_aggr_table(JOIN_TAB *tab, List<Item> *table_fields, ORDER *table_group, bool save_sum_fields, bool distinct, bool keep_row_order) { DBUG_ENTER("JOIN::create_postjoin_aggr_table"); THD_STAGE_INFO(thd, stage_creating_tmp_table); /* Pushing LIMIT to the post-join temporary table creation is not applicable when there is ORDER BY or GROUP BY or there is no GROUP BY, but there are aggregate functions, because in all these cases we need all result rows. We also can not push limit if the limit is WITH TIES, as we do not know how many rows we will actually have. This can happen if ORDER BY was a constant and removed (during remove_const), thus we have an "unlimited" WITH TIES. */ ha_rows table_rows_limit= ((order == NULL || skip_sort_order) && !table_group && !select_lex->with_sum_func && !unit->lim.is_with_ties()) ? select_limit : HA_POS_ERROR; if (!(tab->tmp_table_param= new TMP_TABLE_PARAM(tmp_table_param))) DBUG_RETURN(true); if (tmp_table_keep_current_rowid) add_fields_for_current_rowid(tab, table_fields); tab->tmp_table_param->skip_create_table= true; TABLE* table= create_tmp_table(thd, tab->tmp_table_param, *table_fields, table_group, distinct, save_sum_fields, select_options, table_rows_limit, &empty_clex_str, true, keep_row_order); if (!table) DBUG_RETURN(true); tmp_table_param.using_outer_summary_function= tab->tmp_table_param->using_outer_summary_function; tab->join= this; DBUG_ASSERT(tab > tab->join->join_tab || !top_join_tab_count || !tables_list); tab->table= table; if (tab > join_tab) (tab - 1)->next_select= sub_select_postjoin_aggr; /* if group or order on first table, sort first */ if ((group_list && simple_group) || (implicit_grouping && select_lex->have_window_funcs())) { DBUG_PRINT("info",("Sorting for group")); THD_STAGE_INFO(thd, stage_sorting_for_group); if (ordered_index_usage != ordered_index_group_by && !only_const_tables() && (join_tab + const_tables)->type != JT_CONST && // Don't sort 1 row !implicit_grouping && add_sorting_to_table(join_tab + const_tables, group_list)) goto err; if (alloc_group_fields(this, group_list)) goto err; if (make_sum_func_list(all_fields, fields_list, true)) goto err; if (prepare_sum_aggregators(thd, sum_funcs, !(tables_list && join_tab->is_using_agg_loose_index_scan()))) goto err; if (setup_sum_funcs(thd, sum_funcs)) goto err; group_list= NULL; } else { if (prepare_sum_aggregators(thd, sum_funcs, !join_tab->is_using_agg_loose_index_scan())) goto err; if (setup_sum_funcs(thd, sum_funcs)) goto err; if (!group_list && !table->distinct && order && simple_order && tab == join_tab + const_tables) { DBUG_PRINT("info",("Sorting for order")); THD_STAGE_INFO(thd, stage_sorting_for_order); if (ordered_index_usage != ordered_index_order_by && !only_const_tables() && add_sorting_to_table(join_tab + const_tables, order)) goto err; order= NULL; } } if (!(tab->aggr= new (thd->mem_root) AGGR_OP(tab))) goto err; table->reginfo.join_tab= tab; DBUG_RETURN(false); err: if (table != NULL) free_tmp_table(thd, table); tab->table= NULL; DBUG_RETURN(true); } void JOIN::optimize_distinct() { for (JOIN_TAB *last_join_tab= join_tab + top_join_tab_count - 1; ;) { if (select_lex->select_list_tables & last_join_tab->table->map || last_join_tab->use_join_cache) break; last_join_tab->shortcut_for_distinct= true; if (last_join_tab == join_tab) break; --last_join_tab; } /* Optimize "select distinct b from t1 order by key_part_1 limit #" */ if (order && skip_sort_order && !unit->lim.is_with_ties() && (*order->item)->type() == Item::FIELD_ITEM) { /* Should already have been optimized away */ DBUG_ASSERT(ordered_index_usage == ordered_index_order_by); if (ordered_index_usage == ordered_index_order_by) { order= NULL; } } } /** @brief Add Filesort object to the given table to sort if with filesort @param tab the JOIN_TAB object to attach created Filesort object to @param order List of expressions to sort the table by @note This function moves tab->select, if any, to filesort->select @return false on success, true on OOM */ bool JOIN::add_sorting_to_table(JOIN_TAB *tab, ORDER *order) { tab->filesort= new (thd->mem_root) Filesort(order, HA_ROWS_MAX, tab->keep_current_rowid, tab->select); if (!tab->filesort) return true; TABLE *table= tab->table; if ((tab == join_tab + const_tables) && table->pos_in_table_list->is_sjm_scan_table()) { tab->filesort->set_all_read_bits= TRUE; tab->filesort->unpack= unpack_to_base_table_fields; } /* Select was moved to filesort->select to force join_init_read_record to use sorted result instead of reading table through select. */ if (tab->select) { tab->select= NULL; tab->set_select_cond(NULL, __LINE__); } tab->read_first_record= join_init_read_record; return false; } /** Setup expression caches for subqueries that need them @details The function wraps correlated subquery expressions that return one value into objects of the class Item_cache_wrapper setting up an expression cache for each of them. The result values of the subqueries are to be cached together with the corresponding sets of the parameters - outer references of the subqueries. @retval FALSE OK @retval TRUE Error */ bool JOIN::setup_subquery_caches() { DBUG_ENTER("JOIN::setup_subquery_caches"); /* We have to check all this condition together because items created in one of this clauses can be moved to another one by optimizer */ if (select_lex->expr_cache_may_be_used[IN_WHERE] || select_lex->expr_cache_may_be_used[IN_HAVING] || select_lex->expr_cache_may_be_used[IN_ON] || select_lex->expr_cache_may_be_used[NO_MATTER]) { JOIN_TAB *tab; if (conds && !(conds= conds->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { if (tab->select_cond && !(tab->select_cond= tab->select_cond->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); if (tab->cache_select && tab->cache_select->cond) if (!(tab->cache_select->cond= tab->cache_select-> cond->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); } if (having && !(having= having->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); if (tmp_having) { DBUG_ASSERT(having == NULL); if (!(tmp_having= tmp_having->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); } } if (select_lex->expr_cache_may_be_used[SELECT_LIST] || select_lex->expr_cache_may_be_used[IN_GROUP_BY] || select_lex->expr_cache_may_be_used[NO_MATTER]) { List_iterator<Item> li(all_fields); Item *item; while ((item= li++)) { Item *new_item; if (!(new_item= item->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); if (new_item != item) { thd->change_item_tree(li.ref(), new_item); } } for (ORDER *tmp_group= group_list; tmp_group ; tmp_group= tmp_group->next) { if (!(*tmp_group->item= (*tmp_group->item)->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); } } if (select_lex->expr_cache_may_be_used[NO_MATTER]) { for (ORDER *ord= order; ord; ord= ord->next) { if (!(*ord->item= (*ord->item)->transform(thd, &Item::expr_cache_insert_transformer, NULL))) DBUG_RETURN(TRUE); } } DBUG_RETURN(FALSE); } /* Shrink join buffers used for preceding tables to reduce the occupied space SYNOPSIS shrink_join_buffers() jt table up to which the buffers are to be shrunk curr_space the size of the space used by the buffers for tables 1..jt needed_space the size of the space that has to be used by these buffers DESCRIPTION The function makes an attempt to shrink all join buffers used for the tables starting from the first up to jt to reduce the total size of the space occupied by the buffers used for tables 1,...,jt from curr_space to needed_space. The function assumes that the buffer for the table jt has not been allocated yet. RETURN FALSE if all buffer have been successfully shrunk TRUE otherwise */ bool JOIN::shrink_join_buffers(JOIN_TAB *jt, ulonglong curr_space, ulonglong needed_space) { JOIN_TAB *tab; JOIN_CACHE *cache; for (tab= first_linear_tab(this, WITHOUT_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab != jt; tab= next_linear_tab(this, tab, WITHOUT_BUSH_ROOTS)) { cache= tab->cache; if (cache) { size_t buff_size; if (needed_space < cache->get_min_join_buffer_size()) return TRUE; if (cache->shrink_join_buffer_in_ratio(curr_space, needed_space)) { revise_cache_usage(tab); return TRUE; } buff_size= cache->get_join_buffer_size(); curr_space-= buff_size; if (needed_space < buff_size) { /* Safety: fail if we've exhausted available buffer space with reduced join buffers. */ DBUG_ASSERT(0); return TRUE; } needed_space-= buff_size; } } cache= jt->cache; DBUG_ASSERT(cache); if (needed_space < cache->get_min_join_buffer_size()) return TRUE; cache->set_join_buffer_size((size_t)needed_space); return FALSE; } int JOIN::reinit() { DBUG_ENTER("JOIN::reinit"); first_record= false; group_sent= false; cleaned= false; accepted_rows= 0; if (aggr_tables) { JOIN_TAB *curr_tab= join_tab + exec_join_tab_cnt(); JOIN_TAB *end_tab= curr_tab + aggr_tables; for ( ; curr_tab < end_tab; curr_tab++) { TABLE *tmp_table= curr_tab->table; if (!tmp_table->is_created()) continue; tmp_table->file->extra(HA_EXTRA_RESET_STATE); tmp_table->file->ha_delete_all_rows(); } } clear_sj_tmp_tables(this); if (current_ref_ptrs != items0) { set_items_ref_array(items0); } /* need to reset ref access state (see join_read_key) */ if (join_tab) { JOIN_TAB *tab; for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITH_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { tab->ref.key_err= TRUE; } } clear_sum_funcs(); if (no_rows_in_result_called) { /* Reset effect of possible no_rows_in_result() */ List_iterator_fast<Item> it(fields_list); Item *item; no_rows_in_result_called= 0; while ((item= it++)) item->restore_to_before_no_rows_in_result(); } if (!(select_options & SELECT_DESCRIBE)) if (init_ftfuncs(thd, select_lex, MY_TEST(order))) DBUG_RETURN(1); DBUG_RETURN(0); } /** Prepare join result. @details Prepare join result prior to join execution or describing. Instantiate derived tables and get schema tables result if necessary. @return TRUE An error during derived or schema tables instantiation. FALSE Ok */ bool JOIN::prepare_result(List<Item> **columns_list) { DBUG_ENTER("JOIN::prepare_result"); error= 0; /* Create result tables for materialized views. */ if (!zero_result_cause && select_lex->handle_derived(thd->lex, DT_CREATE)) goto err; if (result->prepare2(this)) goto err; if ((select_lex->options & OPTION_SCHEMA_TABLE) && get_schema_tables_result(this, PROCESSED_BY_JOIN_EXEC)) goto err; DBUG_RETURN(FALSE); err: error= 1; DBUG_RETURN(TRUE); } /** @retval 0 ok 1 error */ bool JOIN::save_explain_data(Explain_query *output, bool can_overwrite, bool need_tmp_table, bool need_order, bool distinct) { DBUG_ENTER("JOIN::save_explain_data"); DBUG_PRINT("enter", ("Save explain Select_lex: %u (%p) parent lex: %p stmt_lex: %p present s select_lex->select_number, select_lex, select_lex->parent_lex, thd->lex->stmt_lex, (output->get_select(select_lex->select_number) ? select_lex->select_number : 0), (output->get_select(select_lex->select_number) ? output->get_select(select_lex->select_number) ->select_lex : NULL))); /* If there is SELECT in this statement with the same number it must be the same SELECT */ DBUG_ASSERT(select_lex->select_number == FAKE_SELECT_LEX_ID || !output || !output->get_select(select_lex->select_number) || output->get_select(select_lex->select_number)->select_lex == select_lex); if (select_lex->select_number != FAKE_SELECT_LEX_ID && have_query_plan != JOIN::QEP_NOT_PRESENT_YET && have_query_plan != JOIN::QEP_DELETED && // this happens when there was // no QEP ever, but then //cleanup() is called multiple times output && // for "SET" command in SPs. (can_overwrite? true: !output->get_select(select_lex->select_number))) { const char *message= NULL; if (!table_count || !tables_list || zero_result_cause) { /* It's a degenerate join */ message= zero_result_cause ? zero_result_cause : "No tables used"; } bool rc= save_explain_data_intern(thd->lex->explain, need_tmp_table, need_order, distinct, message); DBUG_RETURN(rc); } /* Can have join_tab==NULL for degenerate cases (e.g. SELECT .. UNION ... SELECT LIMIT 0) */ if (select_lex == select_lex->master_unit()->fake_select_lex && join_tab) { /* This is fake_select_lex. It has no query plan, but we need to set up a tracker for ANALYZE */ uint nr= select_lex->master_unit()->first_select()->select_number; Explain_union *eu= output->get_union(nr); explain= &eu->fake_select_lex_explain; join_tab[0].tracker= eu->get_fake_select_lex_tracker(); for (uint i=0 ; i < exec_join_tab_cnt() + aggr_tables; i++) { if (join_tab[i].filesort) { if (!(join_tab[i].filesort->tracker= new Filesort_tracker(thd->lex->analyze_stmt))) DBUG_RETURN(1); } } } DBUG_RETURN(0); } int JOIN::exec() { int res; DBUG_ASSERT(optimization_state == OPTIMIZATION_DONE); DBUG_EXECUTE_IF("show_explain_probe_join_exec_start", if (dbug_user_var_equals_int(thd, "show_explain_probe_select_id", select_lex->select_number)) dbug_serve_apcs(thd, 1); ); ANALYZE_START_TRACKING(thd, &explain->time_tracker); res= exec_inner(); ANALYZE_STOP_TRACKING(thd, &explain->time_tracker); DBUG_EXECUTE_IF("show_explain_probe_join_exec_end", if (dbug_user_var_equals_int(thd, "show_explain_probe_select_id", select_lex->select_number)) dbug_serve_apcs(thd, 1); ); return res; } int JOIN::exec_inner() { List<Item> *columns_list= &fields_list; DBUG_ENTER("JOIN::exec_inner"); DBUG_ASSERT(optimization_state == JOIN::OPTIMIZATION_DONE); THD_STAGE_INFO(thd, stage_executing); /* Activate enforcement of LIMIT ROWS EXAMINED during query execution if: (1) This JOIN is the outermost query (not a subquery or derived table) This ensures that the limit is enabled when actual execution begins, and not if a subquery is evaluated during optimization of the outer query. (2) This JOIN is not the result of a UNION. In this case do not apply the limit in order to produce the partial query result stored in the UNION temp table. */ if (!select_lex->outer_select() && // (1) select_lex != select_lex->master_unit()->fake_select_lex) // (2) thd->lex->set_limit_rows_examined(); if (procedure) { procedure_fields_list= fields_list; if (procedure->change_columns(thd, procedure_fields_list) || result->prepare(procedure_fields_list, unit)) { thd->limit_found_rows= 0; DBUG_RETURN(0); } columns_list= &procedure_fields_list; } if (result->prepare2(this)) DBUG_RETURN(error); if (!tables_list && (table_count || !select_lex->with_sum_func) && !select_lex->have_window_funcs()) { // Only test of functions if (select_options & SELECT_DESCRIBE) select_describe(this, FALSE, FALSE, FALSE, (zero_result_cause?zero_result_cause:"No tables used")); else { if (result->send_result_set_metadata(*columns_list, Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) { DBUG_RETURN(error); } /* We have to test for 'conds' here as the WHERE may not be constant even if we don't have any tables for prepared statements or if conds uses something like 'rand()'. If the HAVING clause is either impossible or always true, then JOIN::having is set to NULL by optimize_cond. In this case JOIN::exec must check for JOIN::having_value, in the same way it checks for JOIN::cond_value. */ DBUG_ASSERT(error == 0); if (cond_value != Item::COND_FALSE && having_value != Item::COND_FALSE && (!conds || conds->val_bool()) && (!having || having->val_bool())) { if (do_send_rows && (procedure ? (procedure->send_row(procedure_fields_list) || procedure->end_of_records()): result->send_data_with_check(fields_list, unit, 0)> 0)) error= 1; else send_records= ((select_options & OPTION_FOUND_ROWS) ? 1 : thd->get_sent_row_count()); } else send_records= 0; if (likely(!error)) { join_free(); // Unlock all cursors error= (int) result->send_eof(); } } /* Single select (without union) always returns 0 or 1 row */ thd->limit_found_rows= send_records; DBUG_RETURN(error); } /* Evaluate expensive constant conditions that were not evaluated during optimization. Do not evaluate them for EXPLAIN statements as these conditions may be arbitrarily costly, and because the optimize phase might not have produced a complete executable plan for EXPLAINs. */ if (!zero_result_cause && exec_const_cond && !(select_options & SELECT_DESCRIBE) && !exec_const_cond->val_bool()) zero_result_cause= "Impossible WHERE noticed after reading const tables"; /* We've called exec_const_cond->val_bool(). This may have caused an error. */ if (unlikely(thd->is_error())) { error= thd->is_error(); DBUG_RETURN(error); } if (zero_result_cause) { if (select_lex->have_window_funcs() && send_row_on_empty_set()) { /* The query produces just one row but it has window functions. The only way to compute the value of window function(s) is to run the entire window function computation step (there is no shortcut). */ const_tables= table_count; first_select= sub_select_postjoin_aggr; } else { (void) return_zero_rows(this, result, &select_lex->leaf_tables, columns_list, send_row_on_empty_set(), select_options, zero_result_cause, having ? having : tmp_having, &all_fields); DBUG_RETURN(0); } } /* Evaluate all constant expressions with subqueries in the ORDER/GROUP clauses to make sure that all subqueries return a single row. The evaluation itself will trigger an error if that is not the case. */ if (exec_const_order_group_cond.elements && !(select_options & SELECT_DESCRIBE) && !select_lex->pushdown_select) { List_iterator_fast<Item> const_item_it(exec_const_order_group_cond); Item *cur_const_item; StringBuffer<MAX_FIELD_WIDTH> tmp; while ((cur_const_item= const_item_it++)) { tmp.set_buffer_if_not_allocated(&my_charset_bin); cur_const_item->val_str(&tmp); if (unlikely(thd->is_error())) { error= thd->is_error(); DBUG_RETURN(error); } } } if ((this->select_lex->options & OPTION_SCHEMA_TABLE) && get_schema_tables_result(this, PROCESSED_BY_JOIN_EXEC)) { error= thd->is_error(); DBUG_RETURN(error); } if (select_options & SELECT_DESCRIBE) { select_describe(this, need_tmp, order != 0 && !skip_sort_order, select_distinct, !table_count ? "No tables used" : NullS); DBUG_RETURN(0); } else if (select_lex->pushdown_select) { /* Execute the query pushed into a foreign engine */ error= select_lex->pushdown_select->execute(); DBUG_RETURN(error); } else { /* it's a const select, materialize it. */ select_lex->mark_const_derived(zero_result_cause); } /* XXX: When can we have here thd->is_error() not zero? */ if (unlikely(thd->is_error())) { error= thd->is_error(); DBUG_RETURN(error); } THD_STAGE_INFO(thd, stage_sending_data); DBUG_PRINT("info", ("%s", thd->proc_info)); result->send_result_set_metadata( procedure ? procedure_fields_list : *fields, Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF); error= result->view_structure_only() ? false : do_select(this, procedure); /* Accumulate the counts from all join iterations of all join parts. */ thd->ps_report_examined_row_count(); DBUG_PRINT("counts", ("thd->examined_row_count: %lu", (ulong) thd->get_examined_row_count())); DBUG_RETURN(error); } /** Clean up join. @return Return error that hold JOIN. */ int JOIN::destroy() { DBUG_ENTER("JOIN::destroy"); DBUG_PRINT("info", ("select %p (%u) <> JOIN %p", select_lex, select_lex->select_number, this)); select_lex->join= 0; cond_equal= 0; having_equal= 0; cleanup(1); if (join_tab) { for (JOIN_TAB *tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITH_CONST_TABLES); tab; tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS)) { if (tab->aggr) { free_tmp_table(thd, tab->table); delete tab->tmp_table_param; tab->tmp_table_param= NULL; tab->aggr= NULL; } tab->table= NULL; } } /* Cleanup items referencing temporary table columns */ cleanup_item_list(tmp_all_fields1); cleanup_item_list(tmp_all_fields3); destroy_sj_tmp_tables(this); delete_dynamic(&keyuse); if (save_qep) delete(save_qep); if (ext_keyuses_for_splitting) delete(ext_keyuses_for_splitting); delete procedure; DBUG_RETURN(error); } void JOIN::cleanup_item_list(List<Item> &items) const { DBUG_ENTER("JOIN::cleanup_item_list"); if (!items.is_empty()) { List_iterator_fast<Item> it(items); Item *item; while ((item= it++)) item->cleanup(); } DBUG_VOID_RETURN; } /** @brief Look for provision of the select_handler interface by a foreign engine. Must not be called directly, use find_single_select_handler() or find_partial_select_handler() instead. @param thd The thread handler select_lex SELECT_LEX object, must be passed in the cases of: - single select pushdown - partial pushdown (part of a UNION/EXCEPT/INTERSECT) Must be NULL in case of entire unit pushdown select_lex_unit SELECT_LEX_UNIT object, must be passed in the cases of: - entire unit pushdown - partial pushdown (part of a UNION/EXCEPT/INTERSECT) Must be NULL in case of single select pushdown @details The function checks that this is an upper level select and if so looks through its tables searching for one whose handlerton owns a create_select call-back function. If the call of this function returns a select_handler interface object then the server will push the select query into this engine. This function does not check if the select has tables from different engines. Such a check must be done inside each engine's create_select function. Also the engine's create_select function must perform other checks to make sure the engine can execute the query. @retval the found select_handler if the search is successful 0 otherwise */ static select_handler *find_select_handler_inner(THD *thd, SELECT_LEX *select_lex, SELECT_LEX_UNIT *select_lex_unit) { // Pushdown is not supported for non-top-level SELECTs if (select_lex->master_unit()->outer_select()) return 0; TABLE_LIST *tbl= nullptr; // For SQLCOM_INSERT_SELECT the server takes TABLE_LIST // from thd->lex->query_tables and skips its first table // b/c it is the target table for the INSERT..SELECT. if (thd->lex->sql_command != SQLCOM_INSERT_SELECT) { tbl= select_lex->join->tables_list; } else if (thd->lex->query_tables && thd->lex->query_tables->next_global) { tbl= thd->lex->query_tables->next_global; } else return 0; for (;tbl; tbl= tbl->next_global) { if (!tbl->table) continue; handlerton *ht= tbl->table->file->partition_ht(); if (!ht->create_select) continue; select_handler *sh= ht->create_select(thd, select_lex, select_lex_unit); if (sh) return sh; } return 0; } /** Wrapper for find_select_handler_inner() for the case of single select pushdown. See more comments at the description of find_select_handler_inner() */ select_handler *find_single_select_handler(THD *thd, SELECT_LEX *select_lex) { return find_select_handler_inner(thd, select_lex, nullptr); } /** Wrapper for find_select_handler_inner() for the case of partial select pushdown. Partial pushdown means that a unit (i.e. multiple selects combined with UNION/EXCEPT/INTERSECT operators) cannot be pushed down to the storage engine as a whole but some particular selects of this unit can. For example, SELECT a FROM federated.t1 -- can be pushed down to Federated UNION SELECT b FROM local.t2 -- cannot be pushed down, executed locally See more comments at the description of find_select_handler_inner() */ select_handler * find_partial_select_handler(THD *thd, SELECT_LEX *select_lex, SELECT_LEX_UNIT *select_lex_unit) { return find_select_handler_inner(thd, select_lex, select_lex_unit); } /** An entry point to single-unit select (a select without UNION). @param thd thread handler @param rref_pointer_array a reference to ref_pointer_array of the top-level select_lex for this query @param tables list of all tables used in this query. The tables have been pre-opened. @param fields list of items in SELECT list of the top-level select e.g. SELECT a, b, c FROM t1 will have Item_field for a, b and c in this list. @param conds top level item of an expression representing WHERE clause of the top level select @param og_num total number of ORDER BY and GROUP BY clauses arguments @param order linked list of ORDER BY arguments @param group linked list of GROUP BY arguments @param having top level item of HAVING expression @param proc_param list of PROCEDUREs @param select_options select options (BIG_RESULT, etc) @param result an instance of result set handling class. This object is responsible for send result set rows to the client or inserting them into a table. @param select_lex the only SELECT_LEX of this query @param unit top-level UNIT of this query UNIT is an artificial object created by the parser for every SELECT clause. e.g. SELECT * FROM t1 WHERE a1 IN (SELECT * FROM t2) has 2 unions. @retval FALSE success @retval TRUE an error */ bool mysql_select(THD *thd, TABLE_LIST *tables, List<Item> &fields, COND *conds, uint og_num, ORDER *order, ORDER *group, Item *having, ORDER *proc_param, ulonglong select_options, select_result *result, SELECT_LEX_UNIT *unit, SELECT_LEX *select_lex) { int err= 0; bool free_join= 1, exec_error= 0; DBUG_ENTER("mysql_select"); if (!fields.is_empty()) select_lex->context.resolve_in_select_list= true; JOIN *join; if (select_lex->join != 0) { join= select_lex->join; /* is it single SELECT in derived table, called in derived table creation */ if (select_lex->get_linkage() != DERIVED_TABLE_TYPE || (select_options & SELECT_DESCRIBE)) { if (select_lex->get_linkage() != GLOBAL_OPTIONS_TYPE) { /* Original join tabs might be overwritten at first subselect execution. So we need to restore them. */ Item_subselect *subselect= select_lex->master_unit()->item; if (subselect && subselect->is_uncacheable() && join->reinit()) DBUG_RETURN(TRUE); } else { if (!join->prepared && (err= join->prepare(tables, conds, og_num, order, false, group, having, proc_param, select_lex, unit))) { goto err; } } } free_join= 0; join->select_options= select_options; } else { if (thd->lex->describe) select_options|= SELECT_DESCRIBE; /* When in EXPLAIN, delay deleting the joins so that they are still available when we're producing EXPLAIN EXTENDED warning text. */ if (select_options & SELECT_DESCRIBE) free_join= 0; if (!(join= new (thd->mem_root) JOIN(thd, fields, select_options, result))) DBUG_RETURN(TRUE); THD_STAGE_INFO(thd, stage_init); thd->lex->used_tables=0; if (!join->prepared && (err= join->prepare(tables, conds, og_num, order, false, group, having, proc_param, select_lex, unit))) { goto err; } } thd->get_stmt_da()->reset_current_row_for_warning(1); if (thd->lex->prepare_unreferenced_in_with_clauses()) goto err; /* Look for a table owned by an engine with the select_handler interface */ select_lex->pushdown_select= find_single_select_handler(thd, select_lex); if ((err= join->optimize())) { goto err; // 1 } if (thd->lex->describe & DESCRIBE_EXTENDED) { join->conds_history= join->conds; join->having_history= (join->having?join->having:join->tmp_having); } if (unlikely(thd->is_error())) goto err; exec_error= join->exec(); if (thd->lex->describe & DESCRIBE_EXTENDED) { select_lex->where= join->conds_history; select_lex->having= join->having_history; } err: thd->push_final_warnings(); if (select_lex->pushdown_select) { delete select_lex->pushdown_select; select_lex->pushdown_select= NULL; } if (free_join) { THD_STAGE_INFO(thd, stage_end); err|= (int)(select_lex->cleanup()); DBUG_RETURN(exec_error || err || thd->is_error()); } DBUG_RETURN(exec_error || err); } /** Approximate how many records are going to be returned by this table in this select with this key. @param thd Thread handle @param select Select to be examined @param table The table of interest @param keys The keys of interest @param limit Maximum number of rows of interest @param quick_count Pointer to where we want the estimate written @return Status @retval false Success @retval true Error */ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, TABLE *table, const key_map *keys,ha_rows limit, ha_rows *quick_count) { quick_select_return error; DBUG_ENTER("get_quick_record_count"); uchar buff[STACK_BUFF_ALLOC]; if (unlikely(check_stack_overrun(thd, STACK_MIN_SIZE, buff))) DBUG_RETURN(false); // Fatal error flag is set DEBUG_SYNC(thd, "before_get_quick_record_count"); if (select) { select->head=table; table->reginfo.impossible_range=0; /* EQ_FUNC and EQUAL_FUNC already sent unusable key notes (if any) during update_ref_and_keys(). Have only other functions raise notes from can_optimize_scalar_range(). */ error= select->test_quick_select(thd, *(key_map *)keys, (table_map) 0, limit, 0, FALSE, TRUE, /* remove_where_parts*/ FALSE, Item_func::BITMAP_EXCEPT_ANY_EQUALITY); if (error == SQL_SELECT::OK) { if (select->quick) { /* opt_range_condition_rows was updated in test_quick_select to be the smallest number of rows in any range. select->quick->records is the number of rows in range with smallest cost. */ DBUG_ASSERT(select->quick->records >= table->opt_range_condition_rows); *quick_count= select->quick->records; } DBUG_RETURN(false); } if (error == SQL_SELECT::IMPOSSIBLE_RANGE) { table->reginfo.impossible_range=1; *quick_count= 0; DBUG_RETURN(false); } if (unlikely(error == SQL_SELECT::ERROR)) DBUG_RETURN(true); DBUG_PRINT("warning",("Couldn't use record count on const keypart")); } *quick_count= HA_POS_ERROR; DBUG_RETURN(false); /* This shouldn't happen */ } /* This structure is used to collect info on potentially sargable predicates in order to check whether they become sargable after reading const tables. We form a bitmap of indexes that can be used for sargable predicates. Only such indexes are involved in range analysis. */ struct SARGABLE_PARAM { Field *field; /* field against which to check sargability */ Item **arg_value; /* values of potential keys for lookups */ uint num_values; /* number of values in the above array */ }; /* Mark all tables inside a join nest as constant. @detail This is called when there is a local "Impossible WHERE" inside a multi-table LEFT JOIN. */ void mark_join_nest_as_const(JOIN *join, TABLE_LIST *join_nest, table_map *found_const_table_map, uint *const_count) { List_iterator<TABLE_LIST> it(join_nest->nested_join->join_list); TABLE_LIST *tbl; Json_writer_object emb_obj(join->thd); Json_writer_object trace_obj(join->thd, "mark_join_nest_as_const"); Json_writer_array trace_array(join->thd, "members"); while ((tbl= it++)) { if (tbl->nested_join) { mark_join_nest_as_const(join, tbl, found_const_table_map, const_count); continue; } JOIN_TAB *tab= tbl->table->reginfo.join_tab; if (!(join->const_table_map & tab->table->map)) { tab->type= JT_CONST; tab->info= ET_IMPOSSIBLE_ON_CONDITION; tab->table->const_table= 1; join->const_table_map|= tab->table->map; *found_const_table_map|= tab->table->map; set_position(join,(*const_count)++,tab,(KEYUSE*) 0); mark_as_null_row(tab->table); // All fields are NULL trace_array.add_table_name(tab->table); } } } /* @brief Get the condition that can be used to do range analysis/partition pruning/etc @detail Figure out which condition we can use: - For INNER JOIN, we use the WHERE, - "t1 LEFT JOIN t2 ON ..." uses t2's ON expression - "t1 LEFT JOIN (...) ON ..." uses the join nest's ON expression. */ static Item **get_sargable_cond(JOIN *join, TABLE *table) { Item **retval; if (table->pos_in_table_list->on_expr) { /* This is an inner table from a single-table LEFT JOIN, "t1 LEFT JOIN t2 ON cond". Use the condition cond. */ retval= &table->pos_in_table_list->on_expr; } else if (table->pos_in_table_list->embedding && !table->pos_in_table_list->embedding->sj_on_expr) { /* This is the inner side of a multi-table outer join. Use the appropriate ON expression. */ retval= &(table->pos_in_table_list->embedding->on_expr); } else { /* The table is not inner wrt some LEFT JOIN. Use the WHERE clause */ retval= &join->conds; } return retval; } /** Calculate the best possible join and initialize the join structure. @retval 0 ok @retval 1 Fatal error */ static bool make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, DYNAMIC_ARRAY *keyuse_array) { int error= 0; uint i,table_count,const_count,key; uint sort_space; table_map found_const_table_map, all_table_map; key_map const_ref, eq_part; bool has_expensive_keyparts; TABLE **table_vector; JOIN_TAB *stat,*stat_end,*s,**stat_ref, **stat_vector; KEYUSE *keyuse,*start_keyuse; table_map outer_join=0; table_map no_rows_const_tables= 0; SARGABLE_PARAM *sargables= 0; List_iterator<TABLE_LIST> ti(tables_list); TABLE_LIST *tables; THD *thd= join->thd; DBUG_ENTER("make_join_statistics"); table_count=join->table_count; const uint sj_nests= join->select_lex->sj_nests.elements; // Changed by pull-out /* best_extension_by_limited_search need sort space for 2POSITIION objects per remaining table, which gives us 2*(T + T-1 + T-2 + T-3...1 POSITIONS) = 2*(T+1)/2*T = (T*T+T) */ join->sort_space= sort_space= (table_count*table_count + table_count); /* best_positions is ok to allocate with alloc() as we copy things to it with memcpy() */ if (!multi_alloc_root(join->thd->mem_root, &stat, sizeof(JOIN_TAB)*(table_count), &stat_ref, sizeof(JOIN_TAB*)* MAX_TABLES, &stat_vector, sizeof(JOIN_TAB*)* (table_count +1), &table_vector, sizeof(TABLE*)*(table_count*2), &join->positions, sizeof(POSITION)*(table_count + 1), &join->sort_positions, sizeof(POSITION)*(sort_space), &join->best_positions, sizeof(POSITION)*(table_count + 1), NullS)) DBUG_RETURN(1); /* The following should be optimized to only clear critical things */ bzero((void*)stat, sizeof(JOIN_TAB)* table_count); join->top_join_tab_count= table_count; /* Initialize POSITION objects */ for (i=0 ; i <= table_count ; i++) (void) new ((char*) (join->positions + i)) POSITION; for (i=0 ; i < sort_space ; i++) (void) new ((char*) (join->sort_positions + i)) POSITION; join->best_ref= stat_vector; stat_end=stat+table_count; found_const_table_map= all_table_map=0; const_count=0; for (s= stat, i= 0; (tables= ti++); s++, i++) { TABLE_LIST *embedding= tables->embedding; TABLE *table= tables->table; stat_vector[i]=s; table_vector[i]= s->table= table; s->tab_list= tables; table->pos_in_table_list= tables; error= tables->fetch_number_of_rows(); /* Calculate table->use_stat_records */ set_statistics_for_table(join->thd, table); bitmap_clear_all(&table->cond_set); #ifdef WITH_PARTITION_STORAGE_ENGINE const bool all_partitions_pruned_away= table->all_partitions_pruned_away; #else const bool all_partitions_pruned_away= FALSE; #endif DBUG_EXECUTE_IF("bug11747970_raise_error", { join->thd->set_killed(KILL_QUERY_HARD); }); if (unlikely(error)) { table->file->print_error(error, MYF(0)); goto error; } table->opt_range_keys.clear_all(); table->intersect_keys.clear_all(); table->reginfo.join_tab=s; table->reginfo.not_exists_optimize=0; bzero(table->const_key_parts, sizeof(key_part_map)*table->s->total_keys); all_table_map|= table->map; s->preread_init_done= FALSE; s->join=join; s->dependent= tables->dep_tables; if (tables->schema_table) { /* Information schema is slow and we don't know how many rows we will find. Be setting a moderate amount of rows we are more likely to have it materialized if needed. */ table->file->stats.records= table->used_stat_records= 100; } table->opt_range_condition_rows= table->stat_records(); s->on_expr_ref= &tables->on_expr; if (*s->on_expr_ref) { /* s is the only inner table of an outer join */ if (!table->is_filled_at_execution() && ((!table->file->stats.records && (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT)) || all_partitions_pruned_away) && !embedding) { // Empty table s->dependent= 0; // Ignore LEFT JOIN depend. no_rows_const_tables |= table->map; set_position(join,const_count++,s,(KEYUSE*) 0); continue; } outer_join|= table->map; s->embedding_map= 0; for (;embedding; embedding= embedding->embedding) s->embedding_map|= embedding->nested_join->nj_map; continue; } if (embedding) { /* s belongs to a nested join, maybe to several embedded joins */ s->embedding_map= 0; bool inside_an_outer_join= FALSE; do { /* If this is a semi-join nest, skip it, and proceed upwards. Maybe we're in some outer join nest */ if (embedding->sj_on_expr) { embedding= embedding->embedding; continue; } inside_an_outer_join= TRUE; NESTED_JOIN *nested_join= embedding->nested_join; s->embedding_map|=nested_join->nj_map; s->dependent|= embedding->dep_tables; embedding= embedding->embedding; outer_join|= nested_join->used_tables; } while (embedding); if (inside_an_outer_join) continue; } if (!table->is_filled_at_execution() && (table->s->system || (table->file->stats.records <= 1 && (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT)) || all_partitions_pruned_away) && !s->dependent && !table->fulltext_searched && !join->no_const_tables) { set_position(join,const_count++,s,(KEYUSE*) 0); no_rows_const_tables |= table->map; } /* SJ-Materialization handling: */ if (table->pos_in_table_list->jtbm_subselect && table->pos_in_table_list->jtbm_subselect->is_jtbm_const_tab) { set_position(join,const_count++,s,(KEYUSE*) 0); no_rows_const_tables |= table->map; table->file->stats.records= 0; } } stat_vector[i]=0; join->outer_join=outer_join; if (join->propagate_dependencies(stat)) { // Illegal cross-references found table_count= 0; my_message(ER_WRONG_OUTER_JOIN, ER_THD(thd, ER_WRONG_OUTER_JOIN), MYF(0)); goto error; } { for (JOIN_TAB *s= stat ; s < stat_end ; s++) { TABLE_LIST *tl= s->table->pos_in_table_list; if (tl->embedding && tl->embedding->sj_subq_pred) { s->embedded_dependent= tl->embedding->original_subq_pred_used_tables; } } } if (join->conds || outer_join) { if (update_ref_and_keys(thd, keyuse_array, stat, join->table_count, join->conds, ~outer_join, join->select_lex, &sargables)) goto error; /* Keyparts without prefixes may be useful if this JOIN is a subquery, and if the subquery may be executed via the IN-EXISTS strategy. */ bool skip_unprefixed_keyparts= !(join->is_in_subquery() && join->unit->item->get_IN_subquery()->test_strategy(SUBS_IN_TO_EXISTS)); if (keyuse_array->elements && sort_and_filter_keyuse(join, keyuse_array, skip_unprefixed_keyparts)) goto error; DBUG_EXECUTE("opt", print_keyuse_array(keyuse_array);); if (unlikely(thd->trace_started())) print_keyuse_array_for_trace(thd, keyuse_array); } join->const_table_map= no_rows_const_tables; join->const_tables= const_count; eliminate_tables(join); join->const_table_map &= ~no_rows_const_tables; const_count= join->const_tables; found_const_table_map= join->const_table_map; /* Read tables with 0 or 1 rows (system tables) */ for (POSITION *p_pos=join->positions, *p_end=p_pos+const_count; p_pos < p_end ; p_pos++) { s= p_pos->table; if (! (s->table->map & join->eliminated_tables)) { int tmp; s->type=JT_SYSTEM; join->const_table_map|=s->table->map; if ((tmp=join_read_const_table(join->thd, s, p_pos))) { if (tmp > 0) goto error; // Fatal error } else { found_const_table_map|= s->table->map; s->table->pos_in_table_list->optimized_away= TRUE; } } } /* loop until no more const tables are found */ int ref_changed; do { ref_changed = 0; more_const_tables_found: /* We only have to loop from stat_vector + const_count as set_position() will move all const_tables first in stat_vector */ for (JOIN_TAB **pos=stat_vector+const_count ; (s= *pos) ; pos++) { TABLE *table= s->table; if (table->is_filled_at_execution()) continue; /* If equi-join condition by a key is null rejecting and after a substitution of a const table the key value happens to be null then we can state that there are no matches for this equi-join. */ if ((keyuse= s->keyuse) && *s->on_expr_ref && !s->embedding_map && !(table->map & join->eliminated_tables)) { /* When performing an outer join operation if there are no matching rows for the single row of the outer table all the inner tables are to be null complemented and thus considered as constant tables. Here we apply this consideration to the case of outer join operations with a single inner table only because the case with nested tables would require a more thorough analysis. TODO. Apply single row substitution to null complemented inner tables for nested outer join operations. */ while (keyuse->table == table) { if (!keyuse->is_for_hash_join() && !(keyuse->val->used_tables() & ~join->const_table_map) && keyuse->val->is_null() && keyuse->null_rejecting) { s->type= JT_CONST; s->table->const_table= 1; mark_as_null_row(table); found_const_table_map|= table->map; join->const_table_map|= table->map; set_position(join,const_count++,s,(KEYUSE*) 0); goto more_const_tables_found; } keyuse++; } } if (s->dependent) // If dependent on some table { // All dep. must be constants if (s->dependent & ~(found_const_table_map)) continue; if (table->file->stats.records <= 1L && (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT) && !table->pos_in_table_list->embedding && !((outer_join & table->map) && (*s->on_expr_ref)->is_expensive())) { // system table int tmp= 0; s->type= JT_SYSTEM; join->const_table_map|=table->map; set_position(join,const_count++,s,(KEYUSE*) 0); if ((tmp= join_read_const_table(join->thd, s, join->positions+const_count-1))) { if (tmp > 0) goto error; // Fatal error } else found_const_table_map|= table->map; continue; } } /* check if table can be read by key or table only uses const refs */ if ((keyuse=s->keyuse)) { s->type= JT_REF; while (keyuse->table == table) { if (keyuse->is_for_hash_join()) { keyuse++; continue; } start_keyuse=keyuse; key=keyuse->key; s->keys.set_bit(key); // TODO: remove this ? const_ref.clear_all(); eq_part.clear_all(); has_expensive_keyparts= false; do { if (keyuse->val->type() != Item::NULL_ITEM && !keyuse->optimize && keyuse->keypart != FT_KEYPART) { if (!((~found_const_table_map) & keyuse->used_tables)) { const_ref.set_bit(keyuse->keypart); if (keyuse->val->is_expensive()) has_expensive_keyparts= true; } eq_part.set_bit(keyuse->keypart); } keyuse++; } while (keyuse->table == table && keyuse->key == key); TABLE_LIST *embedding= table->pos_in_table_list->embedding; /* TODO (low priority): currently we ignore the const tables that are within a semi-join nest which is within an outer join nest. The effect of this is that we don't do const substitution for such tables. */ KEY *keyinfo= table->key_info + key; uint key_parts= table->actual_n_key_parts(keyinfo); if (eq_part.is_prefix(key_parts) && !table->fulltext_searched && (!embedding || (embedding->sj_on_expr && !embedding->embedding))) { key_map base_part, base_const_ref, base_eq_part; base_part.set_prefix(keyinfo->user_defined_key_parts); base_const_ref= const_ref; base_const_ref.intersect(base_part); base_eq_part= eq_part; base_eq_part.intersect(base_part); /* We can read the const record if we are using a full unique key and if the table is not an unopened to be materialized table/view. */ if ((table->actual_key_flags(keyinfo) & HA_NOSAME) && (!s->table->pos_in_table_list->is_materialized_derived() || s->table->pos_in_table_list->fill_me)) { if (base_const_ref == base_eq_part && !has_expensive_keyparts && !((outer_join & table->map) && (*s->on_expr_ref)->is_expensive())) { // Found everything for ref. int tmp; ref_changed = 1; s->type= JT_CONST; join->const_table_map|=table->map; set_position(join,const_count++,s,start_keyuse); /* create_ref_for_key will set s->table->const_table */ if (create_ref_for_key(join, s, start_keyuse, FALSE, found_const_table_map)) goto error; if ((tmp=join_read_const_table(join->thd, s, join->positions+const_count-1))) { if (tmp > 0) goto error; // Fatal error } else found_const_table_map|= table->map; break; } } else if (base_const_ref == base_eq_part) s->const_keys.set_bit(key); } } } } } while (ref_changed); join->sort_by_table= get_sort_by_table(join->order, join->group_list, join->select_lex->leaf_tables, join->const_table_map); join->limit_shortcut_applicable= join_limit_shortcut_is_applicable(join); /* Update info on indexes that can be used for search lookups as reading const tables may has added new sargable predicates. */ if (const_count && sargables) { for( ; sargables->field ; sargables++) { Field *field= sargables->field; JOIN_TAB *join_tab= field->table->reginfo.join_tab; key_map possible_keys= field->key_start; possible_keys.intersect(field->table->keys_in_use_for_query); bool is_const= 1; for (uint j=0; j < sargables->num_values; j++) is_const&= sargables->arg_value[j]->const_item(); if (is_const) join_tab[0].const_keys.merge(possible_keys); } } join->impossible_where= false; if (join->conds && const_count) { Item* &conds= join->conds; COND_EQUAL *orig_cond_equal = join->cond_equal; conds->update_used_tables(); conds= conds->remove_eq_conds(join->thd, &join->cond_value, true); if (conds && conds->type() == Item::COND_ITEM && ((Item_cond*) conds)->functype() == Item_func::COND_AND_FUNC) join->cond_equal= &((Item_cond_and*) conds)->m_cond_equal; join->select_lex->where= conds; if (join->cond_value == Item::COND_FALSE) { join->impossible_where= true; conds= (Item*) Item_false; } join->cond_equal= NULL; if (conds) { if (conds->type() == Item::COND_ITEM && ((Item_cond*) conds)->functype() == Item_func::COND_AND_FUNC) join->cond_equal= (&((Item_cond_and *) conds)->m_cond_equal); else if (conds->type() == Item::FUNC_ITEM && ((Item_func*) conds)->functype() == Item_func::MULT_EQUAL_FUNC) { if (!join->cond_equal) join->cond_equal= new COND_EQUAL; join->cond_equal->current_level.empty(); join->cond_equal->current_level.push_back((Item_equal*) conds, join->thd->mem_root); } } if (orig_cond_equal != join->cond_equal) { /* If join->cond_equal has changed all references to it from COND_EQUAL objects associated with ON expressions must be updated. */ for (JOIN_TAB **pos=stat_vector+const_count ; (s= *pos) ; pos++) { if (*s->on_expr_ref && s->cond_equal && s->cond_equal->upper_levels == orig_cond_equal) s->cond_equal->upper_levels= join->cond_equal; } } } join->join_tab= stat; join->make_notnull_conds_for_range_scans(); /* Calc how many (possible) matched records in each table */ /* Todo: add a function so that we can add these Json_writer_objects easily. Another way would be to enclose them in a scope {}; */ { Json_writer_object rows_estimation_wrapper(thd); Json_writer_array rows_estimation(thd, "rows_estimation"); for (s=stat ; s < stat_end ; s++) { s->startup_cost= 0; if (s->type == JT_SYSTEM || s->type == JT_CONST) { Json_writer_object table_records(thd); ha_rows records= 1; if (s->type == JT_SYSTEM || s->table->file->stats.records == 0) records= s->table->file->stats.records; /* zero or one matching row */ s->records= s->found_records= records; s->records_init= s->records_out= rows2double(records); s->read_time=1.0; table_records.add_table_name(s). add("rows", s->found_records). add("cost", s->read_time). add("table_type", s->type == JT_CONST ? "const" : "system"); continue; } /* Approximate found rows and time to read them Update found_records, records, read_time and other scan related variables */ s->estimate_scan_time(); if (s->table->is_splittable()) s->add_keyuses_for_splitting(); /* Add to stat->const_keys those indexes for which all group fields or all select distinct fields participate in one index. */ add_group_and_distinct_keys(join, s); /* This will be updated in calculate_cond_selectivity_for_table() */ s->table->set_cond_selectivity(1.0); DBUG_ASSERT(s->table->used_stat_records == 0 || s->table->cond_selectivity <= s->table->opt_range_condition_rows / s->table->used_stat_records); /* Perform range analysis if we could infer something from it. (1) There are indexes for which we have range conditions, (2) Or there are sargable conditions on the table's columns that we could use for selectivity estimation, (3) Or selectivity estimation via sampling is enabled. (4) Don't do range analysis for materialized subqueries. (5) Don't do range analysis for materialized derived tables/views. */ if ((!s->const_keys.is_clear_all() || // (1) !bitmap_is_clear_all(&s->table->cond_set) || // (2) thd->variables.optimizer_use_condition_selectivity >= 5) && // (3) !s->table->is_filled_at_execution() && // (4) !(s->table->pos_in_table_list->derived && // (5) s->table->pos_in_table_list->is_materialized_derived())) // (5) { bool impossible_range= FALSE; ha_rows records= HA_ROWS_MAX; SQL_SELECT *select= 0; Item **sargable_cond= NULL; if (!s->const_keys.is_clear_all()) { sargable_cond= get_sargable_cond(join, s->table); bool is_sargable_cond_of_where= sargable_cond == &join->conds; select= make_select(s->table, found_const_table_map, found_const_table_map, *sargable_cond, (SORT_INFO*) 0, 1, &error); if (!select) goto error; if (get_quick_record_count(join->thd, select, s->table, &s->const_keys, join->row_limit, &records)) { /* There was an error in test_quick_select */ delete select; goto error; } /* Range analyzer might have modified the condition. Put it the new condition to where we got it from. */ *sargable_cond= select->cond; if (is_sargable_cond_of_where && join->conds && join->conds->type() == Item::COND_ITEM && ((Item_cond*) (join->conds))->functype() == Item_func::COND_AND_FUNC) join->cond_equal= &((Item_cond_and*) (join->conds))->m_cond_equal; s->quick=select->quick; select->quick=0; s->needed_reg=select->needed_reg; impossible_range= records == 0 && s->table->reginfo.impossible_range; s->table->init_cost_info_for_usable_range_rowid_filters(join->thd); } if (!impossible_range) { if (!sargable_cond) sargable_cond= get_sargable_cond(join, s->table); if (join->thd->variables.optimizer_use_condition_selectivity > 1) calculate_cond_selectivity_for_table(join->thd, s->table, sargable_cond); if (s->table->reginfo.impossible_range) { impossible_range= TRUE; records= 0; } } if (impossible_range) { /* Impossible WHERE or ON expression In case of ON, we mark that the we match one empty NULL row. In case of WHERE, don't set found_const_table_map to get the caller to abort with a zero row result. */ TABLE_LIST *emb= s->table->pos_in_table_list->embedding; if (emb && !emb->sj_on_expr && !*s->on_expr_ref) { /* Mark all tables in a multi-table join nest as const */ mark_join_nest_as_const(join, emb, &found_const_table_map, &const_count); } else { double records= 1; join->const_table_map|= s->table->map; set_position(join,const_count++,s,(KEYUSE*) 0); s->type= JT_CONST; s->table->const_table= 1; if (*s->on_expr_ref) { /* Generate empty row */ s->info= ET_IMPOSSIBLE_ON_CONDITION; found_const_table_map|= s->table->map; mark_as_null_row(s->table); // All fields are NULL records= 0; } s->records_init= s->records_out= records; s->found_records= s->records= (ha_rows)records; } } if (records != HA_POS_ERROR) { s->found_records=records; s->read_time= s->quick ? s->quick->read_time : 0.0; } if (select) delete select; else { if (unlikely(thd->trace_started())) add_table_scan_values_to_trace(thd, s); } } else { if (unlikely(thd->trace_started())) add_table_scan_values_to_trace(thd, s); } } } if (pull_out_semijoin_tables(join)) DBUG_RETURN(TRUE); join->join_tab=stat; join->top_join_tab_count= table_count; join->map2table=stat_ref; join->table= table_vector; join->const_tables=const_count; join->found_const_table_map=found_const_table_map; if (join->select_lex->opt_hints_qb) join->select_lex->opt_hints_qb->apply_join_order_hints(join); join->update_key_dependencies(); if (unlikely(thd->trace_started())) trace_table_dependencies(thd, join->join_tab, join->table_count); if (sj_nests) join->select_lex->update_available_semijoin_strategies(thd); if (join->const_tables != join->table_count) optimize_keyuse(join, keyuse_array); DBUG_ASSERT(!join->conds || !join->cond_equal || !join->cond_equal->current_level.elements || (join->conds->type() == Item::COND_ITEM && ((Item_cond*) (join->conds))->functype() == Item_func::COND_AND_FUNC && join->cond_equal == &((Item_cond_and *) (join->conds))->m_cond_equal) || (join->conds->type() == Item::FUNC_ITEM && ((Item_func*) (join->conds))->functype() == Item_func::MULT_EQUAL_FUNC && join->cond_equal->current_level.elements == 1 && join->cond_equal->current_level.head() == join->conds)); if (optimize_semijoin_nests(join, all_table_map)) DBUG_RETURN(TRUE); /* purecov: inspected */ { SELECT_LEX_UNIT *unit= join->select_lex->master_unit(); /* Find an optimal join order of the non-constant tables. */ if (join->const_tables != join->table_count) { if (choose_plan(join, all_table_map & ~join->const_table_map, 0)) goto error; #ifdef HAVE_valgrind // JOIN::positions holds the current query plan. We've already // made the plan choice, so we should only use JOIN::best_positions for (uint k=join->const_tables; k < join->table_count; k++) MEM_UNDEFINED(&join->positions[k], sizeof(join->positions[k])); #endif } else { memcpy((uchar*) join->best_positions,(uchar*) join->positions, sizeof(POSITION)*join->const_tables); join->join_record_count= 1.0; /* Const tables are part of optimizer setup and not counted in cost */ join->best_read=0.0; } if (!(join->select_options & SELECT_DESCRIBE) && unit->derived && unit->derived->is_materialized_derived()) { /* Calculate estimated number of rows for materialized derived table/view. */ double records= 1.0; ha_rows rows; for (i= 0; i < join->table_count ; i++) if (double rr= join->best_positions[i].records_read) records= COST_MULT(records, rr); if (join->group_list) records= estimate_post_group_cardinality(join, records); rows= double_to_rows(records); set_if_smaller(rows, unit->lim.get_select_limit()); join->select_lex->increase_derived_records(rows); } } if (join->choose_subquery_plan(all_table_map & ~join->const_table_map)) goto error; DEBUG_SYNC(join->thd, "inside_make_join_statistics"); DBUG_RETURN(0); error: /* Need to clean up join_tab from TABLEs in case of error. They won't get cleaned up by JOIN::cleanup() because JOIN::join_tab may not be assigned yet by this function (which is building join_tab). Dangling TABLE::reginfo.join_tab may cause part_of_refkey to choke. */ { TABLE_LIST *tmp_table; List_iterator<TABLE_LIST> ti2(tables_list); while ((tmp_table= ti2++)) tmp_table->table->reginfo.join_tab= NULL; } DBUG_RETURN (1); } /* Propagate dependencies between tables. @returns false if success, true if error Build transitive closure for relation 'to be dependent on'. This will speed up the plan search for many cases with outer joins, as well as allow us to catch illegal cross references/ Warshall's algorithm is used to build the transitive closure. As we use bitmaps to represent the relation the complexity of the algorithm is O((number of tables)^2). The classic form of the Warshall's algorithm would look like: for (i= 0; i < table_count; i++) { for (j= 0; j < table_count; j++) { for (k= 0; k < table_count; k++) { if (bitmap_is_set(stat[j].dependent, i) && bitmap_is_set(stat[i].dependent, k)) bitmap_set_bit(stat[j].dependent, k); } } } */ bool JOIN::propagate_dependencies(JOIN_TAB *stat) { for (JOIN_TAB *s= stat; s < stat + table_count; s++) { TABLE *table= s->table; if (outer_join & s->table->map) s->table->maybe_null= 1; if (!table->reginfo.join_tab->dependent) continue; // Add my dependencies to other tables depending on me for (JOIN_TAB *t= stat; t < stat + table_count; t++) { if (t->dependent & table->map) t->dependent |= table->reginfo.join_tab->dependent; } } // Catch illegal cross references for (JOIN_TAB *s= stat; s < stat + table_count; s++) { if (s->dependent & s->table->map) return true; } return false; } void JOIN::update_key_dependencies() { for (JOIN_TAB *tab= join_tab; tab < join_tab + table_count; tab++) tab->key_dependent |= tab->dependent; } /* Export dependencies of the JOIN tables to a newly allocated array of bitmaps (table_map's). This array may be used to restore the original dependencies (see restore_table_dependencies()) */ table_map *JOIN::export_table_dependencies() const { table_map *orig_dep_array= (table_map *)thd->alloc(sizeof(table_map) * table_count); if (orig_dep_array == nullptr) return nullptr; for (uint i= 0; i < table_count; i++) orig_dep_array[i]= join_tab[i].dependent; return orig_dep_array; } /* Restore dependencies of the JOIN tables from a previously exported array of bitmaps (table_map's) (see export_table_dependencies()). This function overwrites the existing dependencies with those from the array. */ void JOIN::restore_table_dependencies(table_map *orig_dep_array) { for (uint i = 0; i < table_count; i++) join_tab[i].dependent= orig_dep_array[i]; } /***************************************************************************** Check with keys are used and with tables references with tables Updates in stat: keys Bitmap of all used keys const_keys Bitmap of all keys with may be used with quick_select keyuse Pointer to possible keys *****************************************************************************/ /** Merge new key definitions to old ones, remove those not used in both. This is called for OR between different levels. That is, the function operates on an array of KEY_FIELD elements which has two parts: $LEFT_PART $RIGHT_PART +-----------------------+-----------------------+ start new_fields end $LEFT_PART and $RIGHT_PART are arrays that have KEY_FIELD elements for two parts of the OR condition. Our task is to produce an array of KEY_FIELD elements that would correspond to "$LEFT_PART OR $RIGHT_PART". The rules for combining elements are as follows: (keyfieldA1 AND keyfieldA2 AND ...) OR (keyfieldB1 AND keyfieldB2 AND ...)= = AND_ij (keyfieldA_i OR keyfieldB_j) We discard all (keyfieldA_i OR keyfieldB_j) that refer to different fields. For those referring to the same field, the logic is as follows: t.keycol=expr1 OR t.keycol=expr2 -> (since expr1 and expr2 are different we can't produce a single equality, so produce nothing) t.keycol=expr1 OR t.keycol=expr1 -> t.keycol=expr1 t.keycol=expr1 OR t.keycol IS NULL -> t.keycol=expr1, and also set KEY_OPTIMIZE_REF_OR_NULL flag The last one is for ref_or_null access. We have handling for this special because it's needed for evaluating IN subqueries that are internally transformed into @code EXISTS(SELECT * FROM t1 WHERE t1.key=outer_ref_field or t1.key IS NULL) @endcode See add_key_fields() for discussion of what is and_level. KEY_FIELD::null_rejecting is processed as follows: @n result has null_rejecting=true if it is set for both ORed references. for example: - (t2.key = t1.field OR t2.key = t1.field) -> null_rejecting=true - (t2.key = t1.field OR t2.key <=> t1.field) -> null_rejecting=false @todo The result of this is that we're missing some 'ref' accesses. OptimizerTeam: Fix this */ static KEY_FIELD * merge_key_fields(KEY_FIELD *start,KEY_FIELD *new_fields,KEY_FIELD *end, uint and_level) { if (start == new_fields) return start; // Impossible or if (new_fields == end) return start; // No new fields, skip all KEY_FIELD *first_free=new_fields; /* Mark all found fields in old array */ for (; new_fields != end ; new_fields++) { for (KEY_FIELD *old=start ; old != first_free ; old++) { if (old->field == new_fields->field) { /* NOTE: below const_item() call really works as "!used_tables()", i.e. it can return FALSE where it is feasible to make it return TRUE. The cause is as follows: Some of the tables are already known to be const tables (the detection code is in make_join_statistics(), above the update_ref_and_keys() call), but we didn't propagate information about this: TABLE::const_table is not set to TRUE, and Item::update_used_tables() hasn't been called for each item. The result of this is that we're missing some 'ref' accesses. TODO: OptimizerTeam: Fix this */ if (!new_fields->val->const_item()) { /* If the value matches, we can use the key reference. If not, we keep it until we have examined all new values */ if (old->val->eq(new_fields->val, old->field->binary())) { old->level= and_level; old->optimize= ((old->optimize & new_fields->optimize & KEY_OPTIMIZE_EXISTS) | ((old->optimize | new_fields->optimize) & KEY_OPTIMIZE_REF_OR_NULL)); old->null_rejecting= (old->null_rejecting && new_fields->null_rejecting); } } else if (old->eq_func && new_fields->eq_func && old->val->eq_by_collation(new_fields->val, old->field->binary(), old->field->charset())) { old->level= and_level; old->optimize= ((old->optimize & new_fields->optimize & KEY_OPTIMIZE_EXISTS) | ((old->optimize | new_fields->optimize) & KEY_OPTIMIZE_REF_OR_NULL)); old->null_rejecting= (old->null_rejecting && new_fields->null_rejecting); } else if (old->eq_func && new_fields->eq_func && ((old->val->can_eval_in_optimize() && old->val->is_null()) || (!new_fields->val->is_expensive() && new_fields->val->is_null()))) { /* field = expression OR field IS NULL */ old->level= and_level; if (old->field->maybe_null()) { old->optimize= KEY_OPTIMIZE_REF_OR_NULL; /* The referred expression can be NULL: */ old->null_rejecting= 0; } /* Remember the NOT NULL value unless the value does not depend on other tables. */ if (!old->val->used_tables() && !old->val->is_expensive() && old->val->is_null()) old->val= new_fields->val; } else { /* We are comparing two different const. In this case we can't use a key-lookup on this so it's better to remove the value and let the range optimzier handle it */ if (old == --first_free) // If last item break; *old= *first_free; // Remove old value old--; // Retry this value } } } } /* Remove all not used items */ for (KEY_FIELD *old=start ; old != first_free ;) { if (old->level != and_level) { // Not used in all levels if (old == --first_free) break; *old= *first_free; // Remove old value continue; } old++; } return first_free; } /* Given a field, return its index in semi-join's select list, or UINT_MAX DESCRIPTION Given a field, we find its table; then see if the table is within a semi-join nest and if the field was in select list of the subselect. If it was, we return field's index in the select list. The value is used by LooseScan strategy. */ static uint get_semi_join_select_list_index(Field *field) { uint res= UINT_MAX; TABLE_LIST *emb_sj_nest; if ((emb_sj_nest= field->table->pos_in_table_list->embedding) && emb_sj_nest->sj_on_expr) { Item_in_subselect *subq_pred= emb_sj_nest->sj_subq_pred; st_select_lex *subq_lex= subq_pred->unit->first_select(); uint ncols= subq_pred->left_exp()->cols(); if (ncols == 1) { Item *sel_item= subq_lex->ref_pointer_array[0]; if (sel_item->type() == Item::FIELD_ITEM && ((Item_field*)sel_item)->field->eq(field)) { res= 0; } } else { for (uint i= 0; i < ncols; i++) { Item *sel_item= subq_lex->ref_pointer_array[i]; if (sel_item->type() == Item::FIELD_ITEM && ((Item_field*)sel_item)->field->eq(field)) { res= i; break; } } } } return res; } /** Add a possible key to array of possible keys if it's usable as a key @param key_fields Pointer to add key, if usable @param and_level And level, to be stored in KEY_FIELD @param cond Condition predicate @param field Field used in comparision @param eq_func True if we used =, <=> or IS NULL @param value Value used for comparison with field @param num_values Number of values[] that we are comparing against @param usable_tables Tables which can be used for key optimization @param sargables IN/OUT Array of found sargable candidates @param row_col_no if = n that > 0 then field is compared only against the n-th component of row values @note If we are doing a NOT NULL comparison on a NOT NULL field in a outer join table, we store this to be able to do not exists optimization later. @returns *key_fields is incremented if we stored a key in the array */ static void add_key_field(JOIN *join, KEY_FIELD **key_fields,uint and_level, Item_bool_func *cond, Field *field, bool eq_func, Item **value, uint num_values, table_map usable_tables, SARGABLE_PARAM **sargables, uint row_col_no= 0) { uint optimize= 0; if (eq_func && ((join->is_allowed_hash_join_access(field->table) && field->hash_join_is_possible() && !(field->table->pos_in_table_list->is_materialized_derived() && field->table->is_created())) || (field->table->pos_in_table_list->is_materialized_derived() && !field->table->is_created() && !(field->flags & BLOB_FLAG)))) { optimize= KEY_OPTIMIZE_EQ; } else if (!(field->flags & PART_KEY_FLAG)) { // Don't remove column IS NULL on a LEFT JOIN table if (eq_func && (*value)->type() == Item::NULL_ITEM && field->table->maybe_null && !field->null_ptr) { optimize= KEY_OPTIMIZE_EXISTS; DBUG_ASSERT(num_values == 1); } } if (optimize != KEY_OPTIMIZE_EXISTS) { table_map used_tables=0; bool optimizable=0; for (uint i=0; i<num_values; i++) { Item *curr_val; if (row_col_no && value[i]->real_item()->type() == Item::ROW_ITEM) { Item_row *value_tuple= (Item_row *) (value[i]->real_item()); curr_val= value_tuple->element_index(row_col_no - 1); } else curr_val= value[i]; table_map value_used_tables= curr_val->used_tables(); used_tables|= value_used_tables; if (!(value_used_tables & (field->table->map | RAND_TABLE_BIT))) optimizable=1; } if (!optimizable) return; if (!(usable_tables & field->table->map)) { if (!eq_func || (*value)->type() != Item::NULL_ITEM || !field->table->maybe_null || field->null_ptr) return; // Can't use left join optimize optimize= KEY_OPTIMIZE_EXISTS; } else { JOIN_TAB *stat=field->table->reginfo.join_tab; key_map possible_keys=field->get_possible_keys(); possible_keys.intersect(field->table->keys_in_use_for_query); stat[0].keys.merge(possible_keys); // Add possible keys /* Save the following cases: Field op constant Field LIKE constant where constant doesn't start with a wildcard Field = field2 where field2 is in a different table Field op formula Field IS NULL Field IS NOT NULL Field BETWEEN ... Field IN ... */ if (field->flags & PART_KEY_FLAG) { stat[0].key_dependent|= used_tables; if (field->key_start.bits_set()) stat[0].key_start_dependent= 1; } bool is_const=1; for (uint i=0; i<num_values; i++) { Item *curr_val; if (row_col_no && value[i]->real_item()->type() == Item::ROW_ITEM) { Item_row *value_tuple= (Item_row *) (value[i]->real_item()); curr_val= value_tuple->element_index(row_col_no - 1); } else curr_val= value[i]; if (!(is_const&= curr_val->const_item())) break; } if (is_const) { stat[0].const_keys.merge(possible_keys); bitmap_set_bit(&field->table->cond_set, field->field_index); } else if (!eq_func) { /* Save info to be able check whether this predicate can be considered as sargable for range analysis after reading const tables. We do not save info about equalities as update_const_equal_items will take care of updating info on keys from sargable equalities. */ (*sargables)--; (*sargables)->field= field; (*sargables)->arg_value= value; (*sargables)->num_values= num_values; } if (!eq_func) // eq_func is NEVER true when num_values > 1 return; } } /* For the moment eq_func is always true. This slot is reserved for future extensions where we want to remembers other things than just eq comparisons */ DBUG_ASSERT(eq_func); /* Store possible eq field */ (*key_fields)->field= field; (*key_fields)->eq_func= eq_func; (*key_fields)->val= *value; (*key_fields)->cond= cond; (*key_fields)->level= and_level; (*key_fields)->optimize= optimize; /* If the condition we are analyzing is NULL-rejecting and at least one side of the equalities is NULLable, mark the KEY_FIELD object as null-rejecting. This property is used by: - add_not_null_conds() to add "column IS NOT NULL" conditions - best_access_path() to produce better estimates for NULL-able unique keys. */ { if ((cond->functype() == Item_func::EQ_FUNC || cond->functype() == Item_func::MULT_EQUAL_FUNC) && ((*value)->maybe_null() || field->real_maybe_null())) (*key_fields)->null_rejecting= true; else (*key_fields)->null_rejecting= false; } (*key_fields)->cond_guard= NULL; (*key_fields)->sj_pred_no= get_semi_join_select_list_index(field); (*key_fields)++; } /** Add possible keys to array of possible keys originated from a simple predicate. @param key_fields Pointer to add key, if usable @param and_level And level, to be stored in KEY_FIELD @param cond Condition predicate @param field_item Field item used for comparison @param eq_func True if we used =, <=> or IS NULL @param value Value used for comparison with field_item @param num_values Number of values[] that we are comparing against @param usable_tables Tables which can be used for key optimization @param sargables IN/OUT Array of found sargable candidates @param row_col_no if = n that > 0 then field is compared only against the n-th component of row values @note If field items f1 and f2 belong to the same multiple equality and a key is added for f1, the same key is added for f2. @returns *key_fields is incremented if we stored a key in the array */ static void add_key_equal_fields(JOIN *join, KEY_FIELD **key_fields, uint and_level, Item_bool_func *cond, Item *field_item, bool eq_func, Item **val, uint num_values, table_map usable_tables, SARGABLE_PARAM **sargables, uint row_col_no= 0) { Field *field= ((Item_field *) (field_item->real_item()))->field; add_key_field(join, key_fields, and_level, cond, field, eq_func, val, num_values, usable_tables, sargables, row_col_no); Item_equal *item_equal= field_item->get_item_equal(); if (item_equal) { /* Add to the set of possible key values every substitution of the field for an equal field included into item_equal */ Item_equal_fields_iterator it(*item_equal); while (it++) { Field *equal_field= it.get_curr_field(); if (!field->eq(equal_field)) { add_key_field(join, key_fields, and_level, cond, equal_field, eq_func, val, num_values, usable_tables, sargables, row_col_no); } } } } /** Check if an expression is a non-outer field. Checks if an expression is a field and belongs to the current select. @param field Item expression to check @return boolean @retval TRUE the expression is a local field @retval FALSE it's something else */ static bool is_local_field (Item *field) { return field->real_item()->type() == Item::FIELD_ITEM && !(field->used_tables() & OUTER_REF_TABLE_BIT) && !((Item_field *)field->real_item())->get_depended_from(); } /* In this and other functions, and_level is a number that is ever-growing and is different for the contents of every AND or OR clause. For example, when processing clause (a AND b AND c) OR (x AND y) we'll have * KEY_FIELD elements for (a AND b AND c) are assigned and_level=1 * KEY_FIELD elements for (x AND y) are assigned and_level=2 * OR operation is performed, and whatever elements are left after it are assigned and_level=3. The primary reason for having and_level attribute is the OR operation which uses and_level to mark KEY_FIELDs that should get into the result of the OR operation */ void Item_cond_and::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { List_iterator_fast<Item> li(*argument_list()); KEY_FIELD *org_key_fields= *key_fields; Item *item; while ((item=li++)) item->add_key_fields(join, key_fields, and_level, usable_tables, sargables); for (; org_key_fields != *key_fields ; org_key_fields++) org_key_fields->level= *and_level; } void Item_cond::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { List_iterator_fast<Item> li(*argument_list()); KEY_FIELD *org_key_fields= *key_fields; (*and_level)++; (li++)->add_key_fields(join, key_fields, and_level, usable_tables, sargables); Item *item; while ((item=li++)) { KEY_FIELD *start_key_fields= *key_fields; (*and_level)++; item->add_key_fields(join, key_fields, and_level, usable_tables, sargables); *key_fields= merge_key_fields(org_key_fields,start_key_fields, *key_fields, ++(*and_level)); } } void Item_func_trig_cond::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { /* Subquery optimization: Conditions that are pushed down into subqueries are wrapped into Item_func_trig_cond. We process the wrapped condition but need to set cond_guard for KEYUSE elements generated from it. */ if (!join->group_list && !join->order && join->unit->item && join->unit->item->substype() == Item_subselect::IN_SUBS && !join->unit->is_unit_op()) { KEY_FIELD *save= *key_fields; args[0]->add_key_fields(join, key_fields, and_level, usable_tables, sargables); // Indicate that this ref access candidate is for subquery lookup: for (; save != *key_fields; save++) save->cond_guard= get_trig_var(); } } void Item_func_between::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { /* Build list of possible keys for 'a BETWEEN low AND high'. It is handled similar to the equivalent condition 'a >= low AND a <= high': */ Item_field *field_item; bool equal_func= false; uint num_values= 2; bool binary_cmp= (args[0]->real_item()->type() == Item::FIELD_ITEM) ? ((Item_field*) args[0]->real_item())->field->binary() : true; /* Additional optimization: If 'low = high': Handle as if the condition was "t.key = low". */ if (!negated && args[1]->eq(args[2], binary_cmp)) { equal_func= true; num_values= 1; } /* Append keys for 'field <cmp> value[]' if the condition is of the form:: '<field> BETWEEN value[1] AND value[2]' */ if (is_local_field(args[0])) { field_item= (Item_field *) (args[0]->real_item()); add_key_equal_fields(join, key_fields, *and_level, this, field_item, equal_func, &args[1], num_values, usable_tables, sargables); } /* Append keys for 'value[0] <cmp> field' if the condition is of the form: 'value[0] BETWEEN field1 AND field2' */ for (uint i= 1; i <= num_values; i++) { if (is_local_field(args[i])) { field_item= (Item_field *) (args[i]->real_item()); add_key_equal_fields(join, key_fields, *and_level, this, field_item, equal_func, args, 1, usable_tables, sargables); } } } void Item_func_in::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { if (is_local_field(args[0]) && !(used_tables() & OUTER_REF_TABLE_BIT)) { DBUG_ASSERT(arg_count != 2); add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) (args[0]->real_item()), false, args + 1, arg_count - 1, usable_tables, sargables); } else if (key_item()->type() == Item::ROW_ITEM && !(used_tables() & OUTER_REF_TABLE_BIT)) { Item_row *key_row= (Item_row *) key_item(); Item **key_col= key_row->addr(0); uint row_cols= key_row->cols(); for (uint i= 0; i < row_cols; i++, key_col++) { if (is_local_field(*key_col)) { Item_field *field_item= (Item_field *)((*key_col)->real_item()); add_key_equal_fields(join, key_fields, *and_level, this, field_item, false, args + 1, arg_count - 1, usable_tables, sargables, i + 1); } } } } void Item_func_ne::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { if (!(used_tables() & OUTER_REF_TABLE_BIT)) { /* QQ: perhaps test for !is_local_field(args[1]) is not really needed here. Other comparison functions, e.g. Item_func_le, Item_func_gt, etc, do not have this test. See Item_bool_func2::add_key_fieldoptimize_op(). Check with the optimizer team. */ if (is_local_field(args[0]) && !is_local_field(args[1])) add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) (args[0]->real_item()), false, &args[1], 1, usable_tables, sargables); /* QQ: perhaps test for !is_local_field(args[0]) is not really needed here. */ if (is_local_field(args[1]) && !is_local_field(args[0])) add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) (args[1]->real_item()), false, &args[0], 1, usable_tables, sargables); } } void Item_func_like::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { if (is_local_field(args[0]) && with_sargable_pattern()) { /* SELECT * FROM t1 WHERE field LIKE const_pattern const_pattern starts with a non-wildcard character */ add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) args[0]->real_item(), false, args + 1, 1, usable_tables, sargables); } } void Item_bool_func2::add_key_fields_optimize_op(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables, bool equal_func) { /* If item is of type 'field op field/constant' add it to key_fields */ if (is_local_field(args[0])) { add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) args[0]->real_item(), equal_func, args + 1, 1, usable_tables, sargables); } else { Item_field *field= NULL; int value_idx= -1; /* Handle SUBSTR(key,1,N)='const', 'const'=SUBSTR(key,1,N), etc */ if (with_sargable_substr(&field, &value_idx)) { add_key_equal_fields(join, key_fields, *and_level, this, field, false, args + value_idx, 1, usable_tables, sargables); } } if (is_local_field(args[1])) { add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) args[1]->real_item(), equal_func, args, 1, usable_tables, sargables); } } void Item_func_truth::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { if (is_local_field(args[0])) { Item *tmp= args[0]->type_handler()->create_boolean_false_item(join->thd); if (unlikely(!tmp)) return; add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) args[0]->real_item(), false/*equal_func*/, &tmp, 1, usable_tables, sargables); } } void Item_func_null_predicate::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { /* column_name IS [NOT] NULL */ if (is_local_field(args[0]) && !(used_tables() & OUTER_REF_TABLE_BIT)) { Item *tmp= new (join->thd->mem_root) Item_null(join->thd); if (unlikely(!tmp)) // Should never be true return; add_key_equal_fields(join, key_fields, *and_level, this, (Item_field*) args[0]->real_item(), functype() == Item_func::ISNULL_FUNC, &tmp, 1, usable_tables, sargables); } } void Item_equal::add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) { Item *const_item2= get_const(); Item_equal_fields_iterator it(*this); if (const_item2) { /* For each field field1 from item_equal consider the equality field1=const_item as a condition allowing an index access of the table with field1 by the keys value of field1. */ while (it++) { Field *equal_field= it.get_curr_field(); add_key_field(join, key_fields, *and_level, this, equal_field, TRUE, &const_item2, 1, usable_tables, sargables); } } else { /* Consider all pairs of different fields included into item_equal. For each of them (field1, field1) consider the equality field1=field2 as a condition allowing an index access of the table with field1 by the keys value of field2. */ Item_equal_fields_iterator fi(*this); while (fi++) { Field *field= fi.get_curr_field(); Item *item; while ((item= it++)) { Field *equal_field= it.get_curr_field(); if (!field->eq(equal_field)) { add_key_field(join, key_fields, *and_level, this, field, TRUE, &item, 1, usable_tables, sargables); } } it.rewind(); } } } static inline uint max_part_bit(key_part_map bits) { if (bits == 0) return 0; /* find first zero bit by reverting all bits and find first bit */ return my_find_first_bit(~(ulonglong) bits); } /** Add a new keuse to the specified array of KEYUSE objects @param[in,out] keyuse_array array of keyuses to be extended @param[in] key_field info on the key use occurrence @param[in] key key number for the keyse to be added @param[in] part key part for the keyuse to be added @note The function builds a new KEYUSE object for a key use utilizing the info on the left and right parts of the given key use extracted from the structure key_field, the key number and key part for this key use. The built object is added to the dynamic array keyuse_array. @retval 0 the built object is successfully added @retval 1 otherwise */ static bool add_keyuse(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field, uint key, uint part) { KEYUSE keyuse; Field *field= key_field->field; keyuse.table= field->table; keyuse.val= key_field->val; keyuse.key= key; if (!is_hash_join_key_no(key)) { keyuse.keypart=part; keyuse.keypart_map= (key_part_map) 1 << part; } else { keyuse.keypart= field->field_index; keyuse.keypart_map= (key_part_map) 0; } keyuse.used_tables= key_field->val->used_tables(); keyuse.optimize= key_field->optimize & KEY_OPTIMIZE_REF_OR_NULL; keyuse.ref_table_rows= 0; keyuse.null_rejecting= key_field->null_rejecting; keyuse.cond_guard= key_field->cond_guard; keyuse.sj_pred_no= key_field->sj_pred_no; keyuse.validity_ref= 0; return (insert_dynamic(keyuse_array,(uchar*) &keyuse)); } /* Add all keys with uses 'field' for some keypart If field->and_level != and_level then only mark key_part as const_part RETURN 0 - OK 1 - Out of memory. */ static LEX_CSTRING equal_str= { STRING_WITH_LEN("=") }; static bool add_key_part(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field) { Field *field=key_field->field; TABLE *form= field->table; THD *thd= form->in_use; if (key_field->eq_func && !(key_field->optimize & KEY_OPTIMIZE_EXISTS)) { for (uint key=0 ; key < form->s->keys ; key++) { if (!(form->keys_in_use_for_query.is_set(key))) continue; if (form->key_info[key].algorithm == HA_KEY_ALG_FULLTEXT || form->key_info[key].algorithm == HA_KEY_ALG_RTREE || form->key_info[key].algorithm == HA_KEY_ALG_VECTOR || form->key_info[key].flags & HA_UNIQUE_HASH) continue; KEY *keyinfo= form->key_info+key; uint key_parts= form->actual_n_key_parts(keyinfo); for (uint part=0 ; part < key_parts ; part++) { if (field->eq(form->key_info[key].key_part[part].field)) { Data_type_compatibility compat= field->can_optimize_keypart_ref(key_field->cond, key_field->val); if (compat == Data_type_compatibility::OK) { if (add_keyuse(keyuse_array, key_field, key, part)) return TRUE; } else if (thd->give_notes_for_unusable_keys()) { field->raise_note_cannot_use_key_part(thd, key, part, equal_str, key_field->cond->compare_collation(), key_field->val, compat); } } } } /* Compressed field cannot be part of a key. For optimizer temporary table compressed fields are replaced by uncompressed, see is_optimizer_tmp_table() and Field_*_compressed::make_new_field(). */ if (!field->compression_method() && field->hash_join_is_possible() && (key_field->optimize & KEY_OPTIMIZE_EQ) && key_field->val->used_tables()) { if (field->can_optimize_hash_join(key_field->cond, key_field->val) != Data_type_compatibility::OK) return false; /* MDEV-24931: For materialized derived tables, don't add hash-join KEYUSE entries beyond max_key_parts(). Excess entries would cause generate_derived_keys_for_table() to build a key with more parts than key_part_map (64 bits) or Bitmap<64> can represent. */ if (form->pos_in_table_list && form->pos_in_table_list->is_materialized_derived()) { uint existing= 0; for (uint k= 0; k < keyuse_array->elements; k++) { KEYUSE *ku= dynamic_element(keyuse_array, k, KEYUSE*); if (ku->table == form && is_hash_join_key_no(ku->key)) existing++; } if (existing >= form->file->max_key_parts()) return FALSE; } if (form->is_splittable()) form->add_splitting_info_for_key_field(key_field); /* If a key use is extracted from an equi-join predicate then it is added not only as a key use for every index whose component can be evaluated utilizing this key use, but also as a key use for hash join. Such key uses are marked with a special key number. */ if (add_keyuse(keyuse_array, key_field, get_hash_join_key_no(), 0)) return TRUE; } } return FALSE; } static bool add_ft_keys(DYNAMIC_ARRAY *keyuse_array, JOIN_TAB *stat,COND *cond,table_map usable_tables) { Item_func_match *cond_func=NULL; if (!cond) return FALSE; if (cond->type() == Item::FUNC_ITEM) { Item_func *func=(Item_func *)cond; Item_func::Functype functype= func->functype(); if (functype == Item_func::FT_FUNC) cond_func=(Item_func_match *)cond; else if (func->argument_count() == 2) { Item *arg0=(Item *)(func->arguments()[0]), *arg1=(Item *)(func->arguments()[1]); if (arg1->const_item() && arg1->cols() == 1 && arg0->type() == Item::FUNC_ITEM && ((Item_func *) arg0)->functype() == Item_func::FT_FUNC && ((functype == Item_func::GE_FUNC && arg1->val_real() > 0) || (functype == Item_func::GT_FUNC && arg1->val_real() >=0))) cond_func= (Item_func_match *) arg0; else if (arg0->const_item() && arg0->cols() == 1 && arg1->type() == Item::FUNC_ITEM && ((Item_func *) arg1)->functype() == Item_func::FT_FUNC && ((functype == Item_func::LE_FUNC && arg0->val_real() > 0) || (functype == Item_func::LT_FUNC && arg0->val_real() >=0))) cond_func= (Item_func_match *) arg1; } } else if (cond->type() == Item::COND_ITEM) { List_iterator_fast<Item> li(*((Item_cond*) cond)->argument_list()); if (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC) { Item *item; while ((item=li++)) { if (add_ft_keys(keyuse_array,stat,item,usable_tables)) return TRUE; } } } if (!cond_func || cond_func->key == NO_SUCH_KEY || !(usable_tables & cond_func->table->map)) return FALSE; KEYUSE keyuse; keyuse.table= cond_func->table; keyuse.val = cond_func; keyuse.key = cond_func->key; keyuse.keypart= FT_KEYPART; keyuse.used_tables=cond_func->key_item()->used_tables(); keyuse.optimize= 0; keyuse.ref_table_rows= 0; keyuse.keypart_map= 0; keyuse.sj_pred_no= UINT_MAX; keyuse.validity_ref= 0; keyuse.null_rejecting= FALSE; return insert_dynamic(keyuse_array,(uchar*) &keyuse); } static int sort_keyuse(const void *a_, const void *b_) { const KEYUSE *a= static_cast<const KEYUSE *>(a_); const KEYUSE *b= static_cast<const KEYUSE *>(b_); int res; if (a->table->tablenr != b->table->tablenr) return (int) (a->table->tablenr - b->table->tablenr); if (a->key != b->key) return (int) (a->key - b->key); if (a->key == MAX_KEY && b->key == MAX_KEY && a->used_tables != b->used_tables) return (int) ((ulong) a->used_tables - (ulong) b->used_tables); if (a->keypart != b->keypart) return (int) (a->keypart - b->keypart); // Place const values before other ones if ((res= MY_TEST((a->used_tables & ~OUTER_REF_TABLE_BIT)) - MY_TEST((b->used_tables & ~OUTER_REF_TABLE_BIT)))) return res; /* Place rows that are not 'OPTIMIZE_REF_OR_NULL' first */ return (int) ((a->optimize & KEY_OPTIMIZE_REF_OR_NULL) - (b->optimize & KEY_OPTIMIZE_REF_OR_NULL)); } /* Add to KEY_FIELD array all 'ref' access candidates within nested join. This function populates KEY_FIELD array with entries generated from the ON condition of the given nested join, and does the same for nested joins contained within this nested join. @param[in] nested_join_table Nested join pseudo-table to process @param[in,out] end End of the key field array @param[in,out] and_level And-level @param[in,out] sargables Array of found sargable candidates @note We can add accesses to the tables that are direct children of this nested join (1), and are not inner tables w.r.t their neighbours (2). Example for #1 (outer brackets pair denotes nested join this function is invoked for): @code ... LEFT JOIN (t1 LEFT JOIN (t2 ... ) ) ON cond @endcode Example for #2: @code ... LEFT JOIN (t1 LEFT JOIN t2 ) ON cond @endcode In examples 1-2 for condition cond, we can add 'ref' access candidates to t1 only. Example #3: @code ... LEFT JOIN (t1, t2 LEFT JOIN t3 ON inner_cond) ON cond @endcode Here we can add 'ref' access candidates for t1 and t2, but not for t3. */ static void add_key_fields_for_nj(JOIN *join, TABLE_LIST *nested_join_table, KEY_FIELD **end, uint *and_level, SARGABLE_PARAM **sargables) { List_iterator<TABLE_LIST> li(nested_join_table->nested_join->join_list); List_iterator<TABLE_LIST> li2(nested_join_table->nested_join->join_list); bool have_another = FALSE; table_map tables= 0; TABLE_LIST *table; DBUG_ASSERT(nested_join_table->nested_join); while ((table= li++) || (have_another && (li=li2, have_another=FALSE, (table= li++)))) { if (table->nested_join) { if (!table->on_expr) { /* It's a semi-join nest. Walk into it as if it wasn't a nest */ have_another= TRUE; li2= li; li= List_iterator<TABLE_LIST>(table->nested_join->join_list); } else add_key_fields_for_nj(join, table, end, and_level, sargables); } else if (!table->on_expr) tables |= table->table->map; } if (nested_join_table->on_expr) nested_join_table->on_expr->add_key_fields(join, end, and_level, tables, sargables); } void count_cond_for_nj(SELECT_LEX *sel, TABLE_LIST *nested_join_table) { List_iterator<TABLE_LIST> li(nested_join_table->nested_join->join_list); List_iterator<TABLE_LIST> li2(nested_join_table->nested_join->join_list); bool have_another = FALSE; TABLE_LIST *table; while ((table= li++) || (have_another && (li=li2, have_another=FALSE, (table= li++)))) if (table->nested_join) { if (!table->on_expr) { /* It's a semi-join nest. Walk into it as if it wasn't a nest */ have_another= TRUE; li2= li; li= List_iterator<TABLE_LIST>(table->nested_join->join_list); } else count_cond_for_nj(sel, table); } if (nested_join_table->on_expr) nested_join_table->on_expr->walk(&Item::count_sargable_conds, sel, 0); } /** Update keyuse array with all possible keys we can use to fetch rows. @param thd @param[out] keyuse Put here ordered array of KEYUSE structures @param join_tab Array in tablenr_order @param tables Number of tables in join @param cond WHERE condition (note that the function analyzes join_tab[i]->on_expr too) @param normal_tables Tables not inner w.r.t some outer join (ones for which we can make ref access based the WHERE clause) @param select_lex current SELECT @param[out] sargables Array of found sargable candidates @retval 0 OK @retval 1 Out of memory. */ static bool update_ref_and_keys(THD *thd, DYNAMIC_ARRAY *keyuse,JOIN_TAB *join_tab, uint tables, COND *cond, table_map normal_tables, SELECT_LEX *select_lex, SARGABLE_PARAM **sargables) { uint and_level,i; KEY_FIELD *key_fields, *end, *field; size_t sz; uint m= MY_MAX(select_lex->max_equal_elems,1); DBUG_ENTER("update_ref_and_keys"); DBUG_PRINT("enter", ("normal_tables: %llx", normal_tables)); SELECT_LEX *sel=thd->lex->current_select; sel->cond_count= 0; sel->between_count= 0; if (cond) cond->walk(&Item::count_sargable_conds, sel, 0); for (i=0 ; i < tables ; i++) { if (*join_tab[i].on_expr_ref) (*join_tab[i].on_expr_ref)->walk(&Item::count_sargable_conds, sel, 0); } { List_iterator<TABLE_LIST> li(*join_tab->join->join_list); TABLE_LIST *table; while ((table= li++)) { if (table->nested_join) count_cond_for_nj(sel, table); } } /* We use the same piece of memory to store both KEY_FIELD and SARGABLE_PARAM structure. KEY_FIELD values are placed at the beginning this memory while SARGABLE_PARAM values are put at the end. All predicates that are used to fill arrays of KEY_FIELD and SARGABLE_PARAM structures have at most 2 arguments except BETWEEN predicates that have 3 arguments and IN predicates. This any predicate if it's not BETWEEN/IN can be used directly to fill at most 2 array elements, either of KEY_FIELD or SARGABLE_PARAM type. For a BETWEEN predicate 3 elements can be filled as this predicate is considered as sargable with respect to each of its argument. An IN predicate can require at most 1 element as currently it is considered as sargable only for its first argument. Multiple equality can add elements that are filled after substitution of field arguments by equal fields. There can be not more than select_lex->max_equal_elems such substitutions. */ sz= MY_MAX(sizeof(KEY_FIELD),sizeof(SARGABLE_PARAM))* ((sel->cond_count*2 + sel->between_count)*m+1); if (!(key_fields=(KEY_FIELD*) thd->alloc(sz))) DBUG_RETURN(TRUE); /* purecov: inspected */ and_level= 0; field= end= key_fields; *sargables= (SARGABLE_PARAM *) key_fields + (sz - sizeof((*sargables)[0].field))/sizeof(SARGABLE_PARAM); /* set a barrier for the array of SARGABLE_PARAM */ (*sargables)[0].field= 0; if (my_init_dynamic_array2(thd->mem_root->psi_key, keyuse, sizeof(KEYUSE), thd->alloc<KEYUSE>(20), 20, 64, MYF(MY_THREAD_SPECIFIC))) DBUG_RETURN(TRUE); if (cond) { KEY_FIELD *saved_field= field; cond->add_key_fields(join_tab->join, &end, &and_level, normal_tables, sargables); for (; field != end ; field++) { /* Mark that we can optimize LEFT JOIN */ if (field->val->type() == Item::NULL_ITEM && !field->field->real_maybe_null()) field->field->table->reginfo.not_exists_optimize=1; } field= saved_field; } for (i=0 ; i < tables ; i++) { /* Block the creation of keys for inner tables of outer joins. Here only the outer joins that can not be converted to inner joins are left and all nests that can be eliminated are flattened. In the future when we introduce conditional accesses for inner tables in outer joins these keys will be taken into account as well. */ if (*join_tab[i].on_expr_ref) (*join_tab[i].on_expr_ref)->add_key_fields(join_tab->join, &end, &and_level, join_tab[i].table->map, sargables); } /* Process ON conditions for the nested joins */ { List_iterator<TABLE_LIST> li(*join_tab->join->join_list); TABLE_LIST *table; while ((table= li++)) { if (table->nested_join) add_key_fields_for_nj(join_tab->join, table, &end, &and_level, sargables); } } /* fill keyuse with found key parts */ for ( ; field != end ; field++) { if (add_key_part(keyuse,field)) DBUG_RETURN(TRUE); } if (select_lex->ftfunc_list->elements) { if (add_ft_keys(keyuse,join_tab,cond,normal_tables)) DBUG_RETURN(TRUE); } DBUG_RETURN(FALSE); } /* check if key could be used with eq_ref The assumption is that all previous key parts where used */ static void remember_if_eq_ref_key(JOIN *join, KEYUSE *use) { DBUG_ASSERT(use->keypart != FT_KEYPART && use->key != MAX_KEY); TABLE *table= use->table; KEY *key= table->key_info+use->key; ulong key_flags= table->actual_key_flags(key); /* Check if possible eq_ref key This may include keys that does not have HA_NULL_PART_KEY set, but this is ok as best_access_path will resolve this. */ if ((key_flags & (HA_NOSAME | HA_EXT_NOSAME))) { uint key_parts= table->actual_n_key_parts(key); if (use->keypart+1 == key_parts) join->eq_ref_tables|= table->map; } } /** Sort the array of possible keys and remove the following key parts: - ref if there is a keypart which is a ref and a const. (e.g. if there is a key(a,b) and the clause is a=3 and b=7 and b=t2.d, then we skip the key part corresponding to b=t2.d) - keyparts without previous keyparts (e.g. if there is a key(a,b,c) but only b < 5 (or a=2 and c < 3) is used in the query, we drop the partial key parts from consideration). Special treatment for ft-keys. Update join->eq_ref_tables with a bitmap of all tables that can possible have a EQ_REF key. Note that the keys are generated to be used by best_access_path() during the optimization stage. Unused keys will later be deleted by JOIN::drop_unused_derived_keys(). */ bool sort_and_filter_keyuse(JOIN *join, DYNAMIC_ARRAY *keyuse, bool skip_unprefixed_keyparts, bool is_splitting) { THD *thd= join->thd; KEYUSE key_end, *prev, *save_pos, *use; uint found_eq_constant, i; bool found_unprefixed_key_part= 0; join->eq_ref_tables= 0; DBUG_ASSERT(keyuse->elements); my_qsort(keyuse->buffer, keyuse->elements, sizeof(KEYUSE), (qsort_cmp) sort_keyuse); bzero((char*) &key_end, sizeof(key_end)); /* Add for easy testing */ if (insert_dynamic(keyuse, (uchar*) &key_end)) return TRUE; if (optimizer_flag(thd, OPTIMIZER_SWITCH_DERIVED_WITH_KEYS)) generate_derived_keys(keyuse); use= save_pos= dynamic_element(keyuse,0,KEYUSE*); prev= &key_end; found_eq_constant= 0; /* Loop over all elements except the last 'key_end' */ for (i=0 ; i < keyuse->elements-1 ; i++,use++) { if (!use->is_for_hash_join()) { if (!is_splitting && !(use->used_tables & ~OUTER_REF_TABLE_BIT) && use->optimize != KEY_OPTIMIZE_REF_OR_NULL) use->table->const_key_parts[use->key]|= use->keypart_map; if (use->keypart != FT_KEYPART) { if (use->key == prev->key && use->table == prev->table) { if (prev->keypart == use->keypart && found_eq_constant) continue; if (prev->keypart+1 < use->keypart) { found_unprefixed_key_part= 1; if (skip_unprefixed_keyparts) continue; /* remove */ } } else { /* Key changed, check if previous key was a primary/unique key lookup */ if (prev != &key_end && !found_unprefixed_key_part) remember_if_eq_ref_key(join, prev); found_unprefixed_key_part= 0; if (use->keypart != 0) { found_unprefixed_key_part= 1; if (skip_unprefixed_keyparts) continue; /* remove - first found key part must be 0 */ } } } else /* FT_KEY_PART */ { if (prev != &key_end && !found_unprefixed_key_part) remember_if_eq_ref_key(join, prev); found_unprefixed_key_part= 1; // This key cannot be EQ_REF } prev= use; found_eq_constant= !use->used_tables; use->table->reginfo.join_tab->checked_keys.set_bit(use->key); } else { if (prev != &key_end && !found_unprefixed_key_part) remember_if_eq_ref_key(join, prev); prev= &key_end; } /* Old gcc used a memcpy(), which is undefined if save_pos==use: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=19410 http://gcc.gnu.org/bugzilla/show_bug.cgi?id=39480 This also disables a valgrind warning, so better to have the test. */ if (save_pos != use) *save_pos= *use; /* Save ptr to first use */ if (!use->table->reginfo.join_tab->keyuse) use->table->reginfo.join_tab->keyuse= save_pos; save_pos++; } if (prev != &key_end && !found_unprefixed_key_part) remember_if_eq_ref_key(join, prev); i= (uint) (save_pos-(KEYUSE*) keyuse->buffer); (void) set_dynamic(keyuse,(uchar*) &key_end,i); keyuse->elements= i; return FALSE; } /** Update some values in keyuse for faster choose_plan() loop. */ void optimize_keyuse(JOIN *join, DYNAMIC_ARRAY *keyuse_array) { KEYUSE *end,*keyuse= dynamic_element(keyuse_array, 0, KEYUSE*); for (end= keyuse+ keyuse_array->elements ; keyuse < end ; keyuse++) { table_map map; /* If we find a ref, assume this table matches a proportional part of this table. For example 100 records matching a table with 5000 records gives 5000/100 = 50 records per key Constant tables are ignored. To avoid bad matches, we don't make ref_table_rows less than 100. */ keyuse->ref_table_rows= ~(ha_rows) 0; // If no ref if (keyuse->used_tables & (map= (keyuse->used_tables & ~join->const_table_map & ~OUTER_REF_TABLE_BIT))) { uint n_tables= my_count_bits(map); if (n_tables == 1) // Only one table { DBUG_ASSERT(!(map & PSEUDO_TABLE_BITS)); // Must be a real table Table_map_iterator it(map); int tablenr= it.next_bit(); DBUG_ASSERT(tablenr != Table_map_iterator::BITMAP_END); TABLE *tmp_table=join->table[tablenr]; if (tmp_table) // already created keyuse->ref_table_rows= MY_MAX(tmp_table->file->stats.records, 100); } } /* Outer reference (external field) is constant for single executing of subquery */ if (keyuse->used_tables == OUTER_REF_TABLE_BIT) keyuse->ref_table_rows= 1; } } /** Check for the presence of AGGFN(DISTINCT a) queries that may be subject to loose index scan. Check if the query is a subject to AGGFN(DISTINCT) using loose index scan (QUICK_GROUP_MIN_MAX_SELECT). Optionally (if out_args is supplied) will push the arguments of AGGFN(DISTINCT) to the list Check for every COUNT(DISTINCT), AVG(DISTINCT) or SUM(DISTINCT). These can be resolved by Loose Index Scan as long as all the aggregate distinct functions refer to the same fields. Thus: SELECT AGGFN(DISTINCT a, b), AGGFN(DISTINCT b, a)... => can use LIS SELECT AGGFN(DISTINCT a), AGGFN(DISTINCT a) ... => can use LIS SELECT AGGFN(DISTINCT a, b), AGGFN(DISTINCT a) ... => cannot use LIS SELECT AGGFN(DISTINCT a), AGGFN(DISTINCT b) ... => cannot use LIS etc. @param join the join to check @param[out] out_args Collect the arguments of the aggregate functions to a list. We don't worry about duplicates as these will be sorted out later in get_best_group_min_max. @return does the query qualify for indexed AGGFN(DISTINCT) @retval true it does @retval false AGGFN(DISTINCT) must apply distinct in it. */ bool is_indexed_agg_distinct(JOIN *join, List<Item_field> *out_args) { Item_sum **sum_item_ptr; bool result= false; if (join->table_count != 1 || /* reference more than 1 table */ join->select_distinct || /* or a DISTINCT */ join->select_lex->olap == ROLLUP_TYPE) /* Check (B3) for ROLLUP */ return false; Bitmap<MAX_FIELDS> first_aggdistinct_fields; bool first_aggdistinct_fields_initialized= false; for (sum_item_ptr= join->sum_funcs; *sum_item_ptr; sum_item_ptr++) { Item_sum *sum_item= *sum_item_ptr; Item *expr; /* aggregate is not AGGFN(DISTINCT) or more than 1 argument to it */ switch (sum_item->sum_func()) { case Item_sum::MIN_FUNC: case Item_sum::MAX_FUNC: continue; case Item_sum::COUNT_DISTINCT_FUNC: break; case Item_sum::AVG_DISTINCT_FUNC: case Item_sum::SUM_DISTINCT_FUNC: if (sum_item->get_arg_count() == 1) break; /* fall through */ default: return false; } /* We arrive here for every COUNT(DISTINCT),AVG(DISTINCT) or SUM(DISTINCT). Collect the arguments of the aggregate functions to a list. We don't worry about duplicates as these will be sorted out later in get_best_group_min_max */ Bitmap<MAX_FIELDS> cur_aggdistinct_fields; cur_aggdistinct_fields.clear_all(); for (uint i= 0; i < sum_item->get_arg_count(); i++) { expr= sum_item->get_arg(i); /* The AGGFN(DISTINCT) arg is not an attribute? */ if (expr->real_item()->type() != Item::FIELD_ITEM) return false; Item_field* item= static_cast<Item_field*>(expr->real_item()); if (out_args) out_args->push_back(item, join->thd->mem_root); cur_aggdistinct_fields.set_bit(item->field->field_index); result= true; } /* If there are multiple aggregate functions, make sure that they all refer to exactly the same set of columns. */ if (!first_aggdistinct_fields_initialized) { first_aggdistinct_fields= cur_aggdistinct_fields; first_aggdistinct_fields_initialized=true; } else if (first_aggdistinct_fields != cur_aggdistinct_fields) return false; } return result; } /** Discover the indexes that can be used for GROUP BY or DISTINCT queries. If the query has a GROUP BY clause, find all indexes that contain all GROUP BY fields, and add those indexes to join->const_keys. If the query has a DISTINCT clause, find all indexes that contain all SELECT fields, and add those indexes to join->const_keys. This allows later on such queries to be processed by a QUICK_GROUP_MIN_MAX_SELECT. @param join @param join_tab @return None */ static void add_group_and_distinct_keys(JOIN *join, JOIN_TAB *join_tab) { List<Item_field> indexed_fields; List_iterator<Item_field> indexed_fields_it(indexed_fields); ORDER *cur_group; Item_field *cur_item; key_map possible_keys(0); if (join->group_list) { /* Collect all query fields referenced in the GROUP clause. */ for (cur_group= join->group_list; cur_group; cur_group= cur_group->next) (*cur_group->item)->walk(&Item::collect_item_field_processor, &indexed_fields, 0); } else if (join->select_distinct) { /* Collect all query fields referenced in the SELECT clause. */ List<Item> &select_items= join->fields_list; List_iterator<Item> select_items_it(select_items); Item *item; while ((item= select_items_it++)) item->walk(&Item::collect_item_field_processor, &indexed_fields, 0); } else if (!join->tmp_table_param.sum_func_count || !is_indexed_agg_distinct(join, &indexed_fields)) { /* There where no GROUP BY fields and also either no aggregate functions or not all aggregate functions where used with the same DISTINCT (or MIN() / MAX() that works similarly). Nothing to do there. */ return; } if (indexed_fields.elements == 0) { /* There where no index we could use to satisfy the GROUP BY */ return; } /* Intersect the keys of all group fields. */ cur_item= indexed_fields_it++; possible_keys.merge(cur_item->field->part_of_key); while ((cur_item= indexed_fields_it++)) { possible_keys.intersect(cur_item->field->part_of_key); } if (!possible_keys.is_clear_all()) join_tab->const_keys.merge(possible_keys); } /***************************************************************************** Go through all combinations of not marked tables and find the one which uses least records *****************************************************************************/ /** Save const tables first as used tables. */ void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key) { join->positions[idx].table= table; join->positions[idx].key=key; join->positions[idx].records_read=1.0; /* This is a const table */ join->positions[idx].records_out=1.0; /* This is a const table */ join->positions[idx].records_init=1.0; /* This is a const table */ join->positions[idx].cond_selectivity= 1.0; join->positions[idx].ref_depend_map= 0; join->positions[idx].partial_join_cardinality= 1; // join->positions[idx].loosescan_key= MAX_KEY; /* Not a LooseScan */ join->positions[idx].sj_strategy= SJ_OPT_NONE; join->positions[idx].use_join_buffer= FALSE; join->positions[idx].range_rowid_filter_info= 0; /* Move the const table as down as possible in best_ref */ JOIN_TAB **pos=join->best_ref+idx+1; JOIN_TAB *next=join->best_ref[idx]; for (;next != table ; pos++) { JOIN_TAB *tmp=pos[0]; pos[0]=next; next=tmp; } join->best_ref[idx]=table; join->positions[idx].spl_plan= 0; join->positions[idx].spl_pd_boundary= 0; } /* Estimate how many records we will get if we read just this table and apply a part of WHERE that can be checked using only the current table and const tables. @param s Current JOIN_TAB @param use_cond_selectivity Value of optimizer_use_condition_selectivity. If > 1 then use table->cond_selecitivity. @return 0.0 No matching rows @return >= 1.0 Number of expected matching rows Estimate how many records we will get if we - read the given table with its "independent" access method (either quick select or full table/index scan), - apply the part of WHERE that refers only to this table and const tables. - The result cannot be bigger than table records @see also table_after_join_selectivity() produces selectivity of condition that is checked after joining rows from this table to rows from preceding tables. */ static double apply_selectivity_for_table(JOIN_TAB *s, uint use_cond_selectivity) { double dbl_records; if (use_cond_selectivity > 1) { TABLE *table= s->table; double sel= table->cond_selectivity; double table_records= rows2double(s->records); DBUG_ASSERT(sel >= 0 && sel <= 1.0); /* table->cond_selectivity will include data from opt_range. Here we check that this is indeed the case. Note that if table_records == 0, then 'sel' is probably 1 */ DBUG_ASSERT(table_records == 0 || sel <= s->table->opt_range_condition_rows / table_records); dbl_records= table_records * sel; } else { /* This is only taking into considering constant key parts used with this table! If no such conditions existed the following should normally hold: s->table->opt_range_condition_rows == s->found_rows == s->records. The case when this does not hold is when using 'best splitting' in which case s->records may be less than s->found_rows; */ DBUG_ASSERT(s->table->opt_range_condition_rows <= s->found_records); dbl_records= rows2double(MY_MIN(s->table->opt_range_condition_rows, s->records)); } DBUG_ASSERT(dbl_records <= s->records); /* Ensure we return at least one row if there is any possibility to have a matching row. Having rows >= 1.0 helps ensure that when we calculate total rows of joins, the number of resulting rows will not be less after the join. In other words, we assume there is at least one matching row when joining a row with the next table. 0.0 is returned only if it is guaranteed there are no matching rows (for example if the table is empty). */ return dbl_records ? MY_MAX(dbl_records, MIN_ROWS_AFTER_FILTERING) : 0.0; } /* Take into account that the table's WHERE clause has conditions on earlier tables that can reduce the number of accepted rows. @param records Number of original rows (after selectivity) If there is a filtering condition on the table (i.e. ref analyzer found at least one "table.keyXpartY= exprZ", where exprZ refers only to tables preceding this table in the join order we're now considering), then assume that 25% of the rows will be filtered out by this condition. This heuristic is supposed to force tables used in exprZ to be before this table in join order. */ inline double use_found_constraint(double records) { records-= records/4; return records ? MY_MAX(records, MIN_ROWS_AFTER_FILTERING) : 0.0; } /* Calculate the cost of reading a set of rows trough an index @param eq_ref True if there is only one matching key (EQ_REF) Logically this is identical to the code in multi_range_read_info_const() excepts the function also takes into account io_blocks and multiple ranges. One main difference between the functions is that multi_range_read_info_const() adds a very small cost per range MULTI_RANGE_READ_SETUP_COST, to ensure that 'ref' is preferred over ranges. Note that this function assumes that index_only_cost is only to be used with filtering (as cost.read_cost takes into account both clustering and covered keys). index_only_cost does not include KEY_COPY_COST as for filtering there is no copying of not accepted keys. If eq_ref is not set, it means that we have to do one extra 'read_next' on the index to verify that there is not more keys with the same value. WHERE_COST cost is not added to any result. */ static ALL_READ_COST cost_for_index_read(THD *thd, const TABLE *table, uint key, ha_rows records, bool eq_ref) { ALL_READ_COST cost; handler *file= table->file; ha_rows max_seeks; ha_rows extra_reads= eq_ref ? 0 : 1; DBUG_ENTER("cost_for_index_read"); max_seeks= (ha_rows) thd->variables.max_seeks_for_key; set_if_bigger(records, 1); if (thd->opt_ctx_replay && !thd->opt_ctx_replay->infuse_cost_for_index_read(table, key, records, eq_ref, &cost)) { /* Ok, Optimizer_context_replay has provided the cost numbers */ } else if (file->is_clustering_key(key)) { cost.index_cost= file->ha_keyread_clustered_time(key, 1, records+extra_reads, 0); cost.copy_cost= rows2double(records) * file->ROW_COPY_COST; /* There is no 'index_only_read' with a clustered index */ cost.row_cost= {0,0}; /* Caping of index_blocks will happen in handler::cost() */ cost.max_index_blocks= MY_MIN(file->row_blocks(), max_seeks); cost.max_row_blocks= 0; } else if (table->covering_keys.is_set(key) && !table->no_keyread) { cost.index_cost= file->ha_keyread_time(key, 1, records + extra_reads, 0); cost.row_cost= {0,0}; cost.copy_cost= rows2double(records) * file->KEY_COPY_COST; cost.max_index_blocks= MY_MIN(file->index_blocks(key), max_seeks); cost.max_row_blocks= 0; } else { cost.index_cost= file->ha_keyread_time(key, 1, records + extra_reads, 0); /* ha_rnd_pos_time() includes time for copying the row */ cost.row_cost= file->ha_rnd_pos_time(records); cost.max_index_blocks= MY_MIN(file->index_blocks(key), max_seeks); cost.max_row_blocks= MY_MIN(file->row_blocks(), max_seeks); cost.copy_cost= 0; } if (Optimizer_context_recorder *recorder= thd->opt_ctx_recorder) { recorder->record_cost_for_index_read(table, key, records, eq_ref, &cost); } DBUG_PRINT("statistics", ("index_cost: %.3f row_cost: %.3f", file->cost(cost.index_cost), file->cost(cost.row_cost))); DBUG_RETURN(cost); } /** Apply filter if the filter is better than the current cost or if it forced by ROWID_FILTER hint @param thd Thread handler @param table Table @param cost Pointer to cost for current cost, which does not include WHERE_COST cost. Will be updated to new cost if filter is chosen. Will be updated to new cost if filter is used. @param records_arg Pointer to number of records for the current key. Will be updated to records after filter, if filter is used. @param startup_cost Startup cost. Will be updated if filter is used. @param fetch_cost Cost of finding the row, without where compare cost @param index_only_cost Cost if fetching '*records_arg' key values @param prev_records Number of record combinations in previous tables @return 'this' Filter is used (and variables are updated) @return 0 Filter is worse than old plan */ Range_rowid_filter_cost_info* Range_rowid_filter_cost_info:: apply_filter(THD *thd, TABLE *table, ALL_READ_COST *cost, double *records_arg, double *startup_cost, uint ranges, double prev_records) { handler *file= table->file; bool use_filter; double new_cost, org_cost, records= *records_arg, new_records; double filter_startup_cost= get_setup_cost(); double filter_lookup_cost= records * lookup_cost(); double tmp; ALL_READ_COST adjusted_cost; /* Calculate number of resulting rows after filtering Here we trust selectivity and do not adjust rows up even if the end result is low. This means that new_records is allowed to be be < 1.0 */ new_records= records * selectivity; /* Calculate the cost of the filter based on that we had originally 'records' rows and after the filter only 'new_records' accepted rows. Note that the rejected rows, we have only done a key read. We only fetch the row and compare the where if the filter accepts the row id. In case of index only read, fetch_cost == index_only_cost. Even in this the filter can give a better plan as we have to do less comparisons with the WHERE clause. The io_cost is used to take into account that we have to do 1 key lookup to find the first matching key in each range. */ adjusted_cost= *cost; /* We are going to read 'selectivity' fewer rows */ adjusted_cost.row_cost.io*= selectivity; adjusted_cost.row_cost.cpu*= selectivity; adjusted_cost.copy_cost*= selectivity; // Cost of copying row or key adjusted_cost.index_cost.cpu+= filter_lookup_cost; tmp= prev_records * WHERE_COST_THD(thd); org_cost= (file->cost_for_reading_multiple_times(prev_records, cost) + records * tmp); new_cost= (file->cost_for_reading_multiple_times(prev_records, &adjusted_cost) + new_records * tmp + filter_startup_cost); DBUG_ASSERT(new_cost >= 0 && new_records >= 0); use_filter= new_cost < org_cost || is_forced_by_hint; if (unlikely(thd->trace_started())) { Json_writer_object trace_filter(thd, "filter"); trace_filter.add("rowid_filter_index", table->key_info[get_key_no()].name). add("index_only_cost", file->cost(cost->index_cost)). add("filter_startup_cost", filter_startup_cost). add("find_key_and_filter_lookup_cost", filter_lookup_cost). add("filter_selectivity", selectivity). add("original_rows", records). add("new_rows", new_records). add("original_access_cost", file->cost(cost)). add("with_filter_access_cost", file->cost(&adjusted_cost)). add("original_found_rows_cost", file->cost(cost->row_cost)). add("with_filter_found_rows_cost", file->cost(adjusted_cost.row_cost)). add("org_cost", org_cost). add("filter_cost", new_cost). add("filter_used", use_filter); } if (use_filter) { cost->row_cost= adjusted_cost.row_cost; cost->index_cost= adjusted_cost.index_cost; cost->copy_cost= adjusted_cost.copy_cost; *records_arg= new_records; (*startup_cost)+= filter_startup_cost; return this; } return 0; } /* @brief Compute the fanout of hash join operation using EITS data @param join JOIN structure @param tab JOIN_TAB for the current table @param remaining_tables Map of tables not yet accessable @param rnd_records Number of accepted rows in the table, after taking selectivity into account. @param hj_start_key Pointer to hash key @param stats_found Is set to 1 if we found any usable hash key part with statistics from analyze. */ double hash_join_fanout(JOIN *join, JOIN_TAB *tab, table_map remaining_tables, double rnd_records, KEYUSE *hj_start_key, bool *stats_found) { THD *thd= join->thd; /* Before doing the hash join, we will scan the table and apply the local part of the WHERE condition. This will produce rnd_records. The EITS statistics describes the entire table. Calling table->field[N]->get_avg_frequency() produces average #rows in the table with some value. What happens if we filter out rows so that rnd_records rows are left? Something between the two outcomes: A. filtering removes a fraction of rows for each value: avg_frequency=avg_frequency * condition_selectivity B. filtering removes entire groups of rows with the same value, but the remaining groups remain of the same size. We make pessimistic assumption and assume B. We also handle an edge case: if rnd_records is less than avg_frequency, assume we'll get rnd_records rows with the same value, and return rnd_records as the fanout estimate. */ double min_freq= (double) tab->table->stat_records(); bool found_not_usable_field= 0; bool found_usable_field __attribute__((unused))= 0; DBUG_ENTER("hash_join_cardinality"); DBUG_ASSERT(rnd_records > 0 && min_freq > 0); Json_writer_object trace_obj(thd, "hash_join_cardinality"); /* There can be multiple KEYUSE referring to same or different columns KEYUSE(tbl.col1 = ...) KEYUSE(tbl.col1 = ...) KEYUSE(tbl.col2 = ...) Hash join code can use multiple columns: (col1, col2) for joining. We need n_distinct({col1, col2}). EITS only has statistics on individual columns: n_distinct(col1), n_distinct(col2). Our current solution is to be very conservative and use selectivity of one column with the lowest avg_frequency. In the future, we should an approach that cautiosly takes into account multiple KEYUSEs either multiply by number of equalities or by sqrt of the second most selective equality. */ Json_writer_array trace_arr(thd, "hash_join_columns"); for (KEYUSE *keyuse= hj_start_key; keyuse->table == tab->table && is_hash_join_key_no(keyuse->key); keyuse++) { if (!(remaining_tables & keyuse->used_tables) && (!keyuse->validity_ref || *keyuse->validity_ref) && tab->access_from_tables_is_allowed(keyuse->used_tables, join->sjm_lookup_tables)) { Field *field= tab->table->field[keyuse->keypart]; found_usable_field= 1; if (is_eits_usable(field)) { double freq= field->read_stats->get_avg_frequency(); Json_writer_object trace_field(thd); trace_field.add("field",field->field_name.str). add("avg_frequency", freq); if (freq < min_freq) min_freq= freq; *stats_found= 1; continue; } } if (!keyuse->validity_ref || *keyuse->validity_ref) found_not_usable_field= 1; } /* Ensure that some part of hash_key is usable */ DBUG_ASSERT(found_usable_field); trace_arr.end(); if (found_not_usable_field) { /* We did not't have data for all key fields. Assume that the hash will at least limit the number of matched rows to HASH_FANOUT. This makes the cost same as when 'hash_join_cardinality=off' in the case when no analyze of the tables have been made. However, it may cause problems when min_freq is higher than HASH_FANOUT as the optimizer will then assume it is better to put the table earlier in the plan when all key parts are not usable. Note that min_freq can become less than 1.0. This is intentional as it matches what happens if OPTIMIZER_SWITCH_HASH_JOIN_CARDINALITY is not used. */ double max_expected_records= rnd_records * HASH_FANOUT; set_if_smaller(min_freq, max_expected_records); trace_obj.add("using_default_hash_fanout", HASH_FANOUT); } else { /* Before joining the table with the contents of join buffer, we will use the quick select and/or apply the table condition. This will reduce the number of rows joined to rnd_records. How will this affect n_distinct? Depending on which rows are removed, this can either leave n_distinct as is (for some value X, some rows are removed but some are left, leaving the number of distinct values the same), or reduce n_distinct in proportion with the fraction of rows removed (for some values of X, either all or none of the rows with that value are removed). We assume the latter: n_distinct is reduced in proportion the condition and quick select's selectivity. This is in effect same as applying apply_selectivity_for_table() on min_freq as we have already done on rnd_records */ min_freq*= rnd_records / tab->table->stat_records(); set_if_bigger(min_freq, HASH_FANOUT); } trace_obj.add("rows", min_freq); DBUG_RETURN(min_freq); } #ifndef DBUG_OFF static char dbug_join_prefix_buf[256]; const char* dbug_print_join_prefix(const POSITION *join_positions, uint idx, JOIN_TAB *s) { char *buf= dbug_join_prefix_buf; String str(buf, sizeof(dbug_join_prefix_buf), &my_charset_bin); str.length(0); for (uint i=0; i!=idx; i++) { str.append(join_positions[i].table->table->alias); str.append(','); } str.append(s->table->alias); if (str.c_ptr_safe() == buf) return buf; else return "Couldn't fit into buffer"; } #endif /** Find the best access path for an extension of a partial execution plan and add this path to the plan. The function finds the best access path to table 's' from the passed partial plan where an access path is the general term for any means to access the data in 's'. An access path may use either an index or a scan, whichever is cheaper. The input partial plan is passed via the array 'join->positions' of length 'idx'. The chosen access method for 's' and its cost are stored in 'join->positions[idx]'. @param join pointer to the structure providing all context info for the query @param s the table to be joined by the function @param thd thread for the connection that submitted the query @param remaining_tables set of tables not included into the partial plan yet @param idx the length of the partial plan @param disable_jbuf TRUE<=> Don't use join buffering @param record_count estimate for the number of records returned by the partial plan @param pos OUT Table access plan @param loose_scan_pos OUT Table plan that uses loosescan, or set cost to DBL_MAX if not possible. @detail Use this to print the current join prefix: dbug_print_join_prefix(join_positions, idx, s) Use this as breakpoint condition to stop at join prefix "t1,t2,t3": $_streq(dbug_print_join_prefix(join_positions, idx, s), "t1,t2,t3") @return None */ struct best_plan { double cost; // Smallest cost found double records; // Old 'Records' double records_read; // Records accessed double records_out; // Smallest record count seen double identical_keys; // Save value from prev_record_reads Range_rowid_filter_cost_info *filter; // Best filter KEYUSE *key; // Best key SplM_plan_info *spl_plan; table_map ref_depends_map; ulonglong refills; // Join cache refills enum join_type type; uint forced_index; uint max_key_part; table_map found_ref; bool use_join_buffer; }; void best_access_path(JOIN *join, JOIN_TAB *s, table_map remaining_tables, const POSITION *join_positions, uint idx, bool disable_jbuf, double record_count, POSITION *pos, POSITION *loose_scan_pos) { THD *thd= join->thd; uint use_cond_selectivity= thd->variables.optimizer_use_condition_selectivity; TABLE *table= s->table; handler *file= table->file; my_bool found_constraint= 0; /* key_dependent is 0 if all key parts could be used or if there was an EQ_REF table found (which uses all key parts). In other words, we cannot find a better key for the table even if remaining_tables is reduced. Otherwise it's a bitmap of tables that could improve key usage. */ table_map key_dependent= 0; ALL_READ_COST tmp; ha_rows rec; MY_BITMAP *eq_join_set= &s->table->eq_join_set; KEYUSE *hj_start_key= 0; table_map spl_pd_boundary= 0; Loose_scan_opt loose_scan_opt; struct best_plan best; Json_writer_object trace_wrapper(thd, "best_access_path"); DBUG_ENTER("best_access_path"); /* Assume that there is at least one accepted row from previous table combinations. This fixes a problem when the selectivity for the preceding table combinations becomes so high that record_count becomes << 1.0, which makes the cost for the current table so low that it does not matter when calculating the best plans. */ set_if_bigger(record_count, 1.0); best.cost= DBL_MAX; best.records= DBL_MAX; best.records_read= DBL_MAX; best.records_out= MY_MIN(table->stat_records() * table->cond_selectivity, table->opt_range_condition_rows); best.identical_keys= 0; best.filter= 0; best.key= 0; best.max_key_part= 0; best.type= JT_UNKNOWN; best.forced_index= MAX_KEY; best.found_ref= 0; best.ref_depends_map= 0; best.refills= 0; best.use_join_buffer= FALSE; best.spl_plan= 0; disable_jbuf= disable_jbuf || idx == join->const_tables; trace_wrapper.add_table_name(s); bitmap_clear_all(eq_join_set); loose_scan_opt.init(join, s, remaining_tables); if (table->is_splittable()) best.spl_plan= s->choose_best_splitting(idx, remaining_tables, join_positions, &spl_pd_boundary); if (unlikely(thd->trace_started())) { Json_writer_object info(thd, "plan_details"); info.add("record_count", record_count); } Json_writer_array trace_paths(thd, "considered_access_paths"); if (s->keyuse) { /* Use key if possible */ KEYUSE *keyuse, *start_key= 0; const char *cause= NULL; uint max_key_part=0; enum join_type type= JT_UNKNOWN; double cur_cost, copy_cost, cached_prev_record_reads= 0.0; table_map cached_prev_ref= ~(table_map) 0; /* Test how we can use keys */ rec= s->records/MATCHING_ROWS_IN_OTHER_TABLE; // Assumed records/key for (keyuse=s->keyuse ; keyuse->table == table ;) { KEY *keyinfo; ulong key_flags; uint key_parts; key_part_map found_part= 0; /* Bitmap indicating which key parts are used with NULL-rejecting conditions. A bit is set to 1 for a key part if it's used with a NULL-rejecting condition (i.e., the condition will never be satisfied when the indexed column contains NULL). A bit is 0 if the key part is used with a non-NULL-rejecting condition (i.e., the condition can be satisfied even when the indexed column contains NULL, e.g., is NULL or <=>). Example: for condition t1.keypart1 = t2.col1 AND t1.keypart2 <=> t2.col2 AND t1.keypart3 = t2.col3 the notnull_part bitmap will be 101 (binary), because: - keypart1: '=' is NULL-rejecting (bit 1) - keypart2: '<=>' is NOT NULL-rejecting (bit 0) - keypart3: '=' is NULL-rejecting (bit 1) */ key_part_map notnull_part=0; table_map found_ref= 0; uint key= keyuse->key; uint max_const_parts; bool ft_key= (keyuse->keypart == FT_KEYPART); /* Bitmap of keyparts where the ref access is over 'keypart=const': */ key_part_map const_part= 0; /* The or-null keypart in ref-or-null access: */ key_part_map ref_or_null_part= 0; key_part_map all_parts= 0; double startup_cost= s->startup_cost; double records_after_filter, records_best_filter, records; Range_rowid_filter_cost_info *filter= 0; double prev_record_count= record_count; double identical_keys= 0; if (is_hash_join_key_no(key)) { /* Hash join as any join employing join buffer can be used to join only those tables that are joined after the first non const table */ if (!(remaining_tables & keyuse->used_tables) && idx > join->const_tables) { if (!hj_start_key) hj_start_key= keyuse; bitmap_set_bit(eq_join_set, keyuse->keypart); } keyuse++; continue; } keyinfo= table->key_info+key; key_parts= table->actual_n_key_parts(keyinfo); key_flags= table->actual_key_flags(keyinfo); /* Calculate how many key segments of the current key we can use */ start_key= keyuse; loose_scan_opt.next_ref_key(); DBUG_PRINT("info", ("Considering ref access on key %s", keyuse->table->key_info[keyuse->key].name.str)); do /* For each keypart */ { uint keypart= keyuse->keypart; table_map best_part_found_ref= 0, key_parts_dependent= 0; double best_prev_record_reads= DBL_MAX; do /* For each way to access the keypart */ { /* If 1. expression does not refer to forward tables 2. we won't get two ref-or-null's */ double ignore; all_parts|= keyuse->keypart_map; if (!(remaining_tables & keyuse->used_tables) && (!keyuse->validity_ref || *keyuse->validity_ref) && s->access_from_tables_is_allowed(keyuse->used_tables, join->sjm_lookup_tables) && !(ref_or_null_part && (keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL))) { found_part|= keyuse->keypart_map; key_parts_dependent= 0; if (!(keyuse->used_tables & ~join->const_table_map)) const_part|= keyuse->keypart_map; if (!keyuse->val->maybe_null() || keyuse->null_rejecting) notnull_part|=keyuse->keypart_map; if ((found_ref | keyuse->used_tables) != cached_prev_ref) { cached_prev_ref= (found_ref | keyuse->used_tables); cached_prev_record_reads= prev_record_reads(join_positions, idx, cached_prev_ref, record_count, &ignore); } if (cached_prev_record_reads < best_prev_record_reads) { best_prev_record_reads= cached_prev_record_reads; best_part_found_ref= (keyuse->used_tables & ~join->const_table_map); } if (rec > keyuse->ref_table_rows) rec= keyuse->ref_table_rows; /* If there is one 'key_column IS NULL' expression, we can use this ref_or_null optimisation of this field */ if (keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL) ref_or_null_part |= keyuse->keypart_map; /* Remember if there is a WHERE condition that contains 'key_part=expression_with_only_accessible_tables' We ignore const tables as these are handled by selectivity code (const table fields are treated as constants). */ found_constraint|= (keyuse->used_tables & ~(remaining_tables | join->const_table_map)); } else if (!(found_part & keyuse->keypart_map)) key_parts_dependent|= keyuse->used_tables; loose_scan_opt.add_keyuse(remaining_tables, keyuse); keyuse++; } while (keyuse->table == table && keyuse->key == key && keyuse->keypart == keypart); /* If we found a usable key, remember the dependent tables */ if (all_parts & 1) key_dependent|= key_parts_dependent; found_ref|= best_part_found_ref; /* Remember if the key expression used previous non const tables */ } while (keyuse->table == table && keyuse->key == key); /* Assume that each key matches a proportional part of table. */ if (!found_part && !ft_key && !loose_scan_opt.have_a_case()) continue; // Nothing usable found if (rec < MATCHING_ROWS_IN_OTHER_TABLE) rec= MATCHING_ROWS_IN_OTHER_TABLE; // Fix for small tables Json_writer_object trace_access_idx(thd); max_const_parts= max_part_bit(const_part); /* full text keys require special treatment */ if (ft_key) { /* Fulltext indexes are preformed the following way: - In the prepare step it performs the search, collects all positions in an array, sorts it. - If optimizer decides to use the ft index access method it simply' returns positions from the array one by one - If optimizer decides to use something else (another index, table scan), then it'll use binary search in the array to find the position. The following code puts the cost down to very small as the prep step will always be done and the cost to fetch the row from memory is very small. Alternatively we could use the cost of an EQ_REF here. */ tmp.reset(); tmp.row_cost.cpu= file->ROW_COPY_COST; /* We don't know how many records will match. However, we want to have the fulltext search done early, so we put the number of records to be very low. */ records= 1.0; type= JT_FT; if (unlikely(trace_access_idx.trace_started())) trace_access_idx. add("access_type", join_type_str[type]). add("full-text index", keyinfo->name); } else { loose_scan_opt.check_ref_access_part1(s, key, start_key, found_part); /* Check if we found full key */ const key_part_map all_key_parts= PREV_BITS(key_part_map, key_parts); if (found_part == all_key_parts && !ref_or_null_part) { /* use eq key */ max_key_part= (uint) ~0; /* If the index is a unique index (1), and - all its columns are not null (2), or - equalities we are using reject NULLs (3) then the estimate is rows=1. */ if ((key_flags & (HA_NOSAME | HA_EXT_NOSAME)) && // (1) (!(key_flags & HA_NULL_PART_KEY) || // (2) all_key_parts == notnull_part)) // (3) { /* Check that eq_ref_tables are correctly updated */ DBUG_ASSERT(join->eq_ref_tables & table->map); type= JT_EQ_REF; if (unlikely(trace_access_idx.trace_started())) trace_access_idx. add("access_type", join_type_str[type]). add("index", keyinfo->name); if (!found_ref && table->opt_range_keys.is_set(key)) { /* Ensure that the cost is identical to the range cost */ table->opt_range[key].get_costs(&tmp); } else { tmp= cost_for_index_read(thd, table, key, 1, 1); } /* Calculate how many record read calls will be made taking into account that we will cache the last read row. */ prev_record_count= prev_record_reads(join_positions, idx, found_ref, record_count, &identical_keys); records= 1.0; } else { type= JT_REF; if (unlikely(trace_access_idx.trace_started())) trace_access_idx. add("access_type", join_type_str[type]). add("index", keyinfo->name); if (!found_ref) { /* We found a const key */ /* ReuseRangeEstimateForRef-1: We get here if we've found a ref(const) (c_i are constants): "(keypart1=c1) AND ... AND (keypartN=cN)" [ref_const_cond] If range optimizer was able to construct a "range" access on this index, then its condition "quick_cond" was equivalent to ref_const_cond (*), and we can re-use E(#rows) from the range optimizer. Proof of (*): By properties of range and ref optimizers quick_cond will be equal or tighter than ref_const_cond. ref_const_cond already covers "smallest" possible interval - a singlepoint interval over all keyparts. Therefore, quick_cond is equivalent to ref_const_cond (if it was an empty interval we wouldn't have got here). */ if (table->opt_range_keys.is_set(key)) { /* Ensure that the cost is identical to the range cost */ records= (double) table->opt_range[key].rows; trace_access_idx.add("used_range_estimates", true); table->opt_range[key].get_costs(&tmp); goto got_cost2; } /* quick_range couldn't use key! */ records= (double) s->records/rec; if (unlikely(trace_access_idx.trace_started())) trace_access_idx. add("used_range_estimates", false). add("reason", "not available"); } else { if (!(records= keyinfo->rec_per_key_null_aware(key_parts-1, notnull_part))) { /* Prefer longer keys */ trace_access_idx.add("rec_per_key_stats_missing", true); records= ((double) s->records / (double) rec * (1.0 + ((double) (table->s->max_key_length-keyinfo->key_length) / (double) table->s->max_key_length))); set_if_smaller(records, (double)s->records); if (records < 1.0) records= 1.0; /* Can't be as good as a unique */ } /* ReuseRangeEstimateForRef-2: We get here if we could not reuse E(#rows) from range optimizer. Make another try: If range optimizer produced E(#rows) for a prefix of the ref access we're considering, and that E(#rows) is lower then our current estimate, make an adjustment. The criteria of when we can make an adjustment is a special case of the criteria used in ReuseRangeEstimateForRef-3. */ if (table->opt_range_keys.is_set(key) && table->opt_range[key].key_parts <= max_const_parts && table->opt_range[key].ranges == 1 && records > (double) table->opt_range[key].rows) { records= (double) table->opt_range[key].rows; trace_access_idx.add("used_range_estimates", "clipped down"); } else if (unlikely(trace_access_idx.trace_started())) { if (table->opt_range_keys.is_set(key)) { trace_access_idx. add("used_range_estimates",false). add("reason", "not better than ref estimates"); } else { trace_access_idx. add("used_range_estimates", false). add("reason", "not available"); } } } /* Calculate the cost of the index access */ tmp= cost_for_index_read(thd, table, key, (ha_rows) records, 0); } } else { type = ref_or_null_part ? JT_REF_OR_NULL : JT_REF; if (unlikely(trace_access_idx.trace_started())) trace_access_idx. add("access_type", join_type_str[type]). add("index", keyinfo->name); /* Use as much key-parts as possible and a uniq key is better than a not unique key Set tmp to the cost of the accessing the expected number of records. */ if ((found_part & 1) && (!(table->key_info[key].index_flags & HA_ONLY_WHOLE_INDEX) || found_part == PREV_BITS(key_part_map, keyinfo->user_defined_key_parts))) { double extra_cost= 0; max_key_part= max_part_bit(found_part); bool all_used_equalities_are_const= (max_key_part == max_const_parts); /* ReuseRangeEstimateForRef-3: We're now considering a ref[or_null] access via (t.keypart1=e1 AND ... AND t.keypartK=eK) [ OR (same-as-above but with one cond replaced with "t.keypart_i IS NULL")] (**) Try re-using E(#rows) from "range" optimizer: We can do so if "range" optimizer used the same intervals as in (**). The intervals used by range optimizer may be not available at this point (as "range" access might have chosen to create quick select over another index), so we can't compare them to (**). We'll make indirect judgements instead. The sufficient conditions for re-use are: (C1) All e_i in (**) are constants (if this is not satisfied we have no way to know which ranges will be actually scanned by 'ref' until we execute the join) (C2) max #key parts in 'range' access == K == max_key_part (this is apparently a necessary requirement) We also have a property that "range optimizer produces equal or tighter set of scan intervals than ref(const) optimizer". Each of the intervals in (**) are "tightest possible" intervals when one limits itself to using keyparts 1..K (which we do in #2). From here it follows that range access used either one, or both of the (I1) and (I2) intervals: (t.keypart1=c1 AND ... AND t.keypartK=eK) (I1) (same-as-above but with one cond replaced with "t.keypart_i IS NULL") (I2) The remaining part is to exclude the situation where range optimizer used one interval while we're considering ref-or-null and looking for estimate for two intervals. This is done by last limitation: (C3) "range optimizer used (have ref_or_null?2:1) intervals" */ if (table->opt_range_keys.is_set(key) && all_used_equalities_are_const && // (C1) table->opt_range[key].key_parts == max_key_part && //(C2) (table->opt_range[key].ranges == 1 + MY_TEST(ref_or_null_part))) //(C3) { records= (double) table->opt_range[key].rows; table->opt_range[key].get_costs(&tmp); /* TODO: Disable opt_range testing below for this range as we can always use this ref instead. */ trace_access_idx.add("used_range_estimates", true); goto got_cost2; } else { /* Check if we have statistic about the distribution */ if ((records= keyinfo->rec_per_key_null_aware(max_key_part-1, notnull_part))) { /* Fix for the case where the index statistics is too optimistic: If (1) We're considering ref(const) and there is quick select on the same index, (2) and that quick select uses more keyparts (i.e. it will scan equal/smaller interval then this ref(const)) (3) and E(#rows) for quick select is higher then our estimate, Then We'll use E(#rows) from quick select. Q: Why do we choose to use 'ref'? Won't quick select be cheaper in some cases ? TODO: figure this out and adjust the plan choice if needed. */ if (table->opt_range_keys.is_set(key)) { double rows; if (table->opt_range[key].key_parts >= max_key_part) // (2) { /* Choose range over REF in the case range will always be as good or better than REF. This is the case when we have only one const range and it consist of more parts than what we used for REF. */ if (all_used_equalities_are_const && table->opt_range[key].key_parts > max_key_part && table->opt_range[key].ranges <= (uint) (1 + MY_TEST(ref_or_null_part))) { trace_access_idx. add("chosen", false). add("cause", "range is simple and more selective"); continue; // continue with next key } } rows= (double) table->opt_range[key].rows; if (all_used_equalities_are_const && // (1) records < rows) // (3) { trace_access_idx.add("used_range_estimates", "clipped up"); records= rows; } } } else { trace_access_idx.add("rec_per_key_stats_missing", true); /* Assume that the first key part matches 1% of the file and that the whole key matches 10 (duplicates) or 1 (unique) records. Assume also that more key matches proportionally more records This gives the formula: records = (x * (b-a) + a*c-b)/(c-1) b = records matched by whole key a = records matched by first key part (1% of all records?) c = number of key parts in key x = used key parts (1 <= x <= c) */ double rec_per_key; if (!(rec_per_key=(double) keyinfo->rec_per_key[keyinfo->user_defined_key_parts-1])) rec_per_key=(double) s->records/rec+1; if (!s->records) records= 0; else if (rec_per_key/(double) s->records >= 0.01) records= rec_per_key; else { double a=s->records*0.01; if (keyinfo->user_defined_key_parts > 1) records= (max_key_part * (rec_per_key - a) + a*keyinfo->user_defined_key_parts - rec_per_key)/ (keyinfo->user_defined_key_parts-1); else records= rows2double(s->records); set_if_bigger(records, MIN_ROWS_AFTER_FILTERING); } } if (ref_or_null_part) { /* We need to do two key searches to find row */ records *= 2.0; extra_cost= s->table->file->KEY_LOOKUP_COST; } /* ReuseRangeEstimateForRef-4: We get here if we could not reuse E(#rows) from range optimizer. Make another try: If range optimizer produced E(#rows) for a prefix of the ref access we're considering, and that E(#rows) is lower then our current estimate, make the adjustment. The decision whether we can re-use the estimate from the range optimizer is the same as in ReuseRangeEstimateForRef-3, applied to first table->quick_key_parts[key] key parts. */ if (table->opt_range_keys.is_set(key) && table->opt_range[key].key_parts <= max_const_parts && table->opt_range[key].ranges == (1 + MY_TEST(ref_or_null_part & const_part)) && records > (double) table->opt_range[key].rows) { // psergey-merge-sept: remove: if (table->opt_range[key].key_parts <= max_const_parts) { trace_access_idx.add("used_range_estimates", true); records= (double) table->opt_range[key].rows; } } } set_if_smaller(records, (double) s->records); tmp= cost_for_index_read(thd, table, key, (ha_rows)records, 0); tmp.copy_cost+= extra_cost; } else { if (!(found_part & 1)) cause= "no predicate for first keypart"; else cause= "No full key found"; trace_access_idx.add("chosen", false).add("cause", cause); continue; } } got_cost2: loose_scan_opt.check_ref_access_part2(key, start_key, records, file->cost(&tmp) + startup_cost, found_ref); } /* not ft_key */ if (records == DBL_MAX) // Key not usable continue; records_best_filter= records_after_filter= records; /* Check if we can use a filter. Records can be 0 in case of empty tables. */ if ((found_part & 1) && records && table->rowid_filter_can_be_applied_to_key(start_key->key)) { /* If we use filter F with selectivity s the cost of fetching data by key using this filter will be cost_of_fetching_1_row * rows * s + cost_of_fetching_1_key_tuple * rows * (1 - s) + cost_of_1_lookup_into_filter * rows Without using any filter the cost would be just cost_of_fetching_1_row * rows So the gain in access cost per row will be cost_of_fetching_1_row * (1 - s) - cost_of_fetching_1_key_tuple * (1 - s) - cost_of_1_lookup_into_filter = (cost_of_fetching_1_row - cost_of_fetching_1_key_tuple) * (1 - s) - cost_of_1_lookup_into_filter Here we have: cost_of_fetching_1_row = tmp/rows cost_of_fetching_1_key_tuple = keyread_tmp/rows Here's a more detailed explanation that uses the formulas behind the function the call filter->get_adjusted_gain(). The function takes as a parameter the number of probes/look-ups into the filter that is equal to the number of fetched key entries that is equal to the number of row fetches when no filter is used (assuming no index condition pushdown is employed for the used key access). Let this number be N. Then the total gain from using the filter is N*a_adj - b where b is the cost of building the filter and a_adj is calculated as follows: a - (1-access_cost_factor)*(1-s) = (1+1_cond_eval_cost)*(1-s)-1_probe_cost - (1-access_cost_factor)*(1-s) = (1-s)*(1_cond_eval_cost+access_cost_factor) - 1_probe_cost. Here ((1-s)*(1_cond_eval_cost) * N is the gain from checking less conditions pushed into the table, 1_probe_cost*N is the cost of the probes and (1*s) * access_cost_factor * N must be the gain from accessing less rows. It does not matter how we calculate the cost of N full row fetches cost_of_fetching_N_rows or how we calculate the cost of fetching N key entries cost_of_fetching_N_key_entries the gain from less row fetches will be (cost_of_fetching_N_rows - cost_of_fetching_N_key_entries) * (1-s) and this should be equal to (1*s) * access_cost_factor * N. Thus access_cost_factor must be calculated as (cost_of_fetching_N_rows - cost_of_fetching_N_key_entries) / N. For safety we clip cost_of_fetching_N_key_entries by the value of cost_of_fetching_N_row though formally it's not necessary. We cannot use filter with JT_EQ_REF as in this case 'tmp' is number of rows from prev_record_read() and keyread_tmp is 0. These numbers are not usable with rowid filter code. */ filter= table->best_range_rowid_filter(start_key->key, records, file->cost(&tmp), file->cost(tmp.index_cost), prev_record_count, &records_best_filter); set_if_smaller(best.records_out, records_best_filter); if (filter) filter= filter->apply_filter(thd, table, &tmp, &records_after_filter, &startup_cost, 1, prev_record_count); } /* Take into account WHERE and setup cost. We have to check the WHERE for all previous row combinations (record_count). 'prev_record_count' is either 'record_count', or in case of EQ_REF the estimated number of index_read() calls to the engine when taking the one row read cache into account. */ copy_cost= (record_count * records_after_filter * WHERE_COST_THD(thd) + startup_cost); cur_cost= (file->cost_for_reading_multiple_times(prev_record_count, &tmp) + copy_cost); if (unlikely(trace_access_idx.trace_started())) { if (prev_record_count != record_count) trace_access_idx.add("prev_record_count", prev_record_count); trace_access_idx. add("rows", records_after_filter). add("cost", cur_cost); } /* The COST_EPS is here to ensure we use the first key if there are two 'identical keys' that could be used. */ if (cur_cost + COST_EPS < best.cost) { trace_access_idx.add("chosen", true); best.cost= cur_cost; /* We use 'records' instead of 'records_after_filter' here as we want to have EXPLAIN print the number of rows found by the key access. */ best.records= records; // Records before filter! best.records_read= records; best.identical_keys= identical_keys; best.key= start_key; best.found_ref= found_ref; best.max_key_part= max_key_part; best.ref_depends_map= found_ref; best.filter= filter; best.type= type; } else if (unlikely(thd->trace_started())) { trace_access_idx. add("chosen", false). add("cause", cause ? cause : "cost"); } set_if_smaller(best.records_out, records); } /* for each key */ } else { /* No usable keys found. However, there may still be an option to use "Range checked for each record" when all depending tables has been read. s->key_dependent tells us which tables these could be and s->key_start_dependent tells us if a first key part was used. s->key_dependent may include more tables than could be used, but this is ok as not having any usable keys is a rare thing and the performance penalty for extra table bits is that best_extension_by_limited_search() would not be able to prune tables earlier. Example query: SELECT * FROM t1,t2 where t1.key1=t2.key1 OR t2.key2<1 */ if (s->key_start_dependent) key_dependent= s->key_dependent; /* Add dependency for sub queries */ key_dependent|= s->embedded_dependent; } /* if (s->keyuse) */ /* Check that s->key_dependent contains all used_tables found in s->keyuse */ key_dependent&= ~PSEUDO_TABLE_BITS; DBUG_ASSERT((key_dependent & (s->key_dependent | s->embedded_dependent)) == key_dependent); /* If there is no key to access the table, but there is an equi-join predicate connecting the table with the previous tables then we consider the possibility of using hash join. We need also to check that: (1) s is inner table of semi-join -> join cache is allowed for semijoins (2) s is inner table of outer join -> join cache is allowed for outer joins */ if (idx > join->const_tables && best.key == 0 && join->is_allowed_hash_join_access(table) && !bitmap_is_clear_all(eq_join_set) && !disable_jbuf && (!s->emb_sj_nest || join->allowed_semijoin_with_cache) && // (1) (!(table->map & join->outer_join) || join->allowed_outer_join_with_cache)) // (2) { Json_writer_object trace_access_hash(thd); double refills, row_copy_cost, copy_cost, cur_cost, where_cost; double matching_combinations, fanout= 0.0, join_sel; trace_access_hash.add("type", "hash"); trace_access_hash.add("index", "hj-key"); /* Estimate the cost of the hash join access to the table */ double rnd_records; bool stats_found= 0; rnd_records= apply_selectivity_for_table(s, use_cond_selectivity); DBUG_ASSERT(rnd_records <= rows2double(s->found_records) + 0.5); DBUG_ASSERT(hj_start_key); fanout= rnd_records; if (optimizer_flag(thd, OPTIMIZER_SWITCH_HASH_JOIN_CARDINALITY) && rnd_records > 0) { /* Starting from this point, rnd_records should not be used anymore. Use "fanout" for an estimate of # matching records. */ fanout= hash_join_fanout(join, s, remaining_tables, rnd_records, hj_start_key, &stats_found); set_if_smaller(best.records_out, fanout); join_sel= 1.0; } if (!stats_found) { /* No OPTIMIZER_SWITCH_HASH_JOIN_CARDINALITY or no field statistics found. Take into account if there is non constant constraints used with earlier tables in the where expression. If yes, this will set fanout to rnd_records/4. We estimate that there will be HASH_FANOUT (10%) hash matches / row. */ fanout= ((found_constraint) ? use_found_constraint(rnd_records) : rnd_records); set_if_smaller(best.records_out, fanout * HASH_FANOUT); join_sel= HASH_FANOUT; } /* The following cost calculation is identical to the cost calculation for the join cache later on, except for the HASH_FANOUT */ if (s->quick) { /* Cost of reading rows through opt_range including comparing the rows with the attached WHERE clause. */ cur_cost= s->quick->read_time; } else cur_cost= s->cached_scan_and_compare_time; /* We read the table as many times as join buffer becomes full. */ refills= (1.0 + floor((double) cache_record_length(join,idx) * record_count / (double) thd->variables.join_buff_size)); cur_cost= COST_MULT(cur_cost, refills); /* Cost of doing the hash lookup and check all matching rows with the WHERE clause. We assume here that, thanks to the hash, we don't have to compare all row combinations, only a fanout or HASH_FANOUT (10%) rows in the cache. */ row_copy_cost= (ROW_COPY_COST_THD(thd) * JOIN_CACHE_ROW_COPY_COST_FACTOR(thd)); matching_combinations= fanout * join_sel * record_count; copy_cost= (record_count * row_copy_cost + matching_combinations * ((idx - join->const_tables) * row_copy_cost)); where_cost= matching_combinations * WHERE_COST_THD(thd); cur_cost= COST_ADD(cur_cost, copy_cost + where_cost); best.cost= cur_cost; best.records_read= rows2double(s->records); best.records= rnd_records; // Records after where (Legacy value) best.key= hj_start_key; best.ref_depends_map= 0; best.use_join_buffer= TRUE; best.filter= 0; best.type= JT_HASH; best.refills= double_to_ulonglong(ceil(refills)); if (unlikely(trace_access_hash.trace_started())) trace_access_hash. add("rows", rnd_records). add("rows_after_hash", fanout * join_sel). add("refills", refills). add("jbuf_use_cost", copy_cost). add("extra_cond_check_cost", where_cost). add("total_cost", best.cost). add("chosen", true); } /* Don't test table scan if it can't be better. Prefer key lookup if we would use the same key for scanning. Don't do a table scan on InnoDB tables, if we can read the used parts of the row from any of the used index. This is because table scans uses index and we would not win anything by using a table scan. A word for word translation of the below if-statement in sergefp's understanding: we check if we should use table scan if: (1) The found 'ref' access produces more records than a table scan (or index scan, or quick select), or 'ref' is more expensive than any of them. (2) This doesn't hold: the best way to perform table scan is to perform 'range' access using index IDX, and the best way to perform 'ref' access is to use the same index IDX, with the same or more key parts. (note: it is not clear how this rule is/should be extended to index_merge quick selects). Also if we have a hash join we prefer that over a table scan. This heuristic doesn't apply if the quick select uses the group-by min-max optimization. (3) See above note about InnoDB. (4) NOT ("FORCE INDEX(...)" is used for table and there is 'ref' access path, but there is no quick select) If the condition in the above brackets holds, then the only possible "table scan" access method is ALL/index (there is no quick select). Since we have a 'ref' access path, and FORCE INDEX instructs us to choose it over ALL/index, there is no need to consider a full table scan. (5) Non-flattenable semi-joins: don't consider doing a scan of temporary table if we had an option to make lookups into it. In real-world cases, lookups are cheaper than full scans, but when the table is small, they can be [considered to be] more expensive, which causes lookups not to be used for cases with small datasets, which is annoying. OR (10) The quick select is an index_merge quick select prescribed by the INDEX_MERGE hint. In this case we try to follow the hint and ignore all other considerations. */ Json_writer_object trace_access_scan(thd); if (((best.records_read >= s->found_records || best.cost > s->read_time) && // (1) !(best.key && best.key->key == MAX_KEY) && // (2) !(s->quick && s->quick->get_type() != QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX && // (2) best.key && s->quick->index == best.key->key && // (2) table->opt_range_keys.is_set(best.key->key) && // (2) best.max_key_part >= table->opt_range[best.key->key].key_parts) &&// (2) !((file->ha_table_flags() & HA_TABLE_SCAN_ON_INDEX) && // (3) !table->covering_keys.is_clear_all() && best.key && !s->quick) &&// (3) !(table->force_index_join && best.key && !s->quick) && // (4) !(best.key && table->pos_in_table_list->jtbm_subselect)) // (5) || (s->quick && s->quick->force_index_merge)) // (10) { // Check full join double records_after_filter, org_records; double records_best_filter, cur_cost; Range_rowid_filter_cost_info *filter= 0; double startup_cost= s->startup_cost; const char *scan_type= ""; enum join_type type; uint forced_index= MAX_KEY; bool force_plan= 0, use_join_buffer= 0; ulonglong refills= 1; ALL_READ_COST cost; /* Range optimizer never proposes a RANGE if it isn't better than FULL: so if RANGE is present, it's always preferred to FULL. Here we estimate its cost. */ if (s->quick) { /* For each record we: - read record range through 'quick' - skip rows which does not satisfy WHERE constraints */ /* Use record count from range optimizer. This is done to make records found comparable to what we get with 'ref' access. */ org_records= records_after_filter= rows2double(s->found_records); records_best_filter= org_records; set_if_smaller(best.records_out, records_best_filter); if (s->quick->get_type() == QUICK_SELECT_I::QS_TYPE_RANGE) { uint key_no= s->quick->index; TABLE::OPT_RANGE *range= &table->opt_range[key_no]; /* Ensure that 'range' and 's' are coming from the same source The complex 'double' comparison is there because floating point registers complications when costs are calculated. */ DBUG_ASSERT(range->rows >= s->found_records); DBUG_ASSERT((range->cost.total_cost() == 0.0 && s->quick->read_time == 0.0) || compare_cost(range->cost.total_cost(), s->quick->read_time)); DBUG_ASSERT(compare_cost(range->cost.comp_cost, range->rows * file->WHERE_COST)); /* Get range cost. This does not include cost of the WHERE */ range->get_costs(&cost); /* Ensure that cost from opt_range are correct */ DBUG_ASSERT(compare_cost(file->cost_no_capping(&cost) + range->cost.comp_cost + range->cost.setup_cost, s->quick->read_time)); if (table->rowid_filter_can_be_applied_to_key(key_no)) { filter= table->best_range_rowid_filter(key_no, rows2double(range->rows), file->cost(&cost), file->cost(cost.index_cost), record_count, &records_best_filter); set_if_smaller(best.records_out, records_best_filter); if (filter) { filter= filter->apply_filter(thd, table, &cost, &records_after_filter, &startup_cost, range->ranges, record_count); if (filter) { set_if_smaller(best.records_out, records_after_filter); table->opt_range[key_no].selectivity= filter->selectivity; } } } if (best.key && key_no == best.key->key && !best.found_ref && best.max_key_part < table->opt_range[best.key->key].key_parts && table->opt_range[best.key->key].ranges == 1) { /* Force to use range as it is using the 'best key' and using more key parts (and thus will read less rows) */ force_plan= 1; } type= JT_RANGE; /* We cannot use range->cost.cmp_cost here as records_after_filter is be different if filter is used. */ cost.copy_cost+= (records_after_filter * file->WHERE_COST + range->cost.setup_cost); } else { type= JT_INDEX_MERGE; /* We don't know exactly from where the costs comes from. Let's store it in copy_cost. Note that s->quick->read_time includes the cost of comparing the row with the where clause (WHERE_COST) */ cost.reset(); cost.copy_cost= s->quick->read_time; force_plan= s->quick->force_index_merge; } loose_scan_opt.check_range_access(join, idx, s->quick); } else { double records_table_filter; /* We will now calculate cost of scan, with or without join buffer */ records_best_filter= records_after_filter= apply_selectivity_for_table(s, use_cond_selectivity); records_table_filter= ((found_constraint) ? use_found_constraint(records_after_filter) : records_after_filter); DBUG_ASSERT(records_after_filter <= s->records); DBUG_ASSERT(records_after_filter <= s->found_records); set_if_smaller(best.records_out, records_table_filter); org_records= rows2double(s->records); /* Estimate cost of reading table. */ if (s->cached_forced_index_type) { type= s->cached_forced_index_type; cost= s->cached_forced_index_cost; forced_index= s->cached_forced_index; } else { if (table->force_index_join && !best.key) { /* The query is using 'forced_index' and we did not find a usable key. Calculate cost of a table scan with the forced index. */ type= JT_NEXT; if (s->cached_covering_key != MAX_KEY) { /* Use value from estimate_scan_time */ forced_index= s->cached_covering_key; cost= s->cached_scan_and_compare_cost; } else { #ifdef FORCE_INDEX_SHOULD_FORCE_INDEX_SCAN /* No cached key, use shortest allowed key */ key_map keys= *file->keys_to_use_for_scanning(); keys.intersect(table->keys_in_use_for_query); if ((forced_index= find_shortest_key(table, &keys)) < MAX_KEY) { cost= cost_for_index_read(thd, table, forced_index, s->records, 0); /* Calculate cost of checking the attached WHERE */ cost.copy_cost+= s->records * file->WHERE_COST; } else #endif { /* No usable key, use table scan */ cost= s->cached_scan_and_compare_cost; type= JT_ALL; } } } else // table scan { cost= s->cached_scan_and_compare_cost; type= JT_ALL; } /* Cache result for other calls */ s->cached_forced_index_type= type; s->cached_forced_index_cost= cost; s->cached_forced_index= forced_index; } } /* Note: the condition checked here is very out of date and incorrect. Below, we use a more accurate check when assigning the value of best.use_join_buffer. */ if ((s->table->map & join->outer_join) || disable_jbuf) { /* Simple scan We estimate we have to read org_records rows. records_after_filter rows will survive the where check of constants. 'best.records_out' rows will survive after the check against columns from previous tables. */ scan_type= "scan"; /* We have to compare each row set against all previous row combinations */ cur_cost= file->cost_for_reading_multiple_times(record_count, &cost); } else { /* Scan trough join cache */ double cmp_time, row_copy_cost, tmp_refills; /* Note that the cost of checking all rows against the table specific WHERE is already included in cur_cost. */ scan_type= "scan_with_join_cache"; /* Calculate cost of refills */ tmp_refills= (1.0 + floor((double) cache_record_length(join,idx) * (record_count / (double) thd->variables.join_buff_size))); cur_cost= file->cost_for_reading_multiple_times(tmp_refills, &cost); refills= double_to_ulonglong(ceil(tmp_refills)); /* We come here only if there are already rows in the join cache */ DBUG_ASSERT(idx != join->const_tables); /* records_after_filter is the number of rows that have survived the table specific WHERE check that only involves constants. Calculate cost of: - Copying all previous record combinations to the join cache - Copying the tables from the join cache to table records - Checking the WHERE against the final row combination */ row_copy_cost= (ROW_COPY_COST_THD(thd) * JOIN_CACHE_ROW_COPY_COST_FACTOR(thd)); cmp_time= (record_count * row_copy_cost + records_after_filter * record_count * ((idx - join->const_tables) * row_copy_cost + WHERE_COST_THD(thd))); cur_cost= COST_ADD(cur_cost, cmp_time); use_join_buffer= 1; } /* Splitting technique cannot be used with join cache */ if (table->is_splittable()) startup_cost+= table->get_materialization_cost(); cur_cost+= startup_cost; if (unlikely(trace_access_scan.trace_started())) { trace_access_scan. add("access_type", type == JT_ALL ? scan_type : join_type_str[type]); if (type == JT_RANGE) trace_access_scan. add("range_index", table->key_info[s->quick->index].name); trace_access_scan. add("rows", org_records). add("rows_after_filter", records_after_filter). add("rows_out", best.records_out). add("cost", cur_cost); if (use_join_buffer) trace_access_scan. add("cost_without_join_buffer", file->cost_for_reading_multiple_times(record_count, &cost)); if (type == JT_ALL) { trace_access_scan.add("index_only", (s->cached_covering_key != MAX_KEY)); } } if (cur_cost + COST_EPS < best.cost || force_plan) { /* If the table has a range (s->quick is set) make_join_select() will ensure that this will be used */ best.cost= cur_cost; best.records_read= org_records; // Records accessed best.records= records_after_filter; // Records to be checked against // previous row combinations /* If we are using 'use_cond_selectivity > 1' then table_after_join_selectivity may take into account other filters that what is currently used so we have to use records_after_filter. If 'use_cond_selectivity <= 1 then we can use information from the best filter. */ best.key= 0; best.forced_index= forced_index; /* filter is only set if s->quick->get_type() == QUICK_SELECT_I::QS_TYPE_RANGE */ best.filter= filter; /* range/index_merge/ALL/index access method are "independent", so: */ best.ref_depends_map= 0; best.use_join_buffer= use_join_buffer || MY_TEST(!disable_jbuf && (join->allowed_outer_join_with_cache || !(s->table->map & join->outer_join))); best.refills= refills; best.spl_plan= 0; best.type= type; trace_access_scan.add("chosen", true); } else trace_access_scan.add("chosen", false); } else { if (unlikely(trace_access_scan.trace_started())) trace_access_scan. add("type", "scan"). add("chosen", false). add("cause", "cost"); } crash_if_first_double_is_bigger(best.records_out, best.records); crash_if_first_double_is_bigger(best.records_out, best.records_read); /* Update the cost information for the current partial plan */ pos->loops= record_count; pos->records_init= best.records_read; pos->records_read= best.records; pos->records_out= best.records_out; pos->identical_keys= best.identical_keys; pos->read_time= best.cost; pos->key= best.key; pos->forced_index= best.forced_index; pos->type= best.type; pos->table= s; pos->ref_depend_map= best.ref_depends_map; pos->loosescan_picker.loosescan_key= MAX_KEY; pos->use_join_buffer= best.use_join_buffer; pos->firstmatch_with_join_buf= 0; pos->spl_plan= best.spl_plan; pos->spl_pd_boundary= best.spl_plan ? spl_pd_boundary: 0; pos->range_rowid_filter_info= best.filter; pos->key_dependent= (best.type == JT_EQ_REF ? (table_map) 0 : key_dependent & remaining_tables); pos->refills= best.refills; loose_scan_opt.save_to_position(s, record_count, pos->records_out, loose_scan_pos); if (!best.key && idx == join->const_tables && // First table table == join->sort_by_table && join->unit->lim.get_select_limit() >= best.records) // QQQ Why? { trace_access_scan.add("use_tmp_table", true); join->sort_by_table= (TABLE*) 1; // Must use temporary table } trace_access_scan.end(); trace_paths.end(); if (unlikely(thd->trace_started())) print_best_access_for_table(thd, pos); DBUG_VOID_RETURN; } /* Find JOIN_TAB's embedding (i.e, parent) subquery. - For merged semi-joins, tables inside the semi-join nest have their semi-join nest as parent. We intentionally ignore results of table pullout action here. - For non-merged semi-joins (JTBM tabs), the embedding subquery is the JTBM join tab itself. */ static TABLE_LIST* get_emb_subq(JOIN_TAB *tab) { TABLE_LIST *tlist= tab->table->pos_in_table_list; if (tlist->jtbm_subselect) return tlist; TABLE_LIST *embedding= tlist->embedding; if (!embedding || !embedding->sj_subq_pred) return NULL; return embedding; } /* Choose initial table order that "helps" semi-join optimizations. The idea is that we should start with the order that is the same as the one we would have had if we had semijoin=off: - Top-level tables go first - subquery tables are grouped together by the subquery they are in, - subquery tables are attached where the subquery predicate would have been attached if we had semi-join off. This function relies on join_tab_cmp()/join_tab_cmp_straight() to produce certain pre-liminary ordering, see compare_embedding_subqueries() for its description. */ static void choose_initial_table_order(JOIN *join) { TABLE_LIST *emb_subq; JOIN_TAB **tab= join->best_ref + join->const_tables; JOIN_TAB **tabs_end= tab + join->table_count - join->const_tables; DBUG_ENTER("choose_initial_table_order"); /* Find where the top-level JOIN_TABs end and subquery JOIN_TABs start */ for (; tab != tabs_end; tab++) { if ((emb_subq= get_emb_subq(*tab))) break; } uint n_subquery_tabs= (uint)(tabs_end - tab); if (!n_subquery_tabs) DBUG_VOID_RETURN; /* Copy the subquery JOIN_TABs to a separate array */ JOIN_TAB *subquery_tabs[MAX_TABLES]; memcpy(subquery_tabs, tab, sizeof(JOIN_TAB*) * n_subquery_tabs); JOIN_TAB **last_top_level_tab= tab; JOIN_TAB **subq_tab= subquery_tabs; JOIN_TAB **subq_tabs_end= subquery_tabs + n_subquery_tabs; TABLE_LIST *cur_subq_nest= NULL; for (; subq_tab < subq_tabs_end; subq_tab++) { if (get_emb_subq(*subq_tab)!= cur_subq_nest) { /* Reached the part of subquery_tabs that covers tables in some subquery. */ cur_subq_nest= get_emb_subq(*subq_tab); /* Determine how many tables the subquery has */ JOIN_TAB **last_tab_for_subq; for (last_tab_for_subq= subq_tab; last_tab_for_subq < subq_tabs_end && get_emb_subq(*last_tab_for_subq) == cur_subq_nest; last_tab_for_subq++) {} uint n_subquery_tables= (uint)(last_tab_for_subq - subq_tab); /* Walk the original array and find where this subquery would have been attached to */ table_map need_tables= cur_subq_nest->original_subq_pred_used_tables; need_tables &= ~(join->const_table_map | PSEUDO_TABLE_BITS); for (JOIN_TAB **top_level_tab= join->best_ref + join->const_tables; top_level_tab < last_top_level_tab; //top_level_tab < join->best_ref + join->table_count; top_level_tab++) { need_tables &= ~(*top_level_tab)->table->map; /* Check if this is the place where subquery should be attached */ if (!need_tables) { /* Move away the top-level tables that are after top_level_tab */ size_t top_tail_len= last_top_level_tab - top_level_tab - 1; memmove(top_level_tab + 1 + n_subquery_tables, top_level_tab + 1, sizeof(JOIN_TAB*)*top_tail_len); last_top_level_tab += n_subquery_tables; memcpy(top_level_tab + 1, subq_tab, sizeof(JOIN_TAB*)*n_subquery_tables); break; } } DBUG_ASSERT(!need_tables); subq_tab += n_subquery_tables - 1; } } DBUG_VOID_RETURN; } /** Selects and invokes a search strategy for an optimal query plan. The function checks user-configurable parameters that control the search strategy for an optimal plan, selects the search method and then invokes it. Each specific optimization procedure stores the final optimal plan in the array 'join->best_positions', and the cost of the plan in 'join->best_read'. @param join pointer to the structure providing all context info for the query @param join_tables set of the tables in the query @param emb_sjm_nest List of tables in case of materialized semi-join nest @retval FALSE ok @retval TRUE Fatal error */ bool choose_plan(JOIN *join, table_map join_tables, TABLE_LIST *emb_sjm_nest) { uint search_depth= join->thd->variables.optimizer_search_depth; uint use_cond_selectivity= join->thd->variables.optimizer_use_condition_selectivity; bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); THD *thd= join->thd; qsort_cmp2 jtab_sort_func; DBUG_ENTER("choose_plan"); join->limit_optimization_mode= false; join->extra_heuristic_pruning= false; join->prune_level= join->thd->variables.optimizer_prune_level; if ((join->emb_sjm_nest= emb_sjm_nest)) { /* We're optimizing semi-join materialization nest, so put the tables from this semi-join as first */ jtab_sort_func= join_tab_cmp_embedded_first; /* If we are searching for the execution plan of a materialized semi-join nest then allowed_tables contains bits only for the tables from this nest. */ join->allowed_tables= (emb_sjm_nest->sj_inner_tables & ~join->const_table_map); } else { /* if (SELECT_STRAIGHT_JOIN option is set) reorder tables so dependent tables come after tables they depend on, otherwise keep tables in the order they were specified in the query else Apply heuristic: pre-sort all access plans with respect to the number of records accessed. */ jtab_sort_func= straight_join ? join_tab_cmp_straight : join_tab_cmp; join->allowed_tables= ~join->const_table_map; } /* psergey-todo: if we're not optimizing an SJM nest, - sort that outer tables are first, and each sjm nest follows - then, put each [sjm_table1, ... sjm_tableN] sub-array right where WHERE clause pushdown would have put it. */ my_qsort2(join->best_ref + join->const_tables, join->table_count - join->const_tables, sizeof(JOIN_TAB*), jtab_sort_func, (void*) emb_sjm_nest); Json_writer_object wrapper(thd); Json_writer_array trace_plan(thd,"considered_execution_plans"); if (!emb_sjm_nest) choose_initial_table_order(join); if (straight_join) { optimize_straight_join(join, join_tables); } else { DBUG_ASSERT(search_depth <= MAX_TABLES + 1); if (search_depth == 0) /* Automatically determine a reasonable value for 'search_depth' */ search_depth= determine_search_depth(join); if (join->prune_level >= 1 && search_depth >= thd->variables.optimizer_extra_pruning_depth) { join->extra_heuristic_pruning= true; } double limit_cost= DBL_MAX; double limit_record_count; POSITION *limit_plan= NULL; /* First, build a join plan that can short-cut ORDER BY...LIMIT. Do it if (1) The SELECT in query makes it possible to do short-cutting for some table TBL. (2) We are optimizing the whole JOIN, not a semi-join nest (3) The table TBL has not been marked as constant (in this case, ORDER BY LIMIT will be optimized away) */ if (join->limit_shortcut_applicable && // (1) !join->emb_sjm_nest && // (2) !(join->sort_by_table->map & join->const_table_map)) //(3) { bool res; Json_writer_object wrapper(join->thd); Json_writer_array trace(join->thd, "join_limit_shortcut_plan_search"); join->limit_optimization_mode= true; res= greedy_search(join, join_tables, search_depth, use_cond_selectivity); join->limit_optimization_mode= false; if (res) DBUG_RETURN(TRUE); DBUG_ASSERT(join->best_read != DBL_MAX); /* We've built a join order. Adjust its cost based on ORDER BY...LIMIT short-cutting. */ limit_plan= join_limit_shortcut_finalize_plan(join, &limit_cost); limit_record_count= join->join_record_count; } /* The main call to search for the query plan: */ if (greedy_search(join, join_tables, search_depth, use_cond_selectivity)) DBUG_RETURN(TRUE); DBUG_ASSERT(join->best_read != DBL_MAX); if (limit_plan && limit_cost < join->best_read) { /* Plan that uses ORDER BY ... LIMIT shortcutting is better. */ memcpy((uchar*)join->best_positions, (uchar*)limit_plan, sizeof(POSITION)*join->table_count); join->best_read= limit_cost; join->join_record_count= limit_record_count; } } join->emb_sjm_nest= 0; DBUG_RETURN(FALSE); } /* Compare two join tabs based on the subqueries they are from. - top-level join tabs go first - then subqueries are ordered by their select_id (we're using this criteria because we need a cross-platform, deterministic ordering) @return 0 - equal -1 - jt1 < jt2 1 - jt1 > jt2 */ static int compare_embedding_subqueries(const JOIN_TAB *jt1, const JOIN_TAB *jt2) { /* Determine if the first table is originally from a subquery */ TABLE_LIST *tbl1= jt1->table->pos_in_table_list; uint tbl1_select_no; if (tbl1->jtbm_subselect) { tbl1_select_no= tbl1->jtbm_subselect->unit->first_select()->select_number; } else if (tbl1->embedding && tbl1->embedding->sj_subq_pred) { tbl1_select_no= tbl1->embedding->sj_subq_pred->unit->first_select()->select_number; } else tbl1_select_no= 1; /* Top-level */ /* Same for the second table */ TABLE_LIST *tbl2= jt2->table->pos_in_table_list; uint tbl2_select_no; if (tbl2->jtbm_subselect) { tbl2_select_no= tbl2->jtbm_subselect->unit->first_select()->select_number; } else if (tbl2->embedding && tbl2->embedding->sj_subq_pred) { tbl2_select_no= tbl2->embedding->sj_subq_pred->unit->first_select()->select_number; } else tbl2_select_no= 1; /* Top-level */ /* Put top-level tables in front. Tables from within subqueries must follow, grouped by their owner subquery. We don't care about the order that subquery groups are in, because choose_initial_table_order() will re-order the groups. */ if (tbl1_select_no != tbl2_select_no) return tbl1_select_no > tbl2_select_no ? 1 : -1; return 0; } /** Compare two JOIN_TAB objects based on the number of accessed records. @param ptr1 pointer to first JOIN_TAB object @param ptr2 pointer to second JOIN_TAB object NOTES The order relation implemented by join_tab_cmp() is not transitive, i.e. it is possible to choose such a, b and c that (a < b) && (b < c) but (c < a). This implies that result of a sort using the relation implemented by join_tab_cmp() depends on the order in which elements are compared, i.e. the result is implementation-specific. Example: a: dependent = 0x0 table->map = 0x1 found_records = 3 ptr = 0x907e6b0 b: dependent = 0x0 table->map = 0x2 found_records = 3 ptr = 0x907e838 c: dependent = 0x6 table->map = 0x10 found_records = 2 ptr = 0x907ecd0 As for subqueries, this function must produce order that can be fed to choose_initial_table_order(). @retval 1 if first is bigger @retval -1 if second is bigger @retval 0 if equal */ static int join_tab_cmp(void *, const void* ptr1, const void* ptr2) { auto jt1= *(static_cast<const JOIN_TAB *const *>(ptr1)); auto jt2= *(static_cast<const JOIN_TAB *const *>(ptr2)); int cmp; if ((cmp= compare_embedding_subqueries(jt1, jt2)) != 0) return cmp; /* After that do ordering according to numbers of records in the table. */ if (jt1->found_records > jt2->found_records) return 1; if (jt1->found_records < jt2->found_records) return -1; return jt1 > jt2 ? 1 : (jt1 < jt2 ? -1 : 0); } /** Same as join_tab_cmp, but for use with SELECT_STRAIGHT_JOIN. */ static int join_tab_cmp_straight(void *, const void* ptr1, const void* ptr2) { auto jt1= *(static_cast<const JOIN_TAB *const *>(ptr1)); auto jt2= *(static_cast<const JOIN_TAB *const *>(ptr2)); /* We don't do subquery flattening if the parent or child select has STRAIGHT_JOIN modifier. It is complicated to implement and the semantics is hardly useful. */ DBUG_ASSERT(!jt1->emb_sj_nest); DBUG_ASSERT(!jt2->emb_sj_nest); int cmp; if ((cmp= compare_embedding_subqueries(jt1, jt2)) != 0) return cmp; /* We have to check dependency with straight_join as we don't reorder later as we do for other plans in best_extension_by_limited_search(). */ if (jt1->dependent & jt2->table->map) return 1; if (jt2->dependent & jt1->table->map) return -1; return jt1 > jt2 ? 1 : (jt1 < jt2 ? -1 : 0); } /* Same as join_tab_cmp but tables from within the given semi-join nest go first. Used when the optimizing semi-join materialization nests. */ static int join_tab_cmp_embedded_first(void *emb, const void* ptr1, const void* ptr2) { TABLE_LIST *emb_nest= static_cast<TABLE_LIST *>(emb); auto jt1= *(static_cast<const JOIN_TAB *const *>(ptr1)); auto jt2= *(static_cast<const JOIN_TAB *const *>(ptr2)); if (jt1->emb_sj_nest == emb_nest && jt2->emb_sj_nest != emb_nest) return -1; if (jt1->emb_sj_nest != emb_nest && jt2->emb_sj_nest == emb_nest) return 1; if (jt1->found_records > jt2->found_records) return 1; if (jt1->found_records < jt2->found_records) return -1; return jt1 > jt2 ? 1 : (jt1 < jt2 ? -1 : 0); } /** Heuristic procedure to automatically guess a reasonable degree of exhaustiveness for the greedy search procedure. The procedure estimates the optimization time and selects a search depth big enough to result in a near-optimal QEP, that doesn't take too long to find. If the number of tables in the query exceeds some constant, then search_depth is set to this constant. @param join pointer to the structure providing all context info for the query @note This is an extremely simplistic implementation that serves as a stub for a more advanced analysis of the join. Ideally the search depth should be determined by learning from previous query optimizations, because it will depend on the CPU power (and other factors). @todo this value should be determined dynamically, based on statistics: uint max_tables_for_exhaustive_opt= 7; @todo this value could be determined by some mapping of the form: depth : table_count -> [max_tables_for_exhaustive_opt..MAX_EXHAUSTIVE] @return A positive integer that specifies the search depth (and thus the exhaustiveness) of the depth-first search algorithm used by 'greedy_search'. */ static uint determine_search_depth(JOIN *join) { uint table_count= join->table_count - join->const_tables; uint search_depth; /* TODO: this value should be determined dynamically, based on statistics: */ uint max_tables_for_exhaustive_opt= 7; if (table_count <= max_tables_for_exhaustive_opt) search_depth= table_count+1; // use exhaustive for small number of tables else /* TODO: this value could be determined by some mapping of the form: depth : table_count -> [max_tables_for_exhaustive_opt..MAX_EXHAUSTIVE] */ search_depth= max_tables_for_exhaustive_opt; // use greedy search return search_depth; } /** Select the best ways to access the tables in a query without reordering them. Find the best access paths for each query table and compute their costs according to their order in the array 'join->best_ref' (thus without reordering the join tables). The function calls sequentially 'best_access_path' for each table in the query to select the best table access method. The final optimal plan is stored in the array 'join->best_positions', and the corresponding cost in 'join->best_read'. @param join pointer to the structure providing all context info for the query @param remaining_tables set of the tables in the query @note This function can be applied to: - queries with STRAIGHT_JOIN - internally to compute the cost of an arbitrary QEP @par Thus 'optimize_straight_join' can be used at any stage of the query optimization process to finalize a QEP as it is. */ static void optimize_straight_join(JOIN *join, table_map remaining_tables) { JOIN_TAB *s; uint idx= join->const_tables; bool disable_jbuf= join->thd->variables.join_cache_level == 0; double record_count= 1.0; double read_time= 0.0; uint use_cond_selectivity= join->thd->variables.optimizer_use_condition_selectivity; POSITION loose_scan_pos; THD *thd= join->thd; init_join_plan_search_state(join); for (JOIN_TAB **pos= join->best_ref + idx ; (s= *pos) ; pos++) { POSITION *position= join->positions + idx; Json_writer_object trace_one_table(thd); double original_record_count, current_record_count; if (unlikely(thd->trace_started())) trace_plan_prefix(&trace_one_table, join, idx, remaining_tables); /* Find the best access method from 's' to the current partial plan */ best_access_path(join, s, remaining_tables, join->positions, idx, disable_jbuf, record_count, position, &loose_scan_pos); /* Compute the cost of the new plan extended with 's' */ current_record_count= COST_MULT(record_count, position->records_out); read_time= COST_ADD(read_time, position->read_time); original_record_count= current_record_count; optimize_semi_joins(join, remaining_tables, idx, ¤t_record_count, &read_time, &loose_scan_pos); if (position->sj_strategy != SJ_OPT_NONE && original_record_count) { /* Adjust records_out to contain the final number of rows */ double ratio= current_record_count / original_record_count; if (ratio < 1) { position->records_out*= ratio; } if (unlikely(trace_one_table.trace_started())) { trace_one_table. add("sj_rows_out", position->records_out). add("sj_rows_for_plan", current_record_count). add("sj_filtered", safe_filtered(position->records_out, position->records_init)); } } remaining_tables&= ~(s->table->map); if (use_cond_selectivity > 1 && position->sj_strategy == SJ_OPT_NONE) { double pushdown_cond_selectivity, records_out; pushdown_cond_selectivity= table_after_join_selectivity(join, idx, s, remaining_tables, &records_out); if (unlikely(thd->trace_started()) && pushdown_cond_selectivity != 1.0) { trace_one_table. add("rows_out", records_out). add("pushdown_cond_selectivity", pushdown_cond_selectivity). add("filtered", safe_filtered(position->records_out, position->records_init)); } position->cond_selectivity= pushdown_cond_selectivity; position->records_out= records_out; current_record_count= COST_MULT(record_count, records_out); } else position->cond_selectivity= 1.0; position->partial_join_cardinality= current_record_count; ++idx; record_count= current_record_count; } if (join->sort_by_table && join->sort_by_table != join->positions[join->const_tables].table->table) { /* We may have to make a temp table, note that this is only a heuristic since we cannot know for sure at this point if we we are going to use addon fields or to have flush sorting to disk. We also don't know the temporary table will be in memory or disk. The following calculation takes a middle ground where assume we can sort the keys in memory but have to use a disk based temporary table to retrieve the rows. This cost is probably much bigger than it has to be... */ double sort_cost; sort_cost= (get_qsort_sort_cost((ha_rows)record_count, 0) + record_count * DISK_TEMPTABLE_LOOKUP_COST(thd)); { if (unlikely(thd->trace_started())) { Json_writer_object trace_one_table(thd); trace_one_table.add("estimated_cost_for_sorting", sort_cost); } } read_time= COST_ADD(read_time, sort_cost); } memcpy((uchar*) join->best_positions, (uchar*) join->positions, sizeof(POSITION)*idx); join->join_record_count= record_count; join->best_read= read_time; } /** Find a good, possibly optimal, query execution plan (QEP) by a greedy search. The search procedure uses a hybrid greedy/exhaustive search with controlled exhaustiveness. The search is performed in N = card(remaining_tables) steps. Each step evaluates how promising is each of the unoptimized tables, selects the most promising table, and extends the current partial QEP with that table. Currenly the most 'promising' table is the one with least expensive extension.\ There are two extreme cases: -# When (card(remaining_tables) < search_depth), the estimate finds the best complete continuation of the partial QEP. This continuation can be used directly as a result of the search. -# When (search_depth == 1) the 'best_extension_by_limited_search' considers the extension of the current QEP with each of the remaining unoptimized tables. All other cases are in-between these two extremes. Thus the parameter 'search_depth' controlls the exhaustiveness of the search. The higher the value, the longer the optimization time and possibly the better the resulting plan. The lower the value, the fewer alternative plans are estimated, but the more likely to get a bad QEP. All intermediate and final results of the procedure are stored in 'join': - join->positions : modified for every partial QEP that is explored - join->best_positions: modified for the current best complete QEP - join->best_read : modified for the current best complete QEP - join->best_ref : might be partially reordered The final optimal plan is stored in 'join->best_positions', and its corresponding cost in 'join->best_read'. @note The following pseudocode describes the algorithm of 'greedy_search': @code procedure greedy_search input: remaining_tables output: pplan; { pplan = <>; do { (t, a) = best_extension(pplan, remaining_tables); pplan = concat(pplan, (t, a)); remaining_tables = remaining_tables - t; } while (remaining_tables != {}) return pplan; } @endcode where 'best_extension' is a placeholder for a procedure that selects the most "promising" of all tables in 'remaining_tables'. Currently this estimate is performed by calling 'best_extension_by_limited_search' to evaluate all extensions of the current QEP of size 'search_depth', thus the complexity of 'greedy_search' mainly depends on that of 'best_extension_by_limited_search'. @par If 'best_extension()' == 'best_extension_by_limited_search()', then the worst-case complexity of this algorithm is <= O(N*N^search_depth/search_depth). When serch_depth >= N, then the complexity of greedy_search is O(N!). @par In the future, 'greedy_search' might be extended to support other implementations of 'best_extension', e.g. some simpler quadratic procedure. @param join pointer to the structure providing all context info for the query @param remaining_tables set of tables not included into the partial plan yet @param search_depth controlls the exhaustiveness of the search @param use_cond_selectivity specifies how the selectivity of the conditions pushed to a table should be taken into account @retval FALSE ok @retval TRUE Fatal error */ static bool greedy_search(JOIN *join, table_map remaining_tables, uint search_depth, uint use_cond_selectivity) { double record_count= 1.0; double read_time= 0.0; uint idx= join->const_tables; // index into 'join->best_ref' uint best_idx; uint size_remain; // cardinality of remaining_tables table_map usable_tables, eq_ref_tables; POSITION best_pos; JOIN_TAB *best_table; // the next plan node to be added to the curr QEP // ==join->tables or # tables in the sj-mat nest we're optimizing uint n_tables __attribute__((unused)); DBUG_ENTER("greedy_search"); DBUG_ASSERT(!(remaining_tables & join->const_table_map)); init_join_plan_search_state(join); /* number of tables that remain to be optimized */ usable_tables= (join->emb_sjm_nest ? (join->emb_sjm_nest->sj_inner_tables & ~join->const_table_map & remaining_tables): remaining_tables); n_tables= size_remain= my_count_bits(usable_tables); join->next_sort_position= join->sort_positions; do { /* Find the extension of the current QEP with the lowest cost We are using remaining_table instead of usable tables here as in case of an emb_sjm_nest, we want to be able to check if an embedded table is depending on an outer table. */ join->best_read= DBL_MAX; if ((int) best_extension_by_limited_search(join, remaining_tables, idx, record_count, read_time, search_depth, use_cond_selectivity, &eq_ref_tables) < (int) SEARCH_OK) DBUG_RETURN(TRUE); /* 'best_read < DBL_MAX' means that optimizer managed to find some plan and updated 'best_positions' array accordingly. */ DBUG_ASSERT(join->best_read < DBL_MAX); if (size_remain <= search_depth) { /* 'join->best_positions' contains a complete optimal extension of the current partial QEP. */ DBUG_EXECUTE("opt", print_plan(join, n_tables, record_count, read_time, read_time, "optimal");); DBUG_RETURN(FALSE); } /* select the first table in the optimal extension as most promising */ best_pos= join->best_positions[idx]; best_table= best_pos.table; /* Each subsequent loop of 'best_extension_by_limited_search' uses 'join->positions' for cost estimates, therefore we have to update its value. */ join->positions[idx]= best_pos; /* Update the interleaving state after extending the current partial plan with a new table. We are doing this here because best_extension_by_limited_search reverts the interleaving state to the one of the non-extended partial plan on exit. */ bool is_interleave_error __attribute__((unused))= check_interleaving_with_nj (best_table); /* This has been already checked by best_extension_by_limited_search */ DBUG_ASSERT(!is_interleave_error); /* Also, update the semi-join optimization state. Information about the picked semi-join operation is in best_pos->...picker, but we need to update the global state in the JOIN object, too. */ if (!join->emb_sjm_nest) update_sj_state(join, best_table, idx, remaining_tables); /* find the position of 'best_table' in 'join->best_ref' */ best_idx= idx; JOIN_TAB *pos= join->best_ref[best_idx]; while (pos && best_table != pos) pos= join->best_ref[++best_idx]; DBUG_ASSERT((pos != NULL)); // should always find 'best_table' /* Move 'best_table' at the first free position in the array of joins We don't need to keep the array sorted as best_extension_by_limited_search() will sort them. */ swap_variables(JOIN_TAB*, join->best_ref[idx], join->best_ref[best_idx]); /* compute the cost of the new plan extended with 'best_table' */ record_count= COST_MULT(record_count, join->positions[idx].records_read); read_time= COST_ADD(read_time, join->positions[idx].read_time); remaining_tables&= ~(best_table->table->map); --size_remain; ++idx; DBUG_EXECUTE("opt", print_plan(join, idx, record_count, read_time, read_time, "extended");); } while (TRUE); } /** Get cost of execution and fanout produced by selected tables in the join prefix (where prefix is defined as prefix in depth-first traversal) @param end_tab_idx The number of last tab to be taken into account (in depth-first traversal prefix) @param filter_map Bitmap of tables whose cost/fanout are to be taken into account. @param read_time_arg [out] store read time here @param record_count_arg [out] store record count here @note @returns read_time_arg and record_count_arg contain the computed cost and fanout */ void JOIN::get_partial_cost_and_fanout(int end_tab_idx, table_map filter_map, double *read_time_arg, double *record_count_arg) { double record_count= 1; double read_time= 0.0; double sj_inner_fanout= 1.0; JOIN_TAB *end_tab= NULL; JOIN_TAB *tab; int i; int last_sj_table= MAX_TABLES; /* Handle a special case where the join is degenerate, and produces no records */ if (table_count == const_tables) { *read_time_arg= 0.0; /* We return 1, because - it is the pessimistic estimate (there might be grouping) - it's safer, as we're less likely to hit the edge cases in calculations. */ *record_count_arg=1.0; return; } for (tab= first_depth_first_tab(this), i= const_tables; tab; tab= next_depth_first_tab(this, tab), i++) { end_tab= tab; if (i == end_tab_idx) break; } for (tab= first_depth_first_tab(this), i= const_tables; ; tab= next_depth_first_tab(this, tab), i++) { if (end_tab->bush_root_tab && end_tab->bush_root_tab == tab) { /* We've entered the SJM nest that contains the end_tab. The caller is - interested in fanout inside the nest (because that's how many times we'll invoke the attached WHERE conditions) - not interested in cost */ record_count= 1.0; read_time= 0.0; } /* Ignore fanout (but not cost) from sj-inner tables, as long as the range that processes them finishes before the end_tab */ if (tab->sj_strategy != SJ_OPT_NONE) { sj_inner_fanout= 1.0; last_sj_table= i + tab->n_sj_tables; } table_map cur_table_map; if (tab->table) cur_table_map= tab->table->map; else { /* This is a SJ-Materialization nest. Check all of its tables */ TABLE *first_child= tab->bush_children->start->table; TABLE_LIST *sjm_nest= first_child->pos_in_table_list->embedding; cur_table_map= sjm_nest->nested_join->used_tables; } if (tab->records_read && (cur_table_map & filter_map)) { record_count= COST_MULT(record_count, tab->records_read); read_time= COST_ADD(read_time, tab->read_time); if (tab->emb_sj_nest) sj_inner_fanout= COST_MULT(sj_inner_fanout, tab->records_read); } if (i == last_sj_table) { record_count /= sj_inner_fanout; sj_inner_fanout= 1.0; last_sj_table= MAX_TABLES; } if (tab == end_tab) break; } *read_time_arg= read_time; *record_count_arg= record_count; } /* Get prefix cost and fanout. This function is different from get_partial_cost_and_fanout: - it operates on a JOIN that haven't yet finished its optimization phase (in particular, fix_semijoin_strategies_for_picked_join_order() and get_best_combination() haven't been called) - it assumes the join prefix doesn't have any semi-join plans These assumptions are met by the caller of the function. */ void JOIN::get_prefix_cost_and_fanout(uint n_tables, double *read_time_arg, double *record_count_arg) { double record_count= 1; double read_time= 0.0; for (uint i= const_tables; i < n_tables + const_tables ; i++) { if (best_positions[i].records_read) { record_count= COST_MULT(record_count, best_positions[i].records_read); read_time= COST_ADD(read_time, best_positions[i].read_time); } } *read_time_arg= read_time; *record_count_arg= record_count; } /** Estimate the number of rows that query execution will read. @todo This is a very pessimistic upper bound. Use join selectivity when available to produce a more realistic number. */ double JOIN::get_examined_rows() { double examined_rows; double prev_fanout= 1; double records; JOIN_TAB *tab= first_breadth_first_tab(); JOIN_TAB *prev_tab= tab; records= (double)tab->get_examined_rows(); while ((tab= next_breadth_first_tab(first_breadth_first_tab(), top_join_tab_count, tab))) { prev_fanout= COST_MULT(prev_fanout, prev_tab->records_read); records= COST_ADD(records, COST_MULT((double) (tab->get_examined_rows()), prev_fanout)); prev_tab= tab; } examined_rows= records; return examined_rows; } /** @brief Get the selectivity of equalities between columns when joining a table @param join The optimized join @param idx The number of tables in the evaluated partual join @param s The table to be joined for evaluation @param rem_tables The bitmap of tables to be joined later @param keyparts The number of key parts to used when joining s @param ref_keyuse_steps Array of references to keyuses employed to join s */ static double table_multi_eq_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, table_map rem_tables, uint keyparts, uint16 *ref_keyuse_steps) { double sel= 1.0; COND_EQUAL *cond_equal= join->cond_equal; if (!cond_equal || !cond_equal->current_level.elements || !s->keyuse) return sel; Item_equal *item_equal; List_iterator_fast<Item_equal> it(cond_equal->current_level); TABLE *table= s->table; table_map table_bit= table->map; POSITION *pos= &join->positions[idx]; while ((item_equal= it++)) { /* Check whether we need to take into account the selectivity of multiple equality item_equal. If this is the case multiply the current value of sel by this selectivity */ table_map used_tables= item_equal->used_tables(); if (!(used_tables & table_bit)) continue; if (item_equal->get_const()) continue; bool adjust_sel= FALSE; Item_equal_fields_iterator fi(*item_equal); while((fi++) && !adjust_sel) { Field *fld= fi.get_curr_field(); if (fld->table->map != table_bit) continue; if (pos->key == 0) adjust_sel= TRUE; else { uint i; KEYUSE *keyuse= pos->key; uint key= keyuse->key; for (i= 0; i < keyparts; i++) { if (i > 0) keyuse+= ref_keyuse_steps[i-1]; uint fldno; if (is_hash_join_key_no(key)) fldno= keyuse->keypart; else fldno= table->key_info[key].key_part[i].fieldnr - 1; if (fld->field_index == fldno) break; } keyuse= pos->key; if (i == keyparts) { /* Field fld is included in multiple equality item_equal and is not a part of the ref key. The selectivity of the multiple equality must be taken into account unless one of the ref arguments is equal to fld. */ adjust_sel= TRUE; for (uint j= 0; j < keyparts && adjust_sel; j++) { if (j > 0) keyuse+= ref_keyuse_steps[j-1]; Item *ref_item= keyuse->val; if (ref_item->real_item()->type() == Item::FIELD_ITEM) { Item_field *field_item= (Item_field *) (ref_item->real_item()); if (item_equal->contains(field_item->field)) adjust_sel= FALSE; } } } } } if (adjust_sel) { /* If ref == 0 and there are no fields in the multiple equality item_equal that belong to the tables joined prior to s then the selectivity of multiple equality will be set to 1.0. */ double eq_fld_sel= 1.0; fi.rewind(); while ((fi++)) { double curr_eq_fld_sel; Field *fld= fi.get_curr_field(); if (!(fld->table->map & ~(table_bit | rem_tables))) continue; curr_eq_fld_sel= get_column_avg_frequency(fld) / fld->table->stat_records(); if (curr_eq_fld_sel < 1.0) set_if_bigger(eq_fld_sel, curr_eq_fld_sel); } sel*= eq_fld_sel; } } return sel; } /** @brief Get the selectivity of conditions when joining a table @param join The optimized join @param s The table to be joined for evaluation @param rem_tables The bitmap of tables to be joined later @param new_records_out OUT Set to number of rows accepted @detail Get selectivity of conditions that can be applied when joining this table with previous tables. For quick selects and full table scans, selectivity of COND(this_table) is accounted for in apply_selectivity_for_table(). Here, we only count selectivity of COND(this_table, previous_tables). For other access methods, we need to calculate selectivity of the whole condition, "COND(this_table) AND COND(this_table, previous_tables)". @retval selectivity of the conditions imposed on the rows of s related to the rows that we are expected to read (position->records_init). */ static double table_after_join_selectivity(JOIN *join, uint idx, JOIN_TAB *s, table_map rem_tables, double *new_records_out) { uint16 ref_keyuse_steps_buf[MAX_REF_PARTS]; uint ref_keyuse_size= MAX_REF_PARTS; uint16 *ref_keyuse_steps= ref_keyuse_steps_buf; Field *field; TABLE *table= s->table; MY_BITMAP *read_set= table->read_set; POSITION *pos= &join->positions[idx]; double sel, records_out= pos->records_out; uint keyparts= 0; uint found_part_ref_or_null= 0; if (pos->key != 0) { sel= table->cond_selectivity; /* A ref access or hash join is used for this table. ref access is created from tbl.keypart1=expr1 AND tbl.keypart2=expr2 AND ... and it will only return rows for which this condition is satisfied. Suppose, certain expr{i} is a constant. Since ref access only returns rows that satisfy tbl.keypart{i}=const (*) then selectivity of this equality should not be counted in return value of this function. This function uses the value of table->cond_selectivity=selectivity(COND(tbl)) (**) as a starting point. This value includes selectivity of equality (*). We should somehow discount it. Looking at calculate_cond_selectivity_for_table(), one can see that the value is not necessarily a direct multiplicand in table->cond_selectivity There are three possible ways to discount 1. There is a potential range access on t.keypart{i}=const. (an important special case: the used ref access has a const prefix for which a range estimate is available) 2. The field has a histogram. field[x]->cond_selectivity has the data. 3. Use index stats on this index: rec_per_key[key_part+1]/rec_per_key[key_part] (TODO: more details about the "t.key=othertable.col" case) */ KEYUSE *keyuse= pos->key; KEYUSE *prev_ref_keyuse= keyuse; uint key= keyuse->key; bool used_range_selectivity= false; /* Check if we have a prefix of key=const that matches a quick select. */ if (!is_hash_join_key_no(key) && table->opt_range_keys.is_set(key)) { key_part_map quick_key_map= (key_part_map(1) << table->opt_range[key].key_parts) - 1; if (s->type == JT_RANGE || (table->opt_range[key].rows && (table->const_key_parts[key] & 1))) { /* We are either using a range or we are using a REF which the same key as an active range and the first key part is a constant. In both cases we have to discount the selectivity for the range as otherwise we are using the selectivity twice. */ for (; quick_key_map & 1 ; quick_key_map>>= 1) { while (keyuse->table == table && keyuse->key == key && keyuse->keypart == keyparts) { keyuse++; } keyparts++; } /* Here we discount selectivity of the constant range CR. To calculate this selectivity we use elements from the quick_rows[] array. If we have indexes i1,...,ik with the same prefix compatible with CR any of the estimate quick_rows[i1], ... quick_rows[ik] could be used for this calculation but here we don't know which one was actually used. So sel could be greater than 1 and we have to cap it. However if sel becomes greater than 2 then with high probability something went wrong. */ DBUG_ASSERT(sel <= 1.0); DBUG_ASSERT(table->opt_range[key].rows <= (double) table->stat_records()); sel /= ((double) table->opt_range[key].rows / (double) table->stat_records()); set_if_smaller(sel, 1.0); used_range_selectivity= true; } } /* Go through the "keypart{N}=..." equalities and find those that were already taken into account in table->cond_selectivity. */ keyuse= pos->key; keyparts=0; while (keyuse->table == table && keyuse->key == key) { if (!(keyuse->used_tables & (rem_tables | table->map))) { if (are_tables_local(s, keyuse->val->used_tables())) { if (is_hash_join_key_no(key)) { if (keyparts == keyuse->keypart) keyparts++; } else { if (keyparts == keyuse->keypart && !((keyuse->val->used_tables()) & ~pos->ref_depend_map) && !(found_part_ref_or_null & keyuse->optimize)) { /* Found a KEYUSE object that will be used by ref access */ keyparts++; found_part_ref_or_null|= keyuse->optimize & ~KEY_OPTIMIZE_EQ; } } if (keyparts > keyuse->keypart) { /* Ok this is the keyuse that will be used for ref access */ if (!used_range_selectivity && keyuse->val->const_item()) { uint fldno; if (is_hash_join_key_no(key)) fldno= keyuse->keypart; else fldno= table->key_info[key].key_part[keyparts-1].fieldnr - 1; if (table->field[fldno]->cond_selectivity > 0) { sel /= table->field[fldno]->cond_selectivity; set_if_smaller(sel, 1.0); } /* TODO: we could do better here: 1. cond_selectivity might be =1 (the default) because quick select on some index prevented us from analyzing histogram for this column. 2. we could get an estimate through this? rec_per_key[key_part-1] / rec_per_key[key_part] */ } if (keyparts > 1) { /* Prepare to set ref_keyuse_steps[keyparts-2]: resize the array if it is not large enough */ if (keyparts - 2 >= ref_keyuse_size) { uint new_size= MY_MAX(ref_keyuse_size*2, keyparts); void *new_buf; if (!(new_buf= my_malloc(PSI_INSTRUMENT_ME, sizeof(*ref_keyuse_steps)*new_size, MYF(0)))) { sel= 1.0; // As if no selectivity was computed goto exit; } memcpy(new_buf, ref_keyuse_steps, sizeof(*ref_keyuse_steps)*ref_keyuse_size); if (ref_keyuse_steps != ref_keyuse_steps_buf) my_free(ref_keyuse_steps); ref_keyuse_steps= (uint16*)new_buf; ref_keyuse_size= new_size; } ref_keyuse_steps[keyparts-2]= (uint16)(keyuse - prev_ref_keyuse); prev_ref_keyuse= keyuse; } } } } keyuse++; } /* If the field f from the table is equal to a field from one the earlier joined tables then the selectivity of the range conditions over the field f must be discounted. We need to discount selectivity only if we're using ref-based access method (and have sel!=1). If we use ALL/range/index_merge, then sel==1, and no need to discount. */ for (Field **f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) { if (!bitmap_is_set(read_set, field->field_index) || !field->next_equal_field) continue; for (Field *next_field= field->next_equal_field; next_field != field; next_field= next_field->next_equal_field) { if (!(next_field->table->map & rem_tables) && next_field->table != table) { if (field->cond_selectivity > 0) { sel/= field->cond_selectivity; set_if_smaller(sel, 1.0); } break; } } } /* We have now calculated a more exact 'records_out' taking more index costs into account. pos->records_out previously contained the smallest record count for all range or ref access, which should not be smaller than what we calculated above. */ records_out= pos->records_init * sel; set_if_smaller(records_out, pos->records_out); } sel= table_multi_eq_cond_selectivity(join, idx, s, rem_tables, keyparts, ref_keyuse_steps); records_out*= sel; /* Update sel to be relative pos->records_read as that is what some old code expects. Newer code should just use 'position->records_out' instead. */ if (pos->records_read == 0) sel= 1.0; else { sel= records_out / pos->records_read; DBUG_ASSERT(sel >= 0.0 && sel <= 1.00001); if (sel > 1.0) sel= 1.0; } exit: *new_records_out= records_out; if (ref_keyuse_steps != ref_keyuse_steps_buf) my_free(ref_keyuse_steps); return sel; } /* Check if the table is an EQ_REF or similar table and there is no cost to gain by moving it to a later stage. We call such a table a edge table (or hanging leaf) as it will read at most one row and will not add to the number of row combinations in the join. */ static inline enum_best_search check_if_edge_table(POSITION *pos, double pushdown_cond_selectivity) { if ((pos->type == JT_EQ_REF || (pos->type == JT_REF && pos->records_init == 1 && !pos->range_rowid_filter_info)) && pushdown_cond_selectivity >= 0.999) return SEARCH_FOUND_EDGE; return SEARCH_OK; } struct SORT_POSITION { JOIN_TAB **join_tab; POSITION *position; }; /* Sort SORT_POSITIONS according to expected number of rows found If number of combinations are the same sort according to join_tab order (same table order as used in the original SQL query) */ static int sort_positions(const void *a_, const void *b_) { const SORT_POSITION *a= static_cast<const SORT_POSITION*>(a_); const SORT_POSITION *b= static_cast<const SORT_POSITION*>(b_); int cmp; if ((cmp= compare_embedding_subqueries(*a->join_tab, *b->join_tab)) != 0) return cmp; if (a->position->records_read > b->position->records_read) return 1; if (a->position->records_read < b->position->records_read) return -1; return CMP_NUM(*a->join_tab, *b->join_tab); } /* Call best_access_path() for a set of tables and collect results @param join JOIN object @param trace_one_table Current optimizer_trace @param pos Pointer to remaining tables @param allowed_tables bitmap of allowed tables. On return set to the collected tables. @param store_poisition Points to where to store next found SORT_POSITION. Will be updated to next free position. @param stop_on_eq_ref Stop searching for more tables if we found an EQ_REF table. @return 0 Normal 1 Eq_ref table found (only if stop_on_eq_ref is used) join->next_sort_position will be update to next free position. */ static bool get_costs_for_tables(JOIN *join, table_map remaining_tables, uint idx, double record_count, Json_writer_object *trace_one_table, JOIN_TAB **pos, SORT_POSITION **store_position, table_map *allowed_tables, bool stop_on_eq_ref) { THD *thd= join->thd; POSITION *sort_position= join->next_sort_position; SORT_POSITION *sort_end= *store_position; JOIN_TAB *s; table_map found_tables= 0; bool found_eq_ref= 0; DBUG_ENTER("get_plans_for_tables"); table_map remaining_allowed_tables= (join->emb_sjm_nest ? (join->emb_sjm_nest->sj_inner_tables & ~join->const_table_map & remaining_tables): remaining_tables); s= *pos; do { table_map real_table_bit= s->table->map; if ((*allowed_tables & real_table_bit) && !(remaining_allowed_tables & s->dependent)) { #ifdef DBUG_ASSERT_EXISTS DBUG_ASSERT(!check_interleaving_with_nj(s)); restore_prev_nj_state(s); // Revert effect of check_... call #endif sort_end->join_tab= pos; sort_end->position= sort_position; bool hint_forces_jbuf= hint_table_state(join->thd, s->table, BNL_HINT_ENUM, false); bool disable_jbuf= (join->thd->variables.join_cache_level == 0) && !hint_forces_jbuf; Json_writer_object wrapper(thd); /* Find the best access method from 's' to the current partial plan */ best_access_path(join, s, remaining_tables, join->positions, idx, disable_jbuf, record_count, sort_position, sort_position + 1); found_tables|= s->table->map; sort_end++; sort_position+= 2; if (unlikely(stop_on_eq_ref) && sort_position[-2].type == JT_EQ_REF) { /* Found an eq_ref tables. Use this, ignoring the other tables */ found_eq_ref= 1; if (found_tables == s->table->map) break; // First table /* Store the found eq_ref table first in store_position */ sort_position-= 2; *allowed_tables= s->table->map; (*store_position)->join_tab= pos; (*store_position)->position= sort_position; (*store_position)++; join->next_sort_position[0]= sort_position[0]; join->next_sort_position[1]= sort_position[1]; join->next_sort_position+= 2; DBUG_RETURN(1); } } else { /* Verify that 'allowed_current_tables' was calculated correctly */ DBUG_ASSERT((remaining_tables & s->dependent) || !(remaining_tables & real_table_bit) || !(*allowed_tables & real_table_bit) || check_interleaving_with_nj(s)); } } while ((s= *++pos)); *allowed_tables= found_tables; *store_position= sort_end; join->next_sort_position= sort_position; DBUG_RETURN(found_eq_ref); } /* @brief Check if it is potentially possible to short-cut the JOIN execution due to ORDER BY ... LIMIT clause @detail It is possible when the join has "ORDER BY ... LIMIT n" clause, and the sort+limit operation is done right after the join operation (there's no grouping or DISTINCT in between). Then we can potentially build a join plan that enumerates rows in the ORDER BY order and so will be able to terminate as soon as it has produced #limit rows. Note that it is not a requirement that sort_by_table has an index that matches ORDER BY. If it doesn't have one, the optimizer will pass sort_by_table to filesort. Reading from sort_by_table won't use short-cutting but the rest of the join will. */ static bool join_limit_shortcut_is_applicable(const JOIN *join) { /* Any post-join operation like GROUP BY or DISTINCT or window functions means we cannot short-cut join execution */ if (!join->thd->variables.optimizer_join_limit_pref_ratio || !join->order || join->select_limit == HA_POS_ERROR || join->group_list || join->select_distinct || join->select_options & SELECT_BIG_RESULT || join->rollup.state != ROLLUP::STATE_NONE || join->select_lex->have_window_funcs() || join->select_lex->with_sum_func) { return false; } /* Cannot do short-cutting if (1) ORDER BY refers to more than one table or (2) the table it refers to cannot be first table in the join order */ if (!join->sort_by_table || // (1) join->sort_by_table->reginfo.join_tab->dependent) // (2) return false; Json_writer_object wrapper(join->thd); Json_writer_object trace(join->thd, "join_limit_shortcut_is_applicable"); trace.add("applicable", 1); /* It looks like we can short-cut limit due to join */ return true; } /* @brief Check if we could use an index-based access method to produce rows in the order for ORDER BY ... LIMIT. @detail This should do what test_if_skip_sort_order() does. We can't use that function directly, because: 1. We're at the join optimization stage and have not done query plan fix-ups done in get_best_combination() and co. 2. The code in test_if_skip_sort_order() does modify query plan structures, for example it may change the table's quick select. This is done even if it's called with no_changes=true parameter. @param access_method_changed OUT Whether the function changed the access method to get rows in desired order. @param new_access_cost OUT if access method changed: its cost. @return true - Can skip sorting false - Cannot skip sorting */ bool test_if_skip_sort_order_early(JOIN *join, bool *access_method_changed, double *new_access_cost) { const POSITION *pos= &join->best_positions[join->const_tables]; TABLE *table= pos->table->table; key_map usable_keys= table->keys_in_use_for_order_by; *access_method_changed= false; // Step #1: Find indexes that produce the required ordering. if (find_indexes_matching_order(join, table, join->order, &usable_keys)) return false; // Cannot skip sorting // Step #2: Check if the index we're using produces the needed ordering uint ref_key; if (pos->key) { // Mirror the (wrong) logic in test_if_skip_sort_order: if (pos->spl_plan || pos->type == JT_REF_OR_NULL) return false; // Use filesort ref_key= pos->key->key; } else { if (pos->table->quick) { if (pos->table->quick->get_type() == QUICK_SELECT_I::QS_TYPE_RANGE) ref_key= pos->table->quick->index; else ref_key= MAX_KEY; } else ref_key= MAX_KEY; } if (ref_key != MAX_KEY && usable_keys.is_set(ref_key)) { return true; // we're using an index that produces the required ordering. } /* Step #3: check if we can switch to using an index that would produce the ordering. (But don't actually switch, this will be done by test_if_skip_sort_order) */ int best_key= -1; uint UNINIT_VAR(best_key_parts); uint saved_best_key_parts= 0; int best_key_direction= 0; JOIN_TAB *tab= pos->table; ha_rows new_limit; double new_read_time; if (test_if_cheaper_ordering(/*in_join_optimizer */TRUE, tab, join->order, table, usable_keys, ref_key, join->select_limit, &best_key, &best_key_direction, &new_limit, &new_read_time, &best_key_parts, &saved_best_key_parts)) { // Ok found a way to skip sorting *access_method_changed= true; *new_access_cost= new_read_time; return true; } return false; } /* Compute the cost of join assuming we only need fraction of the output. */ double recompute_join_cost_with_limit(const JOIN *join, bool skip_sorting, double *first_table_cost, double fraction) { POSITION *pos= join->best_positions + join->const_tables; /* Generally, we assume that producing X% of output takes X% of the cost. best_extension_by_limited_search() subtracts COST_EPS from join->best_read, add it back. (Note: before 11.0, we subtracted COST_EPS here. In 11.0+, there's no need to do this) */ double partial_join_cost= join->best_read * fraction; if (skip_sorting) { /* First table produces rows in required order. Two options: A. first_table_cost=NULL means we use whatever access method the join optimizer has picked. Its cost was included in join->best_read and we've already took a fraction of it. B. first_table_cost!=NULL means we will need to switch to another access method, we have the cost to read rows to produce #LIMIT rows in join output. */ if (first_table_cost) { /* Subtract the remainder of the first table's cost we had in join->best_read. (Before 11.0, we also subtracted pos->records_read/TIME_FOR_COMPARE. In 11.0+, that time is already included in pos->read_time) */ partial_join_cost -= pos->read_time*fraction; DBUG_ASSERT(partial_join_cost >= 0.0); /* Add the cost of the new access method we've got: */ partial_join_cost= COST_ADD(partial_join_cost, *first_table_cost); } } else { DBUG_ASSERT(!first_table_cost); /* Cannot skip sorting. We read the first table entirely, then sort it. partial_join_cost includes pos->read_time*fraction. Add to it pos->read_time*(1-fraction) so we have the cost to read the entire first table. Do the same for costs of checking the WHERE. */ double extra_first_table_cost= pos->read_time * (1.0 - fraction); partial_join_cost= COST_ADD(partial_join_cost, extra_first_table_cost); } return partial_join_cost; } /* @brief Finalize building the join order which allows it to short-cut the join execution. @detail This is called after we have produced a join order that allows short- cutting. Here, we decide if it is cheaper to use this one or the original join order. */ POSITION *join_limit_shortcut_finalize_plan(JOIN *join, double *cost) { Json_writer_object wrapper(join->thd); Json_writer_object trace(join->thd, "join_limit_shortcut_choice"); double fraction= join->select_limit / join->join_record_count; trace.add("limit_fraction", fraction); /* Check which fraction of join output we need */ if (fraction >= 1.0) { trace.add("skip_adjustment", "no short-cutting"); return NULL; } /* Check if the first table's access method produces the required ordering. Possible options: 1. Yes: we can just take a fraction of the execution cost. 2A No: change the access method to one that does produce the required ordering, update the costs. 2B No: Need to pass the first table to filesort(). */ bool skip_sorting; bool access_method_changed; double new_access_cost; { Json_writer_array tmp(join->thd, "test_if_skip_sort_order_early"); skip_sorting= test_if_skip_sort_order_early(join, &access_method_changed, &new_access_cost); } trace.add("can_skip_filesort", skip_sorting); double cost_with_shortcut= recompute_join_cost_with_limit(join, skip_sorting, access_method_changed ? &new_access_cost : (double*)0, fraction); double risk_ratio= (double)join->thd->variables.optimizer_join_limit_pref_ratio; trace.add("full_join_cost", join->best_read); trace.add("risk_ratio", risk_ratio); trace.add("shortcut_join_cost", cost_with_shortcut); cost_with_shortcut *= risk_ratio; trace.add("shortcut_cost_with_risk", cost_with_shortcut); if (cost_with_shortcut < join->best_read) { trace.add("use_shortcut_cost", true); POSITION *pos= (POSITION*)memdup_root(join->thd->mem_root, join->best_positions, sizeof(POSITION)* (join->table_count + 1)); *cost= cost_with_shortcut; return pos; } trace.add("use_shortcut_cost", false); return NULL; } /* @brief If we're in Limit Optimization Mode, allow only join->sort_by_table as the first table in the join order */ static bool join_limit_shortcut_limits_tables(const JOIN *join, uint idx, table_map *map) { if (join->limit_optimization_mode && idx == join->const_tables) { *map= join->sort_by_table->map; return true; } return false; } /** Find a good, possibly optimal, query execution plan (QEP) by a possibly exhaustive search. The procedure searches for the optimal ordering of the query tables in set 'remaining_tables' of size N, and the corresponding optimal access paths to each table. The choice of a table order and an access path for each table constitutes a query execution plan (QEP) that fully specifies how to execute the query. The maximal size of the found plan is controlled by the parameter 'search_depth'. When search_depth == N, the resulting plan is complete and can be used directly as a QEP. If search_depth < N, the found plan consists of only some of the query tables. Such "partial" optimal plans are useful only as input to query optimization procedures, and cannot be used directly to execute a query. The algorithm begins with an empty partial plan stored in 'join->positions' and a set of N tables - 'remaining_tables'. Each step of the algorithm evaluates the cost of the partial plan extended by all access plans for each of the relations in 'remaining_tables', expands the current partial plan with the access plan that results in lowest cost of the expanded partial plan, and removes the corresponding relation from 'remaining_tables'. The algorithm continues until it either constructs a complete optimal plan, or constructs an optimal partial plan with size = search_depth. The final optimal plan is stored in 'join->best_positions'. The corresponding cost of the optimal plan is in 'join->best_read'. @note The procedure uses a recursive depth-first search where the depth of the recursion (and thus the exhaustiveness of the search) is controlled by the parameter 'search_depth'. @note The pseudocode below describes the algorithm of 'best_extension_by_limited_search'. The worst-case complexity of this algorithm is O(N*N^search_depth/search_depth). When serch_depth >= N, then the complexity of greedy_search is O(N!). @code procedure best_extension_by_limited_search( pplan in, // in, partial plan of tables-joined-so-far pplan_cost, // in, cost of pplan remaining_tables, // in, set of tables not referenced in pplan best_plan_so_far, // in/out, best plan found so far best_plan_so_far_cost,// in/out, cost of best_plan_so_far search_depth) // in, maximum size of the plans being considered { for each table T from remaining_tables { // Calculate the cost of using table T as above cost = complex-series-of-calculations; // Add the cost to the cost so far. pplan_cost+= cost; if (pplan_cost >= best_plan_so_far_cost) // pplan_cost already too great, stop search continue; pplan= expand plan by best_access_method; remaining_tables= remaining_tables - table T; if (remaining_tables is not an empty set and search_depth > 1) { best_extension_by_limited_search(pplan, pplan_cost, remaining_tables, best_plan_so_far, best_plan_so_far_cost, search_depth - 1); } else { best_plan_so_far_cost= pplan_cost; best_plan_so_far= pplan; } } } @endcode @note When 'best_extension_by_limited_search' is called for the first time, 'join->best_read' must be set to the largest possible value (e.g. DBL_MAX). The actual implementation provides a way to optionally use pruning heuristic to reduce the search space by skipping some partial plans. @note The parameter 'search_depth' provides control over the recursion depth, and thus the size of the resulting optimal plan. @param join pointer to the structure providing all context info for the query @param remaining_tables set of tables not included into the partial plan yet @param idx length of the partial QEP in 'join->positions'; since a depth-first search is used, also corresponds to the current depth of the search tree; also an index in the array 'join->best_ref'; @param record_count estimate for the number of records returned by the best partial plan @param read_time the cost of the best partial plan @param search_depth maximum depth of the recursion and thus size of the found optimal plan (0 < search_depth <= join->tables+1). (values: 0 = EXHAUSTIVE, 1 = PRUNE_BY_TIME_OR_ROWS) @param use_cond_selectivity specifies how the selectivity of the conditions pushed to a table should be taken into account @retval enum_best_search::SEARCH_OK All fine @retval enum_best_search::SEARCH_FOUND_EDGE All remaining tables are edge tables @retval enum_best_search::SEARCH_ABORT Killed by user @retval enum_best_search::SEARCH_ERROR Fatal error */ static enum_best_search best_extension_by_limited_search(JOIN *join, table_map remaining_tables, uint idx, double record_count, double read_time, uint search_depth, uint use_cond_selectivity, table_map *processed_eq_ref_tables) { THD *thd= join->thd; /* 'join' is a partial plan with lower cost than the best plan so far, so continue expanding it further with the tables in 'remaining_tables'. */ JOIN_TAB *s; double best_record_count= DBL_MAX; double best_read_time= DBL_MAX; enum_best_search best_res; uint tables_left= join->table_count - idx, found_tables; uint accepted_tables __attribute__((unused)); table_map found_eq_ref_tables= 0, used_eq_ref_table= 0; table_map allowed_tables, allowed_current_tables; SORT_POSITION *sort= (SORT_POSITION*) alloca(sizeof(SORT_POSITION)*tables_left); SORT_POSITION *sort_end; DBUG_ENTER("best_extension_by_limited_search"); DBUG_EXECUTE_IF("show_explain_probe_best_ext_lim_search", if (dbug_user_var_equals_int(thd, "show_explain_probe_select_id", join->select_lex->select_number)) dbug_serve_apcs(thd, 1); ); if (unlikely(thd->check_killed())) // Abort DBUG_RETURN(SEARCH_ABORT); DBUG_EXECUTE("opt", print_plan(join, idx, record_count, read_time, read_time, "part_plan");); status_var_increment(thd->status_var.optimizer_join_prefixes_check_calls); if (join->emb_sjm_nest) { /* If we are searching for the execution plan of a materialized semi-join nest then allowed_tables contains bits only for the tables from this nest. */ allowed_tables= (join->emb_sjm_nest->sj_inner_tables & remaining_tables); allowed_current_tables= join->get_allowed_nj_tables(idx) & remaining_tables; } else { /* allowed_tables is used to check if there are tables left that can improve a key search and to see if there are more tables to add in next iteration. allowed_current_tables tells us which tables we can add to the current plan at this stage. */ allowed_tables= remaining_tables; allowed_current_tables= join->get_allowed_nj_tables(idx) & remaining_tables; table_map sort_table; if (join_limit_shortcut_limits_tables(join, idx, &sort_table)) allowed_current_tables= sort_table; } DBUG_ASSERT(allowed_tables & remaining_tables); sort_end= sort; { Json_writer_object trace_one_table(thd); JOIN_TAB **best_ref= join->best_ref + idx; if (unlikely(thd->trace_started())) trace_plan_prefix(&trace_one_table, join, idx, remaining_tables); Json_writer_array arr(thd, "get_costs_for_tables"); if (idx > join->const_tables && join->prune_level >= 2 && join->positions[idx-1].type == JT_EQ_REF && (join->eq_ref_tables & allowed_current_tables)) { /* Previous table was an EQ REF table, only add other possible EQ_REF tables to the chain, stop after first one is found. */ table_map table_map= join->eq_ref_tables & allowed_current_tables; if (get_costs_for_tables(join, remaining_tables, idx, record_count, &trace_one_table, best_ref, &sort_end, &table_map, 1)) used_eq_ref_table= (*sort->join_tab)->table->map; else { /* We didn't find another EQ_REF table, add remaining tables */ if ((table_map= allowed_current_tables & ~table_map)) get_costs_for_tables(join, remaining_tables, idx, record_count, &trace_one_table, best_ref, &sort_end, &table_map, 0); } } else { table_map table_map= allowed_current_tables; get_costs_for_tables(join, remaining_tables, idx, record_count, &trace_one_table, best_ref, &sort_end, &table_map, 0); } found_tables= (uint) (sort_end - sort); DBUG_ASSERT(found_tables > 0); /* Sort tables in ascending order of generated row combinations */ if (found_tables > 1) my_qsort(sort, found_tables, sizeof(SORT_POSITION), sort_positions); } DBUG_ASSERT(join->next_sort_position <= join->sort_positions + join->sort_space); accepted_tables= 0; double min_rec_count= DBL_MAX; double min_rec_count_read_time= DBL_MAX; double min_cost= DBL_MAX; double min_cost_record_count= DBL_MAX; for (SORT_POSITION *pos= sort ; pos < sort_end ; pos++) { s= *pos->join_tab; if (!(found_eq_ref_tables & s->table->map) && !check_interleaving_with_nj(s)) { table_map real_table_bit= s->table->map; double current_record_count, current_read_time, original_record_count; double partial_join_cardinality; POSITION *position= join->positions + idx, *loose_scan_pos; double pushdown_cond_selectivity; Json_writer_object trace_one_table(thd); if (unlikely(thd->trace_started())) { trace_plan_prefix(&trace_one_table, join, idx, remaining_tables); trace_one_table.add_table_name(s); } accepted_tables++; *position= *pos->position; // Get stored result loose_scan_pos= pos->position+1; /* Compute the cost of the new plan extended with 's' */ current_record_count= COST_MULT(record_count, position->records_out); current_read_time= COST_ADD(read_time, position->read_time); if (unlikely(trace_one_table.trace_started())) { trace_one_table. add("rows_for_plan", current_record_count). add("cost_for_plan", current_read_time); } original_record_count= current_record_count; optimize_semi_joins(join, remaining_tables, idx, ¤t_record_count, ¤t_read_time, loose_scan_pos); if (position->sj_strategy != SJ_OPT_NONE) { /* Adjust records_out and current_record_count after semi join */ double ratio= current_record_count / original_record_count; if (ratio < 1.0) position->records_out*= ratio; if (unlikely(trace_one_table.trace_started())) { trace_one_table. add("sj_rows_out", position->records_out). add("sj_rows_for_plan", current_record_count). add("sj_filtered", safe_filtered(position->records_out, position->records_init)); } } /* Expand only partial plans with lower cost than the best QEP so far */ if (current_read_time + COST_EPS >= join->best_read) { DBUG_EXECUTE("opt", print_plan(join, idx+1, current_record_count, read_time, current_read_time, "prune_by_cost");); trace_one_table .add("pruned_by_cost", true) .add("current_cost", current_read_time) .add("best_cost", join->best_read); restore_prev_nj_state(s); restore_prev_sj_state(remaining_tables, s, idx); continue; } /* Prune some less promising partial plans. This heuristic may miss the optimal QEPs, thus it results in a non-exhaustive search. */ if (join->prune_level >= 1) { // Collect the members with min_cost and min_read_time. bool min_rec_hit= false; bool min_cost_hit= false; if (join->extra_heuristic_pruning && (!(position->key_dependent & allowed_tables) || position->records_read < 2.0)) { if (current_record_count < min_rec_count) { min_rec_count= current_record_count; min_rec_count_read_time= current_read_time; min_rec_hit= true; } if (current_read_time < min_cost) { min_cost_record_count= current_record_count; min_cost= current_read_time; min_cost_hit= true; } } if (best_record_count > current_record_count || best_read_time > current_read_time || (idx == join->const_tables && // 's' is the first table in the QEP s->table == join->sort_by_table)) { /* Store the current record count and cost as the best possible cost at this level if the following holds: - It's the lowest record number and cost so far - There is no remaining table that could improve index usage or we found an EQ_REF or REF key with less than 2 matching records (good enough). */ if (best_record_count >= current_record_count && best_read_time >= current_read_time && (!(position->key_dependent & join->allowed_tables) || position->records_read < 2.0)) { best_record_count= current_record_count; best_read_time= current_read_time; } } else { /* Typically, we get here if: best_record_count < current_record_count && best_read_time < current_read_time That is, both record_count and read_time are worse than the best_ ones. This plan doesn't look promising, prune it away. */ DBUG_EXECUTE("opt", print_plan(join, idx+1, current_record_count, read_time, current_read_time, "pruned_by_heuristic");); trace_one_table.add("pruned_by_heuristic", true); restore_prev_nj_state(s); restore_prev_sj_state(remaining_tables, s, idx); continue; } const char* prune_reason= NULL; if (!min_rec_hit && current_record_count >= min_rec_count && current_read_time >= min_rec_count_read_time) prune_reason= "min_record_count"; if (!min_cost_hit && current_record_count >= min_cost_record_count && current_read_time >= min_cost) prune_reason= "min_read_time"; if (prune_reason) { trace_one_table.add("pruned_by_heuristic", prune_reason); restore_prev_nj_state(s); restore_prev_sj_state(remaining_tables, s, idx); continue; } } pushdown_cond_selectivity= 1.0; /* TODO: When a semi-join strategy is applied (sj_strategy!=SJ_OPT_NONE), we should account for selectivity from table_after_join_selectivity(). (Condition filtering is performed before the semi-join removes some fanout so this might require moving the code around) */ if (use_cond_selectivity > 1 && position->sj_strategy == SJ_OPT_NONE) { pushdown_cond_selectivity= table_after_join_selectivity(join, idx, s, remaining_tables & ~real_table_bit, &position->records_out); if (unlikely(trace_one_table.trace_started()) && pushdown_cond_selectivity != 1.0) trace_one_table. add("pushdown_cond_selectivity", pushdown_cond_selectivity). add("filtered", safe_filtered(position->records_out, position->records_init)). add("rows_out", position->records_out); } join->positions[idx].cond_selectivity= pushdown_cond_selectivity; partial_join_cardinality= record_count * position->records_out; join->positions[idx].partial_join_cardinality= partial_join_cardinality; if (unlikely(thd->trace_started()) && pushdown_cond_selectivity < 1.0 && partial_join_cardinality < current_record_count) trace_one_table .add("selectivity", pushdown_cond_selectivity) .add("estimated_join_cardinality", partial_join_cardinality); if ((search_depth > 1) && (remaining_tables & ~real_table_bit) & allowed_tables) { /* Recursively expand the current partial plan */ Json_writer_array trace_rest(thd, "rest_of_plan"); swap_variables(JOIN_TAB*, join->best_ref[idx], *pos->join_tab); best_res= best_extension_by_limited_search(join, remaining_tables & ~real_table_bit, idx + 1, partial_join_cardinality, current_read_time, search_depth - 1, use_cond_selectivity, &found_eq_ref_tables); swap_variables(JOIN_TAB*, join->best_ref[idx], *pos->join_tab); if ((int) best_res < (int) SEARCH_OK) goto end; // Return best_res if (best_res == SEARCH_FOUND_EDGE && check_if_edge_table(join->positions+ idx, pushdown_cond_selectivity) != SEARCH_FOUND_EDGE) best_res= SEARCH_OK; } else { /* 'join' is either the best partial QEP with 'search_depth' relations, or the best complete QEP so far, whichever is smaller. */ if (join->sort_by_table && join->sort_by_table != join->positions[join->const_tables].table->table) { /* We may have to make a temp table, note that this is only a heuristic since we cannot know for sure at this point if we we are going to use addon fields or to have flush sorting to disk. We also don't know the temporary table will be in memory or disk. The following calculation takes a middle ground where assume we can sort the keys in memory but have to use a disk based temporary table to retrieve the rows. This cost is probably much bigger than it has to be... */ double sort_cost; sort_cost= (get_qsort_sort_cost((ha_rows)current_record_count,0) + current_record_count * DISK_TEMPTABLE_LOOKUP_COST(thd)); trace_one_table.add("cost_for_sorting", sort_cost); current_read_time= COST_ADD(current_read_time, sort_cost); } if (current_read_time < join->best_read) { memcpy((uchar*) join->best_positions, (uchar*) join->positions, sizeof(POSITION) * (idx + 1)); join->join_record_count= partial_join_cardinality; join->best_read= current_read_time; } DBUG_EXECUTE("opt", print_plan(join, idx+1, current_record_count, read_time, current_read_time, "full_plan");); best_res= check_if_edge_table(join->positions + idx, pushdown_cond_selectivity); } restore_prev_nj_state(s); restore_prev_sj_state(remaining_tables, s, idx); if (best_res == SEARCH_FOUND_EDGE) { if (pos+1 < sort_end) // If not last table trace_one_table.add("pruned_by_hanging_leaf", true); goto end; } } } DBUG_ASSERT(accepted_tables > 0); best_res= SEARCH_OK; end: join->next_sort_position-= found_tables*2; if (used_eq_ref_table) *processed_eq_ref_tables|= used_eq_ref_table | found_eq_ref_tables; else *processed_eq_ref_tables= 0; DBUG_RETURN(best_res); } /** Find how much space the previous read not const tables takes in cache. */ void JOIN_TAB::calc_used_field_length(bool max_fl) { uint null_fields,blobs,fields; ulong rec_length; Field **f_ptr,*field; uint uneven_bit_fields; MY_BITMAP *read_set= table->read_set; uneven_bit_fields= null_fields= blobs= fields= rec_length=0; for (f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) { if (bitmap_is_set(read_set, field->field_index)) { uint flags=field->flags; fields++; rec_length+=field->pack_length(); if (flags & BLOB_FLAG) blobs++; if (!(flags & NOT_NULL_FLAG)) null_fields++; if (field->type() == MYSQL_TYPE_BIT && ((Field_bit*)field)->bit_len) uneven_bit_fields++; } } if (null_fields || uneven_bit_fields) rec_length+=(table->s->null_fields+7)/8; if (table->maybe_null) rec_length+=sizeof(my_bool); /* Take into account that DuplicateElimination may need to store rowid */ uint rowid_add_size= 0; if (keep_current_rowid) { rowid_add_size= table->file->ref_length; rec_length += rowid_add_size; fields++; } if (max_fl) { // TODO: to improve this estimate for max expected length if (blobs) { ulong blob_length= table->file->stats.mean_rec_length; if (ULONG_MAX - rec_length > blob_length) rec_length+= blob_length; else rec_length= ULONG_MAX; } max_used_fieldlength= rec_length; } else if (table->file->stats.mean_rec_length) set_if_smaller(rec_length, table->file->stats.mean_rec_length + rowid_add_size); used_fields=fields; used_fieldlength=rec_length; used_blobs=blobs; used_null_fields= null_fields; used_uneven_bit_fields= uneven_bit_fields; } /* @brief Extract pushdown conditions for a table scan @details This functions extracts pushdown conditions usable when this table is scanned. The conditions are extracted either from WHERE or from ON expressions. The conditions are attached to the field cache_select of this table. @note Currently the extracted conditions are used only by BNL and BNLH join. algorithms. @retval 0 on success 1 otherwise */ int JOIN_TAB::make_scan_filter() { COND *tmp; DBUG_ENTER("make_scan_filter"); Item *cond= is_inner_table_of_outer_join() ? *get_first_inner_table()->on_expr_ref : join->conds; if (cond) { if ((tmp= make_cond_for_table(join->thd, cond, join->const_table_map | table->map, table->map, -1, FALSE, TRUE))) { DBUG_EXECUTE("where",print_where(tmp,"cache", QT_ORDINARY);); if (!(cache_select= (SQL_SELECT*) join->thd->memdup((uchar*) select, sizeof(SQL_SELECT)))) DBUG_RETURN(1); cache_select->cond= tmp; cache_select->read_tables=join->const_table_map; } else if (join->thd->is_error()) DBUG_RETURN(1); } DBUG_RETURN(0); } /** @brief Check whether hash join algorithm can be used to join this table @details This function finds out whether the ref items that have been chosen by the planner to access this table can be used for hash join algorithms. The answer depends on a certain property of the fields of the joined tables on which the hash join key is built. @note At present the function is supposed to be called only after the function get_best_combination has been called. @retval TRUE it's possible to use hash join to join this table @retval FALSE otherwise */ bool JOIN_TAB::hash_join_is_possible() { if (type != JT_REF && type != JT_EQ_REF) return FALSE; if (!is_ref_for_hash_join()) { KEY *keyinfo= table->key_info + ref.key; return keyinfo->key_part[0].field->hash_join_is_possible(); } return TRUE; } /** @brief Check whether a KEYUSE can be really used for access this join table @param join Join structure with the best join order for which the check is performed @param keyuse Evaluated KEYUSE structure @details This function is supposed to be used after the best execution plan have been already chosen and the JOIN_TAB array for the best join order been already set. For a given KEYUSE to access this JOIN_TAB in the best execution plan the function checks whether it really can be used. The function first performs the check with access_from_tables_is_allowed(). If it succeeds it checks whether the keyuse->val does not use some fields of a materialized semijoin nest that cannot be used to build keys to access outer tables. Such KEYUSEs exists for the query like this: select * from ot where ot.c in (select it1.c from it1, it2 where it1.c=f(it2.c)) Here we have two KEYUSEs to access table ot: with val=it1.c and val=f(it2.c). However if the subquery was materialized the second KEYUSE cannot be employed to access ot. @retval true the given keyuse can be used for ref access of this JOIN_TAB @retval false otherwise */ bool JOIN_TAB::keyuse_is_valid_for_access_in_chosen_plan(JOIN *join, KEYUSE *keyuse) { if (!access_from_tables_is_allowed(keyuse->used_tables, join->sjm_lookup_tables)) return false; if (join->sjm_scan_tables & table->map) return true; table_map keyuse_sjm_scan_tables= keyuse->used_tables & join->sjm_scan_tables; if (!keyuse_sjm_scan_tables) return true; uint sjm_tab_nr= 0; while (!(keyuse_sjm_scan_tables & table_map(1) << sjm_tab_nr)) sjm_tab_nr++; JOIN_TAB *sjm_tab= join->map2table[sjm_tab_nr]; TABLE_LIST *emb_sj_nest= sjm_tab->emb_sj_nest; if (!(emb_sj_nest->sj_mat_info && emb_sj_nest->sj_mat_info->is_used && emb_sj_nest->sj_mat_info->is_sj_scan)) return true; st_select_lex *sjm_sel= emb_sj_nest->sj_subq_pred->unit->first_select(); for (uint i= 0; i < sjm_sel->item_list.elements; i++) { DBUG_ASSERT(sjm_sel->ref_pointer_array[i]->real_item()->type() == Item::FIELD_ITEM); if (keyuse->val->real_item()->type() == Item::FIELD_ITEM) { Field *field = ((Item_field*)sjm_sel->ref_pointer_array[i]->real_item())->field; if (field->eq(((Item_field*)keyuse->val->real_item())->field)) return true; } } return false; } static uint cache_record_length(JOIN *join,uint idx) { uint length=0; JOIN_TAB **pos,**end; for (pos=join->best_ref+join->const_tables,end=join->best_ref+idx ; pos != end ; pos++) { JOIN_TAB *join_tab= *pos; length+= join_tab->get_used_fieldlength(); } return length; } /* Estimate the number of engine ha_index_read_calls for EQ_REF tables when taking into account the one-row-cache in join_read_always_key() SYNOPSIS @param position All previous tables best_access_path() information. @param idx Number of (previous) tables in positions. @param record_count Number of incoming record combinations @param found_ref Bitmap of tables that is used to construct the key used with the index read. @return # The number of estimated calls that cannot be cached by the the one-row-cache. In other words, number of expected calls to engine ha_read_read_map(). Between 1 and record_count or 0 if record_count == 0 DESCRIPTION The one-row-cache gives a great benefit when there are multiple consecutive calls to ha_index_read() with the same key. In this case we can skip calling the engine (and in the future also skip to check the key condition), which can notably increase the performance. Assuming most of the rows are cached, there is no notable saving to be made trying to calculate the total number of distinct key values that will be used. The performance of a ha_index_read_call() is about the same even if we repeatedly read the same set of rows. This code works by calculating the number of identical key sequences found in the record stream. The number of expected distinct calls can then be calculated as records_count / sequences. Some things to note: - record_count == PRODUCT(records_out) over all tables[0...idx-1] - position->prev_record_reads contains the number of identical sequences found for previous EQ_REF tables. Assume a join prefix of t1,t2,t3,t4 and t4 is an EQ_REF table. We have the following combinations that we have to consider: ====== 1) No JOIN_CACHE usage, tables depend only on one previous table Row combinations are generated as: - for all rows in t1 - for all rows in t2 - for all rows in t3 or t1.1,t2.1,t3.1, t1.1,t2.1,t3.2, t1.1,t2.1,t3.3... # Only t3 row changes (until no more rows in t3., ie t3.records_out times) t1.1,t2.2,t3.1, t1.1,t2.2,t3.2, t1.1,t2.2,t3.3... # t2.2 read (above repeated until no more rows in t2 and t3) t1.2,t2.1,t3.1, t1.2,t2.1,t3.2, t1.2,t2.1,t3.3... # t1.2 read If t4 is an EQ_REF table that is depending of one of the previous tables, the number of identical keys can be calculated as the multiplication of records_out of the tables in between the t4 and its first dependency. Let's consider cases where t4 depends on different previous tables: WHERE t4.a=t3.a no caching as t3 can change for each row engine_calls: record_count WHERE t4.a=t2.a t4 is not depending on t3. The number of repeated rows are: t1.1,t2.1,t3.1 to t1.1,t2.1,t3.last # t3.records_out rows t1.1,t2.2,t3.1 to t1.1,t2.2,t3.last # t3.records_out rows ... t1.2,t2.1,t3.1 to t1.2,t2.1,t3.last ... t1.last,t2.last.t3.1 to t1.last,t2.last.1,t3.last For each combination of t1 and t2 there are t3.records_out repeated rows with equal key value engine_calls: record_count / t3.records_out calls = t1.records_out * t2.records_out WHERE t4.a=t1.a The repeated sequences: t1.1,t2.1,t3.1 to t1.1,t2.last,t3.last t1.2,t2.1,t3.1 to t2.1,t2.last,t3.last repeated rows: t2.records_out * t3.records_out engine_calls: record_count/repeated_rows = t1.records_out If t4 depends on a table that uses EQ_REF access, we can multiply that table's repeated_rows with current table's repeated_rows to take that into account. ===== 2) Keys depending on multiple tables In this case we have to stop searching after we find the first table we depend upon. We have to also disregard the number of repeated rows for the found table. This can be seen from (assuming tables t1...t6): WHERE t6.a=t4.a and t6.a=t3.a and t4.a= t2.a - Here t4 is not depending on t3 (and thus there is a t3.records_out identical keys for t4). However t6 key will change for each t3 row and t6 cannot thus use t3.identical_keys WHERE t4.key_part1=t1.a and t4.key_part2= t3.a As t4.key_part2 will change for every row, one-row-cache will not be hit WHERE t4.key_part1=t1.a and t4.key_part2= t2.a t4.key will change when t1 or t2 changes This is the same case as above for WHERE t4.a = t2.a engine_calls: record_count / t3.records_out calls ===== 3) JOIN_CACHE is used If any table is using join_cache as this changes the row combinations seen by following tables. Using join cache for a table T# will have T# rows repeated for the next table as many times there are combinations in the cache. The the cache will re-read and the operations repeats 'refill-1' number of times. Table rows from table just before T# will come in 'random order', from the point of the next tables. Assuming t3 is using a cache, t4 will see the rows coming in the following order: t1.1,t2.1,t3.1, t1.1,t2.2,t3.1, t1.1,t2.3,t3.1... (t3.1 repeated 't2.records_out' times) t1.2,t2.1,t3.1, t1.2,t2.2,t3.1, t1.2,t2.3,t3.1... (Next row in t1 used) t1.1,t2.1,t3.2, t1.1,t2.2,t3.2, t1.1,t2.3,t3.2... (Restarting all t1 & t2 combinations for t3.2) WHERE t4.a=t3.a - There is a repeated sequence of t3.records_out rows for each t1,t2 row combination. engine_calls= record_count / t3.records_out WHERE t4.a=t2.a t2 changes for each row engine_calls= record_count WHERE t4.a=t1.a repeated rows= t2.records_out engine_calls= record_count / t2.records_out A refill of the join cache will restart the row sequences (we have 'refill' more sequences), so we will have to do 'refill' times more engine read calls. ===== Expectations of the accuracy of the return value - The value is always between 1 and record_count - The returned value should almost always larger than the true number of engine calls. - Assuming that every row has different values for all other columns for echo unique key value and record_count is accurate: - If a table is depending on multiple tables, the return value may be notable larger than real value. - If there is no join cache the value should be exact. - If there is a join cache, but no refills calculated or done then the value should be exact. - If there was more join_cache refills than was calculated, the value may be slightly to low. - If the number of refills is equal or less than was calculated the value should be larger than the expected engine read calls. The more refills, the less exact the number will be. */ static double prev_record_reads(const POSITION *position, uint idx, table_map found_ref, double record_count, double *identical_keys) { double found= 1.0; const POSITION *pos_end= position - 1; const POSITION *cur_pos= position + idx; /* Safety against const tables */ if (unlikely(!found_ref)) goto end; for (const POSITION *pos= cur_pos-1; pos != pos_end; pos--) { if (found_ref & pos->table->table->map) { /* Found a table we depend on */ found_ref= ~pos->table->table->map; if (!found_ref) { /* No more dependencies. We can use the cached values to improve things a bit */ if (pos->type == JT_EQ_REF) found= COST_MULT(found, pos->identical_keys); else if (pos->use_join_buffer) found= COST_MULT(found, pos->loops / pos->refills); } break; } if (unlikely(pos->use_join_buffer)) { /* Each refill can change the cached key */ found/= pos->refills; } else { /* We are not depending on the current table. There are 'records_out' rows with identical rows value for our depending tables. */ found= COST_MULT(found, pos->records_out); } } /* In most case found should <= record_count. However if there was a reduction of rows (records_out < 1) before the referencing table then found could be >= record_count. To get resonable numbers, we limit prev_record_read to be between 1.0 and record_count as we have to always do at least one read anyway. */ end: if (unlikely(found > record_count)) found= record_count; if (unlikely(found <= 1.0)) found= 1.0; *identical_keys= found; return record_count / found; } /* Enumerate join tabs in breadth-first fashion, including const tables. */ static JOIN_TAB *next_breadth_first_tab(JOIN_TAB *first_top_tab, uint n_top_tabs_count, JOIN_TAB *tab) { /* tab->join == NULL means that we're performing JOIN::cleanup() after a raised error: on EOM, or on an attempt to create a temporary table with a column of a non allowed data type, such as SYS_REFCURSOR. */ DBUG_ASSERT(tab->join || current_thd->is_error()); if (tab->join) n_top_tabs_count += tab->join->aggr_tables; if (!tab->bush_root_tab) { /* We're at top level. Get the next top-level tab */ tab++; if (tab < first_top_tab + n_top_tabs_count) return tab; /* No more top-level tabs. Switch to enumerating SJM nest children */ tab= first_top_tab; } else { /* We're inside of an SJM nest */ if (!tab->last_leaf_in_bush) { /* There's one more table in the nest, return it. */ return ++tab; } else { /* There are no more tables in this nest. Get out of it and then we'll proceed to the next nest. */ tab= tab->bush_root_tab + 1; } } /* Ok, "tab" points to a top-level table, and we need to find the next SJM nest and enter it. */ for (; tab < first_top_tab + n_top_tabs_count; tab++) { if (tab->bush_children) return tab->bush_children->start; } return NULL; } /* Enumerate JOIN_TABs in "EXPLAIN order". This order - const tabs are included - we enumerate "optimization tabs". - */ JOIN_TAB *first_explain_order_tab(JOIN* join) { JOIN_TAB* tab; tab= join->join_tab; if (!tab) return NULL; /* Can happen when the tables were optimized away */ return (tab->bush_children) ? tab->bush_children->start : tab; } JOIN_TAB *next_explain_order_tab(JOIN* join, JOIN_TAB* tab) { /* If we're inside SJM nest and have reached its end, get out */ if (tab->last_leaf_in_bush) return tab->bush_root_tab; /* Move to next tab in the array we're traversing */ tab++; if (tab == join->join_tab + join->top_join_tab_count) return NULL; /* Outside SJM nest and reached EOF */ if (tab->bush_children) return tab->bush_children->start; return tab; } JOIN_TAB *first_top_level_tab(JOIN *join, enum enum_with_const_tables const_tbls) { JOIN_TAB *tab= join->join_tab; if (const_tbls == WITHOUT_CONST_TABLES) { if (join->const_tables == join->table_count || !tab) return NULL; tab += join->const_tables; } return tab; } JOIN_TAB *next_top_level_tab(JOIN *join, JOIN_TAB *tab) { tab= next_breadth_first_tab(join->first_breadth_first_tab(), join->top_join_tab_count, tab); if (tab && tab->bush_root_tab) tab= NULL; return tab; } JOIN_TAB *first_linear_tab(JOIN *join, enum enum_with_bush_roots include_bush_roots, enum enum_with_const_tables const_tbls) { JOIN_TAB *first= join->join_tab; if (!first) return NULL; if (const_tbls == WITHOUT_CONST_TABLES) first+= join->const_tables; if (first >= join->join_tab + join->top_join_tab_count) return NULL; /* All are const tables */ if (first->bush_children && include_bush_roots == WITHOUT_BUSH_ROOTS) { /* This JOIN_TAB is a SJM nest; Start from first table in nest */ return first->bush_children->start; } return first; } /* A helper function to loop over all join's join_tab in sequential fashion DESCRIPTION Depending on include_bush_roots parameter, JOIN_TABs that represent SJM-scan/lookups are either returned or omitted. SJM-Bush children are returned right after (or in place of) their container join tab (TODO: does anybody depend on this? A: make_join_readinfo() seems to) For example, if we have this structure: ot1--ot2--sjm1----------------ot3-... | +--it1--it2--it3 calls to next_linear_tab( include_bush_roots=TRUE) will return: ot1 ot2 sjm1 it1 it2 it3 ot3 ... while calls to next_linear_tab( include_bush_roots=FALSE) will return: ot1 ot2 it1 it2 it3 ot3 ... (note that sjm1 won't be returned). */ JOIN_TAB *next_linear_tab(JOIN* join, JOIN_TAB* tab, enum enum_with_bush_roots include_bush_roots) { if (include_bush_roots == WITH_BUSH_ROOTS && tab->bush_children) { /* This JOIN_TAB is a SJM nest; Start from first table in nest */ return tab->bush_children->start; } DBUG_ASSERT(!tab->last_leaf_in_bush || tab->bush_root_tab); if (tab->bush_root_tab) /* Are we inside an SJM nest */ { /* Inside SJM nest */ if (!tab->last_leaf_in_bush) return tab+1; /* Return next in nest */ /* Continue from the sjm on the top level */ tab= tab->bush_root_tab; } /* If no more JOIN_TAB's on the top level */ if (++tab >= join->join_tab + join->exec_join_tab_cnt() + join->aggr_tables) return NULL; if (include_bush_roots == WITHOUT_BUSH_ROOTS && tab->bush_children) { /* This JOIN_TAB is a SJM nest; Start from first table in nest */ tab= tab->bush_children->start; } return tab; } /* Start to iterate over all join tables in bush-children-first order, excluding the const tables (see next_depth_first_tab() comment for details) */ JOIN_TAB *first_depth_first_tab(JOIN* join) { JOIN_TAB* tab; /* This means we're starting the enumeration */ if (join->const_tables == join->top_join_tab_count || !join->join_tab) return NULL; tab= join->join_tab + join->const_tables; return (tab->bush_children) ? tab->bush_children->start : tab; } /* A helper function to iterate over all join tables in bush-children-first order DESCRIPTION For example, for this join plan ot1--ot2--sjm1------------ot3-... | | it1--it2--it3 call to first_depth_first_tab() will return ot1, and subsequent calls to next_depth_first_tab() will return: ot2 it1 it2 it3 sjm ot3 ... */ JOIN_TAB *next_depth_first_tab(JOIN* join, JOIN_TAB* tab) { /* If we're inside SJM nest and have reached its end, get out */ if (tab->last_leaf_in_bush) return tab->bush_root_tab; /* Move to next tab in the array we're traversing */ tab++; if (tab == join->join_tab +join->top_join_tab_count) return NULL; /* Outside SJM nest and reached EOF */ if (tab->bush_children) return tab->bush_children->start; return tab; } bool JOIN::check_two_phase_optimization(THD *thd) { if (check_for_splittable_materialized()) return true; return false; } bool JOIN::inject_cond_into_where(Item *injected_cond) { Item *where_item= injected_cond; List<Item> *and_args= NULL; if (conds && conds->type() == Item::COND_ITEM && ((Item_cond*) conds)->functype() == Item_func::COND_AND_FUNC) { and_args= ((Item_cond*) conds)->argument_list(); if (cond_equal) and_args->disjoin((List<Item> *) &cond_equal->current_level); } where_item= and_items(thd, conds, where_item); if (where_item->fix_fields_if_needed(thd, 0)) return true; thd->change_item_tree(&select_lex->where, where_item); select_lex->where->top_level_item(); conds= select_lex->where; if (and_args && cond_equal) { and_args= ((Item_cond*) conds)->argument_list(); List_iterator<Item_equal> li(cond_equal->current_level); Item_equal *elem; while ((elem= li++)) { and_args->push_back(elem, thd->mem_root); } } return false; } static Item * const null_ptr= NULL; /* Set up join struct according to the picked join order in SYNOPSIS get_best_combination() join The join to process (the picked join order is mainly in join->best_positions) DESCRIPTION Setup join structures according the picked join order - finalize semi-join strategy choices (see fix_semijoin_strategies_for_picked_join_order) - create join->join_tab array and put there the JOIN_TABs in the join order - create data structures describing ref access methods. NOTE In this function we switch from pre-join-optimization JOIN_TABs to post-join-optimization JOIN_TABs. This is achieved by copying the entire JOIN_TAB objects. RETURN FALSE OK TRUE Out of memory */ bool JOIN::get_best_combination() { uint tablenr; table_map used_tables; JOIN_TAB *j; KEYUSE *keyuse; JOIN_TAB *sjm_nest_end= NULL; JOIN_TAB *sjm_nest_root= NULL; DBUG_ENTER("get_best_combination"); /* Additional plan nodes for postjoin tmp tables: 1? + // For GROUP BY 1? + // For DISTINCT 1? + // For aggregation functions aggregated in outer query // when used with distinct 1? + // For ORDER BY 1? // buffer result Up to 2 tmp tables are actually used, but it's hard to tell exact number at this stage. */ uint aggr_tables= (group_list ? 1 : 0) + (select_distinct ? (tmp_table_param.using_outer_summary_function ? 2 : 1) : 0) + (order ? 1 : 0) + (select_options & (SELECT_BIG_RESULT | OPTION_BUFFER_RESULT) ? 1 : 0) ; if (aggr_tables == 0) aggr_tables= 1; /* For group by pushdown */ if (select_lex->window_specs.elements) aggr_tables++; if (aggr_tables > 2) aggr_tables= 2; full_join=0; hash_join= FALSE; fix_semijoin_strategies_for_picked_join_order(this); top_join_tab_count= get_number_of_tables_at_top_level(this); #ifndef DBUG_OFF dbug_join_tab_array_size= top_join_tab_count + aggr_tables; #endif /* NOTE: The above computation of aggr_tables can produce wrong result because some of the variables it uses may change their values after we leave this function. Known examples: - Dangerous: using_outer_summary_function=false at this point. Added DBUG_ASSERT below to demonstrate. Can this cause us to allocate less space than we would need? - Not dangerous: select_distinct can be true here but be assigned false afterwards. */ aggr_tables= 2; DBUG_ASSERT(!tmp_table_param.using_outer_summary_function); if (!(join_tab= thd->alloc<JOIN_TAB>(top_join_tab_count + aggr_tables))) DBUG_RETURN(TRUE); if (inject_splitting_cond_for_all_tables_with_split_opt()) goto error; JOIN_TAB_RANGE *root_range; if (!(root_range= new (thd->mem_root) JOIN_TAB_RANGE)) goto error; root_range->start= join_tab; /* root_range->end will be set later */ join_tab_ranges.empty(); if (join_tab_ranges.push_back(root_range, thd->mem_root)) goto error; for (j=join_tab, tablenr=0 ; tablenr < table_count ; tablenr++,j++) { TABLE *form; POSITION *cur_pos= &best_positions[tablenr]; if (cur_pos->sj_strategy == SJ_OPT_MATERIALIZE || cur_pos->sj_strategy == SJ_OPT_MATERIALIZE_SCAN) { /* Ok, we've entered an SJ-Materialization semi-join (note that this can't be done recursively, semi-joins are not allowed to be nested). 1. Put into main join order a JOIN_TAB that represents a lookup or scan in the temptable. */ bzero((void*)j, sizeof(JOIN_TAB)); j->join= this; j->table= NULL; //temporary way to tell SJM tables from others. j->ref.key = -1; j->on_expr_ref= (Item**) &null_ptr; /* The unique index is always in 'possible keys' in EXPLAIN */ j->keys= key_map(1); /* 2. Proceed with processing SJM nest's join tabs, putting them into the sub-order */ SJ_MATERIALIZATION_INFO *sjm= cur_pos->table->emb_sj_nest->sj_mat_info; j->records_read= (sjm->is_sj_scan? sjm->rows : 1.0); j->records_init= j->records_out= j->records_read; j->records= (ha_rows) j->records_read; j->cond_selectivity= 1.0; j->join_read_time= 0.0; /* Not saved currently */ j->join_loops= 0.0; j->bush_children= JOIN_TAB_RANGE::create(thd, sjm->tables); if (!j->bush_children) goto error; join_tab_ranges.push_back(j->bush_children, thd->mem_root); sjm_nest_end= j->bush_children->end; sjm_nest_root= j; j= j->bush_children->start; } *j= *cur_pos->table; j->bush_root_tab= sjm_nest_root; form= table[tablenr]= j->table; form->reginfo.join_tab=j; DBUG_PRINT("info",("type: %d", j->type)); if (j->type == JT_CONST) goto loop_end; // Handled in make_join_stat.. j->loosescan_match_tab= NULL; //non-nulls will be set later j->inside_loosescan_range= FALSE; j->ref.key = -1; j->ref.key_parts=0; if (j->type == JT_SYSTEM) goto loop_end; if (!(keyuse= cur_pos->key)) { if (cur_pos->type == JT_NEXT) // Forced index { j->type= JT_NEXT; j->index= cur_pos->forced_index; } else j->type= JT_ALL; if (cur_pos->use_join_buffer && tablenr != const_tables) full_join= 1; } if ((j->type == JT_REF || j->type == JT_EQ_REF) && is_hash_join_key_no(j->ref.key)) hash_join= TRUE; j->range_rowid_filter_info= cur_pos->range_rowid_filter_info; /* Save records_read in JOIN_TAB so that select_describe()/etc don't have to access join->best_positions[]. */ j->records_init= cur_pos->records_init; j->records_read= cur_pos->records_read; j->records_out= cur_pos->records_out; j->join_read_time= cur_pos->read_time; j->join_loops= cur_pos->loops; loop_end: j->cond_selectivity= cur_pos->cond_selectivity; DBUG_ASSERT(j->cond_selectivity <= 1.0); crash_if_first_double_is_bigger(j->records_out, j->records_init * (j->range_rowid_filter_info ? j->range_rowid_filter_info->selectivity : 1.0)); map2table[j->table->tablenr]= j; /* If we've reached the end of sjm nest, switch back to main sequence */ if (j + 1 == sjm_nest_end) { j->last_leaf_in_bush= TRUE; j= sjm_nest_root; sjm_nest_root= NULL; sjm_nest_end= NULL; } } root_range->end= j; used_tables= OUTER_REF_TABLE_BIT; // Outer row is already read for (j=join_tab, tablenr=0 ; tablenr < table_count ; tablenr++,j++) { if (j->bush_children) j= j->bush_children->start; used_tables|= j->table->map; if (j->type != JT_CONST && j->type != JT_SYSTEM) { if ((keyuse= best_positions[tablenr].key) && create_ref_for_key(this, j, keyuse, TRUE, used_tables)) goto error; // Something went wrong } if (j->last_leaf_in_bush) j= j->bush_root_tab; } top_join_tab_count= (uint)(join_tab_ranges.head()->end - join_tab_ranges.head()->start); if (unlikely(thd->trace_started())) print_final_join_order(this); update_depend_map(this); DBUG_RETURN(0); error: /* join_tab was not correctly setup. Don't use it */ join_tab= 0; DBUG_RETURN(1); } /** Create a descriptor of hash join key to access a given join table @param join join which the join table belongs to @param join_tab the join table to access @param org_keyuse beginning of the key uses to join this table @param used_tables bitmap of the previous tables @details This function first finds key uses that can be utilized by the hash join algorithm to join join_tab to the previous tables marked in the bitmap used_tables. The tested key uses are taken from the array of all key uses for 'join' starting from the position org_keyuse. After all interesting key uses have been found the function builds a descriptor of the corresponding key that is used by the hash join algorithm would it be chosen to join the table join_tab. @retval FALSE the descriptor for a hash join key is successfully created @retval TRUE otherwise */ static bool create_hj_key_for_table(JOIN *join, JOIN_TAB *join_tab, KEYUSE *org_keyuse, table_map used_tables) { KEY *keyinfo; KEY_PART_INFO *key_part_info; KEYUSE *keyuse= org_keyuse; uint key_parts= 0; THD *thd= join->thd; TABLE *table= join_tab->table; bool first_keyuse= TRUE; DBUG_ENTER("create_hj_key_for_table"); do { if (!(~used_tables & keyuse->used_tables) && join_tab->keyuse_is_valid_for_access_in_chosen_plan(join, keyuse) && are_tables_local(join_tab, keyuse->used_tables)) { if (first_keyuse) { key_parts++; } else { KEYUSE *curr= org_keyuse; for( ; curr < keyuse; curr++) { if (curr->keypart == keyuse->keypart && !(~used_tables & curr->used_tables) && join_tab->keyuse_is_valid_for_access_in_chosen_plan(join, curr) && are_tables_local(join_tab, curr->used_tables)) break; } if (curr == keyuse) key_parts++; } } first_keyuse= FALSE; keyuse++; } while (keyuse->table == table && keyuse->is_for_hash_join()); if (!key_parts) DBUG_RETURN(TRUE); /* This memory is allocated only once for the joined table join_tab */ if (!(keyinfo= thd->alloc<KEY>(1)) || !(key_part_info = thd->alloc<KEY_PART_INFO>(key_parts))) DBUG_RETURN(TRUE); keyinfo->usable_key_parts= keyinfo->user_defined_key_parts = key_parts; keyinfo->ext_key_parts= keyinfo->user_defined_key_parts; keyinfo->key_part= key_part_info; keyinfo->key_length=0; keyinfo->algorithm= HA_KEY_ALG_UNDEF; keyinfo->flags= HA_GENERATED_KEY; keyinfo->is_statistics_from_stat_tables= FALSE; keyinfo->all_nulls_key_parts= 0; keyinfo->stat_storage_length= 0; keyinfo->name.str= "$hj"; keyinfo->name.length= 3; keyinfo->rec_per_key= thd->calloc<ulong>(key_parts); if (!keyinfo->rec_per_key) DBUG_RETURN(TRUE); keyinfo->key_part= key_part_info; first_keyuse= TRUE; keyuse= org_keyuse; do { if (!(~used_tables & keyuse->used_tables) && join_tab->keyuse_is_valid_for_access_in_chosen_plan(join, keyuse) && are_tables_local(join_tab, keyuse->used_tables)) { bool add_key_part= TRUE; if (!first_keyuse) { for(KEYUSE *curr= org_keyuse; curr < keyuse; curr++) { if (curr->keypart == keyuse->keypart && !(~used_tables & curr->used_tables) && join_tab->keyuse_is_valid_for_access_in_chosen_plan(join, curr) && are_tables_local(join_tab, curr->used_tables)) { keyuse->keypart= NO_KEYPART; add_key_part= FALSE; break; } } } if (add_key_part) { Field *field= table->field[keyuse->keypart]; uint fieldnr= keyuse->keypart+1; table->create_key_part_by_field(key_part_info, field, fieldnr); keyinfo->key_length += key_part_info->store_length; key_part_info++; } } first_keyuse= FALSE; keyuse++; } while (keyuse->table == table && keyuse->is_for_hash_join()); keyinfo->ext_key_parts= keyinfo->user_defined_key_parts; keyinfo->ext_key_flags= keyinfo->flags; keyinfo->ext_key_part_map= 0; join_tab->hj_key= keyinfo; DBUG_RETURN(FALSE); } /* Check if a set of tables specified by used_tables can be accessed when we're doing scan on join_tab jtab. */ static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables) { if (jtab->bush_root_tab) { /* jtab is inside execution join nest. We may not refer to outside tables, except the const tables. */ table_map local_tables= jtab->emb_sj_nest->nested_join->used_tables | jtab->join->const_table_map | OUTER_REF_TABLE_BIT; return !MY_TEST(used_tables & ~local_tables); } /* If we got here then jtab is at top level. - all other tables at top level are accessible, - tables in join nests are accessible too, because all their columns that are needed at top level will be unpacked when scanning the materialization table. */ return TRUE; } static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, bool allow_full_scan, table_map used_tables) { uint keyparts, length, key; TABLE *table; KEY *keyinfo; KEYUSE *keyuse= org_keyuse; bool ftkey= (keyuse->keypart == FT_KEYPART); THD *thd= join->thd; DBUG_ENTER("create_ref_for_key"); /* Use best key from find_best */ table= j->table; key= keyuse->key; if (!is_hash_join_key_no(key)) keyinfo= table->key_info+key; else { if (create_hj_key_for_table(join, j, org_keyuse, used_tables)) DBUG_RETURN(TRUE); keyinfo= j->hj_key; } if (ftkey) { Item_func_match *ifm=(Item_func_match *)keyuse->val; length=0; keyparts=1; ifm->join_key=1; } else { keyparts=length=0; uint found_part_ref_or_null= 0; /* Calculate length for the used key Stop if there is a missing key part or when we find second key_part with KEY_OPTIMIZE_REF_OR_NULL */ do { if (!(~used_tables & keyuse->used_tables) && (!keyuse->validity_ref || *keyuse->validity_ref) && j->keyuse_is_valid_for_access_in_chosen_plan(join, keyuse)) { if (are_tables_local(j, keyuse->val->used_tables())) { if ((is_hash_join_key_no(key) && keyuse->keypart != NO_KEYPART) || (!is_hash_join_key_no(key) && keyparts == keyuse->keypart && !(found_part_ref_or_null & keyuse->optimize))) { length+= keyinfo->key_part[keyparts].store_length; keyparts++; found_part_ref_or_null|= keyuse->optimize & ~KEY_OPTIMIZE_EQ; } } } keyuse++; } while (keyuse->table == table && keyuse->key == key); if (!keyparts && allow_full_scan) { /* It's a LooseIndexScan strategy scanning whole index */ j->type= JT_ALL; // TODO: Check if this should be JT_NEXT j->index= key; DBUG_RETURN(FALSE); } DBUG_ASSERT(length > 0); DBUG_ASSERT(keyparts != 0); } /* not ftkey */ /* set up fieldref */ j->ref.key_parts= keyparts; j->ref.key_length= length; j->ref.key= (int) key; if (!(j->ref.key_buff= thd->calloc<uchar>(ALIGN_SIZE(length)*2)) || !(j->ref.key_copy= thd->alloc<store_key*>(keyparts+1)) || !(j->ref.items= thd->alloc<Item*>(keyparts)) || !(j->ref.cond_guards= thd->alloc<bool*>(keyparts))) { DBUG_RETURN(TRUE); } j->ref.key_buff2=j->ref.key_buff+ALIGN_SIZE(length); j->ref.key_err=1; j->ref.has_record= FALSE; j->ref.null_rejecting= 0; j->ref.disable_cache= FALSE; j->ref.null_ref_part= NO_REF_PART; j->ref.const_ref_part_map= 0; j->ref.uses_splitting= FALSE; keyuse=org_keyuse; store_key **ref_key= j->ref.key_copy; uchar *key_buff=j->ref.key_buff, *null_ref_key= 0; uint null_ref_part= NO_REF_PART; bool keyuse_uses_no_tables= TRUE; uint not_null_keyparts= 0; if (ftkey) { j->ref.items[0]=((Item_func*)(keyuse->val))->key_item(); /* Predicates pushed down into subquery can't be used FT access */ j->ref.cond_guards[0]= NULL; if (keyuse->used_tables) DBUG_RETURN(TRUE); // not supported yet. SerG j->type=JT_FT; } else { uint i; for (i=0 ; i < keyparts ; keyuse++,i++) { while (((~used_tables) & keyuse->used_tables) || (keyuse->validity_ref && !(*keyuse->validity_ref)) || !j->keyuse_is_valid_for_access_in_chosen_plan(join, keyuse) || keyuse->keypart == NO_KEYPART || (keyuse->keypart != (is_hash_join_key_no(key) ? keyinfo->key_part[i].field->field_index : i)) || !are_tables_local(j, keyuse->val->used_tables())) keyuse++; /* Skip other parts */ uint maybe_null= MY_TEST(keyinfo->key_part[i].null_bit); j->ref.items[i]=keyuse->val; // Save for cond removal j->ref.cond_guards[i]= keyuse->cond_guard; if (!keyuse->val->maybe_null() || keyuse->null_rejecting) not_null_keyparts++; /* Set ref.null_rejecting to true only if we are going to inject a "keyuse->val IS NOT NULL" predicate. */ Item *real= (keyuse->val)->real_item(); if (keyuse->null_rejecting && (real->type() == Item::FIELD_ITEM) && ((Item_field*)real)->field->maybe_null()) j->ref.null_rejecting|= (key_part_map)1 << i; keyuse_uses_no_tables= keyuse_uses_no_tables && !keyuse->used_tables; j->ref.uses_splitting |= (keyuse->validity_ref != NULL); /* We don't want to compute heavy expressions in EXPLAIN, an example would select * from t1 where t1.key=(select thats very heavy); (select thats very heavy) => is a constant here eg: (select avg(order_cost) from orders) => constant but expensive */ if (!keyuse->val->used_tables() && !thd->lex->describe) { // Compare against constant store_key_item tmp(thd, keyinfo->key_part[i].field, key_buff + maybe_null, maybe_null ? key_buff : 0, keyinfo->key_part[i].length, keyuse->val, FALSE); if (unlikely(thd->is_error())) DBUG_RETURN(TRUE); tmp.copy(thd); j->ref.const_ref_part_map |= key_part_map(1) << i ; } else { *ref_key++= get_store_key(thd, keyuse,join->const_table_map, &keyinfo->key_part[i], key_buff, maybe_null); if (!keyuse->val->used_tables()) j->ref.const_ref_part_map |= key_part_map(1) << i ; } /* Remember if we are going to use REF_OR_NULL But only if field _really_ can be null i.e. we force JT_REF instead of JT_REF_OR_NULL in case if field can't be null */ if ((keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL) && maybe_null) { null_ref_key= key_buff; null_ref_part= i; } key_buff+= keyinfo->key_part[i].store_length; } } /* not ftkey */ *ref_key=0; // end_marker if (j->type == JT_FT) DBUG_RETURN(0); ulong key_flags= j->table->actual_key_flags(keyinfo); if (j->type == JT_CONST) j->table->const_table= 1; else if (!((keyparts == keyinfo->user_defined_key_parts && ( (key_flags & (HA_NOSAME | HA_NULL_PART_KEY)) == HA_NOSAME || /* Unique key and all keyparts are NULL rejecting */ ((key_flags & HA_NOSAME) && keyparts == not_null_keyparts) )) || /* true only for extended keys */ (MY_TEST(key_flags & HA_EXT_NOSAME) && keyparts == keyinfo->ext_key_parts) ) || null_ref_key) { /* Must read with repeat */ j->type= null_ref_key ? JT_REF_OR_NULL : JT_REF; j->ref.null_ref_key= null_ref_key; j->ref.null_ref_part= null_ref_part; } else if (keyuse_uses_no_tables) { /* This happen if we are using a constant expression in the ON part of an LEFT JOIN. SELECT * FROM a LEFT JOIN b ON b.key=30 Here we should not mark the table as a 'const' as a field may have a 'normal' value or a NULL value. */ j->type=JT_CONST; } else j->type=JT_EQ_REF; if (j->type == JT_EQ_REF) j->read_record.unlock_row= join_read_key_unlock_row; else if (j->type == JT_CONST) j->read_record.unlock_row= join_const_unlock_row; else j->read_record.unlock_row= rr_unlock_row; DBUG_RETURN(0); } static store_key * get_store_key(THD *thd, KEYUSE *keyuse, table_map used_tables, KEY_PART_INFO *key_part, uchar *key_buff, uint maybe_null) { if (!((~used_tables) & keyuse->used_tables)) // if const item { return new store_key_const_item(thd, key_part->field, key_buff + maybe_null, maybe_null ? key_buff : 0, key_part->length, keyuse->val); } else if (keyuse->val->type() == Item::FIELD_ITEM || (keyuse->val->type() == Item::REF_ITEM && ((((Item_ref*)keyuse->val)->ref_type() == Item_ref::OUTER_REF && (*(Item_ref**)((Item_ref*)keyuse->val)->ref)->ref_type() == Item_ref::DIRECT_REF) || ((Item_ref*)keyuse->val)->ref_type() == Item_ref::VIEW_REF) && keyuse->val->real_item()->type() == Item::FIELD_ITEM)) return new store_key_field(thd, key_part->field, key_buff + maybe_null, maybe_null ? key_buff : 0, key_part->length, ((Item_field*) keyuse->val->real_item())->field, keyuse->val->real_item()->full_name()); return new store_key_item(thd, key_part->field, key_buff + maybe_null, maybe_null ? key_buff : 0, key_part->length, keyuse->val, FALSE); } inline void add_cond_and_fix(THD *thd, Item **e1, Item *e2) { if (*e1) { if (!e2) return; Item *res; if ((res= new (thd->mem_root) Item_cond_and(thd, *e1, e2))) { res->fix_fields(thd, 0); res->update_used_tables(); *e1= res; } } else *e1= e2; } /** Add to join_tab->select_cond[i] "table.field IS NOT NULL" conditions we've inferred from ref/eq_ref access performed. This function is a part of "Early NULL-values filtering for ref access" optimization. Example of this optimization: For query SELECT * FROM t1,t2 WHERE t2.key=t1.field @n and plan " any-access(t1), ref(t2.key=t1.field) " @n add "t1.field IS NOT NULL" to t1's table condition. @n Description of the optimization: We look through equalities chosen to perform ref/eq_ref access, pick equalities that have form "tbl.part_of_key = othertbl.field" (where othertbl is a non-const table and othertbl.field may be NULL) and add them to conditions on corresponding tables (othertbl in this example). Exception from that is the case when referred_tab->join != join. I.e. don't add NOT NULL constraints from any embedded subquery. Consider this query: @code SELECT A.f2 FROM t1 LEFT JOIN t2 A ON A.f2 = f1 WHERE A.f3=(SELECT MIN(f3) FROM t2 C WHERE A.f4 = C.f4) OR A.f3 IS NULL; @endocde Here condition A.f3 IS NOT NULL is going to be added to the WHERE condition of the embedding query. Another example: SELECT * FROM t10, t11 WHERE (t10.a < 10 OR t10.a IS NULL) AND t11.b <=> t10.b AND (t11.a = (SELECT MAX(a) FROM t12 WHERE t12.b = t10.a )); Here condition t10.a IS NOT NULL is going to be added. In both cases addition of NOT NULL condition will erroneously reject some rows of the result set. referred_tab->join != join constraint would disallow such additions. This optimization doesn't affect the choices that ref, range, or join optimizer make. This was intentional because this was added after 4.1 was GA. Implementation overview 1. update_ref_and_keys() accumulates info about null-rejecting predicates in KEY_FIELD::null_rejecting 1.1 add_key_part saves these to KEYUSE. 2. create_ref_for_key copies them to TABLE_REF. 3. add_not_null_conds adds "x IS NOT NULL" to join_tab->select_cond of appropriate JOIN_TAB members. */ static void add_not_null_conds(JOIN *join) { JOIN_TAB *tab; DBUG_ENTER("add_not_null_conds"); for (tab= first_linear_tab(join, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS)) { if (tab->type == JT_REF || tab->type == JT_EQ_REF || tab->type == JT_REF_OR_NULL) { for (uint keypart= 0; keypart < tab->ref.key_parts; keypart++) { if (tab->ref.null_rejecting & ((key_part_map)1 << keypart)) { Item *item= tab->ref.items[keypart]; Item *notnull; Item *real= item->real_item(); if (real->can_eval_in_optimize() && real->type() != Item::FIELD_ITEM) { /* It could be constant instead of field after constant propagation. */ continue; } DBUG_ASSERT(real->type() == Item::FIELD_ITEM); Item_field *not_null_item= (Item_field*)real; JOIN_TAB *referred_tab= not_null_item->field->table->reginfo.join_tab; /* For UPDATE queries such as: UPDATE t1 SET t1.f2=(SELECT MAX(t2.f4) FROM t2 WHERE t2.f3=t1.f1); not_null_item is the t1.f1, but it's referred_tab is 0. */ if (!(notnull= new (join->thd->mem_root) Item_func_isnotnull(join->thd, item))) DBUG_VOID_RETURN; /* We need to do full fix_fields() call here in order to have correct notnull->const_item(). This is needed e.g. by test_quick_select when it is called from make_join_select after this function is called. */ if (notnull->fix_fields(join->thd, ¬null)) DBUG_VOID_RETURN; DBUG_EXECUTE("where",print_where(notnull, (referred_tab ? referred_tab->table->alias.c_ptr() : "outer_ref_cond"), QT_ORDINARY);); if (!tab->first_inner) { COND *new_cond= (referred_tab && referred_tab->join == join) ? referred_tab->select_cond : join->outer_ref_cond; add_cond_and_fix(join->thd, &new_cond, notnull); if (referred_tab && referred_tab->join == join) referred_tab->set_select_cond(new_cond, __LINE__); else join->outer_ref_cond= new_cond; } else add_cond_and_fix(join->thd, tab->first_inner->on_expr_ref, notnull); } } } } DBUG_VOID_RETURN; } /** Build a predicate guarded by match variables for embedding outer joins. The function recursively adds guards for predicate cond assending from tab to the first inner table next embedding nested outer join and so on until it reaches root_tab (root_tab can be 0). In other words: add_found_match_trig_cond(tab->first_inner_tab, y, 0) is the way one should wrap parts of WHERE. The idea is that the part of WHERE should be only evaluated after we've finished figuring out whether outer joins. ^^^ is the above correct? @param tab the first inner table for most nested outer join @param cond the predicate to be guarded (must be set) @param root_tab the first inner table to stop @return - pointer to the guarded predicate, if success - 0, otherwise */ static COND* add_found_match_trig_cond(THD *thd, JOIN_TAB *tab, COND *cond, JOIN_TAB *root_tab) { COND *tmp; DBUG_ASSERT(cond != 0); if (tab == root_tab) return cond; if ((tmp= add_found_match_trig_cond(thd, tab->first_upper, cond, root_tab))) tmp= new (thd->mem_root) Item_func_trig_cond(thd, tmp, &tab->found); if (tmp) { tmp->quick_fix_field(); tmp->update_used_tables(); } return tmp; } bool TABLE_LIST::is_active_sjm() { return sj_mat_info && sj_mat_info->is_used; } /** Fill in outer join related info for the execution plan structure. For each outer join operation left after simplification of the original query the function set up the following pointers in the linear structure join->join_tab representing the selected execution plan. The first inner table t0 for the operation is set to refer to the last inner table tk through the field t0->last_inner. Any inner table ti for the operation are set to refer to the first inner table ti->first_inner. The first inner table t0 for the operation is set to refer to the first inner table of the embedding outer join operation, if there is any, through the field t0->first_upper. The on expression for the outer join operation is attached to the corresponding first inner table through the field t0->on_expr_ref. Here ti are structures of the JOIN_TAB type. In other words, for each join tab, set - first_inner - last_inner - first_upper - on_expr_ref, cond_equal EXAMPLE. For the query: @code SELECT * FROM t1 LEFT JOIN (t2, t3 LEFT JOIN t4 ON t3.a=t4.a) ON (t1.a=t2.a AND t1.b=t3.b) WHERE t1.c > 5, @endcode given the execution plan with the table order t1,t2,t3,t4 is selected, the following references will be set; t4->last_inner=[t4], t4->first_inner=[t4], t4->first_upper=[t2] t2->last_inner=[t4], t2->first_inner=t3->first_inner=[t2], on expression (t1.a=t2.a AND t1.b=t3.b) will be attached to *t2->on_expr_ref, while t3.a=t4.a will be attached to *t4->on_expr_ref. @param join reference to the info fully describing the query @note The function assumes that the simplification procedure has been already applied to the join query (see simplify_joins). This function can be called only after the execution plan has been chosen. */ static bool make_outerjoin_info(JOIN *join) { DBUG_ENTER("make_outerjoin_info"); /* Create temp. tables for merged SJ-Materialization nests. We need to do this now, because further code relies on tab->table and tab->table->pos_in_table_list being set. */ JOIN_TAB *tab; for (tab= first_linear_tab(join, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS)) { if (tab->bush_children) { if (setup_sj_materialization_part1(tab)) DBUG_RETURN(TRUE); tab->table->reginfo.join_tab= tab; } } for (tab= first_linear_tab(join, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES); tab; tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS)) { TABLE *table= tab->table; TABLE_LIST *tbl= table->pos_in_table_list; TABLE_LIST *embedding= tbl->embedding; if (tbl->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) { /* Table tab is the only one inner table for outer join. (Like table t4 for the table reference t3 LEFT JOIN t4 ON t3.a=t4.a is in the query above.) */ tab->last_inner= tab->first_inner= tab; tab->on_expr_ref= &tbl->on_expr; tab->cond_equal= tbl->cond_equal; if (embedding && !embedding->is_active_sjm()) tab->first_upper= embedding->nested_join->first_nested; } else if (!embedding) tab->table->reginfo.not_exists_optimize= 0; for ( ; embedding ; embedding= embedding->embedding) { if (embedding->is_active_sjm()) { /* We're trying to walk out of an SJ-Materialization nest. Don't do this. */ break; } /* Ignore sj-nests: */ if (!(embedding->on_expr && embedding->outer_join)) { tab->table->reginfo.not_exists_optimize= 0; continue; } NESTED_JOIN *nested_join= embedding->nested_join; if (!nested_join->counter) { /* Table tab is the first inner table for nested_join. Save reference to it in the nested join structure. */ nested_join->first_nested= tab; tab->on_expr_ref= &embedding->on_expr; tab->cond_equal= tbl->cond_equal; if (embedding->embedding) tab->first_upper= embedding->embedding->nested_join->first_nested; } if (!tab->first_inner) tab->first_inner= nested_join->first_nested; if (++nested_join->counter < nested_join->n_tables) break; /* Table tab is the last inner table for nested join. */ nested_join->first_nested->last_inner= tab; } } DBUG_RETURN(FALSE); } /* @brief Build a temporary join prefix condition for JOIN_TABs up to the last tab @param ret OUT the condition is returned here @return false OK true Out of memory @detail Walk through the join prefix (from the first table to the last_tab) and build a condition: join_tab_1_cond AND join_tab_2_cond AND ... AND last_tab_conds The condition is only intended to be used by the range optimizer, so: - it is not normalized (can have Item_cond_and inside another Item_cond_and) - it does not include join->exec_const_cond and other similar conditions. */ bool build_tmp_join_prefix_cond(JOIN *join, JOIN_TAB *last_tab, Item **ret) { THD *const thd= join->thd; Item_cond_and *all_conds= NULL; Item *res= NULL; // Pick the ON-expression. Use the same logic as in get_sargable_cond(): if (last_tab->on_expr_ref) res= *last_tab->on_expr_ref; else if (last_tab->table->pos_in_table_list && last_tab->table->pos_in_table_list->embedding && !last_tab->table->pos_in_table_list->embedding->sj_on_expr) { res= last_tab->table->pos_in_table_list->embedding->on_expr; } for (JOIN_TAB *tab= first_depth_first_tab(join); tab; tab= next_depth_first_tab(join, tab)) { if (tab->select_cond) { if (!res) res= tab->select_cond; else { if (!all_conds) { if (!(all_conds= new (thd->mem_root)Item_cond_and(thd, res, tab->select_cond))) return true; res= all_conds; } else all_conds->add(tab->select_cond, thd->mem_root); } } if (tab == last_tab) break; } *ret= all_conds? all_conds: res; return false; } static bool make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) { THD *thd= join->thd; DBUG_ENTER("make_join_select"); if (select) { Json_writer_object trace_wrapper(thd); Json_writer_object trace_conditions(thd, "attaching_conditions_to_tables"); Json_writer_array trace_attached_comp(thd, "attached_conditions_computation"); add_not_null_conds(join); table_map used_tables; /* Step #1: Extract constant condition - Extract and check the constant part of the WHERE - Extract constant parts of ON expressions from outer joins and attach them appropriately. */ if (cond) /* Because of QUICK_GROUP_MIN_MAX_SELECT */ { /* there may be a select without a cond. */ if (join->table_count > 1) cond->update_used_tables(); // Tablenr may have changed /* Extract expressions that depend on constant tables 1. Const part of the join's WHERE clause can be checked immediately and if it is not satisfied then the join has empty result 2. Constant parts of outer joins' ON expressions must be attached there inside the triggers. */ { // Check const tables Item* const_cond= NULL; const_cond= make_cond_for_table(thd, cond, join->const_table_map, (table_map) 0, -1, FALSE, FALSE); if (!const_cond && thd->is_error()) DBUG_RETURN(1); /* Add conditions added by add_not_null_conds(). */ for (uint i= 0 ; i < join->const_tables ; i++) add_cond_and_fix(thd, &const_cond, join->join_tab[i].select_cond); DBUG_EXECUTE("where",print_where(const_cond,"constants", QT_ORDINARY);); if (const_cond) { Json_writer_object trace_const_cond(thd); trace_const_cond.add("condition_on_constant_tables", const_cond); if (const_cond->can_eval_in_optimize()) { bool const_cond_result; { Json_writer_array a(thd, "computing_condition"); const_cond_result= const_cond->val_bool() != 0; } if (!const_cond_result) { DBUG_PRINT("info",("Found impossible WHERE condition")); if (unlikely(trace_const_cond.trace_started())) trace_const_cond. add("evalualted", "true"). add("found", "impossible where"); join->exec_const_cond= NULL; DBUG_RETURN(1); } } else { trace_const_cond.add("evaluated", "false") .add("cause", "expensive cond"); } join->exec_const_cond= const_cond; } if (join->table_count != join->const_tables) { COND *outer_ref_cond= make_cond_for_table(thd, cond, join->const_table_map | OUTER_REF_TABLE_BIT, OUTER_REF_TABLE_BIT, -1, FALSE, FALSE); if (outer_ref_cond) { add_cond_and_fix(thd, &outer_ref_cond, join->outer_ref_cond); join->outer_ref_cond= outer_ref_cond; Json_writer_object trace(thd); trace.add("outer_ref_cond", outer_ref_cond); } else if (thd->is_error()) DBUG_RETURN(1); } else { COND *pseudo_bits_cond= make_cond_for_table(thd, cond, join->const_table_map | PSEUDO_TABLE_BITS, PSEUDO_TABLE_BITS, -1, FALSE, FALSE); if (pseudo_bits_cond) { add_cond_and_fix(thd, &pseudo_bits_cond, join->pseudo_bits_cond); join->pseudo_bits_cond= pseudo_bits_cond; Json_writer_object trace(thd); trace.add("pseudo_bits_cond", pseudo_bits_cond); } else if (thd->is_error()) DBUG_RETURN(1); } } } /* Step #2: Extract WHERE/ON parts */ uint i; for (i= join->top_join_tab_count - 1; i >= join->const_tables; i--) { if (!join->join_tab[i].bush_children) break; } uint last_top_base_tab_idx= i; table_map save_used_tables= 0; used_tables=((select->const_tables=join->const_table_map) | OUTER_REF_TABLE_BIT | RAND_TABLE_BIT); JOIN_TAB *tab; table_map current_map; i= join->const_tables; for (tab= first_depth_first_tab(join); tab; tab= next_depth_first_tab(join, tab)) { bool is_hj; /* first_inner is the X in queries like: SELECT * FROM t1 LEFT OUTER JOIN (t2 JOIN t3) ON X */ JOIN_TAB *first_inner_tab= tab->first_inner; COND *tmp; if (!tab->bush_children) current_map= tab->table->map; else current_map= tab->bush_children->start->emb_sj_nest->sj_inner_tables; /* Tables that are within SJ-Materialization nests cannot have their conditions referring to preceding non-const tables. - If we're looking at the first SJM table, reset used_tables to refer to only allowed tables */ if (tab->emb_sj_nest && tab->emb_sj_nest->sj_mat_info && tab->emb_sj_nest->sj_mat_info->is_used && !(used_tables & tab->emb_sj_nest->sj_inner_tables)) { save_used_tables= used_tables; used_tables= join->const_table_map | OUTER_REF_TABLE_BIT | RAND_TABLE_BIT; } used_tables|=current_map; /* Change from using ref access to using quick select on the same index if the quick select uses more key parts. There are two cases. A. ref access is ref(const). quick select was also constructed using equality restrictions that ref used, and so it will scan a subset of rows that ref access scans. Example: suppose the index is INDEX(kp1, kp2) and the WHERE has: kp1='foo' and kp2 <= 10 here, ref access will use kp1='foo' and quick select will use (foo) <= (kp1,kp2) <=(foo,10) B. ref access is not constant. In this case, quick select was constructed from some other restriction and in general will scan totally different set of rows (it maybe larger or smaller). Example: for INDEX(kp1, kp2) and the WHERE: kp1 <='foo' and kp1=prev_table.col and kp2 <= 10 the ref access will use kp1=prev_table.col, while quick select will use (-inf) < (kp1, kp2) <= ('foo',10). Because of the above, we perform the rewrite ONLY when ref is ref(const). */ if (tab->type == JT_REF && tab->quick && (((uint) tab->ref.key == tab->quick->index && tab->ref.key_length < tab->quick->max_used_key_length) || (!is_hash_join_key_no(tab->ref.key) && tab->table->intersect_keys.is_set(tab->ref.key))) && tab->ref.const_ref_part_map == // (ref-is-const) make_prev_keypart_map(tab->ref.key_parts)) // (ref-is-const) { /* Range uses longer key; Use this instead of ref on key */ if (unlikely(thd->trace_started())) { Json_writer_object ref_to_range(thd); ref_to_range. add("ref_to_range", true). add("cause", "range uses longer key"); } tab->type= JT_RANGE; tab->use_quick=1; if (is_index_merge(tab->quick->get_type())) tab->clear_range_rowid_filter(); tab->ref.key= -1; tab->ref.key_parts=0; // Don't use ref key. join->best_positions[i].records_read= rows2double(tab->quick->records); /* We will use join cache here : prevent sorting of the first table only and sort at the end. */ if (i != join->const_tables && join->table_count > join->const_tables + 1 && join->best_positions[i].use_join_buffer) join->full_join= 1; } tmp= NULL; if (cond) { if (tab->bush_children) { // Reached the materialization tab tmp= make_cond_after_sjm(thd, cond, cond, save_used_tables, used_tables, /*inside_or_clause=*/FALSE); used_tables= save_used_tables | used_tables; save_used_tables= 0; } else { tmp= make_cond_for_table(thd, cond, used_tables, current_map, i, FALSE, FALSE); if (!tmp && thd->is_error()) DBUG_RETURN(1); if (tab == join->join_tab + last_top_base_tab_idx) { /* This pushes conjunctive conditions of WHERE condition such that: - their used_tables() contain RAND_TABLE_BIT - the conditions does not refer to any fields (such like rand() > 0.5) */ table_map rand_table_bit= (table_map) RAND_TABLE_BIT; COND *rand_cond= make_cond_for_table(thd, cond, used_tables, rand_table_bit, -1, FALSE, FALSE); if (rand_cond) add_cond_and_fix(thd, &tmp, rand_cond); else if (thd->is_error()) DBUG_RETURN(1); } } /* Add conditions added by add_not_null_conds(). */ if (tab->select_cond) add_cond_and_fix(thd, &tmp, tab->select_cond); } uint max_jcl= join->max_allowed_join_cache_level; bool is_hash_allowed= join->allowed_join_cache_types & JOIN_CACHE_HASHED_BIT; bool is_bnlh_enabled= ((max_jcl == 3 || max_jcl == 4) && is_hash_allowed) || hint_table_state(thd, tab->table, BNL_HINT_ENUM, false); bool is_bkah_enabled= (max_jcl > 4 && is_hash_allowed) || hint_table_state(thd, tab->table, BKA_HINT_ENUM, false); is_hj= (tab->type == JT_REF || tab->type == JT_EQ_REF) && (is_bnlh_enabled || (is_bkah_enabled && is_hash_join_key_no(tab->ref.key))) && (!tab->emb_sj_nest || join->allowed_semijoin_with_cache) && (!(tab->table->map & join->outer_join) || join->allowed_outer_join_with_cache); if (cond && !tmp && tab->quick) { // Outer join if ((tab->type != JT_ALL && tab->type != JT_RANGE) && !is_hj) { /* Don't use the quick method We come here in the case where we have 'key=constant' and the test is removed by make_cond_for_table() */ delete tab->quick; tab->quick= 0; } else { /* Hack to handle the case where we only refer to a table in the ON part of an OUTER JOIN. In this case we want the code below to check if we should use 'quick' instead. */ DBUG_PRINT("info", ("Item_int")); tmp= (Item*) Item_true; } } if (tmp || !cond || tab->type == JT_REF || tab->type == JT_REF_OR_NULL || tab->type == JT_EQ_REF || first_inner_tab) { DBUG_EXECUTE("where",print_where(tmp, tab->table ? tab->table->alias.c_ptr() :"sjm-nest", QT_ORDINARY);); SQL_SELECT *sel= tab->select= ((SQL_SELECT*) thd->memdup((uchar*) select, sizeof(*select))); if (!sel) DBUG_RETURN(1); // End of memory /* If tab is an inner table of an outer join operation, add a match guard to the pushed down predicate. The guard will turn the predicate on only after the first match for outer tables is encountered. */ if (cond && tmp) { /* Because of QUICK_GROUP_MIN_MAX_SELECT there may be a select without a cond, so neutralize the hack above. */ COND *tmp_cond; if (!(tmp_cond= add_found_match_trig_cond(thd, first_inner_tab, tmp, 0))) DBUG_RETURN(1); sel->cond= tmp_cond; tab->set_select_cond(tmp_cond, __LINE__); /* Push condition to storage engine if this is enabled and the condition is not guarded */ if (tab->table) { tab->table->file->pushed_cond= NULL; if ((tab->table->file->ha_table_flags() & HA_CAN_TABLE_CONDITION_PUSHDOWN) && !first_inner_tab) { Json_writer_object wrap(thd); Json_writer_object trace_cp(thd, "table_condition_pushdown"); trace_cp.add_table_name(tab->table); COND *push_cond= make_cond_for_table(thd, tmp_cond, current_map, current_map, -1, FALSE, FALSE); if (push_cond) { trace_cp.add("push_cond", push_cond); /* Push condition to handler */ if (!tab->table->file->cond_push(push_cond)) tab->table->file->pushed_cond= push_cond; } else if (thd->is_error()) DBUG_RETURN(1); } } } else { sel->cond= NULL; tab->set_select_cond(NULL, __LINE__); } sel->head=tab->table; DBUG_EXECUTE("where", print_where(tmp, tab->table ? tab->table->alias.c_ptr() : "(sjm-nest)", QT_ORDINARY);); if (tab->quick) { /* Use quick key read if it's a constant and it's not used with key reading */ if ((tab->needed_reg.is_clear_all() && tab->type != JT_EQ_REF && tab->type != JT_FT && ((tab->type != JT_CONST && tab->type != JT_REF) || (uint) tab->ref.key == tab->quick->index)) || is_hj) { DBUG_ASSERT(tab->quick->is_valid()); sel->quick=tab->quick; // Use value from get_quick_... sel->quick_keys.clear_all(); sel->needed_reg.clear_all(); if (is_hj && tab->rowid_filter) tab->clear_range_rowid_filter(); } else { delete tab->quick; } tab->quick=0; } uint ref_key= (sel->head ? (uint) sel->head->reginfo.join_tab->ref.key+1 : 0); if (i == join->const_tables && ref_key) { if (!tab->const_keys.is_clear_all() && tab->table->reginfo.impossible_range) DBUG_RETURN(1); } else if ((tab->type == JT_ALL || tab->type == JT_NEXT)) { if (!tab->const_keys.is_clear_all() && tab->table->reginfo.impossible_range) DBUG_RETURN(1); // Impossible range /* We plan to scan all rows either with table or index scan Check again if we should use an index. There are two cases: 1) There could be an index usage the refers to a previous table that we didn't consider before, but could be consider now as a "last resort". For example SELECT * from t1,t2 where t1.a between t2.a and t2.b; 2) If the current table is the first non const table and there is a limit it still possibly beneficial to use the index even if the index range is big as we can stop when we've found limit rows. (1) - Don't switch the used index if we are using semi-join LooseScan on this table. Using different index will not produce the desired ordering and de-duplication. */ if (!tab->table->is_filled_at_execution() && !tab->loosescan_match_tab && // (1) ((cond && (!tab->keys.is_subset(tab->const_keys) && i > join->const_tables)) || (!tab->const_keys.is_clear_all() && i == join->const_tables && join->unit->lim.get_select_limit() < join->best_positions[i].records_read && !(join->select_options & OPTION_FOUND_ROWS)))) { /* Join with outer join condition */ COND *orig_cond=sel->cond; if (build_tmp_join_prefix_cond(join, tab, &sel->cond)) return true; /* To be removed in 11.0+: Caution: we can reach this point with quick=NULL. Below, we'll use tab->keys and not tab->const_keys like get_quick_record_count() did. If we have constructed a group-min-max quick select, make sure we're able to construct it again */ if (sel->quick && sel->quick->get_type() == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) { tab->keys.set_bit(sel->quick->index); } /* We can't call sel->cond->fix_fields, as it will break tab->on_expr if it's AND condition (fix_fields currently removes extra AND/OR levels). Yet attributes of the just built condition are not needed. Thus we call sel->cond->quick_fix_field for safety. */ if (sel->cond && !sel->cond->fixed()) sel->cond->quick_fix_field(); quick_select_return res; if ((res= sel->test_quick_select(thd, tab->keys, ((used_tables & ~ current_map) | OUTER_REF_TABLE_BIT), (join->select_options & OPTION_FOUND_ROWS ? HA_POS_ERROR : join->unit->lim.get_select_limit()), 0, FALSE, FALSE, FALSE, Item_func::BITMAP_ALL)) == SQL_SELECT::IMPOSSIBLE_RANGE) { /* Before reporting "Impossible WHERE" for the whole query we have to check isn't it only "impossible ON" instead */ sel->cond=orig_cond; if (!*tab->on_expr_ref || (res= sel->test_quick_select(thd, tab->keys, used_tables & ~ current_map, (join->select_options & OPTION_FOUND_ROWS ? HA_POS_ERROR : join->unit->lim.get_select_limit()), 0, FALSE, FALSE, FALSE, Item_func::BITMAP_NONE)) == SQL_SELECT::IMPOSSIBLE_RANGE) DBUG_RETURN(1); // Impossible WHERE } else sel->cond=orig_cond; if (res == SQL_SELECT::ERROR) DBUG_RETURN(1); /* Some error in one of test_quick_select calls */ /* Fix for EXPLAIN */ if (sel->quick) { join->best_positions[i].records_read= (double) sel->quick->records; set_if_smaller(join->best_positions[i].records_out, rows2double(sel->head->opt_range_condition_rows)); } else { /* sel->head->opt_range_condition_rows may have been updated to a smaller number than before by a call to test_quick_select. This can happen even if the range optimizer decided to not use the range (sel->quick was not set). */ set_if_smaller(join->best_positions[i].records_out, rows2double(sel->head->opt_range_condition_rows)); } } else { sel->needed_reg=tab->needed_reg; } sel->quick_keys= tab->table->opt_range_keys; if (!sel->quick_keys.is_subset(tab->checked_keys) || !sel->needed_reg.is_subset(tab->checked_keys)) { handler *file= tab->table->file; /* "Range checked for each record" is a "last resort" access method that should only be used when the other option is a cross-product join. We use the following condition (it's approximate): 1. There are potential keys for (sel->needed_reg) 2. There were no possible ways to construct a quick select, or the quick select would be more expensive than the full table scan. */ tab->use_quick= (!sel->needed_reg.is_clear_all() && (sel->quick_keys.is_clear_all() || (sel->quick && sel->quick->read_time > file->cost(file->ha_scan_and_compare_time(tab->table->file-> stats.records))))) ? 2 : 1; sel->read_tables= used_tables & ~current_map; sel->quick_keys.clear_all(); } if (i != join->const_tables && tab->use_quick != 2 && !tab->first_inner) { /* Read with cache */ /* TODO: the execution also gets here when we will not be using join buffer. Review these cases and perhaps, remove this call. (The final decision whether to use join buffer is made in check_join_cache_usage, so we should only call make_scan_filter() there, too). */ if (tab->make_scan_filter()) DBUG_RETURN(1); } } } /* Push down conditions from all ON expressions. Each of these conditions are guarded by a variable that turns if off just before null complemented row for outer joins is formed. Thus, the condition from an 'on expression' are guaranteed not to be checked for the null complemented row. */ /* First push down constant conditions from ON expressions. - Each pushed-down condition is wrapped into trigger which is enabled only for non-NULL-complemented record - The condition is attached to the first_inner_table. With regards to join nests: - if we start at top level, don't walk into nests - if we start inside a nest, stay within that nest. */ JOIN_TAB *start_from= tab->bush_root_tab? tab->bush_root_tab->bush_children->start : join->join_tab + join->const_tables; JOIN_TAB *end_with= tab->bush_root_tab? tab->bush_root_tab->bush_children->end : join->join_tab + join->top_join_tab_count; for (JOIN_TAB *join_tab= start_from; join_tab != end_with; join_tab++) { if (*join_tab->on_expr_ref) { JOIN_TAB *cond_tab= join_tab->first_inner; COND *tmp_cond= make_cond_for_table(thd, *join_tab->on_expr_ref, join->const_table_map, (table_map) 0, -1, FALSE, FALSE); if (!tmp_cond) { if (!thd->is_error()) continue; DBUG_RETURN(1); } tmp_cond= new (thd->mem_root) Item_func_trig_cond(thd, tmp_cond, &cond_tab->not_null_compl); if (!tmp_cond) DBUG_RETURN(1); tmp_cond->quick_fix_field(); cond_tab->select_cond= !cond_tab->select_cond ? tmp_cond : new (thd->mem_root) Item_cond_and(thd, cond_tab->select_cond, tmp_cond); if (!cond_tab->select_cond) DBUG_RETURN(1); cond_tab->select_cond->quick_fix_field(); cond_tab->select_cond->update_used_tables(); if (cond_tab->select) cond_tab->select->cond= cond_tab->select_cond; } } /* Push down non-constant conditions from ON expressions */ JOIN_TAB *last_tab= tab; /* while we're inside of an outer join and last_tab is the last of its tables ... */ while (first_inner_tab && first_inner_tab->last_inner == last_tab) { /* Table tab is the last inner table of an outer join. An on expression is always attached to it. */ COND *on_expr= *first_inner_tab->on_expr_ref; table_map used_tables2= (join->const_table_map | OUTER_REF_TABLE_BIT | RAND_TABLE_BIT); start_from= tab->bush_root_tab? tab->bush_root_tab->bush_children->start : join->join_tab + join->const_tables; for (JOIN_TAB *inner_tab= start_from; inner_tab <= last_tab; inner_tab++) { DBUG_ASSERT(inner_tab->table); current_map= inner_tab->table->map; used_tables2|= current_map; /* psergey: have put the -1 below. It's bad, will need to fix it. */ COND *tmp_cond= make_cond_for_table(thd, on_expr, used_tables2, current_map, /*(inner_tab - first_tab)*/ -1, FALSE, FALSE); if (!tmp_cond && thd->is_error()) DBUG_RETURN(1); if (tab == last_tab) { /* This pushes conjunctive conditions of ON expression of an outer join such that: - their used_tables() contain RAND_TABLE_BIT - the conditions does not refer to any fields (such like rand() > 0.5) */ table_map rand_table_bit= (table_map) RAND_TABLE_BIT; COND *rand_cond= make_cond_for_table(thd, on_expr, used_tables2, rand_table_bit, -1, FALSE, FALSE); if (rand_cond) add_cond_and_fix(thd, &tmp_cond, rand_cond); else if (thd->is_error()) DBUG_RETURN(1); } bool is_sjm_lookup_tab= FALSE; if (inner_tab->bush_children) { /* 'inner_tab' is an SJ-Materialization tab, i.e. we have a join order like this: ot1 sjm_tab LEFT JOIN ot2 ot3 ^ ^ 'tab'-+ +--- left join we're adding triggers for LEFT JOIN's ON expression may not have references to subquery columns. The subquery was in the WHERE clause, so IN-equality is in the WHERE clause, also. However, equality propagation code may have propagated the IN-equality into ON expression, and we may get things like subquery_inner_table=const in the ON expression. We must not check such conditions during SJM-lookup, because 1) subquery_inner_table has no valid current row (materialization temp.table has it instead), and 2) they would be true anyway. */ SJ_MATERIALIZATION_INFO *sjm= inner_tab->bush_children->start->emb_sj_nest->sj_mat_info; if (sjm->is_used && !sjm->is_sj_scan) is_sjm_lookup_tab= TRUE; } if (inner_tab == first_inner_tab && inner_tab->on_precond && !is_sjm_lookup_tab) add_cond_and_fix(thd, &tmp_cond, inner_tab->on_precond); if (tmp_cond && !is_sjm_lookup_tab) { JOIN_TAB *cond_tab= (inner_tab < first_inner_tab ? first_inner_tab : inner_tab); Item **sel_cond_ref= (inner_tab < first_inner_tab ? --> -------------------- --> maximum size reached --> --------------------
¤ Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.4.924Bemerkung: (vorverarbeitet am 2026-10-08) ¤*Bot Zugriff |
|
||||||||||||||
2026-10-09