MDEV-40168 [wip] Add multi valued index over fulltext

TODOs on top of those in the patch diff:

- EXPLAIN output should not say fulltext
- check type match to avoid false negative / positive bugs in mysql
- transcode the value into the index charset in mvi_encode_key

Author: mariadb-YuchenPei

libmysqld/CMakeLists.txt

@@ -72,6 +72,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc
            ../sql/mf_iocache.cc ../sql/my_decimal.cc 
            ../sql/net_serv.cc ../sql/opt_range.cc
            ../sql/opt_group_by_cardinality.cc
+           ../sql/opt_multi_valued_index.cc
            ../sql/opt_rewrite_date_cmp.cc
            ../sql/opt_rewrite_remove_casefold.cc
            ../sql/opt_sargable_left.cc

mysql-test/main/multi_valued_index.opt

@@ -0,0 +1 @@
+--innodb_ft_index_cache

mysql-test/main/multi_valued_index.result

@@ -0,0 +1,51 @@
+# basic test
+SET SESSION debug = '+d,test_invisible_index,test_completely_invisible';
+create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c` int(11) DEFAULT NULL,
+  `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci
+show index from t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t1	1	invisible1	1	invisible1	A	0	NULL	NULL	YES	BTREE			NO
+t1	1	idx	1	DB_MVI_1	NULL	NULL	NULL	NULL	YES	FULLTEXT			NO
+set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table;
+set global innodb_ft_aux_table='test/t1';
+insert into t1 values (1, '{"tags": [1, "abcde", "34567", "", 34567]}');
+SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE;
+WORD	FIRST_DOC_ID	LAST_DOC_ID	DOC_COUNT	DOC_ID	POSITION
+31xx	1	1	1	1	0
+3334353637	1	1	1	1	16
+3334353637	1	1	1	1	16
+6162636465	1	1	1	1	5
+xxxx	1	1	1	1	27
+select json_contains(j->'$.tags', '"abcde"') from t1;
+json_contains(j->'$.tags', '"abcde"')
+1
+explain
+select * from t1 where json_contains(j->'$.tags', '"abcde"');
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	fulltext	idx	idx	0		1	Using where
+select * from t1 where json_contains(j->'$.tags', '"abcde"');
+c	j
+1	{"tags": [1, "abcde", "34567", "", 34567]}
+DROP TABLE t1;
+set global innodb_ft_aux_table=@old_innodb_ft_aux_table;
+# direct call of mvi_encode
+select mvi_encode('[1, 42, "3"]', int);
+mvi_encode('[1, 42, "3"]', int)
+8000000000000001 800000000000002a 8000000000000003
+select mvi_encode('[1, 42, "3"]', unsigned);
+mvi_encode('[1, 42, "3"]', unsigned)
+0000000000000001 000000000000002a 0000000000000003
+select mvi_encode('[1, 42, "3   "]', char(6));
+mvi_encode('[1, 42, "3   "]', char(6))
+31xx 3432 33xx
+select mvi_encode('[1, 42, "   "]', char(6));
+mvi_encode('[1, 42, "   "]', char(6))
+31xx 3432 xxxx
+select mvi_encode('[1, 42, "3   "]', binary(6));
+mvi_encode('[1, 42, "3   "]', binary(6))
+31xx 3432 33xx

mysql-test/main/multi_valued_index.test

@@ -0,0 +1,31 @@
+--source include/have_innodb.inc
+
+--echo # basic test
+SET SESSION debug = '+d,test_invisible_index,test_completely_invisible';
+
+create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb;
+SHOW CREATE TABLE t1;
+show index from t1;
+
+set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table;
+set global innodb_ft_aux_table='test/t1';
+
+insert into t1 values (1, '{"tags": [1, "abcde", "34567", "", 34567]}');
+
+SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE;
+
+select json_contains(j->'$.tags', '"abcde"') from t1;
+explain
+select * from t1 where json_contains(j->'$.tags', '"abcde"');
+select * from t1 where json_contains(j->'$.tags', '"abcde"');
+
+DROP TABLE t1;
+
+set global innodb_ft_aux_table=@old_innodb_ft_aux_table;
+
+--echo # direct call of mvi_encode
+select mvi_encode('[1, 42, "3"]', int);
+select mvi_encode('[1, 42, "3"]', unsigned);
+select mvi_encode('[1, 42, "3   "]', char(6));
+select mvi_encode('[1, 42, "   "]', char(6));
+select mvi_encode('[1, 42, "3   "]', binary(6));

mysql-test/suite/perfschema/r/digest_view.result

@@ -191,17 +191,17 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR
 FROM performance_schema.events_statements_summary_by_digest
 ORDER BY DIGEST_TEXT;
 SCHEMA_NAME	DIGEST	DIGEST_TEXT	COUNT_STAR
-test	cc5e38c5a702f49627052e59a2603818	EXPLAIN SELECT * FROM `test` . `v1` 	1
-test	264b69debfd30bbfe374cdca018fa4f9	EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 	1
-test	75f31bf60b75a4f851ff9c9ee4e19d96	EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 	1
-test	a8f8a85697afacda9f2f3d3b023f9ed0	EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 	1
-test	512598992d37826136f3f1292bb32e1e	EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 	1
-test	54342cc19df16ce54b3ba43d1fd7552d	SELECT * FROM `test` . `v1` 	1
-test	3750b7d8c33e040b90a2cdccb4c642a3	SELECT * FROM `test` . `v1` WHERE `a` = ? 	1
-test	94af70ef76a31364845f534863d99da8	SELECT * FROM `test` . `v1` WHERE `b` > ? 	1
-test	0c87d86b62e664a81a23d4f176fbb377	SELECT `a` , `b` FROM `test` . `v1` 	1
-test	de7e6f1350ff14a952b97b373d76c1a4	SELECT `b` , `a` FROM `test` . `v1` 	1
-test	3468091e6d6ce474aded20beaec36b53	TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 	1
+test	983c118738efc2378bb2a15495c7e6c9	EXPLAIN SELECT * FROM `test` . `v1` 	1
+test	f34ab36bba7121e7953b5c6ca8f58e11	EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 	1
+test	ee2356b394246ac8b7fc5faa97b0c1ad	EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 	1
+test	3e70e03c91932264badade3cb1f2e66e	EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 	1
+test	021f3cff6701828a7f017af34d29ce1c	EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 	1
+test	d2e11df922514a25243fc04ebf839825	SELECT * FROM `test` . `v1` 	1
+test	c967f871a7c1bc86a7667b83a17ca6e1	SELECT * FROM `test` . `v1` WHERE `a` = ? 	1
+test	7c87fc30bf915208e69006449a702b7a	SELECT * FROM `test` . `v1` WHERE `b` > ? 	1
+test	258f5bff705a10fe1c43fd25c29c61fd	SELECT `a` , `b` FROM `test` . `v1` 	1
+test	518f87edc2c96f2b022d6ab3c767df7b	SELECT `b` , `a` FROM `test` . `v1` 	1
+test	d972cde0f9bbec28c84e4dcca8ef3281	TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 	1
 DROP TABLE test.v1;
 CREATE VIEW test.v1 AS SELECT * FROM test.t1;
 EXPLAIN SELECT * from test.v1;
@@ -248,19 +248,19 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR
 FROM performance_schema.events_statements_summary_by_digest
 ORDER BY DIGEST_TEXT;
 SCHEMA_NAME	DIGEST	DIGEST_TEXT	COUNT_STAR
-test	ef8b10f452e117fa96af49687a93a10f	CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 	1
-test	6b99f4d2ad410f9fa4ee1d501b0db571	DROP TABLE `test` . `v1` 	1
-test	cc5e38c5a702f49627052e59a2603818	EXPLAIN SELECT * FROM `test` . `v1` 	2
-test	264b69debfd30bbfe374cdca018fa4f9	EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 	2
-test	75f31bf60b75a4f851ff9c9ee4e19d96	EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 	2
-test	a8f8a85697afacda9f2f3d3b023f9ed0	EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 	2
-test	512598992d37826136f3f1292bb32e1e	EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 	2
-test	54342cc19df16ce54b3ba43d1fd7552d	SELECT * FROM `test` . `v1` 	2
-test	3750b7d8c33e040b90a2cdccb4c642a3	SELECT * FROM `test` . `v1` WHERE `a` = ? 	2
-test	94af70ef76a31364845f534863d99da8	SELECT * FROM `test` . `v1` WHERE `b` > ? 	2
-test	187a846fafe04b746eddaaab80b6a766	SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 	1
-test	0c87d86b62e664a81a23d4f176fbb377	SELECT `a` , `b` FROM `test` . `v1` 	2
-test	de7e6f1350ff14a952b97b373d76c1a4	SELECT `b` , `a` FROM `test` . `v1` 	2
-test	3468091e6d6ce474aded20beaec36b53	TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 	1
+test	bd4716657502ce7922d2ed7e20043b76	CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 	1
+test	2d14a4845ee825e3a1a21bd5799cba3c	DROP TABLE `test` . `v1` 	1
+test	983c118738efc2378bb2a15495c7e6c9	EXPLAIN SELECT * FROM `test` . `v1` 	2
+test	f34ab36bba7121e7953b5c6ca8f58e11	EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 	2
+test	ee2356b394246ac8b7fc5faa97b0c1ad	EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 	2
+test	3e70e03c91932264badade3cb1f2e66e	EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 	2
+test	021f3cff6701828a7f017af34d29ce1c	EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 	2
+test	d2e11df922514a25243fc04ebf839825	SELECT * FROM `test` . `v1` 	2
+test	c967f871a7c1bc86a7667b83a17ca6e1	SELECT * FROM `test` . `v1` WHERE `a` = ? 	2
+test	7c87fc30bf915208e69006449a702b7a	SELECT * FROM `test` . `v1` WHERE `b` > ? 	2
+test	aa329bcb29afc39d9793e4fd6e108722	SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 	1
+test	258f5bff705a10fe1c43fd25c29c61fd	SELECT `a` , `b` FROM `test` . `v1` 	2
+test	518f87edc2c96f2b022d6ab3c767df7b	SELECT `b` , `a` FROM `test` . `v1` 	2
+test	d972cde0f9bbec28c84e4dcca8ef3281	TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 	1
 DROP VIEW test.v1;
 DROP TABLE test.t1;

mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result

@@ -8,5 +8,5 @@ SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1
 ####################################
 SELECT event_name, digest, digest_text, sql_text FROM events_statements_history_long;
 event_name	digest	digest_text	sql_text
-statement/sql/select	c9e142fe40c43498607ca5310e11d2ce	SELECT ? + ? + 	SELECT ...
-statement/sql/truncate	506e3496d92689cd2367329dc7725165	TRUNCATE TABLE 	truncat...
+statement/sql/select	3e19ece73d286977122b52fb027c180c	SELECT ? + ? + 	SELECT ...
+statement/sql/truncate	7ab36a94986dd98b93370b025166a946	TRUNCATE TABLE 	truncat...

sql/CMakeLists.txt

@@ -122,7 +122,6 @@ SET (SQL_SOURCE
                opt_rewrite_remove_casefold.cc
                opt_sargable_left.cc
                opt_sum.cc
-               opt_vcol_substitution.cc
                ../sql-common/pack.c parse_file.cc password.c procedure.cc
                protocol.cc records.cc repl_failsafe.cc rpl_filter.cc
                session_tracker.cc
@@ -199,6 +198,8 @@ SET (SQL_SOURCE
                json_table.cc
                proxy_protocol.cc backup.cc xa.cc
                socketpair.c socketpair.h
+               opt_multi_valued_index.h
+               opt_multi_valued_index.cc
                opt_vcol_substitution.h
                opt_vcol_substitution.cc
                opt_hints_parser.cc opt_hints_parser.h scan_char.h

sql/item.h

@@ -2859,6 +2859,11 @@ class Item :public Value_source,
     DBUG_ASSERT(fixed());
     return false;
   }
+  virtual Item *create_ft_for_mvi(THD *thd, List<Field> *vcol_fields)
+  {
+    return NULL;
+  }
+
 
 protected:
   /*

sql/item_func.h

@@ -109,7 +109,8 @@ class Item_func :public Item_func_or_sum
                   JSON_EXTRACT_FUNC, JSON_VALID_FUNC, ROWNUM_FUNC,
                   CASE_SEARCHED_FUNC, // Used by ColumnStore/Spider
                   CASE_SIMPLE_FUNC,   // Used by ColumnStore/spider,
-                  DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC
+                  DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC,
+                  MVI_ENCODE_FUNC
                 };
 
   /*

sql/item_jsonfunc.h

@@ -376,6 +376,7 @@ class Item_func_json_contains: public Item_bool_func
   }
   bool fix_length_and_dec(THD *thd) override;
   bool val_bool() override;
+  Item *create_ft_for_mvi(THD *thd, List<Field> *vcol_fields) override;
 
 protected:
   Item *shallow_copy(THD *thd) const override