#include "dict0dict.h"
#include "pars0pars.h"
#include "que0que.h"
#include "que0types.h"
#include "fts0types.h"
#include "fts0priv.ic"

Include dependency graph for fts0priv.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	FTS_MAX_BACKGROUND_THREAD_WAIT 10000
#define	FTS_BACKGROUND_THREAD_WAIT_COUNT 1000
#define	FTS_MAX_CONFIG_NAME_LEN 64
#define	FTS_MAX_CONFIG_VALUE_LEN 1024
#define	FTS_ILIST_MAX_SIZE (64 * 1024)
#define	FTS_OPTIMIZE_LIMIT_IN_SECS "optimize_checkpoint_limit"
#define	FTS_SYNCED_DOC_ID "synced_doc_id"
#define	FTS_LAST_OPTIMIZED_WORD "last_optimized_word"
#define	FTS_TOTAL_DELETED_COUNT "deleted_doc_count"
#define	FTS_TOTAL_WORD_COUNT "total_word_count"
#define	FTS_OPTIMIZE_START_TIME "optimize_start_time"
#define	FTS_OPTIMIZE_END_TIME "optimize_end_time"
#define	FTS_STOPWORD_TABLE_NAME "stopword_table_name"
#define	FTS_USE_STOPWORD "use_stopword"
#define	FTS_TABLE_STATE "table_state"
#define	FTS_AUX_MIN_TABLE_ID_LENGTH 48
#define	FTS_MAX_INT_LEN 32
#define	FTS_FETCH_DOC_BY_ID_EQUAL 1
#define	FTS_FETCH_DOC_BY_ID_LARGE 2
#define	FTS_FETCH_DOC_BY_ID_SMALL 3

Typedefs
typedef enum fts_table_state_enum	fts_table_state_t

Enumerations
enum	fts_table_state_enum { FTS_TABLE_STATE_RUNNING = 0, FTS_TABLE_STATE_OPTIMIZING, FTS_TABLE_STATE_DELETED }

Functions
UNIV_INTERN que_t *	fts_parse_sql (fts_table_t fts_table, pars_info_t info, const char *sql)
UNIV_INTERN dberr_t	fts_eval_sql (trx_t trx, que_t graph))
UNIV_INTERN char *	fts_get_table_name (const fts_table_t *fts_table))
UNIV_INTERN const char *	fts_get_select_columns_str (dict_index_t index, pars_info_t info, mem_heap_t *heap))
UNIV_INTERN dberr_t	fts_doc_fetch_by_doc_id (fts_get_doc_t get_doc, doc_id_t doc_id, dict_index_t index_to_use, ulint option, fts_sql_callback callback, void *arg)))
UNIV_INTERN ibool	fts_query_expansion_fetch_doc (void row, void user_arg))
UNIV_INTERN dberr_t	fts_write_node (trx_t trx, que_t graph, fts_table_t fts_table, fts_string_t word, fts_node_t node))
UNIV_INTERN void	fts_tokenize_document (fts_doc_t doc, fts_doc_t result)))
UNIV_INTERN void	fts_tokenize_document_next (fts_doc_t doc, ulint add_pos, fts_doc_t result)))
UNIV_INTERN void	fts_doc_init (fts_doc_t *doc))
UNIV_INTERN int	fts_bsearch (fts_update_t *array, int lower, int upper, doc_id_t doc_id))
UNIV_INTERN void	fts_doc_free (fts_doc_t *doc))
UNIV_INTERN void	fts_word_free (fts_word_t *word))
UNIV_INTERN dberr_t	fts_index_fetch_nodes (trx_t trx, que_t graph, fts_table_t fts_table, const fts_string_t word, fts_fetch_t fetch))
UNIV_INTERN fts_word_t *	fts_word_init (fts_word_t word, byte utf8, ulint len))
UNIV_INLINE int	fts_trx_table_cmp (const void v1, const void v2))
UNIV_INLINE int	fts_trx_table_id_cmp (const void p1, const void p2))
UNIV_INTERN dberr_t	fts_sql_commit (trx_t *trx))
UNIV_INTERN dberr_t	fts_sql_rollback (trx_t *trx))
UNIV_INTERN que_t *	fts_parse_sql_no_dict_lock (fts_table_t fts_table, pars_info_t info, const char *sql)
UNIV_INTERN dberr_t	fts_config_get_value (trx_t trx, fts_table_t fts_table, const char name, fts_string_t value))
UNIV_INTERN dberr_t	fts_config_get_index_value (trx_t trx, dict_index_t index, const char param, fts_string_t value))
UNIV_INTERN dberr_t	fts_config_set_value (trx_t trx, fts_table_t fts_table, const char name, const fts_string_t value))
UNIV_INTERN dberr_t	fts_config_set_ulint (trx_t trx, fts_table_t fts_table, const char *name, ulint int_value))
UNIV_INTERN dberr_t	fts_config_set_index_value (trx_t trx, dict_index_t index, const char param, fts_string_t value))
UNIV_INTERN dberr_t	fts_config_increment_value (trx_t trx, fts_table_t fts_table, const char *name, ulint delta))
UNIV_INTERN dberr_t	fts_config_increment_index_value (trx_t trx, dict_index_t index, const char *name, ulint delta))
UNIV_INTERN dberr_t	fts_config_get_index_ulint (trx_t trx, dict_index_t index, const char name, ulint int_value))
UNIV_INTERN dberr_t	fts_config_set_index_ulint (trx_t trx, dict_index_t index, const char *name, ulint int_value))
UNIV_INTERN dberr_t	fts_config_get_ulint (trx_t trx, fts_table_t fts_table, const char name, ulint int_value))
UNIV_INTERN const ib_vector_t *	fts_cache_find_word (const fts_index_cache_t index_cache, const fts_string_t text))
UNIV_INTERN ibool	fts_cache_is_deleted_doc_id (const fts_cache_t *cache, doc_id_t doc_id))
UNIV_INTERN void	fts_cache_append_deleted_doc_ids (const fts_cache_t cache, ib_vector_t vector)
UNIV_INTERN ibool	fts_wait_for_background_thread_to_start (dict_table_t *table, ulint max_wait)
UNIV_INTERN fts_index_cache_t *	fts_find_index_cache (const fts_cache_t cache, const dict_index_t index))
UNIV_INLINE int	fts_write_object_id (ib_id_t id, char *str, bool hex_format)))
UNIV_INLINE ibool	fts_read_object_id (ib_id_t id, const char str))
UNIV_INTERN int	fts_get_table_id (const fts_table_t fts_table, char table_id))
UNIV_INTERN void	fts_optimize_add_table (dict_table_t *table))
UNIV_INTERN void	fts_optimize_do_table (dict_table_t *table))
UNIV_INTERN char *	fts_get_table_name_prefix (const fts_table_t *fts_table))
UNIV_INTERN void	fts_cache_node_add_positions (fts_cache_t cache, fts_node_t node, doc_id_t doc_id, ib_vector_t *positions)))
UNIV_INTERN char *	fts_config_create_index_param_name (const char param, const dict_index_t index))

Variables
UNIV_INTERN que_t	malloc
UNIV_INTERN que_t	warn_unused_result

Detailed Description

Full text search internal header file

Created 2011/09/02 Sunny Bains

Macro Definition Documentation

#define FTS_AUX_MIN_TABLE_ID_LENGTH 48

The minimum length of an FTS auxiliary table names's id component

e.g., For an auxiliary table name

    FTS_<TABLE_ID>_SUFFIX

This constant is for the minimum length required to store the <TABLE_ID> component.

#define FTS_BACKGROUND_THREAD_WAIT_COUNT 1000

Maximum number of iterations to wait before we complain

#define FTS_FETCH_DOC_BY_ID_EQUAL 1

define for fts_doc_fetch_by_doc_id() "option" value, defines whether

we want to get Doc whose ID is equal to or greater or smaller than supplied ID

#define FTS_ILIST_MAX_SIZE (64 * 1024)

Approx. upper limit of ilist length in bytes.

#define FTS_LAST_OPTIMIZED_WORD "last_optimized_word"

The last word that was OPTIMIZED

#define FTS_MAX_BACKGROUND_THREAD_WAIT 10000

The default time to wait for the background thread (in microsecnds).

#define FTS_MAX_CONFIG_NAME_LEN 64

The maximum length of the config table's value column in bytes

#define FTS_MAX_CONFIG_VALUE_LEN 1024

The maximum length of the config table's value column in bytes

#define FTS_MAX_INT_LEN 32

Maximum length of an integer stored in the config table value column.

#define FTS_OPTIMIZE_END_TIME "optimize_end_time"

End of optimize for an FTS index

#define FTS_OPTIMIZE_LIMIT_IN_SECS "optimize_checkpoint_limit"

FTS config table name parameters The number of seconds after which an OPTIMIZE run will stop

#define FTS_OPTIMIZE_START_TIME "optimize_start_time"

Start of optimize of an FTS index

#define FTS_STOPWORD_TABLE_NAME "stopword_table_name"

User specified stopword table name

#define FTS_SYNCED_DOC_ID "synced_doc_id"

The next doc id

#define FTS_TABLE_STATE "table_state"

State of the FTS system for this table. It can be one of RUNNING, OPTIMIZING, DELETED.

#define FTS_TOTAL_DELETED_COUNT "deleted_doc_count"

Total number of documents that have been deleted. The next_doc_id

minus this count gives us the total number of documents.

#define FTS_TOTAL_WORD_COUNT "total_word_count"

Total number of words parsed from all documents

#define FTS_USE_STOPWORD "use_stopword"

Whether to use (turn on/off) stopword

Enumeration Type Documentation

enum fts_table_state_enum

Enumerator:

FTS_TABLE_STATE_RUNNING	Auxiliary tables created OK
FTS_TABLE_STATE_OPTIMIZING	This is a substate of RUNNING
FTS_TABLE_STATE_DELETED	All aux tables to be dropped when it's safe to do so

Function Documentation

UNIV_INTERN int fts_bsearch	(	fts_update_t *	array,
		int	lower,
		int	upper,
		doc_id_t	doc_id
	)

Do a binary search for a doc id in the array

Returns: +ve index if found -ve index where it should be inserted if not found

Parameters

array	in: array to sort
lower	in: lower bound of array
upper	in: upper bound of array
doc_id	in: doc id to lookup

UNIV_INTERN void fts_cache_append_deleted_doc_ids	(	const fts_cache_t *	cache,
		ib_vector_t *	vector
	)

Append deleted doc ids to vector and sort the vector. in: append to this vector

Parameters

cache in: cache to use

UNIV_INTERN const ib_vector_t* fts_cache_find_word	(	const fts_index_cache_t *	index_cache,
		const fts_string_t *	text
	)

Search cache for word.

Returns: the word node vector if found else NULL

Parameters

index_cache	in: cache to search
text	in: word to search for

UNIV_INTERN ibool fts_cache_is_deleted_doc_id	(	const fts_cache_t *	cache,
		doc_id_t	doc_id
	)

Check cache for deleted doc id.

Returns: TRUE if deleted

Parameters

cache	in: cache ito search
doc_id	in: doc id to search for

UNIV_INTERN void fts_cache_node_add_positions	(	fts_cache_t *	cache,
		fts_node_t *	node,
		doc_id_t	doc_id,
		ib_vector_t *	positions
	)

Add node positions.

Parameters

cache	in: cache
node	in: word node
doc_id	in: doc id
positions	in: fts_token_t::positions

UNIV_INTERN char* fts_config_create_index_param_name	(	const char *	param,
		const dict_index_t *	index
	)

Create the config table name for retrieving index specific value.

Returns: index config parameter name

Parameters

param	in: base name of param
index	in: index for config

UNIV_INTERN dberr_t fts_config_get_index_ulint	(	trx_t *	trx,
		dict_index_t *	index,
		const char *	name,
		ulint *	int_value
	)

Get an ulint value from the config table.

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
index	in: FTS index
name	in: param name
int_value	out: value

UNIV_INTERN dberr_t fts_config_get_index_value	(	trx_t *	trx,
		dict_index_t *	index,
		const char *	param,
		fts_string_t *	value
	)

Get value specific to an FTS index from the config table. The caller must ensure that enough space is allocated for value to hold the column contents.

Returns: DB_SUCCESS or error code

Parameters

trx	transaction
index	in: index
param	in: get config value for this parameter name
value	out: value read from config table

UNIV_INTERN dberr_t fts_config_get_ulint	(	trx_t *	trx,
		fts_table_t *	fts_table,
		const char *	name,
		ulint *	int_value
	)

Get an ulint value from the config table.

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
fts_table	in: the indexed FTS table
name	in: param name
int_value	out: value

UNIV_INTERN dberr_t fts_config_get_value	(	trx_t *	trx,
		fts_table_t *	fts_table,
		const char *	name,
		fts_string_t *	value
	)

Get value from config table. The caller must ensure that enough space is allocated for value to hold the column contents

Returns: DB_SUCCESS or error code

Parameters

fts_table	in: the indexed FTS table
name	in: get config value for this parameter name
value	out: value read from config table

UNIV_INTERN dberr_t fts_config_increment_index_value	(	trx_t *	trx,
		dict_index_t *	index,
		const char *	name,
		ulint	delta
	)

Increment the per index value in the config table for column name.

Returns: DB_SUCCESS or error code

Parameters

trx	transaction
index	in: FTS index
name	in: increment config value for this parameter name
delta	in: increment by this much

UNIV_INTERN dberr_t fts_config_increment_value	(	trx_t *	trx,
		fts_table_t *	fts_table,
		const char *	name,
		ulint	delta
	)

Increment the value in the config table for column name.

Returns: DB_SUCCESS or error code

Parameters

trx	transaction
fts_table	in: the indexed FTS table
name	in: increment config value for this parameter name
delta	in: increment by this much

UNIV_INTERN dberr_t fts_config_set_index_ulint	(	trx_t *	trx,
		dict_index_t *	index,
		const char *	name,
		ulint	int_value
	)

Set an ulint value int the config table.

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
index	in: FTS index
name	in: param name
int_value	in: value

UNIV_INTERN dberr_t fts_config_set_index_value	(	trx_t *	trx,
		dict_index_t *	index,
		const char *	param,
		fts_string_t *	value
	)

Set the value specific to an FTS index in the config table.

Returns: DB_SUCCESS or error code

Parameters

trx	transaction
index	in: index
param	in: get config value for this parameter name
value	out: value read from config table

UNIV_INTERN dberr_t fts_config_set_ulint	(	trx_t *	trx,
		fts_table_t *	fts_table,
		const char *	name,
		ulint	int_value
	)

Set an ulint value in the config table.

Returns: DB_SUCCESS if all OK else error code

Parameters

trx	in: transaction
fts_table	in: the indexed FTS table
name	in: param name
int_value	in: value

UNIV_INTERN dberr_t fts_config_set_value	(	trx_t *	trx,
		fts_table_t *	fts_table,
		const char *	name,
		const fts_string_t *	value
	)

Set the value in the config table for name.

Returns: DB_SUCCESS or error code

Parameters

trx	transaction
fts_table	in: the indexed FTS table
name	in: get config value for this parameter name
value	in: value to update

UNIV_INTERN dberr_t fts_doc_fetch_by_doc_id	(	fts_get_doc_t *	get_doc,
		doc_id_t	doc_id,
		dict_index_t *	index_to_use,
		ulint	option,
		fts_sql_callback	callback,
		void *	arg
	)

Fetch document (= a single row's indexed text) with the given document id.

Returns: : DB_SUCCESS if fetch is successful, else error

Parameters

get_doc	in: state
doc_id	in: id of document to fetch
index_to_use	in: caller supplied FTS index, or NULL
option	in: search option, if it is greater than doc_id or equal
callback	in: callback to read records
arg	in: callback arg

UNIV_INTERN void fts_doc_free ( fts_doc_t * doc )

Free document.

Parameters

doc	in: document

UNIV_INTERN void fts_doc_init ( fts_doc_t * doc )

Initialize a document.

Parameters

doc	in: doc to initialize

UNIV_INTERN dberr_t fts_eval_sql	(	trx_t *	trx,
		que_t *	graph
	)

Evaluate a parsed SQL statement

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
graph	in: Parsed statement

UNIV_INTERN fts_index_cache_t* fts_find_index_cache	(	const fts_cache_t *	cache,
		const dict_index_t *	index
	)

Search the index specific cache for a particular FTS index.

Returns: the index specific cache else NULL

Parameters

cache	in: cache to search
index	in: index to search for

UNIV_INTERN const char* fts_get_select_columns_str	(	dict_index_t *	index,
		pars_info_t *	info,
		mem_heap_t *	heap
	)

Construct the column specification part of the SQL string for selecting the indexed FTS columns for the given table. Adds the necessary bound ids to the given 'info' and returns the SQL string. Examples:

One indexed column named "text":

"$sel0", info/ids: sel0 -> "text"

Two indexed columns named "subject" and "content":

"$sel0, $sel1", info/ids: sel0 -> "subject", sel1 -> "content",

Returns: heap-allocated WHERE string

Parameters

index	in: FTS index
info	in/out: parser info
heap	in: memory heap

UNIV_INTERN int fts_get_table_id	(	const fts_table_t *	fts_table,
		char *	table_id
	)

Get the table id.

Returns: number of bytes written

Parameters

fts_table	in: FTS Auxiliary table
table_id	out: table id, must be at least FTS_AUX_MIN_TABLE_ID_LENGTH bytes long

UNIV_INTERN char* fts_get_table_name ( const fts_table_t * fts_table )

Construct the name of an ancillary FTS table for the given table.

Returns: own: table name, must be freed with mem_free()

Parameters

fts_table in: FTS aux table info

UNIV_INTERN char* fts_get_table_name_prefix ( const fts_table_t * fts_table )

Construct the prefix name of an FTS table.

Returns: own: table name, must be freed with mem_free()

Parameters

fts_table in: Auxiliary table type

UNIV_INTERN dberr_t fts_index_fetch_nodes	(	trx_t *	trx,
		que_t **	graph,
		fts_table_t *	fts_table,
		const fts_string_t *	word,
		fts_fetch_t *	fetch
	)

Read the rows from the FTS inde

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
graph	in: prepared statement
fts_table	in: FTS aux table
word	in: the word to fetch
fetch	in: fetch callback.

UNIV_INTERN void fts_optimize_add_table ( dict_table_t * table )

Add the table to add to the OPTIMIZER's list.

Parameters

table in: table to add

UNIV_INTERN void fts_optimize_do_table ( dict_table_t * table )

Optimize a table.

Parameters

table in: table to optimize

UNIV_INTERN que_t* fts_parse_sql	(	fts_table_t *	fts_table,
		pars_info_t *	info,
		const char *	sql
	)

Parse an SQL string. s is replaced with the table's id.

Returns: query graph

Parameters

fts_table	in: FTS aux table
info	in: info struct, or NULL
sql	in: SQL string to evaluate

UNIV_INTERN que_t* fts_parse_sql_no_dict_lock	(	fts_table_t *	fts_table,
		pars_info_t *	info,
		const char *	sql
	)

Parse an SQL string. s is replaced with the table's id. Don't acquire the dict mutex

Returns: query graph

Parameters

fts_table	in: table with FTS index
info	in: parser info
sql	in: SQL string to evaluate

UNIV_INTERN ibool fts_query_expansion_fetch_doc	(	void *	row,
		void *	user_arg
	)

Callback function for fetch that stores the text of an FTS document, converting each column to UTF-16.

Returns: always FALSE

Parameters

row	in: sel_node_t*
user_arg	in: fts_doc_t*

UNIV_INLINE ibool fts_read_object_id	(	ib_id_t *	id,
		const char *	str
	)

Read the table id from the string generated by fts_write_object_id().

Returns: TRUE if parse successful

Parameters

id	out: a table id
str	in: buffer to read from

UNIV_INTERN dberr_t fts_sql_commit ( trx_t * trx )

Commit a transaction.

Returns: DB_SUCCESS if all OK

Parameters

trx	in: transaction

UNIV_INTERN dberr_t fts_sql_rollback ( trx_t * trx )

Rollback a transaction.

Returns: DB_SUCCESS if all OK

Parameters

trx	in: transaction

UNIV_INTERN void fts_tokenize_document	(	fts_doc_t *	doc,
		fts_doc_t *	result
	)

Tokenize a document.

Parameters

doc	in/out: document to tokenize
result	out: if provided, save result tokens here

UNIV_INTERN void fts_tokenize_document_next	(	fts_doc_t *	doc,
		ulint	add_pos,
		fts_doc_t *	result
	)

Continue to tokenize a document.

Parameters

doc	in/out: document to tokenize
add_pos	in: add this position to all tokens from this tokenization
result	out: if provided, save result tokens here

UNIV_INLINE int fts_trx_table_cmp	(	const void *	p1,
		const void *	p2
	)

Compare two fts_trx_table_t instances, we actually compare the table id's here.

Returns: < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2

Compare two fts_trx_table_t instances.

Returns: < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2

Parameters

p1	in: id1
p2	in: id2

UNIV_INLINE int fts_trx_table_id_cmp	(	const void *	p1,
		const void *	p2
	)

Compare a table id with a trx_table_t table id.

Returns: < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2

Compare a table id with a fts_trx_table_t table id.

Returns: < 0 if n1 < n2, 0 if n1 == n2,> 0 if n1 > n2

Parameters

p1	in: id1
p2	in: id2

UNIV_INTERN ibool fts_wait_for_background_thread_to_start	(	dict_table_t *	table,
		ulint	max_wait
	)

Wait for the background thread to start. We poll to detect change of state, which is acceptable, since the wait should happen only once during startup.

Returns: true if the thread started else FALSE (i.e timed out) in: time in microseconds, if set to 0 then it disables timeout checking

Parameters

table in: table to which the thread is attached

UNIV_INTERN void fts_word_free ( fts_word_t * word )

Free fts_optimizer_word_t instanace.

Parameters

word	in: instance to free.

UNIV_INTERN fts_word_t* fts_word_init	(	fts_word_t *	word,
		byte *	utf8,
		ulint	len
	)

Create a fts_optimizer_word_t instance.

Returns: new instance

Parameters

word	in: word to initialize
utf8	in: UTF-8 string
len	in: length of string in bytes

UNIV_INTERN dberr_t fts_write_node	(	trx_t *	trx,
		que_t **	graph,
		fts_table_t *	fts_table,
		fts_string_t *	word,
		fts_node_t *	node
	)

Parameters

trx	in: transaction
graph	in: query graph
fts_table	in: the FTS aux index
word	in: word in UTF-8
node	in: node columns

UNIV_INLINE int fts_write_object_id	(	ib_id_t	id,
		char *	str,
		bool	hex_format
	)

Write the table id to the given buffer (including final NUL). Buffer must be at least FTS_AUX_MIN_TABLE_ID_LENGTH bytes long.

Returns: number of bytes written

< in: true for fixed hex format, false for old ambiguous format

Write the table id to the given buffer (including final NUL). Buffer must be at least FTS_AUX_MIN_TABLE_ID_LENGTH bytes long.

Returns: number of bytes written

Parameters

id	in: a table/index id
str	in: buffer to write the id to

Macros

Typedefs

Enumerations

Functions

Variables

Detailed Description

Macro Definition Documentation

Enumeration Type Documentation

Function Documentation