Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * ts_typanalyze.c
4 : * functions for gathering statistics from tsvector columns
5 : *
6 : * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
7 : *
8 : *
9 : * IDENTIFICATION
10 : * src/backend/tsearch/ts_typanalyze.c
11 : *
12 : *-------------------------------------------------------------------------
13 : */
14 : #include "postgres.h"
15 :
16 : #include "catalog/pg_collation.h"
17 : #include "catalog/pg_operator.h"
18 : #include "commands/vacuum.h"
19 : #include "common/hashfn.h"
20 : #include "tsearch/ts_type.h"
21 : #include "utils/builtins.h"
22 : #include "varatt.h"
23 :
24 :
25 : /* A hash key for lexemes */
26 : typedef struct
27 : {
28 : char *lexeme; /* lexeme (not NULL terminated!) */
29 : int length; /* its length in bytes */
30 : } LexemeHashKey;
31 :
32 : /* A hash table entry for the Lossy Counting algorithm */
33 : typedef struct
34 : {
35 : LexemeHashKey key; /* This is 'e' from the LC algorithm. */
36 : int frequency; /* This is 'f'. */
37 : int delta; /* And this is 'delta'. */
38 : } TrackItem;
39 :
40 : static void compute_tsvector_stats(VacAttrStats *stats,
41 : AnalyzeAttrFetchFunc fetchfunc,
42 : int samplerows,
43 : double totalrows);
44 : static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current);
45 : static uint32 lexeme_hash(const void *key, Size keysize);
46 : static int lexeme_match(const void *key1, const void *key2, Size keysize);
47 : static int lexeme_compare(const void *key1, const void *key2);
48 : static int trackitem_compare_frequencies_desc(const void *e1, const void *e2,
49 : void *arg);
50 : static int trackitem_compare_lexemes(const void *e1, const void *e2,
51 : void *arg);
52 :
53 :
54 : /*
55 : * ts_typanalyze -- a custom typanalyze function for tsvector columns
56 : */
57 : Datum
58 6 : ts_typanalyze(PG_FUNCTION_ARGS)
59 : {
60 6 : VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
61 6 : Form_pg_attribute attr = stats->attr;
62 :
63 : /* If the attstattarget column is negative, use the default value */
64 : /* NB: it is okay to scribble on stats->attr since it's a copy */
65 6 : if (attr->attstattarget < 0)
66 6 : attr->attstattarget = default_statistics_target;
67 :
68 6 : stats->compute_stats = compute_tsvector_stats;
69 : /* see comment about the choice of minrows in commands/analyze.c */
70 6 : stats->minrows = 300 * attr->attstattarget;
71 :
72 6 : PG_RETURN_BOOL(true);
73 : }
74 :
75 : /*
76 : * compute_tsvector_stats() -- compute statistics for a tsvector column
77 : *
78 : * This functions computes statistics that are useful for determining @@
79 : * operations' selectivity, along with the fraction of non-null rows and
80 : * average width.
81 : *
82 : * Instead of finding the most common values, as we do for most datatypes,
83 : * we're looking for the most common lexemes. This is more useful, because
84 : * there most probably won't be any two rows with the same tsvector and thus
85 : * the notion of a MCV is a bit bogus with this datatype. With a list of the
86 : * most common lexemes we can do a better job at figuring out @@ selectivity.
87 : *
88 : * For the same reasons we assume that tsvector columns are unique when
89 : * determining the number of distinct values.
90 : *
91 : * The algorithm used is Lossy Counting, as proposed in the paper "Approximate
92 : * frequency counts over data streams" by G. S. Manku and R. Motwani, in
93 : * Proceedings of the 28th International Conference on Very Large Data Bases,
94 : * Hong Kong, China, August 2002, section 4.2. The paper is available at
95 : * http://www.vldb.org/conf/2002/S10P03.pdf
96 : *
97 : * The Lossy Counting (aka LC) algorithm goes like this:
98 : * Let s be the threshold frequency for an item (the minimum frequency we
99 : * are interested in) and epsilon the error margin for the frequency. Let D
100 : * be a set of triples (e, f, delta), where e is an element value, f is that
101 : * element's frequency (actually, its current occurrence count) and delta is
102 : * the maximum error in f. We start with D empty and process the elements in
103 : * batches of size w. (The batch size is also known as "bucket size" and is
104 : * equal to 1/epsilon.) Let the current batch number be b_current, starting
105 : * with 1. For each element e we either increment its f count, if it's
106 : * already in D, or insert a new triple into D with values (e, 1, b_current
107 : * - 1). After processing each batch we prune D, by removing from it all
108 : * elements with f + delta <= b_current. After the algorithm finishes we
109 : * suppress all elements from D that do not satisfy f >= (s - epsilon) * N,
110 : * where N is the total number of elements in the input. We emit the
111 : * remaining elements with estimated frequency f/N. The LC paper proves
112 : * that this algorithm finds all elements with true frequency at least s,
113 : * and that no frequency is overestimated or is underestimated by more than
114 : * epsilon. Furthermore, given reasonable assumptions about the input
115 : * distribution, the required table size is no more than about 7 times w.
116 : *
117 : * We set s to be the estimated frequency of the K'th word in a natural
118 : * language's frequency table, where K is the target number of entries in
119 : * the MCELEM array plus an arbitrary constant, meant to reflect the fact
120 : * that the most common words in any language would usually be stopwords
121 : * so we will not actually see them in the input. We assume that the
122 : * distribution of word frequencies (including the stopwords) follows Zipf's
123 : * law with an exponent of 1.
124 : *
125 : * Assuming Zipfian distribution, the frequency of the K'th word is equal
126 : * to 1/(K * H(W)) where H(n) is 1/2 + 1/3 + ... + 1/n and W is the number of
127 : * words in the language. Putting W as one million, we get roughly 0.07/K.
128 : * Assuming top 10 words are stopwords gives s = 0.07/(K + 10). We set
129 : * epsilon = s/10, which gives bucket width w = (K + 10)/0.007 and
130 : * maximum expected hashtable size of about 1000 * (K + 10).
131 : *
132 : * Note: in the above discussion, s, epsilon, and f/N are in terms of a
133 : * lexeme's frequency as a fraction of all lexemes seen in the input.
134 : * However, what we actually want to store in the finished pg_statistic
135 : * entry is each lexeme's frequency as a fraction of all rows that it occurs
136 : * in. Assuming that the input tsvectors are correctly constructed, no
137 : * lexeme occurs more than once per tsvector, so the final count f is a
138 : * correct estimate of the number of input tsvectors it occurs in, and we
139 : * need only change the divisor from N to nonnull_cnt to get the number we
140 : * want.
141 : */
142 : static void
143 6 : compute_tsvector_stats(VacAttrStats *stats,
144 : AnalyzeAttrFetchFunc fetchfunc,
145 : int samplerows,
146 : double totalrows)
147 : {
148 : int num_mcelem;
149 6 : int null_cnt = 0;
150 6 : double total_width = 0;
151 :
152 : /* This is D from the LC algorithm. */
153 : HTAB *lexemes_tab;
154 : HASHCTL hash_ctl;
155 : HASH_SEQ_STATUS scan_status;
156 :
157 : /* This is the current bucket number from the LC algorithm */
158 : int b_current;
159 :
160 : /* This is 'w' from the LC algorithm */
161 : int bucket_width;
162 : int vector_no,
163 : lexeme_no;
164 : LexemeHashKey hash_key;
165 :
166 : /*
167 : * We want statistics_target * 10 lexemes in the MCELEM array. This
168 : * multiplier is pretty arbitrary, but is meant to reflect the fact that
169 : * the number of individual lexeme values tracked in pg_statistic ought to
170 : * be more than the number of values for a simple scalar column.
171 : */
172 6 : num_mcelem = stats->attr->attstattarget * 10;
173 :
174 : /*
175 : * We set bucket width equal to (num_mcelem + 10) / 0.007 as per the
176 : * comment above.
177 : */
178 6 : bucket_width = (num_mcelem + 10) * 1000 / 7;
179 :
180 : /*
181 : * Create the hashtable. It will be in local memory, so we don't need to
182 : * worry about overflowing the initial size. Also we don't need to pay any
183 : * attention to locking and memory management.
184 : */
185 6 : hash_ctl.keysize = sizeof(LexemeHashKey);
186 6 : hash_ctl.entrysize = sizeof(TrackItem);
187 6 : hash_ctl.hash = lexeme_hash;
188 6 : hash_ctl.match = lexeme_match;
189 6 : hash_ctl.hcxt = CurrentMemoryContext;
190 6 : lexemes_tab = hash_create("Analyzed lexemes table",
191 : num_mcelem,
192 : &hash_ctl,
193 : HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
194 :
195 : /* Initialize counters. */
196 6 : b_current = 1;
197 6 : lexeme_no = 0;
198 :
199 : /* Loop over the tsvectors. */
200 3054 : for (vector_no = 0; vector_no < samplerows; vector_no++)
201 : {
202 : Datum value;
203 : bool isnull;
204 : TSVector vector;
205 : WordEntry *curentryptr;
206 : char *lexemesptr;
207 : int j;
208 :
209 3048 : vacuum_delay_point();
210 :
211 3048 : value = fetchfunc(stats, vector_no, &isnull);
212 :
213 : /*
214 : * Check for null/nonnull.
215 : */
216 3048 : if (isnull)
217 : {
218 0 : null_cnt++;
219 0 : continue;
220 : }
221 :
222 : /*
223 : * Add up widths for average-width calculation. Since it's a
224 : * tsvector, we know it's varlena. As in the regular
225 : * compute_minimal_stats function, we use the toasted width for this
226 : * calculation.
227 : */
228 3048 : total_width += VARSIZE_ANY(DatumGetPointer(value));
229 :
230 : /*
231 : * Now detoast the tsvector if needed.
232 : */
233 3048 : vector = DatumGetTSVector(value);
234 :
235 : /*
236 : * We loop through the lexemes in the tsvector and add them to our
237 : * tracking hashtable.
238 : */
239 3048 : lexemesptr = STRPTR(vector);
240 3048 : curentryptr = ARRPTR(vector);
241 175848 : for (j = 0; j < vector->size; j++)
242 : {
243 : TrackItem *item;
244 : bool found;
245 :
246 : /*
247 : * Construct a hash key. The key points into the (detoasted)
248 : * tsvector value at this point, but if a new entry is created, we
249 : * make a copy of it. This way we can free the tsvector value
250 : * once we've processed all its lexemes.
251 : */
252 172800 : hash_key.lexeme = lexemesptr + curentryptr->pos;
253 172800 : hash_key.length = curentryptr->len;
254 :
255 : /* Lookup current lexeme in hashtable, adding it if new */
256 172800 : item = (TrackItem *) hash_search(lexemes_tab,
257 : &hash_key,
258 : HASH_ENTER, &found);
259 :
260 172800 : if (found)
261 : {
262 : /* The lexeme is already on the tracking list */
263 165954 : item->frequency++;
264 : }
265 : else
266 : {
267 : /* Initialize new tracking list element */
268 6846 : item->frequency = 1;
269 6846 : item->delta = b_current - 1;
270 :
271 6846 : item->key.lexeme = palloc(hash_key.length);
272 6846 : memcpy(item->key.lexeme, hash_key.lexeme, hash_key.length);
273 : }
274 :
275 : /* lexeme_no is the number of elements processed (ie N) */
276 172800 : lexeme_no++;
277 :
278 : /* We prune the D structure after processing each bucket */
279 172800 : if (lexeme_no % bucket_width == 0)
280 : {
281 0 : prune_lexemes_hashtable(lexemes_tab, b_current);
282 0 : b_current++;
283 : }
284 :
285 : /* Advance to the next WordEntry in the tsvector */
286 172800 : curentryptr++;
287 : }
288 :
289 : /* If the vector was toasted, free the detoasted copy. */
290 3048 : if (TSVectorGetDatum(vector) != value)
291 384 : pfree(vector);
292 : }
293 :
294 : /* We can only compute real stats if we found some non-null values. */
295 6 : if (null_cnt < samplerows)
296 : {
297 6 : int nonnull_cnt = samplerows - null_cnt;
298 : int i;
299 : TrackItem **sort_table;
300 : TrackItem *item;
301 : int track_len;
302 : int cutoff_freq;
303 : int minfreq,
304 : maxfreq;
305 :
306 6 : stats->stats_valid = true;
307 : /* Do the simple null-frac and average width stats */
308 6 : stats->stanullfrac = (double) null_cnt / (double) samplerows;
309 6 : stats->stawidth = total_width / (double) nonnull_cnt;
310 :
311 : /* Assume it's a unique column (see notes above) */
312 6 : stats->stadistinct = -1.0 * (1.0 - stats->stanullfrac);
313 :
314 : /*
315 : * Construct an array of the interesting hashtable items, that is,
316 : * those meeting the cutoff frequency (s - epsilon)*N. Also identify
317 : * the minimum and maximum frequencies among these items.
318 : *
319 : * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
320 : * frequency is 9*N / bucket_width.
321 : */
322 6 : cutoff_freq = 9 * lexeme_no / bucket_width;
323 :
324 6 : i = hash_get_num_entries(lexemes_tab); /* surely enough space */
325 6 : sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * i);
326 :
327 6 : hash_seq_init(&scan_status, lexemes_tab);
328 6 : track_len = 0;
329 6 : minfreq = lexeme_no;
330 6 : maxfreq = 0;
331 6852 : while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
332 : {
333 6846 : if (item->frequency > cutoff_freq)
334 : {
335 6318 : sort_table[track_len++] = item;
336 6318 : minfreq = Min(minfreq, item->frequency);
337 6318 : maxfreq = Max(maxfreq, item->frequency);
338 : }
339 : }
340 : Assert(track_len <= i);
341 :
342 : /* emit some statistics for debug purposes */
343 6 : elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, "
344 : "# lexemes = %d, hashtable size = %d, usable entries = %d",
345 : num_mcelem, bucket_width, lexeme_no, i, track_len);
346 :
347 : /*
348 : * If we obtained more lexemes than we really want, get rid of those
349 : * with least frequencies. The easiest way is to qsort the array into
350 : * descending frequency order and truncate the array.
351 : */
352 6 : if (num_mcelem < track_len)
353 : {
354 6 : qsort_interruptible(sort_table, track_len, sizeof(TrackItem *),
355 : trackitem_compare_frequencies_desc, NULL);
356 : /* reset minfreq to the smallest frequency we're keeping */
357 6 : minfreq = sort_table[num_mcelem - 1]->frequency;
358 : }
359 : else
360 0 : num_mcelem = track_len;
361 :
362 : /* Generate MCELEM slot entry */
363 6 : if (num_mcelem > 0)
364 : {
365 : MemoryContext old_context;
366 : Datum *mcelem_values;
367 : float4 *mcelem_freqs;
368 :
369 : /*
370 : * We want to store statistics sorted on the lexeme value using
371 : * first length, then byte-for-byte comparison. The reason for
372 : * doing length comparison first is that we don't care about the
373 : * ordering so long as it's consistent, and comparing lengths
374 : * first gives us a chance to avoid a strncmp() call.
375 : *
376 : * This is different from what we do with scalar statistics --
377 : * they get sorted on frequencies. The rationale is that we
378 : * usually search through most common elements looking for a
379 : * specific value, so we can grab its frequency. When values are
380 : * presorted we can employ binary search for that. See
381 : * ts_selfuncs.c for a real usage scenario.
382 : */
383 6 : qsort_interruptible(sort_table, num_mcelem, sizeof(TrackItem *),
384 : trackitem_compare_lexemes, NULL);
385 :
386 : /* Must copy the target values into anl_context */
387 6 : old_context = MemoryContextSwitchTo(stats->anl_context);
388 :
389 : /*
390 : * We sorted statistics on the lexeme value, but we want to be
391 : * able to find out the minimal and maximal frequency without
392 : * going through all the values. We keep those two extra
393 : * frequencies in two extra cells in mcelem_freqs.
394 : *
395 : * (Note: the MCELEM statistics slot definition allows for a third
396 : * extra number containing the frequency of nulls, but we don't
397 : * create that for a tsvector column, since null elements aren't
398 : * possible.)
399 : */
400 6 : mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
401 6 : mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4));
402 :
403 : /*
404 : * See comments above about use of nonnull_cnt as the divisor for
405 : * the final frequency estimates.
406 : */
407 6006 : for (i = 0; i < num_mcelem; i++)
408 : {
409 6000 : TrackItem *titem = sort_table[i];
410 :
411 12000 : mcelem_values[i] =
412 6000 : PointerGetDatum(cstring_to_text_with_len(titem->key.lexeme,
413 : titem->key.length));
414 6000 : mcelem_freqs[i] = (double) titem->frequency / (double) nonnull_cnt;
415 : }
416 6 : mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
417 6 : mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
418 6 : MemoryContextSwitchTo(old_context);
419 :
420 6 : stats->stakind[0] = STATISTIC_KIND_MCELEM;
421 6 : stats->staop[0] = TextEqualOperator;
422 6 : stats->stacoll[0] = DEFAULT_COLLATION_OID;
423 6 : stats->stanumbers[0] = mcelem_freqs;
424 : /* See above comment about two extra frequency fields */
425 6 : stats->numnumbers[0] = num_mcelem + 2;
426 6 : stats->stavalues[0] = mcelem_values;
427 6 : stats->numvalues[0] = num_mcelem;
428 : /* We are storing text values */
429 6 : stats->statypid[0] = TEXTOID;
430 6 : stats->statyplen[0] = -1; /* typlen, -1 for varlena */
431 6 : stats->statypbyval[0] = false;
432 6 : stats->statypalign[0] = 'i';
433 : }
434 : }
435 : else
436 : {
437 : /* We found only nulls; assume the column is entirely null */
438 0 : stats->stats_valid = true;
439 0 : stats->stanullfrac = 1.0;
440 0 : stats->stawidth = 0; /* "unknown" */
441 0 : stats->stadistinct = 0.0; /* "unknown" */
442 : }
443 :
444 : /*
445 : * We don't need to bother cleaning up any of our temporary palloc's. The
446 : * hashtable should also go away, as it used a child memory context.
447 : */
448 6 : }
449 :
450 : /*
451 : * A function to prune the D structure from the Lossy Counting algorithm.
452 : * Consult compute_tsvector_stats() for wider explanation.
453 : */
454 : static void
455 0 : prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
456 : {
457 : HASH_SEQ_STATUS scan_status;
458 : TrackItem *item;
459 :
460 0 : hash_seq_init(&scan_status, lexemes_tab);
461 0 : while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
462 : {
463 0 : if (item->frequency + item->delta <= b_current)
464 : {
465 0 : char *lexeme = item->key.lexeme;
466 :
467 0 : if (hash_search(lexemes_tab, &item->key,
468 : HASH_REMOVE, NULL) == NULL)
469 0 : elog(ERROR, "hash table corrupted");
470 0 : pfree(lexeme);
471 : }
472 : }
473 0 : }
474 :
475 : /*
476 : * Hash functions for lexemes. They are strings, but not NULL terminated,
477 : * so we need a special hash function.
478 : */
479 : static uint32
480 172800 : lexeme_hash(const void *key, Size keysize)
481 : {
482 172800 : const LexemeHashKey *l = (const LexemeHashKey *) key;
483 :
484 172800 : return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
485 : l->length));
486 : }
487 :
488 : /*
489 : * Matching function for lexemes, to be used in hashtable lookups.
490 : */
491 : static int
492 165954 : lexeme_match(const void *key1, const void *key2, Size keysize)
493 : {
494 : /* The keysize parameter is superfluous, the keys store their lengths */
495 165954 : return lexeme_compare(key1, key2);
496 : }
497 :
498 : /*
499 : * Comparison function for lexemes.
500 : */
501 : static int
502 227028 : lexeme_compare(const void *key1, const void *key2)
503 : {
504 227028 : const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
505 227028 : const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
506 :
507 : /* First, compare by length */
508 227028 : if (d1->length > d2->length)
509 0 : return 1;
510 227028 : else if (d1->length < d2->length)
511 0 : return -1;
512 : /* Lengths are equal, do a byte-by-byte comparison */
513 227028 : return strncmp(d1->lexeme, d2->lexeme, d1->length);
514 : }
515 :
516 : /*
517 : * Comparator for sorting TrackItems on frequencies (descending sort)
518 : */
519 : static int
520 38460 : trackitem_compare_frequencies_desc(const void *e1, const void *e2, void *arg)
521 : {
522 38460 : const TrackItem *const *t1 = (const TrackItem *const *) e1;
523 38460 : const TrackItem *const *t2 = (const TrackItem *const *) e2;
524 :
525 38460 : return (*t2)->frequency - (*t1)->frequency;
526 : }
527 :
528 : /*
529 : * Comparator for sorting TrackItems on lexemes
530 : */
531 : static int
532 61074 : trackitem_compare_lexemes(const void *e1, const void *e2, void *arg)
533 : {
534 61074 : const TrackItem *const *t1 = (const TrackItem *const *) e1;
535 61074 : const TrackItem *const *t2 = (const TrackItem *const *) e2;
536 :
537 61074 : return lexeme_compare(&(*t1)->key, &(*t2)->key);
538 : }
|
