Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * tupmacs.h
4 : : * Tuple macros used by both index tuples and heap tuples.
5 : : *
6 : : *
7 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
8 : : * Portions Copyright (c) 1994, Regents of the University of California
9 : : *
10 : : * src/include/access/tupmacs.h
11 : : *
12 : : *-------------------------------------------------------------------------
13 : : */
14 : : #ifndef TUPMACS_H
15 : : #define TUPMACS_H
16 : :
17 : : #include "catalog/pg_type_d.h" /* for TYPALIGN macros */
18 : : #include "common/int.h"
19 : : #include "port/pg_bitutils.h"
20 : : #include "port/pg_bswap.h"
21 : : #include "varatt.h"
22 : :
23 : : /*
24 : : * Check a tuple's null bitmap to determine whether the attribute is null.
25 : : * Note that a 0 in the null bitmap indicates a null, while 1 indicates
26 : : * non-null.
27 : : */
28 : : static inline bool
29 : 147469666 : att_isnull(int ATT, const uint8 *BITS)
30 : : {
31 : 147469666 : return !(BITS[ATT >> 3] & (1 << (ATT & 0x07)));
32 : : }
33 : :
34 : : /*
35 : : * populate_isnull_array
36 : : * Transform a tuple's null bitmap into a boolean array.
37 : : *
38 : : * Caller must ensure that the isnull array is sized so it contains
39 : : * at least as many elements as there are bits in the 'bits' array.
40 : : * Callers should be aware that isnull is populated 8 elements at a time,
41 : : * effectively as if natts is rounded up to the next multiple of 8.
42 : : */
43 : : static inline void
44 : 27806631 : populate_isnull_array(const uint8 *bits, int natts, bool *isnull)
45 : : {
46 : 27806631 : int nbytes = (natts + 7) >> 3;
47 : :
48 : : /*
49 : : * Multiplying the inverted NULL bitmap byte by this value results in the
50 : : * lowest bit in each byte being set the same as each bit of the inverted
51 : : * byte. We perform this as 2 32-bit operations rather than a single
52 : : * 64-bit operation as multiplying by the required value to do this in
53 : : * 64-bits would result in overflowing a uint64 in some cases.
54 : : *
55 : : * XXX if we ever require BMI2 (-march=x86-64-v3), then this could be done
56 : : * more efficiently on most X86-64 CPUs with the PDEP instruction. Beware
57 : : * that some chips (e.g. AMD's Zen2) are horribly inefficient at PDEP.
58 : : */
59 : : #define SPREAD_BITS_MULTIPLIER_32 0x204081U
60 : :
61 [ + + ]: 79315026 : for (int i = 0; i < nbytes; i++, isnull += 8)
62 : : {
63 : : uint64 isnull_8;
64 : 51508395 : uint8 nullbyte = ~bits[i];
65 : :
66 : : /* Convert the lower 4 bits of NULL bitmap word into a 64 bit int */
67 : 51508395 : isnull_8 = (nullbyte & 0xf) * SPREAD_BITS_MULTIPLIER_32;
68 : :
69 : : /*
70 : : * Convert the upper 4 bits of NULL bitmap word into a 64 bit int,
71 : : * shift into the upper 32 bit and bitwise-OR with the result of the
72 : : * lower 4 bits.
73 : : */
74 : 51508395 : isnull_8 |= ((uint64) ((nullbyte >> 4) * SPREAD_BITS_MULTIPLIER_32)) << 32;
75 : :
76 : : /* Mask out all other bits apart from the lowest bit of each byte. */
77 : 51508395 : isnull_8 &= UINT64CONST(0x0101010101010101);
78 : :
79 : : #ifdef WORDS_BIGENDIAN
80 : :
81 : : /*
82 : : * Fix byte order on big-endian machines before copying to the array.
83 : : */
84 : : isnull_8 = pg_bswap64(isnull_8);
85 : : #endif
86 : 51508395 : memcpy(isnull, &isnull_8, sizeof(uint64));
87 : : }
88 : 27806631 : }
89 : :
90 : : #ifndef FRONTEND
91 : : /*
92 : : * Given an attbyval and an attlen from either a Form_pg_attribute or
93 : : * CompactAttribute and a pointer into a tuple's data area, return the
94 : : * correct value or pointer.
95 : : *
96 : : * We return a Datum value in all cases. If attbyval is false, we return the
97 : : * same pointer into the tuple data area that we're passed. Otherwise, we
98 : : * return the correct number of bytes fetched from the data area and extended
99 : : * to Datum form.
100 : : *
101 : : * Note that T must already be properly aligned for this to work correctly.
102 : : */
103 : : #define fetchatt(A,T) fetch_att(T, (A)->attbyval, (A)->attlen)
104 : :
105 : : /*
106 : : * Same, but work from byval/len parameters rather than Form_pg_attribute.
107 : : */
108 : : static inline Datum
109 : 662311555 : fetch_att(const void *T, bool attbyval, int attlen)
110 : : {
111 [ + + ]: 662311555 : if (attbyval)
112 : : {
113 [ + + + + : 547539402 : switch (attlen)
- ]
114 : : {
115 : 17664416 : case sizeof(char):
116 : 17664416 : return CharGetDatum(*((const char *) T));
117 : 29426779 : case sizeof(int16):
118 : 29426779 : return Int16GetDatum(*((const int16 *) T));
119 : 478839901 : case sizeof(int32):
120 : 478839901 : return Int32GetDatum(*((const int32 *) T));
121 : 21608306 : case sizeof(int64):
122 : 21608306 : return Int64GetDatum(*((const int64 *) T));
123 : 0 : default:
124 [ # # ]: 0 : elog(ERROR, "unsupported byval length: %d", attlen);
125 : : return 0;
126 : : }
127 : : }
128 : : else
129 : 114772153 : return PointerGetDatum(T);
130 : : }
131 : :
132 : : /*
133 : : * Same as fetch_att, but no error checking for invalid attlens for byval
134 : : * types. This is safe to use when attlen comes from CompactAttribute as we
135 : : * validate the length when populating that struct.
136 : : */
137 : : static inline Datum
138 : 417521439 : fetch_att_noerr(const void *T, bool attbyval, int attlen)
139 : : {
140 [ + + ]: 417521439 : if (attbyval)
141 : : {
142 [ + + + + ]: 385004331 : switch (attlen)
143 : : {
144 : 254753909 : case sizeof(int32):
145 : 254753909 : return Int32GetDatum(*((const int32 *) T));
146 : 24986879 : case sizeof(int16):
147 : 24986879 : return Int16GetDatum(*((const int16 *) T));
148 : 91730269 : case sizeof(char):
149 : 91730269 : return CharGetDatum(*((const char *) T));
150 : 13533274 : default:
151 : : Assert(attlen == sizeof(int64));
152 : 13533274 : return Int64GetDatum(*((const int64 *) T));
153 : : }
154 : : }
155 : : else
156 : 32517108 : return PointerGetDatum(T);
157 : : }
158 : :
159 : :
160 : : /*
161 : : * align_fetch_then_add
162 : : * Applies all the functionality of att_pointer_alignby(),
163 : : * fetch_att_noerr() and att_addlength_pointer(), resulting in the *off
164 : : * pointer to the perhaps unaligned number of bytes into 'tupptr', ready
165 : : * to deform the next attribute.
166 : : *
167 : : * tupptr: pointer to the beginning of the tuple, after the header and any
168 : : * NULL bitmask.
169 : : * off: offset in bytes for reading tuple data, possibly unaligned.
170 : : * attbyval, attlen and attalignby are values from CompactAttribute.
171 : : */
172 : : static inline Datum
173 : 46813874 : align_fetch_then_add(const char *tupptr, uint32 *off, bool attbyval, int attlen,
174 : : uint8 attalignby)
175 : : {
176 : : Datum res;
177 : :
178 [ + + ]: 46813874 : if (attlen > 0)
179 : : {
180 : : const char *offset_ptr;
181 : :
182 : 16403699 : *off = TYPEALIGN(attalignby, *off);
183 : 16403699 : offset_ptr = tupptr + *off;
184 : 16403699 : *off += attlen;
185 [ + + ]: 16403699 : if (attbyval)
186 : : {
187 [ + + + + ]: 15626921 : switch (attlen)
188 : : {
189 : 1836545 : case sizeof(char):
190 : 1836545 : return CharGetDatum(*((const char *) offset_ptr));
191 : 402069 : case sizeof(int16):
192 : 402069 : return Int16GetDatum(*((const int16 *) offset_ptr));
193 : 11935648 : case sizeof(int32):
194 : 11935648 : return Int32GetDatum(*((const int32 *) offset_ptr));
195 : 1452659 : default:
196 : :
197 : : /*
198 : : * populate_compact_attribute_internal() should have
199 : : * checked
200 : : */
201 : : Assert(attlen == sizeof(int64));
202 : 1452659 : return Int64GetDatum(*((const int64 *) offset_ptr));
203 : : }
204 : : }
205 : 776778 : return PointerGetDatum(offset_ptr);
206 : : }
207 [ + + ]: 30410175 : else if (attlen == -1)
208 : : {
209 [ + + ]: 30399622 : if (!VARATT_IS_SHORT(tupptr + *off))
210 : 3443682 : *off = TYPEALIGN(attalignby, *off);
211 : :
212 : 30399622 : res = PointerGetDatum(tupptr + *off);
213 : 30399622 : *off += VARSIZE_ANY(DatumGetPointer(res));
214 : 30399622 : return res;
215 : : }
216 : : else
217 : : {
218 : : Assert(attlen == -2);
219 : 10553 : *off = TYPEALIGN(attalignby, *off);
220 : 10553 : res = PointerGetDatum(tupptr + *off);
221 : 10553 : *off += strlen(tupptr + *off) + 1;
222 : 10553 : return res;
223 : : }
224 : : }
225 : :
226 : : /*
227 : : * first_null_attr
228 : : * Inspect a NULL bitmap from a tuple and return the 0-based attnum of the
229 : : * first NULL attribute. Returns natts if no NULLs were found.
230 : : *
231 : : * This is coded to expect that 'bits' contains at least one 0 bit somewhere
232 : : * in the array, but not necessarily < natts. Note that natts may be passed
233 : : * as a value lower than the number of bits physically stored in the tuple's
234 : : * NULL bitmap, in which case we may not find a NULL and return natts.
235 : : *
236 : : * The reason we require at least one 0 bit somewhere in the NULL bitmap is
237 : : * that the for loop that checks 0xFF bytes would loop to the last byte in
238 : : * the array if all bytes were 0xFF, and the subsequent code that finds the
239 : : * right-most 0 bit would access the first byte beyond the bitmap. Provided
240 : : * we find a 0 bit before then, that won't happen. Since tuples which have no
241 : : * NULLs don't have a NULL bitmap, this function won't get called for that
242 : : * case.
243 : : */
244 : : static inline int
245 : 129797664 : first_null_attr(const uint8 *bits, int natts)
246 : : {
247 : 129797664 : int nattByte = natts >> 3;
248 : : int bytenum;
249 : : int res;
250 : :
251 : : #ifdef USE_ASSERT_CHECKING
252 : : int firstnull_check = natts;
253 : :
254 : : /* Do it the slow way and check we get the same answer. */
255 : : for (int i = 0; i < natts; i++)
256 : : {
257 : : if (att_isnull(i, bits))
258 : : {
259 : : firstnull_check = i;
260 : : break;
261 : : }
262 : : }
263 : : #endif
264 : :
265 : : /* Process all bytes up to just before the byte for the natts attribute */
266 [ + + ]: 188277043 : for (bytenum = 0; bytenum < nattByte; bytenum++)
267 : : {
268 : : /* break if there's any NULL attrs (a 0 bit) */
269 [ + + ]: 62615473 : if (bits[bytenum] != 0xFF)
270 : 4136094 : break;
271 : : }
272 : :
273 : : /*
274 : : * Look for the highest 0-bit in the 'bytenum' element. To do this, we
275 : : * promote the uint8 to uint32 before performing the bitwise NOT and
276 : : * looking for the first 1-bit. This works even when the byte is 0xFF, as
277 : : * the bitwise NOT of 0xFF in 32 bits is 0xFFFFFF00, in which case
278 : : * pg_rightmost_one_pos32() will return 8. We may end up with a value
279 : : * higher than natts here, but we'll fix that with the Min() below.
280 : : */
281 : 129797664 : res = bytenum << 3;
282 : 129797664 : res += pg_rightmost_one_pos32(~((uint32) bits[bytenum]));
283 : :
284 : : /*
285 : : * Since we did no masking to mask out bits beyond the natts'th bit, we
286 : : * may have found a bit higher than natts, so we must cap res to natts
287 : : */
288 : 129797664 : res = Min(res, natts);
289 : :
290 : : /* Ensure we got the same answer as the att_isnull() loop got */
291 : : Assert(res == firstnull_check);
292 : :
293 : 129797664 : return res;
294 : : }
295 : : #endif /* FRONTEND */
296 : :
297 : : /*
298 : : * typalign_to_alignby: map a TYPALIGN_xxx value to the numeric alignment
299 : : * value it represents. (We store TYPALIGN_xxx codes not the real alignment
300 : : * values mainly so that initial catalog contents can be machine-independent.)
301 : : */
302 : : static inline uint8
303 : 86820373 : typalign_to_alignby(char typalign)
304 : : {
305 : : uint8 alignby;
306 : :
307 [ + + + + : 86820373 : switch (typalign)
- ]
308 : : {
309 : 12993647 : case TYPALIGN_CHAR:
310 : 12993647 : alignby = sizeof(char);
311 : 12993647 : break;
312 : 3380076 : case TYPALIGN_SHORT:
313 : 3380076 : alignby = ALIGNOF_SHORT;
314 : 3380076 : break;
315 : 66219815 : case TYPALIGN_INT:
316 : 66219815 : alignby = ALIGNOF_INT;
317 : 66219815 : break;
318 : 4226835 : case TYPALIGN_DOUBLE:
319 : 4226835 : alignby = ALIGNOF_DOUBLE;
320 : 4226835 : break;
321 : 0 : default:
322 : : #ifndef FRONTEND
323 [ # # ]: 0 : elog(ERROR, "invalid typalign value: %c", typalign);
324 : : #else
325 : : fprintf(stderr, "invalid typalign value: %c\n", typalign);
326 : : exit(1);
327 : : #endif
328 : : alignby = 0;
329 : : break;
330 : : }
331 : 86820373 : return alignby;
332 : : }
333 : :
334 : : /*
335 : : * att_align_datum aligns the given offset as needed for a datum of alignment
336 : : * requirement attalign and typlen attlen. attdatum is the Datum variable
337 : : * we intend to pack into a tuple (it's only accessed if we are dealing with
338 : : * a varlena type). Note that this assumes the Datum will be stored as-is;
339 : : * callers that are intending to convert non-short varlena datums to short
340 : : * format have to account for that themselves.
341 : : */
342 : : #define att_align_datum(cur_offset, attalign, attlen, attdatum) \
343 : : ( \
344 : : ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? \
345 : : (uintptr_t) (cur_offset) : \
346 : : att_align_nominal(cur_offset, attalign) \
347 : : )
348 : :
349 : : /*
350 : : * Similar to att_align_datum, but accepts a number of bytes, typically from
351 : : * CompactAttribute.attalignby to align the Datum by.
352 : : */
353 : : #define att_datum_alignby(cur_offset, attalignby, attlen, attdatum) \
354 : : ( \
355 : : ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? \
356 : : (uintptr_t) (cur_offset) : \
357 : : TYPEALIGN(attalignby, cur_offset))
358 : :
359 : : /*
360 : : * att_align_pointer performs the same calculation as att_align_datum,
361 : : * but is used when walking a tuple. attptr is the current actual data
362 : : * pointer; when accessing a varlena field we have to "peek" to see if we
363 : : * are looking at a pad byte or the first byte of a 1-byte-header datum.
364 : : * (A zero byte must be either a pad byte, or the first byte of a correctly
365 : : * aligned 4-byte length word; in either case we can align safely. A non-zero
366 : : * byte must be either a 1-byte length word, or the first byte of a correctly
367 : : * aligned 4-byte length word; in either case we need not align.)
368 : : *
369 : : * Note: some callers pass a "char *" pointer for cur_offset. This is
370 : : * a bit of a hack but should work all right as long as uintptr_t is the
371 : : * correct width.
372 : : */
373 : : #define att_align_pointer(cur_offset, attalign, attlen, attptr) \
374 : : ( \
375 : : ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? \
376 : : (uintptr_t) (cur_offset) : \
377 : : att_align_nominal(cur_offset, attalign) \
378 : : )
379 : :
380 : : /*
381 : : * Similar to att_align_pointer, but accepts a number of bytes, typically from
382 : : * CompactAttribute.attalignby to align the pointer by.
383 : : */
384 : : #define att_pointer_alignby(cur_offset, attalignby, attlen, attptr) \
385 : : ( \
386 : : ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? \
387 : : (uintptr_t) (cur_offset) : \
388 : : TYPEALIGN(attalignby, cur_offset))
389 : :
390 : : /*
391 : : * att_align_nominal aligns the given offset as needed for a datum of alignment
392 : : * requirement attalign, ignoring any consideration of packed varlena datums.
393 : : * There are three main use cases for using this macro directly:
394 : : * * we know that the att in question is not varlena (attlen != -1);
395 : : * in this case it is cheaper than the above macros and just as good.
396 : : * * we need to estimate alignment padding cost abstractly, ie without
397 : : * reference to a real tuple. We must assume the worst case that
398 : : * all varlenas are aligned.
399 : : * * within arrays and multiranges, we unconditionally align varlenas (XXX this
400 : : * should be revisited, probably).
401 : : *
402 : : * In performance-critical loops, avoid using this macro; instead use
403 : : * att_nominal_alignby with a pre-computed alignby value.
404 : : */
405 : : #define att_align_nominal(cur_offset, attalign) \
406 : : att_nominal_alignby(cur_offset, typalign_to_alignby(attalign))
407 : :
408 : : /*
409 : : * Similar to att_align_nominal, but accepts a number of bytes, typically from
410 : : * CompactAttribute.attalignby to align the offset by.
411 : : */
412 : : #define att_nominal_alignby(cur_offset, attalignby) \
413 : : TYPEALIGN(attalignby, cur_offset)
414 : :
415 : : /*
416 : : * att_addlength_datum increments the given offset by the space needed for
417 : : * the given Datum variable. attdatum is only accessed if we are dealing
418 : : * with a variable-length attribute.
419 : : *
420 : : * att_addlength_datum_overflow sets *overflow true if that occurs
421 : : * in the addition or false otherwise.
422 : : */
423 : : #define att_addlength_datum(cur_offset, attlen, attdatum) \
424 : : att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum))
425 : : #define att_addlength_datum_overflow(cur_offset, attlen, attdatum, overflow) \
426 : : att_addlength_pointer_overflow(cur_offset, attlen, DatumGetPointer(attdatum), overflow)
427 : :
428 : : /*
429 : : * att_addlength_pointer performs the same calculation as att_addlength_datum,
430 : : * but is used when walking a tuple --- attptr is the pointer to the field
431 : : * within the tuple.
432 : : *
433 : : * Note: some callers pass a "char *" pointer for cur_offset. This is
434 : : * actually perfectly OK, but probably should be cleaned up along with
435 : : * the same practice for att_align_pointer.
436 : : */
437 : : #define att_addlength_pointer(cur_offset, attlen, attptr) \
438 : : ( \
439 : : ((attlen) > 0) ? \
440 : : ( \
441 : : (cur_offset) + (attlen) \
442 : : ) \
443 : : : (((attlen) == -1) ? \
444 : : ( \
445 : : (cur_offset) + VARSIZE_ANY(attptr) \
446 : : ) \
447 : : : \
448 : : ( \
449 : : AssertMacro((attlen) == -2), \
450 : : (cur_offset) + (strlen((const char *) (attptr)) + 1) \
451 : : )) \
452 : : )
453 : :
454 : : /*
455 : : * att_addlength_pointer_overflow: as att_addlength_pointer, but set *overflow
456 : : * true if that occurs (false if not). Not a full att_addlength_pointer
457 : : * replacement, because it only allows Size as its first param, not a pointer.
458 : : */
459 : : static inline Size
460 : 156369950 : att_addlength_pointer_overflow(Size cur_offset, int16 attlen,
461 : : const void *attptr, bool *overflow)
462 : : {
463 : : Size add;
464 : :
465 [ + + ]: 156369950 : if (attlen > 0)
466 : 148648651 : add = attlen;
467 [ + + ]: 7721299 : else if (attlen == -1)
468 : 6143978 : add = VARSIZE_ANY(attptr);
469 [ + - ]: 1577321 : else if (attlen == -2)
470 : 1577321 : add = strlen((const char *) attptr) + 1;
471 : : else
472 [ # # ]: 0 : elog(ERROR, "invalid attlen %d", attlen);
473 : :
474 : 156369950 : *overflow = pg_add_size_overflow(cur_offset, add, &cur_offset);
475 : 156369950 : return cur_offset;
476 : : }
477 : :
478 : : #ifndef FRONTEND
479 : : /*
480 : : * store_att_byval is a partial inverse of fetch_att: store a given Datum
481 : : * value into a tuple data area at the specified address. However, it only
482 : : * handles the byval case, because in typical usage the caller needs to
483 : : * distinguish by-val and by-ref cases anyway, and so a do-it-all function
484 : : * wouldn't be convenient.
485 : : */
486 : : static inline void
487 : 127484044 : store_att_byval(void *T, Datum newdatum, int attlen)
488 : : {
489 [ + + + + : 127484044 : switch (attlen)
- ]
490 : : {
491 : 16638231 : case sizeof(char):
492 : 16638231 : *(char *) T = DatumGetChar(newdatum);
493 : 16638231 : break;
494 : 6017330 : case sizeof(int16):
495 : 6017330 : *(int16 *) T = DatumGetInt16(newdatum);
496 : 6017330 : break;
497 : 93448532 : case sizeof(int32):
498 : 93448532 : *(int32 *) T = DatumGetInt32(newdatum);
499 : 93448532 : break;
500 : 11379951 : case sizeof(int64):
501 : 11379951 : *(int64 *) T = DatumGetInt64(newdatum);
502 : 11379951 : break;
503 : 0 : default:
504 [ # # ]: 0 : elog(ERROR, "unsupported byval length: %d", attlen);
505 : : }
506 : 127484044 : }
507 : : #endif /* FRONTEND */
508 : :
509 : : #endif /* TUPMACS_H */
|
