Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * tupmacs.h
4 : : * Tuple macros used by both index tuples and heap tuples.
5 : : *
6 : : *
7 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
8 : : * Portions Copyright (c) 1994, Regents of the University of California
9 : : *
10 : : * src/include/access/tupmacs.h
11 : : *
12 : : *-------------------------------------------------------------------------
13 : : */
14 : : #ifndef TUPMACS_H
15 : : #define TUPMACS_H
16 : :
17 : : #include "catalog/pg_type_d.h" /* for TYPALIGN macros */
18 : : #include "port/pg_bitutils.h"
19 : : #include "port/pg_bswap.h"
20 : : #include "varatt.h"
21 : :
22 : : /*
23 : : * Check a tuple's null bitmap to determine whether the attribute is null.
24 : : * Note that a 0 in the null bitmap indicates a null, while 1 indicates
25 : : * non-null.
26 : : */
27 : : static inline bool
28 : 145265187 : att_isnull(int ATT, const uint8 *BITS)
29 : : {
30 : 145265187 : return !(BITS[ATT >> 3] & (1 << (ATT & 0x07)));
31 : : }
32 : :
33 : : /*
34 : : * populate_isnull_array
35 : : * Transform a tuple's null bitmap into a boolean array.
36 : : *
37 : : * Caller must ensure that the isnull array is sized so it contains
38 : : * at least as many elements as there are bits in the 'bits' array.
39 : : * Callers should be aware that isnull is populated 8 elements at a time,
40 : : * effectively as if natts is rounded up to the next multiple of 8.
41 : : */
42 : : static inline void
43 : 27700288 : populate_isnull_array(const uint8 *bits, int natts, bool *isnull)
44 : : {
45 : 27700288 : int nbytes = (natts + 7) >> 3;
46 : :
47 : : /*
48 : : * Multiplying the inverted NULL bitmap byte by this value results in the
49 : : * lowest bit in each byte being set the same as each bit of the inverted
50 : : * byte. We perform this as 2 32-bit operations rather than a single
51 : : * 64-bit operation as multiplying by the required value to do this in
52 : : * 64-bits would result in overflowing a uint64 in some cases.
53 : : *
54 : : * XXX if we ever require BMI2 (-march=x86-64-v3), then this could be done
55 : : * more efficiently on most X86-64 CPUs with the PDEP instruction. Beware
56 : : * that some chips (e.g. AMD's Zen2) are horribly inefficient at PDEP.
57 : : */
58 : : #define SPREAD_BITS_MULTIPLIER_32 0x204081U
59 : :
60 [ + + ]: 80080017 : for (int i = 0; i < nbytes; i++, isnull += 8)
61 : : {
62 : : uint64 isnull_8;
63 : 52379729 : uint8 nullbyte = ~bits[i];
64 : :
65 : : /* Convert the lower 4 bits of NULL bitmap word into a 64 bit int */
66 : 52379729 : isnull_8 = (nullbyte & 0xf) * SPREAD_BITS_MULTIPLIER_32;
67 : :
68 : : /*
69 : : * Convert the upper 4 bits of NULL bitmap word into a 64 bit int,
70 : : * shift into the upper 32 bit and bitwise-OR with the result of the
71 : : * lower 4 bits.
72 : : */
73 : 52379729 : isnull_8 |= ((uint64) ((nullbyte >> 4) * SPREAD_BITS_MULTIPLIER_32)) << 32;
74 : :
75 : : /* Mask out all other bits apart from the lowest bit of each byte. */
76 : 52379729 : isnull_8 &= UINT64CONST(0x0101010101010101);
77 : :
78 : : #ifdef WORDS_BIGENDIAN
79 : :
80 : : /*
81 : : * Fix byte order on big-endian machines before copying to the array.
82 : : */
83 : : isnull_8 = pg_bswap64(isnull_8);
84 : : #endif
85 : 52379729 : memcpy(isnull, &isnull_8, sizeof(uint64));
86 : : }
87 : 27700288 : }
88 : :
89 : : #ifndef FRONTEND
90 : : /*
91 : : * Given an attbyval and an attlen from either a Form_pg_attribute or
92 : : * CompactAttribute and a pointer into a tuple's data area, return the
93 : : * correct value or pointer.
94 : : *
95 : : * We return a Datum value in all cases. If attbyval is false, we return the
96 : : * same pointer into the tuple data area that we're passed. Otherwise, we
97 : : * return the correct number of bytes fetched from the data area and extended
98 : : * to Datum form.
99 : : *
100 : : * Note that T must already be properly aligned for this to work correctly.
101 : : */
102 : : #define fetchatt(A,T) fetch_att(T, (A)->attbyval, (A)->attlen)
103 : :
104 : : /*
105 : : * Same, but work from byval/len parameters rather than Form_pg_attribute.
106 : : */
107 : : static inline Datum
108 : 631805272 : fetch_att(const void *T, bool attbyval, int attlen)
109 : : {
110 [ + + ]: 631805272 : if (attbyval)
111 : : {
112 [ + + + + : 519087702 : switch (attlen)
- ]
113 : : {
114 : 16682076 : case sizeof(char):
115 : 16682076 : return CharGetDatum(*((const char *) T));
116 : 30067845 : case sizeof(int16):
117 : 30067845 : return Int16GetDatum(*((const int16 *) T));
118 : 458835821 : case sizeof(int32):
119 : 458835821 : return Int32GetDatum(*((const int32 *) T));
120 : 13501960 : case sizeof(int64):
121 : 13501960 : return Int64GetDatum(*((const int64 *) T));
122 : 0 : default:
123 [ # # ]: 0 : elog(ERROR, "unsupported byval length: %d", attlen);
124 : : return 0;
125 : : }
126 : : }
127 : : else
128 : 112717570 : return PointerGetDatum(T);
129 : : }
130 : :
131 : : /*
132 : : * Same as fetch_att, but no error checking for invalid attlens for byval
133 : : * types. This is safe to use when attlen comes from CompactAttribute as we
134 : : * validate the length when populating that struct.
135 : : */
136 : : static inline Datum
137 : 404293727 : fetch_att_noerr(const void *T, bool attbyval, int attlen)
138 : : {
139 [ + + ]: 404293727 : if (attbyval)
140 : : {
141 [ + + + + ]: 368736502 : switch (attlen)
142 : : {
143 : 251405913 : case sizeof(int32):
144 : 251405913 : return Int32GetDatum(*((const int32 *) T));
145 : 23086153 : case sizeof(int16):
146 : 23086153 : return Int16GetDatum(*((const int16 *) T));
147 : 85513686 : case sizeof(char):
148 : 85513686 : return CharGetDatum(*((const char *) T));
149 : 8730750 : default:
150 : : Assert(attlen == sizeof(int64));
151 : 8730750 : return Int64GetDatum(*((const int64 *) T));
152 : : }
153 : : }
154 : : else
155 : 35557225 : return PointerGetDatum(T);
156 : : }
157 : :
158 : :
159 : : /*
160 : : * align_fetch_then_add
161 : : * Applies all the functionality of att_pointer_alignby(),
162 : : * fetch_att_noerr() and att_addlength_pointer(), resulting in the *off
163 : : * pointer to the perhaps unaligned number of bytes into 'tupptr', ready
164 : : * to deform the next attribute.
165 : : *
166 : : * tupptr: pointer to the beginning of the tuple, after the header and any
167 : : * NULL bitmask.
168 : : * off: offset in bytes for reading tuple data, possibly unaligned.
169 : : * attbyval, attlen and attalignby are values from CompactAttribute.
170 : : */
171 : : static inline Datum
172 : 50468536 : align_fetch_then_add(const char *tupptr, uint32 *off, bool attbyval, int attlen,
173 : : uint8 attalignby)
174 : : {
175 : : Datum res;
176 : :
177 [ + + ]: 50468536 : if (attlen > 0)
178 : : {
179 : : const char *offset_ptr;
180 : :
181 : 18414276 : *off = TYPEALIGN(attalignby, *off);
182 : 18414276 : offset_ptr = tupptr + *off;
183 : 18414276 : *off += attlen;
184 [ + + ]: 18414276 : if (attbyval)
185 : : {
186 [ + + + + ]: 17636371 : switch (attlen)
187 : : {
188 : 1843568 : case sizeof(char):
189 : 1843568 : return CharGetDatum(*((const char *) offset_ptr));
190 : 410044 : case sizeof(int16):
191 : 410044 : return Int16GetDatum(*((const int16 *) offset_ptr));
192 : 13950172 : case sizeof(int32):
193 : 13950172 : return Int32GetDatum(*((const int32 *) offset_ptr));
194 : 1432587 : default:
195 : :
196 : : /*
197 : : * populate_compact_attribute_internal() should have
198 : : * checked
199 : : */
200 : : Assert(attlen == sizeof(int64));
201 : 1432587 : return Int64GetDatum(*((const int64 *) offset_ptr));
202 : : }
203 : : }
204 : 777905 : return PointerGetDatum(offset_ptr);
205 : : }
206 [ + + ]: 32054260 : else if (attlen == -1)
207 : : {
208 [ + + ]: 32044431 : if (!VARATT_IS_SHORT(tupptr + *off))
209 : 4085364 : *off = TYPEALIGN(attalignby, *off);
210 : :
211 : 32044431 : res = PointerGetDatum(tupptr + *off);
212 : 32044431 : *off += VARSIZE_ANY(DatumGetPointer(res));
213 : 32044431 : return res;
214 : : }
215 : : else
216 : : {
217 : : Assert(attlen == -2);
218 : 9829 : *off = TYPEALIGN(attalignby, *off);
219 : 9829 : res = PointerGetDatum(tupptr + *off);
220 : 9829 : *off += strlen(tupptr + *off) + 1;
221 : 9829 : return res;
222 : : }
223 : : }
224 : :
225 : : /*
226 : : * first_null_attr
227 : : * Inspect a NULL bitmap from a tuple and return the 0-based attnum of the
228 : : * first NULL attribute. Returns natts if no NULLs were found.
229 : : *
230 : : * This is coded to expect that 'bits' contains at least one 0 bit somewhere
231 : : * in the array, but not necessarily < natts. Note that natts may be passed
232 : : * as a value lower than the number of bits physically stored in the tuple's
233 : : * NULL bitmap, in which case we may not find a NULL and return natts.
234 : : *
235 : : * The reason we require at least one 0 bit somewhere in the NULL bitmap is
236 : : * that the for loop that checks 0xFF bytes would loop to the last byte in
237 : : * the array if all bytes were 0xFF, and the subsequent code that finds the
238 : : * right-most 0 bit would access the first byte beyond the bitmap. Provided
239 : : * we find a 0 bit before then, that won't happen. Since tuples which have no
240 : : * NULLs don't have a NULL bitmap, this function won't get called for that
241 : : * case.
242 : : */
243 : : static inline int
244 : 129402642 : first_null_attr(const uint8 *bits, int natts)
245 : : {
246 : 129402642 : int nattByte = natts >> 3;
247 : : int bytenum;
248 : : int res;
249 : :
250 : : #ifdef USE_ASSERT_CHECKING
251 : : int firstnull_check = natts;
252 : :
253 : : /* Do it the slow way and check we get the same answer. */
254 : : for (int i = 0; i < natts; i++)
255 : : {
256 : : if (att_isnull(i, bits))
257 : : {
258 : : firstnull_check = i;
259 : : break;
260 : : }
261 : : }
262 : : #endif
263 : :
264 : : /* Process all bytes up to just before the byte for the natts attribute */
265 [ + + ]: 185564306 : for (bytenum = 0; bytenum < nattByte; bytenum++)
266 : : {
267 : : /* break if there's any NULL attrs (a 0 bit) */
268 [ + + ]: 61388735 : if (bits[bytenum] != 0xFF)
269 : 5227071 : break;
270 : : }
271 : :
272 : : /*
273 : : * Look for the highest 0-bit in the 'bytenum' element. To do this, we
274 : : * promote the uint8 to uint32 before performing the bitwise NOT and
275 : : * looking for the first 1-bit. This works even when the byte is 0xFF, as
276 : : * the bitwise NOT of 0xFF in 32 bits is 0xFFFFFF00, in which case
277 : : * pg_rightmost_one_pos32() will return 8. We may end up with a value
278 : : * higher than natts here, but we'll fix that with the Min() below.
279 : : */
280 : 129402642 : res = bytenum << 3;
281 : 129402642 : res += pg_rightmost_one_pos32(~((uint32) bits[bytenum]));
282 : :
283 : : /*
284 : : * Since we did no masking to mask out bits beyond the natts'th bit, we
285 : : * may have found a bit higher than natts, so we must cap res to natts
286 : : */
287 : 129402642 : res = Min(res, natts);
288 : :
289 : : /* Ensure we got the same answer as the att_isnull() loop got */
290 : : Assert(res == firstnull_check);
291 : :
292 : 129402642 : return res;
293 : : }
294 : : #endif /* FRONTEND */
295 : :
296 : : /*
297 : : * typalign_to_alignby: map a TYPALIGN_xxx value to the numeric alignment
298 : : * value it represents. (We store TYPALIGN_xxx codes not the real alignment
299 : : * values mainly so that initial catalog contents can be machine-independent.)
300 : : */
301 : : static inline uint8
302 : 84104474 : typalign_to_alignby(char typalign)
303 : : {
304 : : uint8 alignby;
305 : :
306 [ + + + + : 84104474 : switch (typalign)
- ]
307 : : {
308 : 12324957 : case TYPALIGN_CHAR:
309 : 12324957 : alignby = sizeof(char);
310 : 12324957 : break;
311 : 3336846 : case TYPALIGN_SHORT:
312 : 3336846 : alignby = ALIGNOF_SHORT;
313 : 3336846 : break;
314 : 64950872 : case TYPALIGN_INT:
315 : 64950872 : alignby = ALIGNOF_INT;
316 : 64950872 : break;
317 : 3491799 : case TYPALIGN_DOUBLE:
318 : 3491799 : alignby = ALIGNOF_DOUBLE;
319 : 3491799 : break;
320 : 0 : default:
321 : : #ifndef FRONTEND
322 [ # # ]: 0 : elog(ERROR, "invalid typalign value: %c", typalign);
323 : : #else
324 : : fprintf(stderr, "invalid typalign value: %c\n", typalign);
325 : : exit(1);
326 : : #endif
327 : : alignby = 0;
328 : : break;
329 : : }
330 : 84104474 : return alignby;
331 : : }
332 : :
333 : : /*
334 : : * att_align_datum aligns the given offset as needed for a datum of alignment
335 : : * requirement attalign and typlen attlen. attdatum is the Datum variable
336 : : * we intend to pack into a tuple (it's only accessed if we are dealing with
337 : : * a varlena type). Note that this assumes the Datum will be stored as-is;
338 : : * callers that are intending to convert non-short varlena datums to short
339 : : * format have to account for that themselves.
340 : : */
341 : : #define att_align_datum(cur_offset, attalign, attlen, attdatum) \
342 : : ( \
343 : : ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? \
344 : : (uintptr_t) (cur_offset) : \
345 : : att_align_nominal(cur_offset, attalign) \
346 : : )
347 : :
348 : : /*
349 : : * Similar to att_align_datum, but accepts a number of bytes, typically from
350 : : * CompactAttribute.attalignby to align the Datum by.
351 : : */
352 : : #define att_datum_alignby(cur_offset, attalignby, attlen, attdatum) \
353 : : ( \
354 : : ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? \
355 : : (uintptr_t) (cur_offset) : \
356 : : TYPEALIGN(attalignby, cur_offset))
357 : :
358 : : /*
359 : : * att_align_pointer performs the same calculation as att_align_datum,
360 : : * but is used when walking a tuple. attptr is the current actual data
361 : : * pointer; when accessing a varlena field we have to "peek" to see if we
362 : : * are looking at a pad byte or the first byte of a 1-byte-header datum.
363 : : * (A zero byte must be either a pad byte, or the first byte of a correctly
364 : : * aligned 4-byte length word; in either case we can align safely. A non-zero
365 : : * byte must be either a 1-byte length word, or the first byte of a correctly
366 : : * aligned 4-byte length word; in either case we need not align.)
367 : : *
368 : : * Note: some callers pass a "char *" pointer for cur_offset. This is
369 : : * a bit of a hack but should work all right as long as uintptr_t is the
370 : : * correct width.
371 : : */
372 : : #define att_align_pointer(cur_offset, attalign, attlen, attptr) \
373 : : ( \
374 : : ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? \
375 : : (uintptr_t) (cur_offset) : \
376 : : att_align_nominal(cur_offset, attalign) \
377 : : )
378 : :
379 : : /*
380 : : * Similar to att_align_pointer, but accepts a number of bytes, typically from
381 : : * CompactAttribute.attalignby to align the pointer by.
382 : : */
383 : : #define att_pointer_alignby(cur_offset, attalignby, attlen, attptr) \
384 : : ( \
385 : : ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? \
386 : : (uintptr_t) (cur_offset) : \
387 : : TYPEALIGN(attalignby, cur_offset))
388 : :
389 : : /*
390 : : * att_align_nominal aligns the given offset as needed for a datum of alignment
391 : : * requirement attalign, ignoring any consideration of packed varlena datums.
392 : : * There are three main use cases for using this macro directly:
393 : : * * we know that the att in question is not varlena (attlen != -1);
394 : : * in this case it is cheaper than the above macros and just as good.
395 : : * * we need to estimate alignment padding cost abstractly, ie without
396 : : * reference to a real tuple. We must assume the worst case that
397 : : * all varlenas are aligned.
398 : : * * within arrays and multiranges, we unconditionally align varlenas (XXX this
399 : : * should be revisited, probably).
400 : : *
401 : : * In performance-critical loops, avoid using this macro; instead use
402 : : * att_nominal_alignby with a pre-computed alignby value.
403 : : */
404 : : #define att_align_nominal(cur_offset, attalign) \
405 : : att_nominal_alignby(cur_offset, typalign_to_alignby(attalign))
406 : :
407 : : /*
408 : : * Similar to att_align_nominal, but accepts a number of bytes, typically from
409 : : * CompactAttribute.attalignby to align the offset by.
410 : : */
411 : : #define att_nominal_alignby(cur_offset, attalignby) \
412 : : TYPEALIGN(attalignby, cur_offset)
413 : :
414 : : /*
415 : : * att_addlength_datum increments the given offset by the space needed for
416 : : * the given Datum variable. attdatum is only accessed if we are dealing
417 : : * with a variable-length attribute.
418 : : */
419 : : #define att_addlength_datum(cur_offset, attlen, attdatum) \
420 : : att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum))
421 : :
422 : : /*
423 : : * att_addlength_pointer performs the same calculation as att_addlength_datum,
424 : : * but is used when walking a tuple --- attptr is the pointer to the field
425 : : * within the tuple.
426 : : *
427 : : * Note: some callers pass a "char *" pointer for cur_offset. This is
428 : : * actually perfectly OK, but probably should be cleaned up along with
429 : : * the same practice for att_align_pointer.
430 : : */
431 : : #define att_addlength_pointer(cur_offset, attlen, attptr) \
432 : : ( \
433 : : ((attlen) > 0) ? \
434 : : ( \
435 : : (cur_offset) + (attlen) \
436 : : ) \
437 : : : (((attlen) == -1) ? \
438 : : ( \
439 : : (cur_offset) + VARSIZE_ANY(attptr) \
440 : : ) \
441 : : : \
442 : : ( \
443 : : AssertMacro((attlen) == -2), \
444 : : (cur_offset) + (strlen((const char *) (attptr)) + 1) \
445 : : )) \
446 : : )
447 : :
448 : : #ifndef FRONTEND
449 : : /*
450 : : * store_att_byval is a partial inverse of fetch_att: store a given Datum
451 : : * value into a tuple data area at the specified address. However, it only
452 : : * handles the byval case, because in typical usage the caller needs to
453 : : * distinguish by-val and by-ref cases anyway, and so a do-it-all function
454 : : * wouldn't be convenient.
455 : : */
456 : : static inline void
457 : 126608075 : store_att_byval(void *T, Datum newdatum, int attlen)
458 : : {
459 [ + + + + : 126608075 : switch (attlen)
- ]
460 : : {
461 : 16504209 : case sizeof(char):
462 : 16504209 : *(char *) T = DatumGetChar(newdatum);
463 : 16504209 : break;
464 : 6772220 : case sizeof(int16):
465 : 6772220 : *(int16 *) T = DatumGetInt16(newdatum);
466 : 6772220 : break;
467 : 93211016 : case sizeof(int32):
468 : 93211016 : *(int32 *) T = DatumGetInt32(newdatum);
469 : 93211016 : break;
470 : 10120630 : case sizeof(int64):
471 : 10120630 : *(int64 *) T = DatumGetInt64(newdatum);
472 : 10120630 : break;
473 : 0 : default:
474 [ # # ]: 0 : elog(ERROR, "unsupported byval length: %d", attlen);
475 : : }
476 : 126608075 : }
477 : : #endif /* FRONTEND */
478 : :
479 : : #endif /* TUPMACS_H */
|
