Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * tupmacs.h
4 : * Tuple macros used by both index tuples and heap tuples.
5 : *
6 : *
7 : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
8 : * Portions Copyright (c) 1994, Regents of the University of California
9 : *
10 : * src/include/access/tupmacs.h
11 : *
12 : *-------------------------------------------------------------------------
13 : */
14 : #ifndef TUPMACS_H
15 : #define TUPMACS_H
16 :
17 : #include "catalog/pg_type_d.h" /* for TYPALIGN macros */
18 : #include "port/pg_bitutils.h"
19 : #include "port/pg_bswap.h"
20 : #include "varatt.h"
21 :
22 : /*
23 : * Check a tuple's null bitmap to determine whether the attribute is null.
24 : * Note that a 0 in the null bitmap indicates a null, while 1 indicates
25 : * non-null.
26 : */
27 : static inline bool
28 136999393 : att_isnull(int ATT, const bits8 *BITS)
29 : {
30 136999393 : return !(BITS[ATT >> 3] & (1 << (ATT & 0x07)));
31 : }
32 :
33 : /*
34 : * populate_isnull_array
35 : * Transform a tuple's null bitmap into a boolean array.
36 : *
37 : * Caller must ensure that the isnull array is sized so it contains
38 : * at least as many elements as there are bits in the 'bits' array.
39 : * Callers should be aware that isnull is populated 8 elements at a time,
40 : * effectively as if natts is rounded up to the next multiple of 8.
41 : */
42 : static inline void
43 32101880 : populate_isnull_array(const bits8 *bits, int natts, bool *isnull)
44 : {
45 32101880 : int nbytes = (natts + 7) >> 3;
46 :
47 : /*
48 : * Multiplying the inverted NULL bitmap byte by this value results in the
49 : * lowest bit in each byte being set the same as each bit of the inverted
50 : * byte. We perform this as 2 32-bit operations rather than a single
51 : * 64-bit operation as multiplying by the required value to do this in
52 : * 64-bits would result in overflowing a uint64 in some cases.
53 : *
54 : * XXX if we ever require BMI2 (-march=x86-64-v3), then this could be done
55 : * more efficiently on most X86-64 CPUs with the PDEP instruction. Beware
56 : * that some chips (e.g. AMD's Zen2) are horribly inefficient at PDEP.
57 : */
58 : #define SPREAD_BITS_MULTIPLIER_32 0x204081U
59 :
60 90437451 : for (int i = 0; i < nbytes; i++, isnull += 8)
61 : {
62 : uint64 isnull_8;
63 58335571 : bits8 nullbyte = ~bits[i];
64 :
65 : /* Convert the lower 4 bits of NULL bitmap word into a 64 bit int */
66 58335571 : isnull_8 = (nullbyte & 0xf) * SPREAD_BITS_MULTIPLIER_32;
67 :
68 : /*
69 : * Convert the upper 4 bits of NULL bitmap word into a 64 bit int,
70 : * shift into the upper 32 bit and bitwise-OR with the result of the
71 : * lower 4 bits.
72 : */
73 58335571 : isnull_8 |= ((uint64) ((nullbyte >> 4) * SPREAD_BITS_MULTIPLIER_32)) << 32;
74 :
75 : /* Mask out all other bits apart from the lowest bit of each byte. */
76 58335571 : isnull_8 &= UINT64CONST(0x0101010101010101);
77 :
78 : #ifdef WORDS_BIGENDIAN
79 :
80 : /*
81 : * Fix byte order on big-endian machines before copying to the array.
82 : */
83 : isnull_8 = pg_bswap64(isnull_8);
84 : #endif
85 58335571 : memcpy(isnull, &isnull_8, sizeof(uint64));
86 : }
87 32101880 : }
88 :
89 : #ifndef FRONTEND
90 : /*
91 : * Given an attbyval and an attlen from either a Form_pg_attribute or
92 : * CompactAttribute and a pointer into a tuple's data area, return the
93 : * correct value or pointer.
94 : *
95 : * We return a Datum value in all cases. If attbyval is false, we return the
96 : * same pointer into the tuple data area that we're passed. Otherwise, we
97 : * return the correct number of bytes fetched from the data area and extended
98 : * to Datum form.
99 : *
100 : * Note that T must already be properly aligned for this to work correctly.
101 : */
102 : #define fetchatt(A,T) fetch_att(T, (A)->attbyval, (A)->attlen)
103 :
104 : /*
105 : * Same, but work from byval/len parameters rather than Form_pg_attribute.
106 : */
107 : static inline Datum
108 619933769 : fetch_att(const void *T, bool attbyval, int attlen)
109 : {
110 619933769 : if (attbyval)
111 : {
112 508906801 : switch (attlen)
113 : {
114 16177811 : case sizeof(char):
115 16177811 : return CharGetDatum(*((const char *) T));
116 29525038 : case sizeof(int16):
117 29525038 : return Int16GetDatum(*((const int16 *) T));
118 449655028 : case sizeof(int32):
119 449655028 : return Int32GetDatum(*((const int32 *) T));
120 13548924 : case sizeof(int64):
121 13548924 : return Int64GetDatum(*((const int64 *) T));
122 0 : default:
123 0 : elog(ERROR, "unsupported byval length: %d", attlen);
124 : return 0;
125 : }
126 : }
127 : else
128 111026968 : return PointerGetDatum(T);
129 : }
130 :
131 : /*
132 : * Same as fetch_att, but no error checking for invalid attlens for byval
133 : * types. This is safe to use when attlen comes from CompactAttribute as we
134 : * validate the length when populating that struct.
135 : */
136 : static inline Datum
137 426377064 : fetch_att_noerr(const void *T, bool attbyval, int attlen)
138 : {
139 426377064 : if (attbyval)
140 : {
141 384764006 : switch (attlen)
142 : {
143 264792863 : case sizeof(int32):
144 264792863 : return Int32GetDatum(*((const int32 *) T));
145 23019993 : case sizeof(int16):
146 23019993 : return Int16GetDatum(*((const int16 *) T));
147 88435681 : case sizeof(char):
148 88435681 : return CharGetDatum(*((const char *) T));
149 8515469 : default:
150 : Assert(attlen == sizeof(int64));
151 8515469 : return Int64GetDatum(*((const int64 *) T));
152 : }
153 : }
154 : else
155 41613058 : return PointerGetDatum(T);
156 : }
157 :
158 :
159 : /*
160 : * align_fetch_then_add
161 : * Applies all the functionality of att_pointer_alignby(),
162 : * fetch_att_noerr() and att_addlength_pointer(), resulting in the *off
163 : * pointer to the perhaps unaligned number of bytes into 'tupptr', ready
164 : * to deform the next attribute.
165 : *
166 : * tupptr: pointer to the beginning of the tuple, after the header and any
167 : * NULL bitmask.
168 : * off: offset in bytes for reading tuple data, possibly unaligned.
169 : * attbyval, attlen and attalignby are values from CompactAttribute.
170 : */
171 : static inline Datum
172 52097644 : align_fetch_then_add(const char *tupptr, uint32 *off, bool attbyval, int attlen,
173 : uint8 attalignby)
174 : {
175 : Datum res;
176 :
177 52097644 : if (attlen > 0)
178 : {
179 : const char *offset_ptr;
180 :
181 19316685 : *off = TYPEALIGN(attalignby, *off);
182 19316685 : offset_ptr = tupptr + *off;
183 19316685 : *off += attlen;
184 19316685 : if (attbyval)
185 : {
186 18533021 : switch (attlen)
187 : {
188 1880542 : case sizeof(char):
189 1880542 : return CharGetDatum(*((const char *) offset_ptr));
190 399762 : case sizeof(int16):
191 399762 : return Int16GetDatum(*((const int16 *) offset_ptr));
192 14932087 : case sizeof(int32):
193 14932087 : return Int32GetDatum(*((const int32 *) offset_ptr));
194 1320630 : default:
195 :
196 : /*
197 : * populate_compact_attribute_internal() should have
198 : * checked
199 : */
200 : Assert(attlen == sizeof(int64));
201 1320630 : return Int64GetDatum(*((const int64 *) offset_ptr));
202 : }
203 : }
204 783664 : return PointerGetDatum(offset_ptr);
205 : }
206 32780959 : else if (attlen == -1)
207 : {
208 32772068 : if (!VARATT_IS_SHORT(tupptr + *off))
209 4267224 : *off = TYPEALIGN(attalignby, *off);
210 :
211 32772068 : res = PointerGetDatum(tupptr + *off);
212 32772068 : *off += VARSIZE_ANY(DatumGetPointer(res));
213 32772068 : return res;
214 : }
215 : else
216 : {
217 : Assert(attlen == -2);
218 8891 : *off = TYPEALIGN(attalignby, *off);
219 8891 : res = PointerGetDatum(tupptr + *off);
220 8891 : *off += strlen(tupptr + *off) + 1;
221 8891 : return res;
222 : }
223 : }
224 :
225 : /*
226 : * first_null_attr
227 : * Inspect a NULL bitmap from a tuple and return the 0-based attnum of the
228 : * first NULL attribute. Returns natts if no NULLs were found.
229 : *
230 : * This is coded to expect that 'bits' contains at least one 0 bit somewhere
231 : * in the array, but not necessarily < natts. Note that natts may be passed
232 : * as a value lower than the number of bits physically stored in the tuple's
233 : * NULL bitmap, in which case we may not find a NULL and return natts.
234 : *
235 : * The reason we require at least one 0 bit somewhere in the NULL bitmap is
236 : * that the for loop that checks 0xFF bytes would loop to the last byte in
237 : * the array if all bytes were 0xFF, and the subsequent code that finds the
238 : * right-most 0 bit would access the first byte beyond the bitmap. Provided
239 : * we find a 0 bit before then, that won't happen. Since tuples which have no
240 : * NULLs don't have a NULL bitmap, this function won't get called for that
241 : * case.
242 : */
243 : static inline int
244 126927652 : first_null_attr(const bits8 *bits, int natts)
245 : {
246 126927652 : int nattByte = natts >> 3;
247 : int bytenum;
248 : int res;
249 :
250 : #ifdef USE_ASSERT_CHECKING
251 : int firstnull_check = natts;
252 :
253 : /* Do it the slow way and check we get the same answer. */
254 : for (int i = 0; i < natts; i++)
255 : {
256 : if (att_isnull(i, bits))
257 : {
258 : firstnull_check = i;
259 : break;
260 : }
261 : }
262 : #endif
263 :
264 : /* Process all bytes up to just before the byte for the natts attribute */
265 182243704 : for (bytenum = 0; bytenum < nattByte; bytenum++)
266 : {
267 : /* break if there's any NULL attrs (a 0 bit) */
268 60667335 : if (bits[bytenum] != 0xFF)
269 5351283 : break;
270 : }
271 :
272 : /*
273 : * Look for the highest 0-bit in the 'bytenum' element. To do this, we
274 : * promote the uint8 to uint32 before performing the bitwise NOT and
275 : * looking for the first 1-bit. This works even when the byte is 0xFF, as
276 : * the bitwise NOT of 0xFF in 32 bits is 0xFFFFFF00, in which case
277 : * pg_rightmost_one_pos32() will return 8. We may end up with a value
278 : * higher than natts here, but we'll fix that with the Min() below.
279 : */
280 126927652 : res = bytenum << 3;
281 126927652 : res += pg_rightmost_one_pos32(~((uint32) bits[bytenum]));
282 :
283 : /*
284 : * Since we did no masking to mask out bits beyond the natts'th bit, we
285 : * may have found a bit higher than natts, so we must cap res to natts
286 : */
287 126927652 : res = Min(res, natts);
288 :
289 : /* Ensure we got the same answer as the att_isnull() loop got */
290 : Assert(res == firstnull_check);
291 :
292 126927652 : return res;
293 : }
294 : #endif /* FRONTEND */
295 :
296 : /*
297 : * typalign_to_alignby: map a TYPALIGN_xxx value to the numeric alignment
298 : * value it represents. (We store TYPALIGN_xxx codes not the real alignment
299 : * values mainly so that initial catalog contents can be machine-independent.)
300 : */
301 : static inline uint8
302 84442630 : typalign_to_alignby(char typalign)
303 : {
304 : uint8 alignby;
305 :
306 84442630 : switch (typalign)
307 : {
308 12004398 : case TYPALIGN_CHAR:
309 12004398 : alignby = sizeof(char);
310 12004398 : break;
311 3917668 : case TYPALIGN_SHORT:
312 3917668 : alignby = ALIGNOF_SHORT;
313 3917668 : break;
314 65050344 : case TYPALIGN_INT:
315 65050344 : alignby = ALIGNOF_INT;
316 65050344 : break;
317 3470220 : case TYPALIGN_DOUBLE:
318 3470220 : alignby = ALIGNOF_DOUBLE;
319 3470220 : break;
320 0 : default:
321 : #ifndef FRONTEND
322 0 : elog(ERROR, "invalid typalign value: %c", typalign);
323 : #else
324 : fprintf(stderr, "invalid typalign value: %c\n", typalign);
325 : exit(1);
326 : #endif
327 : alignby = 0;
328 : break;
329 : }
330 84442630 : return alignby;
331 : }
332 :
333 : /*
334 : * att_align_datum aligns the given offset as needed for a datum of alignment
335 : * requirement attalign and typlen attlen. attdatum is the Datum variable
336 : * we intend to pack into a tuple (it's only accessed if we are dealing with
337 : * a varlena type). Note that this assumes the Datum will be stored as-is;
338 : * callers that are intending to convert non-short varlena datums to short
339 : * format have to account for that themselves.
340 : */
341 : #define att_align_datum(cur_offset, attalign, attlen, attdatum) \
342 : ( \
343 : ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? \
344 : (uintptr_t) (cur_offset) : \
345 : att_align_nominal(cur_offset, attalign) \
346 : )
347 :
348 : /*
349 : * Similar to att_align_datum, but accepts a number of bytes, typically from
350 : * CompactAttribute.attalignby to align the Datum by.
351 : */
352 : #define att_datum_alignby(cur_offset, attalignby, attlen, attdatum) \
353 : ( \
354 : ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? \
355 : (uintptr_t) (cur_offset) : \
356 : TYPEALIGN(attalignby, cur_offset))
357 :
358 : /*
359 : * att_align_pointer performs the same calculation as att_align_datum,
360 : * but is used when walking a tuple. attptr is the current actual data
361 : * pointer; when accessing a varlena field we have to "peek" to see if we
362 : * are looking at a pad byte or the first byte of a 1-byte-header datum.
363 : * (A zero byte must be either a pad byte, or the first byte of a correctly
364 : * aligned 4-byte length word; in either case we can align safely. A non-zero
365 : * byte must be either a 1-byte length word, or the first byte of a correctly
366 : * aligned 4-byte length word; in either case we need not align.)
367 : *
368 : * Note: some callers pass a "char *" pointer for cur_offset. This is
369 : * a bit of a hack but should work all right as long as uintptr_t is the
370 : * correct width.
371 : */
372 : #define att_align_pointer(cur_offset, attalign, attlen, attptr) \
373 : ( \
374 : ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? \
375 : (uintptr_t) (cur_offset) : \
376 : att_align_nominal(cur_offset, attalign) \
377 : )
378 :
379 : /*
380 : * Similar to att_align_pointer, but accepts a number of bytes, typically from
381 : * CompactAttribute.attalignby to align the pointer by.
382 : */
383 : #define att_pointer_alignby(cur_offset, attalignby, attlen, attptr) \
384 : ( \
385 : ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? \
386 : (uintptr_t) (cur_offset) : \
387 : TYPEALIGN(attalignby, cur_offset))
388 :
389 : /*
390 : * att_align_nominal aligns the given offset as needed for a datum of alignment
391 : * requirement attalign, ignoring any consideration of packed varlena datums.
392 : * There are three main use cases for using this macro directly:
393 : * * we know that the att in question is not varlena (attlen != -1);
394 : * in this case it is cheaper than the above macros and just as good.
395 : * * we need to estimate alignment padding cost abstractly, ie without
396 : * reference to a real tuple. We must assume the worst case that
397 : * all varlenas are aligned.
398 : * * within arrays and multiranges, we unconditionally align varlenas (XXX this
399 : * should be revisited, probably).
400 : *
401 : * In performance-critical loops, avoid using this macro; instead use
402 : * att_nominal_alignby with a pre-computed alignby value.
403 : */
404 : #define att_align_nominal(cur_offset, attalign) \
405 : att_nominal_alignby(cur_offset, typalign_to_alignby(attalign))
406 :
407 : /*
408 : * Similar to att_align_nominal, but accepts a number of bytes, typically from
409 : * CompactAttribute.attalignby to align the offset by.
410 : */
411 : #define att_nominal_alignby(cur_offset, attalignby) \
412 : TYPEALIGN(attalignby, cur_offset)
413 :
414 : /*
415 : * att_addlength_datum increments the given offset by the space needed for
416 : * the given Datum variable. attdatum is only accessed if we are dealing
417 : * with a variable-length attribute.
418 : */
419 : #define att_addlength_datum(cur_offset, attlen, attdatum) \
420 : att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum))
421 :
422 : /*
423 : * att_addlength_pointer performs the same calculation as att_addlength_datum,
424 : * but is used when walking a tuple --- attptr is the pointer to the field
425 : * within the tuple.
426 : *
427 : * Note: some callers pass a "char *" pointer for cur_offset. This is
428 : * actually perfectly OK, but probably should be cleaned up along with
429 : * the same practice for att_align_pointer.
430 : */
431 : #define att_addlength_pointer(cur_offset, attlen, attptr) \
432 : ( \
433 : ((attlen) > 0) ? \
434 : ( \
435 : (cur_offset) + (attlen) \
436 : ) \
437 : : (((attlen) == -1) ? \
438 : ( \
439 : (cur_offset) + VARSIZE_ANY(attptr) \
440 : ) \
441 : : \
442 : ( \
443 : AssertMacro((attlen) == -2), \
444 : (cur_offset) + (strlen((const char *) (attptr)) + 1) \
445 : )) \
446 : )
447 :
448 : #ifndef FRONTEND
449 : /*
450 : * store_att_byval is a partial inverse of fetch_att: store a given Datum
451 : * value into a tuple data area at the specified address. However, it only
452 : * handles the byval case, because in typical usage the caller needs to
453 : * distinguish by-val and by-ref cases anyway, and so a do-it-all function
454 : * wouldn't be convenient.
455 : */
456 : static inline void
457 123935628 : store_att_byval(void *T, Datum newdatum, int attlen)
458 : {
459 123935628 : switch (attlen)
460 : {
461 16166231 : case sizeof(char):
462 16166231 : *(char *) T = DatumGetChar(newdatum);
463 16166231 : break;
464 6349437 : case sizeof(int16):
465 6349437 : *(int16 *) T = DatumGetInt16(newdatum);
466 6349437 : break;
467 89446730 : case sizeof(int32):
468 89446730 : *(int32 *) T = DatumGetInt32(newdatum);
469 89446730 : break;
470 11973230 : case sizeof(int64):
471 11973230 : *(int64 *) T = DatumGetInt64(newdatum);
472 11973230 : break;
473 0 : default:
474 0 : elog(ERROR, "unsupported byval length: %d", attlen);
475 : }
476 123935628 : }
477 : #endif /* FRONTEND */
478 :
479 : #endif /* TUPMACS_H */
|
