Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_deform.c
4 : * Generate code for deforming a heap tuple.
5 : *
6 : * This gains performance benefits over unJITed deforming from compile-time
7 : * knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc
8 : * can be taken advantage of.
9 : *
10 : * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
11 : * Portions Copyright (c) 1994, Regents of the University of California
12 : *
13 : * IDENTIFICATION
14 : * src/backend/jit/llvm/llvmjit_deform.c
15 : *
16 : *-------------------------------------------------------------------------
17 : */
18 :
19 : #include "postgres.h"
20 :
21 : #include <llvm-c/Core.h>
22 :
23 : #include "access/htup_details.h"
24 : #include "access/tupdesc_details.h"
25 : #include "executor/tuptable.h"
26 : #include "jit/llvmjit.h"
27 : #include "jit/llvmjit_emit.h"
28 :
29 :
30 : /*
31 : * Create a function that deforms a tuple of type desc up to natts columns.
32 : */
33 : LLVMValueRef
34 5692 : slot_compile_deform(LLVMJitContext *context, TupleDesc desc,
35 : const TupleTableSlotOps *ops, int natts)
36 : {
37 : char *funcname;
38 :
39 : LLVMModuleRef mod;
40 : LLVMContextRef lc;
41 : LLVMBuilderRef b;
42 :
43 : LLVMTypeRef deform_sig;
44 : LLVMValueRef v_deform_fn;
45 :
46 : LLVMBasicBlockRef b_entry;
47 : LLVMBasicBlockRef b_adjust_unavail_cols;
48 : LLVMBasicBlockRef b_find_start;
49 :
50 : LLVMBasicBlockRef b_out;
51 : LLVMBasicBlockRef b_dead;
52 : LLVMBasicBlockRef *attcheckattnoblocks;
53 : LLVMBasicBlockRef *attstartblocks;
54 : LLVMBasicBlockRef *attisnullblocks;
55 : LLVMBasicBlockRef *attcheckalignblocks;
56 : LLVMBasicBlockRef *attalignblocks;
57 : LLVMBasicBlockRef *attstoreblocks;
58 :
59 : LLVMValueRef v_offp;
60 :
61 : LLVMValueRef v_tupdata_base;
62 : LLVMValueRef v_tts_values;
63 : LLVMValueRef v_tts_nulls;
64 : LLVMValueRef v_slotoffp;
65 : LLVMValueRef v_flagsp;
66 : LLVMValueRef v_nvalidp;
67 : LLVMValueRef v_nvalid;
68 : LLVMValueRef v_maxatt;
69 :
70 : LLVMValueRef v_slot;
71 :
72 : LLVMValueRef v_tupleheaderp;
73 : LLVMValueRef v_tuplep;
74 : LLVMValueRef v_infomask1;
75 : LLVMValueRef v_infomask2;
76 : LLVMValueRef v_bits;
77 :
78 : LLVMValueRef v_hoff;
79 :
80 : LLVMValueRef v_hasnulls;
81 :
82 : /* last column (0 indexed) guaranteed to exist */
83 5692 : int guaranteed_column_number = -1;
84 :
85 : /* current known alignment */
86 5692 : int known_alignment = 0;
87 :
88 : /* if true, known_alignment describes definite offset of column */
89 5692 : bool attguaranteedalign = true;
90 :
91 : int attnum;
92 :
93 : /* virtual tuples never need deforming, so don't generate code */
94 5692 : if (ops == &TTSOpsVirtual)
95 0 : return NULL;
96 :
97 : /* decline to JIT for slot types we don't know to handle */
98 5692 : if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple &&
99 : ops != &TTSOpsMinimalTuple)
100 0 : return NULL;
101 :
102 5692 : mod = llvm_mutable_module(context);
103 5692 : lc = LLVMGetModuleContext(mod);
104 :
105 5692 : funcname = llvm_expand_funcname(context, "deform");
106 :
107 : /*
108 : * Check which columns have to exist, so we don't have to check the row's
109 : * natts unnecessarily.
110 : */
111 29536 : for (attnum = 0; attnum < desc->natts; attnum++)
112 : {
113 23844 : CompactAttribute *att = TupleDescCompactAttr(desc, attnum);
114 :
115 : /*
116 : * If the column is declared NOT NULL then it must be present in every
117 : * tuple, unless there's a "missing" entry that could provide a
118 : * non-NULL value for it. That in turn guarantees that the NULL bitmap
119 : * - if there are any NULLable columns - is at least long enough to
120 : * cover columns up to attnum.
121 : *
122 : * Be paranoid and also check !attisdropped, even though the
123 : * combination of attisdropped && attnotnull combination shouldn't
124 : * exist.
125 : */
126 23844 : if (att->attnotnull &&
127 6038 : !att->atthasmissing &&
128 6038 : !att->attisdropped)
129 6038 : guaranteed_column_number = attnum;
130 : }
131 :
132 : /* Create the signature and function */
133 : {
134 : LLVMTypeRef param_types[1];
135 :
136 5692 : param_types[0] = l_ptr(StructTupleTableSlot);
137 :
138 5692 : deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc),
139 : param_types, lengthof(param_types), 0);
140 : }
141 5692 : v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig);
142 5692 : LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage);
143 5692 : LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF);
144 5692 : llvm_copy_attributes(AttributeTemplate, v_deform_fn);
145 :
146 : b_entry =
147 5692 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry");
148 : b_adjust_unavail_cols =
149 5692 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols");
150 : b_find_start =
151 5692 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock");
152 : b_out =
153 5692 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock");
154 : b_dead =
155 5692 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock");
156 :
157 5692 : b = LLVMCreateBuilderInContext(lc);
158 :
159 5692 : attcheckattnoblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
160 5692 : attstartblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
161 5692 : attisnullblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
162 5692 : attcheckalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
163 5692 : attalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
164 5692 : attstoreblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
165 :
166 5692 : known_alignment = 0;
167 :
168 5692 : LLVMPositionBuilderAtEnd(b, b_entry);
169 :
170 : /* perform allocas first, llvm only converts those to registers */
171 5692 : v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp");
172 :
173 5692 : v_slot = LLVMGetParam(v_deform_fn, 0);
174 :
175 : v_tts_values =
176 5692 : l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES,
177 : "tts_values");
178 : v_tts_nulls =
179 5692 : l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL,
180 : "tts_ISNULL");
181 5692 : v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, "");
182 5692 : v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, "");
183 :
184 5692 : if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple)
185 3174 : {
186 : LLVMValueRef v_heapslot;
187 :
188 : v_heapslot =
189 3174 : LLVMBuildBitCast(b,
190 : v_slot,
191 : l_ptr(StructHeapTupleTableSlot),
192 : "heapslot");
193 3174 : v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, "");
194 : v_tupleheaderp =
195 3174 : l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE,
196 : "tupleheader");
197 : }
198 2518 : else if (ops == &TTSOpsMinimalTuple)
199 : {
200 : LLVMValueRef v_minimalslot;
201 :
202 : v_minimalslot =
203 2518 : LLVMBuildBitCast(b,
204 : v_slot,
205 : l_ptr(StructMinimalTupleTableSlot),
206 : "minimalslot");
207 2518 : v_slotoffp = l_struct_gep(b,
208 : StructMinimalTupleTableSlot,
209 : v_minimalslot,
210 : FIELDNO_MINIMALTUPLETABLESLOT_OFF, "");
211 : v_tupleheaderp =
212 2518 : l_load_struct_gep(b,
213 : StructMinimalTupleTableSlot,
214 : v_minimalslot,
215 : FIELDNO_MINIMALTUPLETABLESLOT_TUPLE,
216 : "tupleheader");
217 : }
218 : else
219 : {
220 : /* should've returned at the start of the function */
221 0 : pg_unreachable();
222 : }
223 :
224 : v_tuplep =
225 5692 : l_load_struct_gep(b,
226 : StructHeapTupleData,
227 : v_tupleheaderp,
228 : FIELDNO_HEAPTUPLEDATA_DATA,
229 : "tuple");
230 : v_bits =
231 5692 : LLVMBuildBitCast(b,
232 : l_struct_gep(b,
233 : StructHeapTupleHeaderData,
234 : v_tuplep,
235 : FIELDNO_HEAPTUPLEHEADERDATA_BITS,
236 : ""),
237 : l_ptr(LLVMInt8TypeInContext(lc)),
238 : "t_bits");
239 : v_infomask1 =
240 5692 : l_load_struct_gep(b,
241 : StructHeapTupleHeaderData,
242 : v_tuplep,
243 : FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK,
244 : "infomask1");
245 : v_infomask2 =
246 5692 : l_load_struct_gep(b,
247 : StructHeapTupleHeaderData,
248 : v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2,
249 : "infomask2");
250 :
251 : /* t_infomask & HEAP_HASNULL */
252 : v_hasnulls =
253 5692 : LLVMBuildICmp(b, LLVMIntNE,
254 : LLVMBuildAnd(b,
255 : l_int16_const(lc, HEAP_HASNULL),
256 : v_infomask1, ""),
257 : l_int16_const(lc, 0),
258 : "hasnulls");
259 :
260 : /* t_infomask2 & HEAP_NATTS_MASK */
261 5692 : v_maxatt = LLVMBuildAnd(b,
262 : l_int16_const(lc, HEAP_NATTS_MASK),
263 : v_infomask2,
264 : "maxatt");
265 :
266 : /*
267 : * Need to zext, as getelementptr otherwise treats hoff as a signed 8bit
268 : * integer, which'd yield a negative offset for t_hoff > 127.
269 : */
270 5692 : v_hoff =
271 5692 : LLVMBuildZExt(b,
272 : l_load_struct_gep(b,
273 : StructHeapTupleHeaderData,
274 : v_tuplep,
275 : FIELDNO_HEAPTUPLEHEADERDATA_HOFF,
276 : ""),
277 : LLVMInt32TypeInContext(lc), "t_hoff");
278 :
279 5692 : v_tupdata_base = l_gep(b,
280 : LLVMInt8TypeInContext(lc),
281 : LLVMBuildBitCast(b,
282 : v_tuplep,
283 : l_ptr(LLVMInt8TypeInContext(lc)),
284 : ""),
285 : &v_hoff, 1,
286 : "v_tupdata_base");
287 :
288 : /*
289 : * Load tuple start offset from slot. Will be reset below in case there's
290 : * no existing deformed columns in slot.
291 : */
292 : {
293 : LLVMValueRef v_off_start;
294 :
295 5692 : v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off");
296 5692 : v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, "");
297 5692 : LLVMBuildStore(b, v_off_start, v_offp);
298 : }
299 :
300 : /* build the basic block for each attribute, need them as jump target */
301 18274 : for (attnum = 0; attnum < natts; attnum++)
302 : {
303 25164 : attcheckattnoblocks[attnum] =
304 12582 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum);
305 25164 : attstartblocks[attnum] =
306 12582 : l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum);
307 25164 : attisnullblocks[attnum] =
308 12582 : l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum);
309 25164 : attcheckalignblocks[attnum] =
310 12582 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum);
311 25164 : attalignblocks[attnum] =
312 12582 : l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum);
313 12582 : attstoreblocks[attnum] =
314 12582 : l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum);
315 : }
316 :
317 : /*
318 : * Check if it is guaranteed that all the desired attributes are available
319 : * in the tuple (but still possibly NULL), by dint of either the last
320 : * to-be-deformed column being NOT NULL, or subsequent ones not accessed
321 : * here being NOT NULL. If that's not guaranteed the tuple headers natt's
322 : * has to be checked, and missing attributes potentially have to be
323 : * fetched (using slot_getmissingattrs().
324 : */
325 5692 : if ((natts - 1) <= guaranteed_column_number)
326 : {
327 : /* just skip through unnecessary blocks */
328 414 : LLVMBuildBr(b, b_adjust_unavail_cols);
329 414 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
330 414 : LLVMBuildBr(b, b_find_start);
331 : }
332 : else
333 : {
334 : LLVMValueRef v_params[3];
335 : LLVMValueRef f;
336 :
337 : /* branch if not all columns available */
338 5278 : LLVMBuildCondBr(b,
339 : LLVMBuildICmp(b, LLVMIntULT,
340 : v_maxatt,
341 : l_int16_const(lc, natts),
342 : ""),
343 : b_adjust_unavail_cols,
344 : b_find_start);
345 :
346 : /* if not, memset tts_isnull of relevant cols to true */
347 5278 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
348 :
349 5278 : v_params[0] = v_slot;
350 5278 : v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), "");
351 5278 : v_params[2] = l_int32_const(lc, natts);
352 5278 : f = llvm_pg_func(mod, "slot_getmissingattrs");
353 5278 : l_call(b,
354 : LLVMGetFunctionType(f), f,
355 : v_params, lengthof(v_params), "");
356 5278 : LLVMBuildBr(b, b_find_start);
357 : }
358 :
359 5692 : LLVMPositionBuilderAtEnd(b, b_find_start);
360 :
361 5692 : v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, "");
362 :
363 : /*
364 : * Build switch to go from nvalid to the right startblock. Callers
365 : * currently don't have the knowledge, but it'd be good for performance to
366 : * avoid this check when it's known that the slot is empty (e.g. in scan
367 : * nodes).
368 : */
369 : if (true)
370 : {
371 5692 : LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid,
372 : b_dead, natts);
373 :
374 18274 : for (attnum = 0; attnum < natts; attnum++)
375 : {
376 12582 : LLVMValueRef v_attno = l_int16_const(lc, attnum);
377 :
378 12582 : LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]);
379 : }
380 : }
381 : else
382 : {
383 : /* jump from entry block to first block */
384 : LLVMBuildBr(b, attcheckattnoblocks[0]);
385 : }
386 :
387 5692 : LLVMPositionBuilderAtEnd(b, b_dead);
388 5692 : LLVMBuildUnreachable(b);
389 :
390 : /*
391 : * Iterate over each attribute that needs to be deformed, build code to
392 : * deform it.
393 : */
394 18274 : for (attnum = 0; attnum < natts; attnum++)
395 : {
396 12582 : CompactAttribute *att = TupleDescCompactAttr(desc, attnum);
397 : LLVMValueRef v_incby;
398 12582 : int alignto = att->attalignby;
399 12582 : LLVMValueRef l_attno = l_int16_const(lc, attnum);
400 : LLVMValueRef v_attdatap;
401 : LLVMValueRef v_resultp;
402 :
403 : /* build block checking whether we did all the necessary attributes */
404 12582 : LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]);
405 :
406 : /*
407 : * If this is the first attribute, slot->tts_nvalid was 0. Therefore
408 : * also reset offset to 0, it may be from a previous execution.
409 : */
410 12582 : if (attnum == 0)
411 : {
412 5692 : LLVMBuildStore(b, l_sizet_const(0), v_offp);
413 : }
414 :
415 : /*
416 : * Build check whether column is available (i.e. whether the tuple has
417 : * that many columns stored). We can avoid the branch if we know
418 : * there's a subsequent NOT NULL column.
419 : */
420 12582 : if (attnum <= guaranteed_column_number)
421 : {
422 1608 : LLVMBuildBr(b, attstartblocks[attnum]);
423 : }
424 : else
425 : {
426 : LLVMValueRef v_islast;
427 :
428 10974 : v_islast = LLVMBuildICmp(b, LLVMIntUGE,
429 : l_attno,
430 : v_maxatt,
431 : "heap_natts");
432 10974 : LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]);
433 : }
434 12582 : LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]);
435 :
436 : /*
437 : * Check for nulls if necessary. No need to take missing attributes
438 : * into account, because if they're present the heaptuple's natts
439 : * would have indicated that a slot_getmissingattrs() is needed.
440 : */
441 12582 : if (!att->attnotnull)
442 : {
443 : LLVMBasicBlockRef b_ifnotnull;
444 : LLVMBasicBlockRef b_ifnull;
445 : LLVMBasicBlockRef b_next;
446 : LLVMValueRef v_attisnull;
447 : LLVMValueRef v_nullbyteno;
448 : LLVMValueRef v_nullbytemask;
449 : LLVMValueRef v_nullbyte;
450 : LLVMValueRef v_nullbit;
451 :
452 10986 : b_ifnotnull = attcheckalignblocks[attnum];
453 10986 : b_ifnull = attisnullblocks[attnum];
454 :
455 10986 : if (attnum + 1 == natts)
456 5278 : b_next = b_out;
457 : else
458 5708 : b_next = attcheckattnoblocks[attnum + 1];
459 :
460 10986 : v_nullbyteno = l_int32_const(lc, attnum >> 3);
461 10986 : v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07));
462 10986 : v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte");
463 :
464 10986 : v_nullbit = LLVMBuildICmp(b,
465 : LLVMIntEQ,
466 : LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""),
467 : l_int8_const(lc, 0),
468 : "attisnull");
469 :
470 10986 : v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, "");
471 :
472 10986 : LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull);
473 :
474 10986 : LLVMPositionBuilderAtEnd(b, b_ifnull);
475 :
476 : /* store null-byte */
477 10986 : LLVMBuildStore(b,
478 : l_int8_const(lc, 1),
479 : l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, ""));
480 : /* store zero datum */
481 10986 : LLVMBuildStore(b,
482 : l_sizet_const(0),
483 : l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, ""));
484 :
485 10986 : LLVMBuildBr(b, b_next);
486 10986 : attguaranteedalign = false;
487 : }
488 : else
489 : {
490 : /* nothing to do */
491 1596 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
492 1596 : LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]);
493 1596 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
494 : }
495 12582 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
496 :
497 : /* ------
498 : * Even if alignment is required, we can skip doing it if provably
499 : * unnecessary:
500 : * - first column is guaranteed to be aligned
501 : * - columns following a NOT NULL fixed width datum have known
502 : * alignment, can skip alignment computation if that known alignment
503 : * is compatible with current column.
504 : * ------
505 : */
506 12582 : if (alignto > 1 &&
507 6014 : (known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment)))
508 : {
509 : /*
510 : * When accessing a varlena field, we have to "peek" to see if we
511 : * are looking at a pad byte or the first byte of a 1-byte-header
512 : * datum. A zero byte must be either a pad byte, or the first
513 : * byte of a correctly aligned 4-byte length word; in either case,
514 : * we can align safely. A non-zero byte must be either a 1-byte
515 : * length word, or the first byte of a correctly aligned 4-byte
516 : * length word; in either case, we need not align.
517 : */
518 5292 : if (att->attlen == -1)
519 : {
520 : LLVMValueRef v_possible_padbyte;
521 : LLVMValueRef v_ispad;
522 : LLVMValueRef v_off;
523 :
524 : /* don't know if short varlena or not */
525 672 : attguaranteedalign = false;
526 :
527 672 : v_off = l_load(b, TypeSizeT, v_offp, "");
528 :
529 : v_possible_padbyte =
530 672 : l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, v_off, "padbyte");
531 : v_ispad =
532 672 : LLVMBuildICmp(b, LLVMIntEQ,
533 : v_possible_padbyte, l_int8_const(lc, 0),
534 : "ispadbyte");
535 672 : LLVMBuildCondBr(b, v_ispad,
536 672 : attalignblocks[attnum],
537 672 : attstoreblocks[attnum]);
538 : }
539 : else
540 : {
541 4620 : LLVMBuildBr(b, attalignblocks[attnum]);
542 : }
543 :
544 5292 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
545 :
546 : /* translation of alignment code (cf TYPEALIGN()) */
547 : {
548 : LLVMValueRef v_off_aligned;
549 5292 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
550 :
551 : /* ((ALIGNVAL) - 1) */
552 5292 : LLVMValueRef v_alignval = l_sizet_const(alignto - 1);
553 :
554 : /* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */
555 5292 : LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, "");
556 :
557 : /* ~((uintptr_t) ((ALIGNVAL) - 1)) */
558 5292 : LLVMValueRef v_rh = l_sizet_const(~(alignto - 1));
559 :
560 5292 : v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset");
561 :
562 5292 : LLVMBuildStore(b, v_off_aligned, v_offp);
563 : }
564 :
565 : /*
566 : * As alignment either was unnecessary or has been performed, we
567 : * now know the current alignment. This is only safe because this
568 : * value isn't used for varlena and nullable columns.
569 : */
570 5292 : if (known_alignment >= 0)
571 : {
572 : Assert(known_alignment != 0);
573 12 : known_alignment = TYPEALIGN(alignto, known_alignment);
574 : }
575 :
576 5292 : LLVMBuildBr(b, attstoreblocks[attnum]);
577 5292 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
578 : }
579 : else
580 : {
581 7290 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
582 7290 : LLVMBuildBr(b, attalignblocks[attnum]);
583 7290 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
584 7290 : LLVMBuildBr(b, attstoreblocks[attnum]);
585 : }
586 12582 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
587 :
588 : /*
589 : * Store the current offset if known to be constant. That allows LLVM
590 : * to generate better code. Without that LLVM can't figure out that
591 : * the offset might be constant due to the jumps for previously
592 : * decoded columns.
593 : */
594 12582 : if (attguaranteedalign)
595 : {
596 : Assert(known_alignment >= 0);
597 1584 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
598 : }
599 :
600 : /* compute what following columns are aligned to */
601 12582 : if (att->attlen < 0)
602 : {
603 : /* can't guarantee any alignment after variable length field */
604 1164 : known_alignment = -1;
605 1164 : attguaranteedalign = false;
606 : }
607 11418 : else if (att->attnotnull && attguaranteedalign && known_alignment >= 0)
608 : {
609 : /*
610 : * If the offset to the column was previously known, a NOT NULL &
611 : * fixed-width column guarantees that alignment is just the
612 : * previous alignment plus column width.
613 : */
614 : Assert(att->attlen > 0);
615 1572 : known_alignment += att->attlen;
616 : }
617 9846 : else if (att->attnotnull && (att->attlen % alignto) == 0)
618 : {
619 : /*
620 : * After a NOT NULL fixed-width column with a length that is a
621 : * multiple of its alignment requirement, we know the following
622 : * column is aligned to at least the current column's alignment.
623 : */
624 : Assert(att->attlen > 0);
625 0 : known_alignment = alignto;
626 : Assert(known_alignment > 0);
627 0 : attguaranteedalign = false;
628 : }
629 : else
630 : {
631 9846 : known_alignment = -1;
632 9846 : attguaranteedalign = false;
633 : }
634 :
635 :
636 : /* compute address to load data from */
637 : {
638 12582 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
639 :
640 12582 : v_attdatap =
641 12582 : l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, "");
642 : }
643 :
644 : /* compute address to store value at */
645 12582 : v_resultp = l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, "");
646 :
647 : /* store null-byte (false) */
648 12582 : LLVMBuildStore(b, l_int8_const(lc, 0),
649 : l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, ""));
650 :
651 : /*
652 : * Store datum. For byval: datums copy the value, extend to Datum's
653 : * width, and store. For byref types: store pointer to data.
654 : */
655 12582 : if (att->attbyval)
656 : {
657 : LLVMValueRef v_tmp_loaddata;
658 10416 : LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8);
659 10416 : LLVMTypeRef vartypep = LLVMPointerType(vartype, 0);
660 :
661 : v_tmp_loaddata =
662 10416 : LLVMBuildPointerCast(b, v_attdatap, vartypep, "");
663 10416 : v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval");
664 10416 : v_tmp_loaddata = LLVMBuildZExt(b, v_tmp_loaddata, TypeSizeT, "");
665 :
666 10416 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
667 : }
668 : else
669 : {
670 : LLVMValueRef v_tmp_loaddata;
671 :
672 : /* store pointer */
673 : v_tmp_loaddata =
674 2166 : LLVMBuildPtrToInt(b,
675 : v_attdatap,
676 : TypeSizeT,
677 : "attr_ptr");
678 2166 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
679 : }
680 :
681 : /* increment data pointer */
682 12582 : if (att->attlen > 0)
683 : {
684 11418 : v_incby = l_sizet_const(att->attlen);
685 : }
686 1164 : else if (att->attlen == -1)
687 : {
688 1164 : v_incby = l_call(b,
689 : llvm_pg_var_func_type("varsize_any"),
690 : llvm_pg_func(mod, "varsize_any"),
691 : &v_attdatap, 1,
692 : "varsize_any");
693 1164 : l_callsite_ro(v_incby);
694 1164 : l_callsite_alwaysinline(v_incby);
695 : }
696 0 : else if (att->attlen == -2)
697 : {
698 0 : v_incby = l_call(b,
699 : llvm_pg_var_func_type("strlen"),
700 : llvm_pg_func(mod, "strlen"),
701 : &v_attdatap, 1, "strlen");
702 :
703 0 : l_callsite_ro(v_incby);
704 :
705 : /* add 1 for NUL byte */
706 0 : v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), "");
707 : }
708 : else
709 : {
710 : Assert(false);
711 0 : v_incby = NULL; /* silence compiler */
712 : }
713 :
714 12582 : if (attguaranteedalign)
715 : {
716 : Assert(known_alignment >= 0);
717 1572 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
718 : }
719 : else
720 : {
721 11010 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
722 :
723 11010 : v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset");
724 11010 : LLVMBuildStore(b, v_off, v_offp);
725 : }
726 :
727 : /*
728 : * jump to next block, unless last possible column, or all desired
729 : * (available) attributes have been fetched.
730 : */
731 12582 : if (attnum + 1 == natts)
732 : {
733 : /* jump out */
734 5692 : LLVMBuildBr(b, b_out);
735 : }
736 : else
737 : {
738 6890 : LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]);
739 : }
740 : }
741 :
742 :
743 : /* build block that returns */
744 5692 : LLVMPositionBuilderAtEnd(b, b_out);
745 :
746 : {
747 5692 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
748 : LLVMValueRef v_flags;
749 :
750 5692 : LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp);
751 5692 : v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), "");
752 5692 : LLVMBuildStore(b, v_off, v_slotoffp);
753 5692 : v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags");
754 5692 : v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), "");
755 5692 : LLVMBuildStore(b, v_flags, v_flagsp);
756 5692 : LLVMBuildRetVoid(b);
757 : }
758 :
759 5692 : LLVMDisposeBuilder(b);
760 :
761 5692 : return v_deform_fn;
762 : }
|
