Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_deform.c
4 : * Generate code for deforming a heap tuple.
5 : *
6 : * This gains performance benefits over unJITed deforming from compile-time
7 : * knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc
8 : * can be taken advantage of.
9 : *
10 : * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
11 : * Portions Copyright (c) 1994, Regents of the University of California
12 : *
13 : * IDENTIFICATION
14 : * src/backend/jit/llvm/llvmjit_deform.c
15 : *
16 : *-------------------------------------------------------------------------
17 : */
18 :
19 : #include "postgres.h"
20 :
21 : #include <llvm-c/Core.h>
22 :
23 : #include "access/htup_details.h"
24 : #include "access/tupdesc_details.h"
25 : #include "executor/tuptable.h"
26 : #include "jit/llvmjit.h"
27 : #include "jit/llvmjit_emit.h"
28 :
29 :
30 : /*
31 : * Create a function that deforms a tuple of type desc up to natts columns.
32 : */
33 : LLVMValueRef
34 6776 : slot_compile_deform(LLVMJitContext *context, TupleDesc desc,
35 : const TupleTableSlotOps *ops, int natts)
36 : {
37 : char *funcname;
38 :
39 : LLVMModuleRef mod;
40 : LLVMContextRef lc;
41 : LLVMBuilderRef b;
42 :
43 : LLVMTypeRef deform_sig;
44 : LLVMValueRef v_deform_fn;
45 :
46 : LLVMBasicBlockRef b_entry;
47 : LLVMBasicBlockRef b_adjust_unavail_cols;
48 : LLVMBasicBlockRef b_find_start;
49 :
50 : LLVMBasicBlockRef b_out;
51 : LLVMBasicBlockRef b_dead;
52 : LLVMBasicBlockRef *attcheckattnoblocks;
53 : LLVMBasicBlockRef *attstartblocks;
54 : LLVMBasicBlockRef *attisnullblocks;
55 : LLVMBasicBlockRef *attcheckalignblocks;
56 : LLVMBasicBlockRef *attalignblocks;
57 : LLVMBasicBlockRef *attstoreblocks;
58 :
59 : LLVMValueRef v_offp;
60 :
61 : LLVMValueRef v_tupdata_base;
62 : LLVMValueRef v_tts_values;
63 : LLVMValueRef v_tts_nulls;
64 : LLVMValueRef v_slotoffp;
65 : LLVMValueRef v_flagsp;
66 : LLVMValueRef v_nvalidp;
67 : LLVMValueRef v_nvalid;
68 : LLVMValueRef v_maxatt;
69 :
70 : LLVMValueRef v_slot;
71 :
72 : LLVMValueRef v_tupleheaderp;
73 : LLVMValueRef v_tuplep;
74 : LLVMValueRef v_infomask1;
75 : LLVMValueRef v_infomask2;
76 : LLVMValueRef v_bits;
77 :
78 : LLVMValueRef v_hoff;
79 :
80 : LLVMValueRef v_hasnulls;
81 :
82 : /* last column (0 indexed) guaranteed to exist */
83 6776 : int guaranteed_column_number = -1;
84 :
85 : /* current known alignment */
86 6776 : int known_alignment = 0;
87 :
88 : /* if true, known_alignment describes definite offset of column */
89 6776 : bool attguaranteedalign = true;
90 :
91 : int attnum;
92 :
93 : /* virtual tuples never need deforming, so don't generate code */
94 6776 : if (ops == &TTSOpsVirtual)
95 0 : return NULL;
96 :
97 : /* decline to JIT for slot types we don't know to handle */
98 6776 : if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple &&
99 : ops != &TTSOpsMinimalTuple)
100 0 : return NULL;
101 :
102 6776 : mod = llvm_mutable_module(context);
103 6776 : lc = LLVMGetModuleContext(mod);
104 :
105 6776 : funcname = llvm_expand_funcname(context, "deform");
106 :
107 : /*
108 : * Check which columns have to exist, so we don't have to check the row's
109 : * natts unnecessarily.
110 : */
111 40424 : for (attnum = 0; attnum < desc->natts; attnum++)
112 : {
113 33648 : Form_pg_attribute att = TupleDescAttr(desc, attnum);
114 :
115 : /*
116 : * If the column is declared NOT NULL then it must be present in every
117 : * tuple, unless there's a "missing" entry that could provide a
118 : * non-NULL value for it. That in turn guarantees that the NULL bitmap
119 : * - if there are any NULLable columns - is at least long enough to
120 : * cover columns up to attnum.
121 : *
122 : * Be paranoid and also check !attisdropped, even though the
123 : * combination of attisdropped && attnotnull combination shouldn't
124 : * exist.
125 : */
126 33648 : if (att->attnotnull &&
127 7882 : !att->atthasmissing &&
128 7882 : !att->attisdropped)
129 7882 : guaranteed_column_number = attnum;
130 : }
131 :
132 : /* Create the signature and function */
133 : {
134 : LLVMTypeRef param_types[1];
135 :
136 6776 : param_types[0] = l_ptr(StructTupleTableSlot);
137 :
138 6776 : deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc),
139 : param_types, lengthof(param_types), 0);
140 : }
141 6776 : v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig);
142 6776 : LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage);
143 6776 : LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF);
144 6776 : llvm_copy_attributes(AttributeTemplate, v_deform_fn);
145 :
146 : b_entry =
147 6776 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry");
148 : b_adjust_unavail_cols =
149 6776 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols");
150 : b_find_start =
151 6776 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock");
152 : b_out =
153 6776 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock");
154 : b_dead =
155 6776 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock");
156 :
157 6776 : b = LLVMCreateBuilderInContext(lc);
158 :
159 6776 : attcheckattnoblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
160 6776 : attstartblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
161 6776 : attisnullblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
162 6776 : attcheckalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
163 6776 : attalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
164 6776 : attstoreblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
165 :
166 6776 : known_alignment = 0;
167 :
168 6776 : LLVMPositionBuilderAtEnd(b, b_entry);
169 :
170 : /* perform allocas first, llvm only converts those to registers */
171 6776 : v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp");
172 :
173 6776 : v_slot = LLVMGetParam(v_deform_fn, 0);
174 :
175 : v_tts_values =
176 6776 : l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES,
177 : "tts_values");
178 : v_tts_nulls =
179 6776 : l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL,
180 : "tts_ISNULL");
181 6776 : v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, "");
182 6776 : v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, "");
183 :
184 6776 : if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple)
185 4112 : {
186 : LLVMValueRef v_heapslot;
187 :
188 : v_heapslot =
189 4112 : LLVMBuildBitCast(b,
190 : v_slot,
191 : l_ptr(StructHeapTupleTableSlot),
192 : "heapslot");
193 4112 : v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, "");
194 : v_tupleheaderp =
195 4112 : l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE,
196 : "tupleheader");
197 : }
198 2664 : else if (ops == &TTSOpsMinimalTuple)
199 : {
200 : LLVMValueRef v_minimalslot;
201 :
202 : v_minimalslot =
203 2664 : LLVMBuildBitCast(b,
204 : v_slot,
205 : l_ptr(StructMinimalTupleTableSlot),
206 : "minimalslot");
207 2664 : v_slotoffp = l_struct_gep(b,
208 : StructMinimalTupleTableSlot,
209 : v_minimalslot,
210 : FIELDNO_MINIMALTUPLETABLESLOT_OFF, "");
211 : v_tupleheaderp =
212 2664 : l_load_struct_gep(b,
213 : StructMinimalTupleTableSlot,
214 : v_minimalslot,
215 : FIELDNO_MINIMALTUPLETABLESLOT_TUPLE,
216 : "tupleheader");
217 : }
218 : else
219 : {
220 : /* should've returned at the start of the function */
221 0 : pg_unreachable();
222 : }
223 :
224 : v_tuplep =
225 6776 : l_load_struct_gep(b,
226 : StructHeapTupleData,
227 : v_tupleheaderp,
228 : FIELDNO_HEAPTUPLEDATA_DATA,
229 : "tuple");
230 : v_bits =
231 6776 : LLVMBuildBitCast(b,
232 : l_struct_gep(b,
233 : StructHeapTupleHeaderData,
234 : v_tuplep,
235 : FIELDNO_HEAPTUPLEHEADERDATA_BITS,
236 : ""),
237 : l_ptr(LLVMInt8TypeInContext(lc)),
238 : "t_bits");
239 : v_infomask1 =
240 6776 : l_load_struct_gep(b,
241 : StructHeapTupleHeaderData,
242 : v_tuplep,
243 : FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK,
244 : "infomask1");
245 : v_infomask2 =
246 6776 : l_load_struct_gep(b,
247 : StructHeapTupleHeaderData,
248 : v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2,
249 : "infomask2");
250 :
251 : /* t_infomask & HEAP_HASNULL */
252 : v_hasnulls =
253 6776 : LLVMBuildICmp(b, LLVMIntNE,
254 : LLVMBuildAnd(b,
255 : l_int16_const(lc, HEAP_HASNULL),
256 : v_infomask1, ""),
257 : l_int16_const(lc, 0),
258 : "hasnulls");
259 :
260 : /* t_infomask2 & HEAP_NATTS_MASK */
261 6776 : v_maxatt = LLVMBuildAnd(b,
262 : l_int16_const(lc, HEAP_NATTS_MASK),
263 : v_infomask2,
264 : "maxatt");
265 :
266 : /*
267 : * Need to zext, as getelementptr otherwise treats hoff as a signed 8bit
268 : * integer, which'd yield a negative offset for t_hoff > 127.
269 : */
270 6776 : v_hoff =
271 6776 : LLVMBuildZExt(b,
272 : l_load_struct_gep(b,
273 : StructHeapTupleHeaderData,
274 : v_tuplep,
275 : FIELDNO_HEAPTUPLEHEADERDATA_HOFF,
276 : ""),
277 : LLVMInt32TypeInContext(lc), "t_hoff");
278 :
279 6776 : v_tupdata_base = l_gep(b,
280 : LLVMInt8TypeInContext(lc),
281 : LLVMBuildBitCast(b,
282 : v_tuplep,
283 : l_ptr(LLVMInt8TypeInContext(lc)),
284 : ""),
285 : &v_hoff, 1,
286 : "v_tupdata_base");
287 :
288 : /*
289 : * Load tuple start offset from slot. Will be reset below in case there's
290 : * no existing deformed columns in slot.
291 : */
292 : {
293 : LLVMValueRef v_off_start;
294 :
295 6776 : v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off");
296 6776 : v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, "");
297 6776 : LLVMBuildStore(b, v_off_start, v_offp);
298 : }
299 :
300 : /* build the basic block for each attribute, need them as jump target */
301 23466 : for (attnum = 0; attnum < natts; attnum++)
302 : {
303 33380 : attcheckattnoblocks[attnum] =
304 16690 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum);
305 33380 : attstartblocks[attnum] =
306 16690 : l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum);
307 33380 : attisnullblocks[attnum] =
308 16690 : l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum);
309 33380 : attcheckalignblocks[attnum] =
310 16690 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum);
311 33380 : attalignblocks[attnum] =
312 16690 : l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum);
313 16690 : attstoreblocks[attnum] =
314 16690 : l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum);
315 : }
316 :
317 : /*
318 : * Check if it is guaranteed that all the desired attributes are available
319 : * in the tuple (but still possibly NULL), by dint of either the last
320 : * to-be-deformed column being NOT NULL, or subsequent ones not accessed
321 : * here being NOT NULL. If that's not guaranteed the tuple headers natt's
322 : * has to be checked, and missing attributes potentially have to be
323 : * fetched (using slot_getmissingattrs().
324 : */
325 6776 : if ((natts - 1) <= guaranteed_column_number)
326 : {
327 : /* just skip through unnecessary blocks */
328 468 : LLVMBuildBr(b, b_adjust_unavail_cols);
329 468 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
330 468 : LLVMBuildBr(b, b_find_start);
331 : }
332 : else
333 : {
334 : LLVMValueRef v_params[3];
335 : LLVMValueRef f;
336 :
337 : /* branch if not all columns available */
338 6308 : LLVMBuildCondBr(b,
339 : LLVMBuildICmp(b, LLVMIntULT,
340 : v_maxatt,
341 : l_int16_const(lc, natts),
342 : ""),
343 : b_adjust_unavail_cols,
344 : b_find_start);
345 :
346 : /* if not, memset tts_isnull of relevant cols to true */
347 6308 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
348 :
349 6308 : v_params[0] = v_slot;
350 6308 : v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), "");
351 6308 : v_params[2] = l_int32_const(lc, natts);
352 6308 : f = llvm_pg_func(mod, "slot_getmissingattrs");
353 6308 : l_call(b,
354 : LLVMGetFunctionType(f), f,
355 : v_params, lengthof(v_params), "");
356 6308 : LLVMBuildBr(b, b_find_start);
357 : }
358 :
359 6776 : LLVMPositionBuilderAtEnd(b, b_find_start);
360 :
361 6776 : v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, "");
362 :
363 : /*
364 : * Build switch to go from nvalid to the right startblock. Callers
365 : * currently don't have the knowledge, but it'd be good for performance to
366 : * avoid this check when it's known that the slot is empty (e.g. in scan
367 : * nodes).
368 : */
369 : if (true)
370 : {
371 6776 : LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid,
372 : b_dead, natts);
373 :
374 23466 : for (attnum = 0; attnum < natts; attnum++)
375 : {
376 16690 : LLVMValueRef v_attno = l_int16_const(lc, attnum);
377 :
378 16690 : LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]);
379 : }
380 : }
381 : else
382 : {
383 : /* jump from entry block to first block */
384 : LLVMBuildBr(b, attcheckattnoblocks[0]);
385 : }
386 :
387 6776 : LLVMPositionBuilderAtEnd(b, b_dead);
388 6776 : LLVMBuildUnreachable(b);
389 :
390 : /*
391 : * Iterate over each attribute that needs to be deformed, build code to
392 : * deform it.
393 : */
394 23466 : for (attnum = 0; attnum < natts; attnum++)
395 : {
396 16690 : Form_pg_attribute att = TupleDescAttr(desc, attnum);
397 : LLVMValueRef v_incby;
398 : int alignto;
399 16690 : LLVMValueRef l_attno = l_int16_const(lc, attnum);
400 : LLVMValueRef v_attdatap;
401 : LLVMValueRef v_resultp;
402 :
403 : /* build block checking whether we did all the necessary attributes */
404 16690 : LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]);
405 :
406 : /*
407 : * If this is the first attribute, slot->tts_nvalid was 0. Therefore
408 : * also reset offset to 0, it may be from a previous execution.
409 : */
410 16690 : if (attnum == 0)
411 : {
412 6776 : LLVMBuildStore(b, l_sizet_const(0), v_offp);
413 : }
414 :
415 : /*
416 : * Build check whether column is available (i.e. whether the tuple has
417 : * that many columns stored). We can avoid the branch if we know
418 : * there's a subsequent NOT NULL column.
419 : */
420 16690 : if (attnum <= guaranteed_column_number)
421 : {
422 1964 : LLVMBuildBr(b, attstartblocks[attnum]);
423 : }
424 : else
425 : {
426 : LLVMValueRef v_islast;
427 :
428 14726 : v_islast = LLVMBuildICmp(b, LLVMIntUGE,
429 : l_attno,
430 : v_maxatt,
431 : "heap_natts");
432 14726 : LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]);
433 : }
434 16690 : LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]);
435 :
436 : /*
437 : * Check for nulls if necessary. No need to take missing attributes
438 : * into account, because if they're present the heaptuple's natts
439 : * would have indicated that a slot_getmissingattrs() is needed.
440 : */
441 16690 : if (!att->attnotnull)
442 : {
443 : LLVMBasicBlockRef b_ifnotnull;
444 : LLVMBasicBlockRef b_ifnull;
445 : LLVMBasicBlockRef b_next;
446 : LLVMValueRef v_attisnull;
447 : LLVMValueRef v_nullbyteno;
448 : LLVMValueRef v_nullbytemask;
449 : LLVMValueRef v_nullbyte;
450 : LLVMValueRef v_nullbit;
451 :
452 14762 : b_ifnotnull = attcheckalignblocks[attnum];
453 14762 : b_ifnull = attisnullblocks[attnum];
454 :
455 14762 : if (attnum + 1 == natts)
456 6308 : b_next = b_out;
457 : else
458 8454 : b_next = attcheckattnoblocks[attnum + 1];
459 :
460 14762 : v_nullbyteno = l_int32_const(lc, attnum >> 3);
461 14762 : v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07));
462 14762 : v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte");
463 :
464 14762 : v_nullbit = LLVMBuildICmp(b,
465 : LLVMIntEQ,
466 : LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""),
467 : l_int8_const(lc, 0),
468 : "attisnull");
469 :
470 14762 : v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, "");
471 :
472 14762 : LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull);
473 :
474 14762 : LLVMPositionBuilderAtEnd(b, b_ifnull);
475 :
476 : /* store null-byte */
477 14762 : LLVMBuildStore(b,
478 : l_int8_const(lc, 1),
479 : l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, ""));
480 : /* store zero datum */
481 14762 : LLVMBuildStore(b,
482 : l_sizet_const(0),
483 : l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, ""));
484 :
485 14762 : LLVMBuildBr(b, b_next);
486 14762 : attguaranteedalign = false;
487 : }
488 : else
489 : {
490 : /* nothing to do */
491 1928 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
492 1928 : LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]);
493 1928 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
494 : }
495 16690 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
496 :
497 : /* determine required alignment */
498 16690 : if (att->attalign == TYPALIGN_INT)
499 13202 : alignto = ALIGNOF_INT;
500 3488 : else if (att->attalign == TYPALIGN_CHAR)
501 1384 : alignto = 1;
502 2104 : else if (att->attalign == TYPALIGN_DOUBLE)
503 1632 : alignto = ALIGNOF_DOUBLE;
504 472 : else if (att->attalign == TYPALIGN_SHORT)
505 472 : alignto = ALIGNOF_SHORT;
506 : else
507 : {
508 0 : elog(ERROR, "unknown alignment");
509 : alignto = 0;
510 : }
511 :
512 : /* ------
513 : * Even if alignment is required, we can skip doing it if provably
514 : * unnecessary:
515 : * - first column is guaranteed to be aligned
516 : * - columns following a NOT NULL fixed width datum have known
517 : * alignment, can skip alignment computation if that known alignment
518 : * is compatible with current column.
519 : * ------
520 : */
521 16690 : if (alignto > 1 &&
522 7262 : (known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment)))
523 : {
524 : /*
525 : * When accessing a varlena field, we have to "peek" to see if we
526 : * are looking at a pad byte or the first byte of a 1-byte-header
527 : * datum. A zero byte must be either a pad byte, or the first
528 : * byte of a correctly aligned 4-byte length word; in either case,
529 : * we can align safely. A non-zero byte must be either a 1-byte
530 : * length word, or the first byte of a correctly aligned 4-byte
531 : * length word; in either case, we need not align.
532 : */
533 8062 : if (att->attlen == -1)
534 : {
535 : LLVMValueRef v_possible_padbyte;
536 : LLVMValueRef v_ispad;
537 : LLVMValueRef v_off;
538 :
539 : /* don't know if short varlena or not */
540 1138 : attguaranteedalign = false;
541 :
542 1138 : v_off = l_load(b, TypeSizeT, v_offp, "");
543 :
544 : v_possible_padbyte =
545 1138 : l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, v_off, "padbyte");
546 : v_ispad =
547 1138 : LLVMBuildICmp(b, LLVMIntEQ,
548 : v_possible_padbyte, l_int8_const(lc, 0),
549 : "ispadbyte");
550 1138 : LLVMBuildCondBr(b, v_ispad,
551 1138 : attalignblocks[attnum],
552 1138 : attstoreblocks[attnum]);
553 : }
554 : else
555 : {
556 6924 : LLVMBuildBr(b, attalignblocks[attnum]);
557 : }
558 :
559 8062 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
560 :
561 : /* translation of alignment code (cf TYPEALIGN()) */
562 : {
563 : LLVMValueRef v_off_aligned;
564 8062 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
565 :
566 : /* ((ALIGNVAL) - 1) */
567 8062 : LLVMValueRef v_alignval = l_sizet_const(alignto - 1);
568 :
569 : /* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */
570 8062 : LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, "");
571 :
572 : /* ~((uintptr_t) ((ALIGNVAL) - 1)) */
573 8062 : LLVMValueRef v_rh = l_sizet_const(~(alignto - 1));
574 :
575 8062 : v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset");
576 :
577 8062 : LLVMBuildStore(b, v_off_aligned, v_offp);
578 : }
579 :
580 : /*
581 : * As alignment either was unnecessary or has been performed, we
582 : * now know the current alignment. This is only safe because this
583 : * value isn't used for varlena and nullable columns.
584 : */
585 8062 : if (known_alignment >= 0)
586 : {
587 : Assert(known_alignment != 0);
588 18 : known_alignment = TYPEALIGN(alignto, known_alignment);
589 : }
590 :
591 8062 : LLVMBuildBr(b, attstoreblocks[attnum]);
592 8062 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
593 : }
594 : else
595 : {
596 8628 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
597 8628 : LLVMBuildBr(b, attalignblocks[attnum]);
598 8628 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
599 8628 : LLVMBuildBr(b, attstoreblocks[attnum]);
600 : }
601 16690 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
602 :
603 : /*
604 : * Store the current offset if known to be constant. That allows LLVM
605 : * to generate better code. Without that LLVM can't figure out that
606 : * the offset might be constant due to the jumps for previously
607 : * decoded columns.
608 : */
609 16690 : if (attguaranteedalign)
610 : {
611 : Assert(known_alignment >= 0);
612 1892 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
613 : }
614 :
615 : /* compute what following columns are aligned to */
616 16690 : if (att->attlen < 0)
617 : {
618 : /* can't guarantee any alignment after variable length field */
619 1928 : known_alignment = -1;
620 1928 : attguaranteedalign = false;
621 : }
622 14762 : else if (att->attnotnull && attguaranteedalign && known_alignment >= 0)
623 : {
624 : /*
625 : * If the offset to the column was previously known, a NOT NULL &
626 : * fixed-width column guarantees that alignment is just the
627 : * previous alignment plus column width.
628 : */
629 : Assert(att->attlen > 0);
630 1874 : known_alignment += att->attlen;
631 : }
632 12888 : else if (att->attnotnull && (att->attlen % alignto) == 0)
633 : {
634 : /*
635 : * After a NOT NULL fixed-width column with a length that is a
636 : * multiple of its alignment requirement, we know the following
637 : * column is aligned to at least the current column's alignment.
638 : */
639 : Assert(att->attlen > 0);
640 0 : known_alignment = alignto;
641 : Assert(known_alignment > 0);
642 0 : attguaranteedalign = false;
643 : }
644 : else
645 : {
646 12888 : known_alignment = -1;
647 12888 : attguaranteedalign = false;
648 : }
649 :
650 :
651 : /* compute address to load data from */
652 : {
653 16690 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
654 :
655 16690 : v_attdatap =
656 16690 : l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, "");
657 : }
658 :
659 : /* compute address to store value at */
660 16690 : v_resultp = l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, "");
661 :
662 : /* store null-byte (false) */
663 16690 : LLVMBuildStore(b, l_int8_const(lc, 0),
664 : l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, ""));
665 :
666 : /*
667 : * Store datum. For byval: datums copy the value, extend to Datum's
668 : * width, and store. For byref types: store pointer to data.
669 : */
670 16690 : if (att->attbyval)
671 : {
672 : LLVMValueRef v_tmp_loaddata;
673 13580 : LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8);
674 13580 : LLVMTypeRef vartypep = LLVMPointerType(vartype, 0);
675 :
676 : v_tmp_loaddata =
677 13580 : LLVMBuildPointerCast(b, v_attdatap, vartypep, "");
678 13580 : v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval");
679 13580 : v_tmp_loaddata = LLVMBuildZExt(b, v_tmp_loaddata, TypeSizeT, "");
680 :
681 13580 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
682 : }
683 : else
684 : {
685 : LLVMValueRef v_tmp_loaddata;
686 :
687 : /* store pointer */
688 : v_tmp_loaddata =
689 3110 : LLVMBuildPtrToInt(b,
690 : v_attdatap,
691 : TypeSizeT,
692 : "attr_ptr");
693 3110 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
694 : }
695 :
696 : /* increment data pointer */
697 16690 : if (att->attlen > 0)
698 : {
699 14762 : v_incby = l_sizet_const(att->attlen);
700 : }
701 1928 : else if (att->attlen == -1)
702 : {
703 1928 : v_incby = l_call(b,
704 : llvm_pg_var_func_type("varsize_any"),
705 : llvm_pg_func(mod, "varsize_any"),
706 : &v_attdatap, 1,
707 : "varsize_any");
708 1928 : l_callsite_ro(v_incby);
709 1928 : l_callsite_alwaysinline(v_incby);
710 : }
711 0 : else if (att->attlen == -2)
712 : {
713 0 : v_incby = l_call(b,
714 : llvm_pg_var_func_type("strlen"),
715 : llvm_pg_func(mod, "strlen"),
716 : &v_attdatap, 1, "strlen");
717 :
718 0 : l_callsite_ro(v_incby);
719 :
720 : /* add 1 for NUL byte */
721 0 : v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), "");
722 : }
723 : else
724 : {
725 : Assert(false);
726 0 : v_incby = NULL; /* silence compiler */
727 : }
728 :
729 16690 : if (attguaranteedalign)
730 : {
731 : Assert(known_alignment >= 0);
732 1874 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
733 : }
734 : else
735 : {
736 14816 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
737 :
738 14816 : v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset");
739 14816 : LLVMBuildStore(b, v_off, v_offp);
740 : }
741 :
742 : /*
743 : * jump to next block, unless last possible column, or all desired
744 : * (available) attributes have been fetched.
745 : */
746 16690 : if (attnum + 1 == natts)
747 : {
748 : /* jump out */
749 6776 : LLVMBuildBr(b, b_out);
750 : }
751 : else
752 : {
753 9914 : LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]);
754 : }
755 : }
756 :
757 :
758 : /* build block that returns */
759 6776 : LLVMPositionBuilderAtEnd(b, b_out);
760 :
761 : {
762 6776 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
763 : LLVMValueRef v_flags;
764 :
765 6776 : LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp);
766 6776 : v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), "");
767 6776 : LLVMBuildStore(b, v_off, v_slotoffp);
768 6776 : v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags");
769 6776 : v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), "");
770 6776 : LLVMBuildStore(b, v_flags, v_flagsp);
771 6776 : LLVMBuildRetVoid(b);
772 : }
773 :
774 6776 : LLVMDisposeBuilder(b);
775 :
776 6776 : return v_deform_fn;
777 : }
|
