Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_deform.c
4 : * Generate code for deforming a heap tuple.
5 : *
6 : * This gains performance benefits over unJITed deforming from compile-time
7 : * knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc
8 : * can be taken advantage of.
9 : *
10 : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
11 : * Portions Copyright (c) 1994, Regents of the University of California
12 : *
13 : * IDENTIFICATION
14 : * src/backend/jit/llvm/llvmjit_deform.c
15 : *
16 : *-------------------------------------------------------------------------
17 : */
18 :
19 : #include "postgres.h"
20 :
21 : #include <llvm-c/Core.h>
22 :
23 : #include "access/htup_details.h"
24 : #include "access/tupdesc_details.h"
25 : #include "executor/tuptable.h"
26 : #include "jit/llvmjit.h"
27 : #include "jit/llvmjit_emit.h"
28 :
29 :
30 : /*
31 : * Create a function that deforms a tuple of type desc up to natts columns.
32 : */
33 : LLVMValueRef
34 2209 : slot_compile_deform(LLVMJitContext *context, TupleDesc desc,
35 : const TupleTableSlotOps *ops, int natts)
36 : {
37 : char *funcname;
38 :
39 : LLVMModuleRef mod;
40 : LLVMContextRef lc;
41 : LLVMBuilderRef b;
42 :
43 : LLVMTypeRef deform_sig;
44 : LLVMValueRef v_deform_fn;
45 :
46 : LLVMBasicBlockRef b_entry;
47 : LLVMBasicBlockRef b_adjust_unavail_cols;
48 : LLVMBasicBlockRef b_find_start;
49 :
50 : LLVMBasicBlockRef b_out;
51 : LLVMBasicBlockRef b_dead;
52 : LLVMBasicBlockRef *attcheckattnoblocks;
53 : LLVMBasicBlockRef *attstartblocks;
54 : LLVMBasicBlockRef *attisnullblocks;
55 : LLVMBasicBlockRef *attcheckalignblocks;
56 : LLVMBasicBlockRef *attalignblocks;
57 : LLVMBasicBlockRef *attstoreblocks;
58 :
59 : LLVMValueRef v_offp;
60 :
61 : LLVMValueRef v_tupdata_base;
62 : LLVMValueRef v_tts_values;
63 : LLVMValueRef v_tts_nulls;
64 : LLVMValueRef v_slotoffp;
65 : LLVMValueRef v_flagsp;
66 : LLVMValueRef v_nvalidp;
67 : LLVMValueRef v_nvalid;
68 : LLVMValueRef v_maxatt;
69 :
70 : LLVMValueRef v_slot;
71 :
72 : LLVMValueRef v_tupleheaderp;
73 : LLVMValueRef v_tuplep;
74 : LLVMValueRef v_infomask1;
75 : LLVMValueRef v_infomask2;
76 : LLVMValueRef v_bits;
77 :
78 : LLVMValueRef v_hoff;
79 :
80 : LLVMValueRef v_hasnulls;
81 :
82 : /* last column (0 indexed) guaranteed to exist */
83 2209 : int guaranteed_column_number = -1;
84 :
85 : /* current known alignment */
86 2209 : int known_alignment = 0;
87 :
88 : /* if true, known_alignment describes definite offset of column */
89 2209 : bool attguaranteedalign = true;
90 :
91 : int attnum;
92 :
93 : /* virtual tuples never need deforming, so don't generate code */
94 2209 : if (ops == &TTSOpsVirtual)
95 0 : return NULL;
96 :
97 : /* decline to JIT for slot types we don't know to handle */
98 2209 : if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple &&
99 : ops != &TTSOpsMinimalTuple)
100 0 : return NULL;
101 :
102 2209 : mod = llvm_mutable_module(context);
103 2209 : lc = LLVMGetModuleContext(mod);
104 :
105 2209 : funcname = llvm_expand_funcname(context, "deform");
106 :
107 : /*
108 : * Check which columns have to exist, so we don't have to check the row's
109 : * natts unnecessarily.
110 : */
111 13102 : for (attnum = 0; attnum < desc->natts; attnum++)
112 : {
113 10893 : CompactAttribute *att = TupleDescCompactAttr(desc, attnum);
114 :
115 : /*
116 : * If the column is declared NOT NULL then it must be present in every
117 : * tuple, unless there's a "missing" entry that could provide a
118 : * non-NULL value for it. That in turn guarantees that the NULL bitmap
119 : * - if there are any NULLable columns - is at least long enough to
120 : * cover columns up to attnum.
121 : *
122 : * Be paranoid and also check !attisdropped, even though the
123 : * combination of attisdropped && attnotnull combination shouldn't
124 : * exist.
125 : */
126 10893 : if (att->attnullability == ATTNULLABLE_VALID &&
127 3053 : !att->atthasmissing &&
128 3053 : !att->attisdropped)
129 3053 : guaranteed_column_number = attnum;
130 : }
131 :
132 : /* Create the signature and function */
133 : {
134 : LLVMTypeRef param_types[1];
135 :
136 2209 : param_types[0] = l_ptr(StructTupleTableSlot);
137 :
138 2209 : deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc),
139 : param_types, lengthof(param_types), 0);
140 : }
141 2209 : v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig);
142 2209 : LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage);
143 2209 : LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF);
144 2209 : llvm_copy_attributes(AttributeTemplate, v_deform_fn);
145 :
146 : b_entry =
147 2209 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry");
148 : b_adjust_unavail_cols =
149 2209 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols");
150 : b_find_start =
151 2209 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock");
152 : b_out =
153 2209 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock");
154 : b_dead =
155 2209 : LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock");
156 :
157 2209 : b = LLVMCreateBuilderInContext(lc);
158 :
159 2209 : attcheckattnoblocks = palloc_array(LLVMBasicBlockRef, natts);
160 2209 : attstartblocks = palloc_array(LLVMBasicBlockRef, natts);
161 2209 : attisnullblocks = palloc_array(LLVMBasicBlockRef, natts);
162 2209 : attcheckalignblocks = palloc_array(LLVMBasicBlockRef, natts);
163 2209 : attalignblocks = palloc_array(LLVMBasicBlockRef, natts);
164 2209 : attstoreblocks = palloc_array(LLVMBasicBlockRef, natts);
165 :
166 2209 : known_alignment = 0;
167 :
168 2209 : LLVMPositionBuilderAtEnd(b, b_entry);
169 :
170 : /* perform allocas first, llvm only converts those to registers */
171 2209 : v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp");
172 :
173 2209 : v_slot = LLVMGetParam(v_deform_fn, 0);
174 :
175 : v_tts_values =
176 2209 : l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES,
177 : "tts_values");
178 : v_tts_nulls =
179 2209 : l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL,
180 : "tts_ISNULL");
181 2209 : v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, "");
182 2209 : v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, "");
183 :
184 2209 : if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple)
185 1032 : {
186 : LLVMValueRef v_heapslot;
187 :
188 : v_heapslot =
189 1032 : LLVMBuildBitCast(b,
190 : v_slot,
191 : l_ptr(StructHeapTupleTableSlot),
192 : "heapslot");
193 1032 : v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, "");
194 : v_tupleheaderp =
195 1032 : l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE,
196 : "tupleheader");
197 : }
198 1177 : else if (ops == &TTSOpsMinimalTuple)
199 : {
200 : LLVMValueRef v_minimalslot;
201 :
202 : v_minimalslot =
203 1177 : LLVMBuildBitCast(b,
204 : v_slot,
205 : l_ptr(StructMinimalTupleTableSlot),
206 : "minimalslot");
207 1177 : v_slotoffp = l_struct_gep(b,
208 : StructMinimalTupleTableSlot,
209 : v_minimalslot,
210 : FIELDNO_MINIMALTUPLETABLESLOT_OFF, "");
211 : v_tupleheaderp =
212 1177 : l_load_struct_gep(b,
213 : StructMinimalTupleTableSlot,
214 : v_minimalslot,
215 : FIELDNO_MINIMALTUPLETABLESLOT_TUPLE,
216 : "tupleheader");
217 : }
218 : else
219 : {
220 : /* should've returned at the start of the function */
221 0 : pg_unreachable();
222 : }
223 :
224 : v_tuplep =
225 2209 : l_load_struct_gep(b,
226 : StructHeapTupleData,
227 : v_tupleheaderp,
228 : FIELDNO_HEAPTUPLEDATA_DATA,
229 : "tuple");
230 : v_bits =
231 2209 : LLVMBuildBitCast(b,
232 : l_struct_gep(b,
233 : StructHeapTupleHeaderData,
234 : v_tuplep,
235 : FIELDNO_HEAPTUPLEHEADERDATA_BITS,
236 : ""),
237 : l_ptr(LLVMInt8TypeInContext(lc)),
238 : "t_bits");
239 : v_infomask1 =
240 2209 : l_load_struct_gep(b,
241 : StructHeapTupleHeaderData,
242 : v_tuplep,
243 : FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK,
244 : "infomask1");
245 : v_infomask2 =
246 2209 : l_load_struct_gep(b,
247 : StructHeapTupleHeaderData,
248 : v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2,
249 : "infomask2");
250 :
251 : /* t_infomask & HEAP_HASNULL */
252 : v_hasnulls =
253 2209 : LLVMBuildICmp(b, LLVMIntNE,
254 : LLVMBuildAnd(b,
255 : l_int16_const(lc, HEAP_HASNULL),
256 : v_infomask1, ""),
257 : l_int16_const(lc, 0),
258 : "hasnulls");
259 :
260 : /* t_infomask2 & HEAP_NATTS_MASK */
261 2209 : v_maxatt = LLVMBuildAnd(b,
262 : l_int16_const(lc, HEAP_NATTS_MASK),
263 : v_infomask2,
264 : "maxatt");
265 :
266 : /*
267 : * Need to zext, as getelementptr otherwise treats hoff as a signed 8bit
268 : * integer, which'd yield a negative offset for t_hoff > 127.
269 : */
270 2209 : v_hoff =
271 2209 : LLVMBuildZExt(b,
272 : l_load_struct_gep(b,
273 : StructHeapTupleHeaderData,
274 : v_tuplep,
275 : FIELDNO_HEAPTUPLEHEADERDATA_HOFF,
276 : ""),
277 : LLVMInt32TypeInContext(lc), "t_hoff");
278 :
279 2209 : v_tupdata_base = l_gep(b,
280 : LLVMInt8TypeInContext(lc),
281 : LLVMBuildBitCast(b,
282 : v_tuplep,
283 : l_ptr(LLVMInt8TypeInContext(lc)),
284 : ""),
285 : &v_hoff, 1,
286 : "v_tupdata_base");
287 :
288 : /*
289 : * Load tuple start offset from slot. Will be reset below in case there's
290 : * no existing deformed columns in slot.
291 : */
292 : {
293 : LLVMValueRef v_off_start;
294 :
295 2209 : v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off");
296 2209 : v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, "");
297 2209 : LLVMBuildStore(b, v_off_start, v_offp);
298 : }
299 :
300 : /* build the basic block for each attribute, need them as jump target */
301 7892 : for (attnum = 0; attnum < natts; attnum++)
302 : {
303 11366 : attcheckattnoblocks[attnum] =
304 5683 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum);
305 11366 : attstartblocks[attnum] =
306 5683 : l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum);
307 11366 : attisnullblocks[attnum] =
308 5683 : l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum);
309 11366 : attcheckalignblocks[attnum] =
310 5683 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum);
311 11366 : attalignblocks[attnum] =
312 5683 : l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum);
313 5683 : attstoreblocks[attnum] =
314 5683 : l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum);
315 : }
316 :
317 : /*
318 : * Check if it is guaranteed that all the desired attributes are available
319 : * in the tuple (but still possibly NULL), by dint of either the last
320 : * to-be-deformed column being NOT NULL, or subsequent ones not accessed
321 : * here being NOT NULL. If that's not guaranteed the tuple headers natt's
322 : * has to be checked, and missing attributes potentially have to be
323 : * fetched (using slot_getmissingattrs().
324 : */
325 2209 : if ((natts - 1) <= guaranteed_column_number)
326 : {
327 : /* just skip through unnecessary blocks */
328 210 : LLVMBuildBr(b, b_adjust_unavail_cols);
329 210 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
330 210 : LLVMBuildBr(b, b_find_start);
331 : }
332 : else
333 : {
334 : LLVMValueRef v_params[3];
335 : LLVMValueRef f;
336 :
337 : /* branch if not all columns available */
338 1999 : LLVMBuildCondBr(b,
339 : LLVMBuildICmp(b, LLVMIntULT,
340 : v_maxatt,
341 : l_int16_const(lc, natts),
342 : ""),
343 : b_adjust_unavail_cols,
344 : b_find_start);
345 :
346 : /* if not, memset tts_isnull of relevant cols to true */
347 1999 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
348 :
349 1999 : v_params[0] = v_slot;
350 1999 : v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), "");
351 1999 : v_params[2] = l_int32_const(lc, natts);
352 1999 : f = llvm_pg_func(mod, "slot_getmissingattrs");
353 1999 : l_call(b,
354 : LLVMGetFunctionType(f), f,
355 : v_params, lengthof(v_params), "");
356 1999 : LLVMBuildBr(b, b_find_start);
357 : }
358 :
359 2209 : LLVMPositionBuilderAtEnd(b, b_find_start);
360 :
361 2209 : v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, "");
362 :
363 : /*
364 : * Build switch to go from nvalid to the right startblock. Callers
365 : * currently don't have the knowledge, but it'd be good for performance to
366 : * avoid this check when it's known that the slot is empty (e.g. in scan
367 : * nodes).
368 : */
369 : if (true)
370 : {
371 2209 : LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid,
372 : b_dead, natts);
373 :
374 7892 : for (attnum = 0; attnum < natts; attnum++)
375 : {
376 5683 : LLVMValueRef v_attno = l_int16_const(lc, attnum);
377 :
378 5683 : LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]);
379 : }
380 : }
381 : else
382 : {
383 : /* jump from entry block to first block */
384 : LLVMBuildBr(b, attcheckattnoblocks[0]);
385 : }
386 :
387 2209 : LLVMPositionBuilderAtEnd(b, b_dead);
388 2209 : LLVMBuildUnreachable(b);
389 :
390 : /*
391 : * Iterate over each attribute that needs to be deformed, build code to
392 : * deform it.
393 : */
394 7892 : for (attnum = 0; attnum < natts; attnum++)
395 : {
396 5683 : CompactAttribute *att = TupleDescCompactAttr(desc, attnum);
397 : LLVMValueRef v_incby;
398 5683 : int alignto = att->attalignby;
399 5683 : LLVMValueRef l_attno = l_int16_const(lc, attnum);
400 : LLVMValueRef v_attdatap;
401 : LLVMValueRef v_resultp;
402 :
403 : /* build block checking whether we did all the necessary attributes */
404 5683 : LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]);
405 :
406 : /*
407 : * If this is the first attribute, slot->tts_nvalid was 0. Therefore
408 : * also reset offset to 0, it may be from a previous execution.
409 : */
410 5683 : if (attnum == 0)
411 : {
412 2209 : LLVMBuildStore(b, l_sizet_const(0), v_offp);
413 : }
414 :
415 : /*
416 : * Build check whether column is available (i.e. whether the tuple has
417 : * that many columns stored). We can avoid the branch if we know
418 : * there's a subsequent NOT NULL column.
419 : */
420 5683 : if (attnum <= guaranteed_column_number)
421 : {
422 812 : LLVMBuildBr(b, attstartblocks[attnum]);
423 : }
424 : else
425 : {
426 : LLVMValueRef v_islast;
427 :
428 4871 : v_islast = LLVMBuildICmp(b, LLVMIntUGE,
429 : l_attno,
430 : v_maxatt,
431 : "heap_natts");
432 4871 : LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]);
433 : }
434 5683 : LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]);
435 :
436 : /*
437 : * Check for nulls if necessary. No need to take missing attributes
438 : * into account, because if they're present the heaptuple's natts
439 : * would have indicated that a slot_getmissingattrs() is needed.
440 : */
441 5683 : if (att->attnullability != ATTNULLABLE_VALID)
442 : {
443 : LLVMBasicBlockRef b_ifnotnull;
444 : LLVMBasicBlockRef b_ifnull;
445 : LLVMBasicBlockRef b_next;
446 : LLVMValueRef v_attisnull;
447 : LLVMValueRef v_nullbyteno;
448 : LLVMValueRef v_nullbytemask;
449 : LLVMValueRef v_nullbyte;
450 : LLVMValueRef v_nullbit;
451 :
452 4877 : b_ifnotnull = attcheckalignblocks[attnum];
453 4877 : b_ifnull = attisnullblocks[attnum];
454 :
455 4877 : if (attnum + 1 == natts)
456 1999 : b_next = b_out;
457 : else
458 2878 : b_next = attcheckattnoblocks[attnum + 1];
459 :
460 4877 : v_nullbyteno = l_int32_const(lc, attnum >> 3);
461 4877 : v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07));
462 4877 : v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte");
463 :
464 4877 : v_nullbit = LLVMBuildICmp(b,
465 : LLVMIntEQ,
466 : LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""),
467 : l_int8_const(lc, 0),
468 : "attisnull");
469 :
470 4877 : v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, "");
471 :
472 4877 : LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull);
473 :
474 4877 : LLVMPositionBuilderAtEnd(b, b_ifnull);
475 :
476 : /* store null-byte */
477 4877 : LLVMBuildStore(b,
478 : l_int8_const(lc, 1),
479 : l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, ""));
480 : /* store zero datum */
481 4877 : LLVMBuildStore(b,
482 : l_datum_const(0),
483 : l_gep(b, TypeDatum, v_tts_values, &l_attno, 1, ""));
484 :
485 4877 : LLVMBuildBr(b, b_next);
486 4877 : attguaranteedalign = false;
487 : }
488 : else
489 : {
490 : /* nothing to do */
491 806 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
492 806 : LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]);
493 806 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
494 : }
495 5683 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
496 :
497 : /* ------
498 : * Even if alignment is required, we can skip doing it if provably
499 : * unnecessary:
500 : * - first column is guaranteed to be aligned
501 : * - columns following a NOT NULL fixed width datum have known
502 : * alignment, can skip alignment computation if that known alignment
503 : * is compatible with current column.
504 : * ------
505 : */
506 5683 : if (alignto > 1 &&
507 2372 : (known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment)))
508 : {
509 : /*
510 : * When accessing a varlena field, we have to "peek" to see if we
511 : * are looking at a pad byte or the first byte of a 1-byte-header
512 : * datum. A zero byte must be either a pad byte, or the first
513 : * byte of a correctly aligned 4-byte length word; in either case,
514 : * we can align safely. A non-zero byte must be either a 1-byte
515 : * length word, or the first byte of a correctly aligned 4-byte
516 : * length word; in either case, we need not align.
517 : */
518 2666 : if (att->attlen == -1)
519 : {
520 : LLVMValueRef v_possible_padbyte;
521 : LLVMValueRef v_ispad;
522 : LLVMValueRef v_off;
523 :
524 : /* don't know if short varlena or not */
525 340 : attguaranteedalign = false;
526 :
527 340 : v_off = l_load(b, TypeSizeT, v_offp, "");
528 :
529 : v_possible_padbyte =
530 340 : l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, v_off, "padbyte");
531 : v_ispad =
532 340 : LLVMBuildICmp(b, LLVMIntEQ,
533 : v_possible_padbyte, l_int8_const(lc, 0),
534 : "ispadbyte");
535 340 : LLVMBuildCondBr(b, v_ispad,
536 340 : attalignblocks[attnum],
537 340 : attstoreblocks[attnum]);
538 : }
539 : else
540 : {
541 2326 : LLVMBuildBr(b, attalignblocks[attnum]);
542 : }
543 :
544 2666 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
545 :
546 : /* translation of alignment code (cf TYPEALIGN()) */
547 : {
548 : LLVMValueRef v_off_aligned;
549 2666 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
550 :
551 : /* ((ALIGNVAL) - 1) */
552 2666 : LLVMValueRef v_alignval = l_sizet_const(alignto - 1);
553 :
554 : /* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */
555 2666 : LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, "");
556 :
557 : /* ~((uintptr_t) ((ALIGNVAL) - 1)) */
558 2666 : LLVMValueRef v_rh = l_sizet_const(~(alignto - 1));
559 :
560 2666 : v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset");
561 :
562 2666 : LLVMBuildStore(b, v_off_aligned, v_offp);
563 : }
564 :
565 : /*
566 : * As alignment either was unnecessary or has been performed, we
567 : * now know the current alignment. This is only safe because this
568 : * value isn't used for varlena and nullable columns.
569 : */
570 2666 : if (known_alignment >= 0)
571 : {
572 : Assert(known_alignment != 0);
573 6 : known_alignment = TYPEALIGN(alignto, known_alignment);
574 : }
575 :
576 2666 : LLVMBuildBr(b, attstoreblocks[attnum]);
577 2666 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
578 : }
579 : else
580 : {
581 3017 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
582 3017 : LLVMBuildBr(b, attalignblocks[attnum]);
583 3017 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
584 3017 : LLVMBuildBr(b, attstoreblocks[attnum]);
585 : }
586 5683 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
587 :
588 : /*
589 : * Store the current offset if known to be constant. That allows LLVM
590 : * to generate better code. Without that LLVM can't figure out that
591 : * the offset might be constant due to the jumps for previously
592 : * decoded columns.
593 : */
594 5683 : if (attguaranteedalign)
595 : {
596 : Assert(known_alignment >= 0);
597 800 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
598 : }
599 :
600 : /* compute what following columns are aligned to */
601 5683 : if (att->attlen < 0)
602 : {
603 : /* can't guarantee any alignment after variable length field */
604 587 : known_alignment = -1;
605 587 : attguaranteedalign = false;
606 : }
607 5096 : else if (att->attnullability == ATTNULLABLE_VALID &&
608 794 : attguaranteedalign && known_alignment >= 0)
609 : {
610 : /*
611 : * If the offset to the column was previously known, a NOT NULL &
612 : * fixed-width column guarantees that alignment is just the
613 : * previous alignment plus column width.
614 : */
615 : Assert(att->attlen > 0);
616 794 : known_alignment += att->attlen;
617 : }
618 4302 : else if (att->attnullability == ATTNULLABLE_VALID &&
619 0 : (att->attlen % alignto) == 0)
620 : {
621 : /*
622 : * After a NOT NULL fixed-width column with a length that is a
623 : * multiple of its alignment requirement, we know the following
624 : * column is aligned to at least the current column's alignment.
625 : */
626 : Assert(att->attlen > 0);
627 0 : known_alignment = alignto;
628 : Assert(known_alignment > 0);
629 0 : attguaranteedalign = false;
630 : }
631 : else
632 : {
633 4302 : known_alignment = -1;
634 4302 : attguaranteedalign = false;
635 : }
636 :
637 :
638 : /* compute address to load data from */
639 : {
640 5683 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
641 :
642 5683 : v_attdatap =
643 5683 : l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, "");
644 : }
645 :
646 : /* compute address to store value at */
647 5683 : v_resultp = l_gep(b, TypeDatum, v_tts_values, &l_attno, 1, "");
648 :
649 : /* store null-byte (false) */
650 5683 : LLVMBuildStore(b, l_int8_const(lc, 0),
651 : l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, ""));
652 :
653 : /*
654 : * Store datum. For byval: datums copy the value, extend to Datum's
655 : * width, and store. For byref types: store pointer to data.
656 : */
657 5683 : if (att->attbyval)
658 : {
659 : LLVMValueRef v_tmp_loaddata;
660 4595 : LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8);
661 4595 : LLVMTypeRef vartypep = LLVMPointerType(vartype, 0);
662 :
663 : v_tmp_loaddata =
664 4595 : LLVMBuildPointerCast(b, v_attdatap, vartypep, "");
665 4595 : v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval");
666 4595 : v_tmp_loaddata = LLVMBuildSExt(b, v_tmp_loaddata, TypeDatum, "");
667 :
668 4595 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
669 : }
670 : else
671 : {
672 : LLVMValueRef v_tmp_loaddata;
673 :
674 : /* store pointer */
675 : v_tmp_loaddata =
676 1088 : LLVMBuildPtrToInt(b,
677 : v_attdatap,
678 : TypeDatum,
679 : "attr_ptr");
680 1088 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
681 : }
682 :
683 : /* increment data pointer */
684 5683 : if (att->attlen > 0)
685 : {
686 5096 : v_incby = l_sizet_const(att->attlen);
687 : }
688 587 : else if (att->attlen == -1)
689 : {
690 587 : v_incby = l_call(b,
691 : llvm_pg_var_func_type("varsize_any"),
692 : llvm_pg_func(mod, "varsize_any"),
693 : &v_attdatap, 1,
694 : "varsize_any");
695 587 : l_callsite_ro(v_incby);
696 587 : l_callsite_alwaysinline(v_incby);
697 : }
698 0 : else if (att->attlen == -2)
699 : {
700 0 : v_incby = l_call(b,
701 : llvm_pg_var_func_type("strlen"),
702 : llvm_pg_func(mod, "strlen"),
703 : &v_attdatap, 1, "strlen");
704 :
705 0 : l_callsite_ro(v_incby);
706 :
707 : /* add 1 for NUL byte */
708 0 : v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), "");
709 : }
710 : else
711 : {
712 : Assert(false);
713 0 : v_incby = NULL; /* silence compiler */
714 : }
715 :
716 5683 : if (attguaranteedalign)
717 : {
718 : Assert(known_alignment >= 0);
719 794 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
720 : }
721 : else
722 : {
723 4889 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
724 :
725 4889 : v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset");
726 4889 : LLVMBuildStore(b, v_off, v_offp);
727 : }
728 :
729 : /*
730 : * jump to next block, unless last possible column, or all desired
731 : * (available) attributes have been fetched.
732 : */
733 5683 : if (attnum + 1 == natts)
734 : {
735 : /* jump out */
736 2209 : LLVMBuildBr(b, b_out);
737 : }
738 : else
739 : {
740 3474 : LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]);
741 : }
742 : }
743 :
744 :
745 : /* build block that returns */
746 2209 : LLVMPositionBuilderAtEnd(b, b_out);
747 :
748 : {
749 2209 : LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
750 : LLVMValueRef v_flags;
751 :
752 2209 : LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp);
753 2209 : v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), "");
754 2209 : LLVMBuildStore(b, v_off, v_slotoffp);
755 2209 : v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags");
756 2209 : v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), "");
757 2209 : LLVMBuildStore(b, v_flags, v_flagsp);
758 2209 : LLVMBuildRetVoid(b);
759 : }
760 :
761 2209 : LLVMDisposeBuilder(b);
762 :
763 2209 : return v_deform_fn;
764 : }
|
