LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit_deform.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 211 222 95.0 %
Date: 2024-05-19 08:10:29 Functions: 1 1 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit_deform.c
       4             :  *    Generate code for deforming a heap tuple.
       5             :  *
       6             :  * This gains performance benefits over unJITed deforming from compile-time
       7             :  * knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc
       8             :  * can be taken advantage of.
       9             :  *
      10             :  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
      11             :  * Portions Copyright (c) 1994, Regents of the University of California
      12             :  *
      13             :  * IDENTIFICATION
      14             :  *    src/backend/jit/llvm/llvmjit_deform.c
      15             :  *
      16             :  *-------------------------------------------------------------------------
      17             :  */
      18             : 
      19             : #include "postgres.h"
      20             : 
      21             : #include <llvm-c/Core.h>
      22             : 
      23             : #include "access/htup_details.h"
      24             : #include "access/tupdesc_details.h"
      25             : #include "executor/tuptable.h"
      26             : #include "jit/llvmjit.h"
      27             : #include "jit/llvmjit_emit.h"
      28             : 
      29             : 
      30             : /*
      31             :  * Create a function that deforms a tuple of type desc up to natts columns.
      32             :  */
      33             : LLVMValueRef
      34        6782 : slot_compile_deform(LLVMJitContext *context, TupleDesc desc,
      35             :                     const TupleTableSlotOps *ops, int natts)
      36             : {
      37             :     char       *funcname;
      38             : 
      39             :     LLVMModuleRef mod;
      40             :     LLVMContextRef lc;
      41             :     LLVMBuilderRef b;
      42             : 
      43             :     LLVMTypeRef deform_sig;
      44             :     LLVMValueRef v_deform_fn;
      45             : 
      46             :     LLVMBasicBlockRef b_entry;
      47             :     LLVMBasicBlockRef b_adjust_unavail_cols;
      48             :     LLVMBasicBlockRef b_find_start;
      49             : 
      50             :     LLVMBasicBlockRef b_out;
      51             :     LLVMBasicBlockRef b_dead;
      52             :     LLVMBasicBlockRef *attcheckattnoblocks;
      53             :     LLVMBasicBlockRef *attstartblocks;
      54             :     LLVMBasicBlockRef *attisnullblocks;
      55             :     LLVMBasicBlockRef *attcheckalignblocks;
      56             :     LLVMBasicBlockRef *attalignblocks;
      57             :     LLVMBasicBlockRef *attstoreblocks;
      58             : 
      59             :     LLVMValueRef v_offp;
      60             : 
      61             :     LLVMValueRef v_tupdata_base;
      62             :     LLVMValueRef v_tts_values;
      63             :     LLVMValueRef v_tts_nulls;
      64             :     LLVMValueRef v_slotoffp;
      65             :     LLVMValueRef v_flagsp;
      66             :     LLVMValueRef v_nvalidp;
      67             :     LLVMValueRef v_nvalid;
      68             :     LLVMValueRef v_maxatt;
      69             : 
      70             :     LLVMValueRef v_slot;
      71             : 
      72             :     LLVMValueRef v_tupleheaderp;
      73             :     LLVMValueRef v_tuplep;
      74             :     LLVMValueRef v_infomask1;
      75             :     LLVMValueRef v_infomask2;
      76             :     LLVMValueRef v_bits;
      77             : 
      78             :     LLVMValueRef v_hoff;
      79             : 
      80             :     LLVMValueRef v_hasnulls;
      81             : 
      82             :     /* last column (0 indexed) guaranteed to exist */
      83        6782 :     int         guaranteed_column_number = -1;
      84             : 
      85             :     /* current known alignment */
      86        6782 :     int         known_alignment = 0;
      87             : 
      88             :     /* if true, known_alignment describes definite offset of column */
      89        6782 :     bool        attguaranteedalign = true;
      90             : 
      91             :     int         attnum;
      92             : 
      93             :     /* virtual tuples never need deforming, so don't generate code */
      94        6782 :     if (ops == &TTSOpsVirtual)
      95           0 :         return NULL;
      96             : 
      97             :     /* decline to JIT for slot types we don't know to handle */
      98        6782 :     if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple &&
      99             :         ops != &TTSOpsMinimalTuple)
     100           0 :         return NULL;
     101             : 
     102        6782 :     mod = llvm_mutable_module(context);
     103        6782 :     lc = LLVMGetModuleContext(mod);
     104             : 
     105        6782 :     funcname = llvm_expand_funcname(context, "deform");
     106             : 
     107             :     /*
     108             :      * Check which columns have to exist, so we don't have to check the row's
     109             :      * natts unnecessarily.
     110             :      */
     111       40448 :     for (attnum = 0; attnum < desc->natts; attnum++)
     112             :     {
     113       33666 :         Form_pg_attribute att = TupleDescAttr(desc, attnum);
     114             : 
     115             :         /*
     116             :          * If the column is declared NOT NULL then it must be present in every
     117             :          * tuple, unless there's a "missing" entry that could provide a
     118             :          * non-NULL value for it. That in turn guarantees that the NULL bitmap
     119             :          * - if there are any NULLable columns - is at least long enough to
     120             :          * cover columns up to attnum.
     121             :          *
     122             :          * Be paranoid and also check !attisdropped, even though the
     123             :          * combination of attisdropped && attnotnull combination shouldn't
     124             :          * exist.
     125             :          */
     126       33666 :         if (att->attnotnull &&
     127        7882 :             !att->atthasmissing &&
     128        7882 :             !att->attisdropped)
     129        7882 :             guaranteed_column_number = attnum;
     130             :     }
     131             : 
     132             :     /* Create the signature and function */
     133             :     {
     134             :         LLVMTypeRef param_types[1];
     135             : 
     136        6782 :         param_types[0] = l_ptr(StructTupleTableSlot);
     137             : 
     138        6782 :         deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc),
     139             :                                       param_types, lengthof(param_types), 0);
     140             :     }
     141        6782 :     v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig);
     142        6782 :     LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage);
     143        6782 :     LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF);
     144        6782 :     llvm_copy_attributes(AttributeTemplate, v_deform_fn);
     145             : 
     146             :     b_entry =
     147        6782 :         LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry");
     148             :     b_adjust_unavail_cols =
     149        6782 :         LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols");
     150             :     b_find_start =
     151        6782 :         LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock");
     152             :     b_out =
     153        6782 :         LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock");
     154             :     b_dead =
     155        6782 :         LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock");
     156             : 
     157        6782 :     b = LLVMCreateBuilderInContext(lc);
     158             : 
     159        6782 :     attcheckattnoblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
     160        6782 :     attstartblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
     161        6782 :     attisnullblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
     162        6782 :     attcheckalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
     163        6782 :     attalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
     164        6782 :     attstoreblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
     165             : 
     166        6782 :     known_alignment = 0;
     167             : 
     168        6782 :     LLVMPositionBuilderAtEnd(b, b_entry);
     169             : 
     170             :     /* perform allocas first, llvm only converts those to registers */
     171        6782 :     v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp");
     172             : 
     173        6782 :     v_slot = LLVMGetParam(v_deform_fn, 0);
     174             : 
     175             :     v_tts_values =
     176        6782 :         l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES,
     177             :                           "tts_values");
     178             :     v_tts_nulls =
     179        6782 :         l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL,
     180             :                           "tts_ISNULL");
     181        6782 :     v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, "");
     182        6782 :     v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, "");
     183             : 
     184        6782 :     if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple)
     185        4130 :     {
     186             :         LLVMValueRef v_heapslot;
     187             : 
     188             :         v_heapslot =
     189        4130 :             LLVMBuildBitCast(b,
     190             :                              v_slot,
     191             :                              l_ptr(StructHeapTupleTableSlot),
     192             :                              "heapslot");
     193        4130 :         v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, "");
     194             :         v_tupleheaderp =
     195        4130 :             l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE,
     196             :                               "tupleheader");
     197             :     }
     198        2652 :     else if (ops == &TTSOpsMinimalTuple)
     199             :     {
     200             :         LLVMValueRef v_minimalslot;
     201             : 
     202             :         v_minimalslot =
     203        2652 :             LLVMBuildBitCast(b,
     204             :                              v_slot,
     205             :                              l_ptr(StructMinimalTupleTableSlot),
     206             :                              "minimalslot");
     207        2652 :         v_slotoffp = l_struct_gep(b,
     208             :                                   StructMinimalTupleTableSlot,
     209             :                                   v_minimalslot,
     210             :                                   FIELDNO_MINIMALTUPLETABLESLOT_OFF, "");
     211             :         v_tupleheaderp =
     212        2652 :             l_load_struct_gep(b,
     213             :                               StructMinimalTupleTableSlot,
     214             :                               v_minimalslot,
     215             :                               FIELDNO_MINIMALTUPLETABLESLOT_TUPLE,
     216             :                               "tupleheader");
     217             :     }
     218             :     else
     219             :     {
     220             :         /* should've returned at the start of the function */
     221           0 :         pg_unreachable();
     222             :     }
     223             : 
     224             :     v_tuplep =
     225        6782 :         l_load_struct_gep(b,
     226             :                           StructHeapTupleData,
     227             :                           v_tupleheaderp,
     228             :                           FIELDNO_HEAPTUPLEDATA_DATA,
     229             :                           "tuple");
     230             :     v_bits =
     231        6782 :         LLVMBuildBitCast(b,
     232             :                          l_struct_gep(b,
     233             :                                       StructHeapTupleHeaderData,
     234             :                                       v_tuplep,
     235             :                                       FIELDNO_HEAPTUPLEHEADERDATA_BITS,
     236             :                                       ""),
     237             :                          l_ptr(LLVMInt8TypeInContext(lc)),
     238             :                          "t_bits");
     239             :     v_infomask1 =
     240        6782 :         l_load_struct_gep(b,
     241             :                           StructHeapTupleHeaderData,
     242             :                           v_tuplep,
     243             :                           FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK,
     244             :                           "infomask1");
     245             :     v_infomask2 =
     246        6782 :         l_load_struct_gep(b,
     247             :                           StructHeapTupleHeaderData,
     248             :                           v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2,
     249             :                           "infomask2");
     250             : 
     251             :     /* t_infomask & HEAP_HASNULL */
     252             :     v_hasnulls =
     253        6782 :         LLVMBuildICmp(b, LLVMIntNE,
     254             :                       LLVMBuildAnd(b,
     255             :                                    l_int16_const(lc, HEAP_HASNULL),
     256             :                                    v_infomask1, ""),
     257             :                       l_int16_const(lc, 0),
     258             :                       "hasnulls");
     259             : 
     260             :     /* t_infomask2 & HEAP_NATTS_MASK */
     261        6782 :     v_maxatt = LLVMBuildAnd(b,
     262             :                             l_int16_const(lc, HEAP_NATTS_MASK),
     263             :                             v_infomask2,
     264             :                             "maxatt");
     265             : 
     266             :     /*
     267             :      * Need to zext, as getelementptr otherwise treats hoff as a signed 8bit
     268             :      * integer, which'd yield a negative offset for t_hoff > 127.
     269             :      */
     270        6782 :     v_hoff =
     271        6782 :         LLVMBuildZExt(b,
     272             :                       l_load_struct_gep(b,
     273             :                                         StructHeapTupleHeaderData,
     274             :                                         v_tuplep,
     275             :                                         FIELDNO_HEAPTUPLEHEADERDATA_HOFF,
     276             :                                         ""),
     277             :                       LLVMInt32TypeInContext(lc), "t_hoff");
     278             : 
     279        6782 :     v_tupdata_base = l_gep(b,
     280             :                            LLVMInt8TypeInContext(lc),
     281             :                            LLVMBuildBitCast(b,
     282             :                                             v_tuplep,
     283             :                                             l_ptr(LLVMInt8TypeInContext(lc)),
     284             :                                             ""),
     285             :                            &v_hoff, 1,
     286             :                            "v_tupdata_base");
     287             : 
     288             :     /*
     289             :      * Load tuple start offset from slot. Will be reset below in case there's
     290             :      * no existing deformed columns in slot.
     291             :      */
     292             :     {
     293             :         LLVMValueRef v_off_start;
     294             : 
     295        6782 :         v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off");
     296        6782 :         v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, "");
     297        6782 :         LLVMBuildStore(b, v_off_start, v_offp);
     298             :     }
     299             : 
     300             :     /* build the basic block for each attribute, need them as jump target */
     301       23466 :     for (attnum = 0; attnum < natts; attnum++)
     302             :     {
     303       33368 :         attcheckattnoblocks[attnum] =
     304       16684 :             l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum);
     305       33368 :         attstartblocks[attnum] =
     306       16684 :             l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum);
     307       33368 :         attisnullblocks[attnum] =
     308       16684 :             l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum);
     309       33368 :         attcheckalignblocks[attnum] =
     310       16684 :             l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum);
     311       33368 :         attalignblocks[attnum] =
     312       16684 :             l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum);
     313       16684 :         attstoreblocks[attnum] =
     314       16684 :             l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum);
     315             :     }
     316             : 
     317             :     /*
     318             :      * Check if it is guaranteed that all the desired attributes are available
     319             :      * in the tuple (but still possibly NULL), by dint of either the last
     320             :      * to-be-deformed column being NOT NULL, or subsequent ones not accessed
     321             :      * here being NOT NULL.  If that's not guaranteed the tuple headers natt's
     322             :      * has to be checked, and missing attributes potentially have to be
     323             :      * fetched (using slot_getmissingattrs().
     324             :      */
     325        6782 :     if ((natts - 1) <= guaranteed_column_number)
     326             :     {
     327             :         /* just skip through unnecessary blocks */
     328         468 :         LLVMBuildBr(b, b_adjust_unavail_cols);
     329         468 :         LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
     330         468 :         LLVMBuildBr(b, b_find_start);
     331             :     }
     332             :     else
     333             :     {
     334             :         LLVMValueRef v_params[3];
     335             :         LLVMValueRef f;
     336             : 
     337             :         /* branch if not all columns available */
     338        6314 :         LLVMBuildCondBr(b,
     339             :                         LLVMBuildICmp(b, LLVMIntULT,
     340             :                                       v_maxatt,
     341             :                                       l_int16_const(lc, natts),
     342             :                                       ""),
     343             :                         b_adjust_unavail_cols,
     344             :                         b_find_start);
     345             : 
     346             :         /* if not, memset tts_isnull of relevant cols to true */
     347        6314 :         LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
     348             : 
     349        6314 :         v_params[0] = v_slot;
     350        6314 :         v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), "");
     351        6314 :         v_params[2] = l_int32_const(lc, natts);
     352        6314 :         f = llvm_pg_func(mod, "slot_getmissingattrs");
     353        6314 :         l_call(b,
     354             :                LLVMGetFunctionType(f), f,
     355             :                v_params, lengthof(v_params), "");
     356        6314 :         LLVMBuildBr(b, b_find_start);
     357             :     }
     358             : 
     359        6782 :     LLVMPositionBuilderAtEnd(b, b_find_start);
     360             : 
     361        6782 :     v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, "");
     362             : 
     363             :     /*
     364             :      * Build switch to go from nvalid to the right startblock.  Callers
     365             :      * currently don't have the knowledge, but it'd be good for performance to
     366             :      * avoid this check when it's known that the slot is empty (e.g. in scan
     367             :      * nodes).
     368             :      */
     369             :     if (true)
     370             :     {
     371        6782 :         LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid,
     372             :                                                 b_dead, natts);
     373             : 
     374       23466 :         for (attnum = 0; attnum < natts; attnum++)
     375             :         {
     376       16684 :             LLVMValueRef v_attno = l_int16_const(lc, attnum);
     377             : 
     378       16684 :             LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]);
     379             :         }
     380             :     }
     381             :     else
     382             :     {
     383             :         /* jump from entry block to first block */
     384             :         LLVMBuildBr(b, attcheckattnoblocks[0]);
     385             :     }
     386             : 
     387        6782 :     LLVMPositionBuilderAtEnd(b, b_dead);
     388        6782 :     LLVMBuildUnreachable(b);
     389             : 
     390             :     /*
     391             :      * Iterate over each attribute that needs to be deformed, build code to
     392             :      * deform it.
     393             :      */
     394       23466 :     for (attnum = 0; attnum < natts; attnum++)
     395             :     {
     396       16684 :         Form_pg_attribute att = TupleDescAttr(desc, attnum);
     397             :         LLVMValueRef v_incby;
     398             :         int         alignto;
     399       16684 :         LLVMValueRef l_attno = l_int16_const(lc, attnum);
     400             :         LLVMValueRef v_attdatap;
     401             :         LLVMValueRef v_resultp;
     402             : 
     403             :         /* build block checking whether we did all the necessary attributes */
     404       16684 :         LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]);
     405             : 
     406             :         /*
     407             :          * If this is the first attribute, slot->tts_nvalid was 0. Therefore
     408             :          * also reset offset to 0, it may be from a previous execution.
     409             :          */
     410       16684 :         if (attnum == 0)
     411             :         {
     412        6782 :             LLVMBuildStore(b, l_sizet_const(0), v_offp);
     413             :         }
     414             : 
     415             :         /*
     416             :          * Build check whether column is available (i.e. whether the tuple has
     417             :          * that many columns stored). We can avoid the branch if we know
     418             :          * there's a subsequent NOT NULL column.
     419             :          */
     420       16684 :         if (attnum <= guaranteed_column_number)
     421             :         {
     422        1964 :             LLVMBuildBr(b, attstartblocks[attnum]);
     423             :         }
     424             :         else
     425             :         {
     426             :             LLVMValueRef v_islast;
     427             : 
     428       14720 :             v_islast = LLVMBuildICmp(b, LLVMIntUGE,
     429             :                                      l_attno,
     430             :                                      v_maxatt,
     431             :                                      "heap_natts");
     432       14720 :             LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]);
     433             :         }
     434       16684 :         LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]);
     435             : 
     436             :         /*
     437             :          * Check for nulls if necessary. No need to take missing attributes
     438             :          * into account, because if they're present the heaptuple's natts
     439             :          * would have indicated that a slot_getmissingattrs() is needed.
     440             :          */
     441       16684 :         if (!att->attnotnull)
     442             :         {
     443             :             LLVMBasicBlockRef b_ifnotnull;
     444             :             LLVMBasicBlockRef b_ifnull;
     445             :             LLVMBasicBlockRef b_next;
     446             :             LLVMValueRef v_attisnull;
     447             :             LLVMValueRef v_nullbyteno;
     448             :             LLVMValueRef v_nullbytemask;
     449             :             LLVMValueRef v_nullbyte;
     450             :             LLVMValueRef v_nullbit;
     451             : 
     452       14756 :             b_ifnotnull = attcheckalignblocks[attnum];
     453       14756 :             b_ifnull = attisnullblocks[attnum];
     454             : 
     455       14756 :             if (attnum + 1 == natts)
     456        6314 :                 b_next = b_out;
     457             :             else
     458        8442 :                 b_next = attcheckattnoblocks[attnum + 1];
     459             : 
     460       14756 :             v_nullbyteno = l_int32_const(lc, attnum >> 3);
     461       14756 :             v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07));
     462       14756 :             v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte");
     463             : 
     464       14756 :             v_nullbit = LLVMBuildICmp(b,
     465             :                                       LLVMIntEQ,
     466             :                                       LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""),
     467             :                                       l_int8_const(lc, 0),
     468             :                                       "attisnull");
     469             : 
     470       14756 :             v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, "");
     471             : 
     472       14756 :             LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull);
     473             : 
     474       14756 :             LLVMPositionBuilderAtEnd(b, b_ifnull);
     475             : 
     476             :             /* store null-byte */
     477       14756 :             LLVMBuildStore(b,
     478             :                            l_int8_const(lc, 1),
     479             :                            l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, ""));
     480             :             /* store zero datum */
     481       14756 :             LLVMBuildStore(b,
     482             :                            l_sizet_const(0),
     483             :                            l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, ""));
     484             : 
     485       14756 :             LLVMBuildBr(b, b_next);
     486       14756 :             attguaranteedalign = false;
     487             :         }
     488             :         else
     489             :         {
     490             :             /* nothing to do */
     491        1928 :             LLVMBuildBr(b, attcheckalignblocks[attnum]);
     492        1928 :             LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]);
     493        1928 :             LLVMBuildBr(b, attcheckalignblocks[attnum]);
     494             :         }
     495       16684 :         LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
     496             : 
     497             :         /* determine required alignment */
     498       16684 :         if (att->attalign == TYPALIGN_INT)
     499       13196 :             alignto = ALIGNOF_INT;
     500        3488 :         else if (att->attalign == TYPALIGN_CHAR)
     501        1384 :             alignto = 1;
     502        2104 :         else if (att->attalign == TYPALIGN_DOUBLE)
     503        1632 :             alignto = ALIGNOF_DOUBLE;
     504         472 :         else if (att->attalign == TYPALIGN_SHORT)
     505         472 :             alignto = ALIGNOF_SHORT;
     506             :         else
     507             :         {
     508           0 :             elog(ERROR, "unknown alignment");
     509             :             alignto = 0;
     510             :         }
     511             : 
     512             :         /* ------
     513             :          * Even if alignment is required, we can skip doing it if provably
     514             :          * unnecessary:
     515             :          * - first column is guaranteed to be aligned
     516             :          * - columns following a NOT NULL fixed width datum have known
     517             :          *   alignment, can skip alignment computation if that known alignment
     518             :          *   is compatible with current column.
     519             :          * ------
     520             :          */
     521       16684 :         if (alignto > 1 &&
     522        7268 :             (known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment)))
     523             :         {
     524             :             /*
     525             :              * When accessing a varlena field, we have to "peek" to see if we
     526             :              * are looking at a pad byte or the first byte of a 1-byte-header
     527             :              * datum.  A zero byte must be either a pad byte, or the first
     528             :              * byte of a correctly aligned 4-byte length word; in either case,
     529             :              * we can align safely.  A non-zero byte must be either a 1-byte
     530             :              * length word, or the first byte of a correctly aligned 4-byte
     531             :              * length word; in either case, we need not align.
     532             :              */
     533        8050 :             if (att->attlen == -1)
     534             :             {
     535             :                 LLVMValueRef v_possible_padbyte;
     536             :                 LLVMValueRef v_ispad;
     537             :                 LLVMValueRef v_off;
     538             : 
     539             :                 /* don't know if short varlena or not */
     540        1138 :                 attguaranteedalign = false;
     541             : 
     542        1138 :                 v_off = l_load(b, TypeSizeT, v_offp, "");
     543             : 
     544             :                 v_possible_padbyte =
     545        1138 :                     l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, v_off, "padbyte");
     546             :                 v_ispad =
     547        1138 :                     LLVMBuildICmp(b, LLVMIntEQ,
     548             :                                   v_possible_padbyte, l_int8_const(lc, 0),
     549             :                                   "ispadbyte");
     550        1138 :                 LLVMBuildCondBr(b, v_ispad,
     551        1138 :                                 attalignblocks[attnum],
     552        1138 :                                 attstoreblocks[attnum]);
     553             :             }
     554             :             else
     555             :             {
     556        6912 :                 LLVMBuildBr(b, attalignblocks[attnum]);
     557             :             }
     558             : 
     559        8050 :             LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
     560             : 
     561             :             /* translation of alignment code (cf TYPEALIGN()) */
     562             :             {
     563             :                 LLVMValueRef v_off_aligned;
     564        8050 :                 LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
     565             : 
     566             :                 /* ((ALIGNVAL) - 1) */
     567        8050 :                 LLVMValueRef v_alignval = l_sizet_const(alignto - 1);
     568             : 
     569             :                 /* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */
     570        8050 :                 LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, "");
     571             : 
     572             :                 /* ~((uintptr_t) ((ALIGNVAL) - 1)) */
     573        8050 :                 LLVMValueRef v_rh = l_sizet_const(~(alignto - 1));
     574             : 
     575        8050 :                 v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset");
     576             : 
     577        8050 :                 LLVMBuildStore(b, v_off_aligned, v_offp);
     578             :             }
     579             : 
     580             :             /*
     581             :              * As alignment either was unnecessary or has been performed, we
     582             :              * now know the current alignment. This is only safe because this
     583             :              * value isn't used for varlena and nullable columns.
     584             :              */
     585        8050 :             if (known_alignment >= 0)
     586             :             {
     587             :                 Assert(known_alignment != 0);
     588          18 :                 known_alignment = TYPEALIGN(alignto, known_alignment);
     589             :             }
     590             : 
     591        8050 :             LLVMBuildBr(b, attstoreblocks[attnum]);
     592        8050 :             LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
     593             :         }
     594             :         else
     595             :         {
     596        8634 :             LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
     597        8634 :             LLVMBuildBr(b, attalignblocks[attnum]);
     598        8634 :             LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
     599        8634 :             LLVMBuildBr(b, attstoreblocks[attnum]);
     600             :         }
     601       16684 :         LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
     602             : 
     603             :         /*
     604             :          * Store the current offset if known to be constant. That allows LLVM
     605             :          * to generate better code. Without that LLVM can't figure out that
     606             :          * the offset might be constant due to the jumps for previously
     607             :          * decoded columns.
     608             :          */
     609       16684 :         if (attguaranteedalign)
     610             :         {
     611             :             Assert(known_alignment >= 0);
     612        1892 :             LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
     613             :         }
     614             : 
     615             :         /* compute what following columns are aligned to */
     616       16684 :         if (att->attlen < 0)
     617             :         {
     618             :             /* can't guarantee any alignment after variable length field */
     619        1928 :             known_alignment = -1;
     620        1928 :             attguaranteedalign = false;
     621             :         }
     622       14756 :         else if (att->attnotnull && attguaranteedalign && known_alignment >= 0)
     623             :         {
     624             :             /*
     625             :              * If the offset to the column was previously known, a NOT NULL &
     626             :              * fixed-width column guarantees that alignment is just the
     627             :              * previous alignment plus column width.
     628             :              */
     629             :             Assert(att->attlen > 0);
     630        1874 :             known_alignment += att->attlen;
     631             :         }
     632       12882 :         else if (att->attnotnull && (att->attlen % alignto) == 0)
     633             :         {
     634             :             /*
     635             :              * After a NOT NULL fixed-width column with a length that is a
     636             :              * multiple of its alignment requirement, we know the following
     637             :              * column is aligned to at least the current column's alignment.
     638             :              */
     639             :             Assert(att->attlen > 0);
     640           0 :             known_alignment = alignto;
     641             :             Assert(known_alignment > 0);
     642           0 :             attguaranteedalign = false;
     643             :         }
     644             :         else
     645             :         {
     646       12882 :             known_alignment = -1;
     647       12882 :             attguaranteedalign = false;
     648             :         }
     649             : 
     650             : 
     651             :         /* compute address to load data from */
     652             :         {
     653       16684 :             LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
     654             : 
     655       16684 :             v_attdatap =
     656       16684 :                 l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, "");
     657             :         }
     658             : 
     659             :         /* compute address to store value at */
     660       16684 :         v_resultp = l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, "");
     661             : 
     662             :         /* store null-byte (false) */
     663       16684 :         LLVMBuildStore(b, l_int8_const(lc, 0),
     664             :                        l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, ""));
     665             : 
     666             :         /*
     667             :          * Store datum. For byval: datums copy the value, extend to Datum's
     668             :          * width, and store. For byref types: store pointer to data.
     669             :          */
     670       16684 :         if (att->attbyval)
     671             :         {
     672             :             LLVMValueRef v_tmp_loaddata;
     673       13574 :             LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8);
     674       13574 :             LLVMTypeRef vartypep = LLVMPointerType(vartype, 0);
     675             : 
     676             :             v_tmp_loaddata =
     677       13574 :                 LLVMBuildPointerCast(b, v_attdatap, vartypep, "");
     678       13574 :             v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval");
     679       13574 :             v_tmp_loaddata = LLVMBuildZExt(b, v_tmp_loaddata, TypeSizeT, "");
     680             : 
     681       13574 :             LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
     682             :         }
     683             :         else
     684             :         {
     685             :             LLVMValueRef v_tmp_loaddata;
     686             : 
     687             :             /* store pointer */
     688             :             v_tmp_loaddata =
     689        3110 :                 LLVMBuildPtrToInt(b,
     690             :                                   v_attdatap,
     691             :                                   TypeSizeT,
     692             :                                   "attr_ptr");
     693        3110 :             LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
     694             :         }
     695             : 
     696             :         /* increment data pointer */
     697       16684 :         if (att->attlen > 0)
     698             :         {
     699       14756 :             v_incby = l_sizet_const(att->attlen);
     700             :         }
     701        1928 :         else if (att->attlen == -1)
     702             :         {
     703        1928 :             v_incby = l_call(b,
     704             :                              llvm_pg_var_func_type("varsize_any"),
     705             :                              llvm_pg_func(mod, "varsize_any"),
     706             :                              &v_attdatap, 1,
     707             :                              "varsize_any");
     708        1928 :             l_callsite_ro(v_incby);
     709        1928 :             l_callsite_alwaysinline(v_incby);
     710             :         }
     711           0 :         else if (att->attlen == -2)
     712             :         {
     713           0 :             v_incby = l_call(b,
     714             :                              llvm_pg_var_func_type("strlen"),
     715             :                              llvm_pg_func(mod, "strlen"),
     716             :                              &v_attdatap, 1, "strlen");
     717             : 
     718           0 :             l_callsite_ro(v_incby);
     719             : 
     720             :             /* add 1 for NUL byte */
     721           0 :             v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), "");
     722             :         }
     723             :         else
     724             :         {
     725             :             Assert(false);
     726           0 :             v_incby = NULL;     /* silence compiler */
     727             :         }
     728             : 
     729       16684 :         if (attguaranteedalign)
     730             :         {
     731             :             Assert(known_alignment >= 0);
     732        1874 :             LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
     733             :         }
     734             :         else
     735             :         {
     736       14810 :             LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
     737             : 
     738       14810 :             v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset");
     739       14810 :             LLVMBuildStore(b, v_off, v_offp);
     740             :         }
     741             : 
     742             :         /*
     743             :          * jump to next block, unless last possible column, or all desired
     744             :          * (available) attributes have been fetched.
     745             :          */
     746       16684 :         if (attnum + 1 == natts)
     747             :         {
     748             :             /* jump out */
     749        6782 :             LLVMBuildBr(b, b_out);
     750             :         }
     751             :         else
     752             :         {
     753        9902 :             LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]);
     754             :         }
     755             :     }
     756             : 
     757             : 
     758             :     /* build block that returns */
     759        6782 :     LLVMPositionBuilderAtEnd(b, b_out);
     760             : 
     761             :     {
     762        6782 :         LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");
     763             :         LLVMValueRef v_flags;
     764             : 
     765        6782 :         LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp);
     766        6782 :         v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), "");
     767        6782 :         LLVMBuildStore(b, v_off, v_slotoffp);
     768        6782 :         v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags");
     769        6782 :         v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), "");
     770        6782 :         LLVMBuildStore(b, v_flags, v_flagsp);
     771        6782 :         LLVMBuildRetVoid(b);
     772             :     }
     773             : 
     774        6782 :     LLVMDisposeBuilder(b);
     775             : 
     776        6782 :     return v_deform_fn;
     777             : }

Generated by: LCOV version 1.14