Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * instr_time.h
4 : * portable high-precision interval timing
5 : *
6 : * This file provides an abstraction layer to hide portability issues in
7 : * interval timing. On x86 we use the RDTSC/RDTSCP instruction directly in
8 : * certain cases, or alternatively clock_gettime() on Unix-like systems and
9 : * QueryPerformanceCounter() on Windows. These macros also give some breathing
10 : * room to use other high-precision-timing APIs.
11 : *
12 : * The basic data type is instr_time, which all callers should treat as an
13 : * opaque typedef. instr_time can store either an absolute time (of
14 : * unspecified reference time) or an interval. The operations provided
15 : * for it are:
16 : *
17 : * INSTR_TIME_IS_ZERO(t) is t equal to zero?
18 : *
19 : * INSTR_TIME_SET_ZERO(t) set t to zero (memset is acceptable too)
20 : *
21 : * INSTR_TIME_SET_CURRENT_FAST(t) set t to current time without waiting
22 : * for instructions in out-of-order window
23 : *
24 : * INSTR_TIME_SET_CURRENT(t) set t to current time while waiting for
25 : * instructions in OOO to retire
26 : *
27 : *
28 : * INSTR_TIME_ADD(x, y) x += y
29 : *
30 : * INSTR_TIME_ADD_NANOSEC(t, n) t += n in nanoseconds (converts to ticks)
31 : *
32 : * INSTR_TIME_SUBTRACT(x, y) x -= y
33 : *
34 : * INSTR_TIME_ACCUM_DIFF(x, y, z) x += (y - z)
35 : *
36 : * INSTR_TIME_GT(x, y) x > y
37 : *
38 : * INSTR_TIME_GET_DOUBLE(t) convert t to double (in seconds)
39 : *
40 : * INSTR_TIME_GET_MILLISEC(t) convert t to double (in milliseconds)
41 : *
42 : * INSTR_TIME_GET_MICROSEC(t) convert t to int64 (in microseconds)
43 : *
44 : * INSTR_TIME_GET_NANOSEC(t) convert t to int64 (in nanoseconds)
45 : *
46 : * Note that INSTR_TIME_SUBTRACT and INSTR_TIME_ACCUM_DIFF convert
47 : * absolute times to intervals. The INSTR_TIME_GET_xxx operations are
48 : * only useful on intervals.
49 : *
50 : * When summing multiple measurements, it's recommended to leave the
51 : * running sum in instr_time form (ie, use INSTR_TIME_ADD or
52 : * INSTR_TIME_ACCUM_DIFF) and convert to a result format only at the end.
53 : *
54 : * Beware of multiple evaluations of the macro arguments.
55 : *
56 : *
57 : * Copyright (c) 2001-2026, PostgreSQL Global Development Group
58 : *
59 : * src/include/portability/instr_time.h
60 : *
61 : *-------------------------------------------------------------------------
62 : */
63 : #ifndef INSTR_TIME_H
64 : #define INSTR_TIME_H
65 :
66 :
67 : /*
68 : * We store interval times as an int64 integer on all platforms, as int64 is
69 : * cheap to add/subtract, the most common operation for instr_time. The
70 : * acquisition of time and converting to specific units of time is platform
71 : * specific.
72 : *
73 : * To avoid users of the API relying on the integer representation, we wrap
74 : * the 64bit integer in a struct.
75 : */
76 : typedef struct instr_time
77 : {
78 : int64 ticks; /* in platforms specific unit */
79 : } instr_time;
80 :
81 :
82 : /* helpers macros used in platform specific code below */
83 :
84 : #define NS_PER_S INT64CONST(1000000000)
85 : #define NS_PER_MS INT64CONST(1000000)
86 : #define NS_PER_US INT64CONST(1000)
87 :
88 : /* Shift amount for fixed-point ticks-to-nanoseconds conversion. */
89 : #define TICKS_TO_NS_SHIFT 14
90 :
91 : /*
92 : * PG_INSTR_TICKS_TO_NS controls whether pg_ticks_to_ns/pg_ns_to_ticks needs to
93 : * check ticks_per_ns_scaled and potentially convert ticks <=> nanoseconds.
94 : *
95 : * PG_INSTR_TSC_CLOCK controls whether the TSC clock source is compiled in, and
96 : * potentially used based on timing_tsc_enabled.
97 : */
98 : #if defined(__x86_64__) || defined(_M_X64)
99 : #define PG_INSTR_TICKS_TO_NS 1
100 : #define PG_INSTR_TSC_CLOCK 1
101 : #elif defined(WIN32)
102 : #define PG_INSTR_TICKS_TO_NS 1
103 : #define PG_INSTR_TSC_CLOCK 0
104 : #else
105 : #define PG_INSTR_TICKS_TO_NS 0
106 : #define PG_INSTR_TSC_CLOCK 0
107 : #endif
108 :
109 : /*
110 : * Variables used to translate ticks to nanoseconds, initialized by
111 : * pg_initialize_timing and adjusted by pg_set_timing_clock_source calls or
112 : * changes of the "timing_clock_source" GUC.
113 : *
114 : * Note that changing these values after setting an instr_time and before
115 : * reading/converting it will lead to incorrect results. This is technically
116 : * possible because the GUC can be changed at runtime, but unlikely, and we
117 : * allow changing this at runtime to simplify testing of different sources.
118 : */
119 : extern PGDLLIMPORT uint64 ticks_per_ns_scaled;
120 : extern PGDLLIMPORT uint64 max_ticks_no_overflow;
121 : extern PGDLLIMPORT bool timing_initialized;
122 :
123 : typedef enum
124 : {
125 : TIMING_CLOCK_SOURCE_AUTO,
126 : TIMING_CLOCK_SOURCE_SYSTEM,
127 : #if PG_INSTR_TSC_CLOCK
128 : TIMING_CLOCK_SOURCE_TSC
129 : #endif
130 : } TimingClockSourceType;
131 :
132 : extern PGDLLIMPORT int timing_clock_source;
133 :
134 : /*
135 : * Initialize timing infrastructure
136 : *
137 : * This must be called at least once before using INSTR_TIME_SET_CURRENT*
138 : * macros.
139 : *
140 : * If you want to use the TSC clock source in a client program,
141 : * pg_set_timing_clock_source() needs to also be called.
142 : */
143 : extern void pg_initialize_timing(void);
144 :
145 : /*
146 : * Sets the time source to be used. Mainly intended for frontend programs,
147 : * the backend should set it via the timing_clock_source GUC instead.
148 : *
149 : * Returns false if the clock source could not be set, for example when TSC
150 : * is not available despite being explicitly set.
151 : */
152 : extern bool pg_set_timing_clock_source(TimingClockSourceType source);
153 :
154 : /* Whether to actually use TSC based on availability and GUC settings. */
155 : extern PGDLLIMPORT bool timing_tsc_enabled;
156 :
157 : /*
158 : * TSC frequency in kHz, set during initialization.
159 : *
160 : * -1 = not yet initialized, 0 = TSC not usable, >0 = frequency in kHz.
161 : */
162 : extern PGDLLIMPORT int32 timing_tsc_frequency_khz;
163 :
164 : #if PG_INSTR_TSC_CLOCK
165 :
166 : extern void pg_initialize_timing_tsc(void);
167 :
168 : typedef struct TscClockSourceInfo
169 : {
170 : int32 frequency_khz; /* from CPUID or calibration */
171 : int32 calibrated_frequency_khz; /* from calibration */
172 : char frequency_source[128]; /* describes how frequency was
173 : * determined */
174 : } TscClockSourceInfo;
175 :
176 : extern const TscClockSourceInfo *pg_timing_tsc_clock_source_info(void);
177 :
178 : #endif /* PG_INSTR_TSC_CLOCK */
179 :
180 : /*
181 : * Returns the current timing clock source effectively in use, resolving
182 : * TIMING_CLOCK_SOURCE_AUTO to either TIMING_CLOCK_SOURCE_SYSTEM or
183 : * TIMING_CLOCK_SOURCE_TSC.
184 : */
185 : static inline TimingClockSourceType
186 1962 : pg_current_timing_clock_source(void)
187 : {
188 : #if PG_INSTR_TSC_CLOCK
189 1962 : if (timing_tsc_enabled)
190 1961 : return TIMING_CLOCK_SOURCE_TSC;
191 : #endif
192 1 : return TIMING_CLOCK_SOURCE_SYSTEM;
193 : }
194 :
195 : #ifndef WIN32
196 :
197 : /* On POSIX, use clock_gettime() for system clock source */
198 :
199 : #include <time.h>
200 :
201 : /*
202 : * The best clockid to use according to the POSIX spec is CLOCK_MONOTONIC,
203 : * since that will give reliable interval timing even in the face of changes
204 : * to the system clock. However, POSIX doesn't require implementations to
205 : * provide anything except CLOCK_REALTIME, so fall back to that if we don't
206 : * find CLOCK_MONOTONIC.
207 : *
208 : * Also, some implementations have nonstandard clockids with better properties
209 : * than CLOCK_MONOTONIC. In particular, as of macOS 10.12, Apple provides
210 : * CLOCK_MONOTONIC_RAW which is both faster to read and higher resolution than
211 : * their version of CLOCK_MONOTONIC.
212 : *
213 : * Note this does not get used in case the TSC clock source logic is used,
214 : * which directly calls architecture specific timing instructions (e.g. RDTSC).
215 : */
216 : #if defined(__darwin__) && defined(CLOCK_MONOTONIC_RAW)
217 : #define PG_INSTR_SYSTEM_CLOCK CLOCK_MONOTONIC_RAW
218 : #define PG_INSTR_SYSTEM_CLOCK_NAME "clock_gettime (CLOCK_MONOTONIC_RAW)"
219 : #elif defined(CLOCK_MONOTONIC)
220 : #define PG_INSTR_SYSTEM_CLOCK CLOCK_MONOTONIC
221 : #define PG_INSTR_SYSTEM_CLOCK_NAME "clock_gettime (CLOCK_MONOTONIC)"
222 : #else
223 : #define PG_INSTR_SYSTEM_CLOCK CLOCK_REALTIME
224 : #define PG_INSTR_SYSTEM_CLOCK_NAME "clock_gettime (CLOCK_REALTIME)"
225 : #endif
226 :
227 : static inline instr_time
228 7709996 : pg_get_ticks_system(void)
229 : {
230 : instr_time now;
231 : struct timespec tmp;
232 :
233 : Assert(timing_initialized);
234 :
235 7709996 : clock_gettime(PG_INSTR_SYSTEM_CLOCK, &tmp);
236 7709996 : now.ticks = tmp.tv_sec * NS_PER_S + tmp.tv_nsec;
237 :
238 7709996 : return now;
239 : }
240 :
241 : #else /* WIN32 */
242 :
243 : /* On Windows, use QueryPerformanceCounter() for system clock source */
244 :
245 : #define PG_INSTR_SYSTEM_CLOCK_NAME "QueryPerformanceCounter"
246 : static inline instr_time
247 : pg_get_ticks_system(void)
248 : {
249 : instr_time now;
250 : LARGE_INTEGER tmp;
251 :
252 : Assert(timing_initialized);
253 :
254 : QueryPerformanceCounter(&tmp);
255 : now.ticks = tmp.QuadPart;
256 :
257 : return now;
258 : }
259 :
260 : #endif /* WIN32 */
261 :
262 : static inline int64
263 64393965 : pg_ticks_to_ns(int64 ticks)
264 : {
265 : #if PG_INSTR_TICKS_TO_NS
266 64393965 : int64 ns = 0;
267 :
268 : Assert(timing_initialized);
269 :
270 : /*
271 : * Avoid doing work if we don't use scaled ticks, e.g. system clock on
272 : * Unix (in that case ticks is counted in nanoseconds)
273 : */
274 64393965 : if (ticks_per_ns_scaled == 0)
275 7707091 : return ticks;
276 :
277 : /*
278 : * Would multiplication overflow? If so perform computation in two parts.
279 : */
280 56686874 : if (unlikely(ticks > (int64) max_ticks_no_overflow))
281 : {
282 : /*
283 : * To avoid overflow, first scale total ticks down by the fixed
284 : * factor, and *afterwards* multiply them by the frequency-based scale
285 : * factor.
286 : *
287 : * The remaining ticks can follow the regular formula, since they
288 : * won't overflow.
289 : */
290 4 : int64 count = ticks >> TICKS_TO_NS_SHIFT;
291 :
292 4 : ns = count * ticks_per_ns_scaled;
293 4 : ticks -= (count << TICKS_TO_NS_SHIFT);
294 : }
295 :
296 56686874 : ns += (ticks * ticks_per_ns_scaled) >> TICKS_TO_NS_SHIFT;
297 :
298 56686874 : return ns;
299 : #else
300 : Assert(timing_initialized);
301 :
302 : return ticks;
303 : #endif /* PG_INSTR_TICKS_TO_NS */
304 : }
305 :
306 : static inline int64
307 15 : pg_ns_to_ticks(int64 ns)
308 : {
309 : #if PG_INSTR_TICKS_TO_NS
310 15 : int64 ticks = 0;
311 :
312 : Assert(timing_initialized);
313 :
314 : /*
315 : * If ticks_per_ns_scaled is zero, ticks are already in nanoseconds (e.g.
316 : * system clock on Unix).
317 : */
318 15 : if (ticks_per_ns_scaled == 0)
319 1 : return ns;
320 :
321 : /*
322 : * The reverse of pg_ticks_to_ns to avoid a similar overflow problem.
323 : */
324 14 : if (unlikely(ns > (INT64_MAX >> TICKS_TO_NS_SHIFT)))
325 : {
326 4 : int64 count = ns / ticks_per_ns_scaled;
327 :
328 4 : ticks = count << TICKS_TO_NS_SHIFT;
329 4 : ns -= count * ticks_per_ns_scaled;
330 : }
331 :
332 14 : ticks += (ns << TICKS_TO_NS_SHIFT) / ticks_per_ns_scaled;
333 :
334 14 : return ticks;
335 : #else
336 : Assert(timing_initialized);
337 :
338 : return ns;
339 : #endif /* PG_INSTR_TICKS_TO_NS */
340 : }
341 :
342 : #if PG_INSTR_TSC_CLOCK
343 :
344 : #define PG_INSTR_TSC_CLOCK_NAME_FAST "RDTSC"
345 : #define PG_INSTR_TSC_CLOCK_NAME "RDTSCP"
346 :
347 : #ifdef _MSC_VER
348 : #include <intrin.h>
349 : #endif /* defined(_MSC_VER) */
350 :
351 : /* Helpers to abstract compiler differences for reading the x86 TSC. */
352 : static inline int64
353 23089668 : pg_rdtsc(void)
354 : {
355 : #ifdef _MSC_VER
356 : return __rdtsc();
357 : #else
358 23089668 : return __builtin_ia32_rdtsc();
359 : #endif /* defined(_MSC_VER) */
360 : }
361 :
362 : static inline int64
363 8282747 : pg_rdtscp(void)
364 : {
365 : uint32 unused;
366 :
367 : #ifdef _MSC_VER
368 : return __rdtscp(&unused);
369 : #else
370 8282747 : return __builtin_ia32_rdtscp(&unused);
371 : #endif /* defined(_MSC_VER) */
372 : }
373 :
374 : /*
375 : * Marked always_inline due to a shortcoming in gcc's heuristics leading to
376 : * only inlining the function partially.
377 : * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124795
378 : */
379 : static pg_attribute_always_inline instr_time
380 13278649 : pg_get_ticks(void)
381 : {
382 13278649 : if (likely(timing_tsc_enabled))
383 : {
384 : instr_time now;
385 :
386 5568653 : now.ticks = pg_rdtscp();
387 5568653 : return now;
388 : }
389 :
390 7709996 : return pg_get_ticks_system();
391 : }
392 :
393 : static pg_attribute_always_inline instr_time
394 23089668 : pg_get_ticks_fast(void)
395 : {
396 23089668 : if (likely(timing_tsc_enabled))
397 : {
398 : instr_time now;
399 :
400 23089668 : now.ticks = pg_rdtsc();
401 23089668 : return now;
402 : }
403 :
404 0 : return pg_get_ticks_system();
405 : }
406 :
407 : #else
408 :
409 : static pg_attribute_always_inline instr_time
410 : pg_get_ticks(void)
411 : {
412 : return pg_get_ticks_system();
413 : }
414 :
415 : static pg_attribute_always_inline instr_time
416 : pg_get_ticks_fast(void)
417 : {
418 : return pg_get_ticks_system();
419 : }
420 :
421 : #endif /* PG_INSTR_TSC_CLOCK */
422 :
423 : /*
424 : * Common macros
425 : */
426 :
427 : #define INSTR_TIME_IS_ZERO(t) ((t).ticks == 0)
428 :
429 : #define INSTR_TIME_SET_ZERO(t) ((t).ticks = 0)
430 :
431 : #define INSTR_TIME_SET_CURRENT_FAST(t) \
432 : ((t) = pg_get_ticks_fast())
433 :
434 : #define INSTR_TIME_SET_CURRENT(t) \
435 : ((t) = pg_get_ticks())
436 :
437 :
438 : #define INSTR_TIME_ADD(x,y) \
439 : ((x).ticks += (y).ticks)
440 :
441 : #define INSTR_TIME_ADD_NANOSEC(t, n) \
442 : ((t).ticks += pg_ns_to_ticks(n))
443 :
444 : #define INSTR_TIME_SUBTRACT(x,y) \
445 : ((x).ticks -= (y).ticks)
446 :
447 : #define INSTR_TIME_ACCUM_DIFF(x,y,z) \
448 : ((x).ticks += (y).ticks - (z).ticks)
449 :
450 : #define INSTR_TIME_GT(x,y) \
451 : ((x).ticks > (y).ticks)
452 :
453 : #define INSTR_TIME_GET_NANOSEC(t) \
454 : (pg_ticks_to_ns((t).ticks))
455 :
456 : #define INSTR_TIME_GET_DOUBLE(t) \
457 : ((double) INSTR_TIME_GET_NANOSEC(t) / NS_PER_S)
458 :
459 : #define INSTR_TIME_GET_MILLISEC(t) \
460 : ((double) INSTR_TIME_GET_NANOSEC(t) / NS_PER_MS)
461 :
462 : #define INSTR_TIME_GET_MICROSEC(t) \
463 : (INSTR_TIME_GET_NANOSEC(t) / NS_PER_US)
464 :
465 : #endif /* INSTR_TIME_H */
|
