Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * pg_wchar.h
4 : : * multibyte-character support
5 : : *
6 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
7 : : * Portions Copyright (c) 1994, Regents of the University of California
8 : : *
9 : : * src/include/mb/pg_wchar.h
10 : : *
11 : : * NOTES
12 : : * This is used both by the backend and by frontends, but should not be
13 : : * included by libpq client programs. In particular, a libpq client
14 : : * should not assume that the encoding IDs used by the version of libpq
15 : : * it's linked to match up with the IDs declared here.
16 : : * To help prevent mistakes, relevant functions that are exported by
17 : : * libpq have a physically different name when being referenced
18 : : * statically.
19 : : *
20 : : *-------------------------------------------------------------------------
21 : : */
22 : : #ifndef PG_WCHAR_H
23 : : #define PG_WCHAR_H
24 : :
25 : : /*
26 : : * The pg_wchar type
27 : : */
28 : : typedef unsigned int pg_wchar;
29 : :
30 : : /*
31 : : * Maximum byte length of multibyte characters in any backend encoding
32 : : */
33 : : #define MAX_MULTIBYTE_CHAR_LEN 4
34 : :
35 : : /*
36 : : * various definitions for EUC
37 : : */
38 : : #define SS2 0x8e /* single shift 2 (JIS0201) */
39 : : #define SS3 0x8f /* single shift 3 (JIS0212) */
40 : :
41 : : /*
42 : : * EUC_TW planes
43 : : */
44 : : #define LC_CNS11643_1 0x95 /* CNS 11643-1992 Plane 1 */
45 : : #define LC_CNS11643_2 0x96 /* CNS 11643-1992 Plane 2 */
46 : : #define LC_CNS11643_3 0xf6 /* CNS 11643-1992 Plane 3 */
47 : : #define LC_CNS11643_4 0xf7 /* CNS 11643-1992 Plane 4 */
48 : : #define LC_CNS11643_5 0xf8 /* CNS 11643-1992 Plane 5 */
49 : : #define LC_CNS11643_6 0xf9 /* CNS 11643-1992 Plane 6 */
50 : : #define LC_CNS11643_7 0xfa /* CNS 11643-1992 Plane 7 */
51 : :
52 : : /*
53 : : * SJIS validation macros
54 : : */
55 : : #define ISSJISHEAD(c) (((c) >= 0x81 && (c) <= 0x9f) || ((c) >= 0xe0 && (c) <= 0xfc))
56 : : #define ISSJISTAIL(c) (((c) >= 0x40 && (c) <= 0x7e) || ((c) >= 0x80 && (c) <= 0xfc))
57 : :
58 : : /*
59 : : * PostgreSQL encoding identifiers
60 : : *
61 : : * WARNING: If you add some encoding don't forget to update
62 : : * the pg_enc2name_tbl[] array (in src/common/encnames.c),
63 : : * the pg_enc2gettext_tbl[] array (in src/common/encnames.c) and
64 : : * the pg_wchar_table[] array (in src/common/wchar.c) and to check
65 : : * PG_ENCODING_BE_LAST macro.
66 : : *
67 : : * PG_SQL_ASCII is default encoding and must be = 0.
68 : : *
69 : : * XXX We must avoid renumbering any backend encoding until libpq's major
70 : : * version number is increased beyond 5; it turns out that the backend
71 : : * encoding IDs are effectively part of libpq's ABI as far as 8.2 initdb and
72 : : * psql are concerned.
73 : : */
74 : : typedef enum pg_enc
75 : : {
76 : : PG_SQL_ASCII = 0, /* SQL/ASCII */
77 : : PG_EUC_JP, /* EUC for Japanese */
78 : : PG_EUC_CN, /* EUC for Chinese */
79 : : PG_EUC_KR, /* EUC for Korean */
80 : : PG_EUC_TW, /* EUC for Taiwan */
81 : : PG_EUC_JIS_2004, /* EUC-JIS-2004 */
82 : : PG_UTF8, /* Unicode UTF8 */
83 : : PG_UNUSED_1, /* (Was Mule internal code) */
84 : : PG_LATIN1, /* ISO-8859-1 Latin 1 */
85 : : PG_LATIN2, /* ISO-8859-2 Latin 2 */
86 : : PG_LATIN3, /* ISO-8859-3 Latin 3 */
87 : : PG_LATIN4, /* ISO-8859-4 Latin 4 */
88 : : PG_LATIN5, /* ISO-8859-9 Latin 5 */
89 : : PG_LATIN6, /* ISO-8859-10 Latin6 */
90 : : PG_LATIN7, /* ISO-8859-13 Latin7 */
91 : : PG_LATIN8, /* ISO-8859-14 Latin8 */
92 : : PG_LATIN9, /* ISO-8859-15 Latin9 */
93 : : PG_LATIN10, /* ISO-8859-16 Latin10 */
94 : : PG_WIN1256, /* windows-1256 */
95 : : PG_WIN1258, /* Windows-1258 */
96 : : PG_WIN866, /* (MS-DOS CP866) */
97 : : PG_WIN874, /* windows-874 */
98 : : PG_KOI8R, /* KOI8-R */
99 : : PG_WIN1251, /* windows-1251 */
100 : : PG_WIN1252, /* windows-1252 */
101 : : PG_ISO_8859_5, /* ISO-8859-5 */
102 : : PG_ISO_8859_6, /* ISO-8859-6 */
103 : : PG_ISO_8859_7, /* ISO-8859-7 */
104 : : PG_ISO_8859_8, /* ISO-8859-8 */
105 : : PG_WIN1250, /* windows-1250 */
106 : : PG_WIN1253, /* windows-1253 */
107 : : PG_WIN1254, /* windows-1254 */
108 : : PG_WIN1255, /* windows-1255 */
109 : : PG_WIN1257, /* windows-1257 */
110 : : PG_KOI8U, /* KOI8-U */
111 : : /* PG_ENCODING_BE_LAST points to the above entry */
112 : :
113 : : /* The following are for client encoding only */
114 : : PG_SJIS, /* Shift JIS (Windows-932) */
115 : : PG_BIG5, /* Big5 (Windows-950) */
116 : : PG_GBK, /* GBK (Windows-936) */
117 : : PG_UHC, /* UHC (Windows-949) */
118 : : PG_GB18030, /* GB18030 */
119 : : PG_JOHAB, /* EUC for Korean JOHAB */
120 : : PG_SHIFT_JIS_2004, /* Shift-JIS-2004 */
121 : : _PG_LAST_ENCODING_ /* mark only */
122 : :
123 : : } pg_enc;
124 : :
125 : : #define PG_ENCODING_BE_LAST PG_KOI8U
126 : :
127 : : #define PG_UNUSED_ENCODING(_enc) \
128 : : ((_enc) == PG_UNUSED_1)
129 : :
130 : : /*
131 : : * Please use these tests before access to pg_enc2name_tbl[]
132 : : * or to other places...
133 : : */
134 : : #define PG_VALID_BE_ENCODING(_enc) \
135 : : ((_enc) >= 0 && (_enc) <= PG_ENCODING_BE_LAST && !PG_UNUSED_ENCODING(_enc))
136 : :
137 : : #define PG_ENCODING_IS_CLIENT_ONLY(_enc) \
138 : : ((_enc) > PG_ENCODING_BE_LAST && (_enc) < _PG_LAST_ENCODING_)
139 : :
140 : : #define PG_VALID_ENCODING(_enc) \
141 : : ((_enc) >= 0 && (_enc) < _PG_LAST_ENCODING_ && !PG_UNUSED_ENCODING(_enc))
142 : :
143 : : /* On FE are possible all encodings */
144 : : #define PG_VALID_FE_ENCODING(_enc) PG_VALID_ENCODING(_enc)
145 : :
146 : : /*
147 : : * When converting strings between different encodings, we assume that space
148 : : * for converted result is 4-to-1 growth in the worst case. The rate for
149 : : * currently supported encoding pairs are within 3 (SJIS JIS X0201 half width
150 : : * kana -> UTF8 is the worst case). So "4" should be enough for the moment.
151 : : *
152 : : * Note that this is not the same as the maximum character width in any
153 : : * particular encoding.
154 : : */
155 : : #define MAX_CONVERSION_GROWTH 4
156 : :
157 : : /*
158 : : * Maximum byte length of a string that's required in any encoding to convert
159 : : * at least one character to any other encoding. In other words, if you feed
160 : : * MAX_CONVERSION_INPUT_LENGTH bytes to any encoding conversion function, it
161 : : * is guaranteed to be able to convert something without needing more input
162 : : * (assuming the input is valid).
163 : : *
164 : : * Currently, the maximum case is the conversion UTF8 -> SJIS JIS X0201 half
165 : : * width kana, where a pair of UTF-8 characters is converted into a single
166 : : * SHIFT_JIS_2004 character (the reverse of the worst case for
167 : : * MAX_CONVERSION_GROWTH). It needs 6 bytes of input. In theory, a
168 : : * user-defined conversion function might have more complicated cases, although
169 : : * for the reverse mapping you would probably also need to bump up
170 : : * MAX_CONVERSION_GROWTH. But there is no need to be stingy here, so make it
171 : : * generous.
172 : : */
173 : : #define MAX_CONVERSION_INPUT_LENGTH 16
174 : :
175 : : /*
176 : : * Maximum byte length of the string equivalent to any one Unicode code point,
177 : : * in any backend encoding. The current value assumes that a 4-byte UTF-8
178 : : * character might expand by MAX_CONVERSION_GROWTH, which is a huge
179 : : * overestimate. But in current usage we don't allocate large multiples of
180 : : * this, so there's little point in being stingy.
181 : : */
182 : : #define MAX_UNICODE_EQUIVALENT_STRING 16
183 : :
184 : : /*
185 : : * Table for mapping an encoding number to official encoding name and
186 : : * possibly other subsidiary data. Be careful to check encoding number
187 : : * before accessing a table entry!
188 : : *
189 : : * if (PG_VALID_ENCODING(encoding))
190 : : * pg_enc2name_tbl[ encoding ];
191 : : */
192 : : typedef struct pg_enc2name
193 : : {
194 : : const char *name;
195 : : pg_enc encoding;
196 : : #ifdef WIN32
197 : : unsigned codepage; /* codepage for WIN32 */
198 : : #endif
199 : : } pg_enc2name;
200 : :
201 : : extern PGDLLIMPORT const pg_enc2name pg_enc2name_tbl[];
202 : :
203 : : /*
204 : : * Encoding names for gettext
205 : : */
206 : : extern PGDLLIMPORT const char *pg_enc2gettext_tbl[];
207 : :
208 : : /*
209 : : * pg_wchar stuff
210 : : */
211 : :
212 : : typedef int (*mblen_function) (const unsigned char *mbstr);
213 : : typedef int (*mbverifychar_function) (const unsigned char *mbstr, int len);
214 : :
215 : : /*
216 : : * Operations on multi-byte encodings are driven by a table of helper
217 : : * functions.
218 : : *
219 : : * All functions are required, except the conversion functions
220 : : * .mb2wchar_with_len and .wchar2mb_with_len, which are only required for
221 : : * server encodings.
222 : : *
223 : : * These functions generally assume that their input is validly formed, and
224 : : * unless specified otherwise have no special return value for errors. The
225 : : * exceptions are the two "verify" functions, which must check their input and
226 : : * signal invalid input through their return value. None of these functions
227 : : * report errors via ereport().
228 : : */
229 : : typedef struct
230 : : {
231 : : /*
232 : : * Convert a multibyte string to a pg_wchar string.
233 : : *
234 : : * "len" is the length of "from" in bytes (not characters). The input
235 : : * string need not be zero-terminated, but conversion will stop early on a
236 : : * zero byte. The output is written to "to". "to" must be allocated to
237 : : * accommodate at least len + 1 pg_wchar's. A terminating 0 is always
238 : : * written. The return value is the number of pg_wchar's written, without
239 : : * the terminator.
240 : : *
241 : : * If a multibyte sequence at the end is cut off by len, it is silently
242 : : * dropped.
243 : : */
244 : : int (*mb2wchar_with_len) (const unsigned char *from, pg_wchar *to, int len);
245 : :
246 : : /*
247 : : * Convert a pg_wchar string to a multibyte string.
248 : : *
249 : : * "len" is the length of "from" in pg_wchar characters (not bytes). The
250 : : * input need not be terminated by a 0 pg_wchar, but conversion will stop
251 : : * early on a 0 pg_wchar. The output is written to "to". "to" must be
252 : : * allocated as at least len * maxmblen + 1 bytes. A terminating 0 byte
253 : : * is always written. The return value is the number of bytes written,
254 : : * without the terminator.
255 : : */
256 : : int (*wchar2mb_with_len) (const pg_wchar *from, unsigned char *to, int len);
257 : :
258 : : /*
259 : : * Get byte length of the first character of the input string.
260 : : *
261 : : * If the input is an empty string (*mbstr == '\0'), then 1 is returned.
262 : : * The return value is always between 1 and maxmblen inclusive.
263 : : *
264 : : * We expect that mblen does not need to examine more than the first byte
265 : : * of the character to discover the correct length. GB18030 is an
266 : : * exception; see pg_gb18030_mblen().
267 : : */
268 : : mblen_function mblen;
269 : :
270 : : /*
271 : : * Get the display width of the first character of the input string.
272 : : *
273 : : * The possible results are -1, 0, 1 and 2. Control characters return -1;
274 : : * callers can handle that specially. Unlike mblen, an empty string
275 : : * returns 0. (But the converse is not true: Zero-width characters also
276 : : * return 0.)
277 : : *
278 : : * Note: for the display output of psql to work properly, the return
279 : : * values of the dsplen functions must conform to the Unicode standard. In
280 : : * particular the NUL character is zero width and control characters are
281 : : * generally width -1. It is recommended that non-ASCII encodings refer
282 : : * their ASCII subset to the ASCII routines to ensure consistency.
283 : : */
284 : : int (*dsplen) (const unsigned char *mbstr);
285 : :
286 : : /*
287 : : * Verify multibyte character.
288 : : *
289 : : * The verifychar functions accept "mbstr", a pointer to the first byte of
290 : : * a string, and "len", the remaining length of the string. If there is a
291 : : * validly encoded character beginning at *mbstr, return its length in
292 : : * bytes; else return -1.
293 : : *
294 : : * The verifychar functions can assume that len > 0 and that *mbstr !=
295 : : * '\0', but they must test for and reject zeroes in any additional bytes
296 : : * of a multibyte character. Note that this definition allows the
297 : : * function for a single-byte encoding to be just "return 1".
298 : : */
299 : : mbverifychar_function mbverifychar;
300 : :
301 : : /*
302 : : * Verify multibyte string.
303 : : *
304 : : * The verifystr functions accept "mbstr", a pointer to a string and
305 : : * "len", the length of the string. They verify the whole string, and
306 : : * return the number of input bytes (<= len) that are valid. In other
307 : : * words, if the whole string is valid, verifystr returns "len", otherwise
308 : : * it returns the byte offset of the first invalid character. The
309 : : * verifystr functions must test for and reject zeroes in the input.
310 : : */
311 : : int (*mbverifystr) (const unsigned char *mbstr, int len);
312 : :
313 : : /*
314 : : * The maximum number of bytes a character in this encoding could occupy.
315 : : *
316 : : * This is used to size output buffers of conversion routines, so it must
317 : : * not be too small.
318 : : */
319 : : int maxmblen;
320 : : } pg_wchar_tbl;
321 : :
322 : : extern PGDLLIMPORT const pg_wchar_tbl pg_wchar_table[];
323 : :
324 : : /*
325 : : * Data structures for conversions between UTF-8 and other encodings
326 : : * (UtfToLocal() and LocalToUtf()). In these data structures, characters of
327 : : * either encoding are represented by uint32 words; hence we can only support
328 : : * characters up to 4 bytes long. For example, the byte sequence 0xC2 0x89
329 : : * would be represented by 0x0000C289, and 0xE8 0xA2 0xB4 by 0x00E8A2B4.
330 : : *
331 : : * There are three possible ways a character can be mapped:
332 : : *
333 : : * 1. Using a radix tree, from source to destination code.
334 : : * 2. Using a sorted array of source -> destination code pairs. This
335 : : * method is used for "combining" characters. There are so few of
336 : : * them that building a radix tree would be wasteful.
337 : : * 3. Using a conversion function.
338 : : */
339 : :
340 : : /*
341 : : * Radix tree for character conversion.
342 : : *
343 : : * Logically, this is actually four different radix trees, for 1-byte,
344 : : * 2-byte, 3-byte and 4-byte inputs. The 1-byte tree is a simple lookup
345 : : * table from source to target code. The 2-byte tree consists of two levels:
346 : : * one lookup table for the first byte, where the value in the lookup table
347 : : * points to a lookup table for the second byte. And so on.
348 : : *
349 : : * Physically, all the trees are stored in one big array, in 'chars16' or
350 : : * 'chars32', depending on the maximum value that needs to be represented. For
351 : : * each level in each tree, we also store lower and upper bound of allowed
352 : : * values - values outside those bounds are considered invalid, and are left
353 : : * out of the tables.
354 : : *
355 : : * In the intermediate levels of the trees, the values stored are offsets
356 : : * into the chars[16|32] array.
357 : : *
358 : : * In the beginning of the chars[16|32] array, there is always a number of
359 : : * zeros, so that you safely follow an index from an intermediate table
360 : : * without explicitly checking for a zero. Following a zero any number of
361 : : * times will always bring you to the dummy, all-zeros table in the
362 : : * beginning. This helps to shave some cycles when looking up values.
363 : : */
364 : : typedef struct
365 : : {
366 : : /*
367 : : * Array containing all the values. Only one of chars16 or chars32 is
368 : : * used, depending on how wide the values we need to represent are.
369 : : */
370 : : const uint16 *chars16;
371 : : const uint32 *chars32;
372 : :
373 : : /* Radix tree for 1-byte inputs */
374 : : uint32 b1root; /* offset of table in the chars[16|32] array */
375 : : uint8 b1_lower; /* min allowed value for a single byte input */
376 : : uint8 b1_upper; /* max allowed value for a single byte input */
377 : :
378 : : /* Radix tree for 2-byte inputs */
379 : : uint32 b2root; /* offset of 1st byte's table */
380 : : uint8 b2_1_lower; /* min/max allowed value for 1st input byte */
381 : : uint8 b2_1_upper;
382 : : uint8 b2_2_lower; /* min/max allowed value for 2nd input byte */
383 : : uint8 b2_2_upper;
384 : :
385 : : /* Radix tree for 3-byte inputs */
386 : : uint32 b3root; /* offset of 1st byte's table */
387 : : uint8 b3_1_lower; /* min/max allowed value for 1st input byte */
388 : : uint8 b3_1_upper;
389 : : uint8 b3_2_lower; /* min/max allowed value for 2nd input byte */
390 : : uint8 b3_2_upper;
391 : : uint8 b3_3_lower; /* min/max allowed value for 3rd input byte */
392 : : uint8 b3_3_upper;
393 : :
394 : : /* Radix tree for 4-byte inputs */
395 : : uint32 b4root; /* offset of 1st byte's table */
396 : : uint8 b4_1_lower; /* min/max allowed value for 1st input byte */
397 : : uint8 b4_1_upper;
398 : : uint8 b4_2_lower; /* min/max allowed value for 2nd input byte */
399 : : uint8 b4_2_upper;
400 : : uint8 b4_3_lower; /* min/max allowed value for 3rd input byte */
401 : : uint8 b4_3_upper;
402 : : uint8 b4_4_lower; /* min/max allowed value for 4th input byte */
403 : : uint8 b4_4_upper;
404 : :
405 : : } pg_mb_radix_tree;
406 : :
407 : : /*
408 : : * UTF-8 to local code conversion map (for combined characters)
409 : : */
410 : : typedef struct
411 : : {
412 : : uint32 utf1; /* UTF-8 code 1 */
413 : : uint32 utf2; /* UTF-8 code 2 */
414 : : uint32 code; /* local code */
415 : : } pg_utf_to_local_combined;
416 : :
417 : : /*
418 : : * local code to UTF-8 conversion map (for combined characters)
419 : : */
420 : : typedef struct
421 : : {
422 : : uint32 code; /* local code */
423 : : uint32 utf1; /* UTF-8 code 1 */
424 : : uint32 utf2; /* UTF-8 code 2 */
425 : : } pg_local_to_utf_combined;
426 : :
427 : : /*
428 : : * callback function for algorithmic encoding conversions (in either direction)
429 : : *
430 : : * if function returns zero, it does not know how to convert the code
431 : : */
432 : : typedef uint32 (*utf_local_conversion_func) (uint32 code);
433 : :
434 : : /*
435 : : * Support macro for encoding conversion functions to validate their
436 : : * arguments. (This could be made more compact if we included fmgr.h
437 : : * here, but we don't want to do that because this header file is also
438 : : * used by frontends.)
439 : : */
440 : : #define CHECK_ENCODING_CONVERSION_ARGS(srcencoding,destencoding) \
441 : : check_encoding_conversion_args(PG_GETARG_INT32(0), \
442 : : PG_GETARG_INT32(1), \
443 : : PG_GETARG_INT32(4), \
444 : : (srcencoding), \
445 : : (destencoding))
446 : :
447 : :
448 : : /*
449 : : * Some handy functions for Unicode-specific tests.
450 : : */
451 : : static inline bool
372 jdavis@postgresql.or 452 :CBC 1785 : is_valid_unicode_codepoint(char32_t c)
453 : : {
2435 tgl@sss.pgh.pa.us 454 [ + - + + ]: 1785 : return (c > 0 && c <= 0x10FFFF);
455 : : }
456 : :
457 : : static inline bool
372 jdavis@postgresql.or 458 : 1220 : is_utf16_surrogate_first(char32_t c)
459 : : {
2488 tgl@sss.pgh.pa.us 460 [ + + + + ]: 1220 : return (c >= 0xD800 && c <= 0xDBFF);
461 : : }
462 : :
463 : : static inline bool
372 jdavis@postgresql.or 464 : 1078 : is_utf16_surrogate_second(char32_t c)
465 : : {
2488 tgl@sss.pgh.pa.us 466 [ + + + + ]: 1078 : return (c >= 0xDC00 && c <= 0xDFFF);
467 : : }
468 : :
469 : : static inline char32_t
372 jdavis@postgresql.or 470 : 42 : surrogate_pair_to_codepoint(char16_t first, char16_t second)
471 : : {
2488 tgl@sss.pgh.pa.us 472 : 42 : return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
473 : : }
474 : :
475 : : /*
476 : : * Convert a UTF-8 character to a Unicode code point.
477 : : * This is a one-character version of pg_utf2wchar_with_len.
478 : : *
479 : : * No error checks here, c must point to a long-enough string.
480 : : */
481 : : static inline char32_t
960 jdavis@postgresql.or 482 : 30708293 : utf8_to_unicode(const unsigned char *c)
483 : : {
484 [ + + ]: 30708293 : if ((*c & 0x80) == 0)
372 485 : 30700980 : return (char32_t) c[0];
960 486 [ + + ]: 7313 : else if ((*c & 0xe0) == 0xc0)
372 487 : 6093 : return (char32_t) (((c[0] & 0x1f) << 6) |
960 488 : 6093 : (c[1] & 0x3f));
489 [ + + ]: 1220 : else if ((*c & 0xf0) == 0xe0)
372 490 : 1155 : return (char32_t) (((c[0] & 0x0f) << 12) |
960 491 : 1155 : ((c[1] & 0x3f) << 6) |
492 : 1155 : (c[2] & 0x3f));
493 [ + - ]: 65 : else if ((*c & 0xf8) == 0xf0)
372 494 : 65 : return (char32_t) (((c[0] & 0x07) << 18) |
960 495 : 65 : ((c[1] & 0x3f) << 12) |
496 : 65 : ((c[2] & 0x3f) << 6) |
497 : 65 : (c[3] & 0x3f));
498 : : else
499 : : /* that is an invalid code on purpose */
960 jdavis@postgresql.or 500 :UBC 0 : return 0xffffffff;
501 : : }
502 : :
503 : : /*
504 : : * Map a Unicode code point to UTF-8. utf8string must have at least
505 : : * unicode_utf8len(c) bytes available.
506 : : */
507 : : static inline unsigned char *
372 jdavis@postgresql.or 508 :CBC 8299277 : unicode_to_utf8(char32_t c, unsigned char *utf8string)
509 : : {
960 510 [ + + ]: 8299277 : if (c <= 0x7F)
511 : : {
512 : 8296812 : utf8string[0] = c;
513 : : }
514 [ + + ]: 2465 : else if (c <= 0x7FF)
515 : : {
516 : 1572 : utf8string[0] = 0xC0 | ((c >> 6) & 0x1F);
517 : 1572 : utf8string[1] = 0x80 | (c & 0x3F);
518 : : }
519 [ + + ]: 893 : else if (c <= 0xFFFF)
520 : : {
521 : 839 : utf8string[0] = 0xE0 | ((c >> 12) & 0x0F);
522 : 839 : utf8string[1] = 0x80 | ((c >> 6) & 0x3F);
523 : 839 : utf8string[2] = 0x80 | (c & 0x3F);
524 : : }
525 : : else
526 : : {
527 : 54 : utf8string[0] = 0xF0 | ((c >> 18) & 0x07);
528 : 54 : utf8string[1] = 0x80 | ((c >> 12) & 0x3F);
529 : 54 : utf8string[2] = 0x80 | ((c >> 6) & 0x3F);
530 : 54 : utf8string[3] = 0x80 | (c & 0x3F);
531 : : }
532 : :
533 : 8299277 : return utf8string;
534 : : }
535 : :
536 : : /*
537 : : * Number of bytes needed to represent the given char in UTF8.
538 : : */
539 : : static inline int
372 540 : 329581 : unicode_utf8len(char32_t c)
541 : : {
973 542 [ + + ]: 329581 : if (c <= 0x7F)
543 : 328277 : return 1;
544 [ + + ]: 1304 : else if (c <= 0x7FF)
545 : 984 : return 2;
546 [ + - ]: 320 : else if (c <= 0xFFFF)
547 : 320 : return 3;
548 : : else
973 jdavis@postgresql.or 549 :UBC 0 : return 4;
550 : : }
551 : :
552 : : /*
553 : : * The functions in this list are exported by libpq, and we need to be sure
554 : : * that we know which calls are satisfied by libpq and which are satisfied
555 : : * by static linkage to libpgcommon. (This is because we might be using a
556 : : * libpq.so that's of a different major version and has encoding IDs that
557 : : * differ from the current version's.) The nominal function names are what
558 : : * are actually used in and exported by libpq, while the names exported by
559 : : * libpgcommon.a and libpgcommon_srv.a end in "_private".
560 : : */
561 : : #if defined(USE_PRIVATE_ENCODING_FUNCS) || !defined(FRONTEND)
562 : : #define pg_char_to_encoding pg_char_to_encoding_private
563 : : #define pg_encoding_to_char pg_encoding_to_char_private
564 : : #define pg_valid_server_encoding pg_valid_server_encoding_private
565 : : #define pg_valid_server_encoding_id pg_valid_server_encoding_id_private
566 : : #define pg_utf_mblen pg_utf_mblen_private
567 : : #endif
568 : :
569 : : /*
570 : : * These functions are considered part of libpq's exported API and
571 : : * are also declared in libpq-fe.h.
572 : : */
573 : : extern int pg_char_to_encoding(const char *name);
574 : : extern const char *pg_encoding_to_char(int encoding);
575 : : extern int pg_valid_server_encoding_id(int encoding);
576 : :
577 : : /*
578 : : * These functions are available to frontend code that links with libpgcommon
579 : : * (in addition to the ones just above). The constant tables declared
580 : : * earlier in this file are also available from libpgcommon.
581 : : */
582 : : extern void pg_encoding_set_invalid(int encoding, char *dst);
583 : : extern int pg_encoding_mblen(int encoding, const char *mbstr);
584 : : extern int pg_encoding_mblen_or_incomplete(int encoding, const char *mbstr,
585 : : size_t remaining);
586 : : extern int pg_encoding_mblen_bounded(int encoding, const char *mbstr);
587 : : extern int pg_encoding_dsplen(int encoding, const char *mbstr);
588 : : extern int pg_encoding_verifymbchar(int encoding, const char *mbstr, int len);
589 : : extern int pg_encoding_verifymbstr(int encoding, const char *mbstr, int len);
590 : : extern int pg_encoding_max_length(int encoding);
591 : : extern int pg_valid_client_encoding(const char *name);
592 : : extern int pg_valid_server_encoding(const char *name);
593 : : extern bool is_encoding_supported_by_icu(int encoding);
594 : : extern const char *get_encoding_name_for_icu(int encoding);
595 : :
596 : : extern bool pg_utf8_islegal(const unsigned char *source, int length);
597 : : extern int pg_utf_mblen(const unsigned char *s);
598 : :
599 : : /*
600 : : * The remaining functions are backend-only.
601 : : */
602 : : extern int pg_mb2wchar(const char *from, pg_wchar *to);
603 : : extern int pg_mb2wchar_with_len(const char *from, pg_wchar *to, int len);
604 : : extern int pg_encoding_mb2wchar_with_len(int encoding,
605 : : const char *from, pg_wchar *to, int len);
606 : : extern int pg_wchar2mb(const pg_wchar *from, char *to);
607 : : extern int pg_wchar2mb_with_len(const pg_wchar *from, char *to, int len);
608 : : extern int pg_encoding_wchar2mb_with_len(int encoding,
609 : : const pg_wchar *from, char *to, int len);
610 : : extern int pg_char_and_wchar_strcmp(const char *s1, const pg_wchar *s2);
611 : : extern int pg_wchar_strncmp(const pg_wchar *s1, const pg_wchar *s2, size_t n);
612 : : extern int pg_char_and_wchar_strncmp(const char *s1, const pg_wchar *s2, size_t n);
613 : : extern size_t pg_wchar_strlen(const pg_wchar *str);
614 : : extern int pg_mblen_cstr(const char *mbstr);
615 : : extern int pg_mblen_range(const char *mbstr, const char *end);
616 : : extern int pg_mblen_with_len(const char *mbstr, int limit);
617 : : extern int pg_mblen_unbounded(const char *mbstr);
618 : :
619 : : /* deprecated */
620 : : extern int pg_mblen(const char *mbstr);
621 : :
622 : : extern int pg_dsplen(const char *mbstr);
623 : : extern int pg_mbstrlen(const char *mbstr);
624 : : extern int pg_mbstrlen_with_len(const char *mbstr, int limit);
625 : : extern int pg_mbcliplen(const char *mbstr, int len, int limit);
626 : : extern int pg_encoding_mbcliplen(int encoding, const char *mbstr,
627 : : int len, int limit);
628 : : extern int pg_mbcharcliplen(const char *mbstr, int len, int limit);
629 : : extern int pg_database_encoding_max_length(void);
630 : : typedef bool (*mbcharacter_incrementer) (unsigned char *mbstr, int len);
631 : : extern mbcharacter_incrementer pg_database_encoding_character_incrementer(void);
632 : :
633 : : extern int PrepareClientEncoding(int encoding);
634 : : extern int SetClientEncoding(int encoding);
635 : : extern void InitializeClientEncoding(void);
636 : : extern int pg_get_client_encoding(void);
637 : : extern const char *pg_get_client_encoding_name(void);
638 : :
639 : : extern void SetDatabaseEncoding(int encoding);
640 : : extern int GetDatabaseEncoding(void);
641 : : extern const char *GetDatabaseEncodingName(void);
642 : : extern void SetMessageEncoding(int encoding);
643 : : extern int GetMessageEncoding(void);
644 : :
645 : : #ifdef ENABLE_NLS
646 : : extern int pg_bind_textdomain_codeset(const char *domainname);
647 : : #endif
648 : :
649 : : extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len,
650 : : int src_encoding,
651 : : int dest_encoding);
652 : : extern int pg_do_encoding_conversion_buf(Oid proc,
653 : : int src_encoding,
654 : : int dest_encoding,
655 : : unsigned char *src, int srclen,
656 : : unsigned char *dest, int destlen,
657 : : bool noError);
658 : :
659 : : extern char *pg_client_to_server(const char *s, int len);
660 : : extern char *pg_server_to_client(const char *s, int len);
661 : : extern char *pg_any_to_server(const char *s, int len, int encoding);
662 : : extern char *pg_server_to_any(const char *s, int len, int encoding);
663 : :
664 : : extern void pg_unicode_to_server(char32_t c, unsigned char *s);
665 : : extern bool pg_unicode_to_server_noerror(char32_t c, unsigned char *s);
666 : :
667 : : extern unsigned short BIG5toCNS(unsigned short big5, unsigned char *lc);
668 : : extern unsigned short CNStoBIG5(unsigned short cns, unsigned char lc);
669 : :
670 : : extern int UtfToLocal(const unsigned char *utf, int len,
671 : : unsigned char *iso,
672 : : const pg_mb_radix_tree *map,
673 : : const pg_utf_to_local_combined *cmap, int cmapsize,
674 : : utf_local_conversion_func conv_func,
675 : : int encoding, bool noError);
676 : : extern int LocalToUtf(const unsigned char *iso, int len,
677 : : unsigned char *utf,
678 : : const pg_mb_radix_tree *map,
679 : : const pg_local_to_utf_combined *cmap, int cmapsize,
680 : : utf_local_conversion_func conv_func,
681 : : int encoding, bool noError);
682 : :
683 : : extern bool pg_verifymbstr(const char *mbstr, int len, bool noError);
684 : : extern bool pg_verify_mbstr(int encoding, const char *mbstr, int len,
685 : : bool noError);
686 : : extern int pg_verify_mbstr_len(int encoding, const char *mbstr, int len,
687 : : bool noError);
688 : :
689 : : extern void check_encoding_conversion_args(int src_encoding,
690 : : int dest_encoding,
691 : : int len,
692 : : int expected_src_encoding,
693 : : int expected_dest_encoding);
694 : :
695 : : pg_noreturn extern void report_invalid_encoding(int encoding, const char *mbstr, int len);
696 : : pg_noreturn extern void report_untranslatable_char(int src_encoding, int dest_encoding,
697 : : const char *mbstr, int len);
698 : :
699 : : extern int local2local(const unsigned char *l, unsigned char *p, int len,
700 : : int src_encoding, int dest_encoding,
701 : : const unsigned char *tab, bool noError);
702 : :
703 : : #ifdef WIN32
704 : : extern WCHAR *pgwin32_message_to_UTF16(const char *str, int len, int *utf16len);
705 : : #endif
706 : :
707 : : #endif /* PG_WCHAR_H */
|
