ilusha/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2026-09-29 12:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Encode surrogates in UTF-8 even for a wide Py_UNICODE.

Browse source 
Implement sys.maxunicode.
Explicitly wrap around upper/lower computations for wide Py_UNICODE.
When decoding large characters with UTF-8, represent expected test
results using the \U notation.
This commit is contained in:
Martin v. Löwis 2001-06-27 06:28:56 +00:00
parent 236d8b7974
commit ce9b5a55e1
5 changed files with 47 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

33
Objects/unicodectype.c
View file

@ -59,14 +59,21 @@ int _PyUnicode_IsLinebreak(register const Py_UNICODE ch)
/* Returns the titlecase Unicode characters corresponding to ch or just
ch if no titlecase mapping is known. */
Py_UNICODE _PyUnicode_ToTitlecase(register const Py_UNICODE ch)
Py_UNICODE _PyUnicode_ToTitlecase(register Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
if (ctype->title)
return ch + ctype->title;
ch += ctype->title;
else
ch += ctype->upper;
return ch + ctype->upper;
#ifdef USE_UCS4_STORAGE
/* The database assumes that the values wrap around at 0x10000. */
if (ch > 0x10000)
ch -= 0x10000;
#endif
return ch;
}
/* Returns 1 for Unicode characters having the category 'Lt', 0
@ -348,21 +355,33 @@ int _PyUnicode_IsUppercase(register const Py_UNICODE ch)
/* Returns the uppercase Unicode characters corresponding to ch or just
ch if no uppercase mapping is known. */
Py_UNICODE _PyUnicode_ToUppercase(register const Py_UNICODE ch)
Py_UNICODE _PyUnicode_ToUppercase(register Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return ch + ctype->upper;
ch += ctype->upper;
#ifdef USE_UCS4_STORAGE
/* The database assumes that the values wrap around at 0x10000. */
if (ch > 0x10000)
ch -= 0x10000;
#endif
return ch;
}
/* Returns the lowercase Unicode characters corresponding to ch or just
ch if no lowercase mapping is known. */
Py_UNICODE _PyUnicode_ToLowercase(register const Py_UNICODE ch)
Py_UNICODE _PyUnicode_ToLowercase(register Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return ch + ctype->lower;
ch += ctype->lower;
#ifdef USE_UCS4_STORAGE
/* The database assumes that the values wrap around at 0x10000. */
if (ch > 0x10000)
ch -= 0x10000;
#endif
return ch;
}
/* Returns 1 for Unicode characters having the category 'Ll', 'Lu', 'Lt',

19
Objects/unicodeobject.c
View file

@ -103,6 +103,18 @@ static PyUnicodeObject *unicode_latin1[256];
*/
static char unicode_default_encoding[100];
Py_UNICODE
PyUnicode_GetMax()
{
#ifdef USE_UCS4_STORAGE
return 0x10FFFF;
#else
/* This is actually an illegal character, so it should
not be passed to unichr. */
return 0xFFFF;
#endif
}
/* --- Unicode Object ----------------------------------------------------- */
static
@ -884,12 +896,6 @@ PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,
cbWritten += 2;
}
else if (ch < 0x10000) {
#if Py_UNICODE_SIZE == 4
*p++ = 0xe0 | (ch>>12);
*p++ = 0x80 | ((ch>>6) & 0x3f);
*p++ = 0x80 | (ch & 0x3f);
cbWritten += 3;
#else
/* Check for high surrogate */
if (0xD800 <= ch && ch <= 0xDBFF) {
if (i != size) {
@ -920,7 +926,6 @@ PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,
}
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f));
#endif
} else {
*p++ = 0xf0 | (ch>>18);
*p++ = 0x80 | ((ch>>12) & 0x3f);