ilusha/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2026-09-29 12:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

bpo-39219: Fix SyntaxError attributes in the tokenizer. (GH-17828)

Browse source 
* Always set the text attribute.
* Correct the offset attribute for non-ascii sources.
This commit is contained in:
Serhiy Storchaka 2020-02-12 12:17:00 +02:00 • committed by GitHub
parent f4f445b693
commit 0cc6b5e559
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

36
Parser/tokenizer.c
View file

@ -1,6 +1,7 @@
/* Tokenizer implementation */
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include <ctype.h>
@ -1034,17 +1035,44 @@ tok_backup(struct tok_state *tok, int c)
static int
syntaxerror(struct tok_state *tok, const char *format, ...)
{
PyObject *errmsg, *errtext, *args;
va_list vargs;
#ifdef HAVE_STDARG_PROTOTYPES
va_start(vargs, format);
#else
va_start(vargs);
#endif
PyErr_FormatV(PyExc_SyntaxError, format, vargs);
errmsg = PyUnicode_FromFormatV(format, vargs);
va_end(vargs);
PyErr_SyntaxLocationObject(tok->filename,
tok->lineno,
(int)(tok->cur - tok->line_start));
if (!errmsg) {
goto error;
}
errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start,
"replace");
if (!errtext) {
goto error;
}
int offset = (int)PyUnicode_GET_LENGTH(errtext);
Py_ssize_t line_len = strcspn(tok->line_start, "\n");
if (line_len != tok->cur - tok->line_start) {
Py_DECREF(errtext);
errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len,
"replace");
}
if (!errtext) {
goto error;
}
args = Py_BuildValue("(O(OiiN))", errmsg,
tok->filename, tok->lineno, offset, errtext);
if (args) {
PyErr_SetObject(PyExc_SyntaxError, args);
Py_DECREF(args);
}
error:
Py_XDECREF(errmsg);
tok->done = E_ERROR;
return ERRORTOKEN;
}