ilusha/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2026-09-29 12:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error.

Browse source
This commit is contained in:
Victor Stinner 2011-04-05 01:48:03 +02:00
parent 7f2fee3640
commit fe7c5b5bdf
6 changed files with 43 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

41
Parser/tokenizer.c
View file

@ -1690,17 +1690,18 @@ PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
return result;
}
/* Get -*- encoding -*- from a Python file.
/* Get the encoding of a Python file. Check for the coding cookie and check if
the file starts with a BOM.
PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
the first or second line of the file (in which case the encoding
should be assumed to be PyUnicode_GetDefaultEncoding()).
PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
encoding in the first or second line of the file (in which case the encoding
should be assumed to be UTF-8).
The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed
by the caller. */
The char * returned is malloc'ed via PyMem_MALLOC() and thus must be freed
by the caller.
*/
char *
PyTokenizer_FindEncoding(int fd)
PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
{
struct tok_state *tok;
FILE *fp;
@ -1720,9 +1721,18 @@ PyTokenizer_FindEncoding(int fd)
return NULL;
}
#ifndef PGEN
tok->filename = PyUnicode_FromString("<string>");
if (tok->filename == NULL)
goto error;
if (filename != NULL) {
Py_INCREF(filename);
tok->filename = filename;
}
else {
tok->filename = PyUnicode_FromString("<string>");
if (tok->filename == NULL) {
fclose(fp);
PyTokenizer_Free(tok);
return encoding;
}
}
#endif
while (tok->lineno < 2 && tok->done == E_OK) {
PyTokenizer_Get(tok, &p_start, &p_end);
@ -1733,13 +1743,16 @@ PyTokenizer_FindEncoding(int fd)
if (encoding)
strcpy(encoding, tok->encoding);
}
#ifndef PGEN
error:
#endif
PyTokenizer_Free(tok);
return encoding;
}
char *
PyTokenizer_FindEncoding(int fd)
{
return PyTokenizer_FindEncodingFilename(fd, NULL);
}
#ifdef Py_DEBUG
void

1
Parser/tokenizer.h
View file

@ -75,7 +75,6 @@ extern void PyTokenizer_Free(struct tok_state *);
extern int PyTokenizer_Get(struct tok_state *, char **, char **);
extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
int len, int *offset);
extern char * PyTokenizer_FindEncoding(int);
#ifdef __cplusplus
}