ilusha/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2026-09-29 12:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge p3yk branch with the trunk up to revision 45595. This breaks a fair

Browse source 
number of tests, all because of the codecs/_multibytecodecs issue described
here (it's not a Py3K issue, just something Py3K discovers):
http://mail.python.org/pipermail/python-dev/2006-April/064051.html

Hye-Shik Chang promised to look for a fix, so no need to fix it here. The
tests that are expected to break are:

test_codecencodings_cn
test_codecencodings_hk
test_codecencodings_jp
test_codecencodings_kr
test_codecencodings_tw
test_codecs
test_multibytecodec

This merge fixes an actual test failure (test_weakref) in this branch,
though, so I believe merging is the right thing to do anyway.
This commit is contained in:
Thomas Wouters 2006-04-21 10:40:58 +00:00
parent 9ada3d6e29
commit 49fd7fa443
640 changed files with 52240 additions and 18408 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Parser/Python.asdl
View file

@ -98,8 +98,11 @@ module Python version "$Revision$"
comprehension = (expr target, expr iter, expr* ifs)
-- not sure what to call the first argument for raise and except
excepthandler = (expr? type, expr? name, stmt* body)
-- TODO(jhylton): Figure out if there is a better way to handle
-- lineno and col_offset fields, particularly when
-- ast is exposed to Python.
excepthandler = (expr? type, expr? name, stmt* body, int lineno,
int col_offset)
arguments = (expr* args, identifier? vararg,
identifier? kwarg, expr* defaults)

94
Parser/asdl_c.py
View file

@ -155,8 +155,10 @@ class StructVisitor(EmitVisitor):
type = sum.types[i]
enum.append("%s_kind=%d" % (type.name, i + 1))
emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")
emit("struct _%(name)s {")
emit("enum { " + ", ".join(enum) + " } kind;", depth + 1)
emit("enum _%(name)s_kind kind;", depth + 1)
emit("union {", depth + 1)
for t in sum.types:
self.visit(t, depth + 2)
@ -186,7 +188,10 @@ class StructVisitor(EmitVisitor):
ctype = get_c_type(field.type)
name = field.name
if field.seq:
self.emit("asdl_seq *%(name)s;" % locals(), depth)
if field.type.value in ('cmpop',):
self.emit("asdl_int_seq *%(name)s;" % locals(), depth)
else:
self.emit("asdl_seq *%(name)s;" % locals(), depth)
else:
self.emit("%(ctype)s %(name)s;" % locals(), depth)
@ -232,7 +237,10 @@ class PrototypeVisitor(EmitVisitor):
name = f.name
# XXX should extend get_c_type() to handle this
if f.seq:
ctype = "asdl_seq *"
if f.type.value in ('cmpop',):
ctype = "asdl_int_seq *"
else:
ctype = "asdl_seq *"
else:
ctype = get_c_type(f.type)
args.append((ctype, name, f.opt or f.seq))
@ -276,7 +284,7 @@ class FunctionVisitor(PrototypeVisitor):
emit("%s p;" % ctype, 1)
for argtype, argname, opt in args:
# XXX hack alert: false is allowed for a bool
if not opt and not argtype == "bool":
if not opt and not (argtype == "bool" or argtype == "int"):
emit("if (!%s) {" % argname, 1)
emit("PyErr_SetString(PyExc_ValueError,", 2)
msg = "field %s is required for %s" % (argname, name)
@ -413,10 +421,10 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
{
int i;
int i, result;
PyObject *s, *l = PyList_New(num_fields);
if (!l) return 0;
for(i=0; i < num_fields; i++) {
for(i = 0; i < num_fields; i++) {
s = PyString_FromString(attrs[i]);
if (!s) {
Py_DECREF(l);
@ -424,7 +432,9 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
}
PyList_SET_ITEM(l, i, s);
}
return PyObject_SetAttrString((PyObject*)type, "_attributes", l) >=0;
result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0;
Py_DECREF(l);
return result;
}
static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
@ -465,9 +475,9 @@ static PyObject* ast2obj_int(bool b)
}
""", 0, reflow=False)
self.emit("static int initialized;", 0)
self.emit("static int init_types(void)",0)
self.emit("{", 0)
self.emit("static int initialized;", 1)
self.emit("if (initialized) return 1;", 1)
self.emit('AST_type = make_type("AST", &PyBaseObject_Type, NULL, 0);', 1)
for dfn in mod.dfns:
@ -543,7 +553,7 @@ class ASTModuleVisitor(PickleVisitor):
self.addObj(cons.name)
def addObj(self, name):
self.emit('if(PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
@ -677,8 +687,8 @@ class ObjVisitor(PickleVisitor):
self.emit("if (!value) goto failed;", depth+1)
self.emit("for(i = 0; i < n; i++)", depth+1)
# This cannot fail, so no need for error handling
self.emit("PyList_SET_ITEM(value, i, ast2obj_%s((%s_ty)asdl_seq_GET(%s, i)));" %
(field.type, field.type, value), depth+2, reflow=False)
self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
depth+2, reflow=False)
self.emit("}", depth)
else:
self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
@ -716,39 +726,35 @@ def main(srcfile):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
else:
p = "%s-ast.h" % mod.name
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "asdl.h"\n'
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
f.close()
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "asdl.h"\n'
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
f.close()
if SRC_DIR:
p = "%s/%s-ast.c" % (SRC_DIR, mod.name)
else:
p = "%s-ast.c" % mod.name
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "Python.h"'
print >> f, '#include "%s-ast.h"' % mod.name
print >> f
print >>f, "static PyTypeObject* AST_type;"
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "Python.h"'
print >> f, '#include "%s-ast.h"' % mod.name
print >> f
print >>f, "static PyTypeObject* AST_type;"
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
if __name__ == "__main__":
import sys
@ -757,6 +763,9 @@ if __name__ == "__main__":
INC_DIR = ''
SRC_DIR = ''
opts, args = getopt.getopt(sys.argv[1:], "h:c:")
if len(opts) != 1:
print "Must specify exactly one output file"
sys.exit(1)
for o, v in opts:
if o == '-h':
INC_DIR = v
@ -764,4 +773,5 @@ if __name__ == "__main__":
SRC_DIR = v
if len(args) != 1:
print "Must specify single input file"
sys.exit(1)
main(args[0])

4
Parser/bitset.c
View file

@ -8,7 +8,7 @@ bitset
newbitset(int nbits)
{
int nbytes = NBYTES(nbits);
bitset ss = PyMem_NEW(BYTE, nbytes);
bitset ss = (char *)PyObject_MALLOC(sizeof(BYTE) * nbytes);
if (ss == NULL)
Py_FatalError("no mem for bitset");
@ -22,7 +22,7 @@ newbitset(int nbits)
void
delbitset(bitset ss)
{
PyMem_DEL(ss);
PyObject_FREE(ss);
}
int

7
Parser/firstsets.c
View file

@ -59,7 +59,7 @@ calcfirstset(grammar *g, dfa *d)
nbits = g->g_ll.ll_nlabels;
result = newbitset(nbits);
sym = PyMem_NEW(int, 1);
sym = (int *)PyObject_MALLOC(sizeof(int));
if (sym == NULL)
Py_FatalError("no mem for new sym in calcfirstset");
nsyms = 1;
@ -73,7 +73,8 @@ calcfirstset(grammar *g, dfa *d)
break;
}
if (j >= nsyms) { /* New label */
PyMem_RESIZE(sym, int, nsyms + 1);
sym = (int *)PyObject_REALLOC(sym,
sizeof(int) * (nsyms + 1));
if (sym == NULL)
Py_FatalError(
"no mem to resize sym in calcfirstset");
@ -108,5 +109,5 @@ calcfirstset(grammar *g, dfa *d)
printf(" }\n");
}
PyMem_FREE(sym);
PyObject_FREE(sym);
}

15
Parser/grammar.c
View file

@ -20,7 +20,7 @@ newgrammar(int start)
{
grammar *g;
g = PyMem_NEW(grammar, 1);
g = (grammar *)PyObject_MALLOC(sizeof(grammar));
if (g == NULL)
Py_FatalError("no mem for new grammar");
g->g_ndfas = 0;
@ -37,7 +37,8 @@ adddfa(grammar *g, int type, char *name)
{
dfa *d;
PyMem_RESIZE(g->g_dfa, dfa, g->g_ndfas + 1);
g->g_dfa = (dfa *)PyObject_REALLOC(g->g_dfa,
sizeof(dfa) * (g->g_ndfas + 1));
if (g->g_dfa == NULL)
Py_FatalError("no mem to resize dfa in adddfa");
d = &g->g_dfa[g->g_ndfas++];
@ -55,7 +56,8 @@ addstate(dfa *d)
{
state *s;
PyMem_RESIZE(d->d_state, state, d->d_nstates + 1);
d->d_state = (state *)PyObject_REALLOC(d->d_state,
sizeof(state) * (d->d_nstates + 1));
if (d->d_state == NULL)
Py_FatalError("no mem to resize state in addstate");
s = &d->d_state[d->d_nstates++];
@ -78,7 +80,7 @@ addarc(dfa *d, int from, int to, int lbl)
assert(0 <= to && to < d->d_nstates);
s = &d->d_state[from];
PyMem_RESIZE(s->s_arc, arc, s->s_narcs + 1);
s->s_arc = (arc *)PyObject_REALLOC(s->s_arc, sizeof(arc) * (s->s_narcs + 1));
if (s->s_arc == NULL)
Py_FatalError("no mem to resize arc list in addarc");
a = &s->s_arc[s->s_narcs++];
@ -97,7 +99,8 @@ addlabel(labellist *ll, int type, char *str)
strcmp(ll->ll_label[i].lb_str, str) == 0)
return i;
}
PyMem_RESIZE(ll->ll_label, label, ll->ll_nlabels + 1);
ll->ll_label = (label *)PyObject_REALLOC(ll->ll_label,
sizeof(label) * (ll->ll_nlabels + 1));
if (ll->ll_label == NULL)
Py_FatalError("no mem to resize labellist in addlabel");
lb = &ll->ll_label[ll->ll_nlabels++];
@ -195,7 +198,7 @@ translabel(grammar *g, label *lb)
name_len = p - src;
else
name_len = strlen(src);
dest = malloc(name_len + 1);
dest = (char *)malloc(name_len + 1);
strncpy(dest, src, name_len);
dest[name_len] = '\0';
free(lb->lb_str);

6
Parser/myreadline.c
View file

@ -111,7 +111,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
size_t n;
char *p;
n = 100;
if ((p = PyMem_MALLOC(n)) == NULL)
if ((p = (char *)PyMem_MALLOC(n)) == NULL)
return NULL;
fflush(sys_stdout);
#ifndef RISCOS
@ -141,7 +141,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
n = strlen(p);
while (n > 0 && p[n-1] != '\n') {
size_t incr = n+2;
p = PyMem_REALLOC(p, n + incr);
p = (char *)PyMem_REALLOC(p, n + incr);
if (p == NULL)
return NULL;
if (incr > INT_MAX) {
@ -151,7 +151,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
break;
n += strlen(p+n);
}
return PyMem_REALLOC(p, n+1);
return (char *)PyMem_REALLOC(p, n+1);
}

2
Parser/node.c
View file

@ -62,7 +62,7 @@ fancy_roundup(int n)
* Win98).
*
* In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
* reported that, with this scheme, 89% of PyMem_RESIZE calls in
* reported that, with this scheme, 89% of PyObject_REALLOC calls in
* PyNode_AddChild passed 1 for the size, and 9% passed 4. So this usually
* wastes very little memory, but is very effective at sidestepping
* platform-realloc disasters on vulnernable platforms.

6
Parser/parser.c
View file

@ -75,7 +75,7 @@ PyParser_New(grammar *g, int start)
if (!g->g_accel)
PyGrammar_AddAccelerators(g);
ps = PyMem_NEW(parser_state, 1);
ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state));
if (ps == NULL)
return NULL;
ps->p_grammar = g;
@ -84,7 +84,7 @@ PyParser_New(grammar *g, int start)
#endif
ps->p_tree = PyNode_New(start);
if (ps->p_tree == NULL) {
PyMem_DEL(ps);
PyMem_FREE(ps);
return NULL;
}
s_reset(&ps->p_stack);
@ -98,7 +98,7 @@ PyParser_Delete(parser_state *ps)
/* NB If you want to save the parse tree,
you must set p_tree to NULL before calling delparser! */
PyNode_Free(ps->p_tree);
PyMem_DEL(ps);
PyMem_FREE(ps);
}

27
Parser/pgen.c
View file

@ -49,7 +49,8 @@ addnfastate(nfa *nf)
{
nfastate *st;
PyMem_RESIZE(nf->nf_state, nfastate, nf->nf_nstates + 1);
nf->nf_state = (nfastate *)PyObject_REALLOC(nf->nf_state,
sizeof(nfastate) * (nf->nf_nstates + 1));
if (nf->nf_state == NULL)
Py_FatalError("out of mem");
st = &nf->nf_state[nf->nf_nstates++];
@ -65,7 +66,8 @@ addnfaarc(nfa *nf, int from, int to, int lbl)
nfaarc *ar;
st = &nf->nf_state[from];
PyMem_RESIZE(st->st_arc, nfaarc, st->st_narcs + 1);
st->st_arc = (nfaarc *)PyObject_REALLOC(st->st_arc,
sizeof(nfaarc) * (st->st_narcs + 1));
if (st->st_arc == NULL)
Py_FatalError("out of mem");
ar = &st->st_arc[st->st_narcs++];
@ -79,7 +81,7 @@ newnfa(char *name)
nfa *nf;
static int type = NT_OFFSET; /* All types will be disjunct */
nf = PyMem_NEW(nfa, 1);
nf = (nfa *)PyObject_MALLOC(sizeof(nfa));
if (nf == NULL)
Py_FatalError("no mem for new nfa");
nf->nf_type = type++;
@ -104,7 +106,7 @@ newnfagrammar(void)
{
nfagrammar *gr;
gr = PyMem_NEW(nfagrammar, 1);
gr = (nfagrammar *)PyObject_MALLOC(sizeof(nfagrammar));
if (gr == NULL)
Py_FatalError("no mem for new nfa grammar");
gr->gr_nnfas = 0;
@ -121,7 +123,8 @@ addnfa(nfagrammar *gr, char *name)
nfa *nf;
nf = newnfa(name);
PyMem_RESIZE(gr->gr_nfa, nfa *, gr->gr_nnfas + 1);
gr->gr_nfa = (nfa **)PyObject_REALLOC(gr->gr_nfa,
sizeof(nfa) * (gr->gr_nnfas + 1));
if (gr->gr_nfa == NULL)
Py_FatalError("out of mem");
gr->gr_nfa[gr->gr_nnfas++] = nf;
@ -361,7 +364,7 @@ typedef struct _ss_arc {
typedef struct _ss_state {
bitset ss_ss;
int ss_narcs;
ss_arc *ss_arc;
struct _ss_arc *ss_arc;
int ss_deleted;
int ss_finish;
int ss_rename;
@ -392,7 +395,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
ss = newbitset(nbits);
addclosure(ss, nf, nf->nf_start);
xx_state = PyMem_NEW(ss_state, 1);
xx_state = (ss_state *)PyObject_MALLOC(sizeof(ss_state));
if (xx_state == NULL)
Py_FatalError("no mem for xx_state in makedfa");
xx_nstates = 1;
@ -411,6 +414,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
/* For each unmarked state... */
for (istate = 0; istate < xx_nstates; ++istate) {
size_t size;
yy = &xx_state[istate];
ss = yy->ss_ss;
/* For all its states... */
@ -430,8 +434,9 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
goto found;
}
/* Add new arc for this state */
PyMem_RESIZE(yy->ss_arc, ss_arc,
yy->ss_narcs + 1);
size = sizeof(ss_arc) * (yy->ss_narcs + 1);
yy->ss_arc = (ss_arc *)PyObject_REALLOC(
yy->ss_arc, size);
if (yy->ss_arc == NULL)
Py_FatalError("out of mem");
zz = &yy->ss_arc[yy->ss_narcs++];
@ -453,7 +458,9 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
goto done;
}
}
PyMem_RESIZE(xx_state, ss_state, xx_nstates + 1);
size = sizeof(ss_state) * (xx_nstates + 1);
xx_state = (ss_state *)PyObject_REALLOC(xx_state,
size);
if (xx_state == NULL)
Py_FatalError("out of mem");
zz->sa_arrow = xx_nstates;

6
Parser/pgenmain.c
View file

@ -104,7 +104,7 @@ getgrammar(char *filename)
putc(' ', stderr);
}
fprintf(stderr, "^\n");
PyMem_DEL(err.text);
PyObject_FREE(err.text);
}
Py_Exit(1);
}
@ -136,7 +136,7 @@ char *
PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
{
size_t n = 1000;
char *p = PyMem_MALLOC(n);
char *p = (char *)PyMem_MALLOC(n);
char *q;
if (p == NULL)
return NULL;
@ -149,7 +149,7 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
n = strlen(p);
if (n > 0 && p[n-1] != '\n')
p[n-1] = '\n';
return PyMem_REALLOC(p, n+1);
return (char *)PyMem_REALLOC(p, n+1);
}
/* No-nonsense fgets */

110
Parser/tokenizer.c
View file

@ -105,7 +105,8 @@ char *_PyParser_TokenNames[] = {
static struct tok_state *
tok_new(void)
{
struct tok_state *tok = PyMem_NEW(struct tok_state, 1);
struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
sizeof(struct tok_state));
if (tok == NULL)
return NULL;
tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
@ -163,7 +164,7 @@ error_ret(struct tok_state *tok) /* XXX */
{
tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
PyMem_DEL(tok->buf);
PyMem_FREE(tok->buf);
tok->buf = NULL;
return NULL; /* as if it were EOF */
}
@ -171,7 +172,7 @@ error_ret(struct tok_state *tok) /* XXX */
static char *
new_string(const char *s, Py_ssize_t len)
{
char* result = PyMem_NEW(char, len + 1);
char* result = (char *)PyMem_MALLOC(len + 1);
if (result != NULL) {
memcpy(result, s, len);
result[len] = '\0';
@ -236,7 +237,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
char* r = new_string(begin, t - begin);
char* q = get_normal_name(r);
if (r != q) {
PyMem_DEL(r);
PyMem_FREE(r);
r = new_string(q, strlen(q));
}
return r;
@ -277,18 +278,18 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
tok->decoding_state = -1;
}
else
PyMem_DEL(cs);
PyMem_FREE(cs);
#else
/* Without Unicode support, we cannot
process the coding spec. Since there
won't be any Unicode literals, that
won't matter. */
PyMem_DEL(cs);
PyMem_FREE(cs);
#endif
}
} else { /* then, compare cs with BOM */
r = (strcmp(tok->encoding, cs) == 0);
PyMem_DEL(cs);
PyMem_FREE(cs);
}
}
if (!r) {
@ -334,7 +335,7 @@ check_bom(int get_char(struct tok_state *),
return 1;
}
if (tok->encoding != NULL)
PyMem_DEL(tok->encoding);
PyMem_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1;
NON_BOM:
@ -345,7 +346,7 @@ check_bom(int get_char(struct tok_state *),
/* Read a line of text from TOK into S, using the stream in TOK.
Return NULL on failure, else S.
On entry, tok->decoding_buffer will be one of:
1) NULL: need to call tok->decoding_readline to get a new line
2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
@ -354,7 +355,7 @@ check_bom(int get_char(struct tok_state *),
(in the s buffer) to copy entire contents of the line read
by tok->decoding_readline. tok->decoding_buffer has the overflow.
In this case, fp_readl is called in a loop (with an expanded buffer)
until the buffer ends with a '\n' (or until the end of the file is
until the buffer ends with a '\n' (or until the end of the file is
reached): see tok_nextc and its calls to decoding_fgets.
*/
@ -470,7 +471,7 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
break;
} else if (tok->decoding_state > 0) {
/* We want a 'raw' read. */
line = Py_UniversalNewlineFgets(s, size,
line = Py_UniversalNewlineFgets(s, size,
tok->fp, NULL);
break;
} else {
@ -502,11 +503,11 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
char buf[500];
/* Need to add 1 to the line number, since this line
has not been counted, yet. */
sprintf(buf,
sprintf(buf,
"Non-ASCII character '\\x%.2x' "
"in file %.200s on line %i, "
"but no encoding declared; "
"see http://www.python.org/peps/pep-0263.html for details",
"see http://www.python.org/peps/pep-0263.html for details",
badchar, tok->filename, tok->lineno + 1);
PyErr_SetString(PyExc_SyntaxError, buf);
return error_ret(tok);
@ -537,13 +538,15 @@ decoding_feof(struct tok_state *tok)
/* Fetch a byte from TOK, using the string buffer. */
static int buf_getc(struct tok_state *tok) {
static int
buf_getc(struct tok_state *tok) {
return Py_CHARMASK(*tok->str++);
}
/* Unfetch a byte from TOK, using the string buffer. */
static void buf_ungetc(int c, struct tok_state *tok) {
static void
buf_ungetc(int c, struct tok_state *tok) {
tok->str--;
assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
}
@ -551,7 +554,8 @@ static void buf_ungetc(int c, struct tok_state *tok) {
/* Set the readline function for TOK to ENC. For the string-based
tokenizer, this means to just record the encoding. */
static int buf_setreadl(struct tok_state *tok, const char* enc) {
static int
buf_setreadl(struct tok_state *tok, const char* enc) {
tok->enc = enc;
return 1;
}
@ -653,7 +657,7 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = PyMem_NEW(char, BUFSIZ)) == NULL) {
if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
@ -672,14 +676,14 @@ void
PyTokenizer_Free(struct tok_state *tok)
{
if (tok->encoding != NULL)
PyMem_DEL(tok->encoding);
PyMem_FREE(tok->encoding);
#ifndef PGEN
Py_XDECREF(tok->decoding_readline);
Py_XDECREF(tok->decoding_buffer);
#endif
if (tok->fp != NULL && tok->buf != NULL)
PyMem_DEL(tok->buf);
PyMem_DEL(tok);
PyMem_FREE(tok->buf);
PyMem_FREE(tok);
}
#if !defined(PGEN) && defined(Py_USING_UNICODE)
@ -711,7 +715,9 @@ tok_stdin_decode(struct tok_state *tok, char **inp)
if (utf8 == NULL)
goto error_clear;
converted = new_string(PyString_AsString(utf8), PyString_Size(utf8));
assert(PyString_Check(utf8));
converted = new_string(PyString_AS_STRING(utf8),
PyString_GET_SIZE(utf8));
Py_DECREF(utf8);
if (converted == NULL)
goto error_nomem;
@ -719,7 +725,7 @@ tok_stdin_decode(struct tok_state *tok, char **inp)
PyMem_FREE(*inp);
*inp = converted;
if (tok->encoding != NULL)
PyMem_DEL(tok->encoding);
PyMem_FREE(tok->encoding);
tok->encoding = new_string(encoding, strlen(encoding));
if (tok->encoding == NULL)
goto error_nomem;
@ -770,38 +776,38 @@ tok_nextc(register struct tok_state *tok)
return Py_CHARMASK(*tok->cur++);
}
if (tok->prompt != NULL) {
char *new = PyOS_Readline(stdin, stdout, tok->prompt);
char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
if (tok->nextprompt != NULL)
tok->prompt = tok->nextprompt;
if (new == NULL)
if (newtok == NULL)
tok->done = E_INTR;
else if (*new == '\0') {
PyMem_FREE(new);
else if (*newtok == '\0') {
PyMem_FREE(newtok);
tok->done = E_EOF;
}
#if !defined(PGEN) && defined(Py_USING_UNICODE)
else if (tok_stdin_decode(tok, &new) != 0)
PyMem_FREE(new);
else if (tok_stdin_decode(tok, &newtok) != 0)
PyMem_FREE(newtok);
#endif
else if (tok->start != NULL) {
size_t start = tok->start - tok->buf;
size_t oldlen = tok->cur - tok->buf;
size_t newlen = oldlen + strlen(new);
size_t newlen = oldlen + strlen(newtok);
char *buf = tok->buf;
PyMem_RESIZE(buf, char, newlen+1);
buf = (char *)PyMem_REALLOC(buf, newlen+1);
tok->lineno++;
if (buf == NULL) {
PyMem_DEL(tok->buf);
PyMem_FREE(tok->buf);
tok->buf = NULL;
PyMem_FREE(new);
PyMem_FREE(newtok);
tok->done = E_NOMEM;
return EOF;
}
tok->buf = buf;
tok->cur = tok->buf + oldlen;
tok->line_start = tok->cur;
strcpy(tok->buf + oldlen, new);
PyMem_FREE(new);
strcpy(tok->buf + oldlen, newtok);
PyMem_FREE(newtok);
tok->inp = tok->buf + newlen;
tok->end = tok->inp + 1;
tok->start = tok->buf + start;
@ -809,8 +815,8 @@ tok_nextc(register struct tok_state *tok)
else {
tok->lineno++;
if (tok->buf != NULL)
PyMem_DEL(tok->buf);
tok->buf = new;
PyMem_FREE(tok->buf);
tok->buf = newtok;
tok->line_start = tok->buf;
tok->cur = tok->buf;
tok->line_start = tok->buf;
@ -824,7 +830,8 @@ tok_nextc(register struct tok_state *tok)
char *pt;
if (tok->start == NULL) {
if (tok->buf == NULL) {
tok->buf = PyMem_NEW(char, BUFSIZ);
tok->buf = (char *)
PyMem_MALLOC(BUFSIZ);
if (tok->buf == NULL) {
tok->done = E_NOMEM;
return EOF;
@ -859,7 +866,8 @@ tok_nextc(register struct tok_state *tok)
Py_ssize_t curvalid = tok->inp - tok->buf;
Py_ssize_t newsize = curvalid + BUFSIZ;
char *newbuf = tok->buf;
PyMem_RESIZE(newbuf, char, newsize);
newbuf = (char *)PyMem_REALLOC(newbuf,
newsize);
if (newbuf == NULL) {
tok->done = E_NOMEM;
tok->cur = tok->inp;
@ -1182,9 +1190,9 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
}
}
}
tok->start = tok->cur;
/* Return pending indents/dedents */
if (tok->pendin != 0) {
if (tok->pendin < 0) {
@ -1196,27 +1204,27 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
return INDENT;
}
}
again:
tok->start = NULL;
/* Skip spaces */
do {
c = tok_nextc(tok);
} while (c == ' ' || c == '\t' || c == '\014');
/* Set start of current token */
tok->start = tok->cur - 1;
/* Skip comment */
if (c == '#')
while (c != EOF && c != '\n')
c = tok_nextc(tok);
/* Check for EOF and errors now */
if (c == EOF) {
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
}
/* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') {
/* Process r"", u"" and ur"" */
@ -1244,7 +1252,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
*p_end = tok->cur;
return NAME;
}
/* Newline */
if (c == '\n') {
tok->atbol = 1;
@ -1255,7 +1263,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
tok->cont_line = 0;
return NEWLINE;
}
/* Period or number starting with period? */
if (c == '.') {
c = tok_nextc(tok);
@ -1418,7 +1426,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
*p_end = tok->cur;
return STRING;
}
/* Line continuation */
if (c == '\\') {
c = tok_nextc(tok);
@ -1430,7 +1438,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
tok->cont_line = 1;
goto again; /* Read next line */
}
/* Check for two-character token */
{
int c2 = tok_nextc(tok);
@ -1449,7 +1457,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
}
tok_backup(tok, c2);
}
/* Keep track of parentheses nesting level */
switch (c) {
case '(':
@ -1463,7 +1471,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
tok->level--;
break;
}
/* Punctuation character */
*p_start = tok->start;
*p_end = tok->cur;