ilusha/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2026-09-29 12:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

gh-92651: Remove the Include/token.h header file (#92652)

Browse source 
Remove the token.h header file. There was never any public tokenizer
C API. The token.h header file was only designed to be used by Python
internals.

Move Include/token.h to Include/internal/pycore_token.h. Including
this header file now requires that the Py_BUILD_CORE macro is
defined. It no longer checks for the Py_LIMITED_API macro.

Rename functions:

* PyToken_OneChar() => _PyToken_OneChar()
* PyToken_TwoChars() => _PyToken_TwoChars()
* PyToken_ThreeChars() => _PyToken_ThreeChars()
This commit is contained in:
Victor Stinner 2022-05-11 23:22:50 +02:00 • committed by GitHub
parent b69297ea23
commit da5727a120
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 52 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Parser/pegen.h
View file

@ -3,8 +3,8 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <token.h>
#include <pycore_ast.h>
#include <pycore_token.h>
#if 0
#define PyPARSE_YIELD_IS_KEYWORD 0x0001

8
Parser/token.c generated
View file

@ -1,7 +1,7 @@
/* Auto-generated by Tools/scripts/generate_token.py */
#include "Python.h"
#include "token.h"
#include "pycore_token.h"
/* Token names */
@ -76,7 +76,7 @@ const char * const _PyParser_TokenNames[] = {
/* Return the token corresponding to a single character */
int
PyToken_OneChar(int c1)
_PyToken_OneChar(int c1)
{
switch (c1) {
case '%': return PERCENT;
@ -107,7 +107,7 @@ PyToken_OneChar(int c1)
}
int
PyToken_TwoChars(int c1, int c2)
_PyToken_TwoChars(int c1, int c2)
{
switch (c1) {
case '!':
@ -191,7 +191,7 @@ PyToken_TwoChars(int c1, int c2)
}
int
PyToken_ThreeChars(int c1, int c2, int c3)
_PyToken_ThreeChars(int c1, int c2, int c3)
{
switch (c1) {
case '*':

6
Parser/tokenizer.c
View file

@ -1992,10 +1992,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
/* Check for two-character token */
{
int c2 = tok_nextc(tok);
int token = PyToken_TwoChars(c, c2);
int token = _PyToken_TwoChars(c, c2);
if (token != OP) {
int c3 = tok_nextc(tok);
int token3 = PyToken_ThreeChars(c, c2, c3);
int token3 = _PyToken_ThreeChars(c, c2, c3);
if (token3 != OP) {
token = token3;
}
@ -2059,7 +2059,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
/* Punctuation character */
*p_start = tok->start;
*p_end = tok->cur;
return PyToken_OneChar(c);
return _PyToken_OneChar(c);
}
int

2
Parser/tokenizer.h
View file

@ -8,7 +8,7 @@ extern "C" {
/* Tokenizer interface */
#include "token.h" /* For token types */
#include "pycore_token.h" /* For token types */
#define MAXINDENT 100 /* Max indentation level */
#define MAXLEVEL 200 /* Max parentheses level */