traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(bindings): support free-threaded python build

Browse source
This commit is contained in:
ObserverOfTime 2024-10-17 17:01:09 +03:00 • committed by Amaan Qureshi
parent 7bf51ae08a
commit a9dbb7257c
3 changed files with 77 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

100
cli/src/init.rs
View file

@ -445,9 +445,44 @@ pub fn generate_grammar_files(
let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
missing_path(&lang_path, create_dir)?;
missing_path(lang_path.join("binding.c"), |path| {
generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts)
})?;
missing_path_else(
lang_path.join("binding.c"),
allow_update,
|path| generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts),
|path| {
let mut contents = fs::read_to_string(path)?;
if !contents.contains("PyModuleDef_Init") {
contents = contents
.replace("PyModule_Create", "PyModuleDef_Init")
.replace(
"static PyMethodDef methods[] = {\n",
indoc! {"
static struct PyModuleDef_Slot slots[] = {
#ifdef Py_GIL_DISABLED
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
#endif
{0, NULL}
};
static PyMethodDef methods[] = {
"},
)
.replace(
indoc! {"
.m_size = -1,
.m_methods = methods
"},
indoc! {"
.m_size = 0,
.m_methods = methods,
.m_slots = slots,
"},
);
write_file(path, contents)?;
}
Ok(())
},
)?;
missing_path(lang_path.join("__init__.py"), |path| {
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)
@ -478,51 +513,27 @@ pub fn generate_grammar_files(
allow_update,
|path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
|path| {
let mut contents = fs::read_to_string(path)?;
if contents.contains("sources.extend") || !contents.contains("egg_info") {
contents = contents
.replace("sources.extend", "sources.append")
.replace(
"from setuptools.command.build import build\n",
indoc! {"
from setuptools.command.build import build
from setuptools.command.egg_info import egg_info
"},
)
.replace(
"setup(\n",
indoc! {r#"
class EggInfo(egg_info):
def find_sources(self):
super().find_sources()
self.filelist.recursive_include("queries", "*.scm")
self.filelist.include("src/tree_sitter/*.h")
setup(
"#},
)
.replace(
"\"bdist_wheel\": BdistWheel\n",
indoc! {r#"
"bdist_wheel": BdistWheel,
"egg_info": EggInfo,
"#},
);
write_file(path, contents)?;
let contents = fs::read_to_string(path)?;
if !contents.contains("egg_info") || !contents.contains("Py_GIL_DISABLED") {
eprintln!("Replacing setup.py");
generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
}
Ok(())
},
)?;
missing_path_else(repo_path.join("pyproject.toml"), allow_update, |path| {
generate_file(
path,
PYPROJECT_TOML_TEMPLATE,
dashed_language_name.as_str(),
&generate_opts,
)
}, |path| {
missing_path_else(
repo_path.join("pyproject.toml"),
allow_update,
|path| {
generate_file(
path,
PYPROJECT_TOML_TEMPLATE,
dashed_language_name.as_str(),
&generate_opts,
)
},
|path| {
let mut contents = fs::read_to_string(path)?;
if !contents.contains("cp310-*") {
contents = contents
@ -532,7 +543,8 @@ pub fn generate_grammar_files(
write_file(path, contents)?;
}
Ok(())
})?;
},
)?;
Ok(())
})?;

14
cli/src/templates/py-binding.c
View file

@ -8,6 +8,13 @@ static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSE
return PyCapsule_New(tree_sitter_LOWER_PARSER_NAME(), "tree_sitter.Language", NULL);
}
static struct PyModuleDef_Slot slots[] = {
#ifdef Py_GIL_DISABLED
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
#endif
{0, NULL}
};
static PyMethodDef methods[] = {
{"language", _binding_language, METH_NOARGS,
"Get the tree-sitter language for this grammar."},
@ -18,10 +25,11 @@ static struct PyModuleDef module = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "_binding",
.m_doc = NULL,
.m_size = -1,
.m_methods = methods
.m_size = 0,
.m_methods = methods,
.m_slots = slots,
};
PyMODINIT_FUNC PyInit__binding(void) {
return PyModule_Create(&module);
return PyModuleDef_Init(&module);
}

16
cli/src/templates/setup.py
View file

@ -1,5 +1,6 @@
from os import path
from platform import system
from sysconfig import get_config_var
from setuptools import Extension, find_packages, setup
from setuptools.command.build import build
@ -13,6 +14,13 @@ sources = [
if path.exists("src/scanner.c"):
sources.append("src/scanner.c")
macros: list[tuple[str, str | None]] = [
("PY_SSIZE_T_CLEAN", None),
("TREE_SITTER_HIDE_SYMBOLS", None),
]
if limited_api := not get_config_var("Py_GIL_DISABLED"):
macros.append(("Py_LIMITED_API", "0x030A0000"))
if system() != "Windows":
cflags = ["-std=c11", "-fvisibility=hidden"]
else:
@ -55,13 +63,9 @@ setup(
name="_binding",
sources=sources,
extra_compile_args=cflags,
define_macros=[
("Py_LIMITED_API", "0x03090000"),
("PY_SSIZE_T_CLEAN", None),
("TREE_SITTER_HIDE_SYMBOLS", None),
],
define_macros=macros,
include_dirs=["src"],
py_limited_api=True,
py_limited_api=limited_api,
)
],
cmdclass={