Wrapping a subset of a very large libraryΒΆ
Sometimes, for a very large library, only a subset of available C++
components is useful for end-users. Wrapping such libraries therefore
requires AutoWIG to be able to consider only a subset of the C++
components during the Generate step. The Clang library is a
complete C/C++ compiler. Clang is a great tool, but its stable
Python interface (i.e. libclang) is lacking some useful features
that are needed by AutoWIG. In particular, class template
specializations are not available in the abstract syntax tree.
Fortunately, most of the classes that would be needed during the
traversal of the C++ abstract syntax tree are not template
specializations. We therefore proposed to bootstrap the Clang
Python bindings using the libclang parser of AutoWIG. This
new Clang Python interface is called PyClangLite and is able
to parse class template specializations. As for libclang, this
interface is proposed only for a subset of the Clang library
sufficient enough for proposing the new pyclanglite parser.
This repository already has wrappers, we therefore need to remove them.
In [ ]:
!git clone https://github.com/StatisKit/ClangLite ClangLite
!git -C ClangLite checkout a13322e37683012ca346595e88abc48ac591112c
In [ ]:
from path import Path
import shutil
srcdir = Path('ClangLite')/'src'/'py'
for wrapper in srcdir.walkfiles('*.cpp'):
wrapper.unlink()
for wrapper in srcdir.walkfiles('*.h'):
wrapper.unlink()
wrapper = srcdir/'clanglite'/'_clanglite.py'
if wrapper.exists():
wrapper.unlink()
blddir = srcdir.parent.parent/'build'
if blddir.exists():
shutil.rmtree(srcdir.parent.parent/'build')
In addition to the Clang libraries, the ClangLite library is
needed in order to have access to some functionalities. The tool.h
header of this ClangLite library includes all necessary Clang
headers. This library is installed using the SCons cpp target.
In [ ]:
!conda remove libclanglite -y
!conda build ClangLite/conda/libclanglite -c statiskit -c conda-forge
!conda install -y libclanglite --use-local -c statiskit -c conda-forge
Once these preliminaries done, we can proceed to the actual generation of wrappers for the Clang library. For this, we import AutoWIG and create an empty Abstract Semantic Graph (ASG).
In [ ]:
import autowig
asg = autowig.AbstractSemanticGraph()
We then parse the tool.h header of the ClangLite library with
relevant compilation flags.
In [ ]:
%%time
import sys
prefix = Path(sys.prefix).abspath()
autowig.parser.plugin = 'libclang'
asg = autowig.parser(asg, [prefix/'include'/'clanglite'/'tool.h'],
flags = ['-x', 'c++', '-std=c++11',
'-D__STDC_LIMIT_MACROS',
'-D__STDC_CONSTANT_MACROS',
'-I' + str((prefix/'include').abspath())],
libpath = prefix/'lib'/'libclang.so',
bootstrap = False,
silent = True)
Since most of AutoWIG guidelines are respected in the Clang
library, the default controller implementation could be
suitable. Nevertheless, we need to force some C++ components to be
wrapped or not. We therefore implements a new controller.
In [ ]:
def clanglite_controller(asg):
for node in asg['::boost::python'].classes(nested = True):
node.is_copyable = True
for node in asg.classes():
node.boost_python_export = False
for node in asg.functions(free=True):
node.boost_python_export = False
for node in asg.variables(free = True):
node.boost_python_export = False
for node in asg.enumerations():
node.boost_python_export = False
for node in asg.enumerators():
if node.parent.boost_python_export:
node.boost_python_export = False
for node in asg.typedefs():
node.boost_python_export = False
from autowig.default_controller import refactoring
asg = refactoring(asg)
if autowig.parser.plugin == 'libclang':
for fct in asg.functions(free=False):
asg._nodes[fct._node]['_is_virtual'] = False
asg._nodes[fct._node]['_is_pure'] = False
asg['class ::clang::QualType'].is_abstract = False
asg['class ::clang::QualType'].is_copyable = True
asg['class ::llvm::StringRef'].is_abstract = False
asg['class ::llvm::StringRef'].is_copyable = True
asg['class ::clang::FileID'].is_abstract = False
asg['class ::clang::FileID'].is_copyable = True
asg['class ::clang::SourceLocation'].is_abstract = False
asg['class ::clang::SourceLocation'].is_copyable = True
asg['class ::clang::TemplateArgument'].is_abstract = False
asg['class ::clang::TemplateArgument'].is_copyable = True
for cls in ['::clang::FriendDecl', '::clang::CapturedDecl', '::clang::OMPThreadPrivateDecl',
'::clang::NonTypeTemplateParmDecl', '::clang::TemplateArgumentList', '::clang::ImportDecl',
'::clang::TemplateTemplateParmDecl', '::clang::CapturedDecl', '::clang::OMPThreadPrivateDecl',
'::clang::NonTypeTemplateParmDecl', '::clang::TemplateArgumentList', '::clang::ImportDecl',
'::clang::TemplateTemplateParmDecl']:
asg['class ' + cls].is_abstract = False
asg['class ::boost::python::api::object'].boost_python_export = True
asg['class ::boost::python::list'].boost_python_export = True
asg['class ::boost::python::str'].boost_python_export = True
subset = []
classes = [asg['class ::clang::QualType'],
asg['class ::clang::Type'],
asg['class ::clang::Decl']]
subset += classes
for cls in classes:
subset += cls.subclasses(recursive=True)
for cls in subset:
if not cls.globalname.strip('class ') in ['::clang::QualType',
'::llvm::StringRef',
'::clang::FileID',
'::clang::SourceLocation',
'::clang::TemplateArgument',
'::clang::FriendDecl',
'::clang::CapturedDecl',
'::clang::OMPThreadPrivateDecl',
'::clang::NonTypeTemplateParmDecl',
'::clang::TemplateArgumentList',
'::clang::ImportDecl',
'::clang::TemplateTemplateParmDecl']:
cls.is_copyable = False
else:
cls.is_copyable = True
subset.append(asg['class ::llvm::StringRef'])
subset.append(asg['class ::clang::ASTUnit'])
subset.append(asg['class ::clang::ASTContext'])
subset.append(asg['class ::clang::SourceManager'])
subset.append(asg['class ::clang::FileID'])
subset.append(asg['class ::clang::SourceLocation'])
subset.append(asg['class ::clang::CXXBaseSpecifier'])
subset.append(asg['class ::clang::DeclContext'])
subset.append(asg['class ::clang::TemplateArgument'])
subset.append(asg['class ::clang::TemplateArgumentList'])
subset.append(asg['enum ::clang::Type::TypeClass'])
subset.append(asg['enum ::clang::AccessSpecifier'])
subset.append(asg['enum ::clang::LinkageSpecDecl::LanguageIDs'])
subset.append(asg['enum ::clang::BuiltinType::Kind'])
subset.append(asg['enum ::clang::TemplateArgument::ArgKind'])
subset.append(asg['enum ::clang::Decl::Kind'])
# subset.extend(asg['::boost::python'].classes(nested = True))
# subset.extend(asg['::boost::python'].enumerations(nested = True))
subset.extend(asg.nodes('::clanglite::build_ast_from_code_with_args'))
for node in subset:
node.boost_python_export = True
for fct in asg['::clanglite'].functions():
if not fct.localname == 'build_ast_from_code_with_args':
fct.parent = fct.parameters[0].qualified_type.desugared_type.unqualified_type
fct.boost_python_export = True
for mtd in asg['class ::clang::ASTContext'].methods(pattern='.*getSourceManager.*'):
if mtd.return_type.globalname == 'class ::clang::SourceManager &':
mtd.boost_python_export = True
break
if autowig.parser.plugin == 'libclang':
for node in (asg.functions(pattern='.*(llvm|clang).*_(begin|end)')
+ asg.functions(pattern='::clang::CXXRecordDecl::getCaptureFields')
+ asg.functions(pattern='.*(llvm|clang).*getNameAsString')
+ asg.nodes('::clang::NamedDecl::getQualifiedNameAsString')
+ asg.functions(pattern='.*::clang::ObjCProtocolDecl')
+ asg.nodes('::clang::ObjCProtocolDecl::collectInheritedProtocolProperties')
+ asg.nodes('::clang::ASTUnit::LoadFromASTFile')
+ asg.nodes('::clang::ASTUnit::getCachedCompletionTypes')
+ asg.nodes('::clang::ASTUnit::getBufferForFile')
+ asg.nodes('::clang::CXXRecordDecl::getCaptureFields')
+ asg.nodes('::clang::ASTContext::SectionInfos')
+ asg.nodes('::clang::ASTContext::getAllocator')
+ asg.nodes('::clang::ASTContext::getObjCEncoding.*')
+ asg.nodes('::clang::ASTContext::getAllocator')
+ asg.nodes('::clang::QualType::getAsString')
+ asg.nodes('::clang::SourceLocation::printToString')
+ asg['class ::llvm::StringRef'].methods()):
node.boost_python_export = False
if autowig.parser.plugin == 'clanglite':
for mtd in asg['class ::clang::Decl'].methods():
if mtd.localname == 'hasAttr':
mtd.boost_python_export = False
import sys
from path import path
for header in (path(sys.prefix)/'include'/'clang').walkfiles('*.h'):
asg[header.abspath()].is_external_dependency = False
return asg
This controller is then dynamically registered and used on the ASG.
In [ ]:
%%time
autowig.controller['clanglite'] = clanglite_controller
autowig.controller.plugin = 'clanglite'
asg = autowig.controller(asg)
In order to wrap a subset of the Clang library, we need to select
the boost_python_internal generator implementation.
In [ ]:
%%time
autowig.generator.plugin = 'boost_python_pattern'
wrappers = autowig.generator(asg,
module = srcdir/'_clanglite.cpp',
decorator = srcdir/'clanglite'/'_clanglite.py',
closure = False)
The wrappers are only generated in-memory. It is therefore needed to write them on the disk to complete the process.
In [ ]:
%%time
wrappers.write()
Here is an example of the generated wrappers. We here present the
wrappers for the clang::Decl class.
In [ ]:
!pygmentize ClangLite/src/py/wrapper_a6aedb4654a55a40aeecf4b1dc5fcc98.cpp
Once the wrappers are written on the disk, the bingings must be compiled
and installed. This can be done using the SCons py target.
In [ ]:
!conda build ClangLite/conda/python-clanglite -c statiskit -c conda-forge
!conda install -y python-clanglite --use-local -c statiskit -c conda-forge
In [ ]:
import autowig
from clanglite.autowig_parser import autowig_parser
autowig.parser['clanglite'] = autowig_parser
autowig.parser.plugin = 'clanglite'
from path import Path
import sys
for wrapper in srcdir.walkfiles('*.cpp'):
wrapper.unlink()
for wrapper in srcdir.walkfiles('*.h'):
wrapper.unlink()
wrapper = srcdir/'clanglite'/'_clanglite.py'
if wrapper.exists():
wrapper.unlink()
prefix = Path(sys.prefix).abspath()
asgbis = autowig.AbstractSemanticGraph()
asgbis = autowig.parser(asgbis, [prefix/'include'/'clanglite'/'tool.h'],
flags = ['-x', 'c++', '-std=c++11',
'-D__STDC_CONSTANT_MACROS',
'-D__STDC_FORMAT_MACROS',
'-D__STDC_LIMIT_MACROS',
'-I' + str((prefix/'include').abspath()),
'-I' + str((prefix/'include'/'python2.7').abspath())],
bootstrap = False,
silent = True)
autowig.controller['clanglite'] = clanglite_controller
autowig.controller.plugin = 'clanglite'
asgbis = autowig.controller(asgbis)
autowig.generator.plugin = 'boost_python_pattern'
wrappers = autowig.generator(asgbis,
module = srcdir/'_clanglite.cpp',
decorator = srcdir/'clanglite'/'_clanglite.py',
closure = False)
wrappers.write()
In [ ]:
!conda remove python-clanglite -y
!conda build ClangLite/conda/python-clanglite -c statiskit -c conda-forge
!conda install -y python-clanglite --use-local -c statiskit -c conda-forge