Yugon
/
pybindmagic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

better WIP

Browse Source
master
Ugo Finnendahl 3 years ago
parent f4c3781f7d
commit ed77a6b400
2 changed files with 255 additions and 122 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 19
      examples/main.py
  2. 358
      pybindmagic/__init__.py

19
examples/main.py
Unescape View File

@ -5,7 +5,7 @@ import pybindmagic
void hello_world()
{
py::print("Hello World");
py::print("Hello World...");
}
#%%
@ -27,11 +27,27 @@ void callothercell()
hello_world();
}
#%%
callthiscell()
callothercell()
#try to update the first cell an run callothercell() again
#%%
%%cpp -f transitiv
void callothercell();
void transitiv()
{
py::print("Cell A:");
callothercell();
}
#%%
transitiv()
#%%
%%cpp -f generateMesh
#include <pybind11/eigen.h>
@ -63,6 +79,7 @@ std::tuple<Eigen::MatrixXd,Eigen::MatrixXi> generateMesh()
return std::make_tuple(V,F);
}
#%%
import JV9NK
V,F = generateMesh()
print("Vertices:\n",V)

358
pybindmagic/__init__.py
Unescape View File

@ -8,6 +8,9 @@ import string
import hashlib
import shutil
import subprocess
import shlex
from lief import parse, ELF
compiler_cmd = "c++"
@ -29,10 +32,29 @@ def_template = '\nm.def("{function}", &{function});\n'
TMP_PATH = "pbm_modules"
if os.path.exists(TMP_PATH):
shutil.rmtree(TMP_PATH)
BUILD_DIR = os.path.join(TMP_PATH, "build")
MODULES_DIR = os.path.join(TMP_PATH, "modules")
LIB_DIR = os.path.join(TMP_PATH, "lib")
# Make sure the correct python executeable is used using cmake
os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"]
# def run_command(command):
# process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
# while True:
# output = process.stdout.readline()
# if output == '' and process.poll() is not None:
# break
# if output:
# print output.strip()
# rc = process.poll()
# return rc
def exec_command(cmd, cwd=None,**kwargs):
if isinstance(cmd, str):
cmd = shlex.split(cmd)
try:
output = subprocess.check_output(cmd,cwd=cwd,stderr=subprocess.STDOUT,**kwargs)
if isinstance(output,bytes):
@ -44,12 +66,11 @@ def exec_command(cmd, cwd=None,**kwargs):
return True, e.output
def ensure_tmp_folder():
path = os.path.join(os.getcwd(), TMP_PATH)
def ensure_tmp_folder(path=TMP_PATH):
path = os.path.join(os.getcwd(), path)
if not os.path.exists(path):
os.makedirs(path)
if not path in sys.path:
sys.path.append(path)
def ensure_pybind(path, force=False):
path = os.path.join(path,"pybind11")
@ -57,7 +78,8 @@ def ensure_pybind(path, force=False):
if exists and force:
shutil.rmtree(os.path.join(path,"pybind11"))
if not exists:
err, output = exec_command(["git", "clone","https://github.com/pybind/pybind11.git",path])
err, output = exec_command(["git", "clone", "https://github.com/pybind/pybind11.git", path])
def ensure_build_dir(path, clear=False):
path = os.path.join(path, "build")
@ -68,6 +90,15 @@ def ensure_build_dir(path, clear=False):
os.makedirs(path)
return exists and not clear
def ensure_linker():
path = os.path.join(TMP_PATH, "liblinker.so")
exists = os.path.exists(path)
if not os.path.exists(path):
return compile_linkerlib()
return
def parse_args(line):
args = line.split(" ")
ret = {}
@ -92,6 +123,7 @@ def get_CMakeLists(args):
CMakeLists = f.read()
return CMakeLists
def cleanup(args, name, CMakeLists):
if args["cmake_path"] is not None:
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
@ -99,20 +131,16 @@ def cleanup(args, name, CMakeLists):
if os.path.exists(os.path.join(args["path"],f"{name}.cpp")):
os.remove(os.path.join(args["path"],f"{name}.cpp"))
def get_gcc_cmd(name):
pyexe = sys.executable
return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {TMP_PATH}/{name}.cpp -o {TMP_PATH}/{name}.o"
return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {BUILD_DIR}/{name}.cpp -o {BUILD_DIR}/{name}.o"
def get_linker_cmd(name):
others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1]])
return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{name}.o -o {TMP_PATH}/{name}.so -L{TMP_PATH} {others}"
def get_unique_linker_cmd(other_path, rnd_path):
# TODO: Merge with get_linker_cmd
other = os.path.basename(other_path)[:-2]
all_other = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1] if os.path.basename(n)[3:-3] != os.path.basename(other_path)[3:-2]])
return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{other}.o -o {rnd_path}/{other}.so -L{TMP_PATH} {all_other}"
def get_linker_cmd(name, others=None):
if others is None:
others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{LIB_DIR}/*.so"), key=os.path.getmtime)[::-1]])
return f"{compiler_cmd} -Wl,-rpath {LIB_DIR} -shared {BUILD_DIR}/{name}.o -o {MODULES_DIR}/{name}.so -L{LIB_DIR} {others}"
def import_to_ip(name):
@ -126,9 +154,147 @@ def import_to_ip(name):
names = [x for x in mdl.__dict__ if not x.startswith("_")]
get_ipython().push({k: getattr(mdl, k) for k in names})
def get_importexport(file):
binary = parse(file)
symbols = binary.symbols
imported = set()
exported = set()
for i,symbol in enumerate(symbols):
if not symbol.imported and not symbol.exported and symbol.binding == ELF.SYMBOL_BINDINGS.GLOBAL:
imported.add(symbol.name)
if symbol.exported and symbol.binding == ELF.SYMBOL_BINDINGS.GLOBAL:
exported.add(symbol.name)
return imported, exported
export_graph = {}
import_graph = {}
symbols = {}
def update_dependencies(fresh_compiled):
global export_graph, import_graph
imports, exports = get_importexport(fresh_compiled)
symbols[fresh_compiled] = (imports, exports)
for inp in imports:
if inp.startswith("Py") or inp.startswith("_Py"):
continue
if inp in import_graph:
import_graph[inp].append(fresh_compiled)
else:
import_graph[inp] = [fresh_compiled]
for ex in exports:
if ex.startswith("PyInit"):
continue
export_graph[ex] = fresh_compiled
def get_list_of_rebuilds(fresh_compiled, need_recompilation=None):
global export_graph, import_graph
if need_recompilation is None:
need_recompilation = set()
need_recompilation.add(fresh_compiled)
exports = symbols[fresh_compiled][1]
for ex in exports:
if ex in import_graph:
for lib in import_graph[ex]:
if lib not in need_recompilation:
need_recompilation.add(lib)
# recursive collect all libs, that depend on this
get_list_of_rebuilds(lib, need_recompilation)
return need_recompilation
def get_list_of_links(fresh_compiled):
global export_graph, import_graph
imports = symbols[fresh_compiled][0]
res = []
for imp in imports:
if imp in export_graph:
lib = export_graph[imp]
if lib not in res:
res.append(lib)
return res
def bulk_import(need_reimport=None):
if need_reimport is None:
need_reimport = sorted(glob.glob(f"{MODULES_DIR}/*.so"), key=os.path.getmtime)
# create unique folder
rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
rnd_path = os.path.join(TMP_PATH, rnd)
if not os.path.exists(rnd_path):
os.makedirs(rnd_path)
with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f:
pass
for lib_path in need_reimport:
lib = os.path.basename(lib_path)[:-3]
shutil.copyfile(lib_path, os.path.join(rnd_path,f"{lib}.so"))
for lib_path in need_reimport:
lib = os.path.basename(lib_path)[:-3]
import_to_ip(f"{rnd}.{lib}")
shutil.rmtree(rnd_path)
def compile_module(cell, args):
name = args["name"]
ensure_tmp_folder(BUILD_DIR)
create_cpp_file(BUILD_DIR, cell, args)
ip = get_ipython()
# We compile the C++ code into an object file
command = get_gcc_cmd(name)
compile = ip.getoutput(command)
# currently warnings will lead to an abort
if len(compile) != 0:
raise Exception("\n".join(compile))
# link into a shared object, without links
linker = get_linker_cmd(name, "")
link = ip.getoutput(linker)
if len(link) != 0:
raise Exception("\n".join(link))
def link_module(name, others=None):
other_name = os.path.basename(name)[:-3]
linker = get_linker_cmd(other_name, others)
link = get_ipython().getoutput(linker)
if len(link) != 0:
raise Exception("\n".join(link))
version = 0
def create_cpp_file(path, cell, args):
name = args["name"]
# add pybind11 function defs
split = cell.split("defs")
if len(split) == 1 and len(args["functions"]) == 0:
raise Exception("You have to name a function as argument ('%%cpp -f <functionname>') or manual set defs")
# make sure that split is at least length 2
split.append("")
curr_defs = split[1]
for function in args["functions"]:
curr_defs += def_template.format(function=function)
with open(os.path.join(path,f"{name}.cpp"), 'w') as f:
f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}"))
def get_newest(name):
return sorted(glob.glob(f"{LIB_DIR}/lib{name}*.so"), key=os.path.getmtime)[-1]
history = []
# TODO: TEST
@register_cell_magic
def cpp(line, cell, *what):
"""Compile, execute C++ code wrapped with pybind11 and import it"""
global version
args = parse_args(line)
ip = get_ipython()
# get name of module
@ -139,109 +305,59 @@ def cpp(line, cell, *what):
compile_id = get_CMakeLists(args)
hash_object = hashlib.sha256((compile_id+"|"+line+"|"+cell).encode())
name = "libcpp_magic_"+hash_object.hexdigest()[:10]
run_cmake = False
name = "cpp_magic_"+hash_object.hexdigest()[:10]
args["name"] = name
version += 1
history.append(os.path.join(MODULES_DIR, f"{name}.so"))
ensure_tmp_folder(TMP_PATH)
# Check if neccessary here
if not TMP_PATH in sys.path:
sys.path.append(TMP_PATH)
ensure_tmp_folder(MODULES_DIR)
ensure_tmp_folder(LIB_DIR)
# if not yet compiled, do so
if not os.path.exists(os.path.join(MODULES_DIR,f"{name}.so")):
# TODO: extend to cmake building
compile_module(cell, args)
update_dependencies(os.path.join(MODULES_DIR,f"{name}.so"))
relinks = list(get_list_of_rebuilds(os.path.join(MODULES_DIR,f"{name}.so")))
relinks = sorted(relinks, key=os.path.getmtime)[::-1]
for re in relinks:
other_name = os.path.basename(re)[:-3]
shutil.copyfile(os.path.join(MODULES_DIR,f"{other_name}.so"), os.path.join(LIB_DIR,f"lib{other_name}{version}.so"))
# touch
open(os.path.join(LIB_DIR,f"lib{other_name}{version}.so"), 'a').close()
# TODO: delete old
# modules = sorted(glob.glob(f"{MODULES_DIR}/*.so"), key=os.path.getmtime)[::-1]
# others = " ".join([f"-l{os.path.basename(get_newest(os.path.basename(n)[:-3]))[3:-3]}" for n in modules])
# print(others)
for re in relinks:
others = " ".join([f"-l{os.path.basename(get_newest(os.path.basename(n)[:-3]))[3:-3]}" for n in sorted(get_list_of_links(re), key=os.path.getmtime)[::-1]])
link_module(re, others)
other_name = os.path.basename(re)[:-3]
shutil.copyfile(os.path.join(MODULES_DIR,f"{other_name}.so"), os.path.join(LIB_DIR,f"lib{other_name}{version}.so"))
bulk_import([h for h in history if h in relinks])
# init temp folder and pybind etc
if args["cmake_path"] is None:
ensure_tmp_folder()
args["path"] = TMP_PATH
CMakeLists = None
else:
ensure_build_dir(args["cmake_path"], clear=args["rebuild"])
p = os.path.abspath(os.path.join(args["cmake_path"],"build"))
if p not in sys.path:
sys.path.append(p)
run_cmake = not os.path.exists(os.path.join(p, "Makefile"))
ensure_pybind(args["cmake_path"])
args["path"] = args["cmake_path"]
CMakeLists = compile_id
build_needed = False
try:
if args["rebuild"]:
raise Exception()
mdl = importlib.import_module(name)
except:
build_needed = True
if build_needed:
try:
# add pybind11 function defs
split = cell.split("defs")
if len(split) == 1 and len(args["functions"]) == 0:
raise Exception("You have to name a function as argument ('%%cpp -f <functionname>') or manual set defs")
# make sure that split is at least length 2
split.append("")
curr_defs = split[1]
for function in args["functions"]:
curr_defs += def_template.format(function=function)
# We write the code to the C++ file.
with open(os.path.join(args["path"],f"{name}.cpp"), 'w') as f:
f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}"))
if args["cmake_path"] is None:
# We compile the C++ code into an object file
command = get_gcc_cmd(name)
compile = ip.getoutput(command)
# currently warnings will lead to an abort
if len(compile) != 0:
raise Exception("\n".join(compile))
# link into a shared object
linker = get_linker_cmd(name)
link = ip.getoutput(linker)
if len(link) != 0:
raise Exception("\n".join(link))
# create unique folder
rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
rnd_path = os.path.join(TMP_PATH, rnd)
if not os.path.exists(rnd_path):
os.makedirs(rnd_path)
with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f:
pass
# Rethink design decision: Maybe do not import all others again, maybe give cells names and reduce the amount of linking
# update all others by relinking and reimporting orderd by compilation time
for other_path in sorted(glob.glob(f"{TMP_PATH}/*.o"), key=os.path.getmtime):
other = os.path.basename(other_path)[:-2]
linker = get_unique_linker_cmd(other_path, rnd_path)
link = ip.getoutput(linker)
if len(link) != 0:
raise Exception("\n".join(link))
import_to_ip(f"{rnd}.{other}")
shutil.rmtree(rnd_path)
else:
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
f.write(CMakeLists.replace("{name}",name))
# TODO: Add option to add cmake flags
if run_cmake:
command = ["cmake", ".."]
print("cmake:")
err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build"))
if err:
raise Exception(output)
print(output)
command = ["make"]
print("make:")
err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build"))
if err:
raise Exception(output)
print(output)
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
f.write(CMakeLists)
except Exception as e:
# clean up even if error occurs
cleanup(args, name, CMakeLists)
raise e
import_to_ip(name)
# clean up
cleanup(args, name, CMakeLists)
"""
CLEAR LIB_FOLDER
#### PIPELINE ####
create name
add version
add name to history
compile to modules_dir/{name}.so (may be already there)
detect_relink_needs
for all relinked mv modules_dir/{name}.so -> libs_dir/lib{name}{version}.so (delete old?)
relink
for all relinked mv modules_dir/{name}.so -> libs_dir/lib{name}{version}.so
import_bulk according to history
"""

Write Preview
Loading…
Cancel
Save