Yugon
/
pybindmagic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd line option links all cpp cells now

Browse Source
master
Ugo Finnendahl 4 years ago
parent 7ecbba7ec2
commit f4c3781f7d
4 changed files with 241 additions and 83 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      .gitignore
  2. 38
      examples/main.py
  3. 66
      examples/test.ipynb
  4. 211
      pybindmagic/__init__.py

1
.gitignore vendored
Unescape View File

@ -5,3 +5,4 @@ pbm_modules
*.pyc
__pycache__
pybind11
*.egg-info

38
examples/main.py
Unescape View File

@ -1,4 +1,37 @@
import pybindmagic
#%%
%%cpp -f hello_world
void hello_world()
{
py::print("Hello World");
}
#%%
hello_world()
#%%
%%cpp -f callothercell -f callthiscell
void hello_world();
void callthiscell()
{
py::print("In this cell");
}
void callothercell()
{
py::print("From Other Cell:");
hello_world();
}
#%%
callthiscell()
callothercell()
#try to update the first cell an run callothercell() again
#%%
%%cpp -f generateMesh
#include <pybind11/eigen.h>
@ -82,13 +115,12 @@ defs
printMesh(*generateMesh())
#%%
%%cpp -c pathtocmake -f viewMeshh
%%cpp -c pathtocmake -f viewMesh
#include <igl/opengl/glfw/Viewer.h>
#include <pybind11/eigen.h>
void viewMeshh(Eigen::Matrix<double, Eigen::Dynamic, 3> V, Eigen::Matrix<int, Eigen::Dynamic, 3> F)
void viewMesh(Eigen::Matrix<double, Eigen::Dynamic, 3> V, Eigen::Matrix<int, Eigen::Dynamic, 3> F)
{
// Plot the mesh
igl::opengl::glfw::Viewer viewer;

66
examples/test.ipynb
Unescape View File

@ -9,6 +9,63 @@
"import pybindmagic"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%cpp -f hello_world\n",
"\n",
"void hello_world()\n",
"{\n",
" py::print(\"Hello World\");\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"hello_world()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%cpp -f callothercell -f callthiscell\n",
"\n",
"void hello_world();\n",
"\n",
"void callthiscell()\n",
"{\n",
" py::print(\"In this cell\");\n",
"}\n",
"\n",
"void callothercell()\n",
"{\n",
" py::print(\"From Other Cell:\");\n",
" hello_world();\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"callthiscell()\n",
"\n",
"callothercell()\n",
"#try to update the first cell an run callothercell() again"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -196,6 +253,13 @@
" ], dtype=np.int32)-1\n",
"viewMesh(V,F)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
@ -218,7 +282,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
"version": "3.8.3"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,

211
pybindmagic/__init__.py
Unescape View File

@ -11,7 +11,9 @@ import subprocess
compiler_cmd = "c++"
cmd_flags = ["$(pkg-config --cflags eigen3)"]
optimize = "-O0"
cmd_flags = ["$(pkg-config --cflags eigen3)", "-Wall"]
template = """
#include <pybind11/pybind11.h>
@ -25,7 +27,7 @@ PYBIND11_MODULE({name}, m)
def_template = '\nm.def("{function}", &{function});\n'
tmp_path = "pbm_modules"
TMP_PATH = "pbm_modules"
# Make sure the correct python executeable is used using cmake
os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"]
@ -42,9 +44,8 @@ def exec_command(cmd, cwd=None,**kwargs):
return True, e.output
def ensure_tmp_folder():
path = os.path.join(os.getcwd(), tmp_path)
path = os.path.join(os.getcwd(), TMP_PATH)
if not os.path.exists(path):
os.makedirs(path)
if not path in sys.path:
@ -67,120 +68,180 @@ def ensure_build_dir(path, clear=False):
os.makedirs(path)
return exists and not clear
@register_cell_magic
def cpp(line, cell, *what):
"""Compile, execute C++ code wrapped with pybind11 and import it"""
def parse_args(line):
args = line.split(" ")
rebuild = False
cmake_path = None
function = ""
run_cmake = False
ret = {}
ret["cmake_path"] = None
ret["functions"] = []
ret["rebuild"] = False
# TODO: Add full argument parser
for i,arg in enumerate(args):
if arg == "-c":
cmake_path = args[i+1].strip()
ret["cmake_path"] = args[i+1].strip()
if arg == "-f":
function = args[i+1].strip()
ret["functions"].append(args[i+1].strip())
if arg == "-rebuild":
rebuild = True
ret["rebuild"] = True
return ret
def get_CMakeLists(args):
CMakeLists = ""
if args["cmake_path"] is not None:
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'r') as f:
CMakeLists = f.read()
return CMakeLists
def cleanup(args, name, CMakeLists):
if args["cmake_path"] is not None:
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
f.write(CMakeLists)
if os.path.exists(os.path.join(args["path"],f"{name}.cpp")):
os.remove(os.path.join(args["path"],f"{name}.cpp"))
def get_gcc_cmd(name):
pyexe = sys.executable
return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {TMP_PATH}/{name}.cpp -o {TMP_PATH}/{name}.o"
def get_linker_cmd(name):
others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1]])
return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{name}.o -o {TMP_PATH}/{name}.so -L{TMP_PATH} {others}"
def get_unique_linker_cmd(other_path, rnd_path):
# TODO: Merge with get_linker_cmd
other = os.path.basename(other_path)[:-2]
all_other = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1] if os.path.basename(n)[3:-3] != os.path.basename(other_path)[3:-2]])
return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{other}.o -o {rnd_path}/{other}.so -L{TMP_PATH} {all_other}"
def import_to_ip(name):
# get a handle on the module
mdl = importlib.import_module(name)
# is there an __all__? if so respect it
if "__all__" in mdl.__dict__:
names = mdl.__dict__["__all__"]
else:
# otherwise we import all names that don't begin with _
names = [x for x in mdl.__dict__ if not x.startswith("_")]
get_ipython().push({k: getattr(mdl, k) for k in names})
# We first retrieve the current IPython interpreter
# instance.
@register_cell_magic
def cpp(line, cell, *what):
"""Compile, execute C++ code wrapped with pybind11 and import it"""
args = parse_args(line)
ip = get_ipython()
# get name of module
if args["cmake_path"] is None:
# TODO!
compile_id = get_gcc_cmd("dummy")
else:
compile_id = get_CMakeLists(args)
cmake = ""
if cmake_path is not None:
with open(os.path.join(cmake_path,"CMakeLists.txt"), 'r') as f:
cmake = f.read()
# name = ''.join(random.choices(string.ascii_lowercase, k=10))
hash_object = hashlib.sha256((cmake+" ".join(line)+function+cell).encode())
name = "cpp_magic_"+hash_object.hexdigest()[:10]
if cmake_path is None:
hash_object = hashlib.sha256((compile_id+"|"+line+"|"+cell).encode())
name = "libcpp_magic_"+hash_object.hexdigest()[:10]
run_cmake = False
# init temp folder and pybind etc
if args["cmake_path"] is None:
ensure_tmp_folder()
args["path"] = TMP_PATH
CMakeLists = None
else:
ensure_build_dir(cmake_path, clear=rebuild)
p = os.path.abspath(os.path.join(cmake_path,"build"))
run_cmake = not os.path.exists(os.path.join(p, "Makefile"))
ensure_build_dir(args["cmake_path"], clear=args["rebuild"])
p = os.path.abspath(os.path.join(args["cmake_path"],"build"))
if p not in sys.path:
sys.path.append(p)
ensure_pybind(cmake_path)
if cmake_path is not None:
path = cmake_path
else:
path = tmp_path
# We define the source and executable filenames.
try:
run_cmake = not os.path.exists(os.path.join(p, "Makefile"))
ensure_pybind(args["cmake_path"])
args["path"] = args["cmake_path"]
CMakeLists = compile_id
build_needed = False
try:
if rebuild:
if args["rebuild"]:
raise Exception()
mdl = importlib.import_module(name)
except:
exe = sys.executable
build_needed = True
if build_needed:
try:
# add pybind11 function defs
split = cell.split("defs")
if len(split) == 1 and function == "":
if len(split) == 1 and len(args["functions"]) == 0:
raise Exception("You have to name a function as argument ('%%cpp -f <functionname>') or manual set defs")
# make sure that split is at least length 2
split.append("")
curr_defs = split[1]
if function != "":
for function in args["functions"]:
curr_defs += def_template.format(function=function)
# We write the code to the C++ file.
with open(os.path.join(path,f"{name}.cpp"), 'w') as f:
with open(os.path.join(args["path"],f"{name}.cpp"), 'w') as f:
f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}"))
if cmake_path is None:
# We compile the C++ code into an executable.
command = f"{compiler_cmd} {' '.join(cmd_flags)} -O3 -Wall -shared -std=c++11 -fPIC $({exe} -m pybind11 --includes) {tmp_path}/{name}.cpp -o {tmp_path}/{name}.so"
if args["cmake_path"] is None:
# We compile the C++ code into an object file
command = get_gcc_cmd(name)
compile = ip.getoutput(command)
# currently warnings will lead to an abort
if len(compile) != 0:
raise Exception("\n".join(compile))
# link into a shared object
linker = get_linker_cmd(name)
link = ip.getoutput(linker)
if len(link) != 0:
raise Exception("\n".join(link))
# create unique folder
rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
rnd_path = os.path.join(TMP_PATH, rnd)
if not os.path.exists(rnd_path):
os.makedirs(rnd_path)
with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f:
pass
# Rethink design decision: Maybe do not import all others again, maybe give cells names and reduce the amount of linking
# update all others by relinking and reimporting orderd by compilation time
for other_path in sorted(glob.glob(f"{TMP_PATH}/*.o"), key=os.path.getmtime):
other = os.path.basename(other_path)[:-2]
linker = get_unique_linker_cmd(other_path, rnd_path)
link = ip.getoutput(linker)
if len(link) != 0:
raise Exception("\n".join(link))
import_to_ip(f"{rnd}.{other}")
shutil.rmtree(rnd_path)
else:
with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f:
f.write(cmake.replace("{name}",name))
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
f.write(CMakeLists.replace("{name}",name))
# TODO: Add option to add cmake flags
if run_cmake:
command = ["cmake", ".."]
print("cmake:")
err, output = exec_command(command, cwd=os.path.join(cmake_path, "build"))
err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build"))
if err:
raise Exception(output)
print(output)
command = ["make"]
print("make:")
err, output = exec_command(command, cwd=os.path.join(cmake_path, "build"))
err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build"))
if err:
raise Exception(output)
print(output)
with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f:
f.write(cmake)
# We execute the executable and return the output.
# get a handle on the module
mdl = importlib.import_module(name)
if os.path.exists(os.path.join(path,f"{name}.cpp")):
os.remove(os.path.join(path,f"{name}.cpp"))
# is there an __all__? if so respect it
if "__all__" in mdl.__dict__:
names = mdl.__dict__["__all__"]
else:
# otherwise we import all names that don't begin with _
names = [x for x in mdl.__dict__ if not x.startswith("_")]
with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
f.write(CMakeLists)
except Exception as e:
if cmake_path is not None:
with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f:
f.write(cmake)
if os.path.exists(os.path.join(path,f"{name}.cpp")):
os.remove(os.path.join(path,f"{name}.cpp"))
# clean up even if error occurs
cleanup(args, name, CMakeLists)
raise e
# now drag them in
ip.push({k: getattr(mdl, k) for k in names})
import_to_ip(name)
# clean up
cleanup(args, name, CMakeLists)

Write Preview
Loading…
Cancel
Save