diff --git a/.gitignore b/.gitignore index e262701..c646c19 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ pbm_modules *.pyc __pycache__ pybind11 +*.egg-info diff --git a/examples/main.py b/examples/main.py index 7349983..0a58be1 100644 --- a/examples/main.py +++ b/examples/main.py @@ -1,4 +1,37 @@ import pybindmagic + +#%% +%%cpp -f hello_world + +void hello_world() +{ + py::print("Hello World"); +} +#%% + +hello_world() + +#%% +%%cpp -f callothercell -f callthiscell + +void hello_world(); + +void callthiscell() +{ + py::print("In this cell"); +} + +void callothercell() +{ + py::print("From Other Cell:"); + hello_world(); +} +#%% +callthiscell() + +callothercell() +#try to update the first cell an run callothercell() again + #%% %%cpp -f generateMesh #include @@ -82,13 +115,12 @@ defs printMesh(*generateMesh()) #%% -%%cpp -c pathtocmake -f viewMeshh +%%cpp -c pathtocmake -f viewMesh #include #include - -void viewMeshh(Eigen::Matrix V, Eigen::Matrix F) +void viewMesh(Eigen::Matrix V, Eigen::Matrix F) { // Plot the mesh igl::opengl::glfw::Viewer viewer; diff --git a/examples/test.ipynb b/examples/test.ipynb index 62b6fe9..42ffbb5 100644 --- a/examples/test.ipynb +++ b/examples/test.ipynb @@ -9,6 +9,63 @@ "import pybindmagic" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%cpp -f hello_world\n", + "\n", + "void hello_world()\n", + "{\n", + " py::print(\"Hello World\");\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hello_world()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%cpp -f callothercell -f callthiscell\n", + "\n", + "void hello_world();\n", + "\n", + "void callthiscell()\n", + "{\n", + " py::print(\"In this cell\");\n", + "}\n", + "\n", + "void callothercell()\n", + "{\n", + " py::print(\"From Other Cell:\");\n", + " hello_world();\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "callthiscell()\n", + "\n", + "callothercell()\n", + "#try to update the first cell an run callothercell() again" + ] + }, { "cell_type": "code", "execution_count": null, @@ -196,6 +253,13 @@ " ], dtype=np.int32)-1\n", "viewMesh(V,F)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -218,7 +282,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.3" }, "latex_envs": { "LaTeX_envs_menu_present": true, diff --git a/pybindmagic/__init__.py b/pybindmagic/__init__.py index e66c3da..b73df8a 100644 --- a/pybindmagic/__init__.py +++ b/pybindmagic/__init__.py @@ -11,7 +11,9 @@ import subprocess compiler_cmd = "c++" -cmd_flags = ["$(pkg-config --cflags eigen3)"] +optimize = "-O0" + +cmd_flags = ["$(pkg-config --cflags eigen3)", "-Wall"] template = """ #include @@ -25,7 +27,7 @@ PYBIND11_MODULE({name}, m) def_template = '\nm.def("{function}", &{function});\n' -tmp_path = "pbm_modules" +TMP_PATH = "pbm_modules" # Make sure the correct python executeable is used using cmake os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"] @@ -42,9 +44,8 @@ def exec_command(cmd, cwd=None,**kwargs): return True, e.output - def ensure_tmp_folder(): - path = os.path.join(os.getcwd(), tmp_path) + path = os.path.join(os.getcwd(), TMP_PATH) if not os.path.exists(path): os.makedirs(path) if not path in sys.path: @@ -67,120 +68,180 @@ def ensure_build_dir(path, clear=False): os.makedirs(path) return exists and not clear -@register_cell_magic -def cpp(line, cell, *what): - """Compile, execute C++ code wrapped with pybind11 and import it""" +def parse_args(line): args = line.split(" ") - rebuild = False - cmake_path = None - function = "" - run_cmake = False + ret = {} + ret["cmake_path"] = None + ret["functions"] = [] + ret["rebuild"] = False # TODO: Add full argument parser for i,arg in enumerate(args): if arg == "-c": - cmake_path = args[i+1].strip() + ret["cmake_path"] = args[i+1].strip() if arg == "-f": - function = args[i+1].strip() + ret["functions"].append(args[i+1].strip()) if arg == "-rebuild": - rebuild = True - + ret["rebuild"] = True + return ret + + +def get_CMakeLists(args): + CMakeLists = "" + if args["cmake_path"] is not None: + with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'r') as f: + CMakeLists = f.read() + return CMakeLists + +def cleanup(args, name, CMakeLists): + if args["cmake_path"] is not None: + with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f: + f.write(CMakeLists) + if os.path.exists(os.path.join(args["path"],f"{name}.cpp")): + os.remove(os.path.join(args["path"],f"{name}.cpp")) + +def get_gcc_cmd(name): + pyexe = sys.executable + return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {TMP_PATH}/{name}.cpp -o {TMP_PATH}/{name}.o" + +def get_linker_cmd(name): + others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1]]) + return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{name}.o -o {TMP_PATH}/{name}.so -L{TMP_PATH} {others}" + +def get_unique_linker_cmd(other_path, rnd_path): + # TODO: Merge with get_linker_cmd + other = os.path.basename(other_path)[:-2] + + all_other = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1] if os.path.basename(n)[3:-3] != os.path.basename(other_path)[3:-2]]) + return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{other}.o -o {rnd_path}/{other}.so -L{TMP_PATH} {all_other}" + + +def import_to_ip(name): + # get a handle on the module + mdl = importlib.import_module(name) + # is there an __all__? if so respect it + if "__all__" in mdl.__dict__: + names = mdl.__dict__["__all__"] + else: + # otherwise we import all names that don't begin with _ + names = [x for x in mdl.__dict__ if not x.startswith("_")] + get_ipython().push({k: getattr(mdl, k) for k in names}) - # We first retrieve the current IPython interpreter - # instance. +@register_cell_magic +def cpp(line, cell, *what): + """Compile, execute C++ code wrapped with pybind11 and import it""" + args = parse_args(line) ip = get_ipython() + # get name of module + if args["cmake_path"] is None: + # TODO! + compile_id = get_gcc_cmd("dummy") + else: + compile_id = get_CMakeLists(args) + + hash_object = hashlib.sha256((compile_id+"|"+line+"|"+cell).encode()) + name = "libcpp_magic_"+hash_object.hexdigest()[:10] + run_cmake = False - cmake = "" - if cmake_path is not None: - with open(os.path.join(cmake_path,"CMakeLists.txt"), 'r') as f: - cmake = f.read() - # name = ''.join(random.choices(string.ascii_lowercase, k=10)) - hash_object = hashlib.sha256((cmake+" ".join(line)+function+cell).encode()) - name = "cpp_magic_"+hash_object.hexdigest()[:10] - if cmake_path is None: + # init temp folder and pybind etc + if args["cmake_path"] is None: ensure_tmp_folder() + args["path"] = TMP_PATH + CMakeLists = None else: - ensure_build_dir(cmake_path, clear=rebuild) - p = os.path.abspath(os.path.join(cmake_path,"build")) - run_cmake = not os.path.exists(os.path.join(p, "Makefile")) + ensure_build_dir(args["cmake_path"], clear=args["rebuild"]) + p = os.path.abspath(os.path.join(args["cmake_path"],"build")) if p not in sys.path: sys.path.append(p) - ensure_pybind(cmake_path) - - if cmake_path is not None: - path = cmake_path - else: - path = tmp_path - - # We define the source and executable filenames. + run_cmake = not os.path.exists(os.path.join(p, "Makefile")) + ensure_pybind(args["cmake_path"]) + args["path"] = args["cmake_path"] + CMakeLists = compile_id + build_needed = False try: + if args["rebuild"]: + raise Exception() + mdl = importlib.import_module(name) + except: + build_needed = True + + if build_needed: try: - if rebuild: - raise Exception() - mdl = importlib.import_module(name) - except: - exe = sys.executable + # add pybind11 function defs split = cell.split("defs") - if len(split) == 1 and function == "": + if len(split) == 1 and len(args["functions"]) == 0: raise Exception("You have to name a function as argument ('%%cpp -f ') or manual set defs") + # make sure that split is at least length 2 split.append("") - curr_defs = split[1] - if function != "": + for function in args["functions"]: curr_defs += def_template.format(function=function) - # We write the code to the C++ file. - with open(os.path.join(path,f"{name}.cpp"), 'w') as f: + with open(os.path.join(args["path"],f"{name}.cpp"), 'w') as f: f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}")) - - if cmake_path is None: - # We compile the C++ code into an executable. - command = f"{compiler_cmd} {' '.join(cmd_flags)} -O3 -Wall -shared -std=c++11 -fPIC $({exe} -m pybind11 --includes) {tmp_path}/{name}.cpp -o {tmp_path}/{name}.so" + if args["cmake_path"] is None: + # We compile the C++ code into an object file + command = get_gcc_cmd(name) compile = ip.getoutput(command) + # currently warnings will lead to an abort if len(compile) != 0: raise Exception("\n".join(compile)) + + # link into a shared object + linker = get_linker_cmd(name) + link = ip.getoutput(linker) + if len(link) != 0: + raise Exception("\n".join(link)) + + + # create unique folder + rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) + rnd_path = os.path.join(TMP_PATH, rnd) + if not os.path.exists(rnd_path): + os.makedirs(rnd_path) + with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f: + pass + # Rethink design decision: Maybe do not import all others again, maybe give cells names and reduce the amount of linking + # update all others by relinking and reimporting orderd by compilation time + for other_path in sorted(glob.glob(f"{TMP_PATH}/*.o"), key=os.path.getmtime): + other = os.path.basename(other_path)[:-2] + + + linker = get_unique_linker_cmd(other_path, rnd_path) + link = ip.getoutput(linker) + if len(link) != 0: + raise Exception("\n".join(link)) + + import_to_ip(f"{rnd}.{other}") + + shutil.rmtree(rnd_path) + else: - with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f: - f.write(cmake.replace("{name}",name)) + with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f: + f.write(CMakeLists.replace("{name}",name)) # TODO: Add option to add cmake flags if run_cmake: command = ["cmake", ".."] print("cmake:") - err, output = exec_command(command, cwd=os.path.join(cmake_path, "build")) + err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build")) if err: raise Exception(output) print(output) command = ["make"] print("make:") - err, output = exec_command(command, cwd=os.path.join(cmake_path, "build")) + err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build")) if err: raise Exception(output) print(output) - with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f: - f.write(cmake) + with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f: + f.write(CMakeLists) + except Exception as e: + # clean up even if error occurs + cleanup(args, name, CMakeLists) + raise e + import_to_ip(name) - # We execute the executable and return the output. - # get a handle on the module - mdl = importlib.import_module(name) - - if os.path.exists(os.path.join(path,f"{name}.cpp")): - os.remove(os.path.join(path,f"{name}.cpp")) - - # is there an __all__? if so respect it - if "__all__" in mdl.__dict__: - names = mdl.__dict__["__all__"] - else: - # otherwise we import all names that don't begin with _ - names = [x for x in mdl.__dict__ if not x.startswith("_")] - except Exception as e: - if cmake_path is not None: - with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f: - f.write(cmake) - if os.path.exists(os.path.join(path,f"{name}.cpp")): - os.remove(os.path.join(path,f"{name}.cpp")) - raise e - - # now drag them in - ip.push({k: getattr(mdl, k) for k in names}) + # clean up + cleanup(args, name, CMakeLists)