From ed77a6b4000d9ef8a7d3319b2220c357db4a49cb Mon Sep 17 00:00:00 2001 From: Ugo Finnendahl Date: Wed, 7 Jul 2021 12:19:36 +0200 Subject: [PATCH] better WIP --- examples/main.py | 19 ++- pybindmagic/__init__.py | 358 ++++++++++++++++++++++++++-------------- 2 files changed, 255 insertions(+), 122 deletions(-) diff --git a/examples/main.py b/examples/main.py index 0a58be1..f2d3921 100644 --- a/examples/main.py +++ b/examples/main.py @@ -5,7 +5,7 @@ import pybindmagic void hello_world() { - py::print("Hello World"); + py::print("Hello World..."); } #%% @@ -27,11 +27,27 @@ void callothercell() hello_world(); } #%% + callthiscell() callothercell() #try to update the first cell an run callothercell() again +#%% +%%cpp -f transitiv + +void callothercell(); + +void transitiv() +{ + py::print("Cell A:"); + callothercell(); +} +#%% + +transitiv() + + #%% %%cpp -f generateMesh #include @@ -63,6 +79,7 @@ std::tuple generateMesh() return std::make_tuple(V,F); } #%% +import JV9NK V,F = generateMesh() print("Vertices:\n",V) diff --git a/pybindmagic/__init__.py b/pybindmagic/__init__.py index b73df8a..cbb91a7 100644 --- a/pybindmagic/__init__.py +++ b/pybindmagic/__init__.py @@ -8,6 +8,9 @@ import string import hashlib import shutil import subprocess +import shlex + +from lief import parse, ELF compiler_cmd = "c++" @@ -29,10 +32,29 @@ def_template = '\nm.def("{function}", &{function});\n' TMP_PATH = "pbm_modules" +if os.path.exists(TMP_PATH): + shutil.rmtree(TMP_PATH) + +BUILD_DIR = os.path.join(TMP_PATH, "build") +MODULES_DIR = os.path.join(TMP_PATH, "modules") +LIB_DIR = os.path.join(TMP_PATH, "lib") # Make sure the correct python executeable is used using cmake os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"] +# def run_command(command): +# process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE) +# while True: +# output = process.stdout.readline() +# if output == '' and process.poll() is not None: +# break +# if output: +# print output.strip() +# rc = process.poll() +# return rc + def exec_command(cmd, cwd=None,**kwargs): + if isinstance(cmd, str): + cmd = shlex.split(cmd) try: output = subprocess.check_output(cmd,cwd=cwd,stderr=subprocess.STDOUT,**kwargs) if isinstance(output,bytes): @@ -44,12 +66,11 @@ def exec_command(cmd, cwd=None,**kwargs): return True, e.output -def ensure_tmp_folder(): - path = os.path.join(os.getcwd(), TMP_PATH) +def ensure_tmp_folder(path=TMP_PATH): + path = os.path.join(os.getcwd(), path) if not os.path.exists(path): os.makedirs(path) - if not path in sys.path: - sys.path.append(path) + def ensure_pybind(path, force=False): path = os.path.join(path,"pybind11") @@ -57,7 +78,8 @@ def ensure_pybind(path, force=False): if exists and force: shutil.rmtree(os.path.join(path,"pybind11")) if not exists: - err, output = exec_command(["git", "clone","https://github.com/pybind/pybind11.git",path]) + err, output = exec_command(["git", "clone", "https://github.com/pybind/pybind11.git", path]) + def ensure_build_dir(path, clear=False): path = os.path.join(path, "build") @@ -68,6 +90,15 @@ def ensure_build_dir(path, clear=False): os.makedirs(path) return exists and not clear + +def ensure_linker(): + path = os.path.join(TMP_PATH, "liblinker.so") + exists = os.path.exists(path) + if not os.path.exists(path): + return compile_linkerlib() + return + + def parse_args(line): args = line.split(" ") ret = {} @@ -92,6 +123,7 @@ def get_CMakeLists(args): CMakeLists = f.read() return CMakeLists + def cleanup(args, name, CMakeLists): if args["cmake_path"] is not None: with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f: @@ -99,20 +131,16 @@ def cleanup(args, name, CMakeLists): if os.path.exists(os.path.join(args["path"],f"{name}.cpp")): os.remove(os.path.join(args["path"],f"{name}.cpp")) + def get_gcc_cmd(name): pyexe = sys.executable - return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {TMP_PATH}/{name}.cpp -o {TMP_PATH}/{name}.o" + return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {BUILD_DIR}/{name}.cpp -o {BUILD_DIR}/{name}.o" -def get_linker_cmd(name): - others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1]]) - return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{name}.o -o {TMP_PATH}/{name}.so -L{TMP_PATH} {others}" -def get_unique_linker_cmd(other_path, rnd_path): - # TODO: Merge with get_linker_cmd - other = os.path.basename(other_path)[:-2] - - all_other = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1] if os.path.basename(n)[3:-3] != os.path.basename(other_path)[3:-2]]) - return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{other}.o -o {rnd_path}/{other}.so -L{TMP_PATH} {all_other}" +def get_linker_cmd(name, others=None): + if others is None: + others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{LIB_DIR}/*.so"), key=os.path.getmtime)[::-1]]) + return f"{compiler_cmd} -Wl,-rpath {LIB_DIR} -shared {BUILD_DIR}/{name}.o -o {MODULES_DIR}/{name}.so -L{LIB_DIR} {others}" def import_to_ip(name): @@ -126,9 +154,147 @@ def import_to_ip(name): names = [x for x in mdl.__dict__ if not x.startswith("_")] get_ipython().push({k: getattr(mdl, k) for k in names}) + +def get_importexport(file): + binary = parse(file) + symbols = binary.symbols + imported = set() + exported = set() + for i,symbol in enumerate(symbols): + if not symbol.imported and not symbol.exported and symbol.binding == ELF.SYMBOL_BINDINGS.GLOBAL: + imported.add(symbol.name) + if symbol.exported and symbol.binding == ELF.SYMBOL_BINDINGS.GLOBAL: + exported.add(symbol.name) + return imported, exported + + +export_graph = {} +import_graph = {} +symbols = {} + +def update_dependencies(fresh_compiled): + global export_graph, import_graph + imports, exports = get_importexport(fresh_compiled) + symbols[fresh_compiled] = (imports, exports) + for inp in imports: + if inp.startswith("Py") or inp.startswith("_Py"): + continue + if inp in import_graph: + import_graph[inp].append(fresh_compiled) + else: + import_graph[inp] = [fresh_compiled] + + for ex in exports: + if ex.startswith("PyInit"): + continue + export_graph[ex] = fresh_compiled + + +def get_list_of_rebuilds(fresh_compiled, need_recompilation=None): + global export_graph, import_graph + if need_recompilation is None: + need_recompilation = set() + need_recompilation.add(fresh_compiled) + exports = symbols[fresh_compiled][1] + for ex in exports: + if ex in import_graph: + for lib in import_graph[ex]: + if lib not in need_recompilation: + need_recompilation.add(lib) + # recursive collect all libs, that depend on this + get_list_of_rebuilds(lib, need_recompilation) + return need_recompilation + +def get_list_of_links(fresh_compiled): + global export_graph, import_graph + imports = symbols[fresh_compiled][0] + res = [] + for imp in imports: + if imp in export_graph: + lib = export_graph[imp] + if lib not in res: + res.append(lib) + return res + + +def bulk_import(need_reimport=None): + if need_reimport is None: + need_reimport = sorted(glob.glob(f"{MODULES_DIR}/*.so"), key=os.path.getmtime) + + # create unique folder + rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) + rnd_path = os.path.join(TMP_PATH, rnd) + if not os.path.exists(rnd_path): + os.makedirs(rnd_path) + with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f: + pass + + for lib_path in need_reimport: + lib = os.path.basename(lib_path)[:-3] + shutil.copyfile(lib_path, os.path.join(rnd_path,f"{lib}.so")) + + for lib_path in need_reimport: + lib = os.path.basename(lib_path)[:-3] + import_to_ip(f"{rnd}.{lib}") + shutil.rmtree(rnd_path) + + +def compile_module(cell, args): + name = args["name"] + ensure_tmp_folder(BUILD_DIR) + + create_cpp_file(BUILD_DIR, cell, args) + + ip = get_ipython() + # We compile the C++ code into an object file + command = get_gcc_cmd(name) + compile = ip.getoutput(command) + # currently warnings will lead to an abort + if len(compile) != 0: + raise Exception("\n".join(compile)) + + # link into a shared object, without links + linker = get_linker_cmd(name, "") + link = ip.getoutput(linker) + if len(link) != 0: + raise Exception("\n".join(link)) + + +def link_module(name, others=None): + other_name = os.path.basename(name)[:-3] + linker = get_linker_cmd(other_name, others) + link = get_ipython().getoutput(linker) + if len(link) != 0: + raise Exception("\n".join(link)) + + +version = 0 + +def create_cpp_file(path, cell, args): + name = args["name"] + # add pybind11 function defs + split = cell.split("defs") + if len(split) == 1 and len(args["functions"]) == 0: + raise Exception("You have to name a function as argument ('%%cpp -f ') or manual set defs") + # make sure that split is at least length 2 + split.append("") + curr_defs = split[1] + for function in args["functions"]: + curr_defs += def_template.format(function=function) + with open(os.path.join(path,f"{name}.cpp"), 'w') as f: + f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}")) + + +def get_newest(name): + return sorted(glob.glob(f"{LIB_DIR}/lib{name}*.so"), key=os.path.getmtime)[-1] + +history = [] + + +# TODO: TEST @register_cell_magic def cpp(line, cell, *what): - """Compile, execute C++ code wrapped with pybind11 and import it""" + global version args = parse_args(line) ip = get_ipython() # get name of module @@ -139,109 +305,59 @@ def cpp(line, cell, *what): compile_id = get_CMakeLists(args) hash_object = hashlib.sha256((compile_id+"|"+line+"|"+cell).encode()) - name = "libcpp_magic_"+hash_object.hexdigest()[:10] - run_cmake = False + name = "cpp_magic_"+hash_object.hexdigest()[:10] + args["name"] = name + + version += 1 + history.append(os.path.join(MODULES_DIR, f"{name}.so")) + + ensure_tmp_folder(TMP_PATH) + # Check if neccessary here + if not TMP_PATH in sys.path: + sys.path.append(TMP_PATH) + ensure_tmp_folder(MODULES_DIR) + ensure_tmp_folder(LIB_DIR) + + # if not yet compiled, do so + if not os.path.exists(os.path.join(MODULES_DIR,f"{name}.so")): + # TODO: extend to cmake building + compile_module(cell, args) + + update_dependencies(os.path.join(MODULES_DIR,f"{name}.so")) + relinks = list(get_list_of_rebuilds(os.path.join(MODULES_DIR,f"{name}.so"))) + relinks = sorted(relinks, key=os.path.getmtime)[::-1] + for re in relinks: + other_name = os.path.basename(re)[:-3] + shutil.copyfile(os.path.join(MODULES_DIR,f"{other_name}.so"), os.path.join(LIB_DIR,f"lib{other_name}{version}.so")) + # touch + open(os.path.join(LIB_DIR,f"lib{other_name}{version}.so"), 'a').close() + # TODO: delete old + + + # modules = sorted(glob.glob(f"{MODULES_DIR}/*.so"), key=os.path.getmtime)[::-1] + # others = " ".join([f"-l{os.path.basename(get_newest(os.path.basename(n)[:-3]))[3:-3]}" for n in modules]) + # print(others) + for re in relinks: + others = " ".join([f"-l{os.path.basename(get_newest(os.path.basename(n)[:-3]))[3:-3]}" for n in sorted(get_list_of_links(re), key=os.path.getmtime)[::-1]]) + link_module(re, others) + other_name = os.path.basename(re)[:-3] + shutil.copyfile(os.path.join(MODULES_DIR,f"{other_name}.so"), os.path.join(LIB_DIR,f"lib{other_name}{version}.so")) + + bulk_import([h for h in history if h in relinks]) - # init temp folder and pybind etc - if args["cmake_path"] is None: - ensure_tmp_folder() - args["path"] = TMP_PATH - CMakeLists = None - else: - ensure_build_dir(args["cmake_path"], clear=args["rebuild"]) - p = os.path.abspath(os.path.join(args["cmake_path"],"build")) - if p not in sys.path: - sys.path.append(p) - run_cmake = not os.path.exists(os.path.join(p, "Makefile")) - ensure_pybind(args["cmake_path"]) - args["path"] = args["cmake_path"] - CMakeLists = compile_id - build_needed = False - try: - if args["rebuild"]: - raise Exception() - mdl = importlib.import_module(name) - except: - build_needed = True - - if build_needed: - try: - # add pybind11 function defs - split = cell.split("defs") - if len(split) == 1 and len(args["functions"]) == 0: - raise Exception("You have to name a function as argument ('%%cpp -f ') or manual set defs") - # make sure that split is at least length 2 - split.append("") - curr_defs = split[1] - for function in args["functions"]: - curr_defs += def_template.format(function=function) - - # We write the code to the C++ file. - with open(os.path.join(args["path"],f"{name}.cpp"), 'w') as f: - f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}")) - - if args["cmake_path"] is None: - # We compile the C++ code into an object file - command = get_gcc_cmd(name) - compile = ip.getoutput(command) - # currently warnings will lead to an abort - if len(compile) != 0: - raise Exception("\n".join(compile)) - - # link into a shared object - linker = get_linker_cmd(name) - link = ip.getoutput(linker) - if len(link) != 0: - raise Exception("\n".join(link)) - - - # create unique folder - rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) - rnd_path = os.path.join(TMP_PATH, rnd) - if not os.path.exists(rnd_path): - os.makedirs(rnd_path) - with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f: - pass - # Rethink design decision: Maybe do not import all others again, maybe give cells names and reduce the amount of linking - # update all others by relinking and reimporting orderd by compilation time - for other_path in sorted(glob.glob(f"{TMP_PATH}/*.o"), key=os.path.getmtime): - other = os.path.basename(other_path)[:-2] - - - linker = get_unique_linker_cmd(other_path, rnd_path) - link = ip.getoutput(linker) - if len(link) != 0: - raise Exception("\n".join(link)) - - import_to_ip(f"{rnd}.{other}") - - shutil.rmtree(rnd_path) - - else: - with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f: - f.write(CMakeLists.replace("{name}",name)) - # TODO: Add option to add cmake flags - if run_cmake: - command = ["cmake", ".."] - print("cmake:") - err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build")) - if err: - raise Exception(output) - print(output) - command = ["make"] - print("make:") - err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build")) - if err: - raise Exception(output) - print(output) - with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f: - f.write(CMakeLists) - except Exception as e: - # clean up even if error occurs - cleanup(args, name, CMakeLists) - raise e - - import_to_ip(name) - - # clean up - cleanup(args, name, CMakeLists) +""" +CLEAR LIB_FOLDER +#### PIPELINE #### + +create name +add version +add name to history +compile to modules_dir/{name}.so (may be already there) +detect_relink_needs + +for all relinked mv modules_dir/{name}.so -> libs_dir/lib{name}{version}.so (delete old?) +relink +for all relinked mv modules_dir/{name}.so -> libs_dir/lib{name}{version}.so +import_bulk according to history + +"""