cmd line option links all cpp cells now

.gitignore

@@ -5,3 +5,4 @@ pbm_modules
 *.pyc
 __pycache__
 pybind11
+*.egg-info

examples/main.py

@@ -1,4 +1,37 @@
 import pybindmagic
+
+#%%
+%%cpp -f hello_world
+
+void hello_world()
+{
+    py::print("Hello World");
+}
+#%%
+
+hello_world()
+
+#%%
+%%cpp -f callothercell -f callthiscell
+
+void hello_world();
+
+void callthiscell()
+{
+    py::print("In this cell");
+}
+
+void callothercell()
+{
+    py::print("From Other Cell:");
+    hello_world();
+}
+#%%
+callthiscell()
+
+callothercell()
+#try to update the first cell an run callothercell() again
+
 #%%
 %%cpp -f generateMesh
 #include <pybind11/eigen.h>
@@ -82,13 +115,12 @@ defs
 printMesh(*generateMesh())
 
 #%%
-%%cpp -c pathtocmake -f viewMeshh
+%%cpp -c pathtocmake -f viewMesh
 
 #include <igl/opengl/glfw/Viewer.h>
 #include <pybind11/eigen.h>
 
-
+void viewMesh(Eigen::Matrix<double, Eigen::Dynamic, 3> V, Eigen::Matrix<int, Eigen::Dynamic, 3> F)
-void viewMeshh(Eigen::Matrix<double, Eigen::Dynamic, 3> V, Eigen::Matrix<int, Eigen::Dynamic, 3> F)
 {
   // Plot the mesh
   igl::opengl::glfw::Viewer viewer;

examples/test.ipynb

@@ -9,6 +9,63 @@
     "import pybindmagic"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%cpp -f hello_world\n",
+    "\n",
+    "void hello_world()\n",
+    "{\n",
+    "    py::print(\"Hello World\");\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hello_world()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%cpp -f callothercell -f callthiscell\n",
+    "\n",
+    "void hello_world();\n",
+    "\n",
+    "void callthiscell()\n",
+    "{\n",
+    "    py::print(\"In this cell\");\n",
+    "}\n",
+    "\n",
+    "void callothercell()\n",
+    "{\n",
+    "    py::print(\"From Other Cell:\");\n",
+    "    hello_world();\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "callthiscell()\n",
+    "\n",
+    "callothercell()\n",
+    "#try to update the first cell an run callothercell() again"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -196,6 +253,13 @@
     "    ], dtype=np.int32)-1\n",
     "viewMesh(V,F)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -218,7 +282,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.6"
+   "version": "3.8.3"
   },
   "latex_envs": {
    "LaTeX_envs_menu_present": true,

pybindmagic/__init__.py

@@ -11,7 +11,9 @@ import subprocess
 
 compiler_cmd = "c++"
 
-cmd_flags = ["$(pkg-config --cflags eigen3)"]
+optimize = "-O0"
+
+cmd_flags = ["$(pkg-config --cflags eigen3)", "-Wall"]
 
 template = """
 #include <pybind11/pybind11.h>
@@ -25,7 +27,7 @@ PYBIND11_MODULE({name}, m)
 
 def_template = '\nm.def("{function}", &{function});\n'
 
-tmp_path = "pbm_modules"
+TMP_PATH = "pbm_modules"
 
 # Make sure the correct python executeable is used using cmake
 os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"]
@@ -42,9 +44,8 @@ def exec_command(cmd, cwd=None,**kwargs):
         return True, e.output
 
 
-
 def ensure_tmp_folder():
-    path = os.path.join(os.getcwd(), tmp_path)
+    path = os.path.join(os.getcwd(), TMP_PATH)
     if not os.path.exists(path):
         os.makedirs(path)
     if not path in sys.path:
@@ -67,120 +68,180 @@ def ensure_build_dir(path, clear=False):
         os.makedirs(path)
     return exists and not clear
 
-@register_cell_magic
+def parse_args(line):
-def cpp(line, cell, *what):
-    """Compile, execute C++ code wrapped with pybind11 and import it"""
     args = line.split(" ")
-    rebuild = False
+    ret = {}
-    cmake_path = None
+    ret["cmake_path"] = None
-    function = ""
+    ret["functions"] = []
-    run_cmake = False
+    ret["rebuild"] = False
     # TODO: Add full argument parser
     for i,arg in enumerate(args):
         if arg == "-c":
-            cmake_path = args[i+1].strip()
+            ret["cmake_path"] = args[i+1].strip()
         if arg == "-f":
-            function = args[i+1].strip()
+            ret["functions"].append(args[i+1].strip())
         if arg == "-rebuild":
-            rebuild = True
+            ret["rebuild"] = True
+    return ret
+
+
+def get_CMakeLists(args):
+    CMakeLists = ""
+    if args["cmake_path"] is not None:
+        with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'r') as f:
+            CMakeLists = f.read()
+    return CMakeLists
 
+def cleanup(args, name, CMakeLists):
+    if args["cmake_path"] is not None:
+        with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
+            f.write(CMakeLists)
+    if os.path.exists(os.path.join(args["path"],f"{name}.cpp")):
+        os.remove(os.path.join(args["path"],f"{name}.cpp"))
+
+def get_gcc_cmd(name):
+    pyexe = sys.executable
+    return f"{compiler_cmd} {' '.join(cmd_flags)} {optimize} -std=c++11 -fPIC $({pyexe} -m pybind11 --includes) -c {TMP_PATH}/{name}.cpp -o {TMP_PATH}/{name}.o"
+
+def get_linker_cmd(name):
+    others = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1]])
+    return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{name}.o -o {TMP_PATH}/{name}.so -L{TMP_PATH} {others}"
+
+def get_unique_linker_cmd(other_path, rnd_path):
+    # TODO: Merge with get_linker_cmd
+    other = os.path.basename(other_path)[:-2]
+
+    all_other = " ".join([f"-l{os.path.basename(n)[3:-3]}" for n in sorted(glob.glob(f"{TMP_PATH}/*.so"), key=os.path.getmtime)[::-1] if os.path.basename(n)[3:-3] != os.path.basename(other_path)[3:-2]])
+    return f"{compiler_cmd} -Wl,-rpath {TMP_PATH} -shared {TMP_PATH}/{other}.o -o {rnd_path}/{other}.so -L{TMP_PATH} {all_other}"
+
+
+def import_to_ip(name):
+    # get a handle on the module
+    mdl = importlib.import_module(name)
+    # is there an __all__?  if so respect it
+    if "__all__" in mdl.__dict__:
+        names = mdl.__dict__["__all__"]
+    else:
+        # otherwise we import all names that don't begin with _
+        names = [x for x in mdl.__dict__ if not x.startswith("_")]
+    get_ipython().push({k: getattr(mdl, k) for k in names})
 
-    # We first retrieve the current IPython interpreter
+@register_cell_magic
-    # instance.
+def cpp(line, cell, *what):
+    """Compile, execute C++ code wrapped with pybind11 and import it"""
+    args = parse_args(line)
     ip = get_ipython()
+    # get name of module
+    if args["cmake_path"] is None:
+        # TODO!
+        compile_id = get_gcc_cmd("dummy")
+    else:
+        compile_id = get_CMakeLists(args)
 
-    cmake = ""
+    hash_object = hashlib.sha256((compile_id+"|"+line+"|"+cell).encode())
-    if cmake_path is not None:
+    name = "libcpp_magic_"+hash_object.hexdigest()[:10]
-        with open(os.path.join(cmake_path,"CMakeLists.txt"), 'r') as f:
+    run_cmake = False
-            cmake = f.read()
+
-    # name = ''.join(random.choices(string.ascii_lowercase, k=10))
+    # init temp folder and pybind etc
-    hash_object = hashlib.sha256((cmake+" ".join(line)+function+cell).encode())
+    if args["cmake_path"] is None:
-    name = "cpp_magic_"+hash_object.hexdigest()[:10]
-    if cmake_path is None:
         ensure_tmp_folder()
+        args["path"] = TMP_PATH
+        CMakeLists = None
     else:
-        ensure_build_dir(cmake_path, clear=rebuild)
+        ensure_build_dir(args["cmake_path"], clear=args["rebuild"])
-        p = os.path.abspath(os.path.join(cmake_path,"build"))
+        p = os.path.abspath(os.path.join(args["cmake_path"],"build"))
-        run_cmake = not os.path.exists(os.path.join(p, "Makefile"))
         if p not in sys.path:
             sys.path.append(p)
-        ensure_pybind(cmake_path)
+        run_cmake = not os.path.exists(os.path.join(p, "Makefile"))
-
+        ensure_pybind(args["cmake_path"])
-    if cmake_path is not None:
+        args["path"] = args["cmake_path"]
-        path = cmake_path
+        CMakeLists = compile_id
-    else:
+    build_needed = False
-        path = tmp_path
-
-    # We define the source and executable filenames.
-    try:
     try:
-            if rebuild:
+        if args["rebuild"]:
             raise Exception()
         mdl = importlib.import_module(name)
     except:
-            exe = sys.executable
+        build_needed = True
+
+    if build_needed:
+        try:
+            # add pybind11 function defs
             split = cell.split("defs")
-            if len(split) == 1 and function == "":
+            if len(split) == 1 and len(args["functions"]) == 0:
                 raise Exception("You have to name a function as argument ('%%cpp -f <functionname>') or manual set defs")
+            # make sure that split is at least length 2
             split.append("")
-
             curr_defs = split[1]
-            if function != "":
+            for function in args["functions"]:
                 curr_defs += def_template.format(function=function)
 
-
             # We write the code to the C++ file.
-            with open(os.path.join(path,f"{name}.cpp"), 'w') as f:
+            with open(os.path.join(args["path"],f"{name}.cpp"), 'w') as f:
                 f.write(template.format(cell_code=split[0],name=name,defs="{"+curr_defs+"}"))
 
-
+            if args["cmake_path"] is None:
-            if cmake_path is None:
+                # We compile the C++ code into an object file
-                # We compile the C++ code into an executable.
+                command = get_gcc_cmd(name)
-                command = f"{compiler_cmd} {' '.join(cmd_flags)} -O3 -Wall -shared -std=c++11 -fPIC $({exe} -m pybind11 --includes) {tmp_path}/{name}.cpp -o {tmp_path}/{name}.so"
                 compile = ip.getoutput(command)
+                # currently warnings will lead to an abort
                 if len(compile) != 0:
                     raise Exception("\n".join(compile))
+
+                # link into a shared object
+                linker = get_linker_cmd(name)
+                link = ip.getoutput(linker)
+                if len(link) != 0:
+                    raise Exception("\n".join(link))
+
+
+                # create unique folder
+                rnd = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
+                rnd_path = os.path.join(TMP_PATH, rnd)
+                if not os.path.exists(rnd_path):
+                    os.makedirs(rnd_path)
+                    with open(os.path.join(rnd_path,"__init__.py"), 'w+') as f:
+                        pass
+                # Rethink design decision: Maybe do not import all others again, maybe give cells names and reduce the amount of linking
+                # update all others by relinking and reimporting orderd by compilation time
+                for other_path in sorted(glob.glob(f"{TMP_PATH}/*.o"), key=os.path.getmtime):
+                    other = os.path.basename(other_path)[:-2]
+
+
+                    linker = get_unique_linker_cmd(other_path, rnd_path)
+                    link = ip.getoutput(linker)
+                    if len(link) != 0:
+                        raise Exception("\n".join(link))
+
+                    import_to_ip(f"{rnd}.{other}")
+
+                shutil.rmtree(rnd_path)
+
             else:
-                with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f:
+                with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
-                    f.write(cmake.replace("{name}",name))
+                    f.write(CMakeLists.replace("{name}",name))
                 # TODO: Add option to add cmake flags
                 if run_cmake:
                     command = ["cmake", ".."]
                     print("cmake:")
-                    err, output = exec_command(command, cwd=os.path.join(cmake_path, "build"))
+                    err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build"))
                     if err:
                         raise Exception(output)
                     print(output)
                 command = ["make"]
                 print("make:")
-                err, output = exec_command(command, cwd=os.path.join(cmake_path, "build"))
+                err, output = exec_command(command, cwd=os.path.join(args["cmake_path"], "build"))
                 if err:
                     raise Exception(output)
                 print(output)
-                with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f:
+                with open(os.path.join(args["cmake_path"],"CMakeLists.txt"), 'w') as f:
-                    f.write(cmake)
+                    f.write(CMakeLists)
-
-
-        # We execute the executable and return the output.
-        # get a handle on the module
-        mdl = importlib.import_module(name)
-
-        if os.path.exists(os.path.join(path,f"{name}.cpp")):
-            os.remove(os.path.join(path,f"{name}.cpp"))
-
-        # is there an __all__?  if so respect it
-        if "__all__" in mdl.__dict__:
-            names = mdl.__dict__["__all__"]
-        else:
-            # otherwise we import all names that don't begin with _
-            names = [x for x in mdl.__dict__ if not x.startswith("_")]
         except Exception as e:
-        if cmake_path is not None:
+            # clean up even if error occurs
-            with open(os.path.join(cmake_path,"CMakeLists.txt"), 'w') as f:
+            cleanup(args, name, CMakeLists)
-                f.write(cmake)
-        if os.path.exists(os.path.join(path,f"{name}.cpp")):
-            os.remove(os.path.join(path,f"{name}.cpp"))
             raise e
 
-    # now drag them in
+    import_to_ip(name)
-    ip.push({k: getattr(mdl, k) for k in names})
+
+    # clean up
+    cleanup(args, name, CMakeLists)