Initial self-hosting commit

This is the initial version of our tool that is able to
build itself. In can be bootstrapped by

  ./bin/bootstrap.py

Co-authored-by: Oliver Reiche <oliver.reiche@huawei.com>
Co-authored-by: Victor Moreno <victor.moreno1@huawei.com>

1 files changed, 185 insertions, 0 deletions

diff --git a/bin/bootstrap.py b/bin/bootstrap.py
new file mode 100755
index 00000000..9eb4a9cc
--- /dev/null
+++ b/bin/bootstrap.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+from pathlib import Path
+
+# path within the repository (constants)
+
+REPOS = "etc/repos.json"
+BOOTSTRAP_CC = ["clang++", "-std=c++20", "-DBOOTSTRAP_BUILD_TOOL"]
+MAIN_MODULE = ""
+MAIN_TARGET = "just"
+MAIN_STAGE = "src/buildtool/main/just"
+
+# relevant directories (global variables)
+
+SRCDIR = os.getcwd()
+WRKDIR = None
+DISTDIR = []
+
+def git_hash(content):
+  header = "blob {}\0".format(len(content)).encode('utf-8')
+  h = hashlib.sha1()
+  h.update(header)
+  h.update(content)
+  return h.hexdigest()
+
+def get_checksum(filename):
+    with open(filename, "rb") as f:
+        data = f.read()
+    return git_hash(data)
+
+def get_archive(*, distfile, fetch):
+    # Fetch the archive, if necessary. Return path to archive
+    for d in DISTDIR:
+        candidate_path = os.path.join(d, distfile)
+        if os.path.isfile(candidate_path):
+            return candidate_path
+    # Fetch to bootstrap working directory
+    fetch_dir = os.path.join(WRKDIR, "fetch")
+    os.makedirs(fetch_dir, exist_ok=True)
+    target = os.path.join(fetch_dir, distfile)
+    subprocess.run(["wget", "-O", target, fetch])
+    return target
+
+def run(cmd, *, cwd, **kwargs):
+    print("Running %r in %r" % (cmd, cwd), flush=True)
+    subprocess.run(cmd, cwd=cwd, check=True, **kwargs)
+
+def setup_deps():
+    # unpack all dependencies and return a list of
+    # additional C++ flags required
+    with open(os.path.join(SRCDIR, REPOS)) as f:
+        config = json.load(f)["repositories"]
+    include_location = os.path.join(WRKDIR, "dep_includes")
+    link_flags = []
+    os.makedirs(include_location)
+    for repo, total_desc in config.items():
+        desc = total_desc.get("repository", {})
+        if not isinstance(desc, dict):
+          # Indirect definition; we will set up the repository at the
+          # resolved place, which also has to be part of the global
+          # repository description.
+          continue
+        hints = total_desc.get("bootstrap", {})
+        if desc.get("type") in ["archive", "zip"]:
+            fetch = desc["fetch"]
+            distfile = desc.get("distfile") or os.path.basename(fetch)
+            archive = get_archive(distfile=distfile, fetch=fetch)
+            actual_checksum = get_checksum(archive)
+            expected_checksum = desc.get("content")
+            if actual_checksum != expected_checksum:
+                print("Checksum mismatch for %r. Expected %r, found %r"
+                      % (archive, expected_checksum, actual_checksum))
+            print("Unpacking %r from %r" % (repo, archive))
+            unpack_location = os.path.join(WRKDIR, "deps", repo)
+            os.makedirs(unpack_location)
+            if desc["type"] == "zip":
+                subprocess.run(["unzip", "-d", ".", archive],
+                               cwd=unpack_location, stdout=subprocess.DEVNULL)
+            else:
+                subprocess.run(["tar", "xf", archive],
+                               cwd=unpack_location)
+            subdir = os.path.join(unpack_location,
+                                  desc.get("subdir", "."))
+            include_dir = os.path.join(subdir,
+                                       hints.get("include_dir", "."))
+            include_name = hints.get("include_name", repo)
+            os.symlink(os.path.normpath(include_dir),
+                       os.path.join(include_location, include_name))
+            if "build" in hints:
+                run(["sh", "-c", hints["build"]], cwd=subdir)
+            if "link" in hints:
+                link_flags.extend(["-L", subdir])
+        if "link" in hints:
+            link_flags.extend(hints["link"])
+
+    return {
+        "include": ["-I", include_location],
+        "link": link_flags
+    }
+
+def bootstrap():
+    # TODO: add package build mode, building against preinstalled dependencies
+    # rather than building dependencies ourselves.
+    print("Bootstrapping in %r from sources %r, taking files from %r"
+          % (WRKDIR, SRCDIR, DISTDIR))
+    os.makedirs(WRKDIR, exist_ok=True)
+    dep_flags = setup_deps();
+    # handle proto
+    src_wrkdir = os.path.join(WRKDIR, "src")
+    shutil.copytree(SRCDIR, src_wrkdir)
+    flags = ["-I", src_wrkdir] + dep_flags["include"]
+    cpp_files = []
+    for root, dirs, files in os.walk(src_wrkdir):
+        if 'test' in dirs:
+            dirs.remove('test')
+        if 'execution_api' in dirs:
+            dirs.remove('execution_api')
+        for f in files:
+            if f.endswith(".cpp"):
+                cpp_files.append(os.path.join(root, f))
+    object_files = []
+    for f in cpp_files:
+        obj_file_name =f[:-len(".cpp")] + ".o"
+        object_files.append(obj_file_name)
+        cmd = BOOTSTRAP_CC + flags + ["-c", f, "-o", obj_file_name]
+        run(cmd, cwd=src_wrkdir)
+    bootstrap_just = os.path.join(WRKDIR, "bootstrap-just")
+    cmd = BOOTSTRAP_CC + ["-o", bootstrap_just] + object_files + dep_flags["link"]
+    run(cmd, cwd=src_wrkdir)
+    CONF_FILE = os.path.join(WRKDIR, "repo-conf.json")
+    LOCAL_ROOT = os.path.join(WRKDIR, ".just")
+    os.makedirs(LOCAL_ROOT, exist_ok=True)
+    run(["sh", "-c",
+         "cp `./bin/just-mr.py --always_file -C %s --local_build_root=%s setup just` %s"
+         % (REPOS, LOCAL_ROOT, CONF_FILE)],
+        cwd=src_wrkdir)
+    GRAPH = os.path.join(WRKDIR, "graph.json")
+    TO_BUILD = os.path.join(WRKDIR, "to_build.json")
+    run([bootstrap_just, "analyse",
+         "-C", CONF_FILE,
+         "--dump_graph", GRAPH,
+         "--dump_artifacts_to_build", TO_BUILD,
+         MAIN_MODULE, MAIN_TARGET],
+        cwd=src_wrkdir)
+    run(["./bin/bootstrap-traverser.py",
+         "-C", CONF_FILE,
+         "--default_workspace", src_wrkdir,
+         GRAPH, TO_BUILD],
+        cwd=src_wrkdir)
+    OUT = os.path.join(WRKDIR, "out")
+    run(["./out-boot/%s" % (MAIN_STAGE,),
+         "install", "-C", CONF_FILE,
+         "-o", OUT, "--local_build_root", LOCAL_ROOT,
+         MAIN_MODULE, MAIN_TARGET],
+        cwd=src_wrkdir)
+
+
+
+def main(args):
+    global SRCDIR
+    global WRKDIR
+    global DISTDIR
+    if len(args) > 1:
+        SRCDIR = os.path.abspath(args[1])
+    if len(args) > 2:
+        WRKDIR = os.path.abspath(args[2])
+
+    if not WRKDIR:
+        WRKDIR = tempfile.mkdtemp()
+    if not DISTDIR:
+        DISTDIR = [os.path.join(Path.home(), ".distfiles")]
+    bootstrap()
+
+if __name__ == "__main__":
+    # Parse options, set DISTDIR
+    main(sys.argv)