Merge pull request #34 from stuartarchibald/wip/scipy2024_demo_update2

Update for SciPy 2024 demo.

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          conda install python=3.12 pytest coverage numpy numba conda-build clangxx=14 numba/label/dev::llvmlite setuptools_scm cython pytest-cov
+          conda install python=3.12 pytest coverage numpy numba conda-build clangxx=14 numba/label/dev::llvmlite setuptools_scm cython pytest-cov pyyaml
       - name : Display packages and system info
         shell: bash -l {0}
         run: |
@@ -80,7 +80,7 @@ jobs:
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          conda install python=3.12 pytest coverage numpy numba conda-build clangxx=14 numba/label/dev::llvmlite setuptools_scm cython pytest-cov
+          conda install python=3.12 pytest coverage numpy numba conda-build clangxx=14 numba/label/dev::llvmlite setuptools_scm cython pytest-cov pyyaml
       - name : Display packages and system info
         shell: bash -l {0}
         run: |

buildscripts/conda_recipes/pixie/meta.yaml

@@ -37,6 +37,7 @@ test:
     - numba
     - clangxx 14.*
     - cython
+    - pyyaml
   commands:
     - pytest --pyargs pixie
 

pixie/cli.py

@@ -1,145 +1,107 @@
 import argparse
+import pathlib
 import os
-import subprocess
-import sysconfig
-import tempfile
 
-from llvmlite import binding as llvm
-
-from pixie.compiler import (
+from pixie import (
     PIXIECompiler,
     TranslationUnit,
     ExportConfiguration,
-    TargetDescription,
+    targets,
 )
 
 
-def tu_from_c_source(fname, build_directory, include_python=False):
-    # TODO put this in support.py or utils.py
-    outfile = os.path.join(build_directory, "tmp.bc")
-    cmd = [
-        "clang",
-        "-x",
-        "c",
-        "-fPIC",
-        "-mcmodel=small",
-        "-emit-llvm",
-        fname,
-        "-o",
-        outfile,
-        "-c",
-    ]
-    if include_python:
-        cmd.extend(["-I", sysconfig.get_path("include")])
-    # TODO: verbose
-    print(" ".join(cmd))
-    subprocess.run(cmd, check=True)
-    with open(outfile, "rb") as f:
-        data = f.read()
-    return TranslationUnit(fname, data)
-
-
-def c_from_cython_source(fname, build_directory):
-    outfile = os.path.join(build_directory, "".join([fname.split(".")[0], ".c"]))
-    cmd = ("cython", "-3", fname, "-o", outfile)
-    subprocess.run(cmd, check=True)
-    outfile = os.path.join(build_directory, "".join([fname.split(".")[0], ".c"]))
-    with open(outfile, "rt") as f:
-        data = f.read()
-    return data, outfile
-
-
-def default_test_config(triple=None):
-    # TODO put this in support.py or utils.py
-    # this just encodes some defaults for testing purposes
-    if triple is None:
-        triple = llvm.get_process_triple()
-
-    arch = triple.split("-")[0]
-    if arch == "x86_64":
-        from pixie.targets.x86_64 import features, predefined
-
-        return TargetDescription(
-            triple,
-            predefined.x86_64.cpu,
-            predefined.x86_64.features,
-            (
-                features.sse2,
-                predefined.x86_64_v2,
-                predefined.x86_64_v3,
-                predefined.x86_64_v4,
-                features.avx512bitalg,
-            ),
-        )
-    elif arch == "arm64":
-        from pixie.targets.arm64 import features, predefined
-
-        return TargetDescription(
-            triple,
-            predefined.apple_m1.cpu,
-            predefined.apple_m1.features,
-            (features.v8_5a, features.v8_6a, features.bf16),
-        )
+def _generate_parser(descr):
+    parser = argparse.ArgumentParser(description=descr)
+    parser.add_argument("-g", action="store_true",
+                        help="compile with debug info")
+    parser.add_argument("-v", action="store_true", help="enable verbose output")
+    parser.add_argument(
+        "-O", metavar="<n>", type=int, choices=(0, 1, 2, 3),
+        help="optimization level", default=3,)
+    parser.add_argument("-o", metavar="<lib>", help="output library")
+    parser.add_argument("files", help="input source files", nargs="+")
+    return parser
+
+
+def _translate_common_options(parser, args):
+    # opt >= 2 adds in vectorization
+    opt_flags = dict()
+    if args['O'] >= 2:
+        opt_flags['loop_vectorize'] = True
+        opt_flags['slp_vectorize'] = True
+
+    # use of `-g` needs to be passed to clang
+    clang_flags = ()
+    if args['g']:
+        clang_flags += ('-g',)
+
+    # work out the library name
+    if args['o'] is not None:
+        library_name = args['o']
+    elif len(args['files']) == 1:
+        base = os.path.basename(args['files'][0])
+        library_name = os.path.splitext(base)[0]
     else:
-        raise ValueError(f"Unsupported triple: '{triple}'.")
+        msg = ("Option -o (output library) is missing and cannot be inferred "
+               "as there are multiple input files.")
+        parser.error(msg)
+
+    return opt_flags, clang_flags, library_name
 
 
 def pixie_cc():
-    parser = argparse.ArgumentParser(description="pixie-cc")
-    parser.add_argument("-g", action="store_true", help="enable debug info")
-    parser.add_argument("-v", action="store_true", help="enable verbose")
-    parser.add_argument("-o", metavar="<lib>", help="output library")
-    parser.add_argument(
-        "-O", metavar="<n>", type=int, nargs=1, help="optimization level"
+
+    parser = _generate_parser("pixie-cc")
+    args = vars(parser.parse_args())
+
+    opt_flags, clang_flags, library_name = \
+        _translate_common_options(parser, args)
+
+    inps = args["files"]
+    tus = []
+    for inp in inps:
+        path = pathlib.Path(inp)
+        tus.append(TranslationUnit.from_c_source(str(path),
+                                                 extra_flags=clang_flags))
+
+    export_config = ExportConfiguration()
+    target_description = targets.get_default_configuration()
+    compiler = PIXIECompiler(
+        library_name=library_name,
+        translation_units=tus,
+        export_configuration=export_config,
+        **target_description,
+        opt_flags=opt_flags,
+        python_cext=True,  # TODO allow users to specify this
+        output_dir=".",  # TODO use $PWD for now
     )
-    parser.add_argument("c-source", help="input source file")
-    args = parser.parse_args()
-    # TODO: verbose
-    print(args)
-
-    with tempfile.TemporaryDirectory(prefix="__pxbld__") as build_directory:
-        tranlastion_units = [tu_from_c_source(vars(args)["c-source"], build_directory)]
-        export_config = ExportConfiguration()
-        target_description = default_test_config()
-        compiler = PIXIECompiler(
-            library_name=args.o,
-            translation_units=tranlastion_units,
-            export_configuration=export_config,
-            baseline_cpu=target_description.baseline_target.cpu,
-            baseline_features=target_description.baseline_target.features,
-            targets_features=target_description.additional_targets,
-            python_cext=True,  # TODO allow users to specify this
-            output_dir=".",  # TODO use $PWD for now
-        )
-        compiler.compile()
+    compiler.compile()
 
 
 def pixie_cythonize():
-    parser = argparse.ArgumentParser(description="pixie-cythonize")
-    parser.add_argument("-g", action="store_true", help="enable debug info")
-    parser.add_argument("-v", action="store_true", help="enable verbose")
-    parser.add_argument("-o", metavar="<lib>", help="output library")
-    parser.add_argument(
-        "-O", metavar="<n>", type=int, nargs=1, help="optimization level"
-    )
-    parser.add_argument("pyx-source", help="input source file")
-    args = parser.parse_args()
-    # TODO: verbose
-    print(args)
+    parser = _generate_parser("pixie-cythonize")
+    args = vars(parser.parse_args())
+
+    opt_flags, clang_flags, library_name = \
+        _translate_common_options(parser, args)
+
     # cythonize the source
-    with tempfile.TemporaryDirectory(prefix="__pxbld__") as build_directory:
-        data, cfile = c_from_cython_source(vars(args)["pyx-source"], build_directory)
-        tranlastion_units = [tu_from_c_source(cfile, build_directory, include_python=True)]
-        export_config = ExportConfiguration()
-        target_description = default_test_config()
-        compiler = PIXIECompiler(
-            library_name=args.o,
-            translation_units=tranlastion_units,
-            export_configuration=export_config,
-            baseline_cpu=target_description.baseline_target.cpu,
-            baseline_features=target_description.baseline_target.features,
-            targets_features=target_description.additional_targets,
-            python_cext=True,  # TODO allow users to specify this
-            output_dir=".",  # TODO use $PWD for now
-        )
-        compiler.compile()
+    inps = args["files"]
+    tus = []
+    for inp in inps:
+        path = pathlib.Path(inp)
+        tus.append(TranslationUnit.from_cython_source(str(path),
+                   extra_clang_flags=clang_flags))
+
+    export_config = ExportConfiguration()
+    target_description = targets.get_default_configuration()
+    compiler = PIXIECompiler(
+        library_name=library_name,
+        translation_units=tus,
+        export_configuration=export_config,
+        **target_description,
+        opt_flags=opt_flags,
+        python_cext=True,  # TODO allow users to specify this
+        output_dir=".",  # TODO use $PWD for now
+    )
+    compiler.compile()

pixie/codegen_helpers.py

@@ -423,10 +423,19 @@ def _pass_manager_builder(opt=2, loop_vectorize=False, slp_vectorize=False,
     return pmb
 
 
-def _module_pass_manager(tm, **kwargs):
+def module_pass_manager(tm, **kwargs):
     pm = llvm.create_module_pass_manager()
     tm.add_analysis_passes(pm)
 
     with _pass_manager_builder(**kwargs) as pmb:
         pmb.populate(pm)
     return pm
+
+
+def function_pass_manager(target_machine, llvm_module, **kwargs):
+    pm = llvm.create_function_pass_manager(llvm_module)
+    pm.add_target_library_info(llvm_module.triple)
+    target_machine.add_analysis_passes(pm)
+    with _pass_manager_builder(**kwargs) as pmb:
+        pmb.populate(pm)
+    return pm