# Copyright (c) 2018, 2025, Oracle and/or its affiliates.

# Copyright (c) 2013, Regents of the University of California

#

# All rights reserved.

#

# Redistribution and use in source and binary forms, with or without modification, are

# permitted provided that the following conditions are met:

#

# 1. Redistributions of source code must retain the above copyright notice, this list of

# conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright notice, this list of

# conditions and the following disclaimer in the documentation and/or other materials provided

# with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF

# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE

# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE

# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING

# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED

# OF THE POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

import contextlib

import datetime

import fnmatch

import glob

import itertools

import os

import pathlib

import re

import shlex

import shutil

import subprocess

import sys

import time

from functools import wraps

from pathlib import Path

from textwrap import dedent

import mx_graalpython_gradleproject

import mx_urlrewrites

if sys.version_info[0] < 3:

raise RuntimeError("The build scripts are no longer compatible with Python 2")

import tempfile

from argparse import ArgumentParser

from dataclasses import dataclass

import stat

from zipfile import ZipFile

import mx

import mx_util

import mx_benchmark

import mx_gate

import mx_native

import mx_unittest

import mx_sdk

import mx_subst

import mx_graalpython_bisect

import mx_graalpython_import

import mx_graalpython_python_benchmarks

# re-export custom mx project classes so they can be used from suite.py

from mx import MavenProject #pylint: disable=unused-import

from mx_cmake import CMakeNinjaProject #pylint: disable=unused-import

from mx_graalpython_gradleproject import GradlePluginProject #pylint: disable=unused-import

from mx_gate import Task

from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JAVA_DRIVER_BENCHMARKS, \

HEAP_BENCHMARKS

from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, \

GraalPythonVm, \

CONFIGURATION_DEFAULT, CONFIGURATION_SANDBOXED, CONFIGURATION_NATIVE, \

CONFIGURATION_DEFAULT_MULTI, CONFIGURATION_SANDBOXED_MULTI, CONFIGURATION_NATIVE_MULTI, \

CONFIGURATION_DEFAULT_MULTI_TIER, CONFIGURATION_NATIVE_MULTI_TIER, \

PythonVmWarmupBenchmarkSuite, \

CONFIGURATION_INTERPRETER, CONFIGURATION_INTERPRETER_MULTI, CONFIGURATION_NATIVE_INTERPRETER, \

CONFIGURATION_NATIVE_INTERPRETER_MULTI, PythonJavaEmbeddingBenchmarkSuite, python_java_embedding_vm_registry, \

GraalPythonJavaDriverVm, CONFIGURATION_JAVA_EMBEDDING_INTERPRETER_MULTI_SHARED, \

CONFIGURATION_JAVA_EMBEDDING_INTERPRETER_MULTI, CONFIGURATION_JAVA_EMBEDDING_MULTI_SHARED, \

CONFIGURATION_JAVA_EMBEDDING_MULTI, CONFIGURATION_PANAMA, PythonJMHDistMxBenchmarkSuite, \

PythonHeapBenchmarkSuite

if not sys.modules.get("__main__"):

# workaround for pdb++

sys.modules["__main__"] = type(sys)("<empty>")

def get_boolean_env(name, default=False):

env = os.environ.get(name)

if env is None:

return default

return env.lower() in ('true', '1')

SUITE = mx.suite('graalpython')

SUITE_COMPILER = mx.suite("compiler", fatalIfMissing=False)

GRAAL_VERSION = SUITE.suiteDict['version']

GRAAL_VERSION_MAJ_MIN = ".".join(GRAAL_VERSION.split(".")[:2])

PYTHON_VERSION = SUITE.suiteDict[f'{SUITE.name}:pythonVersion']

PYTHON_VERSION_MAJ_MIN = ".".join(PYTHON_VERSION.split('.')[:2])

# this environment variable is used by some of our maven projects and jbang integration to build against the unreleased master version during development

os.environ["GRAALPY_VERSION"] = GRAAL_VERSION

MAIN_BRANCH = 'master'

HPY_IMPORT_ORPHAN_BRANCH_NAME = "hpy-import"

GRAALPYTHON_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"

SANDBOXED_OPTIONS = [

'--experimental-options',

'--python.UseSystemToolchain=false', # Use the Sulong toolchain

'--python.NativeModules=false', # Load extensions with Sulong

'--python.PosixModuleBackend=java',

'--python.Sha3ModuleBackend=java',

]

# Allows disabling rebuild for some mx commands such as graalpytest

DISABLE_REBUILD = get_boolean_env('GRAALPYTHON_MX_DISABLE_REBUILD')

_COLLECTING_COVERAGE = False

CI = get_boolean_env("CI")

WIN32 = sys.platform == "win32"

BUILD_NATIVE_IMAGE_WITH_ASSERTIONS = get_boolean_env('BUILD_WITH_ASSERTIONS', CI)

mx_gate.add_jacoco_excludes([

"com.oracle.graal.python.pegparser.sst",

"com.oracle.graal.python.pegparser.test",

"com.oracle.truffle.api.staticobject.test",

"com.oracle.truffle.regex.tregex.test",

"com.oracle.truffle.tck",

"com.oracle.truffle.tools.chromeinspector.test",

"com.oracle.truffle.tools.coverage.test",

"com.oracle.truffle.tools.dap.test",

"com.oracle.truffle.tools.profiler.test",

"org.graalvm.tools.insight.test",

"org.graalvm.tools.lsp.test",

])

if CI and not os.environ.get("GRAALPYTEST_FAIL_FAST"):

os.environ["GRAALPYTEST_FAIL_FAST"] = "true"

def is_collecting_coverage():

return bool(mx_gate.get_jacoco_agent_args() or _COLLECTING_COVERAGE)

def wants_debug_build(flags=os.environ.get("CFLAGS", "")):

return any(x in flags for x in ["-g", "-ggdb", "-ggdb3"])

if wants_debug_build():

mx_native.DefaultNativeProject._original_cflags = mx_native.DefaultNativeProject.cflags

mx_native.DefaultNativeProject.cflags = property(

lambda self: self._original_cflags + (["/Z7"] if WIN32 else ["-fPIC", "-ggdb3"])

)

if WIN32:

# we need the .lib for pythonjni

original_DefaultNativeProject_getArchivableResults = mx_native.DefaultNativeProject.getArchivableResults

def getArchivableResultsWithLib(self, *args, **kwargs):

for result in original_DefaultNativeProject_getArchivableResults(self, *args, **kwargs):

if any(r.endswith("pythonjni.dll") for r in result):

yield tuple(r.replace(".dll", ".lib") for r in result)

yield result

mx_native.DefaultNativeProject.getArchivableResults = getArchivableResultsWithLib

# let's check if VS compilers are on the PATH

if not os.environ.get("LIB"):

mx.log("LIB not in environment, not a VS shell")

elif not os.environ.get("INCLUDE"):

mx.log("INCLUDE not in environment, not a VS shell")

else:

for p in os.environ.get("PATH", "").split(os.pathsep):

if os.path.isfile(os.path.join(os.path.abspath(p), "cl.exe")):

mx.log("LIB and INCLUDE set, cl.exe on PATH, assuming this is a VS shell")

os.environ["DISTUTILS_USE_SDK"] = "1"

break

else:

mx.log("cl.exe not on PATH, not a VS shell")

def _sibling(filename):

return os.path.join(os.path.dirname(__file__), filename)

def _get_core_home():

return os.path.join(SUITE.dir, "graalpython", "lib-graalpython")

def _get_stdlib_home():

return os.path.join(SUITE.dir, "graalpython", "lib-python", "3")

def _get_capi_home(args=None):

return os.path.join(mx.distribution("GRAALPYTHON_NATIVE_LIBS").get_output(), mx.get_os(), mx.get_arch())

_get_jni_home = _get_capi_home

def _extract_graalpython_internal_options(args):

non_internal = []

additional_dists = []

for arg in args:

# Class path extensions

if arg.startswith('-add-dist='):

additional_dists += [arg[10:]]

else:

non_internal += [arg]

return non_internal, additional_dists

def extend_os_env(**kwargs):

env = os.environ.copy()

env.update(**kwargs)

return env

def delete_bad_env_keys(env):

for k in ["SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"]:

if k in env:

del env[k]

def check_vm(vm_warning=True, must_be_jvmci=False):

if not SUITE_COMPILER:

if must_be_jvmci:

mx.abort('** Error ** : graal compiler was not found!')

sys.exit(1)

if vm_warning:

mx.log('** warning ** : graal compiler was not found!! Executing using standard VM..')

def get_jdk():

return mx.get_jdk()

def full_python(args, env=None):

"""Run python from standalone build (unless kwargs are given). Does not build GraalPython sources automatically."""

if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):

args.insert(0, '--python.WithJavaStacktrace=1')

if "--hosted" in args[:2]:

args.remove("--hosted")

return python(args)

if '--vm.da' not in args:

args.insert(0, '--vm.ea')

if not any(arg.startswith('--experimental-options') for arg in args):

args.insert(0, '--experimental-options')

handle_debug_arg(args)

for arg in itertools.chain(

itertools.chain(*map(shlex.split, reversed(mx._opts.java_args_sfx))),

reversed(shlex.split(mx._opts.java_args if mx._opts.java_args else "")),

itertools.chain(*map(shlex.split, reversed(mx._opts.java_args_pfx))),

):

if arg.startswith("-"):

args.insert(0, f"--vm.{arg[1:]}")

else:

mx.warn(f"Dropping {arg}, cannot pass it to launcher")

standalone_home = graalpy_standalone_home('jvm', dev=True, build=False)

graalpy_path = os.path.join(standalone_home, 'bin', _graalpy_launcher())

if not os.path.exists(graalpy_path):

mx.abort("GraalPy standalone doesn't seem to be built.\n" +

"To build it: mx python-jvm\n" +

"Alternatively use: mx python --hosted")

mx.run([graalpy_path] + args, env=env)

def handle_debug_arg(args):

if mx._opts.java_dbg_port:

args.insert(0,

f"--vm.agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=127.0.0.1:{mx._opts.java_dbg_port}")

def python(args, **kwargs):

"""run a Python program or shell"""

if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):

args.insert(0, '--python.WithJavaStacktrace=1')

do_run_python(args, **kwargs)

def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None, cp_prefix=None, cp_suffix=None, main_class=GRAALPYTHON_MAIN_CLASS, minimal=False, **kwargs):

experimental_opt_added = False

if not any(arg.startswith("--python.CAPI") for arg in args):

capi_home = _get_capi_home()

args.insert(0, "--python.CAPI=%s" % capi_home)

args.insert(0, "--experimental-options")

experimental_opt_added = True

if not any(arg.startswith("--python.JNIHome") for arg in args):

args.insert(0, "--python.JNIHome=" + _get_jni_home())

if not experimental_opt_added:

args.insert(0, "--experimental-options")

if not env:

env = os.environ.copy()

env.setdefault("GRAAL_PYTHONHOME", _dev_pythonhome())

delete_bad_env_keys(env)

check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False)

if check_vm_env:

if check_vm_env == '1':

check_vm(must_be_jvmci=True)

elif check_vm_env == '0':

check_vm()

if minimal:

x = [x for x in SUITE.dists if x.name == "GRAALPYTHON"][0]

dists = [dep for dep in x.deps if dep.isJavaProject() or dep.isJARDistribution()]

# Hack: what we should just do is + ['GRAALPYTHON_VERSIONS_MAIN'] and let MX figure out

# the class-path and other VM arguments necessary for it. However, due to a bug in MX,

# LayoutDirDistribution causes an exception if passed to mx.get_runtime_jvm_args,

# because it does not properly initialize its super class ClasspathDependency, see MX PR: 1665.

ver_dep = mx.dependency('GRAALPYTHON_VERSIONS_MAIN').get_output()

cp_prefix = ver_dep if cp_prefix is None else (ver_dep + os.pathsep + cp_prefix)

else:

dists = ['GRAALPYTHON']

dists += ['TRUFFLE_NFI', 'SULONG_NATIVE', 'GRAALPYTHON-LAUNCHER']

vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)

if minimal:

vm_args.insert(0, f"-Dorg.graalvm.language.python.home={_dev_pythonhome()}")

graalpython_args, additional_dists = _extract_graalpython_internal_options(graalpython_args)

dists += additional_dists

if extra_dists:

dists += extra_dists

if not CI:

# Try eagerly to include tools for convenience when running Python

if not mx.suite("tools", fatalIfMissing=False):

SUITE.import_suite("tools", version=None, urlinfos=None, in_subdir=True)

if mx.suite("tools", fatalIfMissing=False):

for tool in ["CHROMEINSPECTOR", "TRUFFLE_COVERAGE"]:

if os.path.exists(mx.suite("tools").dependency(tool).path):

dists.append(tool)

graalpython_args.insert(0, '--experimental-options=true')

vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk, cp_prefix=cp_prefix, cp_suffix=cp_suffix)

if not jdk:

jdk = get_jdk()

# default: assertion checking is enabled

if extra_vm_args is None or '-da' not in extra_vm_args:

vm_args += ['-ea', '-esa']

if extra_vm_args:

vm_args += extra_vm_args

vm_args.append(main_class)

return mx.run_java(vm_args + graalpython_args, jdk=jdk, env=env, **kwargs)

def node_footprint_analyzer(args, **kwargs):

main_class = 'com.oracle.graal.python.test.advanced.NodeFootprintAnalyzer'

vm_args = mx.get_runtime_jvm_args(['GRAALPYTHON_UNIT_TESTS', 'GRAALPYTHON', 'TRUFFLE_NFI', 'SULONG_NATIVE'])

return mx.run_java(vm_args + [main_class] + args, **kwargs)

def _dev_pythonhome_context():

home = os.environ.get("GRAAL_PYTHONHOME", _dev_pythonhome())

return set_env(GRAAL_PYTHONHOME=home)

def _dev_pythonhome():

return os.path.join(SUITE.dir, "graalpython")

def punittest(ars, report=False):

"""

Runs GraalPython junit tests and memory leak tests, which can be skipped using --no-leak-tests.

Pass --regex to further filter the junit and TSK tests. GraalPy tests are always run in two configurations:

with language home on filesystem and with language home served from the Truffle resources.

"""

path = os.environ.get("PATH", "")

if mx.is_linux() and not shutil.which("patchelf"):

venv = Path(SUITE.get_output_root()).absolute() / "patchelf-venv"

path += os.pathsep + str(venv / "bin")

if not shutil.which("patchelf", path=path):

mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [patchelf not found on PATH]")

t0 = time.time()

subprocess.check_call([sys.executable, "-m", "venv", str(venv)])

subprocess.check_call([str(venv / "bin" / "pip"), "install", "patchelf"])

mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [duration: {time.time() - t0}]")

args = [] if ars is None else ars

@dataclass

class TestConfig:

identifier: str

args: list

useResources: bool

reportConfig: bool = report

def __str__(self):

return f"args={self.args!r}, useResources={self.useResources}, report={self.reportConfig}"

configs = []

skip_leak_tests = False

if "--no-leak-tests" in args:

skip_leak_tests = True

args.remove("--no-leak-tests")

if is_collecting_coverage():

skip_leak_tests = True

vm_args = ['-Dpolyglot.engine.WarnInterpreterOnly=false']

# Note: we must use filters instead of --regex so that mx correctly processes the unit test configs,

# but it is OK to apply --regex on top of the filters

graalpy_tests = ['com.oracle.graal.python.test', 'com.oracle.graal.python.pegparser.test', 'org.graalvm.python.embedding.test']

has_compiler = bool(mx.suite('compiler', fatalIfMissing=False))

configs += [

TestConfig("junit", vm_args + graalpy_tests + args, True),

TestConfig("junit", vm_args + graalpy_tests + args, False),

]

if mx.is_linux():

# see GR-60656 and GR-60658 for what's missing in darwin and windows support

configs.append(

# MultiContext cext tests should run by themselves so they aren't influenced by others

TestConfig("multi-cext", vm_args + ['org.graalvm.python.embedding.cext.test'] + args + (["--use-graalvm"] if has_compiler else []), True),

)

if '--regex' not in args:

async_regex = ['--regex', r'com\.oracle\.graal\.python\.test\.integration\.advanced\.AsyncActionThreadingTest']

configs.append(TestConfig("async", vm_args + ['-Dpython.AutomaticAsyncActions=false', 'com.oracle.graal.python.test', 'org.graalvm.python.embedding.test'] + async_regex + args, True, False))

else:

skip_leak_tests = True

for c in configs:

mx.log(f"Python JUnit tests configuration: {c}")

PythonMxUnittestConfig.useResources = c.useResources

with set_env(PATH=path):

mx_unittest.unittest(c.args, test_report_tags=({"task": f"punittest-{c.identifier}-{'w' if c.useResources else 'wo'}-resources"} if c.reportConfig else None))

if skip_leak_tests:

return

# test leaks with Python code only

run_leak_launcher(["--code", "pass", ])

run_leak_launcher(["--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"])

# test leaks when some C module code is involved

run_leak_launcher(["--code", 'import _testcapi, mmap, bz2; print(memoryview(b"").nbytes)'])

# test leaks with shared engine Python code only

run_leak_launcher(["--shared-engine", "--code", "pass"])

run_leak_launcher(["--shared-engine", "--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"])

# test leaks with shared engine when some C module code is involved

run_leak_launcher(["--shared-engine", "--code", 'import _testcapi, mmap, bz2; print(memoryview(b"").nbytes)'])

run_leak_launcher(["--shared-engine", "--code", '[10, 20]', "--python.UseNativePrimitiveStorageStrategy=true",

"--forbidden-class", "com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage",

"--forbidden-class", "com.oracle.graal.python.runtime.native_memory.NativePrimitiveReference"])

run_leak_launcher(["--code", '[10, 20]', "--python.UseNativePrimitiveStorageStrategy=true",

"--forbidden-class", "com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage",

"--forbidden-class", "com.oracle.graal.python.runtime.native_memory.NativePrimitiveReference"])

PYTHON_ARCHIVES = ["GRAALPYTHON_GRAALVM_SUPPORT"]

PYTHON_NATIVE_PROJECTS = ["python-libbz2",

"python-liblzma",

"python-libzsupport",

"python-libposix",

"com.oracle.graal.python.hpy.llvm",

"com.oracle.graal.python.jni",

"com.oracle.graal.python.cext"]

def nativebuild(args):

"Build the non-Java Python projects and archives"

mx.build(["--dependencies", ",".join(PYTHON_NATIVE_PROJECTS + PYTHON_ARCHIVES)])

def nativeclean(args):

"Clean the non-Java Python projects"

mx.clean(["--dependencies", ",".join(PYTHON_NATIVE_PROJECTS + PYTHON_ARCHIVES)])

def python3_unittests(args):

"""run the cPython stdlib unittests"""

mx.run([sys.executable, "graalpython/com.oracle.graal.python.test/src/python_unittests.py", "-v"] + args)

def compare_unittests(args):

"""compare the output of two runs of the cPython stdlib unittests"""

mx.run([sys.executable, "graalpython/com.oracle.graal.python.test/src/compare_unittests.py", "-v"] + args)

class GraalPythonTags(object):

junit = 'python-junit'

junit_maven = 'python-junit-maven'

unittest = 'python-unittest'

unittest_cpython = 'python-unittest-cpython'

unittest_sandboxed = 'python-unittest-sandboxed'

unittest_multi = 'python-unittest-multi-context'

unittest_jython = 'python-unittest-jython'

unittest_arrow = 'python-unittest-arrow-storage'

unittest_hpy = 'python-unittest-hpy'

unittest_hpy_sandboxed = 'python-unittest-hpy-sandboxed'

unittest_posix = 'python-unittest-posix'

unittest_standalone = 'python-unittest-standalone'

unittest_gradle_plugin = 'python-unittest-gradle-plugin'

unittest_gradle_plugin_long_run = 'python-unittest-gradle-plugin-long-run'

unittest_maven_plugin = 'python-unittest-maven-plugin'

unittest_maven_plugin_long_run = 'python-unittest-maven-plugin-long-run'

junit_vfsutils = 'python-junit-vfsutils'

tagged = 'python-tagged-unittest'

svmunit = 'python-svm-unittest'

svmunit_sandboxed = 'python-svm-unittest-sandboxed'

graalvm = 'python-graalvm'

embedding = 'python-standalone-embedding'

graalvm_sandboxed = 'python-graalvm-sandboxed'

svm = 'python-svm'

native_image_embedder = 'python-native-image-embedder'

license = 'python-license'

language_checker = 'python-language-checker'

exclusions_checker = 'python-class-exclusion-checker'

def python_gate(args):

if not os.environ.get("JDT"):

os.environ["JDT"] = "builtin"

if not os.environ.get("ECLIPSE_EXE"):

find_eclipse()

if "--tags" not in args:

args += ["--tags"]

tags = ["style"]

for x in dir(GraalPythonTags):

v = getattr(GraalPythonTags, x)

if isinstance(v, str) and v.startswith("python-"):

if "sandboxed" not in v:

tags.append(v)

args.append(",".join(tags))

mx.log("Running mx python-gate " + " ".join(args))

return mx.command_function("gate")(args)

python_gate.__doc__ = 'Custom gates are %s' % ", ".join([

getattr(GraalPythonTags, t) for t in dir(GraalPythonTags) if not t.startswith("__")

])

def find_eclipse():

pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))

for f in [os.path.join(SUITE.dir, f)

for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f)

for f in os.listdir(pardir)]:

if os.path.basename(f) == "eclipse" and os.path.isdir(f):

mx.log("Automatically choosing %s for Eclipse" % f)

eclipse_exe = os.path.join(f, f"eclipse{'.exe' if mx.is_windows() else ''}")

if os.path.exists(eclipse_exe):

os.environ["ECLIPSE_EXE"] = eclipse_exe

return

@contextlib.contextmanager

def set_env(**environ):

"""Temporarily set the process environment variables"""

old_environ = dict(os.environ)

for k, v in environ.items():

if v is None:

if k in os.environ:

del os.environ[k]

else:

os.environ[k] = v

try:

yield

finally:

os.environ.clear()

os.environ.update(old_environ)

def _graalpy_launcher():

name = 'graalpy'

return f"{name}.exe" if WIN32 else name

def graalpy_standalone_home(standalone_type, enterprise=False, dev=False, build=True):

assert standalone_type in ['native', 'jvm']

assert not (enterprise and dev), "EE dev standalones are not implemented yet"

jdk_version = mx.get_jdk().version

# Check if GRAALPY_HOME points to some compatible pre-built GraalPy standalone

python_home = os.environ.get("GRAALPY_HOME", None)

if python_home and "*" in python_home:

python_home = os.path.abspath(glob.glob(python_home)[0])

mx.log("Using GraalPy standalone from GRAALPY_HOME: " + python_home)

# Try to verify that we're getting what we expect:

has_java = os.path.exists(os.path.join(python_home, 'jvm', 'bin', mx.exe_suffix('java')))

if has_java != (standalone_type == 'jvm'):

mx.abort(f"GRAALPY_HOME is not compatible with the requested distribution type.\n"

f"jvm/bin/java exists?: {has_java}, requested type={standalone_type}.")

line = ''

with open(os.path.join(python_home, 'release'), 'r') as f:

while 'JAVA_VERSION=' not in line:

line = f.readline()

if 'JAVA_VERSION=' not in line:

mx.abort(f"GRAALPY_HOME does not contain 'release' file. Cannot check Java version.")

actual_jdk_version = mx.VersionSpec(line.strip('JAVA_VERSION=').strip(' "\n\r'))

if actual_jdk_version != jdk_version:

mx.abort(f"GRAALPY_HOME is not compatible with the requested JDK version.\n"

f"actual version: '{actual_jdk_version}', version string: {line}, requested version: {jdk_version}.")

launcher = os.path.join(python_home, 'bin', _graalpy_launcher())

out = mx.OutputCapture()

import_ee_status = mx.run([launcher, "-c", "import sys; assert 'Oracle GraalVM' in sys.version"], nonZeroIsFatal=False, out=out, err=out)

if enterprise != (import_ee_status == 0):

mx.abort(f"GRAALPY_HOME is not compatible with requested distribution kind ({import_ee_status=}, {enterprise=}, {out=}).")

return python_home

env_file = 'ce-python'

vm_suite_path = os.path.join(mx.suite('truffle').dir, '..', 'vm')

svm_component = '_SVM'

if enterprise:

env_file = 'ee-python'

enterprise_suite = mx.suite('graal-enterprise', fatalIfMissing=False)

enterprise_suite_dir = enterprise_suite.dir if enterprise_suite else os.path.join(SUITE.dir, '..', 'graal-enterprise', 'graal-enterprise')

vm_suite_path = os.path.join(enterprise_suite_dir, '..', 'vm-enterprise')

svm_component = '_SVM_SVMEE'

if dev:

if standalone_type == 'jvm':

svm_component = ''

mx_args = ['--dy', '/vm']

else:

dev_env_file = os.path.join(os.path.abspath(SUITE.dir), 'mx.graalpython/graalpython-svm-standalone')

mx_args = ['-p', vm_suite_path, '--env', dev_env_file]

else:

mx_args = ['-p', vm_suite_path, '--env', env_file]

mx_args.append("--extra-image-builder-argument=-g")

if BUILD_NATIVE_IMAGE_WITH_ASSERTIONS:

mx_args.append("--extra-image-builder-argument=-ea")

if mx_gate.get_jacoco_agent_args() or (build and not DISABLE_REBUILD):

dep_type = 'JAVA' if standalone_type == 'jvm' else 'NATIVE'

# Example of a string we're building here: PYTHON_JAVA_STANDALONE_SVM_SVMEE_JAVA21

mx.run_mx(mx_args + ["build", "--dep", f"PYTHON_{dep_type}_STANDALONE{svm_component}_JAVA{jdk_version.parts[0]}"])

out = mx.OutputCapture()

# note: 'quiet=True' is important otherwise if the outer MX runs verbose,

# this might fail because of additional output

mx.run_mx(mx_args + ["standalone-home", "--type", standalone_type, "python"], out=out, quiet=True)

python_home = out.data.splitlines()[-1].strip()

if dev and standalone_type == 'native':

path = Path(python_home)

debuginfo = path / '../../libpythonvm.so.image/libpythonvm.so.debug'

if debuginfo.exists():

shutil.copy(debuginfo, path / 'lib')

return python_home

def graalpy_standalone(standalone_type, enterprise=False, dev=False, build=True):

assert standalone_type in ['native', 'jvm']

if standalone_type == 'native' and mx_gate.get_jacoco_agent_args():

return graalpy_standalone('jvm', enterprise=enterprise, dev=dev, build=build)

home = graalpy_standalone_home(standalone_type, enterprise=enterprise, dev=dev, build=build)

launcher = os.path.join(home, 'bin', _graalpy_launcher())

return make_coverage_launcher_if_needed(launcher)

def graalpy_standalone_jvm():

return graalpy_standalone('jvm')

def graalpy_standalone_native():

return graalpy_standalone('native')

def graalpy_standalone_jvm_enterprise():

return os.path.join(graalpy_standalone_home('jvm', enterprise=True), 'bin', _graalpy_launcher())

def graalpy_standalone_native_enterprise():

return os.path.join(graalpy_standalone_home('native', enterprise=True), 'bin', _graalpy_launcher())

def graalvm_jdk():

jdk_version = mx.get_jdk().version

# Check if GRAAL_JDK_HOME points to some compatible pre-built gvm

graal_jdk_home = os.environ.get("GRAAL_JDK_HOME", None)

if graal_jdk_home and "*" in graal_jdk_home:

graal_jdk_home = os.path.abspath(glob.glob(graal_jdk_home)[0])

if sys.platform == "darwin":

graal_jdk_home = os.path.join(graal_jdk_home, 'Contents', 'Home')

mx.log("Using GraalPy standalone from GRAAL_JDK_HOME: " + graal_jdk_home)

# Try to verify that we're getting what we expect:

has_java = os.path.exists(os.path.join(graal_jdk_home, 'bin', mx.exe_suffix('java')))

if not has_java:

mx.abort(f"GRAAL_JDK_HOME does not contain java executable.")

release = os.path.join(graal_jdk_home, 'release')

if not os.path.exists(release):

mx.abort(f"No 'release' file in GRAAL_JDK_HOME.")

java_version = None

implementor = None

with open(release, 'r') as f:

while not (java_version and implementor):

line = f.readline()

if 'JAVA_VERSION=' in line:

java_version = line

if 'IMPLEMENTOR=' in line:

implementor = line

if not java_version:

mx.abort(f"Could not check Java version in GRAAL_JDK_HOME 'release' file.")

actual_jdk_version = mx.VersionSpec(java_version.strip('JAVA_VERSION=').strip(' "\n\r'))

if actual_jdk_version != jdk_version:

mx.abort(f"GRAAL_JDK_HOME is not compatible with the requested JDK version.\n"

f"actual version: '{actual_jdk_version}', version string: {java_version}, requested version: {jdk_version}.")

if not implementor:

mx.abort(f"Could not check implementor in GRAAL_JDK_HOME 'release' file.")

if 'GraalVM' not in implementor:

mx.abort(f"GRAAL_JDK_HOME 'releases' has an unexpected implementor: '{implementor}'.")

return graal_jdk_home

jdk_major_version = mx.get_jdk().version.parts[0]

mx_args = ['-p', os.path.join(mx.suite('truffle').dir, '..', 'vm'), '--env', 'ce']

if not DISABLE_REBUILD:

mx.run_mx(mx_args + ["build", "--dep", f"GRAALVM_COMMUNITY_JAVA{jdk_major_version}"])

out = mx.OutputCapture()

mx.run_mx(mx_args + ["graalvm-home"], out=out)

return out.data.splitlines()[-1].strip()

def get_maven_cache():

buildnr = os.environ.get('BUILD_NUMBER')

# don't worry about maven.repo.local if not running on gate

return os.path.join(SUITE.get_mx_output_dir(), 'm2_cache_' + buildnr) if buildnr else None

def deploy_local_maven_repo():

env = os.environ.copy()

m2_cache = get_maven_cache()

if m2_cache:

mvn_repo_local = f'-Dmaven.repo.local={m2_cache}'

maven_opts = env.get('MAVEN_OPTS')

maven_opts = maven_opts + " " + mvn_repo_local if maven_opts else mvn_repo_local

env['MAVEN_OPTS'] = maven_opts

mx.log(f'Added {mvn_repo_local} to MAVEN_OPTS={maven_opts}')

if not DISABLE_REBUILD:

# build GraalPy and all the necessary dependencies, so that we can deploy them

mx.run_mx(["-p", os.path.join(mx.suite('truffle').dir, '..', 'vm'), "--dy", "graalpython", "build"], env=env)

# deploy maven artifacts

version = GRAAL_VERSION

path = os.path.join(SUITE.get_mx_output_dir(), 'public-maven-repo')

licenses = ['EPL-2.0', 'PSF-License', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-simplified', 'BSD-new', 'UPL', 'MIT']

deploy_args = [

'--tags=public',

'--all-suites',

'--all-distribution-types',

f'--version-string={version}',

'--validate=none',

'--licenses', ','.join(licenses),

'--suppress-javadoc',

'local',

pathlib.Path(path).as_uri(),

]

if not DISABLE_REBUILD:

mx.rmtree(path, ignore_errors=True)

os.mkdir(path)

if m2_cache:

with set_env(MAVEN_OPTS = maven_opts):

mx.maven_deploy(deploy_args)

else:

mx.maven_deploy(deploy_args)

return path, version, env

def deploy_local_maven_repo_wrapper(*args):

p, _, _ = deploy_local_maven_repo()

print(f"local Maven repo path: {p}")

def python_jvm(_=None):

"""Returns the path to GraalPy from 'jvm' standalone dev build. Also builds the standalone."""

launcher = graalpy_standalone('jvm', dev=True)

mx.log(launcher)

return launcher

def python_gvm(_=None):

"""Deprecated, use python-jvm"""

mx.warn("mx python-gvm and the helper function python_gvm are deprecated, use python-jvm/python_jvm")

return python_jvm()

def make_coverage_launcher_if_needed(launcher):

if mx_gate.get_jacoco_agent_args():

# patch our launchers created under jacoco to also run with jacoco.

# do not use is_collecting_coverage() here, we only want to patch when

# jacoco agent is requested.

quote = shlex.quote if sys.platform != 'win32' else lambda x: x

def graalvm_vm_arg(java_arg):

if java_arg.startswith("@") and os.path.exists(java_arg[1:]):

with open(java_arg[1:], "r") as f:

java_arg = f.read()

assert java_arg[0] == "-", java_arg

return quote(f'--vm.{java_arg[1:]}')

agent_args = ' '.join(graalvm_vm_arg(arg) for arg in mx_gate.get_jacoco_agent_args() or [])

# We need to make sure the arguments get passed to subprocesses, so we create a temporary launcher

# with the arguments. We also disable compilation, it hardly helps for this use case

original_launcher = os.path.abspath(os.path.realpath(launcher))

if sys.platform != 'win32':

coverage_launcher = original_launcher + '.sh'

preamble = '#!/bin/sh'

pass_args = '"$@"'

else:

coverage_launcher = original_launcher.replace('.exe', '.cmd')

# Windows looks for libraries on PATH, we need to add the jvm bin dir there or it won't find the instrumentation dlls

jvm_bindir = os.path.join(os.path.dirname(os.path.dirname(original_launcher)), 'jvm', 'bin')

preamble = f'@echo off\nset PATH=%PATH%;{jvm_bindir}'

pass_args = '%*'

with open(coverage_launcher, "w") as f:

f.write(f'{preamble}\n')

exe_arg = quote(f"--python.Executable={coverage_launcher}")

f.write(f'{original_launcher} --jvm {exe_arg} {agent_args} {pass_args}\n')

if sys.platform != 'win32':

os.chmod(coverage_launcher, 0o775)

mx.log(f"Replaced {launcher} with {coverage_launcher} to collect coverage")

launcher = coverage_launcher

return launcher

def python_svm(_=None):

"""Returns the path to GraalPy native image from 'native' standalone dev build.

Also builds the standalone if not built already."""

if mx_gate.get_jacoco_agent_args():

return python_jvm()

launcher = graalpy_standalone('native', dev=True)

mx.log(launcher)

return launcher

def native_image(args):

mx.run_mx([

"-p", os.path.join(mx.suite("truffle").dir, "..", "substratevm"),

"--dy", "graalpython",

"--native-images=",

"build",

])

mx.run_mx([

"-p", os.path.join(mx.suite("truffle").dir, "..", "substratevm"),

"--dy", "graalpython",

"--native-images=",

"native-image",

*args

])

def _python_test_runner():

return os.path.join(SUITE.dir, "graalpython", "com.oracle.graal.python.test", "src", "runner.py")

def _python_unittest_root():

return os.path.join(SUITE.dir, "graalpython", "com.oracle.graal.python.test", "src", "tests")

# name of the project containing the HPy tests

HPY_TEST_PROJECT = "com.oracle.graal.python.hpy.test"

def _hpy_test_root():

return os.path.join(mx.dependency(HPY_TEST_PROJECT).get_output_root(), "bin", "hpytest")

def graalpytest(args):

# help is delegated to the runner, it will fake the mx-specific options as well

parser = ArgumentParser(prog='mx graalpytest', add_help=False)

parser.add_argument('--python')

parser.add_argument('--svm', action='store_true')

args, unknown_args = parser.parse_known_args(args)

env = extend_os_env(

MX_GRAALPYTEST='1',

PYTHONHASHSEED='0',

)

python_args = []

runner_args = []

for arg in unknown_args:

if arg.startswith(('--python.', '--engine.', '--llvm.', '--vm.', '--inspect', '--log.', '--experimental-options')):

python_args.append(arg)

else:

runner_args.append(arg)

# if we got a binary path it's most likely CPython, so don't add graalpython args

is_graalpy = False

python_binary = args.python

if not python_binary:

is_graalpy = True

python_args += ["--experimental-options=true", "--python.EnableDebuggingBuiltins"]

if args.svm:

python_binary = graalpy_standalone_native()

elif 'graalpy' in os.path.basename(python_binary) or 'mxbuild' in python_binary:

is_graalpy = True

gp_args = ["--vm.ea", "--vm.esa", "--experimental-options=true", "--python.EnableDebuggingBuiltins"]

mx.log(f"Executable seems to be GraalPy, prepending arguments: {gp_args}")

python_args += gp_args

runner_args.append(f'--subprocess-args={shlex.join(python_args)}')

cmd_args = [*python_args, _python_test_runner(), 'run', *runner_args]

delete_bad_env_keys(env)

if is_graalpy:

pythonpath = [os.path.join(_dev_pythonhome(), 'lib-python', '3')]

pythonpath += [p for p in env.get('PYTHONPATH', '').split(os.pathsep) if p]

env['PYTHONPATH'] = os.pathsep.join(pythonpath)

if python_binary:

return mx.run([python_binary, *cmd_args], nonZeroIsFatal=True, env=env)

else:

return full_python(cmd_args, env=env)

def _list_graalpython_unittests(paths=None, exclude=None):

exclude = [] if exclude is None else exclude

paths = paths or [_python_unittest_root()]

def is_included(path):

if path.endswith(".py"):

path = path.replace("\\", "/")

basename = os.path.basename(path)

return (

basename.startswith("test_")

and basename not in exclude

and not any(fnmatch.fnmatch(path, pat) for pat in exclude)

)

return False

testfiles = []

for path in paths:

if not os.path.exists(path):

# allow paths relative to the test root

path = os.path.join(_python_unittest_root(), path)

if os.path.isfile(path):

testfiles.append(path)

else:

for testfile in glob.glob(os.path.join(path, "**/test_*.py")):

if is_included(testfile):

testfiles.append(testfile.replace("\\", "/"))

for testfile in glob.glob(os.path.join(path, "test_*.py")):

if is_included(testfile):

testfiles.append(testfile.replace("\\", "/"))

return testfiles

def run_python_unittests(python_binary, args=None, paths=None, exclude=None, env=None,

use_pytest=False, cwd=None, lock=None, out=None, err=None, nonZeroIsFatal=True, timeout=None,

report=False, parallel=None, runner_args=None):

if lock:

lock.acquire()

if parallel is None:

parallel = 4 if paths is None else 1

if sys.platform == 'win32':

# Windows machines don't seem to have much memory

parallel = min(parallel, 2)

args = args or []

args = [

"--vm.ea",

"--experimental-options=true",

"--python.EnableDebuggingBuiltins",

*args,

]

if env is None:

env = os.environ.copy()

env['PYTHONHASHSEED'] = '0'

delete_bad_env_keys(env)

if mx.primary_suite() != SUITE:

env.setdefault("GRAALPYTEST_ALLOW_NO_JAVA_ASSERTIONS", "true")

# just to be able to verify, print C ext mode (also works for CPython)

mx.run(

[

python_binary,

*args,