diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index b4aefacc2..a796424e4 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -190,17 +190,19 @@ internal static ClassBase CreateClass(Type type) impl = new InterfaceObject(type); } - else if (type == typeof(Exception) || - type.IsSubclassOf(typeof(Exception))) - { - impl = new ExceptionClassObject(type); - } - #pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. else if (null != PythonDerivedType.GetPyObjField(type)) #pragma warning restore CS0618 // Type or member is obsolete { - impl = new ClassDerivedObject(type); + impl = type.IsSubclassOf(typeof(Exception)) + ? new ExceptionClassDerivedObject(type) + : new ClassDerivedObject(type); + } + + else if (type == typeof(Exception) || + type.IsSubclassOf(typeof(Exception))) + { + impl = new ExceptionClassObject(type); } else if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(type)) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 6c2c81b13..daeacfcd7 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -269,7 +269,8 @@ internal static Type CreateDerivedType(string name, } } - // add the destructor so the python object created in the constructor gets destroyed + // add the destructor so the python object created in the constructor gets destroyed, + // after the base class finalizers, which may still call the Python overrides MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize", MethodAttributes.Family | MethodAttributes.Virtual | @@ -279,11 +280,11 @@ internal static Type CreateDerivedType(string name, Type.EmptyTypes); ILGenerator il = methodBuilder.GetILGenerator(); il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + il.Emit(OpCodes.Ldarg_0); #pragma warning disable CS0618 // PythonDerivedType is for internal use only il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize))); #pragma warning restore CS0618 // PythonDerivedType is for internal use only - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); il.Emit(OpCodes.Ret); Type type = typeBuilder.CreateType(); @@ -928,7 +929,14 @@ public static void PyFinalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well var self = GetPyObj(obj); + // every generated type in a chain of Python subclasses has its own Finalize calling this on the + // same object, once the finalizers of its base classes ran: only the first one queues its Python object + if (self.Ref == null) + { + return; + } Finalizer.Instance.AddDerivedFinalizedObject(ref self.RawObj, self.Run); + SetPyObj(obj, BorrowedReference.Null); } internal static void Finalize(IntPtr derived) diff --git a/src/runtime/Types/ExceptionClassObject.cs b/src/runtime/Types/ExceptionClassObject.cs index 762c9255a..672d8c86a 100644 --- a/src/runtime/Types/ExceptionClassObject.cs +++ b/src/runtime/Types/ExceptionClassObject.cs @@ -1,4 +1,5 @@ using System; +using System.Reflection; namespace Python.Runtime; @@ -29,17 +30,21 @@ internal ExceptionClassObject(Type tp) : base(tp) { return Exceptions.RaiseTypeError("invalid object"); } - string name = e.GetType().Name; - string message; - if (e.Message != String.Empty) + try { - message = String.Format("{0}('{1}')", name, e.Message); + string name = e.GetType().Name; + string message = e.Message; + if (!string.IsNullOrEmpty(message)) + { + return Runtime.PyString_FromString(String.Format("{0}('{1}')", name, message)); + } + return Runtime.PyString_FromString(String.Format("{0}()", name)); } - else + catch (Exception error) { - message = String.Format("{0}()", name); + SetError(error); + return default; } - return Runtime.PyString_FromString(message); } /// @@ -53,26 +58,84 @@ internal ExceptionClassObject(Type tp) : base(tp) return Exceptions.RaiseTypeError("invalid object"); } - string message = e.ToString(); - string fullTypeName = e.GetType().FullName; - string prefix = fullTypeName + ": "; - if (message.StartsWith(prefix)) + // a None returned by a Python override of ToString reads as an empty message + try { - message = message.Substring(prefix.Length); + string message = e.ToString() ?? string.Empty; + string fullTypeName = e.GetType().FullName; + string prefix = fullTypeName + ": "; + if (message.StartsWith(prefix)) + { + message = message.Substring(prefix.Length); + } + else if (message.StartsWith(fullTypeName)) + { + message = message.Substring(fullTypeName.Length); + } + return Runtime.PyString_FromString(message); } - else if (message.StartsWith(fullTypeName)) + catch (Exception error) { - message = message.Substring(fullTypeName.Length); + SetError(error); + return default; } - return Runtime.PyString_FromString(message); } - public override bool Init(BorrowedReference obj, BorrowedReference args, BorrowedReference kw) + /// + /// Raises to Python the error of an exception member, as it is: a Python override of the member + /// throws its PythonException directly, whose InnerException is its cause, while a member the + /// Python subclass inherits runs through reflection, which wraps what it throws + /// + static void SetError(Exception error) { - if (!base.Init(obj, args, kw)) return false; + var raised = error is TargetInvocationException { InnerException: { } invoked } ? invoked : error; + Exceptions.SetError(raised); + } + public override bool Init(BorrowedReference obj, BorrowedReference args, BorrowedReference kw) + => base.Init(obj, args, kw) && SetArgsAndCause(obj); + + /// + /// Sets the args and __cause__ of the constructed exception, raising to Python an error + /// its members throw, such as a Python override of Message + /// + internal static bool SetArgsAndCause(BorrowedReference obj) + { var e = (CLRObject)GetManagedObject(obj)!; + try + { + return Exceptions.SetArgsAndCause(obj, (Exception)e.inst); + } + catch (Exception error) + { + SetError(error); + return false; + } + } +} - return Exceptions.SetArgsAndCause(obj, (Exception)e.inst); +/// +/// Implementation for the managed types generated for Python subclasses of exceptions +/// that declare __namespace__: the generated object keeps its Python object as derived +/// types do, while it still reads as an exception from Python. +/// +[Serializable] +internal class ExceptionClassDerivedObject : ClassDerivedObject +{ + internal ExceptionClassDerivedObject(Type tp) : base(tp) + { } + + // Type slots are looked up on the implementing type's hierarchy, which here is ClassDerivedObject's + public new static NewReference tp_repr(BorrowedReference ob) => ExceptionClassObject.tp_repr(ob); + + public new static NewReference tp_str(BorrowedReference ob) => ExceptionClassObject.tp_str(ob); + + // ClassDerivedObject's tp_clear leaves the object to the .NET finalizer, but the exception's own fields + // (args, __cause__, __context__, __traceback__, __dict__) can form cycles that only clearing them breaks. + // The wrapper and its handle stay, as the .NET object may still use them. + public new static int tp_clear(BorrowedReference ob) => BaseUnmanagedClear(ob); + + public override bool Init(BorrowedReference obj, BorrowedReference args, BorrowedReference kw) + => base.Init(obj, args, kw) && ExceptionClassObject.SetArgsAndCause(obj); } diff --git a/src/testing/constructortests.cs b/src/testing/constructortests.cs index 5912853b0..d57bcdaf7 100644 --- a/src/testing/constructortests.cs +++ b/src/testing/constructortests.cs @@ -108,4 +108,28 @@ public FinalizedConstructorTest(Exception toThrow) Interlocked.Increment(ref Finalized); } } + + public abstract class AbstractFinalizerCallbackTest + { + public static int Result; + + public abstract int Callback(); + + ~AbstractFinalizerCallbackTest() + { + Result = Callback(); + } + } + + public class VirtualFinalizerCallbackTest + { + public static int Result; + + public virtual int Callback() => 7; + + ~VirtualFinalizerCallbackTest() + { + Result = Callback(); + } + } } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 5334c06a7..0337e7a28 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -2,6 +2,8 @@ """Test exception support.""" +import os +import subprocess import sys import System @@ -447,4 +449,251 @@ def test_dont_leak_exceptions_inner(): except TypeError: print('type error, as expected') except System.ArgumentException: - print('type error, also expected') \ No newline at end of file + print('type error, also expected') + + +_NAMESPACED_EXCEPTION_SCRIPT_SETUP = """ +import os, sys +bin_path = sys.argv[1] +sys.path.append(bin_path) +from clr_loader import get_coreclr +from pythonnet import set_runtime +set_runtime(get_coreclr(runtime_config=os.path.join(bin_path, "Python.Test.runtimeconfig.json"))) +import clr +clr.AddReference("Python.Test") +import gc +from Python.Test import ExtendedException +from System import GC, Object +""" + + +def _run_namespaced_exception_script(script): + """Runs the script in a subprocess, so a native crash fails the test instead of the test run""" + from Python.Test import ExtendedException + + bin_path = os.path.dirname(ExtendedException().GetType().Assembly.Location) + result = subprocess.run([sys.executable, "-c", _NAMESPACED_EXCEPTION_SCRIPT_SETUP + script, bin_path], + capture_output=True, text=True, timeout=300) + + assert result.returncode == 0, result.stderr + return result.stdout.splitlines() + + +_NAMESPACED_EXCEPTION_SCRIPT = """ +from System.Collections.Generic import List + +class WithoutInit(ExtendedException): + __namespace__ = "Python.Test.NamespacedExceptions" + +class InitWithoutSuper(ExtendedException): + __namespace__ = "Python.Test.NamespacedExceptions" + def __init__(self): + pass + def ToString(self): + return "overridden" + +class InitWithSuper(ExtendedException): + __namespace__ = "Python.Test.NamespacedExceptions" + def __init__(self): + super().__init__("message") + +def check(cls): + error = cls() + held = List[Object]() + held.Add(error) + try: + raise error + except ExtendedException as caught: + raised = caught is error + print(cls.__name__, error.extra, held[0] is error, raised, str(error), error.args) + +check(WithoutInit) +check(InitWithoutSuper) +check(InitWithSuper) +gc.collect() +for _ in range(2): + GC.Collect() + GC.WaitForPendingFinalizers() +print("collected") +""" + + +@pytest.mark.skipif(os.environ.get("PYTHONNET_RUNTIME") != "netcore", reason="loads the netcore runtime") +def test_namespaced_exception_subclass_construction(): + """A Python subclass of a .NET exception declaring __namespace__ runs its base + constructor, keeps its Python object for .NET and its overrides, and still + reads as an exception""" + output = _run_namespaced_exception_script(_NAMESPACED_EXCEPTION_SCRIPT) + + overflow = "Arithmetic operation resulted in an overflow." + assert output[-4:] == [ + f"WithoutInit extra True True {overflow} ('{overflow}',)", + f"InitWithoutSuper extra True True overridden ('{overflow}',)", + "InitWithSuper extra True True message ('message',)", + "collected", + ] + + +_NAMESPACED_EXCEPTION_RELEASE_SCRIPT = """ +class Probe(Object): + __namespace__ = "Python.Test.ReleasedExceptions" + def ToString(self): + return "probe" + +class ParentError(ExtendedException): + __namespace__ = "Python.Test.ReleasedExceptions" + def __init__(self): + pass + +class ChildError(ParentError): + __namespace__ = "Python.Test.ReleasedExceptions" + def __init__(self): + pass + +class CycleError(ExtendedException): + __namespace__ = "Python.Test.ReleasedExceptions" + def __init__(self): + pass + +def release(): + for _ in range(3): + gc.collect() + GC.Collect() + GC.WaitForPendingFinalizers() + # each call from .NET into the Python override creates Python objects, which + # periodically release the queued Python objects of finalized .NET objects + for _ in range(300): + str(probe) + +probe = Probe() + +for _ in range(10): + ChildError() +release() +print("released") + +for _ in range(10): + error = CycleError() + error.__cause__ = error + error = CycleError() + error.__context__ = error + error = CycleError() + error.args = (error,) +del error +release() +print(sum(1 for o in gc.get_objects() if type(o) is CycleError)) + +from Python.Test import AbstractFinalizerCallbackTest, VirtualFinalizerCallbackTest + +class AbstractCallback(AbstractFinalizerCallbackTest): + __namespace__ = "Python.Test.ReleasedExceptions" + def Callback(self): + return 42 + +class AbstractCallbackChild(AbstractCallback): + __namespace__ = "Python.Test.ReleasedExceptions" + def Callback(self): + return 43 + +class VirtualCallback(VirtualFinalizerCallbackTest): + __namespace__ = "Python.Test.ReleasedExceptions" + def Callback(self): + return 42 + +class VirtualCallbackChild(VirtualCallback): + __namespace__ = "Python.Test.ReleasedExceptions" + def Callback(self): + return 43 + +for cls, base in ((AbstractCallback, AbstractFinalizerCallbackTest), + (AbstractCallbackChild, AbstractFinalizerCallbackTest), + (VirtualCallback, VirtualFinalizerCallbackTest), + (VirtualCallbackChild, VirtualFinalizerCallbackTest)): + base.Result = 0 + cls() + release() + print(cls.__name__, base.Result) + +""" + + +@pytest.mark.skipif(os.environ.get("PYTHONNET_RUNTIME") != "netcore", reason="loads the netcore runtime") +def test_namespaced_exception_subclass_release(): + """A __namespace__ exception subclass of another one is released once, exceptions + referencing themselves through args, __cause__ or __context__ are collected, and + the finalizer of a .NET base class still reaches the Python overrides of its + __namespace__ subclasses, before their Python object is released""" + output = _run_namespaced_exception_script(_NAMESPACED_EXCEPTION_RELEASE_SCRIPT) + + assert output[-6:] == [ + "released", + "0", + "AbstractCallback 42", + "AbstractCallbackChild 43", + "VirtualCallback 42", + "VirtualCallbackChild 43", + ] + + +_NAMESPACED_EXCEPTION_OVERRIDES_SCRIPT = """ +class OverridingError(ExtendedException): + __namespace__ = "Python.Test.OverridingExceptions" + def __init__(self, failing=False): + self.failing = failing + if failing: + super().__init__() + def fail(self): + raise ValueError("outer") from KeyError("inner") + def get_Message(self): + if self.failing: + self.fail() + return "message" + def ToString(self): + if self.failing: + self.fail() + return None + +error = OverridingError() +print(repr(str(error)), repr(error)) +error.failing = True + +def check(name, call): + try: + call() + except ValueError as raised: + print(name, raised, type(raised.__cause__).__name__, raised.__cause__) + +check("str", lambda: str(error)) +check("repr", lambda: repr(error)) +check("init", lambda: OverridingError(failing=True)) + +class InheritedToStringError(ExtendedException): + __namespace__ = "Python.Test.OverridingExceptions" + def __init__(self): + self.failing = False + def get_Message(self): + if self.failing: + raise ValueError("outer") from KeyError("inner") + return "message" + +inherited = InheritedToStringError() +inherited.failing = True +check("inherited str", lambda: str(inherited)) +""" + + +@pytest.mark.skipif(os.environ.get("PYTHONNET_RUNTIME") != "netcore", reason="loads the netcore runtime") +def test_namespaced_exception_subclass_override_errors(): + """str(), repr() and the constructor of a __namespace__ exception subclass read None + from a Python ToString override as an empty message, and raise the error of a Python + ToString or Message override to their caller as it is, with its cause, also when an + inherited ToString calls the Message override""" + output = _run_namespaced_exception_script(_NAMESPACED_EXCEPTION_OVERRIDES_SCRIPT) + + assert output[-5:] == [ + "'' OverridingError('message')", + "str outer KeyError 'inner'", + "repr outer KeyError 'inner'", + "init outer KeyError 'inner'", + "inherited str outer KeyError 'inner'", + ]