avm2: Coerce function return value to declared return type

This can actually affect runtime behavior - if the return type is
declared as 'int', then an instance of a custom class will get
coerced to 0 when returned by the function.
'Plants vs Zombies Demo' relies on this - it has a function which
incorrectly returns an object instead of an array index, but the
value gets silently coerced to 0 under Flash Player.

Author: Aaron1011

core/src/avm2/activation.rs

@@ -1048,7 +1048,7 @@ impl<'a, 'gc> Activation<'a, 'gc> {
                 Op::CallSuperVoid { index, num_args } => {
                     self.op_call_super_void(method, index, num_args)
                 }
-                Op::ReturnValue => self.op_return_value(),
+                Op::ReturnValue => self.op_return_value(method),
                 Op::ReturnVoid => self.op_return_void(),
                 Op::GetProperty { index } => self.op_get_property(method, index),
                 Op::SetProperty { index } => self.op_set_property(method, index),
@@ -1498,10 +1498,13 @@ impl<'a, 'gc> Activation<'a, 'gc> {
         Ok(FrameControl::Continue)
     }
 
-    fn op_return_value(&mut self) -> Result<FrameControl<'gc>, Error<'gc>> {
+    fn op_return_value(
+        &mut self,
+        method: Gc<'gc, BytecodeMethod<'gc>>,
+    ) -> Result<FrameControl<'gc>, Error<'gc>> {
         let return_value = self.pop_stack();
-
-        Ok(FrameControl::Return(return_value))
+        let coerced = return_value.coerce_to_type_name(self, &method.return_type)?;
+        Ok(FrameControl::Return(coerced))
     }
 
     fn op_return_void(&mut self) -> Result<FrameControl<'gc>, Error<'gc>> {

core/src/avm2/value.rs

@@ -1049,9 +1049,23 @@ impl<'gc> Value<'gc> {
             .name()
             .to_qualified_name_err_message(activation.context.gc_context);
 
+        let debug_str = match self {
+            Value::Object(obj) if obj.as_primitive().is_none() => {
+                // Flash prints the class name (ignoring the toString() impl on the object),
+                // followed by something that looks like an address (it varies between executions).
+                // For now, we just set the "address" to all zeroes, on the off chance that some
+                // application is trying to parse the error message.
+                format!(
+                    "{}@00000000000",
+                    obj.instance_of_class_name(activation.context.gc_context)
+                )
+            }
+            _ => self.coerce_to_debug_string(activation)?.to_string(),
+        };
+
         Err(Error::AvmError(type_error(
             activation,
-            &format!("Error #1034: Type Coercion failed: cannot convert {self:?} to {name}."),
+            &format!("Error #1034: Type Coercion failed: cannot convert {debug_str} to {name}.",),
             1034,
         )?))
     }

tests/tests/swfs/avm2/coerce_return_type/MyClass.as

@@ -0,0 +1,5 @@
+package {
+	public class MyClass {
+		
+	}
+}

tests/tests/swfs/avm2/coerce_return_type/MyOtherClass.as

@@ -0,0 +1,7 @@
+package {
+	public class MyOtherClass {
+		public function toString():String {
+			return "Custom toString";
+		}
+	}
+}

tests/tests/swfs/avm2/coerce_return_type/Test.as

@@ -0,0 +1,39 @@
+package {
+	public class Test {
+		public function Test() {
+			
+		}
+	}
+}
+
+function returnInt(param: *):int {
+	return param;
+}
+
+function returnBool(param: *):Boolean {
+	return param;
+}
+
+function returnNumber(param: *):Number {
+	return param;
+}
+
+function returnString(param: *):String {
+	return param;
+}
+
+function returnMyClass(param: *):MyClass {
+	return param;
+}
+
+for each (var val in [1.0, true, false, null, undefined, "Hello", new MyClass(), new MyOtherClass()]) {
+	trace("returnInt(" + val + ") = " + returnInt(val));
+	trace("returnBool(" + val + ") = " + returnBool(val));
+	trace("returnNumber(" + val + ") = " + returnNumber(val));
+	trace("returnString(" + val + ") = " + returnString(val));
+	try {
+		trace("returnMyClass(" + val + ") = " + returnMyClass(val));
+	} catch (e) {
+		trace("returnMyClass(" + val + ") threw error: " + e);
+	}
+}

tests/tests/swfs/avm2/coerce_return_type/output.txt

@@ -0,0 +1,40 @@
+returnInt(1) = 1
+returnBool(1) = true
+returnNumber(1) = 1
+returnString(1) = 1
+returnMyClass(1) threw error: TypeError: Error #1034: Type Coercion failed: cannot convert 1 to MyClass.
+returnInt(true) = 1
+returnBool(true) = true
+returnNumber(true) = 1
+returnString(true) = true
+returnMyClass(true) threw error: TypeError: Error #1034: Type Coercion failed: cannot convert true to MyClass.
+returnInt(false) = 0
+returnBool(false) = false
+returnNumber(false) = 0
+returnString(false) = false
+returnMyClass(false) threw error: TypeError: Error #1034: Type Coercion failed: cannot convert false to MyClass.
+returnInt(null) = 0
+returnBool(null) = false
+returnNumber(null) = 0
+returnString(null) = null
+returnMyClass(null) = null
+returnInt(undefined) = 0
+returnBool(undefined) = false
+returnNumber(undefined) = NaN
+returnString(undefined) = null
+returnMyClass(undefined) = null
+returnInt(Hello) = 0
+returnBool(Hello) = true
+returnNumber(Hello) = NaN
+returnString(Hello) = Hello
+returnMyClass(Hello) threw error: TypeError: Error #1034: Type Coercion failed: cannot convert "Hello" to MyClass.
+returnInt([object MyClass]) = 0
+returnBool([object MyClass]) = true
+returnNumber([object MyClass]) = NaN
+returnString([object MyClass]) = [object MyClass]
+returnMyClass([object MyClass]) = [object MyClass]
+returnInt(Custom toString) = 0
+returnBool(Custom toString) = true
+returnNumber(Custom toString) = NaN
+returnString(Custom toString) = Custom toString
+returnMyClass(Custom toString) threw error: TypeError: Error #1034: Type Coercion failed: cannot convert MyOtherClass@00000000000 to MyClass.

tests/tests/swfs/avm2/coerce_return_type/test.fla

@@ -0,0 +1 @@
+num_frames = 1