- fixed: Since out types cannot be marked as such in a function prototype (as it'd cause parameter mismatches in the resolving pass) it is necessary to check the argflags as well when determining the register type.

src/d_dehacked.cpp

@@ -749,7 +749,7 @@ static void (*MBFCodePointerFactories[])(FunctionCallEmitter&, int, int) =
 
 void SetDehParams(FState *state, int codepointer)
 {
-	static uint8_t regts[] = { REGT_POINTER, REGT_POINTER, REGT_POINTER };
+	static const uint8_t regts[] = { REGT_POINTER, REGT_POINTER, REGT_POINTER };
 	int value1 = state->GetMisc1();
 	int value2 = state->GetMisc2();
 	if (!(value1|value2)) return;

src/p_map.cpp

@@ -4489,7 +4489,7 @@ DEFINE_ACTION_FUNCTION(AActor, AimLineAttack)
 	PARAM_SELF_PROLOGUE(AActor);
 	PARAM_ANGLE(angle);
 	PARAM_FLOAT(distance);
-	PARAM_POINTER(pLineTarget, FTranslatedLineTarget);
+	PARAM_OUTPOINTER(pLineTarget, FTranslatedLineTarget);
 	PARAM_ANGLE(vrange);
 	PARAM_INT(flags);
 	PARAM_OBJECT(target, AActor);
@@ -4926,7 +4926,7 @@ DEFINE_ACTION_FUNCTION(AActor, LineAttack)
 	PARAM_NAME(damageType);
 	PARAM_CLASS(puffType, AActor);
 	PARAM_INT(flags);
-	PARAM_POINTER(victim, FTranslatedLineTarget);
+	PARAM_OUTPOINTER(victim, FTranslatedLineTarget);
 	PARAM_FLOAT(offsetz);
 	PARAM_FLOAT(offsetforward);
 	PARAM_FLOAT(offsetside);
@@ -5092,7 +5092,7 @@ DEFINE_ACTION_FUNCTION(AActor, LineTrace)
 	PARAM_FLOAT(offsetz);
 	PARAM_FLOAT(offsetforward);
 	PARAM_FLOAT(offsetside);
-	PARAM_POINTER(data, FLineTraceData);
+	PARAM_OUTPOINTER(data, FLineTraceData);
 	ACTION_RETURN_BOOL(P_LineTrace(self,angle,distance,pitch,flags,offsetz,offsetforward,offsetside,data));
 }
 

src/p_mobj.cpp

@@ -7417,7 +7417,7 @@ DEFINE_ACTION_FUNCTION(AActor, SpawnPlayerMissile)
 	PARAM_FLOAT(x);
 	PARAM_FLOAT(y);
 	PARAM_FLOAT(z);
-	PARAM_POINTER(lt, FTranslatedLineTarget);
+	PARAM_OUTPOINTER(lt, FTranslatedLineTarget);
 	PARAM_BOOL(nofreeaim);
 	PARAM_BOOL(noautoaim);
 	PARAM_INT(aimflags);

src/scripting/vm/vm.h

@@ -505,6 +505,7 @@ bool AssertObject(void * ob);
 #define PARAM_STATE_AT(p,x)			assert((p) < numparam); assert(reginfo[p] == REGT_INT); FState *x = (FState *)StateLabels.GetState(param[p].i, self->GetClass());
 #define PARAM_STATE_ACTION_AT(p,x)	assert((p) < numparam); assert(reginfo[p] == REGT_INT); FState *x = (FState *)StateLabels.GetState(param[p].i, stateowner->GetClass());
 #define PARAM_POINTER_AT(p,x,type)	assert((p) < numparam); assert(reginfo[p] == REGT_POINTER); type *x = (type *)param[p].a;
+#define PARAM_OUTPOINTER_AT(p,x,type)	assert((p) < numparam); type *x = (type *)param[p].a;
 #define PARAM_POINTERTYPE_AT(p,x,type)	assert((p) < numparam); assert(reginfo[p] == REGT_POINTER); type x = (type )param[p].a;
 #define PARAM_OBJECT_AT(p,x,type)	assert((p) < numparam); assert(reginfo[p] == REGT_POINTER && AssertObject(param[p].a)); type *x = (type *)param[p].a; assert(x == NULL || x->IsKindOf(RUNTIME_CLASS(type)));
 #define PARAM_CLASS_AT(p,x,base)	assert((p) < numparam); assert(reginfo[p] == REGT_POINTER); base::MetaClass *x = (base::MetaClass *)param[p].a; assert(x == NULL || x->IsDescendantOf(RUNTIME_CLASS(base)));
@@ -528,6 +529,7 @@ bool AssertObject(void * ob);
 #define PARAM_STATE(x)				++paramnum; PARAM_STATE_AT(paramnum,x)
 #define PARAM_STATE_ACTION(x)		++paramnum; PARAM_STATE_ACTION_AT(paramnum,x)
 #define PARAM_POINTER(x,type)		++paramnum; PARAM_POINTER_AT(paramnum,x,type)
+#define PARAM_OUTPOINTER(x,type)		++paramnum; PARAM_OUTPOINTER_AT(paramnum,x,type)
 #define PARAM_POINTERTYPE(x,type)	++paramnum; PARAM_POINTERTYPE_AT(paramnum,x,type)
 #define PARAM_OBJECT(x,type)		++paramnum; PARAM_OBJECT_AT(paramnum,x,type)
 #define PARAM_CLASS(x,base)			++paramnum; PARAM_CLASS_AT(paramnum,x,base)

src/scripting/vm/vmframe.cpp

@@ -80,6 +80,7 @@ void VMFunction::CreateRegUse()
 		return;
 	}
 	assert(Proto->isPrototype());
+
 	for (auto arg : Proto->ArgumentTypes)
 	{
 		count += arg? arg->GetRegCount() : 1;
@@ -87,14 +88,20 @@ void VMFunction::CreateRegUse()
 	uint8_t *regp;
 	RegTypes = regp = (uint8_t*)ClassDataAllocator.Alloc(count);
 	count = 0;
-	for (auto arg : Proto->ArgumentTypes)
+	for (unsigned i = 0; i < Proto->ArgumentTypes.Size(); i++)
 	{
+		auto arg = Proto->ArgumentTypes[i];
+		auto flg = ArgFlags.Size() > i ? ArgFlags[i] : 0;
 		if (arg == nullptr)
 		{
 			// Marker for start of varargs.
 			*regp++ = REGT_NIL;
 		}
-		else for (int i = 0; i < arg->GetRegCount(); i++)
+		else if ((flg & VARF_Out) && !arg->isPointer())
+		{
+			*regp++ = REGT_POINTER;
+		}
+		else for (int j = 0; j < arg->GetRegCount(); j++)
 		{
 			*regp++ = arg->GetRegType();
 		}