I think I found the correct place for the state change

qw/source/cl_parse.c

@@ -353,7 +353,6 @@ Model_NextDownload (void)
 						 va (prespawn_name, cl.servercount,
 							 cl.worldmodel->checksum2));
 	}
-	CL_SetState (ca_active);
 }
 
 static void

qw/source/cl_skin.c

@@ -101,6 +101,7 @@ Skin_NextDownload (void)
 		MSG_WriteString (&cls.netchan.message, va ("begin %i",
 												   cl.servercount));
 		Cache_Report ();				// print remaining memory
+		CL_SetState (ca_active);
 	}
 }