diff --git a/src/v_video.c b/src/v_video.c
index d58104eef..2795a21f4 100644
--- a/src/v_video.c
+++ b/src/v_video.c
@@ -461,7 +461,8 @@ void VID_BlitLinearScreen_ASM(const UINT8 *srcptr, UINT8 *destptr, INT32 width,
 
 static void CV_constextsize_OnChange(void)
 {
-	con_recalc = true;
+	if (!con_refresh)
+		con_recalc = true;
 }