blua: attempt emergency garbo collection on file io operations if they fail

ports of
e885dee5ab

and

3d838f635c

prevents massive memory leaks and other issues due to faulty lua scripts

src/blua/lauxlib.c

@@ -98,6 +98,56 @@ LUALIB_API int luaL_error (lua_State *L, const char *fmt, ...) {
 /* }====================================================== */
 
 
+/*
+** {======================================================
+** 'luaL_resourcetryagain'
+** This function uses 'errno' to check whether the last error was
+** related to lack of resources (e.g., not enough memory or too many
+** open files).  If so, the function performs a full garbage collection
+** to try to release resources, and then it returns 1 to signal to
+** the caller that it is worth trying again the failed operation.
+** Otherwise, it returns 0.  Because error codes are not ANSI C, the
+** code must handle any combination of error codes that are defined.
+** =======================================================
+*/
+
+LUALIB_API int luaL_resourcetryagain (lua_State *L) {
+
+/* these are the resource-related errors in Linux */
+#if defined(EMFILE) || defined(ENFILE) || defined(ENOMEM)
+
+#if !defined(EMFILE)	/* too many open files in the process */
+#define EMFILE		-1	/* if not defined, use an impossible value */
+#endif
+
+#if !defined(ENFILE)	/* too many open files in the system */
+#define ENFILE		-1
+#endif
+
+#if !defined(ENOMEM)	/* not enough memory */
+#define ENOMEM		-1
+#endif
+
+  if (errno == EMFILE || errno == ENFILE || errno == ENOMEM) {
+    lua_gc(L, LUA_GCCOLLECT, 0);  /* try to release resources with a full GC */
+    return 1;  /* signal to try again the creation */
+  }
+
+#endif
+
+  return 0;  /* else, asume errors are not due to lack of resources */
+
+}
+
+/* }====================================================== */
+
+
+/*
+** {======================================================
+** Userdata's metatable manipulation
+** =======================================================
+*/
+
 LUALIB_API int luaL_checkoption (lua_State *L, int narg, const char *def,
                                  const char *const lst[]) {
   const char *name = (def) ? luaL_optstring(L, narg, def) :

src/blua/lauxlib.h

@@ -69,6 +69,7 @@ LUALIB_API int (luaL_error) (lua_State *L, const char *fmt, ...);
 
 LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def,
                                    const char *const lst[]);
+LUALIB_API int (luaL_resourcetryagain) (lua_State *L);
 
 LUALIB_API int (luaL_ref) (lua_State *L, int t);
 LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref);

src/blua/liolib.c

@@ -224,6 +224,27 @@ static int CheckFileName(lua_State* L, const char* filename, boolean extensionch
 	return 0;
 }
 
+/*
+** Equivalent to 'fopen', but if it fails due to a lack of resources
+** (see 'luaL_resourcetryagain'), do an "emergency" garbage collection to try
+** to close some files and then tries to open the file again.
+*/
+static FILE *trytoopen (lua_State *L, const char *path, const char *mode) {
+  FILE *f = fopen(path, mode);
+  if (f == NULL && luaL_resourcetryagain(L))  /* resource failure? */
+    f = fopen(path, mode);  /* try to open again */
+  return f;
+}
+
+static void opencheck (lua_State *L, const char *fname, const char *mode) {
+  FILE **p = newfile(L);
+  *p = trytoopen(L, fname, mode);
+  if (*p == NULL)
+  {
+      I_Error("Can't open file \"%s\"\n", fname); // The file SHOULD exist
+  }
+}
+
 static int io_openlocal (lua_State *L) {
 	FILE **pf;
 	const char *filename = luaL_checkstring(L, 1);
@@ -252,7 +273,7 @@ static int io_openlocal (lua_State *L) {
 
 	// Open and return the file
 	pf = newfile(L);
-	*pf = fopen(realfilename, mode);
+	*pf = trytoopen(L, realfilename, mode);
 	return (*pf == NULL) ? pushresult(L, 0, filename) : 1;
 }
 
@@ -316,10 +337,7 @@ void Got_LuaFile(UINT8 **cp, INT32 playernum)
 		if (!strchr(mode, 'b'))
 			strcat(mode, "b");
 
-		pf = newfile(gL); // Create and push the file handle
-		*pf = fopen(luafiletransfers->realfilename, mode); // Open the file
-		if (!*pf)
-			I_Error("Can't open file \"%s\"\n", luafiletransfers->realfilename); // The file SHOULD exist
+		opencheck(gL, luafiletransfers->realfilename, mode); // Open the file
 	}
 	else
 		lua_pushnil(gL);
@@ -376,6 +394,8 @@ void RemoveLuaFileCallback(INT32 id)
 static int io_tmpfile (lua_State *L) {
   FILE **pf = newfile(L);
   *pf = tmpfile();
+  if (*pf == NULL && luaL_resourcetryagain(L))  /* resource failure? */
+    *pf = tmpfile();  /* try to open again */
   return (*pf == NULL) ? pushresult(L, 0, NULL) : 1;
 }