Support for disabling Buffered output response for realtime data streaming (SSE) (#779)

* Add support for disabled buffered output

* Remove unused code

* Always flush stream in GZIP responses whn flush() is invocated

* Fix compilation error

Author: ggallotti

gxweb/src/main/java/com/genexus/webpanels/WebWrapper.java

@@ -40,7 +40,7 @@ public void setSource(GXWebPanel panel)
 			((HttpContext) context.getHttpContext()).setContext(context);
 			panel.httpContext = (HttpAjaxContext)context.getHttpContext();
 			panel.httpContext.setCompression(false);
-			panel.httpContext.setBuffered(false);
+			panel.httpContext.setResponseBufferMode(HttpContext.ResponseBufferMode.SERVER_DEFAULT);
 			panel.httpContext.useUtf8 = true;
 			panel.httpContext.setOutputStream(new java.io.ByteArrayOutputStream());
 		}

java/src/main/java/com/genexus/GXWebReport.java

@@ -33,7 +33,7 @@ protected void initState(ModelContext context, UserInformation ui)
 	{
 		super.initState(context, ui);
 
-		httpContext.setBuffered(true);
+		httpContext.setResponseBufferMode(HttpContext.ResponseBufferMode.ENABLED);
 		httpContext.setBinary(true);
 		String implementation = com.genexus.Application.getClientContext().getClientPreferences().getPDF_RPT_LIBRARY();
 		if (implementation.equals("ITEXT"))

java/src/main/java/com/genexus/internet/HttpContext.java

@@ -1,21 +1,22 @@
 package com.genexus.internet;
 
-import java.io.*;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Hashtable;
-
+import com.genexus.*;
+import com.genexus.servlet.http.ICookie;
+import com.genexus.servlet.http.IHttpServletRequest;
+import com.genexus.servlet.http.IHttpServletResponse;
+import com.genexus.webpanels.WebSession;
 import json.org.json.IJsonFormattable;
 import json.org.json.JSONArray;
 import json.org.json.JSONException;
 import json.org.json.JSONObject;
 import org.apache.logging.log4j.Logger;
 
-import com.genexus.*;
-import com.genexus.servlet.http.ICookie;
-import com.genexus.servlet.http.IHttpServletRequest;
-import com.genexus.servlet.http.IHttpServletResponse;
-import com.genexus.webpanels.WebSession;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Hashtable;
 
 public abstract class HttpContext implements IHttpContext
 {
@@ -89,11 +90,6 @@ public void disableSpaRequest()
 	{
 		ignoreSpa = true;
 	}
-
-	public void setChunked()
-	{
-		isChunked = true;
-	}
 
 	public boolean isSoapRequest()
 	{
@@ -114,7 +110,7 @@ public boolean isSoapRequest()
 	public com.genexus.xml.XMLWriter GX_xmlwrt = new com.genexus.xml.XMLWriter();
 
 	protected com.genexus.util.FastByteArrayOutputStream buffer;
-	protected boolean buffered;
+	protected ResponseBufferMode bufferMode = ResponseBufferMode.SERVER_DEFAULT;
 	protected boolean compressed;
 	protected boolean doNotCompress;
 	protected ModelContext context;
@@ -131,7 +127,6 @@ public boolean isSoapRequest()
 	protected boolean wrapped = false;
 	protected int drawGridsAtServer = -1;
 	private boolean ignoreSpa = false;
-	private boolean isChunked = false;
 
 	private static HashMap<String, Messages> cachedMessages = new HashMap<String, Messages>();
 	protected String currentLanguage = null;
@@ -180,7 +175,7 @@ protected void copyCommon(HttpContext ctx)
 		ctx.GX_msglist		=  GX_msglist;
 		ctx.GX_xmlwrt		=  GX_xmlwrt;
 		ctx.buffer			=  buffer;
-		ctx.buffered		=  buffered;
+		ctx.bufferMode		=  bufferMode;
 		ctx.compressed		=  compressed;
 		ctx.out				=  out;
 		ctx.writer			=  writer;
@@ -567,11 +562,22 @@ public boolean mustUseWriter()
 	}
 
 
-	public void writeBytes(byte[] bytes) throws IOException
-	{
-            out.write(bytes);
-			if (isChunked || getHttpResponse().getHeader("Transfer-Encoding").equalsIgnoreCase("chunked"))
-				out.flush();
+	public void writeBytes(byte[] bytes) throws IOException {
+		out.write(bytes);
+
+		if (bufferMode == ResponseBufferMode.DISABLED) {
+			tryFlushStream();
+		}
+	}
+
+	private void tryFlushStream() {
+		try {
+			out.flush();
+		}
+		catch (IOException e)
+		{
+			logger.debug("Failed to flush Http stream", e);
+		}
 	}
 
 	public OutputStream getOutputStream()
@@ -605,9 +611,9 @@ public void closeOutputStream()
 		}
 	}
 
-	public void setBuffered(boolean buffered)
+	public void setResponseBufferMode(ResponseBufferMode bufferMode)
 	{
-		this.buffered = buffered;
+		this.bufferMode = bufferMode;
 	}
 
 	public void setCompression(boolean compressed)
@@ -962,4 +968,10 @@ public void readJsonSdtValue(String jsonStr, Object SdtObj)
 		}
 		catch(JSONException exc) {}
 	}
+
+	public enum ResponseBufferMode {
+		DISABLED, // The response buffer is disabled and the server does not use any buffering.
+		SERVER_DEFAULT, // The server uses its default buffering behavior, which includes a buffer size. When the buffer is full, it flushes the response.
+		ENABLED // Not recommended: The response buffer is enabled and actively used by the server with a Custom GeneXus implementation.
+	}
 }

java/src/main/java/com/genexus/internet/HttpResponse.java

@@ -1,21 +1,18 @@
 
 package com.genexus.internet;
 
-import java.io.*;
-import java.util.Hashtable;
-
-import com.genexus.CommonUtil;
-import com.genexus.ModelContext;
 import com.genexus.IHttpContext;
+import com.genexus.ModelContext;
 import com.genexus.PrivateUtilities;
 import com.genexus.com.IHttpResponse;
 import com.genexus.webpanels.FileItemCollection;
 import com.genexus.webpanels.HttpContextWeb;
-import com.genexus.webpanels.HttpUtils;
 import com.genexus.webpanels.WebUtils;
-
 import org.apache.logging.log4j.Logger;
 
+import java.io.*;
+import java.util.Hashtable;
+
 /**
 * Esta clase esta disponible en los webprocs para grabar informacion en el response
 */
@@ -43,23 +40,32 @@ public FileItemCollection getPostedparts()
 	 return httpContext.getPostedparts();
        }
 
-	public void addHeader(String name, String value)
-	{
-		if(name.equalsIgnoreCase("Content-Disposition"))
-		{
-		  value = WebUtils.getEncodedContentDisposition(value, httpContext.getBrowserType());
+	public void addHeader(String name, String value) {
+		final String normalizedName = name.trim().toLowerCase();
+
+		if (normalizedName.equals("content-disposition")) {
+			value = WebUtils.getEncodedContentDisposition(value, httpContext.getBrowserType());
 		}
 
 		httpContext.setHeader(name, value);
 		headers.put(name.toUpperCase(), value);
 
-		if	(name.equalsIgnoreCase("Content-type"))
-		{
-			httpContext.setContentType(value);
-		}
-		else if	(name.equalsIgnoreCase("Content-length"))
-		{
-			httpContext.getResponse().setContentLength((int) CommonUtil.val(value));
+		switch (normalizedName) {
+			case "content-type":
+				httpContext.setContentType(value);
+
+				if (value.equalsIgnoreCase("text/event-stream")) {
+					httpContext.setResponseBufferMode(HttpContext.ResponseBufferMode.DISABLED);
+				}
+				break;
+			case "content-length":
+				try {
+					int length = Integer.parseInt(value);
+					httpContext.getResponse().setContentLength(length);
+				} catch (NumberFormatException ex) {
+					log.warn("Content-Length header could not be set to HttpResponse", ex);
+				}
+				break;
 		}
 	}
 

java/src/main/java/com/genexus/webpanels/GXWebProcedure.java

@@ -69,16 +69,15 @@ protected void initState(ModelContext context, UserInformation ui)
 		super.initState(context, ui);
 
 		if(httpContext.getHttpSecure() == 0)httpContext.setHeader("pragma", "no-cache");
-		if (isChunked()) {
-			httpContext.setChunked();
-			httpContext.setCompression(false);
-		}
 
+		if (!isBufferedResponse()) {
+			httpContext.setResponseBufferMode(HttpContext.ResponseBufferMode.DISABLED);
+		}
 		initialize();
 	}
 
-	protected boolean isChunked() {
-		return false;
+	protected boolean isBufferedResponse() {
+		return true;
 	}
 
 	protected void preExecute()

java/src/main/java/com/genexus/webpanels/HttpContextWeb.java

@@ -1375,7 +1375,7 @@ public void setStream() {
 			if (mustUseWriter()) {
 				setWriter(getResponse().getWriter());
 			} else {
-				if (buffered) {
+				if (bufferMode == ResponseBufferMode.ENABLED) {
 					buffer = new com.genexus.util.FastByteArrayOutputStream();
 					setOutputStream(buffer);
 				} else {
@@ -1386,7 +1386,7 @@ public void setStream() {
 					String accepts = getHeader("Accept-Encoding");
 					if (accepts != null && accepts.indexOf("gzip") >= 0) {
 						setHeader("Content-Encoding", "gzip");
-						setOutputStream(new GZIPOutputStream(getOutputStream()));
+						setOutputStream(new GZIPOutputStream(getOutputStream(), true));
 					}
 				}
 			}
@@ -1399,7 +1399,7 @@ public void flushStream() {
 		proxyCookieValues();
 
 		try {
-			if (buffered) {
+			if (bufferMode == ResponseBufferMode.ENABLED) {
 				// Esto en realidad cierra el ZipOutputStream, o el ByteOutputStream, no cierra
 				// el del
 				// servlet... Es necesario hacerlo, dado que sino el GZip no hace el flush de