on allowed mask mismatch, ensure all inbound frames have same masking status

Author: mostroverkhov

netty-websocket-http1/src/main/java/com/jauntsdn/netty/handler/codec/http/websocketx/WebSocketCallbacksFrameDecoder.java

@@ -28,11 +28,7 @@ void frameListener(
 
   static WebSocketCallbacksFrameDecoder frameDecoder(
       int maxFramePayloadLength, boolean expectMaskedFrames, boolean allowMaskMismatch) {
-
-    Boolean strictExpectMaskedFrames = null;
-    if (!allowMaskMismatch) {
-      strictExpectMaskedFrames = expectMaskedFrames;
-    }
-    return new WebSocketMaskedDecoder(maxFramePayloadLength, strictExpectMaskedFrames);
+    Boolean lenientExpectMaskedFrames = allowMaskMismatch ? null : expectMaskedFrames;
+    return new WebSocketMaskedDecoder(maxFramePayloadLength, lenientExpectMaskedFrames);
   }
 }

netty-websocket-http1/src/main/java/com/jauntsdn/netty/handler/codec/http/websocketx/WebSocketMaskedDecoder.java

@@ -20,11 +20,11 @@
 import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.websocketx.WebSocketCloseStatus;
+import javax.annotation.Nullable;
 
 final class WebSocketMaskedDecoder extends WebSocketDecoder {
-  /*if null, mask mismatch is allowed*/
-  Boolean expectMaskedFrames;
   final int maxFramePayloadLength;
+  Boolean expectMaskedFrames;
 
   WebSocketFrameFactory frameFactory;
 
@@ -43,7 +43,7 @@ final class WebSocketMaskedDecoder extends WebSocketDecoder {
   /* non-negative value means fragmentation is in progress*/
   int fragmentedTotalLength = WebSocketProtocol.VALIDATION_RESULT_NON_FRAGMENTING;
 
-  WebSocketMaskedDecoder(int maxFramePayloadLength, Boolean expectMaskedFrames) {
+  WebSocketMaskedDecoder(int maxFramePayloadLength, @Nullable Boolean expectMaskedFrames) {
     this.maxFramePayloadLength = maxFramePayloadLength;
     this.expectMaskedFrames = expectMaskedFrames;
   }
@@ -96,7 +96,9 @@ void decode(ChannelHandlerContext ctx, ByteBuf in) {
             int code = (prefix >> 24) & 0xF;
 
             Boolean expectMasked = expectMaskedFrames;
-            if (expectMasked != null && maskFlag != expectMasked) {
+            if (expectMasked == null) {
+              expectMaskedFrames = maskFlag;
+            } else if (maskFlag != expectMasked) {
               WebSocketProtocol.close(
                   ctx,
                   this,
@@ -183,7 +185,9 @@ void decode(ChannelHandlerContext ctx, ByteBuf in) {
             boolean maskFlag = masking = (prefix & 0x80) == 0x80;
 
             Boolean expectMasked = expectMaskedFrames;
-            if (expectMasked != null && maskFlag != expectMasked) {
+            if (expectMasked == null) {
+              expectMaskedFrames = maskFlag;
+            } else if (maskFlag != expectMasked) {
               WebSocketProtocol.close(
                   ctx,
                   this,
@@ -319,7 +323,9 @@ void decode(ChannelHandlerContext ctx, ByteBuf in) {
                 boolean maskFlag = masking = (prefix & 0x80) == 0x80;
 
                 Boolean expectMasked = expectMaskedFrames;
-                if (expectMasked != null && maskFlag != expectMasked) {
+                if (expectMasked == null) {
+                  expectMaskedFrames = maskFlag;
+                } else if (maskFlag != expectMasked) {
                   WebSocketProtocol.close(
                       ctx,
                       this,