Fixed upload/download encrypted file API

pubnub-kotlin/pubnub-kotlin-impl/src/integrationTest/kotlin/com/pubnub/api/integration/FilesIntegrationTest.kt

@@ -31,6 +31,111 @@ class FilesIntegrationTest : BaseIntegrationTest() {
         uploadListDownloadDelete(false)
     }
 
+    @Test
+    fun uploadListDownloadDeleteWithCipherUseFileWithLegacyCryptoModule() {
+        uploadListDownloadDeleteWithCipherFile(true)
+    }
+
+    @Test
+    fun uploadListDownloadDeleteWithCipherUseFileWithAesCbcCryptoModule() {
+        uploadListDownloadDeleteWithCipherFile(false)
+    }
+
+    fun uploadListDownloadDeleteWithCipherFile(withLegacyCrypto: Boolean) {
+        if (withLegacyCrypto) {
+            clientConfig = {
+                cryptoModule = CryptoModule.createLegacyCryptoModule("enigma")
+            }
+        } else {
+            clientConfig = {
+                cryptoModule = CryptoModule.createAesCbcCryptoModule("enigma")
+            }
+        }
+
+        val channel: String = randomChannel()
+        val fileName = "logback.xml"
+        val message = "This is message"
+        val meta = "This is meta"
+        val customMessageType = "myCustomType"
+
+        // Read the logback.xml file from resources
+        val logbackResource = this.javaClass.classLoader.getResourceAsStream("logback.xml")
+            ?: throw IllegalStateException("logback.xml not found in resources")
+        val originalContent = logbackResource.readBytes()
+        val originalContentString = String(originalContent, StandardCharsets.UTF_8)
+
+        val connectedLatch = CountDownLatch(1)
+        val fileEventReceived = CountDownLatch(1)
+        pubnub.addListener(
+            object : SubscribeCallback() {
+                override fun status(
+                    pubnub: PubNub,
+                    status: PNStatus,
+                ) {
+                    if (status.category == PNStatusCategory.PNConnectedCategory) {
+                        connectedLatch.countDown()
+                    }
+                }
+
+                override fun file(
+                    pubnub: PubNub,
+                    result: PNFileEventResult,
+                ) {
+                    if (result.file.name == fileName && result.customMessageType == customMessageType) {
+                        fileEventReceived.countDown()
+                    }
+                }
+            },
+        )
+        pubnub.subscribe(channels = listOf(channel))
+        connectedLatch.await(10, TimeUnit.SECONDS)
+
+        val sendResult: PNFileUploadResult? =
+            ByteArrayInputStream(originalContent).use {
+                pubnub.sendFile(
+                    channel = channel,
+                    fileName = fileName,
+                    inputStream = it,
+                    message = message,
+                    meta = meta,
+                    customMessageType = customMessageType
+                ).sync()
+            }
+
+        if (sendResult == null) {
+            Assert.fail()
+            return
+        }
+        fileEventReceived.await(10, TimeUnit.SECONDS)
+
+        val (_, _, _, data) = pubnub.listFiles(channel = channel).sync()
+        val fileFoundOnList = data.find { it.id == sendResult.file.id } != null
+        Assert.assertTrue(fileFoundOnList)
+
+        val (_, byteStream) =
+            pubnub.downloadFile(
+                channel = channel,
+                fileName = fileName,
+                fileId = sendResult.file.id,
+            ).sync()
+
+        byteStream?.use {
+            val downloadedContent = it.readBytes()
+            val downloadedString = String(downloadedContent, StandardCharsets.UTF_8)
+            Assert.assertEquals(
+                "Downloaded content should match original logback.xml",
+                originalContentString,
+                downloadedString
+            )
+        }
+
+        pubnub.deleteFile(
+            channel = channel,
+            fileName = fileName,
+            fileId = sendResult.file.id,
+        ).sync()
+    }
+
     @Test
     fun testSendFileAndDeleteFileOnChannelEntity() {
         val sendFileResultReference: AtomicReference<PNFileUploadResult> = AtomicReference()
@@ -205,6 +310,126 @@ class FilesIntegrationTest : BaseIntegrationTest() {
         ).sync()
     }
 
+    @Test
+    fun uploadLargeEncryptedFileWithLegacyCryptoModule() {
+        uploadLargeEncryptedFileWithCryptoModule(withLegacyCrypto = true)
+    }
+
+    @Test
+    fun uploadLargeEncryptedFileWithAesCbcCryptoModule() {
+        uploadLargeEncryptedFileWithCryptoModule(withLegacyCrypto = false)
+    }
+
+    fun uploadLargeEncryptedFileWithCryptoModule(withLegacyCrypto: Boolean) {
+        clientConfig = {
+            cryptoModule = CryptoModule.createLegacyCryptoModule("enigma")
+        }
+        val channel: String = randomChannel()
+        val fileName = "large_file_${System.currentTimeMillis()}.bin"
+
+        // Create a large binary file (1MB) to test encryption
+        val largeContent = ByteArray(1024 * 1024) { it.toByte() }
+
+        val sendResult: PNFileUploadResult? =
+            ByteArrayInputStream(largeContent).use {
+                pubnub.sendFile(
+                    channel = channel,
+                    fileName = fileName,
+                    inputStream = it,
+                    message = "Large encrypted file test",
+                ).sync()
+            }
+
+        if (sendResult == null) {
+            Assert.fail("Failed to upload large encrypted file")
+            return
+        }
+
+        // Download and verify
+        val (_, byteStream) =
+            pubnub.downloadFile(
+                channel = channel,
+                fileName = fileName,
+                fileId = sendResult.file.id,
+            ).sync()
+
+        byteStream?.use {
+            val downloadedContent = it.readBytes()
+            Assert.assertArrayEquals(
+                "Downloaded encrypted content should match original",
+                largeContent,
+                downloadedContent
+            )
+        }
+
+        // Cleanup
+        pubnub.deleteFile(
+            channel = channel,
+            fileName = fileName,
+            fileId = sendResult.file.id,
+        ).sync()
+    }
+
+    @Test
+    fun uploadMultipleSizesWithEncryption() {
+        clientConfig = {
+            cryptoModule = CryptoModule.createLegacyCryptoModule("enigma")
+        }
+        val channel: String = randomChannel()
+
+        val testSizes = listOf(
+            100, // Small file
+            1024, // 1KB
+            10240, // 10KB
+            102400, // 100KB
+            524288 // 512KB
+        )
+
+        for (size in testSizes) {
+            val fileName = "test_${size}_${System.currentTimeMillis()}.bin"
+            val content = ByteArray(size) { (it % 256).toByte() }
+
+            val sendResult: PNFileUploadResult? =
+                ByteArrayInputStream(content).use {
+                    pubnub.sendFile(
+                        channel = channel,
+                        fileName = fileName,
+                        inputStream = it,
+                        message = "Test file size: $size",
+                    ).sync()
+                }
+
+            if (sendResult == null) {
+                Assert.fail("Failed to upload file of size $size")
+                return
+            }
+
+            // Download and verify
+            val (_, byteStream) =
+                pubnub.downloadFile(
+                    channel = channel,
+                    fileName = fileName,
+                    fileId = sendResult.file.id,
+                ).sync()
+
+            byteStream?.use {
+                val downloadedContent = it.readBytes()
+                Assert.assertArrayEquals(
+                    "Downloaded content should match original for size $size",
+                    content,
+                    downloadedContent
+                )
+            }
+
+            // Cleanup
+            pubnub.deleteFile(
+                channel = channel,
+                fileName = fileName,
+                fileId = sendResult.file.id,
+            ).sync()
+        }
+    }
+
     private fun readToString(inputStream: InputStream): String {
         Scanner(inputStream).useDelimiter("\\A").use { s ->
             return if (s.hasNext()) {

pubnub-kotlin/pubnub-kotlin-impl/src/main/kotlin/com/pubnub/internal/crypto/cryptor/HeaderParser.kt

@@ -36,12 +36,11 @@ internal class HeaderParser(val logConfig: LogConfig?) {
     )
 
     fun parseDataWithHeader(stream: BufferedInputStream): ParseResult<out InputStream> {
-        val bufferedInputStream = stream.buffered()
-        bufferedInputStream.mark(Int.MAX_VALUE) // TODO Can be calculated from spec
+        stream.mark(Int.MAX_VALUE) // TODO Can be calculated from spec
         val possibleInitialHeader = ByteArray(MINIMAL_SIZE_OF_CRYPTO_HEADER)
-        val initiallyRead = bufferedInputStream.read(possibleInitialHeader)
+        val initiallyRead = stream.read(possibleInitialHeader)
         if (!possibleInitialHeader.sliceArray(SENTINEL_STARTING_INDEX..SENTINEL_ENDING_INDEX).contentEquals(SENTINEL)) {
-            bufferedInputStream.reset()
+            stream.reset()
             return ParseResult.NoHeader
         }
 
@@ -58,17 +57,17 @@ internal class HeaderParser(val logConfig: LogConfig?) {
 
         val cryptorData: ByteArray =
             if (cryptorDataSizeFirstByte == THREE_BYTES_SIZE_CRYPTOR_DATA_INDICATOR) {
-                val cryptorDataSizeBytes = readExactlyNBytez(bufferedInputStream, 2)
+                val cryptorDataSizeBytes = readExactlyNBytez(stream, 2)
                 val cryptorDataSize = convertTwoBytesToIntBigEndian(cryptorDataSizeBytes[0], cryptorDataSizeBytes[1])
-                readExactlyNBytez(bufferedInputStream, cryptorDataSize)
+                readExactlyNBytez(stream, cryptorDataSize)
             } else {
                 if (cryptorDataSizeFirstByte == UByte.MIN_VALUE) {
                     byteArrayOf()
                 } else {
-                    readExactlyNBytez(bufferedInputStream, cryptorDataSizeFirstByte.toInt())
+                    readExactlyNBytez(stream, cryptorDataSizeFirstByte.toInt())
                 }
             }
-        return ParseResult.Success(cryptorId, cryptorData, bufferedInputStream)
+        return ParseResult.Success(cryptorId, cryptorData, stream)
     }
 
     private fun readExactlyNBytez(
@@ -130,9 +129,9 @@ internal class HeaderParser(val logConfig: LogConfig?) {
         val finalCryptorDataSize: ByteArray =
             if (cryptorDataSize < THREE_BYTES_SIZE_CRYPTOR_DATA_INDICATOR.toInt()) {
                 byteArrayOf(cryptorDataSize.toByte()) // cryptorDataSize will be stored on 1 byte
-            } else if (cryptorDataSize < MAX_VALUE_THAT_CAN_BE_STORED_ON_TWO_BYTES) {
-                // cryptorDataSize will be stored on 3 byte
-                byteArrayOf(cryptorDataSize.toByte()) + writeNumberOnTwoBytes(cryptorDataSize)
+            } else if (cryptorDataSize <= MAX_VALUE_THAT_CAN_BE_STORED_ON_TWO_BYTES) {
+                // cryptorDataSize will be stored on 3 bytes: indicator (255) + 2 bytes for actual size
+                byteArrayOf(THREE_BYTES_SIZE_CRYPTOR_DATA_INDICATOR.toByte()) + writeNumberOnTwoBytes(cryptorDataSize)
             } else {
                 throw PubNubException(
                     errorMessage = "Cryptor Data Size is: $cryptorDataSize whereas max cryptor data size is: $MAX_VALUE_THAT_CAN_BE_STORED_ON_TWO_BYTES",

pubnub-kotlin/pubnub-kotlin-impl/src/main/kotlin/com/pubnub/internal/endpoints/files/SendFileEndpoint.kt

@@ -86,14 +86,15 @@ class SendFileEndpoint internal constructor(
     ): ExtendedRemoteAction<PNFileUploadResult> {
         val result = AtomicReference<FileUploadRequestDetails>()
 
+        val isEncrypted = cryptoModule != null
         val content =
             cryptoModule?.encryptStream(InputStreamSeparator(inputStream))?.use {
                 it.readBytes()
             } ?: inputStream.readBytes()
         return ComposableRemoteAction.firstDo(generateUploadUrlFactory.create(channel, fileName)) // 1. generateUrl
             .then { res ->
                 result.set(res)
-                sendFileToS3Factory.create(fileName, content, res) // 2. upload to s3
+                sendFileToS3Factory.create(fileName, content, res, isEncrypted) // 2. upload to s3
             }.checkpoint().then {
                 val details = result.get()
                 publishFileMessageFactory.create( // 3. PublishFileMessage

pubnub-kotlin/pubnub-kotlin-impl/src/main/kotlin/com/pubnub/internal/endpoints/files/UploadFileEndpoint.kt

@@ -28,6 +28,7 @@ internal class UploadFileEndpoint(
     private val key: FormField,
     private val formParams: List<FormField>,
     private val baseUrl: String,
+    private val isEncrypted: Boolean = false,
     pubNub: PubNubImpl,
 ) : EndpointCore<Unit, Unit>(pubNub) {
     private val log = LoggerManager.instance.getLogger(pubnub.logConfig, this::class.java)
@@ -65,21 +66,40 @@ internal class UploadFileEndpoint(
         log.debug(
             LogMessage(
                 message = LogMessageContent.Text(
-                    "Initiating S3 upload - fileName: $fileName, contentSize: ${content.size} bytes, contentType: $contentType, formFieldsCount: ${formParams.size}"
+                    "Initiating S3 upload - fileName: $fileName, contentSize: ${content.size} bytes, contentType: $contentType, isEncrypted: $isEncrypted, formFieldsCount: ${formParams.size}"
                 )
             )
         )
 
+        // Override Content-Type for encrypted files to prevent UTF-8 corruption of binary data
+        val modifiedFormParams = if (isEncrypted) {
+            formParams.map { param ->
+                if (param.key.equals(CONTENT_TYPE_HEADER, ignoreCase = true)) {
+                    FormField(param.key, "application/octet-stream")
+                } else {
+                    param
+                }
+            }
+        } else {
+            formParams
+        }
+
         val builder = MultipartBody.Builder().setType(MultipartBody.FORM)
-        addFormParamsWithKeyFirst(key, formParams, builder)
-        val mediaType = getMediaType(contentType)
+        addFormParamsWithKeyFirst(key, modifiedFormParams, builder)
+
+        // For encrypted files, always use application/octet-stream to prevent charset interpretation
+        val mediaType = if (isEncrypted) {
+            APPLICATION_OCTET_STREAM
+        } else {
+            getMediaType(modifiedFormParams.findContentType())
+        }
 
         builder.addFormDataPart(FILE_PART_MULTIPART, fileName, content.toRequestBody(mediaType, 0, content.size))
 
         log.debug(
             LogMessage(
                 message = LogMessageContent.Text(
-                    "Multipart request built - executing upload to S3 for fileName: $fileName"
+                    "Multipart request built - executing upload to S3 for fileName: $fileName with mediaType: $mediaType"
                 )
             )
         )
@@ -288,13 +308,15 @@ internal class UploadFileEndpoint(
             fileName: String,
             content: ByteArray,
             fileUploadRequestDetails: FileUploadRequestDetails,
+            isEncrypted: Boolean = false,
         ): UploadFileEndpoint {
             return UploadFileEndpoint(
                 fileName = fileName,
                 content = content,
                 key = fileUploadRequestDetails.keyFormField,
                 formParams = fileUploadRequestDetails.formFields,
                 baseUrl = fileUploadRequestDetails.url,
+                isEncrypted = isEncrypted,
                 pubNub = pubNub
             )
         }