Skip to content

Commit cee8059

Browse files
guruzguruz
committed
TUS: Use creation with upload again after checksum feature
1 parent b0fbc46 commit cee8059

5 files changed

Lines changed: 110 additions & 41 deletions

File tree

‎opencloudApp/src/main/java/eu/opencloud/android/workers/TusUploadHelper.kt‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ class TusUploadHelper(
5555
) : String? {
5656
// Reset cancelled state for new upload
5757
cancelled = false
58-
val checksum = TusChecksumHelper.parseStoredChecksum(transfer.tusUploadChecksum)
58+
val fileChecksum = TusChecksumHelper.parseStoredChecksum(transfer.tusUploadChecksum)
5959
?.takeIf { it.uploadAlgorithm == TusChecksumHelper.SHA1_WIRE_ALGORITHM }
6060
Timber.d("TUS: starting upload for %s size=%d", remotePath, fileSize)
6161

@@ -69,7 +69,8 @@ class TusUploadHelper(
6969
mimeType = mimeType,
7070
lastModified = lastModified,
7171
spaceWebDavUrl = spaceWebDavUrl,
72-
checksum = checksum,
72+
fileChecksum = fileChecksum,
73+
tusSupport = tusSupport,
7374
)
7475

7576
val offset = fetchCurrentOffset(client, resolvedTusUrl, createdOffset)
@@ -86,7 +87,7 @@ class TusUploadHelper(
8687
progressCallback = progressCallback,
8788
initialOffset = offset,
8889
uploadId = uploadId,
89-
checksum = checksum,
90+
checksumAlgorithm = fileChecksum?.uploadAlgorithm,
9091
)
9192

9293
verifyUploadCompletion(finalOffset, fileSize, uploadId)
@@ -104,7 +105,8 @@ class TusUploadHelper(
104105
mimeType: String,
105106
lastModified: String?,
106107
spaceWebDavUrl: String?,
107-
checksum: TusChecksumHelper.StoredChecksum?,
108+
fileChecksum: TusChecksumHelper.StoredChecksum?,
109+
tusSupport: OCCapability.TusSupport?,
108110
): Pair<String, Long?> {
109111
var tusUrl = transfer.tusUploadUrl
110112
var createdOffset: Long? = null
@@ -116,7 +118,7 @@ class TusUploadHelper(
116118
"mimetype" to mimeType,
117119
)
118120
lastModified?.takeIf { it.isNotBlank() }?.let { metadata["mtime"] = it }
119-
checksum?.let { metadata["checksum"] = it.metadataValue }
121+
fileChecksum?.let { metadata["checksum"] = it.metadataValue }
120122

121123
Timber.d(
122124
"TUS: creating upload resource filename=%s size=%d metadata=%s",
@@ -130,23 +132,25 @@ class TusUploadHelper(
130132
spaceWebDavUrl = spaceWebDavUrl
131133
)
132134

133-
// Checked uploads must send every byte via PATCH so each chunk can carry Upload-Checksum.
134-
val useCreationWithUpload = checksum == null
135-
val firstChunkSize = if (useCreationWithUpload) {
136-
minOf(CreateTusUploadRemoteOperation.DEFAULT_FIRST_CHUNK, fileSize)
137-
} else {
138-
null
139-
}
135+
// Use creation-with-upload like the browser does for OpenCloud compatibility.
136+
// The data part of a creation-with-upload POST follows the same rules as a PATCH
137+
// (TUS spec), so it carries Upload-Checksum for the first chunk just like the
138+
// PATCH requests do for the remaining ones — and like a PATCH it must respect
139+
// the server's max_chunk_size (DEFAULT_FIRST_CHUNK is 10 MiB, but e.g. OpenCloud
140+
// advertises 10_000_000, slightly smaller).
141+
val serverMaxChunk = tusSupport?.maxChunkSize?.takeIf { it > 0 }?.toLong() ?: Long.MAX_VALUE
142+
val firstChunkSize = minOf(CreateTusUploadRemoteOperation.DEFAULT_FIRST_CHUNK, fileSize, serverMaxChunk)
140143
val creationResult = executeRemoteOperation {
141144
CreateTusUploadRemoteOperation(
142145
file = File(localPath),
143146
remotePath = remotePath,
144147
mimetype = mimeType,
145148
metadata = metadata,
146-
useCreationWithUpload = useCreationWithUpload,
149+
useCreationWithUpload = true,
147150
firstChunkSize = firstChunkSize,
148151
tusUrl = "",
149152
collectionUrlOverride = collectionUrl,
153+
checksumAlgorithm = fileChecksum?.uploadAlgorithm,
150154
).execute(client)
151155
}
152156

@@ -163,7 +167,7 @@ class TusUploadHelper(
163167
tusUploadUrl = tusUrl,
164168
tusUploadLength = fileSize,
165169
tusUploadMetadata = metadataString,
166-
tusUploadChecksum = checksum?.storageValue,
170+
tusUploadChecksum = fileChecksum?.storageValue,
167171
tusResumableVersion = "1.0.0",
168172
tusUploadExpires = null,
169173
tusUploadConcat = null,
@@ -243,7 +247,7 @@ class TusUploadHelper(
243247
progressCallback: ((Long, Long) -> Unit)?,
244248
initialOffset: Long,
245249
uploadId: Long,
246-
checksum: TusChecksumHelper.StoredChecksum?,
250+
checksumAlgorithm: String?,
247251
): Pair<Long, String?> {
248252
var offset = initialOffset
249253
var lastEtag: String? = null
@@ -267,7 +271,7 @@ class TusUploadHelper(
267271
offset = offset,
268272
chunkSize = chunkSize,
269273
httpMethodOverride = httpOverride,
270-
checksum = checksum,
274+
checksumAlgorithm = checksumAlgorithm,
271275
).apply {
272276
progressListener?.let { addDataTransferProgressListener(it) }
273277
}
@@ -276,7 +280,7 @@ class TusUploadHelper(
276280
val patchResult = patchOperation.execute(client)
277281
lastEtag = patchOperation.etag.takeIf { it.isNotBlank() }
278282
activePatchOperation = null
279-
if (checksum != null && isChecksumFailure(patchResult.httpCode)) {
283+
if (checksumAlgorithm != null && isChecksumFailure(patchResult.httpCode)) {
280284
clearTusState(uploadId)
281285
throw java.io.IOException(
282286
"TUS: checksum upload rejected with HTTP ${patchResult.httpCode} at offset $offset"

‎opencloudApp/src/test/java/eu/opencloud/android/workers/TusUploadHelperTest.kt‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import io.mockk.verify
3131
import okhttp3.mockwebserver.MockResponse
3232
import okhttp3.mockwebserver.MockWebServer
3333
import org.junit.After
34+
import org.junit.Assert.assertArrayEquals
3435
import org.junit.Assert.assertEquals
3536
import org.junit.Assert.assertFalse
3637
import org.junit.Assert.assertNull
@@ -65,18 +66,19 @@ class TusUploadHelperTest {
6566
}
6667

6768
@Test
68-
fun upload_createsCheckedSessionWithoutFirstChunkAndClearsTusState() {
69+
fun upload_createsCheckedSessionWithFirstChunkChecksumAndClearsTusState() {
6970
val localFile = tempFileWithBytes(byteArrayOf(1, 2, 3, 4, 5))
7071
val sha1Hex = TusChecksumHelper.sha1Hex(localFile)
7172
val storedChecksum = TusChecksumHelper.storedSha1(sha1Hex).storageValue
7273
val uploadUrl = "/uploads/new-session"
74+
// creation-with-upload: the whole 5-byte file fits in the creation POST,
75+
// so the server acknowledges Upload-Offset 5 and no PATCH follows.
7376
server.enqueue(
7477
MockResponse()
7578
.setResponseCode(201)
7679
.addHeader("Location", uploadUrl)
77-
.addHeader("Upload-Offset", "0")
80+
.addHeader("Upload-Offset", "5")
7881
)
79-
server.enqueue(MockResponse().setResponseCode(204).addHeader("Upload-Offset", "5"))
8082
server.enqueue(MockResponse().setResponseCode(404))
8183
val progress = mutableListOf<Long>()
8284

@@ -96,27 +98,24 @@ class TusUploadHelperTest {
9698
)
9799

98100
assertNull(resultEtag)
99-
assertEquals(listOf(0L, 5L), progress)
101+
assertEquals(listOf(5L), progress)
100102

101103
val createRequest = server.takeRequest()
102104
assertEquals("POST", createRequest.method)
103105
assertEquals("/dav/spaces/personal/Photos", createRequest.path)
104-
assertNull(createRequest.getHeader("Upload-Offset"))
106+
assertEquals("0", createRequest.getHeader("Upload-Offset"))
105107
assertEquals("5", createRequest.getHeader("Upload-Length"))
106108
assertTrue(createRequest.getHeader("Upload-Metadata")!!.contains("checksum"))
107-
108-
val patchRequest = server.takeRequest()
109-
assertEquals("PATCH", patchRequest.method)
110-
assertEquals("0", patchRequest.getHeader("Upload-Offset"))
111109
assertEquals(
112110
TusChecksumHelper.uploadChecksumHeader(
113111
file = localFile,
114112
offset = 0,
115113
length = 5,
116114
algorithm = TusChecksumHelper.SHA1_WIRE_ALGORITHM,
117115
),
118-
patchRequest.getHeader("Upload-Checksum")
116+
createRequest.getHeader("Upload-Checksum")
119117
)
118+
assertArrayEquals(byteArrayOf(1, 2, 3, 4, 5), createRequest.body.readByteArray())
120119

121120
verify {
122121
transferRepository.updateTusState(

‎opencloudComLibrary/src/main/java/eu/opencloud/android/lib/resources/files/tus/CreateTusUploadRemoteOperation.kt‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ class CreateTusUploadRemoteOperation(
2929
private val firstChunkSize: Long?,
3030
private val tusUrl: String?,
3131
private val collectionUrlOverride: String? = null,
32-
private val base64Encoder: Base64Encoder = DefaultBase64Encoder()
32+
private val base64Encoder: Base64Encoder = DefaultBase64Encoder(),
33+
/** When set (e.g. "sha1"), an Upload-Checksum header is computed over the first chunk. */
34+
private val checksumAlgorithm: String? = null,
3335
) : RemoteOperation<CreateTusUploadRemoteOperation.CreationResult>() {
3436

3537
data class CreationResult(
@@ -103,7 +105,7 @@ class CreateTusUploadRemoteOperation(
103105

104106
// Set Upload-Offset for creation-with-upload
105107
if (useCreationWithUpload && (firstChunkSize ?: 0L) > 0L) {
106-
postMethod.setRequestHeader(HttpConstants.UPLOAD_OFFSET, "0")
108+
setCreationWithUploadHeaders(postMethod, firstChunkSize!!)
107109
}
108110

109111
val status = client.executeHttpMethod(postMethod)
@@ -164,6 +166,21 @@ class CreateTusUploadRemoteOperation(
164166
result
165167
}
166168

169+
private fun setCreationWithUploadHeaders(postMethod: PostMethod, firstChunkSize: Long) {
170+
postMethod.setRequestHeader(HttpConstants.UPLOAD_OFFSET, "0")
171+
// The data part of a creation-with-upload POST follows the same rules as a
172+
// PATCH (TUS spec), so it carries Upload-Checksum for the first chunk.
173+
checksumAlgorithm?.let { algorithm ->
174+
val chunkChecksumHeader = TusChecksumHelper.uploadChecksumHeader(
175+
file = file,
176+
offset = 0,
177+
length = firstChunkSize,
178+
algorithm = algorithm,
179+
)
180+
postMethod.setRequestHeader(HttpConstants.UPLOAD_CHECKSUM, chunkChecksumHeader)
181+
}
182+
}
183+
167184
private fun isSuccess(status: Int) =
168185
status.isOneOf(HttpConstants.HTTP_CREATED, HttpConstants.HTTP_OK)
169186

‎opencloudComLibrary/src/main/java/eu/opencloud/android/lib/resources/files/tus/PatchTusUploadChunkRemoteOperation.kt‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ class PatchTusUploadChunkRemoteOperation(
3232
private val offset: Long,
3333
private val chunkSize: Long,
3434
private val httpMethodOverride: String? = null,
35-
private val checksum: TusChecksumHelper.StoredChecksum? = null,
35+
/** When set (e.g. "sha1"), an Upload-Checksum header is computed over this chunk's bytes. */
36+
private val checksumAlgorithm: String? = null,
3637
) : RemoteOperation<Long>() {
3738

3839
private val cancellationRequested = AtomicBoolean(false)
@@ -75,16 +76,14 @@ class PatchTusUploadChunkRemoteOperation(
7576
setRequestHeader(HttpConstants.TUS_RESUMABLE, HttpConstants.TUS_RESUMABLE_VERSION_1_0_0)
7677
setRequestHeader(HttpConstants.UPLOAD_OFFSET, offset.toString())
7778
setRequestHeader(HttpConstants.CONTENT_TYPE_HEADER, HttpConstants.CONTENT_TYPE_OFFSET_OCTET_STREAM)
78-
checksum?.let {
79-
setRequestHeader(
80-
HttpConstants.UPLOAD_CHECKSUM,
81-
TusChecksumHelper.uploadChecksumHeader(
82-
file = file,
83-
offset = offset,
84-
length = chunkSize,
85-
algorithm = it.uploadAlgorithm,
86-
)
79+
checksumAlgorithm?.let { algorithm ->
80+
val chunkChecksumHeader = TusChecksumHelper.uploadChecksumHeader(
81+
file = file,
82+
offset = offset,
83+
length = chunkSize,
84+
algorithm = algorithm,
8785
)
86+
setRequestHeader(HttpConstants.UPLOAD_CHECKSUM, chunkChecksumHeader)
8887
}
8988
}
9089

‎opencloudComLibrary/src/test/java/eu/opencloud/android/lib/resources/files/tus/TusIntegrationTest.kt‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,6 @@ class TusIntegrationTest {
243243
val localFile = File.createTempFile("tus", ".bin").apply {
244244
writeBytes(byteArrayOf(1, 2, 3, 4, 5))
245245
}
246-
val checksum = TusChecksumHelper.storedSha1(TusChecksumHelper.sha1Hex(localFile))
247246
server.enqueue(
248247
MockResponse()
249248
.setResponseCode(204)
@@ -255,7 +254,7 @@ class TusIntegrationTest {
255254
uploadUrl = server.url(locationPath).toString(),
256255
offset = 0,
257256
chunkSize = 5,
258-
checksum = checksum,
257+
checksumAlgorithm = TusChecksumHelper.SHA1_WIRE_ALGORITHM,
259258
)
260259
val patchResult = patch.execute(client)
261260

@@ -339,6 +338,57 @@ class TusIntegrationTest {
339338
assertEquals(firstChunkSize.toString(), postReq.getHeader("Content-Length"))
340339
}
341340

341+
@Test
342+
fun creation_with_upload_sendsUploadChecksumForFirstChunk() {
343+
val client = newClient()
344+
val collectionPath = "/remote.php/dav/uploads/$userId"
345+
val locationPath = "$collectionPath/UPLD-WITH-DATA-CHECKSUM"
346+
val localFile = File.createTempFile("tus", ".bin").apply {
347+
writeBytes(ByteArray(100) { it.toByte() })
348+
}
349+
val firstChunkSize = 50L
350+
server.enqueue(
351+
MockResponse()
352+
.setResponseCode(201)
353+
.addHeader("Tus-Resumable", "1.0.0")
354+
.addHeader("Location", locationPath)
355+
.addHeader("Upload-Offset", firstChunkSize.toString())
356+
)
357+
358+
val create = CreateTusUploadRemoteOperation(
359+
file = localFile,
360+
remotePath = "/test-with-data.bin",
361+
mimetype = "application/octet-stream",
362+
metadata = mapOf("filename" to "test-with-data.bin"),
363+
useCreationWithUpload = true,
364+
firstChunkSize = firstChunkSize,
365+
tusUrl = null,
366+
collectionUrlOverride = server.url(collectionPath).toString(),
367+
base64Encoder = object : Base64Encoder {
368+
override fun encode(bytes: ByteArray): String =
369+
Base64.getEncoder().encodeToString(bytes)
370+
},
371+
checksumAlgorithm = TusChecksumHelper.SHA1_WIRE_ALGORITHM,
372+
)
373+
374+
val createResult = create.execute(client)
375+
assertTrue("Create operation failed", createResult.isSuccess)
376+
377+
val postReq = server.takeRequest()
378+
assertEquals("POST", postReq.method)
379+
assertEquals("0", postReq.getHeader("Upload-Offset"))
380+
// The Upload-Checksum header must cover exactly the first chunk, not the whole file.
381+
assertEquals(
382+
TusChecksumHelper.uploadChecksumHeader(
383+
file = localFile,
384+
offset = 0,
385+
length = firstChunkSize,
386+
algorithm = TusChecksumHelper.SHA1_WIRE_ALGORITHM,
387+
),
388+
postReq.getHeader("Upload-Checksum")
389+
)
390+
}
391+
342392
@Test
343393
fun patch_wrong_offset_returns_conflict() {
344394
val client = newClient()

0 commit comments

Comments
 (0)