Merge pull request #15759 from nextcloud/fix/auto-upload-state

fix: auto upload db state

Author: alperozturk96

app/src/androidTest/java/com/owncloud/android/datamodel/UploadStorageManagerTest.java

@@ -18,6 +18,7 @@
 import com.nextcloud.client.account.User;
 import com.nextcloud.client.account.UserAccountManager;
 import com.nextcloud.client.account.UserAccountManagerImpl;
+import com.nextcloud.client.database.entity.UploadEntityKt;
 import com.nextcloud.test.RandomStringGenerator;
 import com.owncloud.android.AbstractIT;
 import com.owncloud.android.MainApp;
@@ -108,7 +109,7 @@ public void largeTest() {
             OCUpload upload = createUpload(account);
 
             uploads.add(upload);
-            uploadsStorageManager.storeUpload(upload);
+            uploadsStorageManager.uploadDao.insertOrReplace(UploadEntityKt.toUploadEntity(upload));
         }
 
         OCUpload[] storedUploads = uploadsStorageManager.getAllStoredUploads();
@@ -151,17 +152,14 @@ public void corruptedUpload() {
                                               account.name);
 
         corruptUpload.setLocalPath(null);
-
-        uploadsStorageManager.storeUpload(corruptUpload);
-
+        uploadsStorageManager.uploadDao.insertOrReplace(UploadEntityKt.toUploadEntity(corruptUpload));
         uploadsStorageManager.getAllStoredUploads();
     }
 
     @Test
     public void getById() {
         OCUpload upload = createUpload(account);
-        long id = uploadsStorageManager.storeUpload(upload);
-
+        long id =  uploadsStorageManager.uploadDao.insertOrReplace(UploadEntityKt.toUploadEntity(upload));
         OCUpload newUpload = uploadsStorageManager.getUploadById(id);
 
         assertNotNull(newUpload);
@@ -178,7 +176,7 @@ public void getByIdNull() {
 
     private void insertUploads(Account account, int rowsToInsert) {
         for (int i = 0; i < rowsToInsert; i++) {
-            uploadsStorageManager.storeUpload(createUpload(account));
+            uploadsStorageManager.uploadDao.insertOrReplace(UploadEntityKt.toUploadEntity(createUpload(account)));
         }
     }
 

app/src/main/java/com/nextcloud/client/database/dao/FileSystemDao.kt

@@ -9,21 +9,24 @@ package com.nextcloud.client.database.dao
 
 import androidx.room.Dao
 import androidx.room.Query
+import com.nextcloud.client.database.entity.FilesystemEntity
 import com.owncloud.android.db.ProviderMeta
 
 @Dao
 interface FileSystemDao {
-
     @Query(
         """
-        SELECT ${ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_LOCAL_PATH}
+        SELECT *
         FROM ${ProviderMeta.ProviderTableMeta.FILESYSTEM_TABLE_NAME}
         WHERE ${ProviderMeta.ProviderTableMeta.FILESYSTEM_SYNCED_FOLDER_ID} = :syncedFolderId
           AND ${ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_SENT_FOR_UPLOAD} = 0
           AND ${ProviderMeta.ProviderTableMeta.FILESYSTEM_FILE_IS_FOLDER} = 0
+          AND ${ProviderMeta.ProviderTableMeta._ID} > :lastId
+        ORDER BY ${ProviderMeta.ProviderTableMeta._ID}
+        LIMIT :limit
     """
     )
-    suspend fun getAutoUploadFiles(syncedFolderId: String): List<String>
+    suspend fun getAutoUploadFilesEntities(syncedFolderId: String, limit: Int, lastId: Int): List<FilesystemEntity>
 
     @Query(
         """

app/src/main/java/com/nextcloud/client/database/dao/UploadDao.kt

@@ -45,4 +45,24 @@ interface UploadDao {
             "AND ${ProviderTableMeta.UPLOADS_REMOTE_PATH} = :remotePath"
     )
     fun deleteByAccountAndRemotePath(accountName: String, remotePath: String)
+
+    @Query(
+        "SELECT * FROM " + ProviderTableMeta.UPLOADS_TABLE_NAME +
+            " WHERE " + ProviderTableMeta._ID + " = :id AND " +
+            ProviderTableMeta.UPLOADS_ACCOUNT_NAME + " = :accountName " +
+            "LIMIT 1"
+    )
+    fun getUploadById(id: Long, accountName: String): UploadEntity?
+
+    @Insert(onConflict = OnConflictStrategy.Companion.REPLACE)
+    fun insertOrReplace(entity: UploadEntity): Long
+
+    @Query(
+        "SELECT * FROM " + ProviderTableMeta.UPLOADS_TABLE_NAME +
+            " WHERE " + ProviderTableMeta.UPLOADS_ACCOUNT_NAME + " = :accountName AND " +
+            ProviderTableMeta.UPLOADS_LOCAL_PATH + " = :localPath AND " +
+            ProviderTableMeta.UPLOADS_REMOTE_PATH + " = :remotePath " +
+            "LIMIT 1"
+    )
+    fun getUploadByAccountAndPaths(accountName: String, localPath: String, remotePath: String): UploadEntity?
 }

app/src/main/java/com/nextcloud/client/database/entity/UploadEntity.kt

@@ -1,6 +1,7 @@
 /*
  * Nextcloud - Android Client
  *
+ * SPDX-FileCopyrightText: 2025 Alper Ozturk <alper.ozturk@nextcloud.com>
  * SPDX-FileCopyrightText: 2022 Álvaro Brey <alvaro@alvarobrey.com>
  * SPDX-FileCopyrightText: 2022 Nextcloud GmbH
  * SPDX-License-Identifier: AGPL-3.0-or-later OR GPL-2.0-only
@@ -32,7 +33,7 @@ data class UploadEntity(
     @ColumnInfo(name = ProviderTableMeta.UPLOADS_FILE_SIZE)
     val fileSize: Long?,
     @ColumnInfo(name = ProviderTableMeta.UPLOADS_STATUS)
-    var status: Int?,
+    val status: Int?,
     @ColumnInfo(name = ProviderTableMeta.UPLOADS_LOCAL_BEHAVIOUR)
     val localBehaviour: Int?,
     @ColumnInfo(name = ProviderTableMeta.UPLOADS_UPLOAD_TIME)
@@ -78,3 +79,33 @@ fun UploadEntity.toOCUpload(capability: OCCapability? = null): OCUpload {
 
     return upload
 }
+
+fun OCUpload.toUploadEntity(): UploadEntity {
+    val id = if (uploadId == -1L) {
+        // needed for the insert new records to the db so that insert DAO function returns new generated id
+        null
+    } else {
+        uploadId
+    }
+
+    return UploadEntity(
+        id = id?.toInt(),
+        localPath = localPath,
+        remotePath = remotePath,
+        accountName = accountName,
+        fileSize = fileSize,
+        status = uploadStatus?.value,
+        localBehaviour = localAction,
+        nameCollisionPolicy = nameCollisionPolicy?.serialize(),
+        isCreateRemoteFolder = if (isCreateRemoteFolder) 1 else 0,
+
+        // uploadEndTimestamp may overflow max int capacity since it is conversion from long to int. coerceAtMost needed
+        uploadEndTimestamp = uploadEndTimestamp.coerceAtMost(Int.MAX_VALUE.toLong()).toInt(),
+        lastResult = lastResult?.value,
+        createdBy = createdBy,
+        isWifiOnly = if (isUseWifiOnly) 1 else 0,
+        isWhileChargingOnly = if (isWhileChargingOnly) 1 else 0,
+        folderUnlockToken = folderUnlockToken,
+        uploadTime = null
+    )
+}

app/src/main/java/com/nextcloud/client/jobs/autoUpload/AutoUploadWorker.kt

@@ -17,12 +17,16 @@ import androidx.work.CoroutineWorker
 import androidx.work.WorkerParameters
 import com.nextcloud.client.account.User
 import com.nextcloud.client.account.UserAccountManager
+import com.nextcloud.client.database.entity.UploadEntity
+import com.nextcloud.client.database.entity.toOCUpload
+import com.nextcloud.client.database.entity.toUploadEntity
 import com.nextcloud.client.device.PowerManagementService
 import com.nextcloud.client.jobs.BackgroundJobManager
 import com.nextcloud.client.jobs.upload.FileUploadWorker
 import com.nextcloud.client.network.ConnectivityService
 import com.nextcloud.client.preferences.SubFolderRule
 import com.nextcloud.utils.ForegroundServiceHelper
+import com.nextcloud.utils.extensions.updateStatus
 import com.owncloud.android.R
 import com.owncloud.android.datamodel.ArbitraryDataProviderImpl
 import com.owncloud.android.datamodel.FileDataStorageManager
@@ -41,7 +45,6 @@ import com.owncloud.android.ui.notifications.NotificationUtils
 import com.owncloud.android.utils.FileStorageUtils
 import com.owncloud.android.utils.FilesSyncHelper
 import com.owncloud.android.utils.MimeType
-import com.owncloud.android.utils.MimeTypeUtil
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.File
@@ -235,37 +238,19 @@ class AutoUploadWorker(
     private fun getUserOrReturn(syncedFolder: SyncedFolder): User? {
         val optionalUser = userAccountManager.getUser(syncedFolder.account)
         if (!optionalUser.isPresent) {
-            Log_OC.w(TAG, "uploadFilesFromFolder skipped user not present")
+            Log_OC.w(TAG, "user not present")
             return null
         }
         return optionalUser.get()
     }
 
-    private fun buildPathsAndMimes(
-        paths: Set<String>,
-        syncedFolder: SyncedFolder,
-        dateFormat: SimpleDateFormat
-    ): List<Triple<String, String, String>> {
-        val lightVersion = context.resources.getBoolean(R.bool.syncedFolder_light)
-        val currentLocale = context.resources.configuration.locales[0]
-
-        return paths.map { path ->
-            val file = File(path)
-            val localPath = file.absolutePath
-            val remotePath =
-                getRemotePath(file, syncedFolder, dateFormat, lightVersion, context.resources, currentLocale)
-            val mimeType = MimeTypeUtil.getBestMimeTypeByFilename(localPath)
-            Triple(localPath, remotePath, mimeType)
-        }
-    }
-
     @Suppress("DEPRECATION")
     private fun getUploadSettings(syncedFolder: SyncedFolder): Triple<Boolean, Boolean, Int> {
         val lightVersion = context.resources.getBoolean(R.bool.syncedFolder_light)
         val accountName = syncedFolder.account
 
         return if (lightVersion) {
-            Log_OC.d(TAG, "uploadFilesFromFolder light version is used")
+            Log_OC.d(TAG, "light version is used")
             val arbitraryDataProvider = ArbitraryDataProviderImpl(context)
             val needsCharging = context.resources.getBoolean(R.bool.syncedFolder_light_on_charging)
             val needsWifi = arbitraryDataProvider.getBooleanValue(
@@ -277,7 +262,7 @@ class AutoUploadWorker(
             Log_OC.d(TAG, "upload action is: $uploadAction")
             Triple(needsCharging, needsWifi, uploadAction)
         } else {
-            Log_OC.d(TAG, "getUploadSettings not light version is used")
+            Log_OC.d(TAG, "not light version is used")
             Triple(syncedFolder.isChargingOnly, syncedFolder.isWifiOnly, syncedFolder.uploadAction)
         }
     }
@@ -286,48 +271,116 @@ class AutoUploadWorker(
     private suspend fun uploadFiles(syncedFolder: SyncedFolder) = withContext(Dispatchers.IO) {
         val dateFormat = prepareDateFormat()
         val user = getUserOrReturn(syncedFolder) ?: return@withContext
-        val paths = repository.getAutoUploadFiles(syncedFolder)
-        if (paths.isEmpty()) {
-            Log_OC.w(TAG, "uploadFiles skipped paths is empty")
-            return@withContext
-        }
-
-        val pathsAndMimes = buildPathsAndMimes(paths, syncedFolder, dateFormat)
-        val (needsCharging, needsWifi, uploadAction) = getUploadSettings(syncedFolder)
-
         val ocAccount = OwnCloudAccount(user.toPlatformAccount(), context)
         val client = OwnCloudClientManagerFactory.getDefaultSingleton()
             .getClientFor(ocAccount, context)
+        val lightVersion = context.resources.getBoolean(R.bool.syncedFolder_light)
+        val currentLocale = context.resources.configuration.locales[0]
 
-        pathsAndMimes.forEach { (localPath, remotePath, _) ->
-            try {
-                Log_OC.d(TAG, "creating oc upload for ${user.accountName}")
-                val upload = OCUpload(localPath, remotePath, user.accountName).apply {
-                    nameCollisionPolicy = syncedFolder.nameCollisionPolicy
-                    isUseWifiOnly = needsWifi
-                    isWhileChargingOnly = needsCharging
-                    uploadStatus = UploadsStorageManager.UploadStatus.UPLOAD_IN_PROGRESS
-                    createdBy = UploadFileOperation.CREATED_AS_INSTANT_PICTURE
-                    isCreateRemoteFolder = true
-                    localAction = uploadAction
+        var lastId = 0
+        while (true) {
+            val filePathsWithIds = repository.getFilePathsWithIds(syncedFolder, lastId)
+
+            if (filePathsWithIds.isEmpty()) {
+                Log_OC.w(TAG, "no more files to upload at lastId: $lastId")
+                break
+            }
+            Log_OC.d(TAG, "Processing batch: lastId=$lastId, count=${filePathsWithIds.size}")
+
+            filePathsWithIds.forEach { (path, id) ->
+                val file = File(path)
+                val localPath = file.absolutePath
+                val remotePath = getRemotePath(
+                    file,
+                    syncedFolder,
+                    dateFormat,
+                    lightVersion,
+                    context.resources,
+                    currentLocale
+                )
+
+                try {
+                    var (uploadEntity, upload) = createEntityAndUpload(user, localPath, remotePath)
+                    try {
+                        // Insert/update to IN_PROGRESS state before starting upload
+                        val generatedId = uploadsStorageManager.uploadDao.insertOrReplace(uploadEntity)
+                        uploadEntity = uploadEntity.copy(id = generatedId.toInt())
+                        upload.uploadId = generatedId
+
+                        val operation = createUploadFileOperation(upload, user)
+                        Log_OC.d(TAG, "🕒 uploading: $localPath, id: $generatedId")
+
+                        val result = operation.execute(client)
+                        uploadsStorageManager.updateStatus(uploadEntity, result.isSuccess)
+
+                        if (result.isSuccess) {
+                            repository.markFileAsUploaded(localPath, syncedFolder)
+                            Log_OC.d(TAG, "✅ upload completed: $localPath")
+                        } else {
+                            Log_OC.e(
+                                TAG,
+                                "❌ upload failed $localPath (${upload.accountName}): ${result.logMessage}"
+                            )
+                        }
+                    } catch (e: Exception) {
+                        uploadsStorageManager.updateStatus(
+                            uploadEntity,
+                            UploadsStorageManager.UploadStatus.UPLOAD_FAILED
+                        )
+                        Log_OC.e(
+                            TAG,
+                            "Exception during upload file, localPath: $localPath, remotePath: $remotePath," +
+                                " exception: $e"
+                        )
+                    }
+                } catch (e: Exception) {
+                    Log_OC.e(
+                        TAG,
+                        "Exception uploadFiles during creating entity and upload, localPath: $localPath, " +
+                            "remotePath: $remotePath, exception: $e"
+                    )
                 }
 
-                uploadsStorageManager.storeUpload(upload)
+                // update last id so upload can continue where it left
+                lastId = id
+            }
+        }
+    }
+
+    private fun createEntityAndUpload(user: User, localPath: String, remotePath: String): Pair<UploadEntity, OCUpload> {
+        val (needsCharging, needsWifi, uploadAction) = getUploadSettings(syncedFolder)
+        Log_OC.d(TAG, "creating oc upload for ${user.accountName}")
 
-                val operation = createUploadFileOperation(upload, user)
-                Log_OC.d(TAG, "🕒 uploading: $localPath")
+        // Get or create upload entity
+        var uploadEntity = uploadsStorageManager.uploadDao.getUploadByAccountAndPaths(
+            localPath = localPath,
+            remotePath = remotePath,
+            accountName = user.accountName
+        )
 
-                val result = operation.execute(client)
-                if (result.isSuccess) {
-                    repository.markFileAsUploaded(localPath, syncedFolder)
-                    Log_OC.d(TAG, "✅ auto upload completed: $localPath")
-                } else {
-                    Log_OC.e(TAG, "❌ auto upload failed: $localPath")
-                }
-            } catch (e: Exception) {
-                Log_OC.e(TAG, "Exception uploadFiles, localPath: $localPath, remotePath: $remotePath, exception: $e")
+        val upload: OCUpload
+        if (uploadEntity != null) {
+            // Existing upload - convert and update status
+
+            upload = uploadEntity.toOCUpload(null)
+            upload.uploadStatus = UploadsStorageManager.UploadStatus.UPLOAD_IN_PROGRESS
+            uploadEntity = upload.toUploadEntity()
+        } else {
+            // New upload - create with all settings
+
+            upload = OCUpload(localPath, remotePath, user.accountName).apply {
+                nameCollisionPolicy = syncedFolder.nameCollisionPolicy
+                isUseWifiOnly = needsWifi
+                isWhileChargingOnly = needsCharging
+                uploadStatus = UploadsStorageManager.UploadStatus.UPLOAD_IN_PROGRESS
+                createdBy = UploadFileOperation.CREATED_AS_INSTANT_PICTURE
+                isCreateRemoteFolder = true
+                localAction = uploadAction
             }
+            uploadEntity = upload.toUploadEntity()
         }
+
+        return uploadEntity to upload
     }
 
     private fun createUploadFileOperation(upload: OCUpload, user: User): UploadFileOperation = UploadFileOperation(