Don't copy existing media files when updating media only

collect_app/src/androidTest/java/org/odk/collect/android/benchmark/EntitiesBenchmarkTest.kt

@@ -1,7 +1,5 @@
 package org.odk.collect.android.benchmark
 
-import android.app.Application
-import androidx.test.core.app.ApplicationProvider
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.hamcrest.MatcherAssert.assertThat
 import org.hamcrest.Matchers.blankOrNullString
@@ -44,7 +42,6 @@ class EntitiesBenchmarkTest {
             ENTITIES_FILTER_PROJECT_URL,
             not(blankOrNullString())
         )
-        clearAndroidCache()
 
         val benchmarker = Benchmarker()
 
@@ -101,6 +98,10 @@ class EntitiesBenchmarkTest {
      * [THOUSAND_MEDIA_FILE_ENTITY_LIST_PROJECT_URL] should be set to a project that contains the
      * "1000-media-files-entity-list" form.
      *
+     * This scenario could also arise when updating a form that has a single new/updated non-entity
+     * media file, but in practice this will probably be most common with entity forms as the list
+     * will always force a media file update.
+     *
      * Devices that currently pass:
      * - Fairphone 3
      */
@@ -127,7 +128,7 @@ class EntitiesBenchmarkTest {
             .clickGetBlankForm()
             .benchmark(
                 "Redownloading a form with 1k media files and entity list when there are no updates",
-                15,
+                5,
                 benchmarker
             ) {
                 it
@@ -138,9 +139,3 @@ class EntitiesBenchmarkTest {
         benchmarker.assertResults()
     }
 }
-
-private fun clearAndroidCache() {
-    val application = ApplicationProvider.getApplicationContext<Application>()
-    application.cacheDir.deleteRecursively()
-    application.cacheDir.mkdir()
-}

collect_app/src/main/java/org/odk/collect/android/activities/FormDownloadListActivity.java

@@ -42,7 +42,6 @@
 import org.odk.collect.android.formmanagement.FormSourceExceptionMapper;
 import org.odk.collect.android.formmanagement.FormsDataService;
 import org.odk.collect.android.formmanagement.ServerFormDetails;
-import org.odk.collect.android.formmanagement.ServerFormsDetailsFetcher;
 import org.odk.collect.android.formmanagement.download.FormDownloadException;
 import org.odk.collect.android.fragments.dialogs.FormsDownloadResultDialog;
 import org.odk.collect.android.injection.DaggerUtils;
@@ -121,9 +120,6 @@ public class FormDownloadListActivity extends FormListActivity implements FormLi
     @Inject
     WebCredentialsUtils webCredentialsUtils;
 
-    @Inject
-    ServerFormsDetailsFetcher serverFormsDetailsFetcher;
-
     @Inject
     NetworkStateProvider connectivityProvider;
 
@@ -294,12 +290,12 @@ private void downloadFormList() {
 
             if (viewModel.isDownloadOnlyMode()) {
                 // Handle external app download case with different server
-                downloadFormListTask = new DownloadFormListTask(serverFormsDetailsFetcher);
+                downloadFormListTask = new DownloadFormListTask();
                 downloadFormListTask.setAlternateCredentials(webCredentialsUtils, viewModel.getUrl(), viewModel.getUsername(), viewModel.getPassword());
                 downloadFormListTask.setDownloaderListener(this);
                 downloadFormListTask.execute();
             } else {
-                downloadFormListTask = new DownloadFormListTask(serverFormsDetailsFetcher);
+                downloadFormListTask = new DownloadFormListTask();
                 downloadFormListTask.setDownloaderListener(this);
                 downloadFormListTask.execute();
             }
@@ -464,7 +460,7 @@ public boolean isLocalFormSuperseded(String formId) {
         }
 
         ServerFormDetails form = viewModel.getFormDetailsByFormId().get(formId);
-        return form.isNotOnDevice() || form.isUpdated();
+        return form.getType() != ServerFormDetails.Type.OnDevice;
     }
 
     /**

collect_app/src/main/java/org/odk/collect/android/formmanagement/FormsDataService.kt

@@ -19,7 +19,6 @@ import org.odk.collect.projects.ProjectDependencyFactory
 import org.odk.collect.settings.keys.ProjectKeys
 import java.io.File
 import java.util.function.Supplier
-import java.util.stream.Collectors
 
 class FormsDataService(
     appState: AppState,
@@ -111,15 +110,14 @@ class FormsDataService(
             if (acquiredLock) {
                 syncWithStorage(projectId)
 
-                val serverFormsDetailsFetcher = serverFormsDetailsFetcher(projectDependencies)
                 val formDownloader = formDownloader(projectDependencies, clock)
 
                 try {
-                    val serverForms: List<ServerFormDetails> =
-                        serverFormsDetailsFetcher.fetchFormDetails()
-                    val updatedForms =
-                        serverForms.stream().filter { obj: ServerFormDetails -> obj.isUpdated }
-                            .collect(Collectors.toList())
+                    val serverForms = ServerFormUseCases.fetchFormDetails(
+                        projectDependencies.formsRepository,
+                        projectDependencies.formSource
+                    )
+                    val updatedForms = serverForms.filter { it.isUpdated() }
                     if (updatedForms.isNotEmpty()) {
                         if (projectDependencies.generalSettings.getBoolean(ProjectKeys.KEY_AUTOMATIC_UPDATE)) {
                             val results = ServerFormUseCases.downloadForms(
@@ -157,14 +155,14 @@ class FormsDataService(
                 startSync(projectId)
                 syncWithStorage(projectId)
 
-                val serverFormsDetailsFetcher = serverFormsDetailsFetcher(projectDependencies)
                 val formDownloader = formDownloader(projectDependencies, clock)
 
                 val serverFormsSynchronizer = ServerFormsSynchronizer(
-                    serverFormsDetailsFetcher,
+                    { repository, source -> ServerFormUseCases.fetchFormDetails(repository, source) },
                     projectDependencies.formsRepository,
                     projectDependencies.instancesRepository,
-                    formDownloader
+                    formDownloader,
+                    projectDependencies.formSource
                 )
 
                 val exception = try {
@@ -243,12 +241,3 @@ private fun formDownloader(
         projectDependencyModule.entitySource
     )
 }
-
-private fun serverFormsDetailsFetcher(
-    projectDependencyModule: ProjectDependencyModule
-): ServerFormsDetailsFetcher {
-    return ServerFormsDetailsFetcher(
-        projectDependencyModule.formsRepository,
-        projectDependencyModule.formSource
-    )
-}

collect_app/src/main/java/org/odk/collect/android/formmanagement/ServerFormDetails.kt

@@ -22,12 +22,68 @@ data class ServerFormDetails @JvmOverloads constructor(
     val formId: String?,
     val formVersion: String?,
     val hash: String?,
-    val isNotOnDevice: Boolean,
-    val isUpdated: Boolean,
-    val manifest: ManifestFile?
+    val manifest: ManifestFile?,
+    val type: Type
 ) : Serializable {
 
+    @Deprecated(message = "Use primary constructor instead")
+    constructor(
+        formName: String?,
+        downloadUrl: String?,
+        formId: String?,
+        formVersion: String?,
+        hash: String?,
+        isNotOnDevice: Boolean,
+        isUpdated: Boolean,
+        manifest: ManifestFile?,
+    ) : this(
+        formName,
+        downloadUrl,
+        formId,
+        formVersion,
+        hash,
+        manifest,
+        if (isNotOnDevice) {
+            Type.New
+        } else if (isUpdated) {
+            Type.UpdatedVersion
+        } else {
+            Type.OnDevice
+        }
+    )
+
+    fun isUpdated(): Boolean {
+        return type != Type.OnDevice && type != Type.New
+    }
+
     companion object {
-        private const val serialVersionUID = 3L
+        private const val serialVersionUID = 4L
+    }
+
+    enum class Type {
+        /**
+         * The form is on the device already
+         */
+        OnDevice,
+
+        /**
+         * The form is not on the device
+         */
+        New,
+
+        /**
+         * The form is on the device, but this is a new version with a new form version and hash
+         */
+        UpdatedVersion,
+
+        /**
+         * The form is on the device, but this is a new version with a new hash
+         */
+        UpdatedHash,
+
+        /**
+         * This version of the form is on the device, but new/updated media files are available
+         */
+        UpdatedMedia
     }
 }

collect_app/src/main/java/org/odk/collect/android/formmanagement/ServerFormUseCases.kt

@@ -6,6 +6,7 @@ import org.odk.collect.android.formmanagement.download.FormDownloadException
 import org.odk.collect.android.formmanagement.download.FormDownloader
 import org.odk.collect.android.instancemanagement.autosend.getLastUpdated
 import org.odk.collect.android.utilities.FileUtils
+import org.odk.collect.android.utilities.FormUtils
 import org.odk.collect.async.OngoingWorkListener
 import org.odk.collect.entities.LocalEntityUseCases
 import org.odk.collect.entities.server.EntitySource
@@ -14,13 +15,75 @@ import org.odk.collect.forms.Form
 import org.odk.collect.forms.FormSource
 import org.odk.collect.forms.FormSourceException
 import org.odk.collect.forms.FormsRepository
+import org.odk.collect.forms.ManifestFile
 import org.odk.collect.forms.MediaFile
 import org.odk.collect.shared.strings.Md5.getMd5Hash
+import timber.log.Timber
 import java.io.File
 import java.io.IOException
 
 object ServerFormUseCases {
 
+    @JvmStatic
+    @Throws(FormSourceException::class)
+    fun fetchFormDetails(
+        formsRepository: FormsRepository,
+        formSource: FormSource
+    ): List<ServerFormDetails> {
+        val formList = formSource.fetchFormList()
+        return formList.map { listItem ->
+            val manifestFile = listItem.manifestURL?.let {
+                getManifestFile(formSource, it)
+            }
+
+            val forms = formsRepository.getAllNotDeletedByFormId(listItem.formID)
+            val thisFormAlreadyDownloaded = forms.isNotEmpty()
+
+            val formHash = listItem.hash
+            val existingForm = if (formHash != null) {
+                formsRepository.getOneByMd5Hash(formHash)
+            } else {
+                null
+            }
+
+            val areNewerMediaFilesAvailable = if (existingForm != null && manifestFile != null) {
+                areNewerMediaFilesAvailable(existingForm, manifestFile.mediaFiles)
+            } else {
+                false
+            }
+
+            val type = if (existingForm != null) {
+                if (existingForm.isDeleted) {
+                    ServerFormDetails.Type.New
+                } else if (areNewerMediaFilesAvailable) {
+                    ServerFormDetails.Type.UpdatedMedia
+                } else {
+                    ServerFormDetails.Type.OnDevice
+                }
+            } else if (thisFormAlreadyDownloaded) {
+                if (listItem.hash == null) {
+                    ServerFormDetails.Type.OnDevice
+                } else if (forms.any { it.version == listItem.version }) {
+                    ServerFormDetails.Type.UpdatedHash
+                } else {
+                    ServerFormDetails.Type.UpdatedVersion
+                }
+            } else {
+                ServerFormDetails.Type.New
+            }
+
+            ServerFormDetails(
+                listItem.name,
+                listItem.downloadURL,
+                listItem.formID,
+                listItem.version,
+                listItem.hash,
+                manifestFile,
+                type
+            )
+        }
+    }
+
     fun downloadForms(
         forms: List<ServerFormDetails>,
         formDownloader: FormDownloader,
@@ -156,7 +219,9 @@ object ServerFormUseCases {
                     val existingFileHash = existingFile.getMd5Hash()
 
                     if (existingFileHash.contentEquals(mediaFile.hash)) {
-                        copyFileToDirectory(existingFile, tempMediaDir)
+                        if (formToDownload.type != ServerFormDetails.Type.UpdatedMedia) {
+                            copyFileToDirectory(existingFile, tempMediaDir)
+                        }
                     } else {
                         downloadMediaFile(
                             formSource,
@@ -233,6 +298,32 @@ object ServerFormUseCases {
             null
         }
     }
+
+    private fun getManifestFile(formSource: FormSource, manifestUrl: String): ManifestFile? {
+        return try {
+            formSource.fetchManifest(manifestUrl)
+        } catch (formSourceException: FormSourceException) {
+            Timber.w(formSourceException)
+            null
+        }
+    }
+
+    private fun areNewerMediaFilesAvailable(
+        existingForm: Form,
+        newMediaFiles: List<MediaFile>
+    ): Boolean {
+        if (newMediaFiles.isEmpty()) {
+            return false
+        }
+
+        val localMediaHashes = FormUtils.getMediaFiles(existingForm)
+            .map { it.getMd5Hash() }
+            .toSet()
+
+        return newMediaFiles.any {
+            !it.filename.endsWith(".zip") && it.hash !in localMediaHashes
+        }
+    }
 }
 
 class EntityListUpdateException(cause: Throwable) : Exception(cause)