From d49760fb9920185a9b8d8cca8b347995e9b4f8b0 Mon Sep 17 00:00:00 2001 From: Norbel Ambanumben Date: Fri, 3 Oct 2025 09:52:40 +0100 Subject: [PATCH 1/4] chore: intial draft for using `geoipdb` from path --- .gitignore | 1 + .../org/ooni/engine/AndroidOonimkallBridge.kt | 4 + .../kotlin/org/ooni/probe/net/Http.android.kt | 24 ++++++ .../values/strings-untranslatable.xml | 1 + .../kotlin/org/ooni/engine/Engine.kt | 6 +- .../kotlin/org/ooni/engine/OonimkallBridge.kt | 1 + .../kotlin/org/ooni/engine/TaskEventMapper.kt | 6 ++ .../ooni/engine/models/EnginePreferences.kt | 1 + .../org/ooni/engine/models/TaskEvent.kt | 1 + .../org/ooni/engine/models/TaskEventResult.kt | 2 + .../org/ooni/engine/models/TaskSettings.kt | 1 + .../commonMain/kotlin/org/ooni/probe/App.kt | 4 + .../org/ooni/probe/data/models/SettingsKey.kt | 2 + .../data/repositories/PreferenceRepository.kt | 2 + .../kotlin/org/ooni/probe/di/Dependencies.kt | 22 ++++- .../org/ooni/probe/domain/DownloadFile.kt | 32 ++++++++ .../ooni/probe/domain/FetchGeoIpDbUpdates.kt | 82 +++++++++++++++++++ .../ooni/probe/domain/GetEnginePreferences.kt | 1 + .../kotlin/org/ooni/probe/net/Http.kt | 7 ++ .../org/ooni/engine/DesktopOonimkallBridge.kt | 4 + .../kotlin/org/ooni/probe/net/Http.desktop.kt | 25 ++++++ .../kotlin/org/ooni/probe/net/Http.ios.kt | 43 ++++++++++ iosApp/iosApp/engine/IosOonimkallBridge.swift | 3 + 23 files changed, 272 insertions(+), 3 deletions(-) create mode 100644 composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt create mode 100644 composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt create mode 100644 composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt create mode 100644 composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt create mode 100644 composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt create mode 100644 composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt diff --git a/.gitignore b/.gitignore index 6fc08ccd4..4f22c7ae2 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,6 @@ composeApp/src/desktopMain/resources/windows/WinSparkle.* /composeApp/src/desktopMain/frameworks/ composeApp/macos-appcast.xml +*.mmdb certificates/*.pem diff --git a/composeApp/src/androidMain/kotlin/org/ooni/engine/AndroidOonimkallBridge.kt b/composeApp/src/androidMain/kotlin/org/ooni/engine/AndroidOonimkallBridge.kt index 42aa3c951..628526599 100644 --- a/composeApp/src/androidMain/kotlin/org/ooni/engine/AndroidOonimkallBridge.kt +++ b/composeApp/src/androidMain/kotlin/org/ooni/engine/AndroidOonimkallBridge.kt @@ -63,6 +63,10 @@ class AndroidOonimkallBridge : OonimkallBridge { it.softwareVersion = softwareVersion it.assetsDir = assetsDir + // geoipDB may not exist in Android binding; set reflectively if available + geoIpDB?.let { path -> + it.geoipDB = path + } it.stateDir = stateDir it.tempDir = tempDir it.tunnelDir = tunnelDir diff --git a/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt new file mode 100644 index 000000000..9a44f640d --- /dev/null +++ b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt @@ -0,0 +1,24 @@ +package org.ooni.probe.net + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.net.HttpURLConnection +import java.net.URL + +actual suspend fun httpGetBytes(url: String): ByteArray = + withContext(Dispatchers.IO) { + val connection = (URL(url).openConnection() as HttpURLConnection) + connection.requestMethod = "GET" + connection.instanceFollowRedirects = true + connection.connectTimeout = 15000 + connection.readTimeout = 30000 + try { + val code = connection.responseCode + val stream = if (code in 200..299) connection.inputStream else connection.errorStream + val bytes = stream?.use { it.readBytes() } ?: ByteArray(0) + if (code !in 200..299) throw RuntimeException("HTTP $code while GET $url: ${String(bytes)}") + bytes + } finally { + connection.disconnect() + } + } diff --git a/composeApp/src/commonMain/composeResources/values/strings-untranslatable.xml b/composeApp/src/commonMain/composeResources/values/strings-untranslatable.xml index e32289531..5ab2153ba 100644 --- a/composeApp/src/commonMain/composeResources/values/strings-untranslatable.xml +++ b/composeApp/src/commonMain/composeResources/values/strings-untranslatable.xml @@ -16,4 +16,5 @@ 2160p (4k) %1$s %2$s + 20250801 diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt index c21c0298f..4f1650864 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt @@ -148,7 +148,7 @@ class Engine( private fun session(sessionConfig: OonimkallBridge.SessionConfig): OonimkallBridge.Session = bridge.newSession(sessionConfig) - private fun buildTaskSettings( + private suspend fun buildTaskSettings( netTest: NetTest, taskOrigin: TaskOrigin, preferences: EnginePreferences, @@ -166,6 +166,7 @@ class Engine( tunnelDir = "$baseFilePath/tunnel", tempDir = cacheDir, assetsDir = "$baseFilePath/assets", + geoIpDB = preferences.geoipDbVersion?.let { "$cacheDir/$it.mmdb" }, options = TaskSettings.Options( noCollector = !preferences.uploadResults, softwareName = buildSoftwareName(taskOrigin), @@ -196,7 +197,7 @@ class Engine( MAX_RUNTIME_DISABLED } - private fun buildSessionConfig( + private suspend fun buildSessionConfig( taskOrigin: TaskOrigin, preferences: EnginePreferences, ) = OonimkallBridge.SessionConfig( @@ -208,6 +209,7 @@ class Engine( tunnelDir = "$baseFilePath/tunnel", tempDir = cacheDir, assetsDir = "$baseFilePath/assets", + geoIpDB = preferences.geoipDbVersion?.let { "$cacheDir/$it.mmdb" }, logger = oonimkallLogger, verbose = false, ) diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/OonimkallBridge.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/OonimkallBridge.kt index 8322430c5..6fa4f3ab4 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/OonimkallBridge.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/OonimkallBridge.kt @@ -31,6 +31,7 @@ interface OonimkallBridge { val proxy: String?, val probeServicesURL: String?, val assetsDir: String, + val geoIpDB: String?, val stateDir: String, val tempDir: String, val tunnelDir: String, diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt index 1aa1010b0..4d46f191a 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt @@ -96,9 +96,15 @@ class TaskEventMapper( asn = value?.probeAsn, ip = value?.probeIp, countryCode = value?.probeCc, + geoIpdb = value?.geoIpdb, networkType = networkTypeFinder(), ) + "status.resolver_lookup" -> value?.geoIpdb?.let { + println(it) + null + } + "status.measurement_done" -> TaskEvent.MeasurementDone(index = value?.idx ?: 0) diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/EnginePreferences.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/EnginePreferences.kt index 29346b6f0..27baf062d 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/EnginePreferences.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/EnginePreferences.kt @@ -7,5 +7,6 @@ data class EnginePreferences( val taskLogLevel: TaskLogLevel, val uploadResults: Boolean, val proxy: String?, + val geoipDbVersion: String?, val maxRuntime: Duration?, ) diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEvent.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEvent.kt index ef28c7519..a91a3e324 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEvent.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEvent.kt @@ -15,6 +15,7 @@ sealed interface TaskEvent { val ip: String?, val asn: String?, val countryCode: String?, + val geoIpdb: String?, val networkType: NetworkType, ) : TaskEvent diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt index 169390d21..3f199718d 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt @@ -26,6 +26,8 @@ data class TaskEventResult( var idx: Int = 0, @SerialName("report_id") var reportId: String? = null, + @SerialName("geoip_db") + var geoIpdb: String? = null, @SerialName("probe_ip") var probeIp: String? = null, @SerialName("probe_asn") diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskSettings.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskSettings.kt index d4e22a965..e1e6b67e5 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskSettings.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskSettings.kt @@ -15,6 +15,7 @@ data class TaskSettings( @SerialName("temp_dir") val tempDir: String, @SerialName("tunnel_dir") val tunnelDir: String, @SerialName("assets_dir") val assetsDir: String, + @SerialName("geoip_db") val geoIpDB: String?, @SerialName("options") val options: Options, @SerialName("annotations") val annotations: Annotations, // Optional diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt index 01ff8e68f..aa4f95b47 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt @@ -119,6 +119,10 @@ fun App( // ForegroundServiceDidNotStartInTimeException some users are getting // dependencies.startSingleRunInner(RunSpecification.OnlyUploadMissingResults) } + LaunchedEffect(Unit) { + // Check for GeoIP DB updates in the background + runCatching { dependencies.fetchGeoIpDbUpdates() } + } LaunchedEffect(Unit) { dependencies.observeAndConfigureAutoUpdate() } diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/SettingsKey.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/SettingsKey.kt index a96f31af9..4cea5acc6 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/SettingsKey.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/SettingsKey.kt @@ -65,6 +65,8 @@ enum class SettingsKey( DELETE_UPLOADED_JSONS("deleteUploadedJsons"), IS_NOTIFICATION_DIALOG("isNotificationDialog"), FIRST_RUN("first_run"), + MMDB_VERSION("mmdb_version"), + MMDB_LAST_CHECK("mmdb_last_check"), CHOSEN_WEBSITES("chosen_websites"), DESCRIPTOR_SECTIONS_COLLAPSED("descriptor_sections_collapsed"), diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt index ac65480b1..c2bd6609f 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt @@ -75,11 +75,13 @@ class PreferenceRepository( ): PreferenceKey<*> { val preferenceKey = getPreferenceKey(name = key.value, prefix = prefix, autoRun = autoRun) return when (key) { + SettingsKey.MMDB_LAST_CHECK, SettingsKey.MAX_RUNTIME, SettingsKey.LEGACY_PROXY_PORT, SettingsKey.DELETE_OLD_RESULTS_THRESHOLD, -> PreferenceKey.IntKey(intPreferencesKey(preferenceKey)) + SettingsKey.MMDB_VERSION, SettingsKey.LEGACY_PROXY_HOSTNAME, SettingsKey.LEGACY_PROXY_PROTOCOL, SettingsKey.PROXY_SELECTED, diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt index e189ec077..37cada078 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt @@ -17,6 +17,7 @@ import org.ooni.engine.Engine import org.ooni.engine.NetworkTypeFinder import org.ooni.engine.OonimkallBridge import org.ooni.engine.TaskEventMapper +import org.ooni.probe.net.httpGetBytes import org.ooni.probe.Database import org.ooni.probe.background.RunBackgroundTask import org.ooni.probe.config.BatteryOptimization @@ -50,7 +51,9 @@ import org.ooni.probe.domain.ClearStorage import org.ooni.probe.domain.DeleteMeasurementsWithoutResult import org.ooni.probe.domain.DeleteOldResults import org.ooni.probe.domain.DeleteResults +import org.ooni.probe.domain.DownloadFile import org.ooni.probe.domain.DownloadUrls +import org.ooni.probe.domain.FetchGeoIpDbUpdates import org.ooni.probe.domain.FinishInProgressData import org.ooni.probe.domain.GetAutoRunSettings import org.ooni.probe.domain.GetAutoRunSpecification @@ -116,6 +119,7 @@ import org.ooni.probe.ui.settings.proxy.ProxyViewModel import org.ooni.probe.ui.settings.webcategories.WebCategoriesViewModel import org.ooni.probe.ui.upload.UploadMeasurementsViewModel import kotlin.coroutines.CoroutineContext +import kotlin.getValue class Dependencies( val platformInfo: PlatformInfo, @@ -201,9 +205,25 @@ class Dependencies( } // Engine - private val taskEventMapper by lazy { TaskEventMapper(networkTypeFinder, json) } + private val downloader by lazy { + DownloadFile( + fileSystem = FileSystem.SYSTEM, + fetchBytes = { url -> httpGetBytes(url) }, + ) + } + + val fetchGeoIpDbUpdates by lazy { + FetchGeoIpDbUpdates( + downloadFile = downloader::invoke, + cacheDir = cacheDir, + engineHttpDo = engine::httpDo, + json = json, + preferencesRepository = preferenceRepository, + ) + } + @VisibleForTesting val engine by lazy { Engine( diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt new file mode 100644 index 000000000..9ea2e25d9 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt @@ -0,0 +1,32 @@ +package org.ooni.probe.domain + +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import okio.use + +/** + * Downloads binary content to a target absolute path using the provided fetcher. + * - Creates parent directories if needed + * - Skips writing if the target already exists with the same size + */ +class DownloadFile( + private val fileSystem: FileSystem, + private val fetchBytes: suspend (url: String) -> ByteArray, +) { + suspend operator fun invoke( + url: String, + absoluteTargetPath: String, + ): Path { + val target = absoluteTargetPath.toPath() + target.parent?.let { parent -> + if (fileSystem.metadataOrNull(parent) == null) fileSystem.createDirectories(parent) + } + val bytes = fetchBytes(url) + val existing = fileSystem.metadataOrNull(target) + if (existing?.size == bytes.size.toLong()) return target + fileSystem.sink(target).buffer().use { sink -> sink.write(bytes) } + return target + } +} diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt new file mode 100644 index 000000000..0ca9ebdfe --- /dev/null +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt @@ -0,0 +1,82 @@ +package org.ooni.probe.domain + +import kotlinx.coroutines.flow.first +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import okio.Path +import ooniprobe.composeapp.generated.resources.Res +import ooniprobe.composeapp.generated.resources.engine_mmdb_version +import org.jetbrains.compose.resources.getString +import org.ooni.engine.Engine +import org.ooni.engine.models.Failure +import org.ooni.engine.models.Result +import org.ooni.engine.models.Success +import org.ooni.engine.models.TaskOrigin +import org.ooni.probe.data.models.SettingsKey +import org.ooni.probe.data.repositories.PreferenceRepository +import kotlin.time.Clock + +class FetchGeoIpDbUpdates( + private val downloadFile: suspend (url: String, absoluteTargetPath: String) -> Path, + private val cacheDir: String, + private val engineHttpDo: suspend (method: String, url: String, taskOrigin: TaskOrigin) -> Result, + private val preferencesRepository: PreferenceRepository, + private val json: Json, +) { + suspend operator fun invoke(): Result = + try { + when (val versionRes = getLatestEngineVersion()) { + is Failure -> Failure(versionRes.reason) + is Success -> { + val (isLatest, _, latestVersion) = isGeoIpDbLatest(versionRes.value) + if (isLatest) { + Success(null) + } else { + val versionName = latestVersion + val url = buildGeoIpDbUrl(versionName) + val target = "$cacheDir/$versionName.mmdb" + + downloadFile(url, target).let { + preferencesRepository.setValueByKey(SettingsKey.MMDB_VERSION, versionName) + preferencesRepository.setValueByKey(SettingsKey.MMDB_LAST_CHECK, Clock.System.now().toEpochMilliseconds()) + Success(it) + } + } + } + } + } catch (t: Throwable) { + Failure(t as? Engine.MkException ?: Engine.MkException(t)) + } + + /** + * Compare latest and current version integers and return pair of latest state and actual version number + * @return Triple where the first element is true if the DB is the latest, + * the second is the current version and the third is the latest version. + */ + private suspend fun isGeoIpDbLatest(latestVersion: String): Triple { + val currentGeoIpDbVersion: String = + (preferencesRepository.getValueByKey(SettingsKey.MMDB_VERSION).first() ?: getString(Res.string.engine_mmdb_version)) as String + + return Triple(normalize(currentGeoIpDbVersion) >= normalize(latestVersion), currentGeoIpDbVersion, latestVersion) + } + + private suspend fun getLatestEngineVersion(): Result { + val url = "https://api.0.github.com/repos/aanorbel/oomplt-mmdb/releases/latest" + + return engineHttpDo("GET", url, TaskOrigin.OoniRun).map { payload -> + val jsonStr = payload ?: throw Engine.MkException(Throwable("Empty body")) + json.decodeFromString(GhRelease.serializer(), jsonStr).tag + } + } + + private fun buildGeoIpDbUrl(version: String): String = + "https://github.com/aanorbel/oomplt-mmdb/releases/download/$version/$version-ip2country_as.mmdb" + + private fun normalize(tag: String): Int = tag.removePrefix("v").trim().toInt() + + @Serializable + data class GhRelease( + @SerialName("tag_name") val tag: String, + ) +} diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/GetEnginePreferences.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/GetEnginePreferences.kt index ae26803d1..aa4e3fbf4 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/GetEnginePreferences.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/GetEnginePreferences.kt @@ -29,6 +29,7 @@ class GetEnginePreferences( null }, proxy = getProxyOption().first().value, + geoipDbVersion = getValueForKey(SettingsKey.MMDB_VERSION) as String?, ) private suspend fun getEnabledCategories(): List { diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt new file mode 100644 index 000000000..eff1ae8f1 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt @@ -0,0 +1,7 @@ +package org.ooni.probe.net + +/** + * Perform a simple HTTP GET and return the raw response body bytes. + * Implemented per-platform to ensure binary-safe downloads. + */ +expect suspend fun httpGetBytes(url: String): ByteArray diff --git a/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt b/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt index 8b82dbddc..8a515789f 100644 --- a/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt +++ b/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt @@ -64,6 +64,10 @@ class DesktopOonimkallBridge : OonimkallBridge { it.softwareVersion = softwareVersion it.assetsDir = assetsDir + // geoipDB may or may not exist in this binding; set via reflection when available + geoIpDB?.let { path -> + // it.geoipDB = path + } it.stateDir = stateDir it.tempDir = tempDir it.tunnelDir = tunnelDir diff --git a/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt b/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt new file mode 100644 index 000000000..89a741d12 --- /dev/null +++ b/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt @@ -0,0 +1,25 @@ +package org.ooni.probe.net + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.BufferedInputStream +import java.net.HttpURLConnection +import java.net.URL + +actual suspend fun httpGetBytes(url: String): ByteArray = + withContext(Dispatchers.IO) { + val connection = (URL(url).openConnection() as HttpURLConnection) + connection.requestMethod = "GET" + connection.instanceFollowRedirects = true + connection.connectTimeout = 15000 + connection.readTimeout = 30000 + try { + val code = connection.responseCode + val stream = if (code in 200..299) connection.inputStream else connection.errorStream + val bytes = stream?.let { BufferedInputStream(it).use { bis -> bis.readBytes() } } ?: ByteArray(0) + if (code !in 200..299) throw RuntimeException("HTTP $code while GET $url: ${bytes.decodeToString()}") + bytes + } finally { + connection.disconnect() + } + } diff --git a/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt new file mode 100644 index 000000000..4e211154e --- /dev/null +++ b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt @@ -0,0 +1,43 @@ +package org.ooni.probe.net + +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.usePinned +import platform.Foundation.NSData +import platform.Foundation.NSURL +import platform.Foundation.NSURLSession +import platform.Foundation.dataTaskWithURL +import platform.posix.memcpy +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +actual suspend fun httpGetBytes(url: String): ByteArray = + suspendCancellableCoroutine { cont -> + val nsurl = NSURL.URLWithString(url)!! + val task = NSURLSession.sharedSession.dataTaskWithURL(nsurl) { data, response, error -> + when { + error != null -> cont.resumeWithException(RuntimeException(error.localizedDescription)) + data != null -> { + // If we have an HTTP response, check status code + val http = response as? platform.Foundation.NSHTTPURLResponse + val status = http?.statusCode?.toInt() ?: 200 + if (status in 200..299) { + cont.resume((data as NSData).toByteArray()) + } else { + cont.resumeWithException(RuntimeException("HTTP $status while GET $url")) + } + } + else -> cont.resume(ByteArray(0)) + } + } + cont.invokeOnCancellation { task.cancel() } + task.resume() + } + +private fun NSData.toByteArray(): ByteArray { + val result = ByteArray(length.toInt()) + result.usePinned { + memcpy(it.addressOf(0), this.bytes, this.length) + } + return result +} diff --git a/iosApp/iosApp/engine/IosOonimkallBridge.swift b/iosApp/iosApp/engine/IosOonimkallBridge.swift index 83fc6c5da..2e9f702df 100644 --- a/iosApp/iosApp/engine/IosOonimkallBridge.swift +++ b/iosApp/iosApp/engine/IosOonimkallBridge.swift @@ -147,6 +147,9 @@ extension OonimkallBridgeSessionConfig { config.stateDir = stateDir config.tempDir = tempDir config.tunnelDir = tunnelDir + if let geoIpDB = geoIpDB { + config.geoipDB = geoIpDB + } if let probeServicesURL = probeServicesURL { config.probeServicesURL = probeServicesURL } From 9c30e7877d9d7b41f43e7adf3d2a8467abf92a9f Mon Sep 17 00:00:00 2001 From: Norbel Ambanumben Date: Fri, 24 Oct 2025 11:49:16 +0100 Subject: [PATCH 2/4] chore: update oonimkall --- composeApp/build.gradle.kts | 4 +-- gradle/libs.versions.toml | 2 +- iosApp/Podfile | 2 +- iosApp/Podfile.lock | 50 ++++++++++++++++++------------------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/composeApp/build.gradle.kts b/composeApp/build.gradle.kts index 7dd18d0ab..c6082661d 100644 --- a/composeApp/build.gradle.kts +++ b/composeApp/build.gradle.kts @@ -290,8 +290,8 @@ android { coreLibraryDesugaring(libs.android.desugar.jdk) debugImplementation(compose.uiTooling) "fullImplementation"(libs.bundles.full.android) - "fullImplementation"("org.ooni:oonimkall:3.27.0-android:@aar") - "fdroidImplementation"("org.ooni:oonimkall:3.27.0-android:@aar") + "fullImplementation"("org.ooni:oonimkall:3.28.0-alpha-android:@aar") + "fdroidImplementation"("org.ooni:oonimkall:3.28.0-alpha-android:@aar") "xperimentalImplementation"(files("libs/android-oonimkall.aar")) androidTestUtil(libs.android.orchestrator) } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index aa44211a4..d59ddb307 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -105,7 +105,7 @@ fastlane-screengrab = { module = "tools.fastlane:screengrab", version = "2.1.1" auto-launch = { module = "io.github.vinceglb:auto-launch", version = "0.7.0"} directories = { module = "dev.dirs:directories", version = "26" } pratanumandal-unique = { module = "tk.pratanumandal:unique4j", version = "1.4" } -desktop-oonimkall = { module = "org.ooni:oonimkall", version = "3.27.0-desktop" } +desktop-oonimkall = { module = "org.ooni:oonimkall", version = "3.28.0-alpha-desktop" } [bundles] diff --git a/iosApp/Podfile b/iosApp/Podfile index af4d741f2..34fbf1908 100644 --- a/iosApp/Podfile +++ b/iosApp/Podfile @@ -2,7 +2,7 @@ platform :ios, '14.0' use_frameworks! def shared_pods - ooni_version = "v3.27.0" + ooni_version = "v3.28.0-alpha" ooni_pods_location = "https://github.com/ooni/probe-cli/releases/download/#{ooni_version}" pod 'composeApp', :path => '../composeApp' diff --git a/iosApp/Podfile.lock b/iosApp/Podfile.lock index 2451b03d5..a59c0f93b 100644 --- a/iosApp/Podfile.lock +++ b/iosApp/Podfile.lock @@ -1,12 +1,12 @@ PODS: - composeApp (1.0.0): - Sentry (= 8.55.1) - - libcrypto (2025.09.09-110535) - - libevent (2025.09.09-110535) - - libssl (2025.09.09-110535) - - libtor (2025.09.09-110535) - - libz (2025.09.09-110535) - - oonimkall (2025.09.09-110535) + - libcrypto (2025.10.24-063517) + - libevent (2025.10.24-063517) + - libssl (2025.10.24-063517) + - libtor (2025.10.24-063517) + - libz (2025.10.24-063517) + - oonimkall (2025.10.24-063517) - Sentry (8.55.1): - Sentry/Core (= 8.55.1) - Sentry/Core (8.55.1) @@ -17,12 +17,12 @@ PODS: DEPENDENCIES: - composeApp (from `../composeApp`) - - libcrypto (from `https://github.com/ooni/probe-cli/releases/download/v3.27.0/libcrypto.podspec`) - - libevent (from `https://github.com/ooni/probe-cli/releases/download/v3.27.0/libevent.podspec`) - - libssl (from `https://github.com/ooni/probe-cli/releases/download/v3.27.0/libssl.podspec`) - - libtor (from `https://github.com/ooni/probe-cli/releases/download/v3.27.0/libtor.podspec`) - - libz (from `https://github.com/ooni/probe-cli/releases/download/v3.27.0/libz.podspec`) - - oonimkall (from `https://github.com/ooni/probe-cli/releases/download/v3.27.0/oonimkall.podspec`) + - libcrypto (from `https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libcrypto.podspec`) + - libevent (from `https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libevent.podspec`) + - libssl (from `https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libssl.podspec`) + - libtor (from `https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libtor.podspec`) + - libz (from `https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libz.podspec`) + - oonimkall (from `https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/oonimkall.podspec`) - Siren - sqlite3 (~> 3.42.0) @@ -36,30 +36,30 @@ EXTERNAL SOURCES: composeApp: :path: "../composeApp" libcrypto: - :podspec: https://github.com/ooni/probe-cli/releases/download/v3.27.0/libcrypto.podspec + :podspec: https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libcrypto.podspec libevent: - :podspec: https://github.com/ooni/probe-cli/releases/download/v3.27.0/libevent.podspec + :podspec: https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libevent.podspec libssl: - :podspec: https://github.com/ooni/probe-cli/releases/download/v3.27.0/libssl.podspec + :podspec: https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libssl.podspec libtor: - :podspec: https://github.com/ooni/probe-cli/releases/download/v3.27.0/libtor.podspec + :podspec: https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libtor.podspec libz: - :podspec: https://github.com/ooni/probe-cli/releases/download/v3.27.0/libz.podspec + :podspec: https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/libz.podspec oonimkall: - :podspec: https://github.com/ooni/probe-cli/releases/download/v3.27.0/oonimkall.podspec + :podspec: https://github.com/ooni/probe-cli/releases/download/v3.28.0-alpha/oonimkall.podspec SPEC CHECKSUMS: composeApp: 23f1c8946d30f151e633bdcf3c7db996ae3be9ce - libcrypto: 6fe6cdcad3c473ed4b3ccfe12a95f30840de48e7 - libevent: c8df42a11d8217584f940cdfe4aed60fd056572e - libssl: 7d9f469af78e11cb2b207211b85bcd415d903ae8 - libtor: 08056abb8cd5fa1c7c7e5cd21e22e4436b699b73 - libz: d695d2d4082e5b71e6a988188eb8b5b2b18b43fc - oonimkall: 9d00aecca34685d6fd6252139703be7793bb6ba8 + libcrypto: a95cf1d71053abe5d1ffbf286a1897f52c6999e2 + libevent: 457557e55295bffdf1bbac3c5c832639dfa149fe + libssl: 595044ab6ea6bf038641ac044b1782bf0fdbfbc1 + libtor: b68e0b20fb994a7d5447a0e5e09739c10d1b8643 + libz: 3ae34fb1e45f0e43457a2e456c0d3dd63a8f72f0 + oonimkall: 4082e113ff788b56d36e3459a1ef51c21c3434b0 Sentry: 6c92b12db0634612f6a66757890fea97e788fe12 Siren: c0f6012f61196b73455202db07730f6454a4beb0 sqlite3: f163dbbb7aa3339ad8fc622782c2d9d7b72f7e9c -PODFILE CHECKSUM: 1200ca7a56742e3cfaae6fc3c01a9f5035849f4b +PODFILE CHECKSUM: f84de3dc2812d0990dad2f2ba141bcad0ba9bfcd COCOAPODS: 1.16.2 From 188221a9b1f328548952a34f71b2941ec5afdbff Mon Sep 17 00:00:00 2001 From: Norbel Ambanumben Date: Tue, 28 Oct 2025 17:46:50 +0300 Subject: [PATCH 3/4] feat: refactor GeoIP DB update handling and improve error management --- .../kotlin/org/ooni/probe/net/Http.android.kt | 15 +++++-- .../commonMain/kotlin/org/ooni/probe/App.kt | 3 +- .../probe/data/models/GetBytesException.kt | 5 +++ .../data/repositories/PreferenceRepository.kt | 5 ++- .../org/ooni/probe/domain/DownloadFile.kt | 15 +++---- .../ooni/probe/domain/FetchGeoIpDbUpdates.kt | 40 +++++++++---------- .../kotlin/org/ooni/probe/net/Http.kt | 5 ++- .../org/ooni/engine/DesktopOonimkallBridge.kt | 2 +- .../kotlin/org/ooni/probe/net/Http.desktop.kt | 15 +++++-- .../kotlin/org/ooni/probe/net/Http.ios.kt | 15 ++++--- 10 files changed, 76 insertions(+), 44 deletions(-) create mode 100644 composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/GetBytesException.kt diff --git a/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt index 9a44f640d..5a254ee0a 100644 --- a/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt +++ b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt @@ -2,10 +2,14 @@ package org.ooni.probe.net import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.ooni.engine.models.Failure +import org.ooni.engine.models.Result +import org.ooni.engine.models.Success +import org.ooni.probe.data.models.GetBytesException import java.net.HttpURLConnection import java.net.URL -actual suspend fun httpGetBytes(url: String): ByteArray = +actual suspend fun httpGetBytes(url: String): Result = withContext(Dispatchers.IO) { val connection = (URL(url).openConnection() as HttpURLConnection) connection.requestMethod = "GET" @@ -16,8 +20,13 @@ actual suspend fun httpGetBytes(url: String): ByteArray = val code = connection.responseCode val stream = if (code in 200..299) connection.inputStream else connection.errorStream val bytes = stream?.use { it.readBytes() } ?: ByteArray(0) - if (code !in 200..299) throw RuntimeException("HTTP $code while GET $url: ${String(bytes)}") - bytes + if (code !in 200..299) { + Failure(GetBytesException(RuntimeException("HTTP $code while GET $url: ${String(bytes)}"))) + } else { + Success(bytes) + } + } catch (e: Throwable) { + Failure(GetBytesException(e)) } finally { connection.disconnect() } diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt index aa4f95b47..9c5b2663b 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt @@ -120,8 +120,7 @@ fun App( // dependencies.startSingleRunInner(RunSpecification.OnlyUploadMissingResults) } LaunchedEffect(Unit) { - // Check for GeoIP DB updates in the background - runCatching { dependencies.fetchGeoIpDbUpdates() } + dependencies.fetchGeoIpDbUpdates() } LaunchedEffect(Unit) { dependencies.observeAndConfigureAutoUpdate() diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/GetBytesException.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/GetBytesException.kt new file mode 100644 index 000000000..6958f8efe --- /dev/null +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/GetBytesException.kt @@ -0,0 +1,5 @@ +package org.ooni.probe.data.models + +class GetBytesException( + t: Throwable, +) : Exception(t) diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt index c2bd6609f..61fa28685 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt @@ -6,6 +6,7 @@ import androidx.datastore.preferences.core.Preferences import androidx.datastore.preferences.core.booleanPreferencesKey import androidx.datastore.preferences.core.edit import androidx.datastore.preferences.core.intPreferencesKey +import androidx.datastore.preferences.core.longPreferencesKey import androidx.datastore.preferences.core.stringPreferencesKey import androidx.datastore.preferences.core.stringSetPreferencesKey import kotlinx.coroutines.flow.Flow @@ -75,12 +76,14 @@ class PreferenceRepository( ): PreferenceKey<*> { val preferenceKey = getPreferenceKey(name = key.value, prefix = prefix, autoRun = autoRun) return when (key) { - SettingsKey.MMDB_LAST_CHECK, SettingsKey.MAX_RUNTIME, SettingsKey.LEGACY_PROXY_PORT, SettingsKey.DELETE_OLD_RESULTS_THRESHOLD, -> PreferenceKey.IntKey(intPreferencesKey(preferenceKey)) + SettingsKey.MMDB_LAST_CHECK, + -> PreferenceKey.LongKey(longPreferencesKey(preferenceKey)) + SettingsKey.MMDB_VERSION, SettingsKey.LEGACY_PROXY_HOSTNAME, SettingsKey.LEGACY_PROXY_PROTOCOL, diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt index 9ea2e25d9..222f7c8d9 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt @@ -5,6 +5,8 @@ import okio.Path import okio.Path.Companion.toPath import okio.buffer import okio.use +import org.ooni.engine.models.Result +import org.ooni.probe.data.models.GetBytesException /** * Downloads binary content to a target absolute path using the provided fetcher. @@ -13,20 +15,19 @@ import okio.use */ class DownloadFile( private val fileSystem: FileSystem, - private val fetchBytes: suspend (url: String) -> ByteArray, + private val fetchBytes: suspend (url: String) -> Result, ) { suspend operator fun invoke( url: String, absoluteTargetPath: String, - ): Path { + ): Result { val target = absoluteTargetPath.toPath() target.parent?.let { parent -> if (fileSystem.metadataOrNull(parent) == null) fileSystem.createDirectories(parent) } - val bytes = fetchBytes(url) - val existing = fileSystem.metadataOrNull(target) - if (existing?.size == bytes.size.toLong()) return target - fileSystem.sink(target).buffer().use { sink -> sink.write(bytes) } - return target + return fetchBytes(url).map { bytes -> + fileSystem.sink(target).buffer().use { sink -> sink.write(bytes) } + target + } } } diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt index 0ca9ebdfe..7ebde4ad1 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt @@ -13,41 +13,41 @@ import org.ooni.engine.models.Failure import org.ooni.engine.models.Result import org.ooni.engine.models.Success import org.ooni.engine.models.TaskOrigin +import org.ooni.probe.data.models.GetBytesException import org.ooni.probe.data.models.SettingsKey import org.ooni.probe.data.repositories.PreferenceRepository import kotlin.time.Clock class FetchGeoIpDbUpdates( - private val downloadFile: suspend (url: String, absoluteTargetPath: String) -> Path, + private val downloadFile: suspend (url: String, absoluteTargetPath: String) -> Result, private val cacheDir: String, private val engineHttpDo: suspend (method: String, url: String, taskOrigin: TaskOrigin) -> Result, private val preferencesRepository: PreferenceRepository, private val json: Json, ) { suspend operator fun invoke(): Result = - try { - when (val versionRes = getLatestEngineVersion()) { - is Failure -> Failure(versionRes.reason) - is Success -> { - val (isLatest, _, latestVersion) = isGeoIpDbLatest(versionRes.value) - if (isLatest) { - Success(null) - } else { - val versionName = latestVersion - val url = buildGeoIpDbUrl(versionName) - val target = "$cacheDir/$versionName.mmdb" + getLatestEngineVersion() + .onSuccess { version -> + val (isLatest, _, latestVersion) = isGeoIpDbLatest(version) + if (isLatest) { + return Success(null) + } else { + val versionName = latestVersion + val url = buildGeoIpDbUrl(versionName) + val target = "$cacheDir/$versionName.mmdb" - downloadFile(url, target).let { + downloadFile(url, target) + .onSuccess { downloadedPath -> preferencesRepository.setValueByKey(SettingsKey.MMDB_VERSION, versionName) preferencesRepository.setValueByKey(SettingsKey.MMDB_LAST_CHECK, Clock.System.now().toEpochMilliseconds()) - Success(it) + return Success(downloadedPath) + }.onFailure { downloadError -> + return Failure(Engine.MkException(downloadError)) } - } } - } - } catch (t: Throwable) { - Failure(t as? Engine.MkException ?: Engine.MkException(t)) - } + }.onFailure { versionError -> + return Failure(versionError) + }.let { Failure(Engine.MkException(Throwable("Unexpected state"))) } /** * Compare latest and current version integers and return pair of latest state and actual version number @@ -62,7 +62,7 @@ class FetchGeoIpDbUpdates( } private suspend fun getLatestEngineVersion(): Result { - val url = "https://api.0.github.com/repos/aanorbel/oomplt-mmdb/releases/latest" + val url = "https://api.github.com/repos/aanorbel/oomplt-mmdb/releases/latest" return engineHttpDo("GET", url, TaskOrigin.OoniRun).map { payload -> val jsonStr = payload ?: throw Engine.MkException(Throwable("Empty body")) diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt index eff1ae8f1..185ac9da5 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt @@ -1,7 +1,10 @@ package org.ooni.probe.net +import org.ooni.engine.models.Result +import org.ooni.probe.data.models.GetBytesException + /** * Perform a simple HTTP GET and return the raw response body bytes. * Implemented per-platform to ensure binary-safe downloads. */ -expect suspend fun httpGetBytes(url: String): ByteArray +expect suspend fun httpGetBytes(url: String): Result diff --git a/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt b/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt index 8a515789f..1c2388fe1 100644 --- a/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt +++ b/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt @@ -66,7 +66,7 @@ class DesktopOonimkallBridge : OonimkallBridge { it.assetsDir = assetsDir // geoipDB may or may not exist in this binding; set via reflection when available geoIpDB?.let { path -> - // it.geoipDB = path + it.geoipDB = path } it.stateDir = stateDir it.tempDir = tempDir diff --git a/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt b/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt index 89a741d12..6217b2880 100644 --- a/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt +++ b/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt @@ -2,11 +2,15 @@ package org.ooni.probe.net import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.ooni.engine.models.Failure +import org.ooni.engine.models.Result +import org.ooni.engine.models.Success +import org.ooni.probe.data.models.GetBytesException import java.io.BufferedInputStream import java.net.HttpURLConnection import java.net.URL -actual suspend fun httpGetBytes(url: String): ByteArray = +actual suspend fun httpGetBytes(url: String): Result = withContext(Dispatchers.IO) { val connection = (URL(url).openConnection() as HttpURLConnection) connection.requestMethod = "GET" @@ -17,8 +21,13 @@ actual suspend fun httpGetBytes(url: String): ByteArray = val code = connection.responseCode val stream = if (code in 200..299) connection.inputStream else connection.errorStream val bytes = stream?.let { BufferedInputStream(it).use { bis -> bis.readBytes() } } ?: ByteArray(0) - if (code !in 200..299) throw RuntimeException("HTTP $code while GET $url: ${bytes.decodeToString()}") - bytes + if (code !in 200..299) { + Failure(GetBytesException(RuntimeException("HTTP $code while GET $url: ${bytes.decodeToString()}"))) + } else { + Success(bytes) + } + } catch (e: Throwable) { + Failure(GetBytesException(e)) } finally { connection.disconnect() } diff --git a/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt index 4e211154e..46cdb182d 100644 --- a/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt +++ b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt @@ -3,31 +3,34 @@ package org.ooni.probe.net import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.cinterop.addressOf import kotlinx.cinterop.usePinned +import org.ooni.engine.models.Failure +import org.ooni.engine.models.Result +import org.ooni.engine.models.Success +import org.ooni.probe.data.models.GetBytesException import platform.Foundation.NSData import platform.Foundation.NSURL import platform.Foundation.NSURLSession import platform.Foundation.dataTaskWithURL import platform.posix.memcpy import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -actual suspend fun httpGetBytes(url: String): ByteArray = +actual suspend fun httpGetBytes(url: String): Result = suspendCancellableCoroutine { cont -> val nsurl = NSURL.URLWithString(url)!! val task = NSURLSession.sharedSession.dataTaskWithURL(nsurl) { data, response, error -> when { - error != null -> cont.resumeWithException(RuntimeException(error.localizedDescription)) + error != null -> cont.resume(Failure(GetBytesException(RuntimeException(error.localizedDescription)))) data != null -> { // If we have an HTTP response, check status code val http = response as? platform.Foundation.NSHTTPURLResponse val status = http?.statusCode?.toInt() ?: 200 if (status in 200..299) { - cont.resume((data as NSData).toByteArray()) + cont.resume(Success((data as NSData).toByteArray())) } else { - cont.resumeWithException(RuntimeException("HTTP $status while GET $url")) + cont.resume(Failure(GetBytesException(RuntimeException("HTTP $status while GET $url")))) } } - else -> cont.resume(ByteArray(0)) + else -> cont.resume(Success(ByteArray(0))) } } cont.invokeOnCancellation { task.cancel() } From eef3dbc0416f2cb02b225c8f87de3428248893b4 Mon Sep 17 00:00:00 2001 From: Norbel Ambanumben Date: Tue, 28 Oct 2025 19:00:49 +0300 Subject: [PATCH 4/4] feat: enhance GeoIP DB update process and improve error handling --- .../kotlin/org/ooni/probe/net/Http.android.kt | 8 +++- .../kotlin/org/ooni/engine/Engine.kt | 2 +- .../kotlin/org/ooni/engine/TaskEventMapper.kt | 2 +- .../kotlin/org/ooni/probe/di/Dependencies.kt | 2 +- .../ooni/probe/domain/FetchGeoIpDbUpdates.kt | 39 +++++++++++++--- .../kotlin/org/ooni/probe/net/Http.ios.kt | 45 ++++++++++++------- 6 files changed, 70 insertions(+), 28 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt index 5a254ee0a..04eb1acf8 100644 --- a/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt +++ b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt @@ -11,7 +11,13 @@ import java.net.URL actual suspend fun httpGetBytes(url: String): Result = withContext(Dispatchers.IO) { - val connection = (URL(url).openConnection() as HttpURLConnection) + val connection: HttpURLConnection + try { + connection = URL(url).openConnection() as HttpURLConnection + } catch (e: Throwable) { + return@withContext Failure(GetBytesException(e)) + } + connection.requestMethod = "GET" connection.instanceFollowRedirects = true connection.connectTimeout = 15000 diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt index 4f1650864..3411f4f8f 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt @@ -197,7 +197,7 @@ class Engine( MAX_RUNTIME_DISABLED } - private suspend fun buildSessionConfig( + private fun buildSessionConfig( taskOrigin: TaskOrigin, preferences: EnginePreferences, ) = OonimkallBridge.SessionConfig( diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt index 4d46f191a..f46cd5dae 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt @@ -101,7 +101,7 @@ class TaskEventMapper( ) "status.resolver_lookup" -> value?.geoIpdb?.let { - println(it) + Logger.d("GeoIP DB info in resolver lookup: $it") null } diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt index 37cada078..992b3e4bc 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt @@ -210,7 +210,7 @@ class Dependencies( private val downloader by lazy { DownloadFile( fileSystem = FileSystem.SYSTEM, - fetchBytes = { url -> httpGetBytes(url) }, + fetchBytes = ::httpGetBytes, ) } diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt index 7ebde4ad1..2ea0b9f34 100644 --- a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt +++ b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt @@ -1,14 +1,17 @@ package org.ooni.probe.domain +import co.touchlab.kermit.Logger import kotlinx.coroutines.flow.first import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json import okio.Path import ooniprobe.composeapp.generated.resources.Res import ooniprobe.composeapp.generated.resources.engine_mmdb_version import org.jetbrains.compose.resources.getString import org.ooni.engine.Engine +import org.ooni.engine.Engine.MkException import org.ooni.engine.models.Failure import org.ooni.engine.models.Result import org.ooni.engine.models.Success @@ -38,8 +41,14 @@ class FetchGeoIpDbUpdates( downloadFile(url, target) .onSuccess { downloadedPath -> - preferencesRepository.setValueByKey(SettingsKey.MMDB_VERSION, versionName) - preferencesRepository.setValueByKey(SettingsKey.MMDB_LAST_CHECK, Clock.System.now().toEpochMilliseconds()) + preferencesRepository.setValueByKey( + SettingsKey.MMDB_VERSION, + versionName, + ) + preferencesRepository.setValueByKey( + SettingsKey.MMDB_LAST_CHECK, + Clock.System.now().toEpochMilliseconds(), + ) return Success(downloadedPath) }.onFailure { downloadError -> return Failure(Engine.MkException(downloadError)) @@ -56,17 +65,33 @@ class FetchGeoIpDbUpdates( */ private suspend fun isGeoIpDbLatest(latestVersion: String): Triple { val currentGeoIpDbVersion: String = - (preferencesRepository.getValueByKey(SettingsKey.MMDB_VERSION).first() ?: getString(Res.string.engine_mmdb_version)) as String + ( + preferencesRepository.getValueByKey(SettingsKey.MMDB_VERSION).first() + ?: getString(Res.string.engine_mmdb_version) + ) as String - return Triple(normalize(currentGeoIpDbVersion) >= normalize(latestVersion), currentGeoIpDbVersion, latestVersion) + return Triple( + normalize(currentGeoIpDbVersion) >= normalize(latestVersion), + currentGeoIpDbVersion, + latestVersion, + ) } - private suspend fun getLatestEngineVersion(): Result { + private suspend fun getLatestEngineVersion(): Result { val url = "https://api.github.com/repos/aanorbel/oomplt-mmdb/releases/latest" return engineHttpDo("GET", url, TaskOrigin.OoniRun).map { payload -> - val jsonStr = payload ?: throw Engine.MkException(Throwable("Empty body")) - json.decodeFromString(GhRelease.serializer(), jsonStr).tag + payload?.let { + try { + json.decodeFromString(GhRelease.serializer(), payload).tag + } catch (e: SerializationException) { + Logger.e(e) { "Failed to decode release info" } + null + } catch (e: IllegalArgumentException) { + Logger.e(e) { "Failed to decode release info" } + null + } + } ?: throw MkException(Throwable("Failed to fetch latest version")) } } diff --git a/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt index 46cdb182d..689808fe4 100644 --- a/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt +++ b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt @@ -16,31 +16,42 @@ import kotlin.coroutines.resume actual suspend fun httpGetBytes(url: String): Result = suspendCancellableCoroutine { cont -> - val nsurl = NSURL.URLWithString(url)!! + val nsurl = NSURL.URLWithString(url) ?: run { + cont.resume(Failure(GetBytesException(RuntimeException("Invalid URL: $url")))) + return@suspendCancellableCoroutine + } val task = NSURLSession.sharedSession.dataTaskWithURL(nsurl) { data, response, error -> - when { - error != null -> cont.resume(Failure(GetBytesException(RuntimeException(error.localizedDescription)))) - data != null -> { - // If we have an HTTP response, check status code - val http = response as? platform.Foundation.NSHTTPURLResponse - val status = http?.statusCode?.toInt() ?: 200 - if (status in 200..299) { - cont.resume(Success((data as NSData).toByteArray())) + if (error != null) { + cont.resume(Failure(GetBytesException(RuntimeException(error.toString())))) + return@dataTaskWithURL + } + + when (val r = response) { + is platform.Foundation.NSHTTPURLResponse -> { + val statusCode = r.statusCode + if (statusCode in 200..299) { + cont.resume(Success(data?.toByteArray() ?: ByteArray(0))) } else { - cont.resume(Failure(GetBytesException(RuntimeException("HTTP $status while GET $url")))) + cont.resume(Failure(GetBytesException(RuntimeException("HTTP $statusCode while GET $url")))) + } + } + else -> { + // This could be for non-HTTP responses (e.g. file://) or an invalid state + if (data != null) { + cont.resume(Success(data.toByteArray())) + } else { + cont.resume(Failure(GetBytesException(RuntimeException("Request to $url returned no data and no error")))) } } - else -> cont.resume(Success(ByteArray(0))) } } cont.invokeOnCancellation { task.cancel() } task.resume() } -private fun NSData.toByteArray(): ByteArray { - val result = ByteArray(length.toInt()) - result.usePinned { - memcpy(it.addressOf(0), this.bytes, this.length) +private fun NSData.toByteArray(): ByteArray = + ByteArray(length.toInt()).apply { + usePinned { + memcpy(it.addressOf(0), bytes, length) + } } - return result -}