From 3890c4ffb9d3044fd6b4ffc8c17af7a3fe4b8382 Mon Sep 17 00:00:00 2001 From: Imran Remtulla Date: Fri, 24 Nov 2023 16:39:44 -0500 Subject: [PATCH] Support for fixed APK URL in HTML source (#1101) --- assets/translations/bs.json | 1 + assets/translations/cs.json | 1 + assets/translations/de.json | 1 + assets/translations/en.json | 1 + assets/translations/es.json | 1 + assets/translations/fa.json | 1 + assets/translations/fr.json | 1 + assets/translations/hu.json | 1 + assets/translations/it.json | 1 + assets/translations/ja.json | 1 + assets/translations/nl.json | 1 + assets/translations/pl.json | 1 + assets/translations/pt.json | 1 + assets/translations/ru.json | 1 + assets/translations/tr.json | 1 + assets/translations/vi.json | 1 + assets/translations/zh.json | 1 + lib/app_sources/html.dart | 13 ++- lib/providers/apps_provider.dart | 168 ++++++++++++++++++------------- pubspec.lock | 2 +- pubspec.yaml | 1 + 21 files changed, 124 insertions(+), 77 deletions(-) diff --git a/assets/translations/bs.json b/assets/translations/bs.json index d3bf65a..f76a4a1 100644 --- a/assets/translations/bs.json +++ b/assets/translations/bs.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Dovršite instalaciju aplikacije", "checkingForUpdatesNotifChannel": "Tražim moguće nadogradnje", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Želite li ukloniti aplikaciju?", "other": "Želite li ukloniti aplikacije?" diff --git a/assets/translations/cs.json b/assets/translations/cs.json index b4b052e..613a348 100644 --- a/assets/translations/cs.json +++ b/assets/translations/cs.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Dokončit instalaci aplikace", "checkingForUpdatesNotifChannel": "Zkontrolovat aktualizace", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Odstranit Apku?", "other": "Odstranit Apky?" diff --git a/assets/translations/de.json b/assets/translations/de.json index 5b22bfd..f0352ec 100644 --- a/assets/translations/de.json +++ b/assets/translations/de.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "App Installation abschließen", "checkingForUpdatesNotifChannel": "Nach Aktualisierungen suchen", "onlyCheckInstalledOrTrackOnlyApps": "Überprüfe nur installierte und mit „nur Nachverfolgen“ markierte Apps nach Aktualisierungen", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "App entfernen?", "other": "Apps entfernen?" diff --git a/assets/translations/en.json b/assets/translations/en.json index 4e4626a..ed85d8d 100644 --- a/assets/translations/en.json +++ b/assets/translations/en.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Complete App Installation", "checkingForUpdatesNotifChannel": "Checking for Updates", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Remove App?", "other": "Remove Apps?" diff --git a/assets/translations/es.json b/assets/translations/es.json index 481838a..9ffd926 100644 --- a/assets/translations/es.json +++ b/assets/translations/es.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Instalación Completa de la Aplicación", "checkingForUpdatesNotifChannel": "Buscando Actualizaciones", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "¿Eliminar Aplicación?", "other": "¿Eliminar Aplicaciones?" diff --git a/assets/translations/fa.json b/assets/translations/fa.json index 013ee34..1563556 100644 --- a/assets/translations/fa.json +++ b/assets/translations/fa.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "نصب کامل برنامه", "checkingForUpdatesNotifChannel": "بررسی به‌روزرسانی‌ها", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "برنامه حذف شود؟", "other": "برنامه ها حذف شوند؟" diff --git a/assets/translations/fr.json b/assets/translations/fr.json index e5d3bba..eaab8c0 100644 --- a/assets/translations/fr.json +++ b/assets/translations/fr.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Installation complète de l'application", "checkingForUpdatesNotifChannel": "Vérification des mises à jour", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Supprimer l'application ?", "other": "Supprimer les applications ?" diff --git a/assets/translations/hu.json b/assets/translations/hu.json index 23ff3d3..e1258dd 100644 --- a/assets/translations/hu.json +++ b/assets/translations/hu.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Teljes app telepítés", "checkingForUpdatesNotifChannel": "Frissítések keresése", "onlyCheckInstalledOrTrackOnlyApps": "Csak a telepített és a csak követhető appokat ellenőrizze frissítésekért", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Eltávolítja az alkalmazást?", "other": "Eltávolítja az alkalmazást?" diff --git a/assets/translations/it.json b/assets/translations/it.json index 945dfc0..277c1aa 100644 --- a/assets/translations/it.json +++ b/assets/translations/it.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Completa l'installazione dell'app", "checkingForUpdatesNotifChannel": "Controllo degli aggiornamenti in corso", "onlyCheckInstalledOrTrackOnlyApps": "Cerca aggiornamenti solo per app installate e app in Solo-Monitoraggio", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Rimuovere l'app?", "other": "Rimuovere le app?" diff --git a/assets/translations/ja.json b/assets/translations/ja.json index e176194..ff7cdda 100644 --- a/assets/translations/ja.json +++ b/assets/translations/ja.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "アプリのインストールを完了する", "checkingForUpdatesNotifChannel": "アップデートを確認中", "onlyCheckInstalledOrTrackOnlyApps": "インストール済みのアプリと「追跡のみ」のアプリのアップデートのみを確認する", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "アプリを削除しますか？", "other": "アプリを削除しますか？" diff --git a/assets/translations/nl.json b/assets/translations/nl.json index 7c953da..9b6addc 100644 --- a/assets/translations/nl.json +++ b/assets/translations/nl.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Voltooien van de app-installatie", "checkingForUpdatesNotifChannel": "Controleren op updates", "onlyCheckInstalledOrTrackOnlyApps": "Alleen geïnstalleerde en Track-Only apps controleren op updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "App verwijderen?", "other": "Apps verwijderen?" diff --git a/assets/translations/pl.json b/assets/translations/pl.json index e36ef9c..a06fd38 100644 --- a/assets/translations/pl.json +++ b/assets/translations/pl.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Ukończenie instalacji aplikacji", "checkingForUpdatesNotifChannel": "Sprawdzanie dostępności aktualizacji", "onlyCheckInstalledOrTrackOnlyApps": "Sprawdzaj tylko zainstalowane i obserwowane aplikacje pod kątem aktualizacji", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Usunąć aplikację?", "few": "Usunąć aplikacje?", diff --git a/assets/translations/pt.json b/assets/translations/pt.json index 9611468..b828ee0 100644 --- a/assets/translations/pt.json +++ b/assets/translations/pt.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Instalação completa do App", "checkingForUpdatesNotifChannel": "Checando por Atualizações", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Remover App?", "other": "Remover Apps?" diff --git a/assets/translations/ru.json b/assets/translations/ru.json index f02a7da..1dcb20d 100644 --- a/assets/translations/ru.json +++ b/assets/translations/ru.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Завершение установки приложения", "checkingForUpdatesNotifChannel": "Проверка обновлений", "onlyCheckInstalledOrTrackOnlyApps": "Only check installed and Track-Only apps for updates", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Удалить приложение?", "other": "Удалить приложения?" diff --git a/assets/translations/tr.json b/assets/translations/tr.json index a969e57..04dc7ff 100644 --- a/assets/translations/tr.json +++ b/assets/translations/tr.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Uygulama Kurulumu Tamamlandı", "checkingForUpdatesNotifChannel": "Güncellemeler Kontrol Ediliyor", "onlyCheckInstalledOrTrackOnlyApps": "Yalnızca yüklü ve Yalnızca İzleme Uygulamalarını güncelleme", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "Uygulamayı Kaldır?", "other": "Uygulamaları Kaldır?" diff --git a/assets/translations/vi.json b/assets/translations/vi.json index 4b66a9e..4f13aa9 100644 --- a/assets/translations/vi.json +++ b/assets/translations/vi.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "Hoàn tất cài đặt ứng dụng", "checkingForUpdatesNotifChannel": "Đang kiểm tra cập nhật", "onlyCheckInstalledOrTrackOnlyApps": "Chỉ kiểm tra các ứng dụng đã cài đặt và Chỉ-Theo dõi để biết các bản cập nhật", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion":{ "one": "Gỡ ứng dụng?", "other": "Gỡ ứng dụng?" diff --git a/assets/translations/zh.json b/assets/translations/zh.json index 314d9fe..bf031fd 100644 --- a/assets/translations/zh.json +++ b/assets/translations/zh.json @@ -275,6 +275,7 @@ "completeAppInstallationNotifChannel": "完成应用安装", "checkingForUpdatesNotifChannel": "正在检查更新", "onlyCheckInstalledOrTrackOnlyApps": "只对已安装和“仅追踪”的应用进行更新检查", + "fixedAPKURL": "APK URL is fixed", "removeAppQuestion": { "one": "是否删除应用？", "other": "是否删除应用？" diff --git a/lib/app_sources/html.dart b/lib/app_sources/html.dart index f6dab8a..2a44f23 100644 --- a/lib/app_sources/html.dart +++ b/lib/app_sources/html.dart @@ -4,6 +4,7 @@ import 'package:html/parser.dart'; import 'package:http/http.dart'; import 'package:obtainium/components/generated_form.dart'; import 'package:obtainium/custom_errors.dart'; +import 'package:obtainium/providers/apps_provider.dart'; import 'package:obtainium/providers/source_provider.dart'; String ensureAbsoluteUrl(String ambiguousUrl, Uri referenceAbsoluteUrl) { @@ -94,6 +95,7 @@ class HTML extends AppSource { label: tr('sortByFileNamesNotLinks')) ], [GeneratedFormSwitch('reverseSort', label: tr('reverseSort'))], + [GeneratedFormSwitch('fixedAPKURL', label: tr('fixedAPKURL'))], [ GeneratedFormTextField('customLinkFilterRegex', label: tr('customLinkFilterRegex'), @@ -222,7 +224,10 @@ class HTML extends AppSource { throw NoReleasesError(); } var rel = links.last; - String? version = rel.hashCode.toString(); + String? version; + if (additionalSettings['fixedAPKURL'] != true) { + version = rel.hashCode.toString(); + } var versionExtractionRegEx = additionalSettings['versionExtractionRegEx'] as String?; if (versionExtractionRegEx?.isNotEmpty == true) { @@ -243,9 +248,9 @@ class HTML extends AppSource { throw NoVersionError(); } } - List apkUrls = - [rel].map((e) => ensureAbsoluteUrl(e, uri)).toList(); - return APKDetails(version!, apkUrls.map((e) => MapEntry(e, e)).toList(), + rel = ensureAbsoluteUrl(rel, uri); + version ??= (await checkDownloadHash(rel)).toString(); + return APKDetails(version, [rel].map((e) => MapEntry(e, e)).toList(), AppNames(uri.host, tr('app'))); } else { throw getObtainiumHttpError(res); diff --git a/lib/providers/apps_provider.dart b/lib/providers/apps_provider.dart index ae0e992..dae50c4 100644 --- a/lib/providers/apps_provider.dart +++ b/lib/providers/apps_provider.dart @@ -6,6 +6,7 @@ import 'dart:convert'; import 'dart:io'; import 'dart:math'; import 'package:http/http.dart' as http; +import 'package:crypto/crypto.dart'; import 'package:android_alarm_manager_plus/android_alarm_manager_plus.dart'; import 'package:android_intent_plus/flag.dart'; @@ -139,6 +140,100 @@ List> moveStrToEndMapEntryWithCount( return arr; } +Future downloadFileWithRetry( + String url, String fileNameNoExt, Function? onProgress, String destDir, + {bool useExisting = true, + Map? headers, + int retries = 3}) async { + try { + return await downloadFile(url, fileNameNoExt, onProgress, destDir, + useExisting: useExisting, headers: headers); + } catch (e) { + if (retries > 0 && e is ClientException) { + await Future.delayed(const Duration(seconds: 5)); + return await downloadFileWithRetry( + url, fileNameNoExt, onProgress, destDir, + useExisting: useExisting, headers: headers, retries: (retries - 1)); + } else { + rethrow; + } + } +} + +String hashListOfLists(List> data) { + var bytes = utf8.encode(jsonEncode(data)); + var digest = sha256.convert(bytes); + var hash = digest.toString(); + return hash.hashCode.toString(); +} + +Future checkDownloadHash(String url, + {int bytesToGrab = 1024, Map? headers}) async { + var req = Request('GET', Uri.parse(url)); + if (headers != null) { + req.headers.addAll(headers); + } + req.headers[HttpHeaders.rangeHeader] = 'bytes=0-$bytesToGrab'; + var client = http.Client(); + var response = await client.send(req); + if (response.statusCode < 200 || response.statusCode > 299) { + throw ObtainiumError(response.reasonPhrase ?? tr('unexpectedError')); + } + List> bytes = await response.stream.take(bytesToGrab).toList(); + return hashListOfLists(bytes); +} + +Future downloadFile( + String url, String fileNameNoExt, Function? onProgress, String destDir, + {bool useExisting = true, Map? headers}) async { + var req = Request('GET', Uri.parse(url)); + if (headers != null) { + req.headers.addAll(headers); + } + var client = http.Client(); + StreamedResponse response = await client.send(req); + String ext = + response.headers['content-disposition']?.split('.').last ?? 'apk'; + if (ext.endsWith('"') || ext.endsWith("other")) { + ext = ext.substring(0, ext.length - 1); + } + if (url.toLowerCase().endsWith('.apk') && ext != 'apk') { + ext = 'apk'; + } + File downloadedFile = File('$destDir/$fileNameNoExt.$ext'); + if (!(downloadedFile.existsSync() && useExisting)) { + File tempDownloadedFile = File('${downloadedFile.path}.part'); + if (tempDownloadedFile.existsSync()) { + tempDownloadedFile.deleteSync(recursive: true); + } + var length = response.contentLength; + var received = 0; + double? progress; + var sink = tempDownloadedFile.openWrite(); + await response.stream.map((s) { + received += s.length; + progress = (length != null ? received / length * 100 : 30); + if (onProgress != null) { + onProgress(progress); + } + return s; + }).pipe(sink); + await sink.close(); + progress = null; + if (onProgress != null) { + onProgress(progress); + } + if (response.statusCode != 200) { + tempDownloadedFile.deleteSync(recursive: true); + throw response.reasonPhrase ?? tr('unexpectedError'); + } + tempDownloadedFile.renameSync(downloadedFile.path); + } else { + client.close(); + } + return downloadedFile; +} + class AppsProvider with ChangeNotifier { // In memory App state (should always be kept in sync with local storage versions) Map apps = {}; @@ -192,77 +287,6 @@ class AppsProvider with ChangeNotifier { }(); } - Future downloadFileWithRetry( - String url, String fileNameNoExt, Function? onProgress, - {bool useExisting = true, - Map? headers, - int retries = 3}) async { - try { - return await downloadFile(url, fileNameNoExt, onProgress, - useExisting: useExisting, headers: headers); - } catch (e) { - if (retries > 0 && e is ClientException) { - await Future.delayed(const Duration(seconds: 5)); - return await downloadFileWithRetry(url, fileNameNoExt, onProgress, - useExisting: useExisting, headers: headers, retries: (retries - 1)); - } else { - rethrow; - } - } - } - - Future downloadFile( - String url, String fileNameNoExt, Function? onProgress, - {bool useExisting = true, Map? headers}) async { - var destDir = APKDir.path; - var req = Request('GET', Uri.parse(url)); - if (headers != null) { - req.headers.addAll(headers); - } - var client = http.Client(); - StreamedResponse response = await client.send(req); - String ext = - response.headers['content-disposition']?.split('.').last ?? 'apk'; - if (ext.endsWith('"') || ext.endsWith("other")) { - ext = ext.substring(0, ext.length - 1); - } - if (url.toLowerCase().endsWith('.apk') && ext != 'apk') { - ext = 'apk'; - } - File downloadedFile = File('$destDir/$fileNameNoExt.$ext'); - if (!(downloadedFile.existsSync() && useExisting)) { - File tempDownloadedFile = File('${downloadedFile.path}.part'); - if (tempDownloadedFile.existsSync()) { - tempDownloadedFile.deleteSync(recursive: true); - } - var length = response.contentLength; - var received = 0; - double? progress; - var sink = tempDownloadedFile.openWrite(); - await response.stream.map((s) { - received += s.length; - progress = (length != null ? received / length * 100 : 30); - if (onProgress != null) { - onProgress(progress); - } - return s; - }).pipe(sink); - await sink.close(); - progress = null; - if (onProgress != null) { - onProgress(progress); - } - if (response.statusCode != 200) { - tempDownloadedFile.deleteSync(recursive: true); - throw response.reasonPhrase ?? tr('unexpectedError'); - } - tempDownloadedFile.renameSync(downloadedFile.path); - } else { - client.close(); - } - return downloadedFile; - } - Future handleAPKIDChange(App app, PackageInfo? newInfo, File downloadedFile, String downloadUrl) async { // If the APK package ID is different from the App ID, it is either new (using a placeholder ID) or the ID has changed @@ -322,7 +346,7 @@ class AppsProvider with ChangeNotifier { notificationsProvider?.notify(notif); } prevProg = prog; - }); + }, APKDir.path); // Set to 90 for remaining steps, will make null in 'finally' if (apps[app.id] != null) { apps[app.id]!.downloadProgress = -1; diff --git a/pubspec.lock b/pubspec.lock index 9bc0922..be0a2b6 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -147,7 +147,7 @@ packages: source: hosted version: "0.3.3+6" crypto: - dependency: transitive + dependency: "direct main" description: name: crypto sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab diff --git a/pubspec.yaml b/pubspec.yaml index 8859d6b..e986429 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -66,6 +66,7 @@ dependencies: hsluv: ^1.1.3 connectivity_plus: ^5.0.0 shared_storage: ^0.8.0 + crypto: ^3.0.3 dev_dependencies: flutter_test: