rewrite apkpure source to use api instead of web scraping

lib/app_sources/apkpure.dart

@@ -1,24 +1,18 @@
+import 'dart:convert';
+
 import 'package:device_info_plus/device_info_plus.dart';
 import 'package:easy_localization/easy_localization.dart';
-import 'package:html/parser.dart';
-import 'package:obtainium/app_sources/html.dart';
 import 'package:obtainium/components/generated_form.dart';
 import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/providers/source_provider.dart';
 
-parseDateTimeMMMddCommayyyy(String? dateString) {
+extension Unique<E, Id> on List<E> {
-  DateTime? releaseDate;
+  List<E> unique([Id Function(E element)? id, bool inplace = true]) {
-  try {
+    final ids = Set();
-    releaseDate = dateString != null
+    var list = inplace ? this : List<E>.from(this);
-        ? DateFormat('MMM dd, yyyy').parse(dateString)
+    list.retainWhere((x) => ids.add(id != null ? id(x) : x as Id));
-        : null;
+    return list;
-    releaseDate = dateString != null && releaseDate == null
-        ? DateFormat('MMMM dd, yyyy').parse(dateString)
-        : releaseDate;
-  } catch (err) {
-    // ignore
   }
-  return releaseDate;
 }
 
 class APKPure extends AppSource {
@@ -35,6 +29,10 @@ class APKPure extends AppSource {
       [
         GeneratedFormSwitch('stayOneVersionBehind',
             label: tr('stayOneVersionBehind'), defaultValue: false)
+      ],
+      [
+        GeneratedFormSwitch('selectNewestApk',
+            label: tr('selectNewestApk'), defaultValue: true)
       ]
     ];
   }
@@ -65,109 +63,65 @@ class APKPure extends AppSource {
     return Uri.parse(standardUrl).pathSegments.last;
   }
 
-  getDetailsForVersionLink(
+  getDetailsForVersion(List<Map> versionVariants, List<String> supportedArchs,
-      String standardUrl,
-      String appId,
-      String host,
-      List<String> supportedArchs,
-      String link,
       Map<String, dynamic> additionalSettings) async {
-    var res = await sourceRequest(link, additionalSettings);
+    var apkUrls = versionVariants
-    if (res.statusCode == 200) {
+        .map((e) {
-      var html = parse(res.body);
+          String appId = e['package_name'];
-      var apksDiv =
+          String versionCode = e['version_code'];
-          html.querySelector('#version-list div div.show-more-content');
+
-      DateTime? topReleaseDate;
+          List<String> architectures = e['native_code']?.cast<String>();
-      var apkUrls = apksDiv
+          String architectureString = architectures.join(',');
-              ?.querySelectorAll('div.group-title')
+          if (additionalSettings['autoApkFilterByArch'] == true &&
-              .map((e) {
+              architectures.isNotEmpty &&
-                String architectureString = e.text.trim();
+              architectures.where((a) => supportedArchs.contains(a)).isEmpty) {
-                if (architectureString.toLowerCase() == 'unlimited' ||
+            return null;
-                    architectureString.toLowerCase() == 'universal') {
+          }
-                  architectureString = '';
+
-                }
+          String type = e['asset']['type'];
-                List<String> architectures = architectureString
+          if (e['is_a_p_ks'] == true) {
-                    .split(',')
+            type = 'APK';
-                    .map((e) => e.trim())
+          }
-                    .where((e) => e.isNotEmpty)
+
-                    .toList();
+          return MapEntry(
-                // Only take the first APK for each architecture, ignore others for now, for simplicity
+              '$appId-$versionCode-$architectureString.${type.toLowerCase()}',
-                // Unclear why there can even be multiple APKs for the same version and arch
+              'https://d.cdnpure.com/b/$type/$appId?versionCode=$versionCode&nc=$architectureString');
-                var apkInfo = e.nextElementSibling?.querySelector('div.info');
+        })
-                String? versionCode = RegExp('[0-9]+')
+        .nonNulls
-                    .firstMatch(
+        .toList()
-                        apkInfo?.querySelector('div.info-top .code')?.text ??
+        .unique((e) => e.key);
-                            '')
+
-                    ?.group(0)
+    // get version details from first variant
-                    ?.trim();
+    var v = versionVariants.first;
-                var types = apkInfo
+    String version = v['version_name'];
-                        ?.querySelectorAll('div.info-top span.tag')
+    String author = v['developer'];
-                        .map((e) => e.text.trim())
+    String appName = v['title'];
-                        .map((t) => t == 'APKs' ? 'APK' : t) ??
+    DateTime releaseDate = DateTime.parse(v['update_date']);
-                    [];
+    String? changeLog = v['whatsnew'];
-                String type = types.isEmpty ? 'APK' : types.first;
+    if (changeLog != null && changeLog.isEmpty) {
-                String? dateString = apkInfo
+      changeLog = null;
-                    ?.querySelector('div.info-bottom span.time')
+    }
-                    ?.text
+
-                    .trim();
+    if (additionalSettings['selectNewestApk'] == true) {
-                DateTime? releaseDate = parseDateTimeMMMddCommayyyy(dateString);
+      apkUrls = [apkUrls.first];
-                if (additionalSettings['autoApkFilterByArch'] == true &&
+    }
-                    architectures.isNotEmpty &&
+
-                    architectures
+    return APKDetails(version, apkUrls, AppNames(author, appName),
-                        .where((a) => supportedArchs.contains(a))
+        releaseDate: releaseDate, changeLog: changeLog);
-                        .isEmpty) {
+  }
-                  return const MapEntry('', '');
+
-                }
+  @override
-                topReleaseDate ??=
+  Future<Map<String, String>?> getRequestHeaders(
-                    releaseDate; // Just use the release date of the first APK in the list as the release date for this version
+      Map<String, dynamic> additionalSettings,
-                return MapEntry(
+      {bool forAPKDownload = false}) async {
-                    '$appId-$versionCode-$architectureString.${type.toLowerCase()}',
+    if (forAPKDownload) {
-                    'https://d.${hosts.contains(host) ? 'cdnpure.com' : host}/b/$type/$appId?versionCode=$versionCode');
+      return null;
-              })
-              .where((e) => e.key.isNotEmpty)
-              .toList() ??
-          [];
-      if (apkUrls.isEmpty) {
-        var link =
-            html.querySelector("a.download-start-btn")?.attributes['href'];
-        RegExp downloadLinkRegEx = RegExp(
-            r'^https:\/\/d\.[^/]+\/b\/([^/]+)\/[^/?]+\?versionCode=([0-9]+)$',
-            caseSensitive: false);
-        RegExpMatch? match = downloadLinkRegEx.firstMatch(link ?? '');
-        if (match == null) {
-          throw NoAPKError();
-        }
-        String type = match.group(1)!;
-        String versionCode = match.group(2)!;
-        apkUrls = [
-          MapEntry('$appId-$versionCode-.${type.toLowerCase()}',
-              'https://d.${hosts.contains(host) ? 'cdnpure.com' : host}/b/$type/$appId?versionCode=$versionCode')
-        ];
-      }
-      String version = Uri.parse(link).pathSegments.last;
-      String? author;
-      try {
-        author = html
-                .querySelector('span.info-sdk')
-                ?.text
-                .trim()
-                .substring(version.length + 4) ??
-            Uri.parse(standardUrl).pathSegments.reversed.last;
-      } catch (e) {
-        author = html.querySelector('span.info-sdk')?.text.trim() ??
-            Uri.parse(standardUrl).pathSegments.reversed.last;
-      }
-      String appName =
-          html.querySelector('h1.info-title')?.text.trim() ?? appId;
-      String? changeLog = html
-          .querySelector('div.module.change-log')
-          ?.innerHtml
-          .trim()
-          .replaceAll("<br>", "  \n");
-      return APKDetails(version, apkUrls, AppNames(author, appName),
-          releaseDate: topReleaseDate, changeLog: changeLog);
     } else {
-      throw getObtainiumHttpError(res);
+      return {
+        "Ual-Access-Businessid": "projecta",
+        "Ual-Access-ProjectA":
+            '{"device_info":{"os_ver":"${((await DeviceInfoPlugin().androidInfo).version.sdkInt)}"}}',
+      };
     }
   }
 
@@ -177,41 +131,46 @@ class APKPure extends AppSource {
     Map<String, dynamic> additionalSettings,
   ) async {
     String appId = (await tryInferringAppId(standardUrl))!;
-    String host = Uri.parse(standardUrl).host;
 
-    var res0 = await sourceRequest('$standardUrl/versions', additionalSettings);
+    List<String> supportedArchs =
-    var decodedStandardUrl = standardUrl;
+        (await DeviceInfoPlugin().androidInfo).supportedAbis;
-    try {
+
-      decodedStandardUrl = Uri.decodeFull(decodedStandardUrl);
+    // request versions from API
-    } catch (e) {
+    var res = await sourceRequest(
-      //
+        "https://tapi.pureapk.com/v3/get_app_his_version?package_name=$appId&hl=en",
+        additionalSettings);
+    if (res.statusCode != 200) {
+      throw getObtainiumHttpError(res);
     }
-    var versionLinks = await grabLinksCommon(res0, {
+    List<Map<String, dynamic>> apks =
-      'skipSort': true,
+        jsonDecode(res.body)['version_list'].cast<Map<String, dynamic>>();
-      'customLinkFilterRegex': '$decodedStandardUrl/download/[^/]+\$'
-    });
 
-    var supportedArchs = (await DeviceInfoPlugin().androidInfo).supportedAbis;
+    // group by version
+    List versions = apks
+        .fold<Map<String, List<Map<String, dynamic>>>>({},
+            (Map<String, List<Map<String, dynamic>>> val,
+                Map<String, dynamic> element) {
+          String v = element['version_name'];
+          if (!val.containsKey(v)) {
+            val[v] = [];
+          }
+          val[v]?.add(element);
+          return val;
+        })
+        .values
+        .toList();
 
-    if (additionalSettings['autoApkFilterByArch'] != true) {
+    for (var i = 0; i < versions.length; i++) {
-      // No need to request multiple versions when we're not going to filter them (always pick the top one)
+      var v = versions[i];
-      versionLinks = versionLinks.sublist(0, 1);
-    }
-    if (versionLinks.isEmpty) {
-      throw NoReleasesError();
-    }
-
-    for (var i = 0; i < versionLinks.length; i++) {
-      var link = versionLinks[i];
       try {
         if (i == 0 && additionalSettings['stayOneVersionBehind'] == true) {
           throw NoReleasesError();
         }
-        return await getDetailsForVersionLink(standardUrl, appId, host,
+        return await getDetailsForVersion(
-            supportedArchs, link.key, additionalSettings);
+            v, supportedArchs, additionalSettings);
       } catch (e) {
         if (additionalSettings['fallbackToOlderReleases'] != true ||
-            i == versionLinks.length - 1) {
+            i == versions.length - 1) {
           rethrow;
         }
       }

lib/app_sources/uptodown.dart

@@ -1,9 +1,23 @@
 import 'package:easy_localization/easy_localization.dart';
 import 'package:html/parser.dart';
-import 'package:obtainium/app_sources/apkpure.dart';
 import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/providers/source_provider.dart';
 
+parseDateTimeMMMddCommayyyy(String? dateString) {
+  DateTime? releaseDate;
+  try {
+    releaseDate = dateString != null
+        ? DateFormat('MMM dd, yyyy').parse(dateString)
+        : null;
+    releaseDate = dateString != null && releaseDate == null
+        ? DateFormat('MMMM dd, yyyy').parse(dateString)
+        : releaseDate;
+  } catch (err) {
+    // ignore
+  }
+  return releaseDate;
+}
+
 class Uptodown extends AppSource {
   Uptodown() {
     hosts = ['uptodown.com'];