APKPure: Filter releases by available architecture (#598)

lib/app_sources/apkpure.dart

@@ -1,5 +1,7 @@
+import 'package:device_info_plus/device_info_plus.dart';
 import 'package:easy_localization/easy_localization.dart';
 import 'package:html/parser.dart';
+import 'package:obtainium/app_sources/html.dart';
 import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/providers/source_provider.dart';
 
@@ -58,40 +60,102 @@ class APKPure extends AppSource {
   ) async {
     String appId = (await tryInferringAppId(standardUrl))!;
     String host = Uri.parse(standardUrl).host;
-    var res = await sourceRequest('$standardUrl/download', additionalSettings);
+
-    var resChangelog = await sourceRequest(standardUrl, additionalSettings);
+    var res0 = await sourceRequest('$standardUrl/versions', additionalSettings);
-    if (res.statusCode == 200 && resChangelog.statusCode == 200) {
+    var versionLinks = await grabLinksCommon(res0, {
-      var html = parse(res.body);
+      'skipSort': true,
-      var htmlChangelog = parse(resChangelog.body);
+      'customLinkFilterRegex': '$standardUrl/download/[^/]+\$'
-      String? version = html.querySelector('span.info-sdk span')?.text.trim();
+    });
-      if (version == null) {
+
-        throw NoVersionError();
+    // if (versionLinks.length > 7) {
-      }
+    //   // Returns up to 30 which is too much - would take too long and possibly get blocked/rate-limited
-      String? dateString =
+    //   versionLinks = versionLinks.sublist(0, 7);
-          html.querySelector('span.info-other span.date')?.text.trim();
+    // }
-      DateTime? releaseDate = parseDateTimeMMMddCommayyyy(dateString);
+
-      String type = html.querySelector('a.info-tag')?.text.trim() ?? 'APK';
+    var supportedArchs = (await DeviceInfoPlugin().androidInfo).supportedAbis;
-      List<MapEntry<String, String>> apkUrls = [
+
-        MapEntry('$appId.apk',
+    if (additionalSettings['autoApkFilterByArch'] != true) {
-            'https://d.${hosts.contains(host) ? 'cdnpure.com' : host}/b/$type/$appId?version=latest')
+      // No need to request multiple versions when we're not going to filter them (always pick the top one)
-      ];
+      versionLinks = versionLinks.sublist(0, 1);
-      String author = html
-              .querySelector('span.info-sdk')
-              ?.text
-              .trim()
-              .substring(version.length + 4) ??
-          Uri.parse(standardUrl).pathSegments.reversed.last;
-      String appName =
-          html.querySelector('h1.info-title')?.text.trim() ?? appId;
-      String? changeLog = htmlChangelog
-          .querySelector("div.whats-new-info p:not(.date)")
-          ?.innerHtml
-          .trim()
-          .replaceAll("<br>", "  \n");
-      return APKDetails(version, apkUrls, AppNames(author, appName),
-          releaseDate: releaseDate, changeLog: changeLog);
-    } else {
-      throw getObtainiumHttpError(res);
     }
+    if (versionLinks.isEmpty) {
+      throw NoReleasesError();
+    }
+
+    List<APKDetails?> versionDetails =
+        (await Future.wait(versionLinks.map((link) async {
+      var res = await sourceRequest(link.key, additionalSettings);
+      if (res.statusCode == 200) {
+        var html = parse(res.body);
+        var apksDiv =
+            html.querySelector('#version-list div div.show-more-content');
+        DateTime? topReleaseDate;
+        var apkUrls = apksDiv
+                ?.querySelectorAll('div.group-title')
+                .map((e) {
+                  String? architecture = e.text.trim();
+                  // Only take the first APK for each architecture, ignore others for now, for simplicity
+                  // Unclear why there can even be multiple APKs for the same version and arch
+                  var apkInfo = e.nextElementSibling?.querySelector('div.info');
+                  String? versionCode = RegExp('[0-9]+')
+                      .firstMatch(apkInfo
+                              ?.querySelector('div.info-top span.code')
+                              ?.text ??
+                          '')
+                      ?.group(0)
+                      ?.trim();
+                  String? type = apkInfo
+                          ?.querySelector('div.info-top span.tag')
+                          ?.text
+                          .trim() ??
+                      'APK';
+                  String? dateString = apkInfo
+                      ?.querySelector('div.info-bottom span.time')
+                      ?.text
+                      .trim();
+                  DateTime? releaseDate =
+                      parseDateTimeMMMddCommayyyy(dateString);
+                  if (additionalSettings['autoApkFilterByArch'] == true &&
+                      !supportedArchs.contains(architecture)) {
+                    return const MapEntry('', '');
+                  }
+                  topReleaseDate ??=
+                      releaseDate; // Just use the release date of the first APK in the list as the release date for this version
+                  return MapEntry(
+                      '$appId-$versionCode-$architecture.${type.toLowerCase()}',
+                      'https://d.${hosts.contains(host) ? 'cdnpure.com' : host}/b/$type/$appId?versionCode=$versionCode');
+                })
+                .where((e) => e.key.isNotEmpty)
+                .toList() ??
+            [];
+        if (apkUrls.isEmpty) {
+          return null;
+        }
+        String version = Uri.parse(link.key).pathSegments.last;
+        String author = html
+                .querySelector('span.info-sdk')
+                ?.text
+                .trim()
+                .substring(version.length + 4) ??
+            Uri.parse(standardUrl).pathSegments.reversed.last;
+        String appName =
+            html.querySelector('h1.info-title')?.text.trim() ?? appId;
+        String? changeLog = html
+            .querySelector('div.module.change-log')
+            ?.innerHtml
+            .trim()
+            .replaceAll("<br>", "  \n");
+        return APKDetails(version, apkUrls, AppNames(author, appName),
+            releaseDate: topReleaseDate, changeLog: changeLog);
+      } else {
+        throw getObtainiumHttpError(res);
+      }
+    })))
+            .where((e) => e != null)
+            .toList();
+    if (versionDetails.isEmpty) {
+      throw NoAPKError();
+    }
+    return versionDetails[0]!;
   }
 }

lib/app_sources/html.dart

@@ -92,6 +92,73 @@ bool _isNumeric(String s) {
   return s.codeUnitAt(0) >= 48 && s.codeUnitAt(0) <= 57;
 }
 
+// Given an HTTP response, grab some links according to the common additional settings
+// (those that apply to intermediate and final steps)
+Future<List<MapEntry<String, String>>> grabLinksCommon(
+    Response res, Map<String, dynamic> additionalSettings) async {
+  if (res.statusCode != 200) {
+    throw getObtainiumHttpError(res);
+  }
+  var html = parse(res.body);
+  List<MapEntry<String, String>> allLinks = html
+      .querySelectorAll('a')
+      .map((element) => MapEntry(
+          element.attributes['href'] ?? '',
+          element.text.isNotEmpty
+              ? element.text
+              : (element.attributes['href'] ?? '').split('/').last))
+      .where((element) => element.key.isNotEmpty)
+      .map((e) => MapEntry(ensureAbsoluteUrl(e.key, res.request!.url), e.value))
+      .toList();
+  if (allLinks.isEmpty) {
+    allLinks = RegExp(
+            r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?')
+        .allMatches(res.body)
+        .map((match) =>
+            MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? ''))
+        .toList();
+  }
+  List<MapEntry<String, String>> links = [];
+  bool skipSort = additionalSettings['skipSort'] == true;
+  bool filterLinkByText = additionalSettings['filterByLinkText'] == true;
+  if ((additionalSettings['customLinkFilterRegex'] as String?)?.isNotEmpty ==
+      true) {
+    var reg = RegExp(additionalSettings['customLinkFilterRegex']);
+    links = allLinks.where((element) {
+      var link = element.key;
+      try {
+        link = Uri.decodeFull(element.key);
+      } catch (e) {
+        // Some links may not have valid encoding
+      }
+      return reg.hasMatch(filterLinkByText ? element.value : link);
+    }).toList();
+  } else {
+    links = allLinks.where((element) {
+      var link = element.key;
+      try {
+        link = Uri.decodeFull(element.key);
+      } catch (e) {
+        // Some links may not have valid encoding
+      }
+      return Uri.parse(filterLinkByText ? element.value : link)
+          .path
+          .toLowerCase()
+          .endsWith('.apk');
+    }).toList();
+  }
+  if (!skipSort) {
+    links.sort((a, b) => additionalSettings['sortByLastLinkSegment'] == true
+        ? compareAlphaNumeric(a.key.split('/').where((e) => e.isNotEmpty).last,
+            b.key.split('/').where((e) => e.isNotEmpty).last)
+        : compareAlphaNumeric(a.key, b.key));
+  }
+  if (additionalSettings['reverseSort'] == true) {
+    links = links.reversed.toList();
+  }
+  return links;
+}
+
 class HTML extends AppSource {
   @override
   List<List<GeneratedFormItem>> get combinedAppSpecificSettingFormItems {
@@ -225,75 +292,6 @@ class HTML extends AppSource {
     return url;
   }
 
-  // Given an HTTP response, grab some links according to the common additional settings
-  // (those that apply to intermediate and final steps)
-  Future<List<MapEntry<String, String>>> grabLinksCommon(
-      Response res, Map<String, dynamic> additionalSettings) async {
-    if (res.statusCode != 200) {
-      throw getObtainiumHttpError(res);
-    }
-    var html = parse(res.body);
-    List<MapEntry<String, String>> allLinks = html
-        .querySelectorAll('a')
-        .map((element) => MapEntry(
-            element.attributes['href'] ?? '',
-            element.text.isNotEmpty
-                ? element.text
-                : (element.attributes['href'] ?? '').split('/').last))
-        .where((element) => element.key.isNotEmpty)
-        .map((e) =>
-            MapEntry(ensureAbsoluteUrl(e.key, res.request!.url), e.value))
-        .toList();
-    if (allLinks.isEmpty) {
-      allLinks = RegExp(
-              r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?')
-          .allMatches(res.body)
-          .map((match) =>
-              MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? ''))
-          .toList();
-    }
-    List<MapEntry<String, String>> links = [];
-    bool skipSort = additionalSettings['skipSort'] == true;
-    bool filterLinkByText = additionalSettings['filterByLinkText'] == true;
-    if ((additionalSettings['customLinkFilterRegex'] as String?)?.isNotEmpty ==
-        true) {
-      var reg = RegExp(additionalSettings['customLinkFilterRegex']);
-      links = allLinks.where((element) {
-        var link = element.key;
-        try {
-          link = Uri.decodeFull(element.key);
-        } catch (e) {
-          // Some links may not have valid encoding
-        }
-        return reg.hasMatch(filterLinkByText ? element.value : link);
-      }).toList();
-    } else {
-      links = allLinks.where((element) {
-        var link = element.key;
-        try {
-          link = Uri.decodeFull(element.key);
-        } catch (e) {
-          // Some links may not have valid encoding
-        }
-        return Uri.parse(filterLinkByText ? element.value : link)
-            .path
-            .toLowerCase()
-            .endsWith('.apk');
-      }).toList();
-    }
-    if (!skipSort) {
-      links.sort((a, b) => additionalSettings['sortByLastLinkSegment'] == true
-          ? compareAlphaNumeric(
-              a.key.split('/').where((e) => e.isNotEmpty).last,
-              b.key.split('/').where((e) => e.isNotEmpty).last)
-          : compareAlphaNumeric(a.key, b.key));
-    }
-    if (additionalSettings['reverseSort'] == true) {
-      links = links.reversed.toList();
-    }
-    return links;
-  }
-
   @override
   Future<APKDetails> getLatestAPKDetails(
     String standardUrl,