Update checking improvements (#38)

Still no auto retry for rate-limit. Instead, rate-limit errors are ignored and the unchecked Apps have to wait until the next cycle. Even this needs more testing before release.
2025-11-04 07:13:28 +01:00 · 2022-09-27 23:20:39 -04:00
parent 77e1768f3b
commit dd193d62f2
10 changed files with 185 additions and 90 deletions
--- a/lib/app_sources/github.dart
+++ b/lib/app_sources/github.dart
@@ -1,6 +1,7 @@
 import 'dart:convert';
 import 'package:http/http.dart';
 import 'package:obtainium/components/generated_form.dart';
+import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/providers/source_provider.dart';

 class GitHub implements AppSource {
@@ -76,7 +77,10 @@ class GitHub implements AppSource {
      return APKDetails(version, targetRelease['apkUrls']);
    } else {
      if (res.headers['x-ratelimit-remaining'] == '0') {
-        throw 'Rate limit reached - try again in ${(int.parse(res.headers['x-ratelimit-reset'] ?? '1800000000') / 60000000).toString()} minutes';
+        throw RateLimitError(
+            (int.parse(res.headers['x-ratelimit-reset'] ?? '1800000000') /
+                    60000000)
+                .round());
      }

      throw couldNotFindReleases;
--- a/lib/custom_errors.dart
+++ b/lib/custom_errors.dart
@@ -0,0 +1,8 @@
+class RateLimitError {
+  late int remainingMinutes;
+  RateLimitError(this.remainingMinutes);
+
+  @override
+  String toString() =>
+      'Rate limit reached - try again in $remainingMinutes minutes';
+}
--- a/lib/main.dart
+++ b/lib/main.dart
@@ -1,6 +1,7 @@
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:obtainium/app_sources/github.dart';
+import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/pages/home.dart';
 import 'package:obtainium/providers/apps_provider.dart';
 import 'package:obtainium/providers/notifications_provider.dart';
@@ -15,20 +16,42 @@ import 'package:device_info_plus/device_info_plus.dart';
 const String currentReleaseTag =
    'v0.4.1-beta'; // KEEP THIS IN SYNC WITH GITHUB RELEASES

-@pragma('vm:entry-point')
-void bgTaskCallback() {
-  // Background update checking process
-  Workmanager().executeTask((task, taskName) async {
+const String bgUpdateCheckTaskName = 'bg-update-check';
+
+bgUpdateCheck(int? ignoreAfterMicroseconds) async {
+  DateTime? ignoreAfter = ignoreAfterMicroseconds != null
+      ? DateTime.fromMicrosecondsSinceEpoch(ignoreAfterMicroseconds)
+      : null;
  var notificationsProvider = NotificationsProvider();
  await notificationsProvider.notify(checkingUpdatesNotification);
  try {
    var appsProvider = AppsProvider();
-      await notificationsProvider
-          .cancel(ErrorCheckingUpdatesNotification('').id);
+    await notificationsProvider.cancel(ErrorCheckingUpdatesNotification('').id);
    await appsProvider.loadApps();
    // List<String> existingUpdateIds = // TODO: Uncomment this and below when it works
    //     appsProvider.getExistingUpdates(installedOnly: true);
-      List<App> newUpdates = await appsProvider.checkUpdates();
+    List<String> existingUpdateIds =
+        appsProvider.getExistingUpdates(installedOnly: true);
+    // DateTime nextIgnoreAfter = DateTime.now();
+    try {
+      await appsProvider.checkUpdates(ignoreAfter: ignoreAfter);
+    } catch (e) {
+      if (e is RateLimitError) {
+        // Ignore these (scheduling another task as below does not work)
+        // Workmanager().registerOneOffTask(
+        //     bgUpdateCheckTaskName, bgUpdateCheckTaskName,
+        //     constraints: Constraints(networkType: NetworkType.connected),
+        //     initialDelay: Duration(minutes: e.remainingMinutes),
+        //     inputData: {'ignoreAfter': nextIgnoreAfter.microsecondsSinceEpoch});
+      } else {
+        rethrow;
+      }
+    }
+    List<App> newUpdates = appsProvider
+        .getExistingUpdates(installedOnly: true)
+        .where((id) => !existingUpdateIds.contains(id))
+        .map((e) => appsProvider.apps[e]!.app)
+        .toList();
    // List<String> silentlyUpdated = await appsProvider
    //     .downloadAndInstallLatestApp(
    //         [...newUpdates.map((e) => e.id), ...existingUpdateIds], null);
@@ -47,13 +70,19 @@ void bgTaskCallback() {
    }
    return Future.value(true);
  } catch (e) {
-      notificationsProvider.notify(
-          ErrorCheckingUpdatesNotification(e.toString()),
+    notificationsProvider.notify(ErrorCheckingUpdatesNotification(e.toString()),
        cancelExisting: true);
-      return Future.value(false);
+    return Future.error(false);
  } finally {
    await notificationsProvider.cancel(checkingUpdatesNotification.id);
  }
+}
+
+@pragma('vm:entry-point')
+void bgTaskCallback() {
+  // Background process callback
+  Workmanager().executeTask((task, inputData) async {
+    return await bgUpdateCheck(inputData?['ignoreAfter']);
  });
 }

@@ -95,16 +124,6 @@ class MyApp extends StatelessWidget {
    if (settingsProvider.prefs == null) {
      settingsProvider.initializeSettings();
    } else {
-      // Register the background update task according to the user's setting
-      if (settingsProvider.updateInterval > 0) {
-        Workmanager().registerPeriodicTask('bg-update-check', 'bg-update-check',
-            frequency: Duration(minutes: settingsProvider.updateInterval),
-            initialDelay: Duration(minutes: settingsProvider.updateInterval),
-            constraints: Constraints(networkType: NetworkType.connected),
-            existingWorkPolicy: ExistingWorkPolicy.replace);
-      } else {
-        Workmanager().cancelByUniqueName('bg-update-check');
-      }
      bool isFirstRun = settingsProvider.checkAndFlipFirstRun();
      if (isFirstRun) {
        // If this is the first run, ask for notification permissions and add Obtainium to the Apps list
@@ -119,9 +138,24 @@ class MyApp extends StatelessWidget {
              currentReleaseTag,
              [],
              0,
-              ['true'])
+              ['true'],
+              null)
        ]);
      }
+      // Register the background update task according to the user's setting
+      if (settingsProvider.updateInterval == 0) {
+        Workmanager().cancelByUniqueName(bgUpdateCheckTaskName);
+      } else {
+        Workmanager().registerPeriodicTask(
+            bgUpdateCheckTaskName, bgUpdateCheckTaskName,
+            frequency: Duration(minutes: settingsProvider.updateInterval),
+            initialDelay: Duration(minutes: settingsProvider.updateInterval),
+            constraints: Constraints(networkType: NetworkType.connected),
+            existingWorkPolicy: ExistingWorkPolicy.keep,
+            backoffPolicy: BackoffPolicy.linear,
+            backoffPolicyDelay:
+                const Duration(minutes: minUpdateIntervalMinutes));
+      }
    }

    return DynamicColorBuilder(
--- a/lib/mass_app_sources/githubstars.dart
+++ b/lib/mass_app_sources/githubstars.dart
@@ -15,8 +15,8 @@ class GitHubStars implements MassAppSource {
    if (args.length != requiredArgs.length) {
      throw 'Wrong number of arguments provided';
    }
-    Response res =
-        await get(Uri.parse('https://api.github.com/users/${args[0]}/starred'));
+    Response res = await get(Uri.parse(
+        'https://api.github.com/users/${args[0]}/starred?per_page=100')); //TODO: Make requests for more pages until you run out
    if (res.statusCode == 200) {
      return (jsonDecode(res.body) as List<dynamic>)
          .map((e) => e['html_url'] as String)
--- a/lib/pages/app.dart
+++ b/lib/pages/app.dart
@@ -85,6 +85,15 @@ class _AppPageState extends State<AppPage> {
                  textAlign: TextAlign.center,
                  style: Theme.of(context).textTheme.bodyLarge,
                ),
+                const SizedBox(
+                  height: 32,
+                ),
+                Text(
+                  'Last Update Check: ${app?.app.lastUpdateCheck == null ? 'Never' : '\n${app?.app.lastUpdateCheck?.toLocal()}'}',
+                  textAlign: TextAlign.center,
+                  style: const TextStyle(
+                      fontStyle: FontStyle.italic, fontSize: 12),
+                )
              ],
            ),
      bottomSheet: Padding(
--- a/lib/pages/import_export.dart
+++ b/lib/pages/import_export.dart
@@ -40,7 +40,8 @@ class _ImportExportPageState extends State<ImportExportPage> {

    Future<List<List<String>>> addApps(List<String> urls) async {
      await settingsProvider.getInstallPermission();
-      List<dynamic> results = await sourceProvider.getApps(urls);
+      List<dynamic> results = await sourceProvider.getApps(urls,
+          ignoreUrls: appsProvider.apps.values.map((e) => e.app.url).toList());
      List<App> apps = results[0];
      Map<String, dynamic> errorsMap = results[1];
      for (var app in apps) {
--- a/lib/pages/settings.dart
+++ b/lib/pages/settings.dart
@@ -169,41 +169,41 @@ class _SettingsPageState extends State<SettingsPage> {
                                    labelText:
                                        'Background Update Checking Interval'),
                                value: settingsProvider.updateInterval,
-                                items: const [
-                                  DropdownMenuItem(
-                                    value: 15,
-                                    child: Text('15 Minutes'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 30,
-                                    child: Text('30 Minutes'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 60,
-                                    child: Text('1 Hour'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 360,
-                                    child: Text('6 Hours'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 720,
-                                    child: Text('12 Hours'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 1440,
-                                    child: Text('1 Day'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 0,
-                                    child: Text('Never - Manual Only'),
-                                  ),
-                                ],
+                                items: updateIntervals.map((e) {
+                                  int displayNum = (e < 60
+                                          ? e
+                                          : e < 1440
+                                              ? e / 60
+                                              : e / 1440)
+                                      .round();
+                                  var displayUnit = (e < 60
+                                      ? 'Minute'
+                                      : e < 1440
+                                          ? 'Hour'
+                                          : 'Day');
+
+                                  String display = e == 0
+                                      ? 'Never - Manual Only'
+                                      : '$displayNum $displayUnit${displayNum == 1 ? '' : 's'}';
+                                  return DropdownMenuItem(
+                                      value: e, child: Text(display));
+                                }).toList(),
                                onChanged: (value) {
                                  if (value != null) {
                                    settingsProvider.updateInterval = value;
                                  }
                                }),
+                            const SizedBox(
+                              height: 8,
+                            ),
+                            Text(
+                              'Large App collections may require multiple cycles',
+                              style: Theme.of(context)
+                                  .textTheme
+                                  .labelMedium!
+                                  .merge(const TextStyle(
+                                      fontStyle: FontStyle.italic)),
+                            ),
                            const Spacer(),
                            Row(
                              mainAxisAlignment: MainAxisAlignment.center,
--- a/lib/providers/apps_provider.dart
+++ b/lib/providers/apps_provider.dart
@@ -297,12 +297,23 @@ class AppsProvider with ChangeNotifier {
    return null;
  }

-  Future<List<App>> checkUpdates() async {
+  Future<List<App>> checkUpdates({DateTime? ignoreAfter}) async {
    List<App> updates = [];
    if (!gettingUpdates) {
      gettingUpdates = true;

      List<String> appIds = apps.keys.toList();
+      if (ignoreAfter != null) {
+        appIds = appIds
+            .where((id) =>
+                apps[id]!.app.lastUpdateCheck != null &&
+                apps[id]!.app.lastUpdateCheck!.isBefore(ignoreAfter))
+            .toList();
+      }
+      appIds.sort((a, b) => (apps[a]!.app.lastUpdateCheck ??
+              DateTime.fromMicrosecondsSinceEpoch(0))
+          .compareTo(apps[b]!.app.lastUpdateCheck ??
+              DateTime.fromMicrosecondsSinceEpoch(0)));
      for (int i = 0; i < appIds.length; i++) {
        App? newApp = await getUpdate(appIds[i]);
        if (newApp != null) {
--- a/lib/providers/settings_provider.dart
+++ b/lib/providers/settings_provider.dart
@@ -13,6 +13,16 @@ enum SortColumnSettings { added, nameAuthor, authorName }

 enum SortOrderSettings { ascending, descending }

+const maxAPIRateLimitMinutes = 30;
+const minUpdateIntervalMinutes = maxAPIRateLimitMinutes + 30;
+const maxUpdateIntervalMinutes = 4320;
+List<int> updateIntervals = [15, 30, 60, 120, 180, 360, 720, 1440, 4320, 0]
+    .where((element) =>
+        (element >= minUpdateIntervalMinutes &&
+            element <= maxUpdateIntervalMinutes) ||
+        element == 0)
+    .toList();
+
 class SettingsProvider with ChangeNotifier {
  SharedPreferences? prefs;

@@ -45,7 +55,17 @@ class SettingsProvider with ChangeNotifier {
  }

  int get updateInterval {
-    return prefs?.getInt('updateInterval') ?? 1440;
+    var min = prefs?.getInt('updateInterval') ?? 180;
+    if (!updateIntervals.contains(min)) {
+      var temp = updateIntervals[0];
+      for (var i in updateIntervals) {
+        if (min > i && i != 0) {
+          temp = i;
+        }
+      }
+      min = temp;
+    }
+    return min;
  }

  set updateInterval(int min) {
--- a/lib/providers/source_provider.dart
+++ b/lib/providers/source_provider.dart
@@ -38,6 +38,7 @@ class App {
  List<String> apkUrls = [];
  late int preferredApkIndex;
  late List<String> additionalData;
+  late DateTime? lastUpdateCheck;
  App(
      this.id,
      this.url,
@@ -47,7 +48,8 @@ class App {
      this.latestVersion,
      this.apkUrls,
      this.preferredApkIndex,
-      this.additionalData);
+      this.additionalData,
+      this.lastUpdateCheck);

  @override
  String toString() {
@@ -69,7 +71,10 @@ class App {
      json['preferredApkIndex'] == null ? 0 : json['preferredApkIndex'] as int,
      json['additionalData'] == null
          ? SourceProvider().getSource(json['url']).additionalDataDefaults
-          : List<String>.from(jsonDecode(json['additionalData'])));
+          : List<String>.from(jsonDecode(json['additionalData'])),
+      json['lastUpdateCheck'] == null
+          ? null
+          : DateTime.fromMicrosecondsSinceEpoch(json['lastUpdateCheck']));

  Map<String, dynamic> toJson() => {
        'id': id,
@@ -80,7 +85,8 @@ class App {
        'latestVersion': latestVersion,
        'apkUrls': jsonEncode(apkUrls),
        'preferredApkIndex': preferredApkIndex,
-        'additionalData': jsonEncode(additionalData)
+        'additionalData': jsonEncode(additionalData),
+        'lastUpdateCheck': lastUpdateCheck?.microsecondsSinceEpoch
      };
 }

@@ -195,15 +201,17 @@ class SourceProvider {
        apk.version,
        apk.apkUrls,
        apk.apkUrls.length - 1,
-        additionalData);
+        additionalData,
+        DateTime.now());
  }

  /// Returns a length 2 list, where the first element is a list of Apps and
  /// the second is a Map<String, dynamic> of URLs and errors
-  Future<List<dynamic>> getApps(List<String> urls) async {
+  Future<List<dynamic>> getApps(List<String> urls,
+      {List<String> ignoreUrls = const []}) async {
    List<App> apps = [];
    Map<String, dynamic> errors = {};
-    for (var url in urls) {
+    for (var url in urls.where((element) => !ignoreUrls.contains(element))) {
      try {
        var source = getSource(url);
        apps.add(await getApp(source, url, source.additionalDataDefaults));