Update checking improvements (#38)

Still no auto retry for rate-limit. Instead, rate-limit errors are ignored and the unchecked Apps have to wait until the next cycle. Even this needs more testing before release.

lib/app_sources/github.dart

@@ -1,6 +1,7 @@
 import 'dart:convert';
 import 'package:http/http.dart';
 import 'package:obtainium/components/generated_form.dart';
+import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/providers/source_provider.dart';
 
 class GitHub implements AppSource {
@@ -76,7 +77,10 @@ class GitHub implements AppSource {
       return APKDetails(version, targetRelease['apkUrls']);
     } else {
       if (res.headers['x-ratelimit-remaining'] == '0') {
-        throw 'Rate limit reached - try again in ${(int.parse(res.headers['x-ratelimit-reset'] ?? '1800000000') / 60000000).toString()} minutes';
+        throw RateLimitError(
+            (int.parse(res.headers['x-ratelimit-reset'] ?? '1800000000') /
+                    60000000)
+                .round());
       }
 
       throw couldNotFindReleases;

lib/custom_errors.dart

@@ -0,0 +1,8 @@
+class RateLimitError {
+  late int remainingMinutes;
+  RateLimitError(this.remainingMinutes);
+
+  @override
+  String toString() =>
+      'Rate limit reached - try again in $remainingMinutes minutes';
+}

lib/main.dart

@@ -1,6 +1,7 @@
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:obtainium/app_sources/github.dart';
+import 'package:obtainium/custom_errors.dart';
 import 'package:obtainium/pages/home.dart';
 import 'package:obtainium/providers/apps_provider.dart';
 import 'package:obtainium/providers/notifications_provider.dart';
@@ -15,45 +16,73 @@ import 'package:device_info_plus/device_info_plus.dart';
 const String currentReleaseTag =
     'v0.4.1-beta'; // KEEP THIS IN SYNC WITH GITHUB RELEASES
 
+const String bgUpdateCheckTaskName = 'bg-update-check';
+
+bgUpdateCheck(int? ignoreAfterMicroseconds) async {
+  DateTime? ignoreAfter = ignoreAfterMicroseconds != null
+      ? DateTime.fromMicrosecondsSinceEpoch(ignoreAfterMicroseconds)
+      : null;
+  var notificationsProvider = NotificationsProvider();
+  await notificationsProvider.notify(checkingUpdatesNotification);
+  try {
+    var appsProvider = AppsProvider();
+    await notificationsProvider.cancel(ErrorCheckingUpdatesNotification('').id);
+    await appsProvider.loadApps();
+    // List<String> existingUpdateIds = // TODO: Uncomment this and below when it works
+    //     appsProvider.getExistingUpdates(installedOnly: true);
+    List<String> existingUpdateIds =
+        appsProvider.getExistingUpdates(installedOnly: true);
+    // DateTime nextIgnoreAfter = DateTime.now();
+    try {
+      await appsProvider.checkUpdates(ignoreAfter: ignoreAfter);
+    } catch (e) {
+      if (e is RateLimitError) {
+        // Ignore these (scheduling another task as below does not work)
+        // Workmanager().registerOneOffTask(
+        //     bgUpdateCheckTaskName, bgUpdateCheckTaskName,
+        //     constraints: Constraints(networkType: NetworkType.connected),
+        //     initialDelay: Duration(minutes: e.remainingMinutes),
+        //     inputData: {'ignoreAfter': nextIgnoreAfter.microsecondsSinceEpoch});
+      } else {
+        rethrow;
+      }
+    }
+    List<App> newUpdates = appsProvider
+        .getExistingUpdates(installedOnly: true)
+        .where((id) => !existingUpdateIds.contains(id))
+        .map((e) => appsProvider.apps[e]!.app)
+        .toList();
+    // List<String> silentlyUpdated = await appsProvider
+    //     .downloadAndInstallLatestApp(
+    //         [...newUpdates.map((e) => e.id), ...existingUpdateIds], null);
+    // if (silentlyUpdated.isNotEmpty) {
+    //   newUpdates
+    //       .where((element) => !silentlyUpdated.contains(element.id))
+    //       .toList();
+    //   notificationsProvider.notify(
+    //       SilentUpdateNotification(
+    //           silentlyUpdated.map((e) => appsProvider.apps[e]!.app).toList()),
+    //       cancelExisting: true);
+    // }
+    if (newUpdates.isNotEmpty) {
+      notificationsProvider.notify(UpdateNotification(newUpdates),
+          cancelExisting: true);
+    }
+    return Future.value(true);
+  } catch (e) {
+    notificationsProvider.notify(ErrorCheckingUpdatesNotification(e.toString()),
+        cancelExisting: true);
+    return Future.error(false);
+  } finally {
+    await notificationsProvider.cancel(checkingUpdatesNotification.id);
+  }
+}
+
 @pragma('vm:entry-point')
 void bgTaskCallback() {
-  // Background update checking process
+  // Background process callback
-  Workmanager().executeTask((task, taskName) async {
+  Workmanager().executeTask((task, inputData) async {
-    var notificationsProvider = NotificationsProvider();
+    return await bgUpdateCheck(inputData?['ignoreAfter']);
-    await notificationsProvider.notify(checkingUpdatesNotification);
-    try {
-      var appsProvider = AppsProvider();
-      await notificationsProvider
-          .cancel(ErrorCheckingUpdatesNotification('').id);
-      await appsProvider.loadApps();
-      // List<String> existingUpdateIds = // TODO: Uncomment this and below when it works
-      //     appsProvider.getExistingUpdates(installedOnly: true);
-      List<App> newUpdates = await appsProvider.checkUpdates();
-      // List<String> silentlyUpdated = await appsProvider
-      //     .downloadAndInstallLatestApp(
-      //         [...newUpdates.map((e) => e.id), ...existingUpdateIds], null);
-      // if (silentlyUpdated.isNotEmpty) {
-      //   newUpdates
-      //       .where((element) => !silentlyUpdated.contains(element.id))
-      //       .toList();
-      //   notificationsProvider.notify(
-      //       SilentUpdateNotification(
-      //           silentlyUpdated.map((e) => appsProvider.apps[e]!.app).toList()),
-      //       cancelExisting: true);
-      // }
-      if (newUpdates.isNotEmpty) {
-        notificationsProvider.notify(UpdateNotification(newUpdates),
-            cancelExisting: true);
-      }
-      return Future.value(true);
-    } catch (e) {
-      notificationsProvider.notify(
-          ErrorCheckingUpdatesNotification(e.toString()),
-          cancelExisting: true);
-      return Future.value(false);
-    } finally {
-      await notificationsProvider.cancel(checkingUpdatesNotification.id);
-    }
   });
 }
 
@@ -95,16 +124,6 @@ class MyApp extends StatelessWidget {
     if (settingsProvider.prefs == null) {
       settingsProvider.initializeSettings();
     } else {
-      // Register the background update task according to the user's setting
-      if (settingsProvider.updateInterval > 0) {
-        Workmanager().registerPeriodicTask('bg-update-check', 'bg-update-check',
-            frequency: Duration(minutes: settingsProvider.updateInterval),
-            initialDelay: Duration(minutes: settingsProvider.updateInterval),
-            constraints: Constraints(networkType: NetworkType.connected),
-            existingWorkPolicy: ExistingWorkPolicy.replace);
-      } else {
-        Workmanager().cancelByUniqueName('bg-update-check');
-      }
       bool isFirstRun = settingsProvider.checkAndFlipFirstRun();
       if (isFirstRun) {
         // If this is the first run, ask for notification permissions and add Obtainium to the Apps list
@@ -119,9 +138,24 @@ class MyApp extends StatelessWidget {
               currentReleaseTag,
               [],
               0,
-              ['true'])
+              ['true'],
+              null)
         ]);
       }
+      // Register the background update task according to the user's setting
+      if (settingsProvider.updateInterval == 0) {
+        Workmanager().cancelByUniqueName(bgUpdateCheckTaskName);
+      } else {
+        Workmanager().registerPeriodicTask(
+            bgUpdateCheckTaskName, bgUpdateCheckTaskName,
+            frequency: Duration(minutes: settingsProvider.updateInterval),
+            initialDelay: Duration(minutes: settingsProvider.updateInterval),
+            constraints: Constraints(networkType: NetworkType.connected),
+            existingWorkPolicy: ExistingWorkPolicy.keep,
+            backoffPolicy: BackoffPolicy.linear,
+            backoffPolicyDelay:
+                const Duration(minutes: minUpdateIntervalMinutes));
+      }
     }
 
     return DynamicColorBuilder(

lib/mass_app_sources/githubstars.dart

@@ -15,8 +15,8 @@ class GitHubStars implements MassAppSource {
     if (args.length != requiredArgs.length) {
       throw 'Wrong number of arguments provided';
     }
-    Response res =
+    Response res = await get(Uri.parse(
-        await get(Uri.parse('https://api.github.com/users/${args[0]}/starred'));
+        'https://api.github.com/users/${args[0]}/starred?per_page=100')); //TODO: Make requests for more pages until you run out
     if (res.statusCode == 200) {
       return (jsonDecode(res.body) as List<dynamic>)
           .map((e) => e['html_url'] as String)

lib/pages/app.dart

@@ -85,6 +85,15 @@ class _AppPageState extends State<AppPage> {
                   textAlign: TextAlign.center,
                   style: Theme.of(context).textTheme.bodyLarge,
                 ),
+                const SizedBox(
+                  height: 32,
+                ),
+                Text(
+                  'Last Update Check: ${app?.app.lastUpdateCheck == null ? 'Never' : '\n${app?.app.lastUpdateCheck?.toLocal()}'}',
+                  textAlign: TextAlign.center,
+                  style: const TextStyle(
+                      fontStyle: FontStyle.italic, fontSize: 12),
+                )
               ],
             ),
       bottomSheet: Padding(

lib/pages/import_export.dart

@@ -40,7 +40,8 @@ class _ImportExportPageState extends State<ImportExportPage> {
 
     Future<List<List<String>>> addApps(List<String> urls) async {
       await settingsProvider.getInstallPermission();
-      List<dynamic> results = await sourceProvider.getApps(urls);
+      List<dynamic> results = await sourceProvider.getApps(urls,
+          ignoreUrls: appsProvider.apps.values.map((e) => e.app.url).toList());
       List<App> apps = results[0];
       Map<String, dynamic> errorsMap = results[1];
       for (var app in apps) {

lib/pages/settings.dart

@@ -169,41 +169,41 @@ class _SettingsPageState extends State<SettingsPage> {
                                     labelText:
                                         'Background Update Checking Interval'),
                                 value: settingsProvider.updateInterval,
-                                items: const [
+                                items: updateIntervals.map((e) {
-                                  DropdownMenuItem(
+                                  int displayNum = (e < 60
-                                    value: 15,
+                                          ? e
-                                    child: Text('15 Minutes'),
+                                          : e < 1440
-                                  ),
+                                              ? e / 60
-                                  DropdownMenuItem(
+                                              : e / 1440)
-                                    value: 30,
+                                      .round();
-                                    child: Text('30 Minutes'),
+                                  var displayUnit = (e < 60
-                                  ),
+                                      ? 'Minute'
-                                  DropdownMenuItem(
+                                      : e < 1440
-                                    value: 60,
+                                          ? 'Hour'
-                                    child: Text('1 Hour'),
+                                          : 'Day');
-                                  ),
+
-                                  DropdownMenuItem(
+                                  String display = e == 0
-                                    value: 360,
+                                      ? 'Never - Manual Only'
-                                    child: Text('6 Hours'),
+                                      : '$displayNum $displayUnit${displayNum == 1 ? '' : 's'}';
-                                  ),
+                                  return DropdownMenuItem(
-                                  DropdownMenuItem(
+                                      value: e, child: Text(display));
-                                    value: 720,
+                                }).toList(),
-                                    child: Text('12 Hours'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 1440,
-                                    child: Text('1 Day'),
-                                  ),
-                                  DropdownMenuItem(
-                                    value: 0,
-                                    child: Text('Never - Manual Only'),
-                                  ),
-                                ],
                                 onChanged: (value) {
                                   if (value != null) {
                                     settingsProvider.updateInterval = value;
                                   }
                                 }),
+                            const SizedBox(
+                              height: 8,
+                            ),
+                            Text(
+                              'Large App collections may require multiple cycles',
+                              style: Theme.of(context)
+                                  .textTheme
+                                  .labelMedium!
+                                  .merge(const TextStyle(
+                                      fontStyle: FontStyle.italic)),
+                            ),
                             const Spacer(),
                             Row(
                               mainAxisAlignment: MainAxisAlignment.center,

lib/providers/apps_provider.dart

@@ -297,12 +297,23 @@ class AppsProvider with ChangeNotifier {
     return null;
   }
 
-  Future<List<App>> checkUpdates() async {
+  Future<List<App>> checkUpdates({DateTime? ignoreAfter}) async {
     List<App> updates = [];
     if (!gettingUpdates) {
       gettingUpdates = true;
 
       List<String> appIds = apps.keys.toList();
+      if (ignoreAfter != null) {
+        appIds = appIds
+            .where((id) =>
+                apps[id]!.app.lastUpdateCheck != null &&
+                apps[id]!.app.lastUpdateCheck!.isBefore(ignoreAfter))
+            .toList();
+      }
+      appIds.sort((a, b) => (apps[a]!.app.lastUpdateCheck ??
+              DateTime.fromMicrosecondsSinceEpoch(0))
+          .compareTo(apps[b]!.app.lastUpdateCheck ??
+              DateTime.fromMicrosecondsSinceEpoch(0)));
       for (int i = 0; i < appIds.length; i++) {
         App? newApp = await getUpdate(appIds[i]);
         if (newApp != null) {

lib/providers/settings_provider.dart

@@ -13,6 +13,16 @@ enum SortColumnSettings { added, nameAuthor, authorName }
 
 enum SortOrderSettings { ascending, descending }
 
+const maxAPIRateLimitMinutes = 30;
+const minUpdateIntervalMinutes = maxAPIRateLimitMinutes + 30;
+const maxUpdateIntervalMinutes = 4320;
+List<int> updateIntervals = [15, 30, 60, 120, 180, 360, 720, 1440, 4320, 0]
+    .where((element) =>
+        (element >= minUpdateIntervalMinutes &&
+            element <= maxUpdateIntervalMinutes) ||
+        element == 0)
+    .toList();
+
 class SettingsProvider with ChangeNotifier {
   SharedPreferences? prefs;
 
@@ -45,7 +55,17 @@ class SettingsProvider with ChangeNotifier {
   }
 
   int get updateInterval {
-    return prefs?.getInt('updateInterval') ?? 1440;
+    var min = prefs?.getInt('updateInterval') ?? 180;
+    if (!updateIntervals.contains(min)) {
+      var temp = updateIntervals[0];
+      for (var i in updateIntervals) {
+        if (min > i && i != 0) {
+          temp = i;
+        }
+      }
+      min = temp;
+    }
+    return min;
   }
 
   set updateInterval(int min) {

lib/providers/source_provider.dart

@@ -38,6 +38,7 @@ class App {
   List<String> apkUrls = [];
   late int preferredApkIndex;
   late List<String> additionalData;
+  late DateTime? lastUpdateCheck;
   App(
       this.id,
       this.url,
@@ -47,7 +48,8 @@ class App {
       this.latestVersion,
       this.apkUrls,
       this.preferredApkIndex,
-      this.additionalData);
+      this.additionalData,
+      this.lastUpdateCheck);
 
   @override
   String toString() {
@@ -69,7 +71,10 @@ class App {
       json['preferredApkIndex'] == null ? 0 : json['preferredApkIndex'] as int,
       json['additionalData'] == null
           ? SourceProvider().getSource(json['url']).additionalDataDefaults
-          : List<String>.from(jsonDecode(json['additionalData'])));
+          : List<String>.from(jsonDecode(json['additionalData'])),
+      json['lastUpdateCheck'] == null
+          ? null
+          : DateTime.fromMicrosecondsSinceEpoch(json['lastUpdateCheck']));
 
   Map<String, dynamic> toJson() => {
         'id': id,
@@ -80,7 +85,8 @@ class App {
         'latestVersion': latestVersion,
         'apkUrls': jsonEncode(apkUrls),
         'preferredApkIndex': preferredApkIndex,
-        'additionalData': jsonEncode(additionalData)
+        'additionalData': jsonEncode(additionalData),
+        'lastUpdateCheck': lastUpdateCheck?.microsecondsSinceEpoch
       };
 }
 
@@ -195,15 +201,17 @@ class SourceProvider {
         apk.version,
         apk.apkUrls,
         apk.apkUrls.length - 1,
-        additionalData);
+        additionalData,
+        DateTime.now());
   }
 
   /// Returns a length 2 list, where the first element is a list of Apps and
   /// the second is a Map<String, dynamic> of URLs and errors
-  Future<List<dynamic>> getApps(List<String> urls) async {
+  Future<List<dynamic>> getApps(List<String> urls,
+      {List<String> ignoreUrls = const []}) async {
     List<App> apps = [];
     Map<String, dynamic> errors = {};
-    for (var url in urls) {
+    for (var url in urls.where((element) => !ignoreUrls.contains(element))) {
       try {
         var source = getSource(url);
         apps.add(await getApp(source, url, source.additionalDataDefaults));