Merge pull request #2265 from ImranR98/dev

- Always follow redirects and store cookies between redirects, including for downloads —useful for https://xeiaso.net/blog/2025/anubis (https://github.com/ImranR98/Obtainium/issues/2264)
- Even more flexibility in the HTML source — JSON string extraction fallback (https://github.com/ImranR98/Obtainium/issues/2262)

lib/app_sources/html.dart

@@ -1,3 +1,5 @@
+import 'dart:convert';
+
 import 'package:easy_localization/easy_localization.dart';
 import 'package:html/parser.dart';
 import 'package:http/http.dart';
@@ -67,6 +69,27 @@ int compareAlphaNumeric(String a, String b) {
   return aParts.length.compareTo(bParts.length);
 }
 
+List<String> collectAllStringsFromJSONObject(dynamic obj) {
+  List<String> extractor(dynamic obj) {
+    final results = <String>[];
+    if (obj is String) {
+      results.add(obj);
+    } else if (obj is List) {
+      for (final item in obj) {
+        results.addAll(extractor(item));
+      }
+    } else if (obj is Map<String, dynamic>) {
+      for (final value in obj.values) {
+        results.addAll(extractor(value));
+      }
+    }
+
+    return results;
+  }
+
+  return extractor(obj);
+}
+
 List<String> _splitAlphaNumeric(String s) {
   List<String> parts = [];
   StringBuffer sb = StringBuffer();
@@ -95,6 +118,13 @@ bool _isNumeric(String s) {
   return s.codeUnitAt(0) >= 48 && s.codeUnitAt(0) <= 57;
 }
 
+List<MapEntry<String, String>> getLinksInLines(String lines) => RegExp(
+        r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?')
+    .allMatches(lines)
+    .map((match) =>
+        MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? ''))
+    .toList();
+
 // Given an HTTP response, grab some links according to the common additional settings
 // (those that apply to intermediate and final steps)
 Future<List<MapEntry<String, String>>> grabLinksCommon(
@@ -114,12 +144,21 @@ Future<List<MapEntry<String, String>>> grabLinksCommon(
       .map((e) => MapEntry(ensureAbsoluteUrl(e.key, res.request!.url), e.value))
       .toList();
   if (allLinks.isEmpty) {
-    allLinks = RegExp(
+    allLinks = getLinksInLines(res.body);
-            r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?')
+  }
-        .allMatches(res.body)
+  if (allLinks.isEmpty) {
-        .map((match) =>
+    // Getting desperate
-            MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? ''))
+    try {
-        .toList();
+      var jsonStrings = collectAllStringsFromJSONObject(jsonDecode(res.body));
+      allLinks = getLinksInLines(jsonStrings.join('\n'));
+      if (allLinks.isEmpty) {
+        allLinks = getLinksInLines(jsonStrings.map((l) {
+          return ensureAbsoluteUrl(l, res.request!.url);
+        }).join('\n'));
+      }
+    } catch (e) {
+      //
+    }
   }
   List<MapEntry<String, String>> links = [];
   bool skipSort = additionalSettings['skipSort'] == true;

lib/providers/apps_provider.dart

@@ -7,7 +7,6 @@ import 'dart:io';
 import 'dart:math';
 import 'package:battery_plus/battery_plus.dart';
 import 'package:fluttertoast/fluttertoast.dart';
-import 'package:http/http.dart' as http;
 import 'package:crypto/crypto.dart';
 import 'dart:typed_data';
 
@@ -246,9 +245,9 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
   var reqHeaders = headers ?? {};
   var req = Request('GET', Uri.parse(url));
   req.headers.addAll(reqHeaders);
-  var client = IOClient(createHttpClient(allowInsecure));
+  var headersClient = IOClient(createHttpClient(allowInsecure));
-  StreamedResponse response = await client.send(req);
+  StreamedResponse headersResponse = await headersClient.send(req);
-  var resHeaders = response.headers;
+  var resHeaders = headersResponse.headers;
 
   // Use the headers to decide what the file extension is, and
   // whether it supports partial downloads (range request), and
@@ -276,21 +275,20 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
     rangeFeatureEnabled =
         resHeaders['accept-ranges']?.trim().toLowerCase() == 'bytes';
   }
+  headersClient.close();
 
   // If you have an existing file that is usable,
   // decide whether you can use it (either return full or resume partial)
-  var fullContentLength = response.contentLength;
+  var fullContentLength = headersResponse.contentLength;
   if (useExisting && downloadedFile.existsSync()) {
     var length = downloadedFile.lengthSync();
     if (fullContentLength == null || !rangeFeatureEnabled) {
       // If there is no content length reported, assume it the existing file is fully downloaded
       // Also if the range feature is not supported, don't trust the content length if any (#1542)
-      client.close();
       return downloadedFile;
     } else {
       // Check if resume needed/possible
       if (length == fullContentLength) {
-        client.close();
         return downloadedFile;
       }
       if (length > fullContentLength) {
@@ -330,7 +328,6 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
     if (shouldReturn) {
       logs?.add(
           'Existing partial download completed - not repeating: ${tempDownloadedFile.uri.pathSegments.last}');
-      client.close();
       return downloadedFile;
     } else {
       logs?.add(
@@ -346,17 +343,18 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
       : null;
   int rangeStart = targetFileLength ?? 0;
   IOSink? sink;
-  if (rangeFeatureEnabled && fullContentLength != null && rangeStart > 0) {
-    client.close();
-    client = IOClient(createHttpClient(allowInsecure));
   req = Request('GET', Uri.parse(url));
   req.headers.addAll(reqHeaders);
-    req.headers.addAll({'range': 'bytes=$rangeStart-${fullContentLength - 1}'});
+  if (rangeFeatureEnabled && fullContentLength != null && rangeStart > 0) {
-    response = await client.send(req);
+    reqHeaders.addAll({'range': 'bytes=$rangeStart-${fullContentLength - 1}'});
     sink = tempDownloadedFile.openWrite(mode: FileMode.writeOnlyAppend);
   } else if (tempDownloadedFile.existsSync()) {
     tempDownloadedFile.deleteSync(recursive: true);
   }
+  var responseWithClient =
+      await sourceRequestStreamResponse('GET', url, reqHeaders, {});
+  HttpClient responseClient = responseWithClient.key;
+  HttpClientResponse response = responseWithClient.value;
   sink ??= tempDownloadedFile.openWrite(mode: FileMode.writeOnly);
 
   // Perform the download
@@ -369,7 +367,8 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
   const downloadUIUpdateInterval = Duration(milliseconds: 500);
   const downloadBufferSize = 32 * 1024; // 32KB
   final downloadBuffer = BytesBuilder();
-  await response.stream
+  await response
+      .asBroadcastStream()
       .map((chunk) {
         received += chunk.length;
         final now = DateTime.now();
@@ -407,31 +406,15 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
   }
   if (response.statusCode < 200 || response.statusCode > 299) {
     tempDownloadedFile.deleteSync(recursive: true);
-    throw response.reasonPhrase ?? tr('unexpectedError');
+    throw response.reasonPhrase;
   }
   if (tempDownloadedFile.existsSync()) {
     tempDownloadedFile.renameSync(downloadedFile.path);
   }
-  client.close();
+  responseClient.close();
   return downloadedFile;
 }
 
-Future<Map<String, String>> getHeaders(String url,
-    {Map<String, String>? headers, bool allowInsecure = false}) async {
-  var req = http.Request('GET', Uri.parse(url));
-  if (headers != null) {
-    req.headers.addAll(headers);
-  }
-  var client = IOClient(createHttpClient(allowInsecure));
-  var response = await client.send(req);
-  if (response.statusCode < 200 || response.statusCode > 299) {
-    throw ObtainiumError(response.reasonPhrase ?? tr('unexpectedError'));
-  }
-  var returnHeaders = response.headers;
-  client.close();
-  return returnHeaders;
-}
-
 Future<List<PackageInfo>> getAllInstalledInfo() async {
   return await pm.getInstalledPackages() ?? [];
 }

lib/providers/source_provider.dart

@@ -510,6 +510,75 @@ HttpClient createHttpClient(bool insecure) {
   return client;
 }
 
+Future<MapEntry<HttpClient, HttpClientResponse>> sourceRequestStreamResponse(
+      String method,
+      String url,
+      Map<String, String>? requestHeaders,
+      Map<String, dynamic> additionalSettings,
+      {bool followRedirects = true,
+      Object? postBody}) async {
+    var currentUrl = Uri.parse(url);
+    var redirectCount = 0;
+    const maxRedirects = 10;
+    List<Cookie> cookies = [];
+    while (redirectCount < maxRedirects) {
+      var httpClient =
+          createHttpClient(additionalSettings['allowInsecure'] == true);
+      var request = await httpClient.openUrl(method, currentUrl);
+      if (requestHeaders != null) {
+        requestHeaders.forEach((key, value) {
+          request.headers.set(key, value);
+        });
+      }
+      request.cookies.addAll(cookies);
+      request.followRedirects = false;
+      if (postBody != null) {
+        request.headers.contentType = ContentType.json;
+        request.write(jsonEncode(postBody));
+      }
+      final response = await request.close();
+
+      if (followRedirects &&
+          (response.statusCode >= 300 && response.statusCode <= 399)) {
+        final location = response.headers.value(HttpHeaders.locationHeader);
+        if (location != null) {
+          currentUrl = Uri.parse(ensureAbsoluteUrl(location, currentUrl));
+          redirectCount++;
+          cookies = response.cookies;
+          httpClient.close();
+          continue;
+        }
+      }
+
+      return MapEntry(httpClient, response);
+    }
+    throw ObtainiumError('Too many redirects ($maxRedirects)');
+  }
+
+  Future<Response> httpClientResponseStreamToFinalResponse(
+      HttpClient httpClient,
+      String method,
+      String url,
+      HttpClientResponse response) async {
+    final bytes =
+        (await response.fold<BytesBuilder>(BytesBuilder(), (b, d) => b..add(d)))
+            .toBytes();
+
+    final headers = <String, String>{};
+    response.headers.forEach((name, values) {
+      headers[name] = values.join(', ');
+    });
+
+    httpClient.close();
+
+    return http.Response.bytes(
+      bytes,
+      response.statusCode,
+      headers: headers,
+      request: http.Request(method, Uri.parse(url)),
+    );
+  }
+
 abstract class AppSource {
   List<String> hosts = [];
   bool hostChanged = false;
@@ -567,64 +636,15 @@ abstract class AppSource {
   Future<Response> sourceRequest(
       String url, Map<String, dynamic> additionalSettings,
       {bool followRedirects = true, Object? postBody}) async {
-    var requestHeaders = await getRequestHeaders(additionalSettings);
-
-    if (requestHeaders != null || followRedirects == false) {
     var method = postBody == null ? 'GET' : 'POST';
-      var currentUrl = url;
+    var requestHeaders = await getRequestHeaders(additionalSettings);
-      var redirectCount = 0;
+    var streamedResponseAndClient = await sourceRequestStreamResponse(
-      const maxRedirects = 10;
+        method, url, requestHeaders, additionalSettings);
-      while (redirectCount < maxRedirects) {
+    return await httpClientResponseStreamToFinalResponse(
-        var httpClient =
+        streamedResponseAndClient.key,
-            createHttpClient(additionalSettings['allowInsecure'] == true);
+        method,
-        var request = await httpClient.openUrl(method, Uri.parse(currentUrl));
+        url,
-        if (requestHeaders != null) {
+        streamedResponseAndClient.value);
-          requestHeaders.forEach((key, value) {
-            request.headers.set(key, value);
-          });
-        }
-        request.followRedirects = false;
-        if (postBody != null) {
-          request.headers.contentType = ContentType.json;
-          request.write(jsonEncode(postBody));
-        }
-        final response = await request.close();
-
-        if (followRedirects &&
-            (response.statusCode == 301 || response.statusCode == 302)) {
-          final location = response.headers.value(HttpHeaders.locationHeader);
-          if (location != null) {
-            currentUrl = location;
-            redirectCount++;
-            httpClient.close();
-            continue;
-          }
-        }
-
-        final bytes = (await response.fold<BytesBuilder>(
-                BytesBuilder(), (b, d) => b..add(d)))
-            .toBytes();
-
-        final headers = <String, String>{};
-        response.headers.forEach((name, values) {
-          headers[name] = values.join(', ');
-        });
-
-        httpClient.close();
-
-        return http.Response.bytes(
-          bytes,
-          response.statusCode,
-          headers: headers,
-          request: http.Request(method, Uri.parse(url)),
-        );
-      }
-      throw ObtainiumError('Too many redirects ($maxRedirects)');
-    } else {
-      return postBody == null
-          ? http.get(Uri.parse(url))
-          : http.post(Uri.parse(url), body: jsonEncode(postBody));
-    }
   }
 
   void runOnAddAppInputChange(String inputUrl) {

pubspec.lock

@@ -80,10 +80,10 @@ packages:
     dependency: transitive
     description:
       name: archive
-      sha256: a7f37ff061d7abc2fcf213554b9dcaca713c5853afa5c065c44888bc9ccaf813
+      sha256: "2fde1607386ab523f7a36bb3e7edb43bd58e6edaf2ffb29d8a6d578b297fdbbd"
       url: "https://pub.dev"
     source: hosted
-    version: "4.0.6"
+    version: "4.0.7"
   args:
     dependency: transitive
     description:
@@ -564,10 +564,10 @@ packages:
     dependency: "direct main"
     description:
       name: html
-      sha256: "9475be233c437f0e3637af55e7702cbbe5c23a68bd56e8a5fa2d426297b7c6c8"
+      sha256: "6d1264f2dffa1b1101c25a91dff0dc2daee4c18e87cd8538729773c073dbf602"
       url: "https://pub.dev"
     source: hosted
-    version: "0.15.5+1"
+    version: "0.15.6"
   http:
     dependency: "direct main"
     description:
@@ -1147,10 +1147,10 @@ packages:
     dependency: transitive
     description:
       name: url_launcher_web
-      sha256: "3ba963161bd0fe395917ba881d320b9c4f6dd3c4a233da62ab18a5025c85f1e9"
+      sha256: "4bd2b7b4dc4d4d0b94e5babfffbca8eac1a126c7f3d6ecbc1a11013faa3abba2"
       url: "https://pub.dev"
     source: hosted
-    version: "2.4.0"
+    version: "2.4.1"
   url_launcher_windows:
     dependency: transitive
     description:
@@ -1203,10 +1203,10 @@ packages:
     dependency: transitive
     description:
       name: webview_flutter_android
-      sha256: "5c3b6f992d123084903ec091b84f021c413a92a9af49038e4564a1b26c8452cf"
+      sha256: "6b0eae02b7604954b80ee9a29507ac38f5de74b712faa6fee33abc1cdedc1b21"
       url: "https://pub.dev"
     source: hosted
-    version: "4.4.1"
+    version: "4.4.2"
   webview_flutter_platform_interface:
     dependency: transitive
     description:

pubspec.yaml

@@ -16,7 +16,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
 # In Windows, build-name is used as the major, minor, and patch parts
 # of the product and file versions while build-number is used as the build suffix.
-version: 1.1.51+2308
+version: 1.1.52+2309
 
 environment:
   sdk: ^3.6.0