diff --git a/lib/app_sources/html.dart b/lib/app_sources/html.dart index 6c8269c..af26cec 100644 --- a/lib/app_sources/html.dart +++ b/lib/app_sources/html.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:easy_localization/easy_localization.dart'; import 'package:html/parser.dart'; import 'package:http/http.dart'; @@ -67,6 +69,27 @@ int compareAlphaNumeric(String a, String b) { return aParts.length.compareTo(bParts.length); } +List collectAllStringsFromJSONObject(dynamic obj) { + List extractor(dynamic obj) { + final results = []; + if (obj is String) { + results.add(obj); + } else if (obj is List) { + for (final item in obj) { + results.addAll(extractor(item)); + } + } else if (obj is Map) { + for (final value in obj.values) { + results.addAll(extractor(value)); + } + } + + return results; + } + + return extractor(obj); +} + List _splitAlphaNumeric(String s) { List parts = []; StringBuffer sb = StringBuffer(); @@ -95,6 +118,13 @@ bool _isNumeric(String s) { return s.codeUnitAt(0) >= 48 && s.codeUnitAt(0) <= 57; } +List> getLinksInLines(String lines) => RegExp( + r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?') + .allMatches(lines) + .map((match) => + MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? '')) + .toList(); + // Given an HTTP response, grab some links according to the common additional settings // (those that apply to intermediate and final steps) Future>> grabLinksCommon( @@ -114,12 +144,21 @@ Future>> grabLinksCommon( .map((e) => MapEntry(ensureAbsoluteUrl(e.key, res.request!.url), e.value)) .toList(); if (allLinks.isEmpty) { - allLinks = RegExp( - r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?') - .allMatches(res.body) - .map((match) => - MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? '')) - .toList(); + allLinks = getLinksInLines(res.body); + } + if (allLinks.isEmpty) { + // Getting desperate + try { + var jsonStrings = collectAllStringsFromJSONObject(jsonDecode(res.body)); + allLinks = getLinksInLines(jsonStrings.join('\n')); + if (allLinks.isEmpty) { + allLinks = getLinksInLines(jsonStrings.map((l) { + return ensureAbsoluteUrl(l, res.request!.url); + }).join('\n')); + } + } catch (e) { + // + } } List> links = []; bool skipSort = additionalSettings['skipSort'] == true; diff --git a/lib/providers/apps_provider.dart b/lib/providers/apps_provider.dart index 833f1c7..3fe46be 100644 --- a/lib/providers/apps_provider.dart +++ b/lib/providers/apps_provider.dart @@ -7,7 +7,6 @@ import 'dart:io'; import 'dart:math'; import 'package:battery_plus/battery_plus.dart'; import 'package:fluttertoast/fluttertoast.dart'; -import 'package:http/http.dart' as http; import 'package:crypto/crypto.dart'; import 'dart:typed_data'; @@ -246,9 +245,9 @@ Future downloadFile(String url, String fileName, bool fileNameHasExt, var reqHeaders = headers ?? {}; var req = Request('GET', Uri.parse(url)); req.headers.addAll(reqHeaders); - var client = IOClient(createHttpClient(allowInsecure)); - StreamedResponse response = await client.send(req); - var resHeaders = response.headers; + var headersClient = IOClient(createHttpClient(allowInsecure)); + StreamedResponse headersResponse = await headersClient.send(req); + var resHeaders = headersResponse.headers; // Use the headers to decide what the file extension is, and // whether it supports partial downloads (range request), and @@ -276,21 +275,20 @@ Future downloadFile(String url, String fileName, bool fileNameHasExt, rangeFeatureEnabled = resHeaders['accept-ranges']?.trim().toLowerCase() == 'bytes'; } + headersClient.close(); // If you have an existing file that is usable, // decide whether you can use it (either return full or resume partial) - var fullContentLength = response.contentLength; + var fullContentLength = headersResponse.contentLength; if (useExisting && downloadedFile.existsSync()) { var length = downloadedFile.lengthSync(); if (fullContentLength == null || !rangeFeatureEnabled) { // If there is no content length reported, assume it the existing file is fully downloaded // Also if the range feature is not supported, don't trust the content length if any (#1542) - client.close(); return downloadedFile; } else { // Check if resume needed/possible if (length == fullContentLength) { - client.close(); return downloadedFile; } if (length > fullContentLength) { @@ -330,7 +328,6 @@ Future downloadFile(String url, String fileName, bool fileNameHasExt, if (shouldReturn) { logs?.add( 'Existing partial download completed - not repeating: ${tempDownloadedFile.uri.pathSegments.last}'); - client.close(); return downloadedFile; } else { logs?.add( @@ -346,17 +343,18 @@ Future downloadFile(String url, String fileName, bool fileNameHasExt, : null; int rangeStart = targetFileLength ?? 0; IOSink? sink; + req = Request('GET', Uri.parse(url)); + req.headers.addAll(reqHeaders); if (rangeFeatureEnabled && fullContentLength != null && rangeStart > 0) { - client.close(); - client = IOClient(createHttpClient(allowInsecure)); - req = Request('GET', Uri.parse(url)); - req.headers.addAll(reqHeaders); - req.headers.addAll({'range': 'bytes=$rangeStart-${fullContentLength - 1}'}); - response = await client.send(req); + reqHeaders.addAll({'range': 'bytes=$rangeStart-${fullContentLength - 1}'}); sink = tempDownloadedFile.openWrite(mode: FileMode.writeOnlyAppend); } else if (tempDownloadedFile.existsSync()) { tempDownloadedFile.deleteSync(recursive: true); } + var responseWithClient = + await sourceRequestStreamResponse('GET', url, reqHeaders, {}); + HttpClient responseClient = responseWithClient.key; + HttpClientResponse response = responseWithClient.value; sink ??= tempDownloadedFile.openWrite(mode: FileMode.writeOnly); // Perform the download @@ -369,7 +367,8 @@ Future downloadFile(String url, String fileName, bool fileNameHasExt, const downloadUIUpdateInterval = Duration(milliseconds: 500); const downloadBufferSize = 32 * 1024; // 32KB final downloadBuffer = BytesBuilder(); - await response.stream + await response + .asBroadcastStream() .map((chunk) { received += chunk.length; final now = DateTime.now(); @@ -407,31 +406,15 @@ Future downloadFile(String url, String fileName, bool fileNameHasExt, } if (response.statusCode < 200 || response.statusCode > 299) { tempDownloadedFile.deleteSync(recursive: true); - throw response.reasonPhrase ?? tr('unexpectedError'); + throw response.reasonPhrase; } if (tempDownloadedFile.existsSync()) { tempDownloadedFile.renameSync(downloadedFile.path); } - client.close(); + responseClient.close(); return downloadedFile; } -Future> getHeaders(String url, - {Map? headers, bool allowInsecure = false}) async { - var req = http.Request('GET', Uri.parse(url)); - if (headers != null) { - req.headers.addAll(headers); - } - var client = IOClient(createHttpClient(allowInsecure)); - var response = await client.send(req); - if (response.statusCode < 200 || response.statusCode > 299) { - throw ObtainiumError(response.reasonPhrase ?? tr('unexpectedError')); - } - var returnHeaders = response.headers; - client.close(); - return returnHeaders; -} - Future> getAllInstalledInfo() async { return await pm.getInstalledPackages() ?? []; } diff --git a/lib/providers/source_provider.dart b/lib/providers/source_provider.dart index 423b1d3..a70d0d3 100644 --- a/lib/providers/source_provider.dart +++ b/lib/providers/source_provider.dart @@ -510,6 +510,75 @@ HttpClient createHttpClient(bool insecure) { return client; } +Future> sourceRequestStreamResponse( + String method, + String url, + Map? requestHeaders, + Map additionalSettings, + {bool followRedirects = true, + Object? postBody}) async { + var currentUrl = Uri.parse(url); + var redirectCount = 0; + const maxRedirects = 10; + List cookies = []; + while (redirectCount < maxRedirects) { + var httpClient = + createHttpClient(additionalSettings['allowInsecure'] == true); + var request = await httpClient.openUrl(method, currentUrl); + if (requestHeaders != null) { + requestHeaders.forEach((key, value) { + request.headers.set(key, value); + }); + } + request.cookies.addAll(cookies); + request.followRedirects = false; + if (postBody != null) { + request.headers.contentType = ContentType.json; + request.write(jsonEncode(postBody)); + } + final response = await request.close(); + + if (followRedirects && + (response.statusCode >= 300 && response.statusCode <= 399)) { + final location = response.headers.value(HttpHeaders.locationHeader); + if (location != null) { + currentUrl = Uri.parse(ensureAbsoluteUrl(location, currentUrl)); + redirectCount++; + cookies = response.cookies; + httpClient.close(); + continue; + } + } + + return MapEntry(httpClient, response); + } + throw ObtainiumError('Too many redirects ($maxRedirects)'); + } + + Future httpClientResponseStreamToFinalResponse( + HttpClient httpClient, + String method, + String url, + HttpClientResponse response) async { + final bytes = + (await response.fold(BytesBuilder(), (b, d) => b..add(d))) + .toBytes(); + + final headers = {}; + response.headers.forEach((name, values) { + headers[name] = values.join(', '); + }); + + httpClient.close(); + + return http.Response.bytes( + bytes, + response.statusCode, + headers: headers, + request: http.Request(method, Uri.parse(url)), + ); + } + abstract class AppSource { List hosts = []; bool hostChanged = false; @@ -567,64 +636,15 @@ abstract class AppSource { Future sourceRequest( String url, Map additionalSettings, {bool followRedirects = true, Object? postBody}) async { + var method = postBody == null ? 'GET' : 'POST'; var requestHeaders = await getRequestHeaders(additionalSettings); - - if (requestHeaders != null || followRedirects == false) { - var method = postBody == null ? 'GET' : 'POST'; - var currentUrl = url; - var redirectCount = 0; - const maxRedirects = 10; - while (redirectCount < maxRedirects) { - var httpClient = - createHttpClient(additionalSettings['allowInsecure'] == true); - var request = await httpClient.openUrl(method, Uri.parse(currentUrl)); - if (requestHeaders != null) { - requestHeaders.forEach((key, value) { - request.headers.set(key, value); - }); - } - request.followRedirects = false; - if (postBody != null) { - request.headers.contentType = ContentType.json; - request.write(jsonEncode(postBody)); - } - final response = await request.close(); - - if (followRedirects && - (response.statusCode == 301 || response.statusCode == 302)) { - final location = response.headers.value(HttpHeaders.locationHeader); - if (location != null) { - currentUrl = location; - redirectCount++; - httpClient.close(); - continue; - } - } - - final bytes = (await response.fold( - BytesBuilder(), (b, d) => b..add(d))) - .toBytes(); - - final headers = {}; - response.headers.forEach((name, values) { - headers[name] = values.join(', '); - }); - - httpClient.close(); - - return http.Response.bytes( - bytes, - response.statusCode, - headers: headers, - request: http.Request(method, Uri.parse(url)), - ); - } - throw ObtainiumError('Too many redirects ($maxRedirects)'); - } else { - return postBody == null - ? http.get(Uri.parse(url)) - : http.post(Uri.parse(url), body: jsonEncode(postBody)); - } + var streamedResponseAndClient = await sourceRequestStreamResponse( + method, url, requestHeaders, additionalSettings); + return await httpClientResponseStreamToFinalResponse( + streamedResponseAndClient.key, + method, + url, + streamedResponseAndClient.value); } void runOnAddAppInputChange(String inputUrl) { diff --git a/pubspec.lock b/pubspec.lock index 77a7349..34730b9 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -80,10 +80,10 @@ packages: dependency: transitive description: name: archive - sha256: a7f37ff061d7abc2fcf213554b9dcaca713c5853afa5c065c44888bc9ccaf813 + sha256: "2fde1607386ab523f7a36bb3e7edb43bd58e6edaf2ffb29d8a6d578b297fdbbd" url: "https://pub.dev" source: hosted - version: "4.0.6" + version: "4.0.7" args: dependency: transitive description: @@ -564,10 +564,10 @@ packages: dependency: "direct main" description: name: html - sha256: "9475be233c437f0e3637af55e7702cbbe5c23a68bd56e8a5fa2d426297b7c6c8" + sha256: "6d1264f2dffa1b1101c25a91dff0dc2daee4c18e87cd8538729773c073dbf602" url: "https://pub.dev" source: hosted - version: "0.15.5+1" + version: "0.15.6" http: dependency: "direct main" description: @@ -1147,10 +1147,10 @@ packages: dependency: transitive description: name: url_launcher_web - sha256: "3ba963161bd0fe395917ba881d320b9c4f6dd3c4a233da62ab18a5025c85f1e9" + sha256: "4bd2b7b4dc4d4d0b94e5babfffbca8eac1a126c7f3d6ecbc1a11013faa3abba2" url: "https://pub.dev" source: hosted - version: "2.4.0" + version: "2.4.1" url_launcher_windows: dependency: transitive description: @@ -1203,10 +1203,10 @@ packages: dependency: transitive description: name: webview_flutter_android - sha256: "5c3b6f992d123084903ec091b84f021c413a92a9af49038e4564a1b26c8452cf" + sha256: "6b0eae02b7604954b80ee9a29507ac38f5de74b712faa6fee33abc1cdedc1b21" url: "https://pub.dev" source: hosted - version: "4.4.1" + version: "4.4.2" webview_flutter_platform_interface: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index accf96e..2ffab17 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -16,7 +16,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html # In Windows, build-name is used as the major, minor, and patch parts # of the product and file versions while build-number is used as the build suffix. -version: 1.1.51+2308 +version: 1.1.52+2309 environment: sdk: ^3.6.0