peter/Obtainium
1
0
Fork 0
mirror of https://github.com/ImranR98/Obtainium.git synced 2025-07-13 05:16:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2265 from ImranR98/dev

Browse Source 
- Always follow redirects and store cookies between redirects, including for downloads —useful for https://xeiaso.net/blog/2025/anubis (https://github.com/ImranR98/Obtainium/issues/2264)
- Even more flexibility in the HTML source — JSON string extraction fallback (https://github.com/ImranR98/Obtainium/issues/2262)
This commit is contained in:
Imran
2025-04-27 04:15:43 +00:00
committed by GitHub
parent 98182d9873 f159c0bd44
commit e639758b15
5 changed files with 147 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
lib/app_sources/html.dart
View File

@ -1,3 +1,5 @@
import 'dart:convert';
import 'package:easy_localization/easy_localization.dart';
import 'package:html/parser.dart';
import 'package:http/http.dart';
@ -67,6 +69,27 @@ int compareAlphaNumeric(String a, String b) {
return aParts.length.compareTo(bParts.length);
}
List<String> collectAllStringsFromJSONObject(dynamic obj) {
List<String> extractor(dynamic obj) {
final results = <String>[];
if (obj is String) {
results.add(obj);
} else if (obj is List) {
for (final item in obj) {
results.addAll(extractor(item));
}
} else if (obj is Map<String, dynamic>) {
for (final value in obj.values) {
results.addAll(extractor(value));
}
}
return results;
}
return extractor(obj);
}
List<String> _splitAlphaNumeric(String s) {
List<String> parts = [];
StringBuffer sb = StringBuffer();
@ -95,6 +118,13 @@ bool _isNumeric(String s) {
return s.codeUnitAt(0) >= 48 && s.codeUnitAt(0) <= 57;
}
List<MapEntry<String, String>> getLinksInLines(String lines) => RegExp(
r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?')
.allMatches(lines)
.map((match) =>
MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? ''))
.toList();
// Given an HTTP response, grab some links according to the common additional settings
// (those that apply to intermediate and final steps)
Future<List<MapEntry<String, String>>> grabLinksCommon(
@ -114,12 +144,21 @@ Future<List<MapEntry<String, String>>> grabLinksCommon(
.map((e) => MapEntry(ensureAbsoluteUrl(e.key, res.request!.url), e.value))
.toList();
if (allLinks.isEmpty) {
allLinks = RegExp(
r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?')
.allMatches(res.body)
.map((match) =>
MapEntry(match.group(0)!, match.group(0)?.split('/').last ?? ''))
.toList();
allLinks = getLinksInLines(res.body);
}
if (allLinks.isEmpty) {
// Getting desperate
try {
var jsonStrings = collectAllStringsFromJSONObject(jsonDecode(res.body));
allLinks = getLinksInLines(jsonStrings.join('\n'));
if (allLinks.isEmpty) {
allLinks = getLinksInLines(jsonStrings.map((l) {
return ensureAbsoluteUrl(l, res.request!.url);
}).join('\n'));
}
} catch (e) {
//
}
}
List<MapEntry<String, String>> links = [];
bool skipSort = additionalSettings['skipSort'] == true;

49
lib/providers/apps_provider.dart
View File

@ -7,7 +7,6 @@ import 'dart:io';
import 'dart:math';
import 'package:battery_plus/battery_plus.dart';
import 'package:fluttertoast/fluttertoast.dart';
import 'package:http/http.dart' as http;
import 'package:crypto/crypto.dart';
import 'dart:typed_data';
@ -246,9 +245,9 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
var reqHeaders = headers ?? {};
var req = Request('GET', Uri.parse(url));
req.headers.addAll(reqHeaders);
var client = IOClient(createHttpClient(allowInsecure));
StreamedResponse response = await client.send(req);
var resHeaders = response.headers;
var headersClient = IOClient(createHttpClient(allowInsecure));
StreamedResponse headersResponse = await headersClient.send(req);
var resHeaders = headersResponse.headers;
// Use the headers to decide what the file extension is, and
// whether it supports partial downloads (range request), and
@ -276,21 +275,20 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
rangeFeatureEnabled =
resHeaders['accept-ranges']?.trim().toLowerCase() == 'bytes';
}
headersClient.close();
// If you have an existing file that is usable,
// decide whether you can use it (either return full or resume partial)
var fullContentLength = response.contentLength;
var fullContentLength = headersResponse.contentLength;
if (useExisting && downloadedFile.existsSync()) {
var length = downloadedFile.lengthSync();
if (fullContentLength == null || !rangeFeatureEnabled) {
// If there is no content length reported, assume it the existing file is fully downloaded
// Also if the range feature is not supported, don't trust the content length if any (#1542)
client.close();
return downloadedFile;
} else {
// Check if resume needed/possible
if (length == fullContentLength) {
client.close();
return downloadedFile;
}
if (length > fullContentLength) {
@ -330,7 +328,6 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
if (shouldReturn) {
logs?.add(
'Existing partial download completed - not repeating: ${tempDownloadedFile.uri.pathSegments.last}');
client.close();
return downloadedFile;
} else {
logs?.add(
@ -346,17 +343,18 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
: null;
int rangeStart = targetFileLength ?? 0;
IOSink? sink;
req = Request('GET', Uri.parse(url));
req.headers.addAll(reqHeaders);
if (rangeFeatureEnabled && fullContentLength != null && rangeStart > 0) {
client.close();
client = IOClient(createHttpClient(allowInsecure));
req = Request('GET', Uri.parse(url));
req.headers.addAll(reqHeaders);
req.headers.addAll({'range': 'bytes=$rangeStart-${fullContentLength - 1}'});
response = await client.send(req);
reqHeaders.addAll({'range': 'bytes=$rangeStart-${fullContentLength - 1}'});
sink = tempDownloadedFile.openWrite(mode: FileMode.writeOnlyAppend);
} else if (tempDownloadedFile.existsSync()) {
tempDownloadedFile.deleteSync(recursive: true);
}
var responseWithClient =
await sourceRequestStreamResponse('GET', url, reqHeaders, {});
HttpClient responseClient = responseWithClient.key;
HttpClientResponse response = responseWithClient.value;
sink ??= tempDownloadedFile.openWrite(mode: FileMode.writeOnly);
// Perform the download
@ -369,7 +367,8 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
const downloadUIUpdateInterval = Duration(milliseconds: 500);
const downloadBufferSize = 32 * 1024; // 32KB
final downloadBuffer = BytesBuilder();
await response.stream
await response
.asBroadcastStream()
.map((chunk) {
received += chunk.length;
final now = DateTime.now();
@ -407,31 +406,15 @@ Future<File> downloadFile(String url, String fileName, bool fileNameHasExt,
}
if (response.statusCode < 200 || response.statusCode > 299) {
tempDownloadedFile.deleteSync(recursive: true);
throw response.reasonPhrase ?? tr('unexpectedError');
throw response.reasonPhrase;
}
if (tempDownloadedFile.existsSync()) {
tempDownloadedFile.renameSync(downloadedFile.path);
}
client.close();
responseClient.close();
return downloadedFile;
}
Future<Map<String, String>> getHeaders(String url,
{Map<String, String>? headers, bool allowInsecure = false}) async {
var req = http.Request('GET', Uri.parse(url));
if (headers != null) {
req.headers.addAll(headers);
}
var client = IOClient(createHttpClient(allowInsecure));
var response = await client.send(req);
if (response.statusCode < 200 || response.statusCode > 299) {
throw ObtainiumError(response.reasonPhrase ?? tr('unexpectedError'));
}
var returnHeaders = response.headers;
client.close();
return returnHeaders;
}
Future<List<PackageInfo>> getAllInstalledInfo() async {
return await pm.getInstalledPackages() ?? [];
}

134
lib/providers/source_provider.dart
View File

@ -510,6 +510,75 @@ HttpClient createHttpClient(bool insecure) {
return client;
}
Future<MapEntry<HttpClient, HttpClientResponse>> sourceRequestStreamResponse(
String method,
String url,
Map<String, String>? requestHeaders,
Map<String, dynamic> additionalSettings,
{bool followRedirects = true,
Object? postBody}) async {
var currentUrl = Uri.parse(url);
var redirectCount = 0;
const maxRedirects = 10;
List<Cookie> cookies = [];
while (redirectCount < maxRedirects) {
var httpClient =
createHttpClient(additionalSettings['allowInsecure'] == true);
var request = await httpClient.openUrl(method, currentUrl);
if (requestHeaders != null) {
requestHeaders.forEach((key, value) {
request.headers.set(key, value);
});
}
request.cookies.addAll(cookies);
request.followRedirects = false;
if (postBody != null) {
request.headers.contentType = ContentType.json;
request.write(jsonEncode(postBody));
}
final response = await request.close();
if (followRedirects &&
(response.statusCode >= 300 && response.statusCode <= 399)) {
final location = response.headers.value(HttpHeaders.locationHeader);
if (location != null) {
currentUrl = Uri.parse(ensureAbsoluteUrl(location, currentUrl));
redirectCount++;
cookies = response.cookies;
httpClient.close();
continue;
}
}
return MapEntry(httpClient, response);
}
throw ObtainiumError('Too many redirects ($maxRedirects)');
}
Future<Response> httpClientResponseStreamToFinalResponse(
HttpClient httpClient,
String method,
String url,
HttpClientResponse response) async {
final bytes =
(await response.fold<BytesBuilder>(BytesBuilder(), (b, d) => b..add(d)))
.toBytes();
final headers = <String, String>{};
response.headers.forEach((name, values) {
headers[name] = values.join(', ');
});
httpClient.close();
return http.Response.bytes(
bytes,
response.statusCode,
headers: headers,
request: http.Request(method, Uri.parse(url)),
);
}
abstract class AppSource {
List<String> hosts = [];
bool hostChanged = false;
@ -567,64 +636,15 @@ abstract class AppSource {
Future<Response> sourceRequest(
String url, Map<String, dynamic> additionalSettings,
{bool followRedirects = true, Object? postBody}) async {
var method = postBody == null ? 'GET' : 'POST';
var requestHeaders = await getRequestHeaders(additionalSettings);
if (requestHeaders != null || followRedirects == false) {
var method = postBody == null ? 'GET' : 'POST';
var currentUrl = url;
var redirectCount = 0;
const maxRedirects = 10;
while (redirectCount < maxRedirects) {
var httpClient =
createHttpClient(additionalSettings['allowInsecure'] == true);
var request = await httpClient.openUrl(method, Uri.parse(currentUrl));
if (requestHeaders != null) {
requestHeaders.forEach((key, value) {
request.headers.set(key, value);
});
}
request.followRedirects = false;
if (postBody != null) {
request.headers.contentType = ContentType.json;
request.write(jsonEncode(postBody));
}
final response = await request.close();
if (followRedirects &&
(response.statusCode == 301 || response.statusCode == 302)) {
final location = response.headers.value(HttpHeaders.locationHeader);
if (location != null) {
currentUrl = location;
redirectCount++;
httpClient.close();
continue;
}
}
final bytes = (await response.fold<BytesBuilder>(
BytesBuilder(), (b, d) => b..add(d)))
.toBytes();
final headers = <String, String>{};
response.headers.forEach((name, values) {
headers[name] = values.join(', ');
});
httpClient.close();
return http.Response.bytes(
bytes,
response.statusCode,
headers: headers,
request: http.Request(method, Uri.parse(url)),
);
}
throw ObtainiumError('Too many redirects ($maxRedirects)');
} else {
return postBody == null
? http.get(Uri.parse(url))
: http.post(Uri.parse(url), body: jsonEncode(postBody));
}
var streamedResponseAndClient = await sourceRequestStreamResponse(
method, url, requestHeaders, additionalSettings);
return await httpClientResponseStreamToFinalResponse(
streamedResponseAndClient.key,
method,
url,
streamedResponseAndClient.value);
}
void runOnAddAppInputChange(String inputUrl) {

16
pubspec.lock
View File

@ -80,10 +80,10 @@ packages:
dependency: transitive
description:
name: archive
sha256: a7f37ff061d7abc2fcf213554b9dcaca713c5853afa5c065c44888bc9ccaf813
sha256: "2fde1607386ab523f7a36bb3e7edb43bd58e6edaf2ffb29d8a6d578b297fdbbd"
url: "https://pub.dev"
source: hosted
version: "4.0.6"
version: "4.0.7"
args:
dependency: transitive
description:
@ -564,10 +564,10 @@ packages:
dependency: "direct main"
description:
name: html
sha256: "9475be233c437f0e3637af55e7702cbbe5c23a68bd56e8a5fa2d426297b7c6c8"
sha256: "6d1264f2dffa1b1101c25a91dff0dc2daee4c18e87cd8538729773c073dbf602"
url: "https://pub.dev"
source: hosted
version: "0.15.5+1"
version: "0.15.6"
http:
dependency: "direct main"
description:
@ -1147,10 +1147,10 @@ packages:
dependency: transitive
description:
name: url_launcher_web
sha256: "3ba963161bd0fe395917ba881d320b9c4f6dd3c4a233da62ab18a5025c85f1e9"
sha256: "4bd2b7b4dc4d4d0b94e5babfffbca8eac1a126c7f3d6ecbc1a11013faa3abba2"
url: "https://pub.dev"
source: hosted
version: "2.4.0"
version: "2.4.1"
url_launcher_windows:
dependency: transitive
description:
@ -1203,10 +1203,10 @@ packages:
dependency: transitive
description:
name: webview_flutter_android
sha256: "5c3b6f992d123084903ec091b84f021c413a92a9af49038e4564a1b26c8452cf"
sha256: "6b0eae02b7604954b80ee9a29507ac38f5de74b712faa6fee33abc1cdedc1b21"
url: "https://pub.dev"
source: hosted
version: "4.4.1"
version: "4.4.2"
webview_flutter_platform_interface:
dependency: transitive
description:

2
pubspec.yaml
View File

@ -16,7 +16,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
# https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
# In Windows, build-name is used as the major, minor, and patch parts
# of the product and file versions while build-number is used as the build suffix.
version: 1.1.51+2308
version: 1.1.52+2309
environment:
sdk: ^3.6.0