From 22fc894c9d94bb527f59c754f0ea0aaa0c747913 Mon Sep 17 00:00:00 2001 From: Austin Huang Date: Tue, 22 Dec 2020 11:54:52 -0500 Subject: [PATCH] implement app feed endpoint instead of browser --- .../asyncs/FeedPostFetchService.java | 33 ++++- .../repositories/FeedRepository.java | 14 ++ .../instagrabber/webservices/FeedService.java | 137 ++++++++++++++++++ .../webservices/GraphQLService.java | 36 ----- .../webservices/LocationService.java | 2 - .../webservices/ProfileService.java | 2 - .../instagrabber/webservices/TagsService.java | 2 - 7 files changed, 178 insertions(+), 48 deletions(-) create mode 100644 app/src/main/java/awais/instagrabber/repositories/FeedRepository.java create mode 100644 app/src/main/java/awais/instagrabber/webservices/FeedService.java diff --git a/app/src/main/java/awais/instagrabber/asyncs/FeedPostFetchService.java b/app/src/main/java/awais/instagrabber/asyncs/FeedPostFetchService.java index a3792921..65cf3e12 100644 --- a/app/src/main/java/awais/instagrabber/asyncs/FeedPostFetchService.java +++ b/app/src/main/java/awais/instagrabber/asyncs/FeedPostFetchService.java @@ -1,34 +1,55 @@ package awais.instagrabber.asyncs; +import android.util.Log; + +import java.util.ArrayList; import java.util.List; import awais.instagrabber.customviews.helpers.PostFetcher; import awais.instagrabber.interfaces.FetchListener; import awais.instagrabber.models.FeedModel; import awais.instagrabber.repositories.responses.PostsFetchResponse; -import awais.instagrabber.webservices.GraphQLService; +import awais.instagrabber.utils.Constants; +import awais.instagrabber.utils.CookieUtils; +import awais.instagrabber.webservices.FeedService; import awais.instagrabber.webservices.ServiceCallback; +import static awais.instagrabber.utils.Utils.settingsHelper; + public class FeedPostFetchService implements PostFetcher.PostFetchService { private static final String TAG = "FeedPostFetchService"; - private final GraphQLService graphQLService; + private final FeedService feedService; private String nextCursor; private boolean hasNextPage; public FeedPostFetchService() { - graphQLService = GraphQLService.getInstance(); + feedService = FeedService.getInstance(); } @Override public void fetch(final FetchListener> fetchListener) { - graphQLService.fetchFeed(25, nextCursor, new ServiceCallback() { + final List feedModels = new ArrayList<>(); + final String cookie = settingsHelper.getString(Constants.COOKIE); + final String csrfToken = CookieUtils.getCsrfTokenFromCookie(cookie); + feedModels.clear(); + feedService.fetch(csrfToken, nextCursor, new ServiceCallback() { @Override public void onSuccess(final PostsFetchResponse result) { - if (result == null) return; + if (result == null && feedModels.size() > 0) { + fetchListener.onResult(feedModels); + return; + } + else if (result == null) return; nextCursor = result.getNextCursor(); hasNextPage = result.hasNextPage(); + feedModels.addAll(result.getFeedModels()); if (fetchListener != null) { - fetchListener.onResult(result.getFeedModels()); + if (feedModels.size() < 15 && hasNextPage) { + feedService.fetch(csrfToken, nextCursor, this); + } + else { + fetchListener.onResult(feedModels); + } } } diff --git a/app/src/main/java/awais/instagrabber/repositories/FeedRepository.java b/app/src/main/java/awais/instagrabber/repositories/FeedRepository.java new file mode 100644 index 00000000..24ac5e01 --- /dev/null +++ b/app/src/main/java/awais/instagrabber/repositories/FeedRepository.java @@ -0,0 +1,14 @@ +package awais.instagrabber.repositories; + +import java.util.Map; + +import retrofit2.Call; +import retrofit2.http.FieldMap; +import retrofit2.http.FormUrlEncoded; +import retrofit2.http.POST; + +public interface FeedRepository { + @FormUrlEncoded + @POST("/api/v1/feed/timeline/") + Call fetch(@FieldMap final Map signedForm); +} diff --git a/app/src/main/java/awais/instagrabber/webservices/FeedService.java b/app/src/main/java/awais/instagrabber/webservices/FeedService.java new file mode 100644 index 00000000..fd491293 --- /dev/null +++ b/app/src/main/java/awais/instagrabber/webservices/FeedService.java @@ -0,0 +1,137 @@ +package awais.instagrabber.webservices; + +import android.os.Handler; +import android.util.Log; + +import androidx.annotation.NonNull; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.UUID; + +import awais.instagrabber.models.FeedModel; +import awais.instagrabber.models.PostChild; +import awais.instagrabber.models.ProfileModel; +import awais.instagrabber.models.enums.MediaItemType; +import awais.instagrabber.repositories.FeedRepository; +import awais.instagrabber.repositories.responses.PostsFetchResponse; +import awais.instagrabber.utils.Constants; +import awais.instagrabber.utils.ResponseBodyUtils; +import awais.instagrabber.utils.TextUtils; +import retrofit2.Call; +import retrofit2.Callback; +import retrofit2.Response; +import retrofit2.Retrofit; + +public class FeedService extends BaseService { + private static final String TAG = "FeedService"; + + private final FeedRepository repository; + + private static FeedService instance; + + private FeedService() { + final Retrofit retrofit = getRetrofitBuilder() + .baseUrl("https://i.instagram.com") + .build(); + repository = retrofit.create(FeedRepository.class); + } + + public static FeedService getInstance() { + if (instance == null) { + instance = new FeedService(); + } + return instance; + } + + public void fetch(final String csrfToken, + final String cursor, + final ServiceCallback callback) { + final Map form = new HashMap<>(); + form.put("_uuid", UUID.randomUUID().toString()); + form.put("_csrftoken", csrfToken); + form.put("phone_id", UUID.randomUUID().toString()); + form.put("device_id", UUID.randomUUID().toString()); + form.put("client_session_id", UUID.randomUUID().toString()); + form.put("is_prefetch", "0"); + form.put("timezone_offset", String.valueOf(TimeZone.getDefault().getRawOffset() / 1000)); + if (!TextUtils.isEmpty(cursor)) { + form.put("max_id", cursor); + form.put("reason", "pagination"); + } + else { + form.put("is_pull_to_refresh", "1"); + form.put("reason", "pull_to_refresh"); + } + final Call request = repository.fetch(form); + request.enqueue(new Callback() { + @Override + public void onResponse(@NonNull final Call call, @NonNull final Response response) { + try { + // Log.d(TAG, "onResponse: body: " + response.body()); + final PostsFetchResponse postsFetchResponse = parseResponse(response); + if (callback != null) { + callback.onSuccess(postsFetchResponse); + } + } catch (JSONException e) { + Log.e(TAG, "onResponse", e); + if (callback != null) { + callback.onFailure(e); + } + } + } + + @Override + public void onFailure(@NonNull final Call call, @NonNull final Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); + + } + + @NonNull + private PostsFetchResponse parseResponse(@NonNull final Response response) throws JSONException { + if (TextUtils.isEmpty(response.body())) { + Log.e(TAG, "parseResponse: feed response body is empty with status code: " + response.code()); + return new PostsFetchResponse(Collections.emptyList(), false, null); + } + return parseResponseBody(response.body()); + } + + @NonNull + private PostsFetchResponse parseResponseBody(@NonNull final String body) + throws JSONException { + final JSONObject root = new JSONObject(body); + final boolean moreAvailable = root.optBoolean("more_available"); + final String nextMaxId = root.optString("next_max_id"); + final JSONArray feedItems = root.optJSONArray("items"); + final List feedModels = new ArrayList<>(); + for (int i = 0; i < feedItems.length(); ++i) { + final JSONObject itemJson = feedItems.optJSONObject(i); + if (itemJson == null || itemJson.has("injected") + ) { + continue; + } + final FeedModel feedModel = ResponseBodyUtils.parseItem(itemJson); + if (feedModel != null) { + feedModels.add(feedModel); + } + } + return new PostsFetchResponse(feedModels, moreAvailable, nextMaxId); + } +} \ No newline at end of file diff --git a/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java b/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java index 94d9f38f..d21c7870 100644 --- a/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java +++ b/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java @@ -88,42 +88,6 @@ public class GraphQLService extends BaseService { }); } - public void fetchFeed(final int maxItemsToLoad, - final String cursor, - final ServiceCallback callback) { - if (loadFromMock) { - final Handler handler = new Handler(); - handler.postDelayed(() -> { - final ClassLoader classLoader = getClass().getClassLoader(); - if (classLoader == null) { - Log.e(TAG, "fetch: classLoader is null!"); - return; - } - try (InputStream resourceAsStream = classLoader.getResourceAsStream("feed_response.json"); - Reader in = new InputStreamReader(resourceAsStream, StandardCharsets.UTF_8)) { - final int bufferSize = 1024; - final char[] buffer = new char[bufferSize]; - final StringBuilder out = new StringBuilder(); - int charsRead; - while ((charsRead = in.read(buffer, 0, buffer.length)) > 0) { - out.append(buffer, 0, charsRead); - } - callback.onSuccess(parseResponseBody(out.toString(), Constants.EXTRAS_USER, "edge_web_feed_timeline")); - } catch (IOException | JSONException e) { - Log.e(TAG, "fetch: ", e); - } - }, 1000); - return; - } - fetch("c699b185975935ae2a457f24075de8c7", - "{\"fetch_media_item_count\":" + maxItemsToLoad + "," + - "\"fetch_like\":3,\"has_stories\":false,\"has_stories\":false,\"has_threaded_comments\":true," + - "\"fetch_media_item_cursor\":\"" + (cursor == null ? "" : cursor) + "\"}", - Constants.EXTRAS_USER, - "edge_web_feed_timeline", - callback); - } - public void fetchLocationPosts(@NonNull final String locationId, final String maxId, final ServiceCallback callback) { diff --git a/app/src/main/java/awais/instagrabber/webservices/LocationService.java b/app/src/main/java/awais/instagrabber/webservices/LocationService.java index d0bbe7c2..a9b40981 100644 --- a/app/src/main/java/awais/instagrabber/webservices/LocationService.java +++ b/app/src/main/java/awais/instagrabber/webservices/LocationService.java @@ -89,8 +89,6 @@ public class LocationService extends BaseService { final JSONObject root = new JSONObject(body); final boolean moreAvailable = root.optBoolean("more_available"); final String nextMaxId = root.optString("next_max_id"); - final int numResults = root.optInt("num_results"); - final String status = root.optString("status"); final JSONArray itemsJson = root.optJSONArray("items"); final List items = parseItems(itemsJson); return new PostsFetchResponse( diff --git a/app/src/main/java/awais/instagrabber/webservices/ProfileService.java b/app/src/main/java/awais/instagrabber/webservices/ProfileService.java index 8e3c4c6a..056463d9 100644 --- a/app/src/main/java/awais/instagrabber/webservices/ProfileService.java +++ b/app/src/main/java/awais/instagrabber/webservices/ProfileService.java @@ -228,8 +228,6 @@ public class ProfileService extends BaseService { final JSONObject root = new JSONObject(body); final boolean moreAvailable = root.optBoolean("more_available"); final String nextMaxId = root.optString("next_max_id"); - final int numResults = root.optInt("num_results"); - final String status = root.optString("status"); final JSONArray itemsJson = root.optJSONArray("items"); final List items = parseItems(itemsJson, isInMedia); return new PostsFetchResponse( diff --git a/app/src/main/java/awais/instagrabber/webservices/TagsService.java b/app/src/main/java/awais/instagrabber/webservices/TagsService.java index 5786e82f..7f3dc490 100644 --- a/app/src/main/java/awais/instagrabber/webservices/TagsService.java +++ b/app/src/main/java/awais/instagrabber/webservices/TagsService.java @@ -158,8 +158,6 @@ public class TagsService extends BaseService { final JSONObject root = new JSONObject(body); final boolean moreAvailable = root.optBoolean("more_available"); final String nextMaxId = root.optString("next_max_id"); - final int numResults = root.optInt("num_results"); - final String status = root.optString("status"); final JSONArray itemsJson = root.optJSONArray("items"); final List items = parseItems(itemsJson); return new PostsFetchResponse(