|
|
|
@ -25,29 +25,50 @@ import com.android.volley.Response.Listener; |
|
|
|
|
import org.floens.chan.Chan; |
|
|
|
|
import org.floens.chan.chan.ChanUrls; |
|
|
|
|
import org.floens.chan.core.database.DatabaseManager; |
|
|
|
|
import org.floens.chan.core.database.DatabaseSavedReplyManager; |
|
|
|
|
import org.floens.chan.core.manager.FilterEngine; |
|
|
|
|
import org.floens.chan.core.model.Filter; |
|
|
|
|
import org.floens.chan.core.model.Loadable; |
|
|
|
|
import org.floens.chan.core.model.Post; |
|
|
|
|
import org.floens.chan.utils.Logger; |
|
|
|
|
import org.floens.chan.utils.Time; |
|
|
|
|
|
|
|
|
|
import java.util.ArrayList; |
|
|
|
|
import java.util.HashMap; |
|
|
|
|
import java.util.List; |
|
|
|
|
import java.util.Map; |
|
|
|
|
import java.util.concurrent.Callable; |
|
|
|
|
import java.util.concurrent.ExecutionException; |
|
|
|
|
import java.util.concurrent.ExecutorService; |
|
|
|
|
import java.util.concurrent.Executors; |
|
|
|
|
import java.util.concurrent.Future; |
|
|
|
|
|
|
|
|
|
public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanReaderResponse> { |
|
|
|
|
private static final String TAG = "ChanReaderRequest"; |
|
|
|
|
private static final boolean LOG_TIMING = false; |
|
|
|
|
|
|
|
|
|
private static final int THREAD_COUNT; |
|
|
|
|
private static ExecutorService EXECUTOR; |
|
|
|
|
|
|
|
|
|
static { |
|
|
|
|
THREAD_COUNT = Runtime.getRuntime().availableProcessors(); |
|
|
|
|
EXECUTOR = Executors.newFixedThreadPool(THREAD_COUNT); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private Loadable loadable; |
|
|
|
|
private List<Post> cached; |
|
|
|
|
private Post op; |
|
|
|
|
private FilterEngine filterEngine; |
|
|
|
|
private DatabaseManager databaseManager; |
|
|
|
|
private DatabaseSavedReplyManager databaseSavedReplyManager; |
|
|
|
|
|
|
|
|
|
private List<Filter> filters; |
|
|
|
|
private long startLoad; |
|
|
|
|
|
|
|
|
|
private ChanReaderRequest(String url, Listener<ChanReaderResponse> listener, ErrorListener errorListener) { |
|
|
|
|
super(url, listener, errorListener); |
|
|
|
|
filterEngine = FilterEngine.getInstance(); |
|
|
|
|
databaseManager = Chan.getDatabaseManager(); |
|
|
|
|
databaseSavedReplyManager = databaseManager.getDatabaseSavedReplyManager(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public static ChanReaderRequest newInstance(Loadable loadable, List<Post> cached, Listener<ChanReaderResponse> listener, ErrorListener errorListener) { |
|
|
|
@ -87,6 +108,8 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
request.startLoad = Time.startTiming(); |
|
|
|
|
|
|
|
|
|
return request; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -97,84 +120,172 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public ChanReaderResponse readJson(JsonReader reader) throws Exception { |
|
|
|
|
List<Post> list; |
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("Network", startLoad); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
long load = Time.startTiming(); |
|
|
|
|
|
|
|
|
|
ProcessingQueue processing = new ProcessingQueue(); |
|
|
|
|
|
|
|
|
|
Map<Integer, Post> cachedByNo = new HashMap<>(); |
|
|
|
|
for (int i = 0; i < cached.size(); i++) { |
|
|
|
|
Post cache = cached.get(i); |
|
|
|
|
cachedByNo.put(cache.no, cache); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (loadable.isThreadMode()) { |
|
|
|
|
list = loadThread(reader); |
|
|
|
|
loadThread(reader, processing, cachedByNo); |
|
|
|
|
} else if (loadable.isCatalogMode()) { |
|
|
|
|
list = loadCatalog(reader); |
|
|
|
|
loadCatalog(reader, processing, cachedByNo); |
|
|
|
|
} else { |
|
|
|
|
throw new IllegalArgumentException("Unknown mode"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("Load json", load); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
List<Post> list = parsePosts(processing); |
|
|
|
|
return processPosts(list); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private ChanReaderResponse processPosts(List<Post> serverList) throws Exception { |
|
|
|
|
// Concurrently parses the new posts with an executor
|
|
|
|
|
private List<Post> parsePosts(ProcessingQueue queue) throws InterruptedException, ExecutionException { |
|
|
|
|
long parsePosts = Time.startTiming(); |
|
|
|
|
|
|
|
|
|
List<Post> total = new ArrayList<>(); |
|
|
|
|
|
|
|
|
|
total.addAll(queue.cached); |
|
|
|
|
|
|
|
|
|
List<Callable<Post>> tasks = new ArrayList<>(queue.toParse.size()); |
|
|
|
|
for (int i = 0; i < queue.toParse.size(); i++) { |
|
|
|
|
Post post = queue.toParse.get(i); |
|
|
|
|
tasks.add(new PostParseCallable(filterEngine, filters, databaseSavedReplyManager, post)); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (!tasks.isEmpty()) { |
|
|
|
|
List<Future<Post>> futures = EXECUTOR.invokeAll(tasks); |
|
|
|
|
for (int i = 0; i < futures.size(); i++) { |
|
|
|
|
Future<Post> future = futures.get(i); |
|
|
|
|
Post parsedPost = future.get(); |
|
|
|
|
if (parsedPost != null) { |
|
|
|
|
total.add(parsedPost); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("Parse posts with " + THREAD_COUNT + " threads", parsePosts); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return total; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private ChanReaderResponse processPosts(List<Post> serverPosts) throws Exception { |
|
|
|
|
ChanReaderResponse response = new ChanReaderResponse(); |
|
|
|
|
response.posts = new ArrayList<>(serverList.size()); |
|
|
|
|
response.posts = new ArrayList<>(serverPosts.size()); |
|
|
|
|
response.op = op; |
|
|
|
|
|
|
|
|
|
List<Post> cachedPosts = new ArrayList<>(); |
|
|
|
|
List<Post> newPosts = new ArrayList<>(); |
|
|
|
|
if (cached.size() > 0) { |
|
|
|
|
long deleteCheck = Time.startTiming(); |
|
|
|
|
// Add all posts that were parsed before
|
|
|
|
|
response.posts.addAll(cached); |
|
|
|
|
cachedPosts.addAll(cached); |
|
|
|
|
|
|
|
|
|
Map<Integer, Post> cachedPostsByNo = new HashMap<>(); |
|
|
|
|
for (int i = 0; i < cachedPosts.size(); i++) { |
|
|
|
|
Post post = cachedPosts.get(i); |
|
|
|
|
cachedPostsByNo.put(post.no, post); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Map<Integer, Post> serverPostsByNo = new HashMap<>(); |
|
|
|
|
for (int i = 0; i < serverPosts.size(); i++) { |
|
|
|
|
Post post = serverPosts.get(i); |
|
|
|
|
serverPostsByNo.put(post.no, post); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// If there's a cached post but it's not in the list received from the server, mark it as deleted
|
|
|
|
|
if (loadable.isThreadMode()) { |
|
|
|
|
boolean serverHas; |
|
|
|
|
for (Post cache : cached) { |
|
|
|
|
serverHas = false; |
|
|
|
|
for (Post b : serverList) { |
|
|
|
|
if (b.no == cache.no) { |
|
|
|
|
serverHas = true; |
|
|
|
|
break; |
|
|
|
|
for (int i = 0; i < cachedPosts.size(); i++) { |
|
|
|
|
Post cachedPost = cachedPosts.get(i); |
|
|
|
|
cachedPost.deleted.set(!serverPostsByNo.containsKey(cachedPost.no)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("Delete check", deleteCheck); |
|
|
|
|
} |
|
|
|
|
long newCheck = Time.startTiming(); |
|
|
|
|
|
|
|
|
|
cache.deleted.set(!serverHas); |
|
|
|
|
// If there's a post in the list from the server, that's not in the cached list, add it.
|
|
|
|
|
for (int i = 0; i < serverPosts.size(); i++) { |
|
|
|
|
Post serverPost = serverPosts.get(i); |
|
|
|
|
if (!cachedPostsByNo.containsKey(serverPost.no)) { |
|
|
|
|
newPosts.add(serverPost); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("New check", newCheck); |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
newPosts.addAll(serverPosts); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// If there's a post in the list from the server, that's not in the cached list, add it.
|
|
|
|
|
boolean known; |
|
|
|
|
for (Post post : serverList) { |
|
|
|
|
known = false; |
|
|
|
|
List<Post> allPosts = new ArrayList<>(cachedPosts.size() + newPosts.size()); |
|
|
|
|
allPosts.addAll(cachedPosts); |
|
|
|
|
allPosts.addAll(newPosts); |
|
|
|
|
|
|
|
|
|
for (Post cache : cached) { |
|
|
|
|
if (cache.no == post.no) { |
|
|
|
|
known = true; |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
if (loadable.isThreadMode()) { |
|
|
|
|
Map<Integer, Post> postsByNo = new HashMap<>(); |
|
|
|
|
for (int i = 0; i < allPosts.size(); i++) { |
|
|
|
|
Post post = allPosts.get(i); |
|
|
|
|
postsByNo.put(post.no, post); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (!known) { |
|
|
|
|
response.posts.add(post); |
|
|
|
|
// Maps post no's to a list of no's that that post received replies from
|
|
|
|
|
Map<Integer, List<Integer>> replies = new HashMap<>(); |
|
|
|
|
|
|
|
|
|
long collectReplies = Time.startTiming(); |
|
|
|
|
for (int i = 0; i < allPosts.size(); i++) { |
|
|
|
|
Post sourcePost = allPosts.get(i); |
|
|
|
|
|
|
|
|
|
for (int replyTo : sourcePost.repliesTo) { |
|
|
|
|
List<Integer> value = replies.get(replyTo); |
|
|
|
|
if (value == null) { |
|
|
|
|
value = new ArrayList<>(3); |
|
|
|
|
replies.put(replyTo, value); |
|
|
|
|
} |
|
|
|
|
value.add(sourcePost.no); |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
response.posts.addAll(serverList); |
|
|
|
|
} |
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("Collect replies", collectReplies); |
|
|
|
|
} |
|
|
|
|
long mapReplies = Time.startTiming(); |
|
|
|
|
|
|
|
|
|
for (int i = 0; i < response.posts.size(); i++) { |
|
|
|
|
Post sourcePost = response.posts.get(i); |
|
|
|
|
synchronized (sourcePost.repliesFrom) { |
|
|
|
|
sourcePost.repliesFrom.clear(); |
|
|
|
|
for (Map.Entry<Integer, List<Integer>> entry : replies.entrySet()) { |
|
|
|
|
int key = entry.getKey(); |
|
|
|
|
List<Integer> value = entry.getValue(); |
|
|
|
|
|
|
|
|
|
for (int j = i + 1; j < response.posts.size(); j++) { |
|
|
|
|
Post replyToSource = response.posts.get(j); |
|
|
|
|
if (replyToSource.repliesTo.contains(sourcePost.no)) { |
|
|
|
|
sourcePost.repliesFrom.add(replyToSource.no); |
|
|
|
|
Post subject = postsByNo.get(key); |
|
|
|
|
synchronized (subject.repliesFrom) { |
|
|
|
|
subject.repliesFrom.clear(); |
|
|
|
|
subject.repliesFrom.addAll(value); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (LOG_TIMING) { |
|
|
|
|
Time.endTiming("Map replies", mapReplies); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
response.posts.addAll(allPosts); |
|
|
|
|
|
|
|
|
|
return response; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private List<Post> loadThread(JsonReader reader) throws Exception { |
|
|
|
|
ArrayList<Post> list = new ArrayList<>(); |
|
|
|
|
|
|
|
|
|
private void loadThread(JsonReader reader, ProcessingQueue queue, Map<Integer, Post> cachedByNo) throws Exception { |
|
|
|
|
reader.beginObject(); |
|
|
|
|
// Page object
|
|
|
|
|
while (reader.hasNext()) { |
|
|
|
@ -184,10 +295,7 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
// Thread array
|
|
|
|
|
while (reader.hasNext()) { |
|
|
|
|
// Thread object
|
|
|
|
|
Post post = readPostObject(reader); |
|
|
|
|
if (post != null) { |
|
|
|
|
list.add(post); |
|
|
|
|
} |
|
|
|
|
readPostObject(reader, queue, cachedByNo); |
|
|
|
|
} |
|
|
|
|
reader.endArray(); |
|
|
|
|
} else { |
|
|
|
@ -195,13 +303,9 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
reader.endObject(); |
|
|
|
|
|
|
|
|
|
return list; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private List<Post> loadCatalog(JsonReader reader) throws Exception { |
|
|
|
|
ArrayList<Post> list = new ArrayList<>(); |
|
|
|
|
|
|
|
|
|
private void loadCatalog(JsonReader reader, ProcessingQueue queue, Map<Integer, Post> cachedByNo) throws Exception { |
|
|
|
|
reader.beginArray(); // Array of pages
|
|
|
|
|
|
|
|
|
|
while (reader.hasNext()) { |
|
|
|
@ -212,10 +316,7 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
reader.beginArray(); // Threads array
|
|
|
|
|
|
|
|
|
|
while (reader.hasNext()) { |
|
|
|
|
Post post = readPostObject(reader); |
|
|
|
|
if (post != null) { |
|
|
|
|
list.add(post); |
|
|
|
|
} |
|
|
|
|
readPostObject(reader, queue, cachedByNo); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
reader.endArray(); |
|
|
|
@ -228,11 +329,9 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
reader.endArray(); |
|
|
|
|
|
|
|
|
|
return list; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private Post readPostObject(JsonReader reader) throws Exception { |
|
|
|
|
private void readPostObject(JsonReader reader, ProcessingQueue queue, Map<Integer, Post> cachedByNo) throws Exception { |
|
|
|
|
Post post = new Post(); |
|
|
|
|
post.board = loadable.board; |
|
|
|
|
|
|
|
|
@ -242,12 +341,7 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
|
|
|
|
|
switch (key) { |
|
|
|
|
case "no": |
|
|
|
|
// Post number
|
|
|
|
|
post.no = reader.nextInt(); |
|
|
|
|
/*} else if (key.equals("time")) { |
|
|
|
|
// Time
|
|
|
|
|
long time = reader.nextLong(); |
|
|
|
|
post.date = new Date(time * 1000);*/ |
|
|
|
|
break; |
|
|
|
|
case "now": |
|
|
|
|
post.date = reader.nextString(); |
|
|
|
@ -340,59 +434,23 @@ public class ChanReaderRequest extends JsonReaderRequest<ChanReaderRequest.ChanR |
|
|
|
|
op.uniqueIps = post.uniqueIps; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Post cached = null; |
|
|
|
|
for (Post item : this.cached) { |
|
|
|
|
if (item.no == post.no) { |
|
|
|
|
cached = item; |
|
|
|
|
|
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Post cached = cachedByNo.get(post.no); |
|
|
|
|
if (cached != null) { |
|
|
|
|
return cached; |
|
|
|
|
queue.cached.add(cached); |
|
|
|
|
} else { |
|
|
|
|
// Process the filters before finish, because parsing the html is dependent on filter matches
|
|
|
|
|
processPostFilter(post); |
|
|
|
|
if (!post.finish()) { |
|
|
|
|
Logger.e(TAG, "Incorrect data about post received for post " + post.no); |
|
|
|
|
return null; |
|
|
|
|
} else { |
|
|
|
|
processPostAfterFinish(post); |
|
|
|
|
return post; |
|
|
|
|
} |
|
|
|
|
queue.toParse.add(post); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void processPostFilter(Post post) { |
|
|
|
|
int filterSize = filters.size(); |
|
|
|
|
for (int i = 0; i < filterSize; i++) { |
|
|
|
|
Filter filter = filters.get(i); |
|
|
|
|
if (filterEngine.matches(filter, post)) { |
|
|
|
|
FilterEngine.FilterAction action = FilterEngine.FilterAction.forId(filter.action); |
|
|
|
|
switch (action) { |
|
|
|
|
case COLOR: |
|
|
|
|
post.filterHighlightedColor = filter.color; |
|
|
|
|
break; |
|
|
|
|
case HIDE: |
|
|
|
|
post.filterStub = true; |
|
|
|
|
break; |
|
|
|
|
case REMOVE: |
|
|
|
|
post.filterRemove = true; |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void processPostAfterFinish(Post post) { |
|
|
|
|
post.isSavedReply = databaseManager.getDatabaseSavedReplyManager().isSaved(post.board, post.no); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public static class ChanReaderResponse { |
|
|
|
|
// Op Post that is created new each time.
|
|
|
|
|
// Used to later copy members like image count to the real op on the main thread.
|
|
|
|
|
public Post op; |
|
|
|
|
public List<Post> posts; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private static class ProcessingQueue { |
|
|
|
|
public List<Post> cached = new ArrayList<>(); |
|
|
|
|
public List<Post> toParse = new ArrayList<>(); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|