--- src/com/zoffcc/applications/aagtl/HTMLDownloader.java 2012/08/05 14:00:28 3
+++ src/com/zoffcc/applications/aagtl/HTMLDownloader.java 2015/08/01 08:47:10 4
@@ -115,6 +115,11 @@
private static final Pattern patternViewstates = Pattern.compile("id=\"__VIEWSTATE(\\d*)\"[^(value)]+value=\"([^\"]+)\"[^>]+>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
private static final Pattern patternUserToken = Pattern.compile("userToken\\s*=\\s*'([^']+)'");
+ private static Segment GC_DOWNLOAD__s_;
+ private static List GC_DOWNLOAD__gc_list = new ArrayList();
+ private static int GC_DOWNLOAD__count_p = 0;
+ private static final int GC_DOWNLOAD__max_threads_concurrent = 5;
+
// patterns from c:geo opensource
public HTMLDownloader(aagtl main)
@@ -338,7 +343,19 @@
public get_geocaches_ret get_geocaches(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
{
- return get_geocaches_v3(location, count_p, max_p, rec_depth, h, zoom_level);
+ get_geocaches_ret r = null;
+
+ try
+ {
+ r = get_geocaches_v3(location, count_p, max_p, rec_depth, h, zoom_level);
+ }
+ catch (Exception e)
+ {
+ r = new get_geocaches_ret();
+ r.points = null;
+ r.count_p = 0;
+ }
+ return r;
}
public get_geocaches_ret get_geocaches_v3(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
@@ -388,7 +405,7 @@
Boolean cont = true;
Source source = null;
- List gc_list = new ArrayList();
+ GC_DOWNLOAD__gc_list = new ArrayList();
count_p = 0;
max_p = 0;
while (cont)
@@ -414,67 +431,147 @@
// displaySegments(segments2);
try
{
+ int count_list_elements = 0;
for (Segment s_ : segments2)
{
- guid = "";
- disabled = false;
- gccode = null;
- try
+ count_list_elements++;
+ }
+ //System.out.println("AAGTL:count=" + count_list_elements);
+ Thread tmp_thread_array[] = new Thread[count_list_elements];
+ int threads_running = 0;
+ int cur_thread = 0;
+ GC_DOWNLOAD__count_p = count_p;
+ for (Segment s_ : segments2)
+ {
+ GC_DOWNLOAD__s_ = s_;
+ // --------------------------------------------------------
+ // ---------------- download a single cache ---------------
+ // --------------------------------------------------------
+ final class MyRunnable implements Runnable
{
- List segments3 = s_.getAllElements("class", "Merge", false);
- // displaySegments(segments2);
- guid = segments3.get(0).getFirstElement(HTMLElementName.A).getAttributeValue("href");
- guid = guid.split("guid=", 3)[1];
- //System.out.println("guid=:" + guid);
+ public Handler h_2;
+ public int max_p_2;
- try
+ MyRunnable(List gc_list, Handler h, int max_p)
{
- // Cineasten sehen mehr
- if (segments3.get(1).getFirstElement(HTMLElementName.A).getAttributeValue("class").equalsIgnoreCase("lnk Strike"))
- {
- // System.out.println("disabled=:" + disabled);
- disabled = true;
- }
+ this.h_2 = h;
+ this.max_p_2 = max_p;
}
- catch (Exception e3)
+
+ public void run()
{
- }
- gccode = segments3.get(1).getFirstElement("class", "small", false).getTextExtractor().toString();
- gccode = gccode.split("\\|")[1].trim();
- //System.out.println("gccode=:" + gccode);
- }
- catch (Exception e2)
- {
- e2.printStackTrace();
+ String guid2 = "";
+ String gccode2 = "";
+ Boolean disabled2 = false;
+
+ guid2 = "";
+ disabled2 = false;
+ gccode2 = null;
+ try
+ {
+ List segments3 = GC_DOWNLOAD__s_.getAllElements("class", "Merge", false);
+ // displaySegments(segments2);
+ guid2 = segments3.get(0).getFirstElement(HTMLElementName.A).getAttributeValue("href");
+ guid2 = guid2.split("guid=", 3)[1];
+ //System.out.println("guid=:" + guid2);
+
+ try
+ {
+ // Cineasten sehen mehr
+ if (segments3.get(1).getFirstElement(HTMLElementName.A).getAttributeValue("class").equalsIgnoreCase("lnk Strike"))
+ {
+ // System.out.println("disabled=:" + disabled);
+ disabled2 = true;
+ }
+ }
+ catch (Exception e3)
+ {
+ }
+
+ gccode2 = segments3.get(1).getFirstElement("class", "small", false).getTextExtractor().toString();
+ gccode2 = gccode2.split("\\|")[1].trim();
+ //System.out.println("gccode=:" + gccode2);
+ }
+ catch (Exception e2)
+ {
+ e2.printStackTrace();
+ }
+
+ if (gccode2 != null)
+ {
+ GeocacheCoordinate c__ = null;
+ c__ = new GeocacheCoordinate(0, 0, gccode2);
+ if (disabled2)
+ {
+ c__.status = GeocacheCoordinate.STATUS_DISABLED;
+ }
+
+ String url2 = "http://www.geocaching.com/seek/cdpf.aspx?guid=" + guid2;
+ //System.out.println("url=" + url2);
+
+ List values_list_2 = new ArrayList();
+ //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
+ //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
+ // bs = new ByteArrayOutputStream();
+ String the_page2 = get_reader_stream(url2, values_list_2, null, true);
+ c__ = CacheDownloader.__parse_cache_page_print(the_page2, c__);
+ if (c__ != null)
+ {
+ synchronized (GC_DOWNLOAD__gc_list)
+ {
+ GC_DOWNLOAD__gc_list.add(c__);
+ GC_DOWNLOAD__count_p = GC_DOWNLOAD__count_p + 1;
+ main_aagtl.set_bar_slow(h_2, "get geocaches", c__.title, GC_DOWNLOAD__count_p, max_p_2, true);
+ }
+ }
+ }
+
+ }
}
+ //
+ tmp_thread_array[cur_thread] = new Thread(new MyRunnable(GC_DOWNLOAD__gc_list, h, max_p));
+ tmp_thread_array[cur_thread].start();
+ System.out.println("++ start thread ++");
+ cur_thread++;
+ threads_running++;
- if (gccode != null)
+ if (threads_running > (GC_DOWNLOAD__max_threads_concurrent - 1))
{
- GeocacheCoordinate c__ = null;
- c__ = new GeocacheCoordinate(0, 0, gccode);
- if (disabled)
+ try
{
- c__.status = GeocacheCoordinate.STATUS_DISABLED;
+ while (threads_running > 0)
+ {
+ System.out.println("** waiting for thread to finish **");
+ tmp_thread_array[cur_thread - 1].join();
+ threads_running--;
+ }
}
-
- String url2 = "http://www.geocaching.com/seek/cdpf.aspx?guid=" + guid;
- //System.out.println("url=" + url);
-
- values_list = new ArrayList();
- //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
- //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
- // bs = new ByteArrayOutputStream();
- String the_page2 = get_reader_stream(url2, values_list, null, true);
- c__ = CacheDownloader.__parse_cache_page_print(the_page2, c__);
- if (c__ != null)
+ catch (Exception e)
{
- gc_list.add(c__);
- count_p = count_p + 1;
- this.main_aagtl.set_bar_slow(h, "get geocaches", c__.title, count_p, max_p, true);
+ e.printStackTrace();
}
}
+ // --------------------------------------------------------
+ // ---------------- download a single cache ---------------
+ // --------------------------------------------------------
+
+ } // -- end "for"-loop
+
+ int i7 = 0;
+ for (i7 = 0; i7 < cur_thread; i7++)
+ {
+ try
+ {
+ tmp_thread_array[i7].join();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ }
}
+
+ count_p = GC_DOWNLOAD__count_p;
}
catch (Exception e)
{
@@ -525,12 +622,12 @@
}
int jk;
- r2.count_p = gc_list.size();
- r2.points = new GeocacheCoordinate[gc_list.size()];
+ r2.count_p = GC_DOWNLOAD__gc_list.size();
+ r2.points = new GeocacheCoordinate[GC_DOWNLOAD__gc_list.size()];
- for (jk = 0; jk < gc_list.size(); jk++)
+ for (jk = 0; jk < GC_DOWNLOAD__gc_list.size(); jk++)
{
- r2.points[jk] = gc_list.get(jk);
+ r2.points[jk] = GC_DOWNLOAD__gc_list.get(jk);
}
// gc_list.clear();
@@ -1147,7 +1244,7 @@
public boolean login()
{
// System.out.println("--L--- LOGIN START -----");
-
+
String login_url = "https://www.geocaching.com/login/default.aspx";
DefaultHttpClient client2 = null;
@@ -1659,7 +1756,7 @@
// System.out.println("created cookie: ->" +
// String.valueOf(this_cookie));
- this.cookie_jar.addCookie(this_cookie);
+ HTMLDownloader.cookie_jar.addCookie(this_cookie);
}
}
@@ -1708,7 +1805,7 @@
// System.out.println("created cookie: ->" +
// String.valueOf(this_cookie));
- this.cookie_jar.addCookie(this_cookie);
+ HTMLDownloader.cookie_jar.addCookie(this_cookie);
}
}