/[aagtl_public1]/src/com/zoffcc/applications/aagtl/HTMLDownloader.java
aagtl

Contents of /src/com/zoffcc/applications/aagtl/HTMLDownloader.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 4 - (show annotations) (download)
Sat Aug 1 08:47:10 2015 UTC (8 years, 7 months ago) by zoffadmin
File size: 55733 byte(s)
1.0.35
1 /**
2 * aagtl Advanced Geocaching Tool for Android
3 * loosely based on agtl by Daniel Fett <fett@danielfett.de>
4 * Copyright (C) 2010 - 2012 Zoff <aagtl@work.zoff.cc>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * version 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 package com.zoffcc.applications.aagtl;
22
23 import java.io.BufferedReader;
24 import java.io.ByteArrayOutputStream;
25 import java.io.File;
26 import java.io.FileInputStream;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStream;
32 import java.io.OutputStreamWriter;
33 import java.io.Reader;
34 import java.io.StringWriter;
35 import java.io.UnsupportedEncodingException;
36 import java.io.Writer;
37 import java.net.HttpURLConnection;
38 import java.net.Socket;
39 import java.net.SocketTimeoutException;
40 import java.net.URI;
41 import java.net.URISyntaxException;
42 import java.net.URL;
43 import java.net.UnknownHostException;
44 import java.security.NoSuchAlgorithmException;
45 import java.security.SecureRandom;
46 import java.text.DecimalFormat;
47 import java.text.DecimalFormatSymbols;
48 import java.util.ArrayList;
49 import java.util.List;
50 import java.util.Locale;
51 import java.util.regex.Matcher;
52 import java.util.regex.Pattern;
53
54 import javax.net.ssl.HostnameVerifier;
55 import javax.net.ssl.HttpsURLConnection;
56 import javax.net.ssl.SSLContext;
57 import javax.net.ssl.SSLSession;
58 import javax.net.ssl.X509TrustManager;
59
60 import net.htmlparser.jericho.FormFields;
61 import net.htmlparser.jericho.HTMLElementName;
62 import net.htmlparser.jericho.Segment;
63 import net.htmlparser.jericho.Source;
64
65 import org.apache.http.HttpEntity;
66 import org.apache.http.HttpHost;
67 import org.apache.http.HttpResponse;
68 import org.apache.http.HttpVersion;
69 import org.apache.http.NameValuePair;
70 import org.apache.http.client.ClientProtocolException;
71 import org.apache.http.client.CookieStore;
72 import org.apache.http.client.entity.UrlEncodedFormEntity;
73 import org.apache.http.client.methods.HttpGet;
74 import org.apache.http.client.methods.HttpPost;
75 import org.apache.http.conn.ClientConnectionManager;
76 import org.apache.http.conn.params.ConnManagerPNames;
77 import org.apache.http.conn.params.ConnPerRouteBean;
78 import org.apache.http.conn.params.ConnRoutePNames;
79 import org.apache.http.conn.scheme.PlainSocketFactory;
80 import org.apache.http.conn.scheme.Scheme;
81 import org.apache.http.conn.scheme.SchemeRegistry;
82 import org.apache.http.cookie.Cookie;
83 import org.apache.http.impl.client.DefaultHttpClient;
84 import org.apache.http.impl.conn.SingleClientConnManager;
85 import org.apache.http.impl.cookie.BasicClientCookie;
86 import org.apache.http.message.BasicNameValuePair;
87 import org.apache.http.params.BasicHttpParams;
88 import org.apache.http.params.HttpConnectionParams;
89 import org.apache.http.params.HttpParams;
90 import org.apache.http.params.HttpProtocolParams;
91 import org.apache.http.protocol.HTTP;
92 import org.json.JSONArray;
93 import org.json.JSONException;
94 import org.json.JSONObject;
95
96 import android.os.Handler;
97 import android.util.Log;
98
99 import com.byarger.exchangeit.EasySSLSocketFactory;
100
101 public class HTMLDownloader
102 {
103
104 boolean logged_in = false;
105 public static CookieStore cookie_jar = null;
106 public static int GC_DOWNLOAD_MAX_REC_DEPTH = 6;
107 aagtl main_aagtl;
108 public static int large_buffer_size = 50 * 1024;
109 public static int default_buffer_size = 50 * 1024;
110
111 // patterns from c:geo opensource
112 private static final Pattern patternLoggedIn = Pattern.compile("<span class=\"Success\">You are logged in as[^<]*<strong[^>]*>([^<]+)</strong>[^<]*</span>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
113 private static final Pattern patternLogged2In = Pattern.compile("<strong>\\W*Hello,[^<]*<a[^>]+>([^<]+)</a>[^<]*</strong>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
114 private static final Pattern patternViewstateFieldCount = Pattern.compile("id=\"__VIEWSTATEFIELDCOUNT\"[^(value)]+value=\"(\\d+)\"[^>]+>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
115 private static final Pattern patternViewstates = Pattern.compile("id=\"__VIEWSTATE(\\d*)\"[^(value)]+value=\"([^\"]+)\"[^>]+>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
116 private static final Pattern patternUserToken = Pattern.compile("userToken\\s*=\\s*'([^']+)'");
117
118 private static Segment GC_DOWNLOAD__s_;
119 private static List<GeocacheCoordinate> GC_DOWNLOAD__gc_list = new ArrayList<GeocacheCoordinate>();
120 private static int GC_DOWNLOAD__count_p = 0;
121 private static final int GC_DOWNLOAD__max_threads_concurrent = 5;
122
123 // patterns from c:geo opensource
124
125 public HTMLDownloader(aagtl main)
126 {
127 this.main_aagtl = main;
128 }
129
130 class get_geocaches_ret
131 {
132 int count_p;
133 GeocacheCoordinate[] points;
134 }
135
136 public String convertStreamToString(InputStream is) throws IOException
137 {
138 /*
139 * To convert the InputStream to String we use the
140 * Reader.read(char[] buffer) method. We iterate until the
141 * Reader return -1 which means there's no more data to
142 * read. We use the StringWriter class to produce the string.
143 */
144 if (is != null)
145 {
146 Writer writer = new StringWriter();
147
148 char[] buffer = new char[default_buffer_size];
149 try
150 {
151 Reader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"), large_buffer_size);
152 int n;
153 while ((n = reader.read(buffer)) != -1)
154 {
155 writer.write(buffer, 0, n);
156 }
157 }
158 finally
159 {
160 is.close();
161 }
162 return writer.toString();
163 }
164 else
165 {
166 return null;
167 }
168 }
169
170 public String get_reader_stream(String url, List<NameValuePair> values, ByteArrayOutputStream data, Boolean need_login)
171 {
172
173 if ((need_login) && (!this.logged_in))
174 {
175 System.out.println("--2--- LOGIN START -----");
176 this.logged_in = login();
177 System.out.println("--2--- LOGIN END -----");
178 }
179
180 if ((values == null) && (data == null))
181 {
182 return null;
183 }
184 else if (data == null)
185 {
186 String websiteData = null;
187 URI uri = null;
188 DefaultHttpClient client = new DefaultHttpClient();
189
190 // insert cookies from cookie_jar
191 client.setCookieStore(cookie_jar);
192
193 try
194 {
195 uri = new URI(url);
196 }
197 catch (URISyntaxException e)
198 {
199 e.printStackTrace();
200 return null;
201 }
202
203 HttpPost method = new HttpPost(uri);
204 method.addHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)");
205 method.addHeader("Pragma", "no-cache");
206 method.addHeader("Content-Type", "application/x-www-form-urlencoded");
207
208 HttpEntity entity = null;
209 try
210 {
211 entity = new UrlEncodedFormEntity(values, HTTP.UTF_8);
212 }
213 catch (UnsupportedEncodingException e)
214 {
215 e.printStackTrace();
216 return null;
217 }
218
219 method.addHeader(entity.getContentType());
220 method.setEntity(entity);
221
222 try
223 {
224 HttpResponse res2 = client.execute(method);
225 // System.out.println("login response ->" +
226 // String.valueOf(res2.getStatusLine()));
227 InputStream data2 = res2.getEntity().getContent();
228 websiteData = generateString(data2);
229
230 }
231 catch (ClientProtocolException e)
232 {
233 // e.printStackTrace();
234 return null;
235 }
236 catch (IOException e)
237 {
238 // e.printStackTrace();
239 return null;
240 }
241
242 client.getConnectionManager().shutdown();
243
244 return websiteData;
245
246 }
247 else
248 {
249 try
250 {
251 // url Header textdata
252 InputStream i = this.doPost2(url, values, data);
253 return this.convertStreamToString(i);
254 }
255 catch (IOException e)
256 {
257 e.printStackTrace();
258 return null;
259 }
260 }
261 }
262
263 private InputStream doPost2(String urlString, List<NameValuePair> values, ByteArrayOutputStream content) throws IOException
264 {
265 URL url = new URL(urlString);
266 HttpURLConnection con = (HttpURLConnection) url.openConnection();
267 InputStream in = null;
268 OutputStream out;
269 byte[] buff;
270 con.setRequestMethod("POST");
271 for (int j = 0; j < values.size(); j++)
272 {
273 con.addRequestProperty(values.get(j).getName(), values.get(j).getValue());
274 }
275 String my_cookies = this.getCookies();
276 con.addRequestProperty("Cookie", my_cookies);
277 con.setDoOutput(true);
278 con.setDoInput(true);
279 con.connect();
280 out = con.getOutputStream();
281 buff = content.toByteArray();
282 out.write(buff);
283 out.flush();
284 out.close();
285 in = con.getInputStream();
286
287 return in;
288 }
289
290 public String get_user_token()
291 {
292 String ret = "";
293 String url = "http://www.geocaching.com/map/default.aspx?lat=6&lng=9";
294 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
295 String the_page = get_reader_stream(url, values_list, null, true);
296
297 if (the_page == null)
298 {
299 if (CacheDownloader.DEBUG_) System.out.println("page = NULL");
300 return "";
301 }
302
303 String[] response_lines = the_page.split("\n");
304 String line = null;
305 Pattern p = null;
306 Matcher m = null;
307 for (int i = 0; i < response_lines.length; i++)
308 {
309 line = response_lines[i];
310 // remove spaces at start and end of string
311 line = line.trim();
312
313 if (line.startsWith("var uvtoken"))
314 {
315 if (CacheDownloader.DEBUG_) System.out.println("usertoken=" + line);
316 p = Pattern.compile("userToken[ =]+'([^']+)'");
317 m = p.matcher(line);
318 m.find();
319 if (m.groupCount() > 0)
320 {
321 if (CacheDownloader.DEBUG_) System.out.println("usertoken parsed=" + m.group(1));
322 return m.group(1);
323 }
324 }
325 }
326
327 // + for line in page:
328 //
329 // + if line.startswith('var uvtoken'):
330 //
331 // + self.user_token =
332 // re.compile("userToken[ =]+'([^']+)'").search(line).group(1)
333 //
334 // + page.close()
335 //
336 // + return
337 //
338 // + raise
339 // Exception("Website contents unexpected. Please check connection.")
340
341 return ret;
342 }
343
344 public get_geocaches_ret get_geocaches(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
345 {
346 get_geocaches_ret r = null;
347
348 try
349 {
350 r = get_geocaches_v3(location, count_p, max_p, rec_depth, h, zoom_level);
351 }
352 catch (Exception e)
353 {
354 r = new get_geocaches_ret();
355 r.points = null;
356 r.count_p = 0;
357 }
358 return r;
359 }
360
361 public get_geocaches_ret get_geocaches_v3(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
362 {
363 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
364
365 Coordinate c1 = location[0];
366 Coordinate c2 = location[1];
367
368 Coordinate center = new Coordinate((c1.lat + c2.lat) / 2, (c1.lon + c2.lon) / 2);
369 double dist = (center.distance_to(c1) / 1000) / 2;
370 //System.out.println("distance is " + dist + " meters");
371
372 if (dist > 100)
373 {
374 // dist too large
375 count_p = count_p + 1;
376 get_geocaches_ret r = new get_geocaches_ret();
377 r.count_p = count_p;
378 r.points = null;
379 return r;
380 }
381
382 // use "." as comma seperator!!
383 DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US);
384 otherSymbols.setDecimalSeparator('.');
385 otherSymbols.setGroupingSeparator(',');
386 DecimalFormat format_lat_lon = new DecimalFormat("#.#####", otherSymbols);
387 // use "." as comma seperator!!
388
389 String lat_str = format_lat_lon.format(center.lat);
390 String lon_str = format_lat_lon.format(center.lon);
391 String dist_str = format_lat_lon.format(dist);
392 String url = "http://www.geocaching.com/seek/nearest.aspx?lat=" + lat_str + "&lng=" + lon_str + "&dist=" + dist_str;
393 //System.out.println("url=" + url);
394
395 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
396 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
397 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
398 // ByteArrayOutputStream bs = new ByteArrayOutputStream();
399 String the_page = get_reader_stream(url, values_list, null, true);
400
401 get_geocaches_ret r2 = new get_geocaches_ret();
402 r2.count_p = count_p;
403 r2.points = null;
404
405 Boolean cont = true;
406 Source source = null;
407
408 GC_DOWNLOAD__gc_list = new ArrayList<GeocacheCoordinate>();
409 count_p = 0;
410 max_p = 0;
411 while (cont)
412 {
413 source = new Source(the_page);
414 List<? extends Segment> segments = (source.getFirstElement("id", "ctl00_ContentBody_ResultsPanel", false).getContent().getFirstElement("class", "PageBuilderWidget", false).getContent().getAllElements("b"));
415 if (segments.size() < 3)
416 {
417 // no results
418 return r2;
419 }
420
421 int count = Integer.parseInt(segments.get(0).getTextExtractor().toString());
422 int page_current = Integer.parseInt(segments.get(1).getTextExtractor().toString());
423 int page_max = Integer.parseInt(segments.get(2).getTextExtractor().toString());
424 //System.out.println("count=" + count + " cur=" + page_current + " max=" + page_max);
425 max_p = count;
426
427 String guid = "";
428 String gccode = "";
429 Boolean disabled = false;
430 List<? extends Segment> segments2 = (source.getFirstElement("class", "SearchResultsTable Table", false).getContent().getAllElements(HTMLElementName.TR));
431 // displaySegments(segments2);
432 try
433 {
434 int count_list_elements = 0;
435 for (Segment s_ : segments2)
436 {
437 count_list_elements++;
438 }
439 //System.out.println("AAGTL:count=" + count_list_elements);
440 Thread tmp_thread_array[] = new Thread[count_list_elements];
441 int threads_running = 0;
442 int cur_thread = 0;
443 GC_DOWNLOAD__count_p = count_p;
444 for (Segment s_ : segments2)
445 {
446 GC_DOWNLOAD__s_ = s_;
447 // --------------------------------------------------------
448 // ---------------- download a single cache ---------------
449 // --------------------------------------------------------
450 final class MyRunnable implements Runnable
451 {
452 public Handler h_2;
453 public int max_p_2;
454
455 MyRunnable(List<GeocacheCoordinate> gc_list, Handler h, int max_p)
456 {
457 this.h_2 = h;
458 this.max_p_2 = max_p;
459 }
460
461 public void run()
462 {
463
464 String guid2 = "";
465 String gccode2 = "";
466 Boolean disabled2 = false;
467
468 guid2 = "";
469 disabled2 = false;
470 gccode2 = null;
471 try
472 {
473 List<? extends Segment> segments3 = GC_DOWNLOAD__s_.getAllElements("class", "Merge", false);
474 // displaySegments(segments2);
475 guid2 = segments3.get(0).getFirstElement(HTMLElementName.A).getAttributeValue("href");
476 guid2 = guid2.split("guid=", 3)[1];
477 //System.out.println("guid=:" + guid2);
478
479 try
480 {
481 // <a href="/seek/cache_details.aspx?guid=d9dbf39a-e2e6-4640-b951-d1d6307b16bd" class="lnk Strike"><span>Cineasten sehen mehr</span></a>
482 if (segments3.get(1).getFirstElement(HTMLElementName.A).getAttributeValue("class").equalsIgnoreCase("lnk Strike"))
483 {
484 // System.out.println("disabled=:" + disabled);
485 disabled2 = true;
486 }
487 }
488 catch (Exception e3)
489 {
490 }
491
492 gccode2 = segments3.get(1).getFirstElement("class", "small", false).getTextExtractor().toString();
493 gccode2 = gccode2.split("\\|")[1].trim();
494 //System.out.println("gccode=:" + gccode2);
495 }
496 catch (Exception e2)
497 {
498 e2.printStackTrace();
499 }
500
501 if (gccode2 != null)
502 {
503 GeocacheCoordinate c__ = null;
504 c__ = new GeocacheCoordinate(0, 0, gccode2);
505 if (disabled2)
506 {
507 c__.status = GeocacheCoordinate.STATUS_DISABLED;
508 }
509
510 String url2 = "http://www.geocaching.com/seek/cdpf.aspx?guid=" + guid2;
511 //System.out.println("url=" + url2);
512
513 List<NameValuePair> values_list_2 = new ArrayList<NameValuePair>();
514 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
515 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
516 // bs = new ByteArrayOutputStream();
517 String the_page2 = get_reader_stream(url2, values_list_2, null, true);
518 c__ = CacheDownloader.__parse_cache_page_print(the_page2, c__);
519 if (c__ != null)
520 {
521 synchronized (GC_DOWNLOAD__gc_list)
522 {
523 GC_DOWNLOAD__gc_list.add(c__);
524 GC_DOWNLOAD__count_p = GC_DOWNLOAD__count_p + 1;
525 main_aagtl.set_bar_slow(h_2, "get geocaches", c__.title, GC_DOWNLOAD__count_p, max_p_2, true);
526 }
527 }
528 }
529
530 }
531 }
532 //
533 tmp_thread_array[cur_thread] = new Thread(new MyRunnable(GC_DOWNLOAD__gc_list, h, max_p));
534 tmp_thread_array[cur_thread].start();
535 System.out.println("++ start thread ++");
536 cur_thread++;
537 threads_running++;
538
539 if (threads_running > (GC_DOWNLOAD__max_threads_concurrent - 1))
540 {
541 try
542 {
543 while (threads_running > 0)
544 {
545 System.out.println("** waiting for thread to finish **");
546 tmp_thread_array[cur_thread - 1].join();
547 threads_running--;
548 }
549 }
550 catch (Exception e)
551 {
552 e.printStackTrace();
553 }
554 }
555 // --------------------------------------------------------
556 // ---------------- download a single cache ---------------
557 // --------------------------------------------------------
558
559 } // -- end "for"-loop
560
561 int i7 = 0;
562 for (i7 = 0; i7 < cur_thread; i7++)
563 {
564 try
565 {
566 tmp_thread_array[i7].join();
567 }
568 catch (Exception e)
569 {
570 e.printStackTrace();
571 }
572 }
573
574 count_p = GC_DOWNLOAD__count_p;
575 }
576 catch (Exception e)
577 {
578 e.printStackTrace();
579 }
580
581 cont = false;
582
583 // ----------- check for paging ----------------
584 // ----------- and run through pages -----------
585 //
586 if (page_current < page_max)
587 {
588 FormFields formFields = source.getFormFields();
589 String vs1 = null;
590 try
591 {
592 vs1 = formFields.getValues("__VIEWSTATE1").get(0);
593 }
594 catch (Exception e)
595 {
596
597 }
598 String vs = null;
599 try
600 {
601 vs = formFields.getValues("__VIEWSTATE").get(0);
602 }
603 catch (Exception e)
604 {
605
606 }
607
608 //System.out.println("vs=" + vs);
609 //System.out.println("vs1=" + vs1);
610
611 List<NameValuePair> values_list2 = new ArrayList<NameValuePair>();
612 values_list2.add(new BasicNameValuePair("__EVENTTARGET", "ctl00$ContentBody$pgrTop$ctl08"));
613 values_list2.add(new BasicNameValuePair("__VIEWSTATEFIELDCOUNT", "2"));
614 values_list2.add(new BasicNameValuePair("__VIEWSTATE", vs));
615 values_list2.add(new BasicNameValuePair("__VIEWSTATE1", vs1));
616 // ByteArrayOutputStream bs = new ByteArrayOutputStream();
617 the_page = get_reader_stream(url, values_list2, null, true);
618 cont = true;
619 }
620 // ----------- check for paging ----------------
621 // ----------- and run through pages -----------
622 }
623
624 int jk;
625 r2.count_p = GC_DOWNLOAD__gc_list.size();
626 r2.points = new GeocacheCoordinate[GC_DOWNLOAD__gc_list.size()];
627
628 for (jk = 0; jk < GC_DOWNLOAD__gc_list.size(); jk++)
629 {
630 r2.points[jk] = GC_DOWNLOAD__gc_list.get(jk);
631 }
632 // gc_list.clear();
633
634 return r2;
635 }
636
637 private static void displaySegments(List<? extends Segment> segmentsx)
638 {
639 for (Segment segment1 : segmentsx)
640 {
641 System.out.println("-------------------------------------------------------------------------------");
642 System.out.println(segment1.getDebugInfo());
643 System.out.println(segment1);
644 }
645 System.out.println("\n*******************************************************************************\n");
646 }
647
648 public get_geocaches_ret get_geocaches_v2(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
649 {
650 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
651
652 if (zoom_level > 16)
653 {
654 zoom_level = 16;
655 }
656 else if (zoom_level < 6)
657 {
658 zoom_level = 6;
659 }
660
661 get_geocaches_ret rr2 = new get_geocaches_ret();
662 Coordinate c1 = location[0];
663 Coordinate c2 = location[1];
664
665 String zoomlevel_ = "" + zoom_level;
666
667 // use "." as comma seperator!!
668 DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US);
669 otherSymbols.setDecimalSeparator('.');
670 otherSymbols.setGroupingSeparator(',');
671 DecimalFormat format_lat_lon = new DecimalFormat("#.#####", otherSymbols);
672 // use "." as comma seperator!!
673
674 // http://www.geocaching.com/map/default.aspx?ll=48.22607,16.36997
675 // GET http://www.geocaching.com/map/default.aspx?ll=48.22607,16.36997 HTTP/1.1
676 String lat_str = format_lat_lon.format((c1.lat + c2.lat) / 2);
677 String lon_str = format_lat_lon.format((c1.lon + c2.lon) / 2);
678 //String url = "http://www.geocaching.com/map/?ll=" + lat_str + "," + lon_str + "&z=" + zoomlevel_;
679 String url = "http://www.geocaching.com/map/default.aspx?ll=" + lat_str + "," + lon_str;
680 System.out.println("url=" + url);
681
682 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
683 //*values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
684 //*values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
685 // values_list.add(new BasicNameValuePair("Referer","http://www.geocaching.com/map/default.aspx"));
686 //ByteArrayOutputStream bs = new ByteArrayOutputStream();
687 String the_page = get_reader_stream(url, values_list, null, true);
688 //String the_page = getUrlData(url);
689
690 //System.out.println(the_page);
691
692 // get values --------------------------
693 // get values --------------------------
694 // Groundspeak.Map.UserSession('xxxx',
695 Pattern p = Pattern.compile("Groundspeak\\.Map\\.UserSession.'([^']*)'");
696 Matcher m = p.matcher(the_page);
697 Boolean has_found = true;
698 String kk_ = "";
699 try
700 {
701 has_found = m.find();
702 kk_ = m.group(1);
703 }
704 catch (Exception e3)
705 {
706 e3.printStackTrace();
707 }
708 System.out.println("kk=" + kk_);
709
710 // sessionToken:'
711 p = Pattern.compile("sessionToken:'([^']*)'");
712 m = p.matcher(the_page);
713 has_found = m.find();
714 String sess_token_ = m.group(1);
715
716 System.out.println("sess_token=" + sess_token_);
717 // get values --------------------------
718 // get values --------------------------
719
720 java.util.Date date = new java.util.Date();
721 System.out.println("ts=" + date.getTime());
722
723 String timestamp_ = "" + date.getTime(); // ..seconds..
724
725 // lat, lon -> tile
726 Coordinate coord = new Coordinate((c1.lat + c2.lat) / 2, (c1.lon + c2.lon) / 2);
727 double[] tile = main_aagtl.rose.deg2num_give_zoom(coord, zoom_level);
728 String xtile_num = "" + ((int) tile[0]);
729 String ytile_num = "" + ((int) tile[1]);
730 // lat, lon -> tile
731
732 // X-Requested-With: XMLHttpRequest
733 //Referer: http://www.geocaching.com/map/default.aspx?ll=48.22607,16.36997
734 values_list = new ArrayList<NameValuePair>();
735 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
736 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
737 //values_list.add(new BasicNameValuePair("X-Requested-With", "XMLHttpRequest"));
738 //values_list.add(new BasicNameValuePair("Referer", "http://www.geocaching.com/map/default.aspx?ll=" + lat_str + "," + lon_str));
739
740 String url2 = "http://www.geocaching.com/map/map.info?x=" + xtile_num + "&y=" + ytile_num + "&z=" + zoomlevel_ + "&k=" + kk_ + "&st=" + sess_token_ + "&ep=1&_=" + timestamp_;
741
742 System.out.println("url2=" + url2);
743
744 the_page = get_reader_stream(url2, values_list, null, true);
745 // the_page = getUrlData(url2);
746 try
747 {
748 System.out.println(the_page);
749 }
750 catch (Exception e3)
751 {
752
753 }
754
755 // we need to add one, in any case!
756 count_p = count_p + 1;
757 get_geocaches_ret r = new get_geocaches_ret();
758 r.count_p = count_p;
759 r.points = null;
760 return r;
761 }
762
763 public get_geocaches_ret get_geocaches_v1(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
764 {
765
766 if (rec_depth > GC_DOWNLOAD_MAX_REC_DEPTH)
767 {
768 // we need to add one, in any case!
769 count_p = count_p + 1;
770 get_geocaches_ret r = new get_geocaches_ret();
771 r.count_p = count_p;
772 r.points = null;
773 return r;
774 }
775
776 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
777
778 get_geocaches_ret rr2 = new get_geocaches_ret();
779
780 Coordinate c1 = location[0];
781 Coordinate c2 = location[1];
782
783 String url = "http://www.geocaching.com/map/default.aspx/MapAction?lat=9&lng=6";
784
785 String user_token = "";
786 user_token = get_user_token();
787
788 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
789 DecimalFormat format = new DecimalFormat("#.##########");
790 String body_data = String.format("{\"dto\":{\"data\":{\"c\":1,\"m\":\"\",\"d\":\"%s|%s|%s|%s\"},\"ut\":\"%s\"}}", format.format(Math.max(c1.lat, c2.lat)).replace(",", "."), format.format(Math.min(c1.lat, c2.lat)).replace(",", "."), format.format(Math.max(c1.lon, c2.lon)).replace(",", "."), format.format(Math.min(c1.lon, c2.lon)).replace(",", "."), user_token);
791
792 // System.out.println(body_data);
793
794 values_list.add(new BasicNameValuePair("Content-Type", "application/json"));
795 values_list.add(new BasicNameValuePair("Content-Length", String.valueOf(body_data.length())));
796 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
797 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
798 ByteArrayOutputStream bs = new ByteArrayOutputStream();
799 try
800 {
801 bs.write(body_data.getBytes());
802 }
803 catch (IOException e1)
804 {
805 e1.printStackTrace();
806 }
807 String the_page = get_reader_stream(url, values_list, bs, true);
808
809 Boolean found = false;
810 JSONObject a = null;
811
812 try
813 {
814 a = new JSONObject(the_page);
815 // System.out.println(a.get("d"));
816 // get "d:" which seems to be only String? whatever, this works
817 // anyway
818 a = new JSONObject(a.get("d").toString());
819 found = true;
820 }
821 catch (JSONException e)
822 {
823 e.printStackTrace();
824 System.out.println("get_geocaches_ret: JSONException1");
825
826 rr2.count_p = count_p;
827 rr2.points = null;
828 return rr2;
829 }
830 catch (Exception e)
831 {
832 e.printStackTrace();
833 System.out.println("get_geocaches_ret: general Exception");
834 }
835
836 // string not found -> no caches got from website!
837 if (!found)
838 {
839 rr2.count_p = count_p;
840 rr2.points = null;
841 return rr2;
842 }
843
844 try
845 {
846 JSONObject _cs = a.getJSONObject("cs");
847
848 Boolean has_cc = false;
849 try
850 {
851 JSONArray _check = _cs.getJSONArray("cc");
852 has_cc = true;
853 }
854 catch (JSONException e)
855 {
856 // cs not found
857 has_cc = false;
858 }
859
860 if (!has_cc)
861 {
862 // 1nd part ----------------------
863 // 1nd part ----------------------
864 // System.out.println("part 1");
865
866 Boolean has_count_greater_zero = false;
867 int count_parsed = 0;
868 try
869 {
870 count_parsed = _cs.getInt("count");
871 if (count_parsed > 0)
872 {
873 // System.out.println("count_parsed: " + count_parsed);
874 has_count_greater_zero = true;
875 }
876 }
877 catch (JSONException e)
878 {
879 // has_count_greater_zero not found
880 has_count_greater_zero = false;
881 }
882
883 if (has_count_greater_zero)
884 {
885
886 double middle_lat = (c1.lat + c2.lat) / 2;
887 double middle_lon = (c1.lon + c2.lon) / 2;
888 double lat_left_top = Math.min(c1.lat, c2.lat);
889 double lon_left_top = Math.min(c1.lon, c2.lon);
890 double lat_right_bottom = Math.max(c1.lat, c2.lat);
891 double lon_right_bottom = Math.max(c1.lon, c2.lon);
892
893 Coordinate t1 = new Coordinate(lat_left_top, lon_left_top);
894 Coordinate t2 = new Coordinate(middle_lat, middle_lon);
895 Coordinate[] coord_temp = new Coordinate[2];
896 coord_temp[0] = t1;
897 coord_temp[1] = t2;
898 get_geocaches_ret r_temp = new get_geocaches_ret();
899 r_temp = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
900 count_p = r_temp.count_p;
901
902 t1 = new Coordinate(middle_lat, lon_left_top);
903 t2 = new Coordinate(lat_right_bottom, middle_lon);
904 coord_temp = new Coordinate[2];
905 coord_temp[0] = t1;
906 coord_temp[1] = t2;
907 get_geocaches_ret r_temp2 = new get_geocaches_ret();
908 r_temp2 = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
909 count_p = r_temp2.count_p;
910
911 t1 = new Coordinate(lat_left_top, middle_lon);
912 t2 = new Coordinate(middle_lat, lon_right_bottom);
913 coord_temp = new Coordinate[2];
914 coord_temp[0] = t1;
915 coord_temp[1] = t2;
916 get_geocaches_ret r_temp3 = new get_geocaches_ret();
917 r_temp3 = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
918 count_p = r_temp3.count_p;
919
920 t1 = new Coordinate(middle_lat, middle_lon);
921 t2 = new Coordinate(lat_right_bottom, lon_right_bottom);
922 coord_temp = new Coordinate[2];
923 coord_temp[0] = t1;
924 coord_temp[1] = t2;
925 get_geocaches_ret r_temp4 = new get_geocaches_ret();
926 r_temp4 = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
927 count_p = r_temp4.count_p;
928
929 int l1 = 0;
930 int l2 = 0;
931 int l3 = 0;
932 int l4 = 0;
933 if (r_temp.points != null)
934 {
935 l1 = r_temp.points.length;
936 }
937 if (r_temp2.points != null)
938 {
939 l2 = r_temp2.points.length;
940 }
941 if (r_temp3.points != null)
942 {
943 l3 = r_temp3.points.length;
944 }
945 if (r_temp4.points != null)
946 {
947 l4 = r_temp4.points.length;
948 }
949 GeocacheCoordinate[] points2 = new GeocacheCoordinate[l1 + l2 + l3 + l4];
950 int j = 0;
951 int j_old = 0;
952 if (l1 > 0)
953 {
954 for (j = 0; j < r_temp.points.length; j++)
955 {
956 points2[j_old] = r_temp.points[j];
957 j_old++;
958 }
959 }
960 if (l2 > 0)
961 {
962 for (j = 0; j < r_temp2.points.length; j++)
963 {
964 points2[j_old] = r_temp2.points[j];
965 j_old++;
966 }
967 }
968 if (l3 > 0)
969 {
970
971 for (j = 0; j < r_temp3.points.length; j++)
972 {
973 points2[j_old] = r_temp3.points[j];
974 j_old++;
975 }
976 }
977 if (l4 > 0)
978 {
979
980 for (j = 0; j < r_temp4.points.length; j++)
981 {
982 points2[j_old] = r_temp4.points[j];
983 j_old++;
984 }
985 }
986
987 rr2.count_p = count_p;
988 rr2.points = new GeocacheCoordinate[l1 + l2 + l3 + l4];
989 rr2.points = points2;
990 return rr2;
991
992 }
993
994 // 1nd part ----------------------
995 // 1nd part ----------------------
996
997 }
998 else
999 {
1000 // 2nd part ----------------------
1001 // 2nd part ----------------------
1002 // System.out.println("part 2");
1003
1004 // make progressbar look better
1005 count_p = count_p + 3;
1006
1007 JSONArray _b = _cs.getJSONArray("cc");
1008 GeocacheCoordinate[] points2 = new GeocacheCoordinate[_b.length()];
1009
1010 int zw_count = 0;
1011 int zw_count_draw = 0;
1012 int zw_max_p_2 = _b.length();
1013 int zw_draw_every = 1 + (int) ((float) (zw_max_p_2) / (float) (11));
1014
1015 for (int i = 0; i < _b.length(); i++)
1016 {
1017 GeocacheCoordinate c__ = null;
1018 c__ = new GeocacheCoordinate(_b.getJSONObject(i).getDouble("lat"), _b.getJSONObject(i).getDouble("lon"), _b.getJSONObject(i).getString("gc").toString());
1019
1020 // System.out.println("11 " +
1021 // _b.getJSONObject(i).getDouble("lat") + " "
1022 // + _b.getJSONObject(i).getDouble("lon"));
1023
1024 // System.out.println("22 " + c__.lat + " " + c__.lon);
1025 // System.out.println("33 " +
1026 // _b.getJSONObject(i).toString());
1027
1028 // title
1029 c__.title = _b.getJSONObject(i).getString("nn");
1030
1031 // System.out.println("dd1");
1032
1033 // type
1034 String temp = _b.getJSONObject(i).getString("ctid");
1035
1036 try
1037 {
1038 c__.type = GeocacheCoordinate.GC_TYPE_HASH.get(temp);
1039 }
1040 catch (Exception e)
1041 {
1042 System.out.println("Unknown GC Type!!");
1043 c__.type = GeocacheCoordinate.TYPE_UNKNOWN;
1044 }
1045
1046 // System.out.println("dd2");
1047
1048 // found
1049 c__.found = _b.getJSONObject(i).getBoolean("f");
1050
1051 // System.out.println("" + _b.getJSONObject(i).toString());
1052
1053 // status
1054 Boolean temp_b = _b.getJSONObject(i).getBoolean("ia");
1055 if (temp_b)
1056 {
1057 c__.status = GeocacheCoordinate.STATUS_NORMAL;
1058 }
1059 else
1060 {
1061 c__.status = GeocacheCoordinate.STATUS_DISABLED;
1062 }
1063
1064 points2[i] = c__;
1065
1066 zw_count++;
1067 zw_count_draw++;
1068 this.main_aagtl.set_bar_slow(h, "get geocaches", c__.title, zw_count, zw_max_p_2, true);
1069
1070 if (zw_count_draw >= zw_draw_every)
1071 {
1072 zw_count_draw = 0;
1073 }
1074
1075 }
1076
1077 count_p = count_p + 1;
1078 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
1079
1080 rr2.count_p = count_p;
1081 rr2.points = points2;
1082 return rr2;
1083
1084 // 2nd part ----------------------
1085 // 2nd part ----------------------
1086 }
1087 }
1088 catch (JSONException e)
1089 {
1090 System.out.println("get_geocaches_ret: JSONException2");
1091
1092 rr2.count_p = count_p;
1093 rr2.points = null;
1094 return rr2;
1095 }
1096
1097 return rr2;
1098 }
1099
1100 public static boolean isEmpty(String[] a)
1101 {
1102 if (a == null)
1103 {
1104 return true;
1105 }
1106
1107 for (String s : a)
1108 {
1109 if ((s == null) || (s.equals("")))
1110 {
1111 return true;
1112 }
1113 }
1114 return false;
1115 }
1116
1117 /**
1118 * read all viewstates from page
1119 *
1120 * @return String[] with all view states
1121 */
1122 public static String[] getViewstates(String page)
1123 {
1124 // Get the number of viewstates.
1125 // If there is only one viewstate, __VIEWSTATEFIELDCOUNT is not present
1126 int count = 1;
1127 final Matcher matcherViewstateCount = patternViewstateFieldCount.matcher(page);
1128 if (matcherViewstateCount.find())
1129 {
1130 count = Integer.parseInt(matcherViewstateCount.group(1));
1131 }
1132
1133 String[] viewstates = new String[count];
1134
1135 // Get the viewstates
1136 int no;
1137 final Matcher matcherViewstates = patternViewstates.matcher(page);
1138 while (matcherViewstates.find())
1139 {
1140 String sno = matcherViewstates.group(1); // number of viewstate
1141 if ("".equals(sno))
1142 {
1143 no = 0;
1144 }
1145 else
1146 {
1147 no = Integer.parseInt(sno);
1148 }
1149 // System.out.println("1 v " + no + "=" + matcherViewstates.group(2));
1150 viewstates[no] = matcherViewstates.group(2);
1151 }
1152
1153 return viewstates;
1154 }
1155
1156 /**
1157 * put viewstates into request parameters
1158 */
1159 private static void putViewstates(List<NameValuePair> params, String[] viewstates)
1160 {
1161 if (isEmpty(viewstates))
1162 {
1163 //System.out.println("EEEEEEEEEEEEEEE*EEEEEEEE");
1164 return;// params;
1165 }
1166 // System.out.println("***** __VIEWSTATE=" + viewstates[0]);
1167 params.add(new BasicNameValuePair("__VIEWSTATE", viewstates[0]));
1168 if (viewstates.length > 1)
1169 {
1170 for (int i = 1; i < viewstates.length; i++)
1171 {
1172 params.add(new BasicNameValuePair("__VIEWSTATE" + i, viewstates[i]));
1173 }
1174 params.add(new BasicNameValuePair("__VIEWSTATEFIELDCOUNT", viewstates.length + ""));
1175 }
1176 return;// params;
1177 }
1178
1179 /**
1180 * transfers the viewstates variables from a page (response) to parameters
1181 * (next request)
1182 */
1183 public static void transferViewstates(String page, List<NameValuePair> params)
1184 {
1185 putViewstates(params, getViewstates(page));
1186 }
1187
1188 private void trust_Every_ssl_cert()
1189 {
1190 // NEVER enable this on a production release!!!!!!!!!!
1191 try
1192 {
1193 HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier()
1194 {
1195 public boolean verify(String hostname, SSLSession session)
1196 {
1197 Log.d("aagtl", "DANGER !!! trusted hostname=" + hostname + " DANGER !!!");
1198 // return true -> mean we trust this cert !! DANGER !! DANGER !!
1199 return true;
1200 }
1201 });
1202 SSLContext context = SSLContext.getInstance("TLS");
1203 context.init(null, new X509TrustManager[] { new X509TrustManager()
1204 {
1205 public java.security.cert.X509Certificate[] getAcceptedIssuers()
1206 {
1207 Log.d("aagtl", "DANGER !!! 222222222");
1208 return new java.security.cert.X509Certificate[0];
1209 }
1210
1211 public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) throws java.security.cert.CertificateException
1212 {
1213 Log.d("aagtl", "DANGER !!! 333333333");
1214 }
1215
1216 public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) throws java.security.cert.CertificateException
1217 {
1218 Log.d("aagtl", "DANGER !!! 444444444444");
1219 }
1220 } }, new SecureRandom());
1221 HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
1222 }
1223 catch (Exception e)
1224 {
1225 e.printStackTrace();
1226 }
1227 // NEVER enable this on a production release!!!!!!!!!!
1228 }
1229
1230 public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException
1231 {
1232 SSLContext sslContext = null;
1233 try
1234 {
1235 sslContext = SSLContext.getInstance("TLS");
1236 }
1237 catch (NoSuchAlgorithmException e)
1238 {
1239 e.printStackTrace();
1240 }
1241 return sslContext.getSocketFactory().createSocket(socket, host, port, autoClose);
1242 }
1243
1244 public boolean login()
1245 {
1246 // System.out.println("--L--- LOGIN START -----");
1247
1248 String login_url = "https://www.geocaching.com/login/default.aspx";
1249
1250 DefaultHttpClient client2 = null;
1251 HttpHost proxy = null;
1252
1253 if (CacheDownloader.DEBUG2_)
1254 {
1255 // NEVER enable this on a production release!!!!!!!!!!
1256 // NEVER enable this on a production release!!!!!!!!!!
1257 trust_Every_ssl_cert();
1258
1259 SchemeRegistry schemeRegistry = new SchemeRegistry();
1260 schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
1261 schemeRegistry.register(new Scheme("https", new EasySSLSocketFactory(), 443));
1262
1263 HttpParams params = new BasicHttpParams();
1264 params.setParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 30);
1265 params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRouteBean(30));
1266 params.setParameter(HttpProtocolParams.USE_EXPECT_CONTINUE, false);
1267
1268 proxy = new HttpHost("192.168.0.1", 8888);
1269 params.setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
1270
1271 HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
1272
1273 ClientConnectionManager cm = new SingleClientConnManager(params, schemeRegistry);
1274 client2 = new DefaultHttpClient(cm, params);
1275 // NEVER enable this on a production release!!!!!!!!!!
1276 // NEVER enable this on a production release!!!!!!!!!!
1277 }
1278 else
1279 {
1280 HttpParams httpParameters = new BasicHttpParams();
1281 int timeoutConnection = 10000;
1282 HttpConnectionParams.setConnectionTimeout(httpParameters, timeoutConnection);
1283 int timeoutSocket = 10000;
1284 HttpConnectionParams.setSoTimeout(httpParameters, timeoutSocket);
1285
1286 client2 = new DefaultHttpClient(httpParameters);
1287 }
1288
1289 if (CacheDownloader.DEBUG2_)
1290 {
1291 client2.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
1292 }
1293
1294 // read first page to check for login
1295 // read first page to check for login
1296 // read first page to check for login
1297
1298 String websiteData2 = null;
1299 URI uri2 = null;
1300 try
1301 {
1302 uri2 = new URI(login_url);
1303 }
1304 catch (URISyntaxException e)
1305 {
1306 e.printStackTrace();
1307 return false;
1308 }
1309
1310 client2.setCookieStore(cookie_jar);
1311
1312 HttpGet method2 = new HttpGet(uri2);
1313
1314 method2.addHeader("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.10) Gecko/20071115 Firefox/2.0.0.10");
1315 method2.addHeader("Pragma", "no-cache");
1316 method2.addHeader("Accept-Language", "en");
1317
1318 HttpResponse res = null;
1319 try
1320 {
1321 res = client2.execute(method2);
1322 }
1323 catch (ClientProtocolException e1)
1324 {
1325 e1.printStackTrace();
1326 return false;
1327 }
1328 catch (IOException e1)
1329 {
1330 e1.printStackTrace();
1331 return false;
1332 }
1333 catch (Exception e)
1334 {
1335 e.printStackTrace();
1336 return false;
1337 }
1338
1339 InputStream data = null;
1340 try
1341 {
1342 data = res.getEntity().getContent();
1343 }
1344 catch (IllegalStateException e1)
1345 {
1346 e1.printStackTrace();
1347 return false;
1348 }
1349 catch (IOException e1)
1350 {
1351 e1.printStackTrace();
1352 return false;
1353 }
1354 catch (Exception e)
1355 {
1356 e.printStackTrace();
1357 return false;
1358 }
1359
1360 websiteData2 = generateString(data);
1361
1362 if (CacheDownloader.DEBUG_) System.out.println("111 cookies=" + dumpCookieStore(client2.getCookieStore()));
1363 cookie_jar = client2.getCookieStore();
1364
1365 // on every page
1366 final Matcher matcherLogged2In = patternLogged2In.matcher(websiteData2);
1367 if (matcherLogged2In.find())
1368 {
1369 if (CacheDownloader.DEBUG_) System.out.println("login: -> already logged in (1)");
1370 return true;
1371 }
1372
1373 // after login
1374 final Matcher matcherLoggedIn = patternLoggedIn.matcher(websiteData2);
1375 if (matcherLoggedIn.find())
1376 {
1377 if (CacheDownloader.DEBUG_) System.out.println("login: -> already logged in (2)");
1378 return true;
1379 }
1380 // read first page to check for login
1381 // read first page to check for login
1382 // read first page to check for login
1383
1384 // ok post login data as formdata
1385 // ok post login data as formdata
1386 // ok post login data as formdata
1387 // ok post login data as formdata
1388 HttpPost method = new HttpPost(uri2);
1389
1390 method.addHeader("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0) Firefox/7.0");
1391 method.addHeader("Pragma", "no-cache");
1392 method.addHeader("Content-Type", "application/x-www-form-urlencoded");
1393 method.addHeader("Accept-Language", "en");
1394
1395 List<NameValuePair> loginInfo = new ArrayList<NameValuePair>();
1396
1397 loginInfo.add(new BasicNameValuePair("__EVENTTARGET", ""));
1398 loginInfo.add(new BasicNameValuePair("__EVENTARGUMENT", ""));
1399
1400 String[] viewstates = getViewstates(websiteData2);
1401 if (CacheDownloader.DEBUG_) System.out.println("vs==" + viewstates[0]);
1402 putViewstates(loginInfo, viewstates);
1403
1404 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$tbUsername", this.main_aagtl.global_settings.options_username));
1405 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$tbPassword", this.main_aagtl.global_settings.options_password));
1406 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$btnSignIn", "Login"));
1407 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$cbRememberMe", "on"));
1408
1409 //for (int i = 0; i < loginInfo.size(); i++)
1410 //{
1411 // System.out.println("x*X " + loginInfo.get(i).getName() + " " + loginInfo.get(i).getValue());
1412 //}
1413
1414 // set cookies
1415 client2.setCookieStore(cookie_jar);
1416
1417 HttpEntity entity = null;
1418 try
1419 {
1420 entity = new UrlEncodedFormEntity(loginInfo, HTTP.UTF_8);
1421 }
1422 catch (UnsupportedEncodingException e)
1423 {
1424 e.printStackTrace();
1425 return false;
1426 }
1427
1428 // try
1429 // {
1430 // System.out.println("1 " + entity.toString());
1431 // InputStream in2 = entity.getContent();
1432 // InputStreamReader ir2 = new InputStreamReader(in2, "utf-8");
1433 // BufferedReader r = new BufferedReader(ir2);
1434 // System.out.println("line=" + r.readLine());
1435 // }
1436 // catch (Exception e2)
1437 // {
1438 // e2.printStackTrace();
1439 // }
1440
1441 method.setEntity(entity);
1442
1443 HttpResponse res2 = null;
1444 try
1445 {
1446 res2 = client2.execute(method);
1447 if (CacheDownloader.DEBUG_) System.out.println("login response ->" + String.valueOf(res2.getStatusLine()));
1448 }
1449 catch (ClientProtocolException e)
1450 {
1451 e.printStackTrace();
1452 return false;
1453 }
1454 catch (IOException e)
1455 {
1456 e.printStackTrace();
1457 return false;
1458 }
1459 catch (Exception e)
1460 {
1461 e.printStackTrace();
1462 return false;
1463 }
1464
1465 // for (int x = 0; x < res2.getAllHeaders().length; x++)
1466 // {
1467 // System.out.println("## n=" + res2.getAllHeaders()[x].getName());
1468 // System.out.println("## v=" + res2.getAllHeaders()[x].getValue());
1469 // }
1470
1471 InputStream data2 = null;
1472 try
1473 {
1474 data2 = res2.getEntity().getContent();
1475 }
1476 catch (IllegalStateException e1)
1477 {
1478 e1.printStackTrace();
1479 return false;
1480 }
1481 catch (IOException e1)
1482 {
1483 e1.printStackTrace();
1484 return false;
1485 }
1486 catch (Exception e)
1487 {
1488 e.printStackTrace();
1489 return false;
1490 }
1491
1492 websiteData2 = generateString(data2);
1493
1494 if (CacheDownloader.DEBUG_) System.out.println("2222 cookies=" + dumpCookieStore(client2.getCookieStore()));
1495 cookie_jar = client2.getCookieStore();
1496
1497 String[] viewstates2 = getViewstates(websiteData2);
1498 if (CacheDownloader.DEBUG_) System.out.println("vs==" + viewstates2[0]);
1499
1500 // System.out.println("******");
1501 // System.out.println("******");
1502 // System.out.println("******");
1503 // System.out.println(websiteData2);
1504 // System.out.println("******");
1505 // System.out.println("******");
1506 // System.out.println("******");
1507
1508 return true;
1509
1510 }
1511
1512 public static String dumpCookieStore(CookieStore cookieStore)
1513 {
1514 StringBuilder cookies = new StringBuilder();
1515 for (Cookie cookie : cookieStore.getCookies())
1516 {
1517 System.out.println("CC**\n\n");
1518 cookies.append(cookie.getName());
1519 cookies.append("=");
1520 cookies.append(cookie.getValue());
1521 cookies.append("=");
1522 cookies.append(cookie.getDomain());
1523 cookies.append(";" + "\n");
1524 }
1525 return cookies.toString();
1526 }
1527
1528 public void saveCookies()
1529 {
1530 FileOutputStream fOut = null;
1531 OutputStreamWriter osw = null;
1532 // make dirs
1533 File dir1 = new File(this.main_aagtl.main_dir + "/config");
1534 dir1.mkdirs();
1535 // save cookies to file
1536 try
1537 {
1538 File cookie_file = new File(this.main_aagtl.main_dir + "/config/cookie.txt");
1539 fOut = new FileOutputStream(cookie_file);
1540 osw = new OutputStreamWriter(fOut);
1541 osw.write(cookie_jar.toString());
1542 osw.flush();
1543 }
1544 catch (Exception e)
1545 {
1546 e.printStackTrace();
1547 System.out.println("saveCookies: Exception1");
1548 }
1549 finally
1550 {
1551 try
1552 {
1553 osw.close();
1554 fOut.close();
1555 }
1556 catch (IOException e)
1557 {
1558 e.printStackTrace();
1559 System.out.println("saveCookies: Exception2");
1560 }
1561 }
1562 }
1563
1564 public String getCookies()
1565 {
1566 String ret = "";
1567
1568 // System.out.println("getCookies size: " +
1569 // cookie_jar.getCookies().size());
1570 try
1571 {
1572 for (int i = 0; i < cookie_jar.getCookies().size(); i++)
1573 {
1574 // System.out.println("getCookies: " +
1575 // cookie_jar.getCookies().get(i).toString());
1576 if (i > 0)
1577 {
1578 ret = ret + "; ";
1579 }
1580 ret = ret + cookie_jar.getCookies().get(i).getName() + "=" + cookie_jar.getCookies().get(i).getValue();
1581 }
1582 }
1583 catch (Exception e)
1584 {
1585 e.printStackTrace();
1586 // System.out.println("getCookies: Exception");
1587 }
1588 // System.out.println("getCookies: ret=" + ret);
1589 return ret;
1590 }
1591
1592 public String[] getCookies2()
1593 {
1594 if (cookie_jar.getCookies().size() == 0)
1595 {
1596 return null;
1597 }
1598 String[] ret = new String[cookie_jar.getCookies().size() * 2];
1599
1600 // System.out.println("getCookies size: " +
1601 // cookie_jar.getCookies().size());
1602 try
1603 {
1604 for (int i = 0; i < cookie_jar.getCookies().size(); i++)
1605 {
1606 // System.out.println("getCookies: " +
1607 // cookie_jar.getCookies().get(i).toString());
1608 ret[i * 2] = cookie_jar.getCookies().get(i).getName();
1609 ret[(i * 2) + 1] = cookie_jar.getCookies().get(i).getValue();
1610 }
1611 }
1612 catch (Exception e)
1613 {
1614 e.printStackTrace();
1615 // System.out.println("getCookies: Exception");
1616 }
1617 return ret;
1618 }
1619
1620 public void loadCookies()
1621 {
1622 String[] ret = new String[2];
1623
1624 // make dirs
1625 File dir1 = new File(this.main_aagtl.main_dir + "/config");
1626 dir1.mkdirs();
1627 // load cookies from file
1628 File cookie_file = new File(this.main_aagtl.main_dir + "/config/cookie.txt");
1629
1630 FileInputStream fIn = null;
1631 InputStreamReader isr = null;
1632 char[] inputBuffer = new char[255];
1633 Writer writer = new StringWriter();
1634 String data = null;
1635 try
1636 {
1637 fIn = new FileInputStream(cookie_file);
1638 isr = new InputStreamReader(fIn);
1639 int n = 0;
1640 while ((n = isr.read(inputBuffer)) != -1)
1641 {
1642 writer.write(inputBuffer, 0, n);
1643 }
1644 data = writer.toString();
1645 }
1646 catch (Exception e)
1647 {
1648 e.printStackTrace();
1649 System.out.println("loadCookies: Exception1");
1650 return;
1651 }
1652 finally
1653 {
1654 try
1655 {
1656 isr.close();
1657 fIn.close();
1658 }
1659 catch (NullPointerException e2)
1660 {
1661 System.out.println("loadCookies: Exception2");
1662 return;
1663 }
1664 catch (IOException e)
1665 {
1666 System.out.println("loadCookies: Exception3");
1667
1668 e.printStackTrace();
1669 return;
1670 }
1671 }
1672
1673 if (cookie_jar == null)
1674 {
1675 // cookie_jar = new CookieStore();
1676 return;
1677 }
1678 else
1679 {
1680 cookie_jar.clear();
1681 }
1682
1683 // Log.d("load cookie:", "->" + String.valueOf(data));
1684
1685 // [[version: 0][name: ASP.NET_SessionId]
1686 // [value: cyuoctxrwio1x13vivqzlxgi][domain: www.geocaching.com]
1687 // [path: /][expiry: null], [version: 0][name: userid]
1688 // [value: 8a72e55f-419c-4da7-8de3-7813a3fda9c7][domain:
1689 // www.geocaching.com]
1690 // [path: /][expiry: Tue Apr 26 15:41:14 Europe/Belgrade 2011]]
1691
1692 if (data.length() > 1)
1693 {
1694 // check for multpile cookies
1695 if (data.startsWith("[["))
1696 {
1697 // strip [ and ] at begin and end of string
1698 data = data.substring(1, data.length() - 1);
1699 String s3 = "\\], \\[";
1700 String[] a3 = data.split(s3);
1701 String data_cookie;
1702 for (int j3 = 0; j3 < a3.length; j3++)
1703 {
1704 data_cookie = a3[j3];
1705 if (j3 == 0)
1706 {
1707 data_cookie = data_cookie + "]";
1708 }
1709 else
1710 {
1711 data_cookie = "[" + data_cookie;
1712 }
1713 // System.out.println("parsing cookie #" + j3 + ": " +
1714 // data_cookie);
1715
1716 String s2 = "]";
1717 String[] a = data_cookie.split(s2);
1718 String x = null;
1719 String c1, c2 = null;
1720 String c_name = null, c_value = null, c_domain = null, c_path = null;
1721 String c_version = null;
1722 BasicClientCookie this_cookie = null;
1723
1724 for (int j = 0; j < a.length; j++)
1725 {
1726 x = a[j].replace("[", "").trim();
1727 c1 = x.split(":")[0];
1728 c2 = x.split(":")[1].substring(1);
1729 // Log.d("load cookie:", "->" + String.valueOf(c1));
1730 // Log.d("load cookie:", "->" + String.valueOf(c2));
1731 if (c1.matches("name") == true)
1732 {
1733 // Log.d("name:", "->" + String.valueOf(c1));
1734 c_name = c2;
1735 }
1736 else if (c1.matches("value") == true)
1737 {
1738 c_value = c2;
1739 }
1740 else if (c1.matches("domain") == true)
1741 {
1742 c_domain = c2;
1743 }
1744 else if (c1.matches("path") == true)
1745 {
1746 c_path = c2;
1747 }
1748 else if (c1.matches("version") == true)
1749 {
1750 c_version = c2;
1751 }
1752 }
1753 this_cookie = new BasicClientCookie(c_name, c_value);
1754 this_cookie.setDomain(c_domain);
1755 this_cookie.setPath(c_path);
1756 // System.out.println("created cookie: ->" +
1757 // String.valueOf(this_cookie));
1758
1759 HTMLDownloader.cookie_jar.addCookie(this_cookie);
1760
1761 }
1762 }
1763 // single cookie
1764 else
1765 {
1766 String s2 = "]";
1767 String[] a = data.split(s2);
1768 String x = null;
1769 String c1, c2 = null;
1770 String c_name = null, c_value = null, c_domain = null, c_path = null;
1771 String c_version = null;
1772 BasicClientCookie this_cookie = null;
1773 for (int j = 0; j < a.length; j++)
1774 {
1775 x = a[j].replace("[", "").trim();
1776 c1 = x.split(":")[0];
1777 c2 = x.split(":")[1].substring(1);
1778 // Log.d("load cookie:", "->" + String.valueOf(c1));
1779 // Log.d("load cookie:", "->" + String.valueOf(c2));
1780 if (c1.matches("name") == true)
1781 {
1782 // Log.d("name:", "->" + String.valueOf(c1));
1783 c_name = c2;
1784 }
1785 else if (c1.matches("value") == true)
1786 {
1787 c_value = c2;
1788 }
1789 else if (c1.matches("domain") == true)
1790 {
1791 c_domain = c2;
1792 }
1793 else if (c1.matches("path") == true)
1794 {
1795 c_path = c2;
1796 }
1797 else if (c1.matches("version") == true)
1798 {
1799 c_version = c2;
1800 }
1801 }
1802 this_cookie = new BasicClientCookie(c_name, c_value);
1803 this_cookie.setDomain(c_domain);
1804 this_cookie.setPath(c_path);
1805 // System.out.println("created cookie: ->" +
1806 // String.valueOf(this_cookie));
1807
1808 HTMLDownloader.cookie_jar.addCookie(this_cookie);
1809 }
1810 }
1811
1812 return;
1813 }
1814
1815 public String getUrlData(String url)
1816 {
1817 String websiteData = null;
1818
1819 if (!this.logged_in)
1820 {
1821 System.out.println("--1--- LOGIN START -----");
1822 this.logged_in = login();
1823 System.out.println("--1--- LOGIN END -----");
1824 }
1825
1826 try
1827 {
1828 HttpParams httpParameters = new BasicHttpParams();
1829 // Set the timeout in milliseconds until a connection is
1830 // established.
1831 int timeoutConnection = 10000;
1832 HttpConnectionParams.setConnectionTimeout(httpParameters, timeoutConnection);
1833 // Set the default socket timeout (SO_TIMEOUT)
1834 // in milliseconds which is the timeout for waiting for data.
1835 int timeoutSocket = 10000;
1836 HttpConnectionParams.setSoTimeout(httpParameters, timeoutSocket);
1837
1838 DefaultHttpClient client = new DefaultHttpClient(httpParameters);
1839 client.setCookieStore(cookie_jar);
1840 // Log.d("cookie_jar", "->" + String.valueOf(cookie_jar));
1841
1842 URI uri = new URI(url);
1843 HttpGet method = new HttpGet(uri);
1844
1845 method.addHeader("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.10) Gecko/20071115 Firefox/2.0.0.10");
1846 method.addHeader("Pragma", "no-cache");
1847
1848 HttpResponse res = client.execute(method);
1849 InputStream data = res.getEntity().getContent();
1850 websiteData = generateString(data);
1851 client.getConnectionManager().shutdown();
1852
1853 }
1854 catch (SocketTimeoutException e2)
1855 {
1856 Log.d("HTMLDownloader", "Connectiont timout: " + e2);
1857 }
1858 catch (ClientProtocolException e)
1859 {
1860 e.printStackTrace();
1861 }
1862 catch (UnknownHostException x)
1863 {
1864 Log.d("HTMLDownloader", ": " + x);
1865 }
1866 catch (IOException e)
1867 {
1868 e.printStackTrace();
1869 }
1870 catch (URISyntaxException e)
1871 {
1872 e.printStackTrace();
1873 }
1874 catch (Exception e)
1875 {
1876 e.printStackTrace();
1877 }
1878
1879 return websiteData;
1880 }
1881
1882 public String generateString(InputStream stream)
1883 {
1884 InputStreamReader reader = new InputStreamReader(stream);
1885 BufferedReader buffer = new BufferedReader(reader, large_buffer_size);
1886 // StringBuilder sb = new StringBuilder();
1887 String ret = "";
1888
1889 try
1890 {
1891 // ***DEBUG*** System.out.println("int=" +(char) buffer.read());
1892 String cur;
1893 while ((cur = buffer.readLine()) != null)
1894 {
1895 // System.out.println("int=" + cur);
1896 // System.out.println("**"+cur);
1897 // sb.append(cur + "\n");
1898 ret = ret + cur + "\n";
1899 }
1900 }
1901 catch (IOException e)
1902 {
1903 e.printStackTrace();
1904 }
1905 catch (Exception e)
1906 {
1907 e.printStackTrace();
1908 }
1909
1910 try
1911 {
1912 stream.close();
1913 }
1914 catch (IOException e)
1915 {
1916 e.printStackTrace();
1917 }
1918 catch (Exception e)
1919 {
1920 e.printStackTrace();
1921 }
1922
1923 //return sb.toString();
1924 return ret;
1925 }
1926
1927 }

   
Visit the aagtl Website