/[aagtl_public1]/src/com/zoffcc/applications/aagtl/HTMLDownloader.java
aagtl

Contents of /src/com/zoffcc/applications/aagtl/HTMLDownloader.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3 - (show annotations) (download)
Sun Aug 5 14:00:28 2012 UTC (11 years, 7 months ago) by zoffadmin
File size: 52894 byte(s)
license text correction
1 /**
2 * aagtl Advanced Geocaching Tool for Android
3 * loosely based on agtl by Daniel Fett <fett@danielfett.de>
4 * Copyright (C) 2010 - 2012 Zoff <aagtl@work.zoff.cc>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * version 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 package com.zoffcc.applications.aagtl;
22
23 import java.io.BufferedReader;
24 import java.io.ByteArrayOutputStream;
25 import java.io.File;
26 import java.io.FileInputStream;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStream;
32 import java.io.OutputStreamWriter;
33 import java.io.Reader;
34 import java.io.StringWriter;
35 import java.io.UnsupportedEncodingException;
36 import java.io.Writer;
37 import java.net.HttpURLConnection;
38 import java.net.Socket;
39 import java.net.SocketTimeoutException;
40 import java.net.URI;
41 import java.net.URISyntaxException;
42 import java.net.URL;
43 import java.net.UnknownHostException;
44 import java.security.NoSuchAlgorithmException;
45 import java.security.SecureRandom;
46 import java.text.DecimalFormat;
47 import java.text.DecimalFormatSymbols;
48 import java.util.ArrayList;
49 import java.util.List;
50 import java.util.Locale;
51 import java.util.regex.Matcher;
52 import java.util.regex.Pattern;
53
54 import javax.net.ssl.HostnameVerifier;
55 import javax.net.ssl.HttpsURLConnection;
56 import javax.net.ssl.SSLContext;
57 import javax.net.ssl.SSLSession;
58 import javax.net.ssl.X509TrustManager;
59
60 import net.htmlparser.jericho.FormFields;
61 import net.htmlparser.jericho.HTMLElementName;
62 import net.htmlparser.jericho.Segment;
63 import net.htmlparser.jericho.Source;
64
65 import org.apache.http.HttpEntity;
66 import org.apache.http.HttpHost;
67 import org.apache.http.HttpResponse;
68 import org.apache.http.HttpVersion;
69 import org.apache.http.NameValuePair;
70 import org.apache.http.client.ClientProtocolException;
71 import org.apache.http.client.CookieStore;
72 import org.apache.http.client.entity.UrlEncodedFormEntity;
73 import org.apache.http.client.methods.HttpGet;
74 import org.apache.http.client.methods.HttpPost;
75 import org.apache.http.conn.ClientConnectionManager;
76 import org.apache.http.conn.params.ConnManagerPNames;
77 import org.apache.http.conn.params.ConnPerRouteBean;
78 import org.apache.http.conn.params.ConnRoutePNames;
79 import org.apache.http.conn.scheme.PlainSocketFactory;
80 import org.apache.http.conn.scheme.Scheme;
81 import org.apache.http.conn.scheme.SchemeRegistry;
82 import org.apache.http.cookie.Cookie;
83 import org.apache.http.impl.client.DefaultHttpClient;
84 import org.apache.http.impl.conn.SingleClientConnManager;
85 import org.apache.http.impl.cookie.BasicClientCookie;
86 import org.apache.http.message.BasicNameValuePair;
87 import org.apache.http.params.BasicHttpParams;
88 import org.apache.http.params.HttpConnectionParams;
89 import org.apache.http.params.HttpParams;
90 import org.apache.http.params.HttpProtocolParams;
91 import org.apache.http.protocol.HTTP;
92 import org.json.JSONArray;
93 import org.json.JSONException;
94 import org.json.JSONObject;
95
96 import android.os.Handler;
97 import android.util.Log;
98
99 import com.byarger.exchangeit.EasySSLSocketFactory;
100
101 public class HTMLDownloader
102 {
103
104 boolean logged_in = false;
105 public static CookieStore cookie_jar = null;
106 public static int GC_DOWNLOAD_MAX_REC_DEPTH = 6;
107 aagtl main_aagtl;
108 public static int large_buffer_size = 50 * 1024;
109 public static int default_buffer_size = 50 * 1024;
110
111 // patterns from c:geo opensource
112 private static final Pattern patternLoggedIn = Pattern.compile("<span class=\"Success\">You are logged in as[^<]*<strong[^>]*>([^<]+)</strong>[^<]*</span>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
113 private static final Pattern patternLogged2In = Pattern.compile("<strong>\\W*Hello,[^<]*<a[^>]+>([^<]+)</a>[^<]*</strong>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
114 private static final Pattern patternViewstateFieldCount = Pattern.compile("id=\"__VIEWSTATEFIELDCOUNT\"[^(value)]+value=\"(\\d+)\"[^>]+>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
115 private static final Pattern patternViewstates = Pattern.compile("id=\"__VIEWSTATE(\\d*)\"[^(value)]+value=\"([^\"]+)\"[^>]+>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
116 private static final Pattern patternUserToken = Pattern.compile("userToken\\s*=\\s*'([^']+)'");
117
118 // patterns from c:geo opensource
119
120 public HTMLDownloader(aagtl main)
121 {
122 this.main_aagtl = main;
123 }
124
125 class get_geocaches_ret
126 {
127 int count_p;
128 GeocacheCoordinate[] points;
129 }
130
131 public String convertStreamToString(InputStream is) throws IOException
132 {
133 /*
134 * To convert the InputStream to String we use the
135 * Reader.read(char[] buffer) method. We iterate until the
136 * Reader return -1 which means there's no more data to
137 * read. We use the StringWriter class to produce the string.
138 */
139 if (is != null)
140 {
141 Writer writer = new StringWriter();
142
143 char[] buffer = new char[default_buffer_size];
144 try
145 {
146 Reader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"), large_buffer_size);
147 int n;
148 while ((n = reader.read(buffer)) != -1)
149 {
150 writer.write(buffer, 0, n);
151 }
152 }
153 finally
154 {
155 is.close();
156 }
157 return writer.toString();
158 }
159 else
160 {
161 return null;
162 }
163 }
164
165 public String get_reader_stream(String url, List<NameValuePair> values, ByteArrayOutputStream data, Boolean need_login)
166 {
167
168 if ((need_login) && (!this.logged_in))
169 {
170 System.out.println("--2--- LOGIN START -----");
171 this.logged_in = login();
172 System.out.println("--2--- LOGIN END -----");
173 }
174
175 if ((values == null) && (data == null))
176 {
177 return null;
178 }
179 else if (data == null)
180 {
181 String websiteData = null;
182 URI uri = null;
183 DefaultHttpClient client = new DefaultHttpClient();
184
185 // insert cookies from cookie_jar
186 client.setCookieStore(cookie_jar);
187
188 try
189 {
190 uri = new URI(url);
191 }
192 catch (URISyntaxException e)
193 {
194 e.printStackTrace();
195 return null;
196 }
197
198 HttpPost method = new HttpPost(uri);
199 method.addHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)");
200 method.addHeader("Pragma", "no-cache");
201 method.addHeader("Content-Type", "application/x-www-form-urlencoded");
202
203 HttpEntity entity = null;
204 try
205 {
206 entity = new UrlEncodedFormEntity(values, HTTP.UTF_8);
207 }
208 catch (UnsupportedEncodingException e)
209 {
210 e.printStackTrace();
211 return null;
212 }
213
214 method.addHeader(entity.getContentType());
215 method.setEntity(entity);
216
217 try
218 {
219 HttpResponse res2 = client.execute(method);
220 // System.out.println("login response ->" +
221 // String.valueOf(res2.getStatusLine()));
222 InputStream data2 = res2.getEntity().getContent();
223 websiteData = generateString(data2);
224
225 }
226 catch (ClientProtocolException e)
227 {
228 // e.printStackTrace();
229 return null;
230 }
231 catch (IOException e)
232 {
233 // e.printStackTrace();
234 return null;
235 }
236
237 client.getConnectionManager().shutdown();
238
239 return websiteData;
240
241 }
242 else
243 {
244 try
245 {
246 // url Header textdata
247 InputStream i = this.doPost2(url, values, data);
248 return this.convertStreamToString(i);
249 }
250 catch (IOException e)
251 {
252 e.printStackTrace();
253 return null;
254 }
255 }
256 }
257
258 private InputStream doPost2(String urlString, List<NameValuePair> values, ByteArrayOutputStream content) throws IOException
259 {
260 URL url = new URL(urlString);
261 HttpURLConnection con = (HttpURLConnection) url.openConnection();
262 InputStream in = null;
263 OutputStream out;
264 byte[] buff;
265 con.setRequestMethod("POST");
266 for (int j = 0; j < values.size(); j++)
267 {
268 con.addRequestProperty(values.get(j).getName(), values.get(j).getValue());
269 }
270 String my_cookies = this.getCookies();
271 con.addRequestProperty("Cookie", my_cookies);
272 con.setDoOutput(true);
273 con.setDoInput(true);
274 con.connect();
275 out = con.getOutputStream();
276 buff = content.toByteArray();
277 out.write(buff);
278 out.flush();
279 out.close();
280 in = con.getInputStream();
281
282 return in;
283 }
284
285 public String get_user_token()
286 {
287 String ret = "";
288 String url = "http://www.geocaching.com/map/default.aspx?lat=6&lng=9";
289 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
290 String the_page = get_reader_stream(url, values_list, null, true);
291
292 if (the_page == null)
293 {
294 if (CacheDownloader.DEBUG_) System.out.println("page = NULL");
295 return "";
296 }
297
298 String[] response_lines = the_page.split("\n");
299 String line = null;
300 Pattern p = null;
301 Matcher m = null;
302 for (int i = 0; i < response_lines.length; i++)
303 {
304 line = response_lines[i];
305 // remove spaces at start and end of string
306 line = line.trim();
307
308 if (line.startsWith("var uvtoken"))
309 {
310 if (CacheDownloader.DEBUG_) System.out.println("usertoken=" + line);
311 p = Pattern.compile("userToken[ =]+'([^']+)'");
312 m = p.matcher(line);
313 m.find();
314 if (m.groupCount() > 0)
315 {
316 if (CacheDownloader.DEBUG_) System.out.println("usertoken parsed=" + m.group(1));
317 return m.group(1);
318 }
319 }
320 }
321
322 // + for line in page:
323 //
324 // + if line.startswith('var uvtoken'):
325 //
326 // + self.user_token =
327 // re.compile("userToken[ =]+'([^']+)'").search(line).group(1)
328 //
329 // + page.close()
330 //
331 // + return
332 //
333 // + raise
334 // Exception("Website contents unexpected. Please check connection.")
335
336 return ret;
337 }
338
339 public get_geocaches_ret get_geocaches(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
340 {
341 return get_geocaches_v3(location, count_p, max_p, rec_depth, h, zoom_level);
342 }
343
344 public get_geocaches_ret get_geocaches_v3(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
345 {
346 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
347
348 Coordinate c1 = location[0];
349 Coordinate c2 = location[1];
350
351 Coordinate center = new Coordinate((c1.lat + c2.lat) / 2, (c1.lon + c2.lon) / 2);
352 double dist = (center.distance_to(c1) / 1000) / 2;
353 //System.out.println("distance is " + dist + " meters");
354
355 if (dist > 100)
356 {
357 // dist too large
358 count_p = count_p + 1;
359 get_geocaches_ret r = new get_geocaches_ret();
360 r.count_p = count_p;
361 r.points = null;
362 return r;
363 }
364
365 // use "." as comma seperator!!
366 DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US);
367 otherSymbols.setDecimalSeparator('.');
368 otherSymbols.setGroupingSeparator(',');
369 DecimalFormat format_lat_lon = new DecimalFormat("#.#####", otherSymbols);
370 // use "." as comma seperator!!
371
372 String lat_str = format_lat_lon.format(center.lat);
373 String lon_str = format_lat_lon.format(center.lon);
374 String dist_str = format_lat_lon.format(dist);
375 String url = "http://www.geocaching.com/seek/nearest.aspx?lat=" + lat_str + "&lng=" + lon_str + "&dist=" + dist_str;
376 //System.out.println("url=" + url);
377
378 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
379 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
380 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
381 // ByteArrayOutputStream bs = new ByteArrayOutputStream();
382 String the_page = get_reader_stream(url, values_list, null, true);
383
384 get_geocaches_ret r2 = new get_geocaches_ret();
385 r2.count_p = count_p;
386 r2.points = null;
387
388 Boolean cont = true;
389 Source source = null;
390
391 List<GeocacheCoordinate> gc_list = new ArrayList<GeocacheCoordinate>();
392 count_p = 0;
393 max_p = 0;
394 while (cont)
395 {
396 source = new Source(the_page);
397 List<? extends Segment> segments = (source.getFirstElement("id", "ctl00_ContentBody_ResultsPanel", false).getContent().getFirstElement("class", "PageBuilderWidget", false).getContent().getAllElements("b"));
398 if (segments.size() < 3)
399 {
400 // no results
401 return r2;
402 }
403
404 int count = Integer.parseInt(segments.get(0).getTextExtractor().toString());
405 int page_current = Integer.parseInt(segments.get(1).getTextExtractor().toString());
406 int page_max = Integer.parseInt(segments.get(2).getTextExtractor().toString());
407 //System.out.println("count=" + count + " cur=" + page_current + " max=" + page_max);
408 max_p = count;
409
410 String guid = "";
411 String gccode = "";
412 Boolean disabled = false;
413 List<? extends Segment> segments2 = (source.getFirstElement("class", "SearchResultsTable Table", false).getContent().getAllElements(HTMLElementName.TR));
414 // displaySegments(segments2);
415 try
416 {
417 for (Segment s_ : segments2)
418 {
419 guid = "";
420 disabled = false;
421 gccode = null;
422 try
423 {
424 List<? extends Segment> segments3 = s_.getAllElements("class", "Merge", false);
425 // displaySegments(segments2);
426 guid = segments3.get(0).getFirstElement(HTMLElementName.A).getAttributeValue("href");
427 guid = guid.split("guid=", 3)[1];
428 //System.out.println("guid=:" + guid);
429
430 try
431 {
432 // <a href="/seek/cache_details.aspx?guid=d9dbf39a-e2e6-4640-b951-d1d6307b16bd" class="lnk Strike"><span>Cineasten sehen mehr</span></a>
433 if (segments3.get(1).getFirstElement(HTMLElementName.A).getAttributeValue("class").equalsIgnoreCase("lnk Strike"))
434 {
435 // System.out.println("disabled=:" + disabled);
436 disabled = true;
437 }
438 }
439 catch (Exception e3)
440 {
441 }
442
443 gccode = segments3.get(1).getFirstElement("class", "small", false).getTextExtractor().toString();
444 gccode = gccode.split("\\|")[1].trim();
445 //System.out.println("gccode=:" + gccode);
446 }
447 catch (Exception e2)
448 {
449 e2.printStackTrace();
450 }
451
452 if (gccode != null)
453 {
454 GeocacheCoordinate c__ = null;
455 c__ = new GeocacheCoordinate(0, 0, gccode);
456 if (disabled)
457 {
458 c__.status = GeocacheCoordinate.STATUS_DISABLED;
459 }
460
461 String url2 = "http://www.geocaching.com/seek/cdpf.aspx?guid=" + guid;
462 //System.out.println("url=" + url);
463
464 values_list = new ArrayList<NameValuePair>();
465 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
466 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
467 // bs = new ByteArrayOutputStream();
468 String the_page2 = get_reader_stream(url2, values_list, null, true);
469 c__ = CacheDownloader.__parse_cache_page_print(the_page2, c__);
470 if (c__ != null)
471 {
472 gc_list.add(c__);
473 count_p = count_p + 1;
474 this.main_aagtl.set_bar_slow(h, "get geocaches", c__.title, count_p, max_p, true);
475 }
476 }
477 }
478 }
479 catch (Exception e)
480 {
481 e.printStackTrace();
482 }
483
484 cont = false;
485
486 // ----------- check for paging ----------------
487 // ----------- and run through pages -----------
488 //
489 if (page_current < page_max)
490 {
491 FormFields formFields = source.getFormFields();
492 String vs1 = null;
493 try
494 {
495 vs1 = formFields.getValues("__VIEWSTATE1").get(0);
496 }
497 catch (Exception e)
498 {
499
500 }
501 String vs = null;
502 try
503 {
504 vs = formFields.getValues("__VIEWSTATE").get(0);
505 }
506 catch (Exception e)
507 {
508
509 }
510
511 //System.out.println("vs=" + vs);
512 //System.out.println("vs1=" + vs1);
513
514 List<NameValuePair> values_list2 = new ArrayList<NameValuePair>();
515 values_list2.add(new BasicNameValuePair("__EVENTTARGET", "ctl00$ContentBody$pgrTop$ctl08"));
516 values_list2.add(new BasicNameValuePair("__VIEWSTATEFIELDCOUNT", "2"));
517 values_list2.add(new BasicNameValuePair("__VIEWSTATE", vs));
518 values_list2.add(new BasicNameValuePair("__VIEWSTATE1", vs1));
519 // ByteArrayOutputStream bs = new ByteArrayOutputStream();
520 the_page = get_reader_stream(url, values_list2, null, true);
521 cont = true;
522 }
523 // ----------- check for paging ----------------
524 // ----------- and run through pages -----------
525 }
526
527 int jk;
528 r2.count_p = gc_list.size();
529 r2.points = new GeocacheCoordinate[gc_list.size()];
530
531 for (jk = 0; jk < gc_list.size(); jk++)
532 {
533 r2.points[jk] = gc_list.get(jk);
534 }
535 // gc_list.clear();
536
537 return r2;
538 }
539
540 private static void displaySegments(List<? extends Segment> segmentsx)
541 {
542 for (Segment segment1 : segmentsx)
543 {
544 System.out.println("-------------------------------------------------------------------------------");
545 System.out.println(segment1.getDebugInfo());
546 System.out.println(segment1);
547 }
548 System.out.println("\n*******************************************************************************\n");
549 }
550
551 public get_geocaches_ret get_geocaches_v2(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
552 {
553 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
554
555 if (zoom_level > 16)
556 {
557 zoom_level = 16;
558 }
559 else if (zoom_level < 6)
560 {
561 zoom_level = 6;
562 }
563
564 get_geocaches_ret rr2 = new get_geocaches_ret();
565 Coordinate c1 = location[0];
566 Coordinate c2 = location[1];
567
568 String zoomlevel_ = "" + zoom_level;
569
570 // use "." as comma seperator!!
571 DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US);
572 otherSymbols.setDecimalSeparator('.');
573 otherSymbols.setGroupingSeparator(',');
574 DecimalFormat format_lat_lon = new DecimalFormat("#.#####", otherSymbols);
575 // use "." as comma seperator!!
576
577 // http://www.geocaching.com/map/default.aspx?ll=48.22607,16.36997
578 // GET http://www.geocaching.com/map/default.aspx?ll=48.22607,16.36997 HTTP/1.1
579 String lat_str = format_lat_lon.format((c1.lat + c2.lat) / 2);
580 String lon_str = format_lat_lon.format((c1.lon + c2.lon) / 2);
581 //String url = "http://www.geocaching.com/map/?ll=" + lat_str + "," + lon_str + "&z=" + zoomlevel_;
582 String url = "http://www.geocaching.com/map/default.aspx?ll=" + lat_str + "," + lon_str;
583 System.out.println("url=" + url);
584
585 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
586 //*values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
587 //*values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
588 // values_list.add(new BasicNameValuePair("Referer","http://www.geocaching.com/map/default.aspx"));
589 //ByteArrayOutputStream bs = new ByteArrayOutputStream();
590 String the_page = get_reader_stream(url, values_list, null, true);
591 //String the_page = getUrlData(url);
592
593 //System.out.println(the_page);
594
595 // get values --------------------------
596 // get values --------------------------
597 // Groundspeak.Map.UserSession('xxxx',
598 Pattern p = Pattern.compile("Groundspeak\\.Map\\.UserSession.'([^']*)'");
599 Matcher m = p.matcher(the_page);
600 Boolean has_found = true;
601 String kk_ = "";
602 try
603 {
604 has_found = m.find();
605 kk_ = m.group(1);
606 }
607 catch (Exception e3)
608 {
609 e3.printStackTrace();
610 }
611 System.out.println("kk=" + kk_);
612
613 // sessionToken:'
614 p = Pattern.compile("sessionToken:'([^']*)'");
615 m = p.matcher(the_page);
616 has_found = m.find();
617 String sess_token_ = m.group(1);
618
619 System.out.println("sess_token=" + sess_token_);
620 // get values --------------------------
621 // get values --------------------------
622
623 java.util.Date date = new java.util.Date();
624 System.out.println("ts=" + date.getTime());
625
626 String timestamp_ = "" + date.getTime(); // ..seconds..
627
628 // lat, lon -> tile
629 Coordinate coord = new Coordinate((c1.lat + c2.lat) / 2, (c1.lon + c2.lon) / 2);
630 double[] tile = main_aagtl.rose.deg2num_give_zoom(coord, zoom_level);
631 String xtile_num = "" + ((int) tile[0]);
632 String ytile_num = "" + ((int) tile[1]);
633 // lat, lon -> tile
634
635 // X-Requested-With: XMLHttpRequest
636 //Referer: http://www.geocaching.com/map/default.aspx?ll=48.22607,16.36997
637 values_list = new ArrayList<NameValuePair>();
638 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
639 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
640 //values_list.add(new BasicNameValuePair("X-Requested-With", "XMLHttpRequest"));
641 //values_list.add(new BasicNameValuePair("Referer", "http://www.geocaching.com/map/default.aspx?ll=" + lat_str + "," + lon_str));
642
643 String url2 = "http://www.geocaching.com/map/map.info?x=" + xtile_num + "&y=" + ytile_num + "&z=" + zoomlevel_ + "&k=" + kk_ + "&st=" + sess_token_ + "&ep=1&_=" + timestamp_;
644
645 System.out.println("url2=" + url2);
646
647 the_page = get_reader_stream(url2, values_list, null, true);
648 // the_page = getUrlData(url2);
649 try
650 {
651 System.out.println(the_page);
652 }
653 catch (Exception e3)
654 {
655
656 }
657
658 // we need to add one, in any case!
659 count_p = count_p + 1;
660 get_geocaches_ret r = new get_geocaches_ret();
661 r.count_p = count_p;
662 r.points = null;
663 return r;
664 }
665
666 public get_geocaches_ret get_geocaches_v1(Coordinate[] location, int count_p, int max_p, int rec_depth, Handler h, int zoom_level)
667 {
668
669 if (rec_depth > GC_DOWNLOAD_MAX_REC_DEPTH)
670 {
671 // we need to add one, in any case!
672 count_p = count_p + 1;
673 get_geocaches_ret r = new get_geocaches_ret();
674 r.count_p = count_p;
675 r.points = null;
676 return r;
677 }
678
679 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
680
681 get_geocaches_ret rr2 = new get_geocaches_ret();
682
683 Coordinate c1 = location[0];
684 Coordinate c2 = location[1];
685
686 String url = "http://www.geocaching.com/map/default.aspx/MapAction?lat=9&lng=6";
687
688 String user_token = "";
689 user_token = get_user_token();
690
691 List<NameValuePair> values_list = new ArrayList<NameValuePair>();
692 DecimalFormat format = new DecimalFormat("#.##########");
693 String body_data = String.format("{\"dto\":{\"data\":{\"c\":1,\"m\":\"\",\"d\":\"%s|%s|%s|%s\"},\"ut\":\"%s\"}}", format.format(Math.max(c1.lat, c2.lat)).replace(",", "."), format.format(Math.min(c1.lat, c2.lat)).replace(",", "."), format.format(Math.max(c1.lon, c2.lon)).replace(",", "."), format.format(Math.min(c1.lon, c2.lon)).replace(",", "."), user_token);
694
695 // System.out.println(body_data);
696
697 values_list.add(new BasicNameValuePair("Content-Type", "application/json"));
698 values_list.add(new BasicNameValuePair("Content-Length", String.valueOf(body_data.length())));
699 //values_list.add(new BasicNameValuePair("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)"));
700 //values_list.add(new BasicNameValuePair("Pragma", "no-cache"));
701 ByteArrayOutputStream bs = new ByteArrayOutputStream();
702 try
703 {
704 bs.write(body_data.getBytes());
705 }
706 catch (IOException e1)
707 {
708 e1.printStackTrace();
709 }
710 String the_page = get_reader_stream(url, values_list, bs, true);
711
712 Boolean found = false;
713 JSONObject a = null;
714
715 try
716 {
717 a = new JSONObject(the_page);
718 // System.out.println(a.get("d"));
719 // get "d:" which seems to be only String? whatever, this works
720 // anyway
721 a = new JSONObject(a.get("d").toString());
722 found = true;
723 }
724 catch (JSONException e)
725 {
726 e.printStackTrace();
727 System.out.println("get_geocaches_ret: JSONException1");
728
729 rr2.count_p = count_p;
730 rr2.points = null;
731 return rr2;
732 }
733 catch (Exception e)
734 {
735 e.printStackTrace();
736 System.out.println("get_geocaches_ret: general Exception");
737 }
738
739 // string not found -> no caches got from website!
740 if (!found)
741 {
742 rr2.count_p = count_p;
743 rr2.points = null;
744 return rr2;
745 }
746
747 try
748 {
749 JSONObject _cs = a.getJSONObject("cs");
750
751 Boolean has_cc = false;
752 try
753 {
754 JSONArray _check = _cs.getJSONArray("cc");
755 has_cc = true;
756 }
757 catch (JSONException e)
758 {
759 // cs not found
760 has_cc = false;
761 }
762
763 if (!has_cc)
764 {
765 // 1nd part ----------------------
766 // 1nd part ----------------------
767 // System.out.println("part 1");
768
769 Boolean has_count_greater_zero = false;
770 int count_parsed = 0;
771 try
772 {
773 count_parsed = _cs.getInt("count");
774 if (count_parsed > 0)
775 {
776 // System.out.println("count_parsed: " + count_parsed);
777 has_count_greater_zero = true;
778 }
779 }
780 catch (JSONException e)
781 {
782 // has_count_greater_zero not found
783 has_count_greater_zero = false;
784 }
785
786 if (has_count_greater_zero)
787 {
788
789 double middle_lat = (c1.lat + c2.lat) / 2;
790 double middle_lon = (c1.lon + c2.lon) / 2;
791 double lat_left_top = Math.min(c1.lat, c2.lat);
792 double lon_left_top = Math.min(c1.lon, c2.lon);
793 double lat_right_bottom = Math.max(c1.lat, c2.lat);
794 double lon_right_bottom = Math.max(c1.lon, c2.lon);
795
796 Coordinate t1 = new Coordinate(lat_left_top, lon_left_top);
797 Coordinate t2 = new Coordinate(middle_lat, middle_lon);
798 Coordinate[] coord_temp = new Coordinate[2];
799 coord_temp[0] = t1;
800 coord_temp[1] = t2;
801 get_geocaches_ret r_temp = new get_geocaches_ret();
802 r_temp = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
803 count_p = r_temp.count_p;
804
805 t1 = new Coordinate(middle_lat, lon_left_top);
806 t2 = new Coordinate(lat_right_bottom, middle_lon);
807 coord_temp = new Coordinate[2];
808 coord_temp[0] = t1;
809 coord_temp[1] = t2;
810 get_geocaches_ret r_temp2 = new get_geocaches_ret();
811 r_temp2 = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
812 count_p = r_temp2.count_p;
813
814 t1 = new Coordinate(lat_left_top, middle_lon);
815 t2 = new Coordinate(middle_lat, lon_right_bottom);
816 coord_temp = new Coordinate[2];
817 coord_temp[0] = t1;
818 coord_temp[1] = t2;
819 get_geocaches_ret r_temp3 = new get_geocaches_ret();
820 r_temp3 = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
821 count_p = r_temp3.count_p;
822
823 t1 = new Coordinate(middle_lat, middle_lon);
824 t2 = new Coordinate(lat_right_bottom, lon_right_bottom);
825 coord_temp = new Coordinate[2];
826 coord_temp[0] = t1;
827 coord_temp[1] = t2;
828 get_geocaches_ret r_temp4 = new get_geocaches_ret();
829 r_temp4 = this.get_geocaches(coord_temp, count_p, max_p, rec_depth + 1, h, zoom_level);
830 count_p = r_temp4.count_p;
831
832 int l1 = 0;
833 int l2 = 0;
834 int l3 = 0;
835 int l4 = 0;
836 if (r_temp.points != null)
837 {
838 l1 = r_temp.points.length;
839 }
840 if (r_temp2.points != null)
841 {
842 l2 = r_temp2.points.length;
843 }
844 if (r_temp3.points != null)
845 {
846 l3 = r_temp3.points.length;
847 }
848 if (r_temp4.points != null)
849 {
850 l4 = r_temp4.points.length;
851 }
852 GeocacheCoordinate[] points2 = new GeocacheCoordinate[l1 + l2 + l3 + l4];
853 int j = 0;
854 int j_old = 0;
855 if (l1 > 0)
856 {
857 for (j = 0; j < r_temp.points.length; j++)
858 {
859 points2[j_old] = r_temp.points[j];
860 j_old++;
861 }
862 }
863 if (l2 > 0)
864 {
865 for (j = 0; j < r_temp2.points.length; j++)
866 {
867 points2[j_old] = r_temp2.points[j];
868 j_old++;
869 }
870 }
871 if (l3 > 0)
872 {
873
874 for (j = 0; j < r_temp3.points.length; j++)
875 {
876 points2[j_old] = r_temp3.points[j];
877 j_old++;
878 }
879 }
880 if (l4 > 0)
881 {
882
883 for (j = 0; j < r_temp4.points.length; j++)
884 {
885 points2[j_old] = r_temp4.points[j];
886 j_old++;
887 }
888 }
889
890 rr2.count_p = count_p;
891 rr2.points = new GeocacheCoordinate[l1 + l2 + l3 + l4];
892 rr2.points = points2;
893 return rr2;
894
895 }
896
897 // 1nd part ----------------------
898 // 1nd part ----------------------
899
900 }
901 else
902 {
903 // 2nd part ----------------------
904 // 2nd part ----------------------
905 // System.out.println("part 2");
906
907 // make progressbar look better
908 count_p = count_p + 3;
909
910 JSONArray _b = _cs.getJSONArray("cc");
911 GeocacheCoordinate[] points2 = new GeocacheCoordinate[_b.length()];
912
913 int zw_count = 0;
914 int zw_count_draw = 0;
915 int zw_max_p_2 = _b.length();
916 int zw_draw_every = 1 + (int) ((float) (zw_max_p_2) / (float) (11));
917
918 for (int i = 0; i < _b.length(); i++)
919 {
920 GeocacheCoordinate c__ = null;
921 c__ = new GeocacheCoordinate(_b.getJSONObject(i).getDouble("lat"), _b.getJSONObject(i).getDouble("lon"), _b.getJSONObject(i).getString("gc").toString());
922
923 // System.out.println("11 " +
924 // _b.getJSONObject(i).getDouble("lat") + " "
925 // + _b.getJSONObject(i).getDouble("lon"));
926
927 // System.out.println("22 " + c__.lat + " " + c__.lon);
928 // System.out.println("33 " +
929 // _b.getJSONObject(i).toString());
930
931 // title
932 c__.title = _b.getJSONObject(i).getString("nn");
933
934 // System.out.println("dd1");
935
936 // type
937 String temp = _b.getJSONObject(i).getString("ctid");
938
939 try
940 {
941 c__.type = GeocacheCoordinate.GC_TYPE_HASH.get(temp);
942 }
943 catch (Exception e)
944 {
945 System.out.println("Unknown GC Type!!");
946 c__.type = GeocacheCoordinate.TYPE_UNKNOWN;
947 }
948
949 // System.out.println("dd2");
950
951 // found
952 c__.found = _b.getJSONObject(i).getBoolean("f");
953
954 // System.out.println("" + _b.getJSONObject(i).toString());
955
956 // status
957 Boolean temp_b = _b.getJSONObject(i).getBoolean("ia");
958 if (temp_b)
959 {
960 c__.status = GeocacheCoordinate.STATUS_NORMAL;
961 }
962 else
963 {
964 c__.status = GeocacheCoordinate.STATUS_DISABLED;
965 }
966
967 points2[i] = c__;
968
969 zw_count++;
970 zw_count_draw++;
971 this.main_aagtl.set_bar_slow(h, "get geocaches", c__.title, zw_count, zw_max_p_2, true);
972
973 if (zw_count_draw >= zw_draw_every)
974 {
975 zw_count_draw = 0;
976 }
977
978 }
979
980 count_p = count_p + 1;
981 this.main_aagtl.set_bar_slow(h, "get geocaches", "downloading ...", count_p, max_p, true);
982
983 rr2.count_p = count_p;
984 rr2.points = points2;
985 return rr2;
986
987 // 2nd part ----------------------
988 // 2nd part ----------------------
989 }
990 }
991 catch (JSONException e)
992 {
993 System.out.println("get_geocaches_ret: JSONException2");
994
995 rr2.count_p = count_p;
996 rr2.points = null;
997 return rr2;
998 }
999
1000 return rr2;
1001 }
1002
1003 public static boolean isEmpty(String[] a)
1004 {
1005 if (a == null)
1006 {
1007 return true;
1008 }
1009
1010 for (String s : a)
1011 {
1012 if ((s == null) || (s.equals("")))
1013 {
1014 return true;
1015 }
1016 }
1017 return false;
1018 }
1019
1020 /**
1021 * read all viewstates from page
1022 *
1023 * @return String[] with all view states
1024 */
1025 public static String[] getViewstates(String page)
1026 {
1027 // Get the number of viewstates.
1028 // If there is only one viewstate, __VIEWSTATEFIELDCOUNT is not present
1029 int count = 1;
1030 final Matcher matcherViewstateCount = patternViewstateFieldCount.matcher(page);
1031 if (matcherViewstateCount.find())
1032 {
1033 count = Integer.parseInt(matcherViewstateCount.group(1));
1034 }
1035
1036 String[] viewstates = new String[count];
1037
1038 // Get the viewstates
1039 int no;
1040 final Matcher matcherViewstates = patternViewstates.matcher(page);
1041 while (matcherViewstates.find())
1042 {
1043 String sno = matcherViewstates.group(1); // number of viewstate
1044 if ("".equals(sno))
1045 {
1046 no = 0;
1047 }
1048 else
1049 {
1050 no = Integer.parseInt(sno);
1051 }
1052 // System.out.println("1 v " + no + "=" + matcherViewstates.group(2));
1053 viewstates[no] = matcherViewstates.group(2);
1054 }
1055
1056 return viewstates;
1057 }
1058
1059 /**
1060 * put viewstates into request parameters
1061 */
1062 private static void putViewstates(List<NameValuePair> params, String[] viewstates)
1063 {
1064 if (isEmpty(viewstates))
1065 {
1066 //System.out.println("EEEEEEEEEEEEEEE*EEEEEEEE");
1067 return;// params;
1068 }
1069 // System.out.println("***** __VIEWSTATE=" + viewstates[0]);
1070 params.add(new BasicNameValuePair("__VIEWSTATE", viewstates[0]));
1071 if (viewstates.length > 1)
1072 {
1073 for (int i = 1; i < viewstates.length; i++)
1074 {
1075 params.add(new BasicNameValuePair("__VIEWSTATE" + i, viewstates[i]));
1076 }
1077 params.add(new BasicNameValuePair("__VIEWSTATEFIELDCOUNT", viewstates.length + ""));
1078 }
1079 return;// params;
1080 }
1081
1082 /**
1083 * transfers the viewstates variables from a page (response) to parameters
1084 * (next request)
1085 */
1086 public static void transferViewstates(String page, List<NameValuePair> params)
1087 {
1088 putViewstates(params, getViewstates(page));
1089 }
1090
1091 private void trust_Every_ssl_cert()
1092 {
1093 // NEVER enable this on a production release!!!!!!!!!!
1094 try
1095 {
1096 HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier()
1097 {
1098 public boolean verify(String hostname, SSLSession session)
1099 {
1100 Log.d("aagtl", "DANGER !!! trusted hostname=" + hostname + " DANGER !!!");
1101 // return true -> mean we trust this cert !! DANGER !! DANGER !!
1102 return true;
1103 }
1104 });
1105 SSLContext context = SSLContext.getInstance("TLS");
1106 context.init(null, new X509TrustManager[] { new X509TrustManager()
1107 {
1108 public java.security.cert.X509Certificate[] getAcceptedIssuers()
1109 {
1110 Log.d("aagtl", "DANGER !!! 222222222");
1111 return new java.security.cert.X509Certificate[0];
1112 }
1113
1114 public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) throws java.security.cert.CertificateException
1115 {
1116 Log.d("aagtl", "DANGER !!! 333333333");
1117 }
1118
1119 public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) throws java.security.cert.CertificateException
1120 {
1121 Log.d("aagtl", "DANGER !!! 444444444444");
1122 }
1123 } }, new SecureRandom());
1124 HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
1125 }
1126 catch (Exception e)
1127 {
1128 e.printStackTrace();
1129 }
1130 // NEVER enable this on a production release!!!!!!!!!!
1131 }
1132
1133 public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException
1134 {
1135 SSLContext sslContext = null;
1136 try
1137 {
1138 sslContext = SSLContext.getInstance("TLS");
1139 }
1140 catch (NoSuchAlgorithmException e)
1141 {
1142 e.printStackTrace();
1143 }
1144 return sslContext.getSocketFactory().createSocket(socket, host, port, autoClose);
1145 }
1146
1147 public boolean login()
1148 {
1149 // System.out.println("--L--- LOGIN START -----");
1150
1151 String login_url = "https://www.geocaching.com/login/default.aspx";
1152
1153 DefaultHttpClient client2 = null;
1154 HttpHost proxy = null;
1155
1156 if (CacheDownloader.DEBUG2_)
1157 {
1158 // NEVER enable this on a production release!!!!!!!!!!
1159 // NEVER enable this on a production release!!!!!!!!!!
1160 trust_Every_ssl_cert();
1161
1162 SchemeRegistry schemeRegistry = new SchemeRegistry();
1163 schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
1164 schemeRegistry.register(new Scheme("https", new EasySSLSocketFactory(), 443));
1165
1166 HttpParams params = new BasicHttpParams();
1167 params.setParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 30);
1168 params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRouteBean(30));
1169 params.setParameter(HttpProtocolParams.USE_EXPECT_CONTINUE, false);
1170
1171 proxy = new HttpHost("192.168.0.1", 8888);
1172 params.setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
1173
1174 HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
1175
1176 ClientConnectionManager cm = new SingleClientConnManager(params, schemeRegistry);
1177 client2 = new DefaultHttpClient(cm, params);
1178 // NEVER enable this on a production release!!!!!!!!!!
1179 // NEVER enable this on a production release!!!!!!!!!!
1180 }
1181 else
1182 {
1183 HttpParams httpParameters = new BasicHttpParams();
1184 int timeoutConnection = 10000;
1185 HttpConnectionParams.setConnectionTimeout(httpParameters, timeoutConnection);
1186 int timeoutSocket = 10000;
1187 HttpConnectionParams.setSoTimeout(httpParameters, timeoutSocket);
1188
1189 client2 = new DefaultHttpClient(httpParameters);
1190 }
1191
1192 if (CacheDownloader.DEBUG2_)
1193 {
1194 client2.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
1195 }
1196
1197 // read first page to check for login
1198 // read first page to check for login
1199 // read first page to check for login
1200
1201 String websiteData2 = null;
1202 URI uri2 = null;
1203 try
1204 {
1205 uri2 = new URI(login_url);
1206 }
1207 catch (URISyntaxException e)
1208 {
1209 e.printStackTrace();
1210 return false;
1211 }
1212
1213 client2.setCookieStore(cookie_jar);
1214
1215 HttpGet method2 = new HttpGet(uri2);
1216
1217 method2.addHeader("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.10) Gecko/20071115 Firefox/2.0.0.10");
1218 method2.addHeader("Pragma", "no-cache");
1219 method2.addHeader("Accept-Language", "en");
1220
1221 HttpResponse res = null;
1222 try
1223 {
1224 res = client2.execute(method2);
1225 }
1226 catch (ClientProtocolException e1)
1227 {
1228 e1.printStackTrace();
1229 return false;
1230 }
1231 catch (IOException e1)
1232 {
1233 e1.printStackTrace();
1234 return false;
1235 }
1236 catch (Exception e)
1237 {
1238 e.printStackTrace();
1239 return false;
1240 }
1241
1242 InputStream data = null;
1243 try
1244 {
1245 data = res.getEntity().getContent();
1246 }
1247 catch (IllegalStateException e1)
1248 {
1249 e1.printStackTrace();
1250 return false;
1251 }
1252 catch (IOException e1)
1253 {
1254 e1.printStackTrace();
1255 return false;
1256 }
1257 catch (Exception e)
1258 {
1259 e.printStackTrace();
1260 return false;
1261 }
1262
1263 websiteData2 = generateString(data);
1264
1265 if (CacheDownloader.DEBUG_) System.out.println("111 cookies=" + dumpCookieStore(client2.getCookieStore()));
1266 cookie_jar = client2.getCookieStore();
1267
1268 // on every page
1269 final Matcher matcherLogged2In = patternLogged2In.matcher(websiteData2);
1270 if (matcherLogged2In.find())
1271 {
1272 if (CacheDownloader.DEBUG_) System.out.println("login: -> already logged in (1)");
1273 return true;
1274 }
1275
1276 // after login
1277 final Matcher matcherLoggedIn = patternLoggedIn.matcher(websiteData2);
1278 if (matcherLoggedIn.find())
1279 {
1280 if (CacheDownloader.DEBUG_) System.out.println("login: -> already logged in (2)");
1281 return true;
1282 }
1283 // read first page to check for login
1284 // read first page to check for login
1285 // read first page to check for login
1286
1287 // ok post login data as formdata
1288 // ok post login data as formdata
1289 // ok post login data as formdata
1290 // ok post login data as formdata
1291 HttpPost method = new HttpPost(uri2);
1292
1293 method.addHeader("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0) Firefox/7.0");
1294 method.addHeader("Pragma", "no-cache");
1295 method.addHeader("Content-Type", "application/x-www-form-urlencoded");
1296 method.addHeader("Accept-Language", "en");
1297
1298 List<NameValuePair> loginInfo = new ArrayList<NameValuePair>();
1299
1300 loginInfo.add(new BasicNameValuePair("__EVENTTARGET", ""));
1301 loginInfo.add(new BasicNameValuePair("__EVENTARGUMENT", ""));
1302
1303 String[] viewstates = getViewstates(websiteData2);
1304 if (CacheDownloader.DEBUG_) System.out.println("vs==" + viewstates[0]);
1305 putViewstates(loginInfo, viewstates);
1306
1307 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$tbUsername", this.main_aagtl.global_settings.options_username));
1308 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$tbPassword", this.main_aagtl.global_settings.options_password));
1309 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$btnSignIn", "Login"));
1310 loginInfo.add(new BasicNameValuePair("ctl00$ContentBody$cbRememberMe", "on"));
1311
1312 //for (int i = 0; i < loginInfo.size(); i++)
1313 //{
1314 // System.out.println("x*X " + loginInfo.get(i).getName() + " " + loginInfo.get(i).getValue());
1315 //}
1316
1317 // set cookies
1318 client2.setCookieStore(cookie_jar);
1319
1320 HttpEntity entity = null;
1321 try
1322 {
1323 entity = new UrlEncodedFormEntity(loginInfo, HTTP.UTF_8);
1324 }
1325 catch (UnsupportedEncodingException e)
1326 {
1327 e.printStackTrace();
1328 return false;
1329 }
1330
1331 // try
1332 // {
1333 // System.out.println("1 " + entity.toString());
1334 // InputStream in2 = entity.getContent();
1335 // InputStreamReader ir2 = new InputStreamReader(in2, "utf-8");
1336 // BufferedReader r = new BufferedReader(ir2);
1337 // System.out.println("line=" + r.readLine());
1338 // }
1339 // catch (Exception e2)
1340 // {
1341 // e2.printStackTrace();
1342 // }
1343
1344 method.setEntity(entity);
1345
1346 HttpResponse res2 = null;
1347 try
1348 {
1349 res2 = client2.execute(method);
1350 if (CacheDownloader.DEBUG_) System.out.println("login response ->" + String.valueOf(res2.getStatusLine()));
1351 }
1352 catch (ClientProtocolException e)
1353 {
1354 e.printStackTrace();
1355 return false;
1356 }
1357 catch (IOException e)
1358 {
1359 e.printStackTrace();
1360 return false;
1361 }
1362 catch (Exception e)
1363 {
1364 e.printStackTrace();
1365 return false;
1366 }
1367
1368 // for (int x = 0; x < res2.getAllHeaders().length; x++)
1369 // {
1370 // System.out.println("## n=" + res2.getAllHeaders()[x].getName());
1371 // System.out.println("## v=" + res2.getAllHeaders()[x].getValue());
1372 // }
1373
1374 InputStream data2 = null;
1375 try
1376 {
1377 data2 = res2.getEntity().getContent();
1378 }
1379 catch (IllegalStateException e1)
1380 {
1381 e1.printStackTrace();
1382 return false;
1383 }
1384 catch (IOException e1)
1385 {
1386 e1.printStackTrace();
1387 return false;
1388 }
1389 catch (Exception e)
1390 {
1391 e.printStackTrace();
1392 return false;
1393 }
1394
1395 websiteData2 = generateString(data2);
1396
1397 if (CacheDownloader.DEBUG_) System.out.println("2222 cookies=" + dumpCookieStore(client2.getCookieStore()));
1398 cookie_jar = client2.getCookieStore();
1399
1400 String[] viewstates2 = getViewstates(websiteData2);
1401 if (CacheDownloader.DEBUG_) System.out.println("vs==" + viewstates2[0]);
1402
1403 // System.out.println("******");
1404 // System.out.println("******");
1405 // System.out.println("******");
1406 // System.out.println(websiteData2);
1407 // System.out.println("******");
1408 // System.out.println("******");
1409 // System.out.println("******");
1410
1411 return true;
1412
1413 }
1414
1415 public static String dumpCookieStore(CookieStore cookieStore)
1416 {
1417 StringBuilder cookies = new StringBuilder();
1418 for (Cookie cookie : cookieStore.getCookies())
1419 {
1420 System.out.println("CC**\n\n");
1421 cookies.append(cookie.getName());
1422 cookies.append("=");
1423 cookies.append(cookie.getValue());
1424 cookies.append("=");
1425 cookies.append(cookie.getDomain());
1426 cookies.append(";" + "\n");
1427 }
1428 return cookies.toString();
1429 }
1430
1431 public void saveCookies()
1432 {
1433 FileOutputStream fOut = null;
1434 OutputStreamWriter osw = null;
1435 // make dirs
1436 File dir1 = new File(this.main_aagtl.main_dir + "/config");
1437 dir1.mkdirs();
1438 // save cookies to file
1439 try
1440 {
1441 File cookie_file = new File(this.main_aagtl.main_dir + "/config/cookie.txt");
1442 fOut = new FileOutputStream(cookie_file);
1443 osw = new OutputStreamWriter(fOut);
1444 osw.write(cookie_jar.toString());
1445 osw.flush();
1446 }
1447 catch (Exception e)
1448 {
1449 e.printStackTrace();
1450 System.out.println("saveCookies: Exception1");
1451 }
1452 finally
1453 {
1454 try
1455 {
1456 osw.close();
1457 fOut.close();
1458 }
1459 catch (IOException e)
1460 {
1461 e.printStackTrace();
1462 System.out.println("saveCookies: Exception2");
1463 }
1464 }
1465 }
1466
1467 public String getCookies()
1468 {
1469 String ret = "";
1470
1471 // System.out.println("getCookies size: " +
1472 // cookie_jar.getCookies().size());
1473 try
1474 {
1475 for (int i = 0; i < cookie_jar.getCookies().size(); i++)
1476 {
1477 // System.out.println("getCookies: " +
1478 // cookie_jar.getCookies().get(i).toString());
1479 if (i > 0)
1480 {
1481 ret = ret + "; ";
1482 }
1483 ret = ret + cookie_jar.getCookies().get(i).getName() + "=" + cookie_jar.getCookies().get(i).getValue();
1484 }
1485 }
1486 catch (Exception e)
1487 {
1488 e.printStackTrace();
1489 // System.out.println("getCookies: Exception");
1490 }
1491 // System.out.println("getCookies: ret=" + ret);
1492 return ret;
1493 }
1494
1495 public String[] getCookies2()
1496 {
1497 if (cookie_jar.getCookies().size() == 0)
1498 {
1499 return null;
1500 }
1501 String[] ret = new String[cookie_jar.getCookies().size() * 2];
1502
1503 // System.out.println("getCookies size: " +
1504 // cookie_jar.getCookies().size());
1505 try
1506 {
1507 for (int i = 0; i < cookie_jar.getCookies().size(); i++)
1508 {
1509 // System.out.println("getCookies: " +
1510 // cookie_jar.getCookies().get(i).toString());
1511 ret[i * 2] = cookie_jar.getCookies().get(i).getName();
1512 ret[(i * 2) + 1] = cookie_jar.getCookies().get(i).getValue();
1513 }
1514 }
1515 catch (Exception e)
1516 {
1517 e.printStackTrace();
1518 // System.out.println("getCookies: Exception");
1519 }
1520 return ret;
1521 }
1522
1523 public void loadCookies()
1524 {
1525 String[] ret = new String[2];
1526
1527 // make dirs
1528 File dir1 = new File(this.main_aagtl.main_dir + "/config");
1529 dir1.mkdirs();
1530 // load cookies from file
1531 File cookie_file = new File(this.main_aagtl.main_dir + "/config/cookie.txt");
1532
1533 FileInputStream fIn = null;
1534 InputStreamReader isr = null;
1535 char[] inputBuffer = new char[255];
1536 Writer writer = new StringWriter();
1537 String data = null;
1538 try
1539 {
1540 fIn = new FileInputStream(cookie_file);
1541 isr = new InputStreamReader(fIn);
1542 int n = 0;
1543 while ((n = isr.read(inputBuffer)) != -1)
1544 {
1545 writer.write(inputBuffer, 0, n);
1546 }
1547 data = writer.toString();
1548 }
1549 catch (Exception e)
1550 {
1551 e.printStackTrace();
1552 System.out.println("loadCookies: Exception1");
1553 return;
1554 }
1555 finally
1556 {
1557 try
1558 {
1559 isr.close();
1560 fIn.close();
1561 }
1562 catch (NullPointerException e2)
1563 {
1564 System.out.println("loadCookies: Exception2");
1565 return;
1566 }
1567 catch (IOException e)
1568 {
1569 System.out.println("loadCookies: Exception3");
1570
1571 e.printStackTrace();
1572 return;
1573 }
1574 }
1575
1576 if (cookie_jar == null)
1577 {
1578 // cookie_jar = new CookieStore();
1579 return;
1580 }
1581 else
1582 {
1583 cookie_jar.clear();
1584 }
1585
1586 // Log.d("load cookie:", "->" + String.valueOf(data));
1587
1588 // [[version: 0][name: ASP.NET_SessionId]
1589 // [value: cyuoctxrwio1x13vivqzlxgi][domain: www.geocaching.com]
1590 // [path: /][expiry: null], [version: 0][name: userid]
1591 // [value: 8a72e55f-419c-4da7-8de3-7813a3fda9c7][domain:
1592 // www.geocaching.com]
1593 // [path: /][expiry: Tue Apr 26 15:41:14 Europe/Belgrade 2011]]
1594
1595 if (data.length() > 1)
1596 {
1597 // check for multpile cookies
1598 if (data.startsWith("[["))
1599 {
1600 // strip [ and ] at begin and end of string
1601 data = data.substring(1, data.length() - 1);
1602 String s3 = "\\], \\[";
1603 String[] a3 = data.split(s3);
1604 String data_cookie;
1605 for (int j3 = 0; j3 < a3.length; j3++)
1606 {
1607 data_cookie = a3[j3];
1608 if (j3 == 0)
1609 {
1610 data_cookie = data_cookie + "]";
1611 }
1612 else
1613 {
1614 data_cookie = "[" + data_cookie;
1615 }
1616 // System.out.println("parsing cookie #" + j3 + ": " +
1617 // data_cookie);
1618
1619 String s2 = "]";
1620 String[] a = data_cookie.split(s2);
1621 String x = null;
1622 String c1, c2 = null;
1623 String c_name = null, c_value = null, c_domain = null, c_path = null;
1624 String c_version = null;
1625 BasicClientCookie this_cookie = null;
1626
1627 for (int j = 0; j < a.length; j++)
1628 {
1629 x = a[j].replace("[", "").trim();
1630 c1 = x.split(":")[0];
1631 c2 = x.split(":")[1].substring(1);
1632 // Log.d("load cookie:", "->" + String.valueOf(c1));
1633 // Log.d("load cookie:", "->" + String.valueOf(c2));
1634 if (c1.matches("name") == true)
1635 {
1636 // Log.d("name:", "->" + String.valueOf(c1));
1637 c_name = c2;
1638 }
1639 else if (c1.matches("value") == true)
1640 {
1641 c_value = c2;
1642 }
1643 else if (c1.matches("domain") == true)
1644 {
1645 c_domain = c2;
1646 }
1647 else if (c1.matches("path") == true)
1648 {
1649 c_path = c2;
1650 }
1651 else if (c1.matches("version") == true)
1652 {
1653 c_version = c2;
1654 }
1655 }
1656 this_cookie = new BasicClientCookie(c_name, c_value);
1657 this_cookie.setDomain(c_domain);
1658 this_cookie.setPath(c_path);
1659 // System.out.println("created cookie: ->" +
1660 // String.valueOf(this_cookie));
1661
1662 this.cookie_jar.addCookie(this_cookie);
1663
1664 }
1665 }
1666 // single cookie
1667 else
1668 {
1669 String s2 = "]";
1670 String[] a = data.split(s2);
1671 String x = null;
1672 String c1, c2 = null;
1673 String c_name = null, c_value = null, c_domain = null, c_path = null;
1674 String c_version = null;
1675 BasicClientCookie this_cookie = null;
1676 for (int j = 0; j < a.length; j++)
1677 {
1678 x = a[j].replace("[", "").trim();
1679 c1 = x.split(":")[0];
1680 c2 = x.split(":")[1].substring(1);
1681 // Log.d("load cookie:", "->" + String.valueOf(c1));
1682 // Log.d("load cookie:", "->" + String.valueOf(c2));
1683 if (c1.matches("name") == true)
1684 {
1685 // Log.d("name:", "->" + String.valueOf(c1));
1686 c_name = c2;
1687 }
1688 else if (c1.matches("value") == true)
1689 {
1690 c_value = c2;
1691 }
1692 else if (c1.matches("domain") == true)
1693 {
1694 c_domain = c2;
1695 }
1696 else if (c1.matches("path") == true)
1697 {
1698 c_path = c2;
1699 }
1700 else if (c1.matches("version") == true)
1701 {
1702 c_version = c2;
1703 }
1704 }
1705 this_cookie = new BasicClientCookie(c_name, c_value);
1706 this_cookie.setDomain(c_domain);
1707 this_cookie.setPath(c_path);
1708 // System.out.println("created cookie: ->" +
1709 // String.valueOf(this_cookie));
1710
1711 this.cookie_jar.addCookie(this_cookie);
1712 }
1713 }
1714
1715 return;
1716 }
1717
1718 public String getUrlData(String url)
1719 {
1720 String websiteData = null;
1721
1722 if (!this.logged_in)
1723 {
1724 System.out.println("--1--- LOGIN START -----");
1725 this.logged_in = login();
1726 System.out.println("--1--- LOGIN END -----");
1727 }
1728
1729 try
1730 {
1731 HttpParams httpParameters = new BasicHttpParams();
1732 // Set the timeout in milliseconds until a connection is
1733 // established.
1734 int timeoutConnection = 10000;
1735 HttpConnectionParams.setConnectionTimeout(httpParameters, timeoutConnection);
1736 // Set the default socket timeout (SO_TIMEOUT)
1737 // in milliseconds which is the timeout for waiting for data.
1738 int timeoutSocket = 10000;
1739 HttpConnectionParams.setSoTimeout(httpParameters, timeoutSocket);
1740
1741 DefaultHttpClient client = new DefaultHttpClient(httpParameters);
1742 client.setCookieStore(cookie_jar);
1743 // Log.d("cookie_jar", "->" + String.valueOf(cookie_jar));
1744
1745 URI uri = new URI(url);
1746 HttpGet method = new HttpGet(uri);
1747
1748 method.addHeader("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.10) Gecko/20071115 Firefox/2.0.0.10");
1749 method.addHeader("Pragma", "no-cache");
1750
1751 HttpResponse res = client.execute(method);
1752 InputStream data = res.getEntity().getContent();
1753 websiteData = generateString(data);
1754 client.getConnectionManager().shutdown();
1755
1756 }
1757 catch (SocketTimeoutException e2)
1758 {
1759 Log.d("HTMLDownloader", "Connectiont timout: " + e2);
1760 }
1761 catch (ClientProtocolException e)
1762 {
1763 e.printStackTrace();
1764 }
1765 catch (UnknownHostException x)
1766 {
1767 Log.d("HTMLDownloader", ": " + x);
1768 }
1769 catch (IOException e)
1770 {
1771 e.printStackTrace();
1772 }
1773 catch (URISyntaxException e)
1774 {
1775 e.printStackTrace();
1776 }
1777 catch (Exception e)
1778 {
1779 e.printStackTrace();
1780 }
1781
1782 return websiteData;
1783 }
1784
1785 public String generateString(InputStream stream)
1786 {
1787 InputStreamReader reader = new InputStreamReader(stream);
1788 BufferedReader buffer = new BufferedReader(reader, large_buffer_size);
1789 // StringBuilder sb = new StringBuilder();
1790 String ret = "";
1791
1792 try
1793 {
1794 // ***DEBUG*** System.out.println("int=" +(char) buffer.read());
1795 String cur;
1796 while ((cur = buffer.readLine()) != null)
1797 {
1798 // System.out.println("int=" + cur);
1799 // System.out.println("**"+cur);
1800 // sb.append(cur + "\n");
1801 ret = ret + cur + "\n";
1802 }
1803 }
1804 catch (IOException e)
1805 {
1806 e.printStackTrace();
1807 }
1808 catch (Exception e)
1809 {
1810 e.printStackTrace();
1811 }
1812
1813 try
1814 {
1815 stream.close();
1816 }
1817 catch (IOException e)
1818 {
1819 e.printStackTrace();
1820 }
1821 catch (Exception e)
1822 {
1823 e.printStackTrace();
1824 }
1825
1826 //return sb.toString();
1827 return ret;
1828 }
1829
1830 }

   
Visit the aagtl Website