HtmlUnit使用 - package com.lanyotech.www.wordbank;
-
- import java.io.FileOutputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.OutputStream;
- import java.net.MalformedURLException;
- import java.util.List;
- import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
- import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
- import com.gargoylesoftware.htmlunit.ScriptResult;
- import com.gargoylesoftware.htmlunit.WebClient;
- import com.gargoylesoftware.htmlunit.html.HtmlOption;
- import com.gargoylesoftware.htmlunit.html.HtmlPage;
- import com.gargoylesoftware.htmlunit.html.HtmlSelect;
-
- public class WorldBankCrawl {
-
- private static String TARGET_URL = “http:
-
- public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
-
- WebClient webClient = new WebClient();
-
- webClient.setJavaScriptEnabled(true);
- webClient.setCssEnabled(false);
- webClient.setAjaxController(new NicelyResynchronizingAjaxController());
- webClient.setTimeout(35000);
- webClient.setThrowExceptionOnScriptError(false);
-
- HtmlPage rootPage= webClient.getPage(TARGET_URL);
-
- HtmlSelect hs = (HtmlSelect) rootPage.getElementById(“lstCubes”);
-
- hs.getOption(0).setSelected(true);
-
- System.out.println(“正在跳转…”);
-
- ScriptResult sr = rootPage.executeJavaScript(“javascript:setCubeData(2,-1,4,’/ddp’);”);
-
-
- HtmlPage countrySelect = (HtmlPage) sr.getNewPage();
-
- HtmlPage framePage=(HtmlPage)countrySelect.getFrameByName(“frmTree1″).getEnclosedPage();
-
- framePage.executeJavaScript(“javascript:TransferListAll(‘countrylst’,'countrylstselected’,'no’);SetSelectedCount(‘countrylstselected’,'tdcount’);”);
-
- ScriptResult electricityScriptResult = framePage.executeJavaScript(“javascript:wrapperSetCube(‘/ddp’)”);
-
- System.out.println(“正在跳转…”);
-
- HtmlPage electricitySelect = (HtmlPage) electricityScriptResult.getNewPage();
-
- HtmlPage electricityFrame = (HtmlPage) electricitySelect.getFrameByName(“frmTree1″).getEnclosedPage();
-
- HtmlSelect seriesSelect = (HtmlSelect) electricityFrame.getElementById(“countrylst”);
-
- List optionList = seriesSelect.getOptions();
-
- optionList.get(1).setSelected(true);
-
- electricityFrame.executeJavaScript(“javascript:TransferList(‘countrylst’,'countrylstselected’,'no’);SetSelectedCount(‘countrylstselected’,'tdcount’);”);
-
- HtmlSelect electricitySelected = (HtmlSelect) electricityFrame.getElementById(“countrylstselected”);
- List list = electricitySelected.getOptions();
-
- ScriptResult timeScriptResult = electricityFrame.executeJavaScript(“javascript:wrapperSetCube(‘/ddp’)”);
-
- System.out.println(“正在跳转…”);
- HtmlPage timeSelectPage = (HtmlPage) timeScriptResult.getNewPage();
-
- timeSelectPage = (HtmlPage) timeSelectPage.getFrameByName(“frmTree1″).getEnclosedPage();
-
- timeSelectPage.executeJavaScript(“javascript:TransferListAll(‘countrylst’,'countrylstselected’,'no’);SetSelectedCount(‘countrylstselected’,'tdcount’);”);
-
- ScriptResult exportResult = timeSelectPage.executeJavaScript(“javascript:wrapperSetCube(‘/ddp’)”);
-
- System.out.println(“正在跳转…”);
-
- HtmlPage exportPage = (HtmlPage) exportResult.getNewPage();
-
- ScriptResult downResult = exportPage.executeJavaScript(“javascript:exportData(‘/ddp’ ,’EXT_BULK’ ,’WDI_Time=51||WDI_Series=1||WDI_Ctry=244||’ );”);
-
- System.out.println(“正在跳转…”);
- HtmlPage downLoadPage = (HtmlPage) downResult.getNewPage();
-
- ScriptResult downLoadResult = downLoadPage.executeJavaScript(“javascript:exportData(‘/ddp’,'BULKEXCEL’);”);
-
- InputStream is = downLoadResult.getNewPage().getWebResponse().getContentAsStream();
-
- OutputStream fos = new FileOutputStream(“d:
- byte[] buffer=new byte[1024*30];
- int len=-1;
- while((len=is.read(buffer))>0){
- fos.write(buffer, 0, len);
- }
- fos.close();
- fos.close();
- System.out.println(“Success!”);
- }
- }
|