前几天刚完成对网页数据抓取的工作，对些想法。下面的例子作为一个参照（不是工作的源CODE），供赏。
package com.zjf.websplider;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.Socket;
import java.net.URL;
import java.util.Properties;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.tidy.Tidy;
/**
* Web页面抓取,负责从页面上抓取相应的信息
*
* @author Administrator
*/
public class WebSplider
{
/**
* 根据请求的Url路径,和输出流进行模拟Http请求,得到页面数据
*
* @throws Exception
*/
public static void getURL(String url, OutputStream out) throws Exception
{
// 创建URL对象
URL http_URL = new URL(url);
// 请求协议
String protocol = http_URL.getProtocol(）;
// 服务器主机名称
String host = http_URL.getHost(）;
// 请求的文件名称
String filename = http_URL.getFile(）;
// 端口
int port = http_URL.getPort(）;
// 如果不是http请求
if (!protocol.equals("http"））
{
// 抛出异常
throw new IllegalArgumentException("仅仅支持http请求协议"）;
}
// 请求的服务器主机名称为null
if (host == null) { throw new IllegalArgumentException("无效的服务主机帐户"); }
// 如果服务器端口等于-1
if (port == -1)
{
// 则使用默认的端口80
port = 80;
}
// 如果请求的文件不存在
if (filename == "")
{
// 则,路径默认到服务器的当前目录下
filename = "/";
}
//
建立底层的Socket通讯
Socket socket = new Socket(host, port）;
// 得到输入流数据
InputStream from = socket.getInputStream(）;
// 构造http请求头
PrintWriter to = new PrintWriter(socket.getOutputStream()）;
// 设置http请求头类型及信息
to.print("GET " + filename + " HTTP/1.0\n"）;
to.print("Accept: */*\n");
to.print("Accept-Language: zh-cn\n");
to.print("Accept-Encoding: gb2312, deflate\n"）;
to
.print("User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\n"）;
to.print("Host: " + http_URL.getHost() + "\n"）;
to.print("Connection: Keep-Alive\n\n"）;
to.flush(）;
byte[] buf = new byte[4096];
int bytes_read;
// 读取数据流
while ((bytes_read = from.read(buf)) != -1)
{
out.write(buf, 0, bytes_read）;
}
// 关闭socket通讯,及数据流对象
socket.close(）;
out.close(）;
}
/**
* 得到请求的页面信息
*/
public static String getRequestPage(String url)
{
ByteArrayOutputStream outStream = new ByteArrayOutputStream(）;
// 返回的html代码
String html = "";
try
{
// 要读取的页面地址
getURL(url, outStream）;
html = outStream.toString().toLowerCase(）;
html = getList(html).toLowerCase(）;
}
catch (IOException e)
{
System.out.println("写入文件错误" + e.getMessage()）;
}
catch (Exception e)
{
e.printStackTrace(）;
}
return html;
}
/**
* 截取HTML标签中的内容
*
* @param html
* @return
*/
public static String getList(String html)
{
int StartPos = -1;
int EndPos = -1;
StartPos = html.indexOf("", 1）;
//EndPos = html.indexOf("",1）;
return html.substring(StartPos）;
}
/**
* 将得到的html页面转换成DOM文档
*/
public static Document getDocument(String url)
{
String html = getRequestPage(url）;
ByteArrayInputStream reader = new ByteArrayInputStream(html.getBytes()）;
Tidy tidy = new Tidy(）;
// 将html文档转换成符合规范的Dom文档
tidy.setXHTML(true）;
Document doc = null;
try
{
doc = tidy.parseDOM(reader, System.out）;
}
catch (Exception e)
{
e.printStackTrace(）;
}
return doc;
}
/**
* 创建Dom转换对象
*
* @return
*/
public static Transformer newTransformer()
{
try
{
Transformer transformer = TransformerFactory.newInstance()
.newTransformer();
Properties properties = transformer.getOutputProperties();
properties.setProperty(OutputKeys.ENCODING, "GBK");
properties.setProperty(OutputKeys.METHOD, "xml");
properties.setProperty(OutputKeys.VERSION, "1.0");
properties.setProperty(OutputKeys.INDENT, "no");
transformer.setOutputProperties(properties);
return transformer;
}
catch (TransformerConfigurationException tce)
{
throw new RuntimeException(tce.getMessage());
}
}
/**
* 取得dom文档相应的字符串
* @param args
*/
public String getStringByDocument(String url)
{
Document doc = this.getDocument(url);
//取得所有的节点
NodeList list = doc.getElementsByTagName("html"）;
Transformer transformer = newTransformer(）;
StringBuffer buffer = new StringBuffer(）;
//对于每个节点
for (int i = 0; i < list.getLength(）; i++)
{
StringWriter sw = new StringWriter(）;
Node node = list.item(i）;
try
{
//将节点转换成字符串
transformer.transform(new DOMSource(node), new StreamResult(sw)）;
buffer.append(new String(sw.toString().getBytes("ISO8859-1"))）;
System.out.println(i + " : " + new String(sw.toString().getBytes("ISO8859-1"))）;
}
catch (Exception e)
{
e.printStackTrace(）;
}
}
return buffer.toString(）;
}
public static void main(String args[])
{
WebSplider web = new WebSplider(）;
}
}
