在webForm中WebRequest/WebClient/WebBrowser获取远程页面源码的三种方式（downmoon)
一个小需求，获取远程页面的源码，主要用于抓数据。原来用的好好的，最近突然不能获取页面源码了，但是仍然可以用浏览器正常浏览。（文后附源码下载。^_^）
经过分析，原来用的代码如下：
StreamReader sreader = null ;
string result = string .Empty;
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(Url);
// httpWebRequest.Timeout = 20;
httpWebRequest.KeepAlive = false ;
#endregion
HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
sreader = new StreamReader(httpWebResponse.GetResponseStream(), encoding);
result = reader.ReadToEnd();
if ( null != httpWebResponse) { httpWebResponse.Close(); }
return result;
}
return result; ;
}
catch (WebException e) { return null ; }
finally { if (sreader != null ) { sreader.Close(); } }
查了下资料，原来需要加参数。
#region 关键参数，否则会取不到内容 Important Parameters,else get nothing.
httpWebRequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
httpWebRequest.Accept = "*/*";
httpWebRequest.KeepAlive = true;
httpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
#endregion
修正后的代码如下：
#region 读取页面详细信息
/**/ /// <summary> /// 读取页面详细信息
/// </summary>
/// <param name="Url"> 需要读取的地址 </param>
/// <param name="encoding"> 读取的编码方式 </param>
/// <returns></returns>
public static string GetStringByUrl( string Url, System.Text.Encoding encoding)
{
if (Url.Equals( " about:blank " )) return null ; ;
if ( ! Url.StartsWith( " http:// " ) && ! Url.StartsWith( " https:// " )) { Url = " http:// " + Url; }
int dialCount = 0 ;
loop:
StreamReader sreader = null ;
string result = string .Empty;
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(Url);
// httpWebRequest.Timeout = 20;
关键参数，否则会取不到内容 Important Parameters,else get nothing. #region 关键参数，否则会取不到内容 Important Parameters,else get nothing.
httpWebRequest.UserAgent = " User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;) " ;
httpWebRequest.Accept = " */* " ;
httpWebRequest.KeepAlive = true ;
httpWebRequest.Headers.Add( " Accept-Language " , " zh-cn,en-us;q=0.5 " );
#endregion
HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
sreader = new StreamReader(httpWebResponse.GetResponseStream(), encoding);
char [] cCont = new char [ 256 ];
int count = sreader.Read(cCont, 0 , 256 );
while (count > 0 )
{ // Dumps the 256 characters on a string and displays the string to the console.
String str = new String(cCont, 0 , count);
result += str;
count = sreader.Read(cCont, 0 , 256 );
}
}
if ( null != httpWebResponse) { httpWebResponse.Close(); }
return result;
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ConnectFailure) { dialCount ++ ; ReDial(); }
if (dialCount < 5 ) { goto loop; }
return null ;
}
finally { if (sreader != null ) { sreader.Close(); } }
}
#endregion
public static void ReDial()
{
int res = 1 ;
/**/ /// /while (res != 0)
/// /{
/// / CSDNWebTest.RASDisplay ras = new RASDisplay();
/// / ras.Disconnect();
/// / res = ras.Connect("asdl");
/// / System.Threading.Thread.Sleep(TimeSpan.FromSeconds(10));
/// /}
}
问题是解决了，后来再想了想，可以用WebClient先把页面download到本地临时文件，再读取文本内容。
代码如下：
private string GetPageByWebClient( string url)
{
string result = null ;
if (url.Equals( " about:blank " )) return null ; ;
if ( ! url.StartsWith( " http:// " ) && ! url.StartsWith( " https:// " )) { url = " http:// " + url; }
string filename = RandomKey( 1111 , 9999 ) + " .txt " ;
DownloadOneFileByURLWithWebClient(filename, url, " C:// " );
StreamReader sr = new StreamReader( " c:// " + filename, System.Text.Encoding.Default);
try { result = sr.ReadToEnd(); return result; }
catch { return null ; }
finally
{
if (sr != null ) { sr.Close(); }
}
}
private string RandomKey( int b, int e)
{
return DateTime.Now.ToString( " yyyyMMdd-HHmmss-fff- " ) + this .getRandomID(b, e);
}
private int getRandomID( int minValue, int maxValue)
{
Random ri = new Random( unchecked (( int )DateTime.Now.Ticks));
int k = ri.Next(minValue, maxValue);
return k;
}
private string GuidString
{
get { return Guid.NewGuid().ToString(); }
}
/**/ /// Web Client Method ,only For Small picture
/// </summary>
/// <param name="fileName"></param>
/// <param name="url"></param>
/// <param name="localPath"></param>
public static void DownloadOneFileByURLWithWebClient( string fileName, string url, string localPath)
{
System.Net.WebClient wc = new System.Net.WebClient();
if (File.Exists(localPath + fileName)) { File.Delete(localPath + fileName); }
if (Directory.Exists(localPath) == false ) { Directory.CreateDirectory(localPath); }
wc.DownloadFile(url + fileName, localPath + fileName);
}
结果不能获取源码。错误如下：
再想想，还有Webbrowser控件可以用啊。在WinFrom下只要在主线程前加[STAThread]即可。
[STAThread]
public void GetURLContentByWebBrowser()
{
try
{
// webBrowser1 = new WebBrowser();
string url = txtUrl.Text.Trim();
string result = null ;
WebBrowser wb = new WebBrowser();
/**/ /// /if (wb != null){ wb.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(wb_DocumentCompleted); }
if (String.IsNullOrEmpty(url)) return ;
if (url.Equals( " about:blank " )) return ; ;
if ( ! url.StartsWith( " http:// " ) && ! url.StartsWith( " https:// " )) { url = " http:// " + url; }
try
{
wb.Navigate( new Uri(url));
result = wb.DocumentText;
lbResult.Text = result;
}
catch (System.UriFormatException)
{ }
return ;
}
catch (Exception ex)
{
// WriteLog.Writelog("这是获取页面全部html代码时发生的错误：" + url, ex);
throw ex;
// return ;
}
}
在WebForm就麻烦些了，出现错误，线程不在单线程单元中，故无法实例化ActiveX控件“8856f961-340a-11d0-a96b-00c04fd705a2”
代码如下：
private string GetPageStringbyWebBrowser( string url)
{
if (url.Equals( " about:blank " )) return null ; ;
if ( ! url.StartsWith( " http:// " ) && ! url.StartsWith( " https:// " )) { url = " http:// " + url; }
WebBrowser myWB = new WebBrowser();
myWB.ScrollBarsEnabled = false ;
myWB.Navigate(url);
while (myWB.ReadyState != WebBrowserReadyState.Complete)
{
System.Windows.Forms.Application.DoEvents();
}
if (myWB != null )
{
System.IO.StreamReader getReader = null ;
try
{
getReader = new System.IO.StreamReader(myWB.DocumentStream, System.Text.Encoding.GetEncoding(myWB.Document.Encoding));
string gethtml = getReader.ReadToEnd();
return gethtml;
}
catch { return null ; }
finally
{
if (getReader != null ) { getReader.Close(); }
myWB.Dispose();
}
}
return null ;
}
后来搜索N小时(N>=5)后，终于找到可行解决方案,在WebPage页面头部加入AspCompat="true"
即<%@ Page Language="C#" AspCompat="true"
******/>
MSDN给出的解释是：
在 ASP .NET 网页的 <%@Page> 标记中包含兼容性属性 aspcompat=true，如 <%@Page aspcompat=true Language=VB%>。使用此属性将强制网页以 STA 模式执行，从而确保您的组件可以继续正确运行。如果试图使用 STA 组件但没有指定此标记，运行时将会发生异常情况。
将此属性的值设置为 true 时，将允许网页调用 COM+ 1.0 组件，该组件需要访问非管理的 ASP 内置对象。可以通过 ObjectContext 对象进行访问。
如果将此标记的值设为 true，性能会稍微有些下降。建议只在确实需要时才这样做。
终于可以了！ 不知道有没有更好的方法？？
附：源码下载。
邀月注：
如果不能测试，请注意是否在域（AD)环境下，如果是！ 请注意设置代理和防火墙
请参考：
http://dev.csdn.net/article/83914.shtm
或http://blog.csdn.net/downmoon/archive/2006/04/14/663337.aspx
或http://www.cnblogs.com/downmoon/archive/2007/12/29/1019701.html