开发笔记 第620页

#region 取得大类

httpHelper.Encoding = Encoding.Default;
string content = httpHelper.GetHtml(url);
Regex re = new Regex(mainTypeReg, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.IgnorePatternWhitespace);
Match mc = re.Match(content);
Dictionary<string, string> typeDict = new Dictionary<string, string>();
if (mc.Success)
{
MatchCollection mcs = re.Matches(content);
foreach (Match me in mcs)
{
string strKey = me.Groups[“key“].Value;
string strValue = me.Groups[“value“].Value;
//截取连接前面部分作为大类标识
string newValue = strValue.TrimEnd(‘/‘);
int eIndex = newValue.LastIndexOf(‘/‘);
newValue = newValue.Substring(0, eIndex) + “/“;

if (!typeDict.ContainsKey(strKey))
{
typeDict.Add(strKey, newValue);
}
}
}
#endregion

#region 取得子类
Dictionary<string, CircleSubTypeInfo> circleDict = new Dictionary<string, CircleSubTypeInfo>();
re = new Regex(subTypeReg, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.IgnorePatternWhitespace);
mc = re.Match(content);
if (mc.Success)
{
MatchCollection mcs = re.Matches(content);
foreach (Match me in mcs)
{
string strKey = me.Groups[“key“].Value;
string strValue = me.Groups[“value“].Value;
//截取连接前面部分作为大类标识
string typeValue = strValue.TrimEnd(‘/‘);
int eIndex = typeValue.LastIndexOf(‘/‘);
typeValue = typeValue.Substring(0, eIndex) + “/“;

if (!circleDict.ContainsKey(strKey))
{
CircleSubTypeInfo info = new CircleSubTypeInfo();
info.Name = strKey;
info.LinkUrl = strValue;
info.TypeUrlValue = typeValue;

circleDict.Add(strKey, info);
}
}
}
#endregion

#region 保存数据
Database db = DatabaseFactory.CreateDatabase();
DbCommand command = null;
string SQL = “”;

foreach (string key in typeDict.Keys)
{
SQL = string.Format(“Insert into CircleType(TypeName, TypeValue) values(‘{0}’, ‘{1}’) “, key, typeDict[key]);
command = db.GetSqlStringCommand(sql);
db.ExecuteNonQuery(command);
}

foreach (string key in circleDict.Keys)
{
CircleSubTypeInfo info = circleDict[key];
sql = string.Format(“Insert into CircleSubType(SubTypeName, LinkUrl, TypeUrlValue) values(‘{0}’, ‘{1}’, ‘{2}’) “, info.Name, info.LinkUrl, info.TypeUrlValue);
command = db.GetSqlStringCommand(sql);
db.ExecuteNonQuery(command);
}
#endregion

this.lblTips.Text = “获取分类操作完成“;

LogTextHelper.WriteLine(string.Format(“正在处理类型：{0}“, urlDict[key]));
cookie = new System.Net.CookieContainer();

string urlKey = key;
Match mc = re.Match(urlKey);
string d1 = mc.Groups[“d1“].Value;
string d2 = mc.Groups[“d2“].Value;
string d3 = mc.Groups[“d3“].Value;
int pageSize = 30;

urlKey = urlKey.Trim(‘/‘);//清除前后的/字符
string url = “http://q.163.com/dwr/call/plaincall/CircleMainpageBean.getCircleByType2IdInMemberOrder.dwr“;
//string refUrl = “http://q.163.com/mapCircleList/2/11/48/?fromCircleCircleMap“;
string refUrl = string.Format(“http://q.163.com/{0}/?fromCircleCircleMap“, urlKey);
#region 内容正则表达式
StringBuilder circleReg = new StringBuilder();
circleReg.Append(“s[0-9]\\d*.circleId=(?<circleId>[0-9]\\d*[^;])“);
circleReg.Append(“.*?s[0-9]\\d*.circleType1Str=\”(?<circleType1Str>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.circleType2Str=\”(?<circleType2Str>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.createDateStr=\”(?<createDateStr>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.creatorId=(?<creatorId>[0-9]\\d*[^;])“);
circleReg.Append(“.*?s[0-9]\\d*.creatorName=\”(?<creatorName>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.creatorSpaceUrl=\”(?<creatorSpaceUrl>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.description=\”(?<description>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.joinDeclaration=\”(?<joinDeclaration>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.linkImgUrl=\”(?<linkImgUrl>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.memberNum=(?<memberNum>[0-9]\\d*[^;])“);
circleReg.Append(“.*?s[0-9]\\d*.name=\”(?<name>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.urlName=\”(?<urlName>.*?)\”“);
circleReg.Append( “.*?s[0-9]\\d*.visitNum=(?<visitNum>[0-9]\\d*[^;])“);

MatchCollection mcs = re.Matches(content);
foreach (Match me in mcs)
{
#region MyRegion
j++;

int memberNum = 0;
try
{
memberNum = Convert.ToInt32(me.Groups[“memberNum“].Value);
}
catch { }
if (memberNum < 50)
{
flag = false;
break;
}

sql = string.Format(@”insert into Circle(circleId,circleType1Str,circleType2Str,createDateStr,creatorId,
creatorName,creatorSpaceUrl,description,joinDeclaration,linkImgUrl,memberNum,name2,urlName,SubTypeName)
values(‘{0}’,'{1}’,'{2}’,'{3}’,'{4}’,'{5}’,'{6}’,'{7}’,'{8}’,'{9}’,'{10}’,'{11}’,'{12}’,'{13}’) “, me.Groups[“circleId“].Value,
UnicodeHelper.UnicodeToString(me.Groups[“circleType1Str“].Value.Replace(“‘“, “”)), UnicodeHelper.UnicodeToString(me.Groups[“circleType2Str“].Value.Replace(“‘“, “”)),
me.Groups[“createDateStr“].Value, me.Groups[“creatorId“].Value, UnicodeHelper.UnicodeToString(me.Groups[“creatorName“].Value),
me.Groups[“creatorSpaceUrl“].Value, UnicodeHelper.UnicodeToString(me.Groups[“description“].Value.Replace(“‘“, “”)), UnicodeHelper.UnicodeToString(me.Groups[“joinDeclaration“].Value.Replace(“‘“, “”)),
me.Groups[“linkImgUrl“].Value, me.Groups[“memberNum“].Value, UnicodeHelper.UnicodeToString(me.Groups[“name“].Value.Replace(“‘“, “”)), me.Groups[“urlName“].Value, urlDict[key]);
command = db.GetSqlStringCommand(sql);
try
{
db.ExecuteNonQuery(command);
}
catch (Exception ex)
{
LogTextHelper.WriteLine(sql);
LogTextHelper.WriteLine(ex.ToString());
}

message = string.Format(“正在处理{0}:{1} 正在写入数据{2}次“, urlDict[key], url, j);
CallCtrlWithThreadSafety.SetText(this.lblTips, message, this);
Application.DoEvents();
Thread.Sleep(10);

#endregion
}
}
else
{
flag = false;//没有匹配就停止
break;
}

foreach (string key in circlelDict.Keys)
{
string circleId = key;
string urlName = circlelDict[key];
string refUrl = string.Format(“http://q.163.com/{0}/members/“, urlName);

#region 内容正则表达式
StringBuilder circleReg = new StringBuilder();
circleReg.Append(“s[0-9]\\d*.ageStr=\”(?<ageStr>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.city=\”(?<city>.*?[^;])\”“);
circleReg.Append(“.*?s[0-9]\\d*.hometownCity=(\”(?<hometownCity>.*?[^;])\”|(?<hometownCity>null))“);
circleReg.Append(“.*?s[0-9]\\d*.hometownProvince=(\”(?<hometownProvince>.*?)\”|(?<hometownProvince>null))“);
circleReg.Append(“.*?s[0-9]\\d*.name=(\”(?<name>.*?)\”|(?<name>null))“);
circleReg.Append(“.*?s[0-9]\\d*.nickname=(\”(?<nickname>.*?)\”|(?<nickname>null))“);
circleReg.Append(“.*?s[0-9]\\d*.profileImage140=\”(?<profileImage140>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.profileImage60=\”(?<profileImage60>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.province=\”(?<province>.*?)\”“);
circleReg.Append(“.*?s[0-9]\\d*.qq=(\”(?<qq>.*?)\”|(?<qq>null))“);
circleReg.Append(“.*?s[0-9]\\d*.realName=(\”(?<realName>.*?)\”|(?<realName>null))“);
circleReg.Append(“.*?s[0-9]\\d*.spaceName=(\”(?<spaceName>.*?)\”|(?<spaceName>null))“);
circleReg.Append(“.*?s[0-9]\\d*.userId=(?<userId>[0-9]\\d*[^;])“);
circleReg.Append(“.*?s[0-9]\\d*.userName=\”(?<userName>.*?)\”“);
#endregion

bool flag = true;
int i = 0;
int j = 0;
List<CircleMemberInfo> entityList = new List<CircleMemberInfo>();
while (flag)
{
#region 构造提交参数
StringBuilder sb = new StringBuilder();
sb.AppendFormat(“callCount=1“);
sb.AppendFormat(“&page=/{0}/members/“, urlName);
sb.AppendFormat(“&httpSessionId=“);
sb.AppendFormat(“&scriptSessionId=D4DAC4AD9C3BF9B71C82802BDDBA0C25369“);
sb.AppendFormat(“&c0-scriptName=CircleBean“);
sb.AppendFormat(“&c0-methodName=getNewCircleUsers“);
sb.AppendFormat(“&c0-id=0“);//保留字符
sb.AppendFormat(“&c0-param0=number:{0}“, circleId);//11
sb.AppendFormat(“&c0-param1=number:{0}“, pageSize);//数量
sb.AppendFormat(“&c0-param2=number:{0}“, pageSize * i);//0,30,60
sb.AppendFormat(“&c0-param3=boolean:true“);
sb.AppendFormat(“&batchId={0}“, i);

sql = string.Format(@”insert into CircleMember(userId,userName,realName,nickname,circleId)
values(‘{0}’,'{1}’,'{2}’,'{3}’,'{4}’) “, me.Groups[“userId“].Value, httpHelper.RemoveHtml(UnicodeHelper.UnicodeToString(me.Groups[“userName“].Value)),
httpHelper.RemoveHtml(UnicodeHelper.UnicodeToString(me.Groups[“realName“].Value.Replace(“‘“, “”))), httpHelper.RemoveHtml(UnicodeHelper.UnicodeToString(me.Groups[“nickname“].Value.Replace(“‘“, “”))), circleId);
command = db.GetSqlStringCommand(sql);
try
{
db.ExecuteNonQuery(command);
}
catch (Exception ex)
{
LogTextHelper.WriteLine(sql);
LogTextHelper.WriteLine(ex.ToString());
}

message = string.Format(“正在处理{0} 正在写入数据{1}次“, urlName, j);
CallCtrlWithThreadSafety.SetText(this.lblTips, message, this);
Application.DoEvents();
Thread.Sleep(10);
#endregion