@bergus
2015-12-31T10:39:17.000000Z
字数 14884
阅读 2270
spyder
# encoding=utf-8import reimport urllib2from math import *from bs4 import BeautifulSoup# 请求头部信息header = {'User-Agent':'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'}# {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0"}#测试brand_namebrand_name=["星巴克","麦当劳","肯德基","小南国","真功夫","汉拿山","新辣道","新疆火宴山","味千拉面"]city_code='''1,上海2,北京3,杭州4,广州5,南京6,苏州7,深圳8,成都9,重庆10,天津11,宁波12,扬州13,无锡14,福州15,厦门16,武汉17,西安18,沈阳19,大连20,其它城市21,青岛22,济南23,海口24,石家庄25,唐山26,秦皇岛27,邯郸28,邢台29,保定30,张家口31,承德32,沧州33,廊坊34,衡水35,太原36,大同37,阳泉38,长治39,晋城40,朔州41,晋中42,运城43,忻州44,临汾45,吕梁46,呼和浩特47,包头48,乌海49,赤峰50,通辽51,鄂尔多斯52,呼伦贝尔53,兴安盟54,锡林郭勒55,乌兰察布56,巴彦淖尔57,阿拉善58,鞍山59,抚顺60,本溪61,丹东62,锦州63,营口64,阜新65,辽阳66,盘锦67,铁岭68,朝阳69,葫芦岛70,长春71,吉林72,四平73,辽源74,通化75,白山76,松原77,白城78,延边79,哈尔滨80,齐齐哈尔81,鸡西82,鹤岗83,双鸭山84,大庆85,伊春86,佳木斯87,七台河88,牡丹江89,黑河90,绥化91,大兴安岭92,徐州93,常州94,南通95,连云港96,淮安97,盐城98,镇江99,泰州100,宿迁101,温州102,嘉兴103,湖州104,绍兴105,金华106,衢州107,舟山108,台州109,丽水110,合肥111,芜湖112,蚌埠113,淮南114,马鞍山115,淮北116,铜陵117,安庆118,黄山119,滁州120,阜阳121,宿州122,巢湖123,六安124,亳州125,池州126,宣城127,莆田128,三明129,泉州130,漳州131,南平132,龙岩133,宁德134,南昌135,景德镇136,萍乡137,九江138,新余139,鹰潭140,赣州141,吉安142,宜春143,抚州144,上饶145,淄博146,枣庄147,东营148,烟台149,潍坊150,济宁151,泰安152,威海153,日照154,莱芜155,临沂156,德州157,聊城158,滨州159,菏泽160,郑州161,开封162,洛阳163,平顶山164,安阳165,鹤壁166,新乡167,焦作168,濮阳169,许昌170,漯河171,三门峡172,南阳173,商丘174,信阳175,周口176,驻马店177,黄石178,十堰179,宜昌180,襄阳181,鄂州182,荆门183,孝感184,荆州185,黄冈186,咸宁187,随州188,恩施189,仙桃190,潜江191,天门192,株洲193,湘潭194,衡阳195,邵阳196,岳阳197,常德198,张家界199,益阳200,郴州201,永州202,怀化203,娄底204,湘西205,韶关206,珠海207,汕头208,佛山209,江门210,湛江211,茂名212,肇庆213,惠州214,梅州215,汕尾216,河源217,阳江218,清远219,东莞220,中山221,潮州222,揭阳223,云浮224,南宁225,柳州226,桂林227,梧州228,北海229,防城港230,钦州231,贵港232,玉林233,百色234,贺州235,河池238,自贡239,攀枝花240,泸州241,德阳242,绵阳243,广元244,遂宁245,内江246,乐山247,南充248,眉山249,宜宾250,广安251,达州252,雅安253,巴中254,资阳255,阿坝256,甘孜州257,凉山258,贵阳259,六盘水260,遵义261,安顺262,铜仁地区263,黔西南264,毕节地区265,黔东南266,黔南267,昆明268,曲靖269,玉溪270,保山271,昭通272,楚雄州273,红河274,文山州275,普洱276,西双版纳277,大理州278,德宏279,丽江280,怒江281,迪庆282,临沧283,拉萨284,昌都地区285,山南286,日喀则地区287,那曲288,阿里289,林芝地区290,铜川291,宝鸡292,咸阳293,渭南294,延安295,汉中296,榆林297,安康298,商洛299,兰州300,嘉峪关301,金昌302,白银303,天水304,武威305,张掖306,平凉307,酒泉308,庆阳309,定西310,陇南311,临夏州312,甘南313,西宁314,海东315,海北316,黄南318,果洛319,玉树320,海西321,银川322,石嘴山323,吴忠324,固原325,乌鲁木齐326,克拉玛依327,吐鲁番地区328,哈密地区329,昌吉州330,博尔塔拉331,巴音郭楞332,阿克苏地区333,克孜勒苏334,喀什地区335,和田地区336,伊犁337,塔城地区338,阿勒泰地区339,石河子340,台湾341,香港342,澳门343,国外其他344,长沙345,三亚346,北屯351,中卫353,格尔木市354,奎屯市355,库尔勒市358,儋州359,潢川县361,延吉市362,珲春市363,梅河口市367,锡林浩特市368,乌兰浩特市382,西昌市383,康定县385,义乌389,阿拉尔390,白沙391,保亭392,昌江393,澄迈县394,崇左395,定安县396,东方397,济源398,来宾399,乐东400,临高县401,陵水402,琼海403,琼中404,神农架林区405,图木舒克406,屯昌县407,万宁408,文昌409,五家渠410,五指山411,海南州416,昆山417,常熟418,张家港419,吴江420,太仓421,慈溪422,奉化423,余姚424,仪征市426,高邮市427,江阴428,宜兴429,瓦房店市430,普兰店市431,庄河市432,长海县433,福清434,密云县435,延庆县439,辛集市440,藁城市441,晋州市442,新乐市443,鹿泉市444,井陉县445,正定县446,栾城县447,行唐县448,灵寿县449,高邑县450,深泽县451,赞皇县452,无极县453,平山县454,元氏县455,赵县456,宣化县457,张北县458,康保县459,沽源县460,尚义县461,蔚县462,阳原县463,怀安县464,万全县465,怀来县466,涿鹿县467,赤城县468,崇礼县469,承德县470,兴隆县471,平泉县472,滦平县473,隆化县474,丰宁满族自治县475,宽城满族自治县476,围场满族蒙古族自治县477,昌黎县478,抚宁县479,卢龙县480,青龙满族自治县481,遵化市482,迁安市483,滦县484,滦南县485,乐亭县486,迁西县487,玉田县489,霸州市490,三河市491,固安县492,永清县493,香河县494,大城县495,文安县496,大厂回族自治县497,定州市498,涿州市499,安国市500,高碑店市501,满城县502,清苑县503,易县504,徐水县505,涞源县506,定兴县507,顺平县508,唐县509,望都县510,涞水县511,高阳县512,安新县513,雄县514,曲阳县515,阜平县516,博野县517,蠡县518,冀州市519,深州市520,枣强县521,武邑县522,武强县523,饶阳县524,安平县525,故城县526,景县527,阜城县528,泊头市529,任丘市530,黄骅市531,河间市532,沧县533,青县534,东光县535,海兴县536,盐山县537,肃宁县538,南皮县539,吴桥县540,献县541,孟村回族自治县542,南宫市543,沙河市544,邢台县545,临城县546,内丘县547,柏乡县548,隆尧县549,任县550,南和县551,宁晋县552,巨鹿县553,新河县554,广宗县555,平乡县556,威县557,清河县558,临西县559,武安市560,邯郸县561,临漳县562,成安县563,大名县564,涉县565,磁县566,肥乡县567,永年县568,邱县569,鸡泽县570,广平县571,馆陶县572,魏县573,曲周县574,古交市575,清徐县576,阳曲县577,娄烦县578,山阴县579,应县580,右玉县581,怀仁县582,阳高县583,天镇县584,广灵县585,灵丘县586,浑源县587,左云县588,大同县589,平定县590,盂县591,潞城市592,长治县593,襄垣县594,屯留县595,平顺县596,黎城县597,壶关县598,长子县599,武乡县600,沁县601,沁源县602,高平市603,泽州县604,沁水县605,阳城县606,陵川县607,原平市608,定襄县609,五台县610,代县611,繁峙县612,宁武县613,静乐县614,神池县615,五寨县616,岢岚县617,河曲县618,保德县619,偏关县620,介休市621,榆社县622,左权县623,和顺县624,昔阳县625,寿阳县626,太谷县627,祁县628,平遥县629,灵石县630,侯马市631,霍州市632,曲沃县633,翼城县634,襄汾县635,洪洞县636,古县637,安泽县638,浮山县639,吉县640,乡宁县641,蒲县642,大宁县643,永和县644,隰县645,汾西县646,孝义市647,汾阳市648,文水县649,中阳县650,兴县651,临县652,方山县653,柳林县654,岚县655,交口县656,交城县657,石楼县658,托克托县659,武川县660,和林格尔县661,清水河县662,固阳县663,宁城县664,林西县665,霍林郭勒市666,开鲁县667,满洲里市668,扎兰屯市669,牙克石市670,根河市671,额尔古纳市672,丰镇市673,卓资县674,化德县675,商都县676,兴和县677,凉城县678,五原县679,磴口县680,阿尔山市681,突泉县682,二连浩特市683,多伦县684,新民市685,辽中县686,康平县687,法库县688,北票市689,凌源市690,朝阳县691,建平县692,喀喇沁左翼蒙古族自治县693,彰武县694,阜新蒙古族自治县695,调兵山市696,开原市697,铁岭县698,西丰县699,昌图县700,抚顺县701,新宾满族自治县702,清原满族自治县703,本溪满族自治县704,桓仁满族自治县705,灯塔市706,辽阳县707,海城市708,台安县709,岫岩满族自治县710,凤城市711,东港市712,宽甸满族自治县713,大石桥市714,盖州市715,大洼县716,盘山县717,凌海市718,北宁市719,黑山县720,义县721,兴城市722,绥中县723,建昌县724,德惠市725,九台市726,榆树市727,农安县728,大安市729,洮南市730,镇赉县731,通榆县732,扶余县733,长岭县734,乾安县735,前郭尔罗斯蒙古族自治县736,磐石市737,蛟河市738,桦甸市739,舒兰市740,永吉县741,双辽市742,公主岭市743,梨树县744,伊通满族自治县745,东丰县746,东辽县747,集安市748,通化县749,辉南县750,柳河县751,临江市753,抚松县754,靖宇县755,长白朝鲜族自治县756,图们市757,敦化市758,龙井市759,和龙市760,汪清县761,安图县768,宾县769,巴彦县773,勃利县774,讷河市775,龙江县776,依安县777,泰来县778,甘南县779,富裕县780,克山县781,克东县782,拜泉县783,北安市784,五大连池市785,嫩江县786,逊克县787,孙吴县788,肇州县789,肇源县790,林甸县791,杜尔伯特蒙古族自治县792,萝北县793,绥滨县794,铁力市795,嘉荫县796,同江市797,富锦市798,桦南县799,桦川县800,汤原县801,抚远县802,集贤县803,友谊县804,宝清县805,饶河县806,虎林市807,密山市808,鸡东县809,穆棱市810,绥芬河市811,海林市812,宁安市813,东宁县814,林口县815,安达市816,肇东市817,海伦市818,望奎县819,兰西县820,青冈县821,庆安县822,明水县823,绥棱县824,呼玛县825,塔河县826,漠河县828,邳州市829,新沂市831,睢宁县832,沛县833,丰县834,赣榆县835,灌云县836,东海县837,灌南县838,沭阳县839,泗阳县840,泗洪县841,金湖县842,盱眙县843,洪泽县844,涟水县845,东台市846,大丰市847,射阳县848,阜宁县849,滨海县850,响水县851,建湖县852,宝应县853,靖江854,泰兴市856,兴化市857,海门市858,启东市860,如皋市862,海安县863,扬中市864,丹阳市865,句容市866,金坛市867,溧阳868,临安869,富阳870,建德市871,桐庐县872,淳安县873,长兴县874,德清县875,安吉县876,平湖市877,海宁市878,桐乡市879,嘉善县880,海盐县881,岱山县882,嵊泗县883,诸暨884,上虞区885,嵊州市886,柯桥区888,江山市889,常山县890,开化县891,龙游县892,兰溪市893,永康894,东阳895,武义县896,浦江县897,磐安县898,临海市899,温岭市900,三门县901,天台县902,仙居县903,玉环县904,瑞安905,乐清908,平阳911,苍南912,龙泉市913,缙云县914,青田县915,云和县916,遂昌县917,松阳县918,庆元县919,景宁畲族自治县920,长丰县921,肥东县922,肥西县923,砀山县924,萧县925,灵璧县926,泗县927,濉溪县928,涡阳县929,蒙城县930,利辛县931,界首市932,临泉县933,太和县934,阜南县935,颍上县936,怀远县937,五河县938,固镇县939,凤台县940,明光市941,天长市942,来安县943,全椒县944,定远县945,凤阳县946,当涂县947,芜湖县948,繁昌县949,南陵县950,铜陵县951,桐城市952,怀宁县953,枞阳县954,潜山县955,太湖县956,宿松县957,望江县958,岳西县959,歙县960,休宁县961,黟县962,祁门县963,寿县964,霍邱县965,舒城县966,金寨县967,霍山县968,庐江县969,无为县970,含山县971,和县972,东至县973,石台县974,青阳县975,宁国市976,郎溪县977,广德县978,泾县979,旌德县980,绩溪县981,长乐983,连江县984,罗源县986,永泰县987,平潭县988,邵武市989,武夷山990,建瓯市991,建阳市992,顺昌县993,浦城县994,光泽县995,松溪县996,政和县998,永安市999,明溪县1000,清流县1001,宁化县1002,大田县1003,尤溪县1004,沙县1005,将乐县1006,泰宁县1007,建宁县1008,石狮1009,晋江1010,南安市1011,惠安县1012,安溪县1013,永春县1014,德化县1015,龙海1017,漳浦县1024,漳平市1025,长汀县1026,永定县1027,上杭县1028,武平县1029,连城县1030,福安市1031,福鼎1032,寿宁县1033,霞浦县1034,柘荣县1035,屏南县1036,古田县1037,周宁县1040,安义县1042,瑞昌市1043,九江县1044,武宁县1045,修水县1046,永修县1047,德安县1048,星子县1049,都昌县1050,湖口县1051,彭泽县1052,乐平市1053,浮梁县1054,贵溪市1055,余江县1056,分宜县1057,莲花县1058,上栗县1059,芦溪县1060,瑞金市1061,南康市1062,赣县1063,信丰县1064,大余县1065,上犹县1066,崇义县1067,安远县1068,龙南县1069,定南县1070,全南县1071,宁都县1072,于都县1073,兴国县1074,会昌县1075,寻乌县1076,石城县1077,德兴市1078,上饶县1079,广丰县1080,玉山县1081,铅山县1082,横峰县1083,弋阳县1084,余干县1085,鄱阳县1086,万年县1087,婺源县1088,南城县1089,黎川县1090,南丰县1091,崇仁县1092,乐安县1093,宜黄县1094,金溪县1095,资溪县1096,东乡县1097,广昌县1098,丰城市1099,樟树市1100,高安市1101,奉新县1102,万载县1103,上高县1104,宜丰县1105,靖安县1106,铜鼓县1107,井冈山市1108,吉安县1109,吉水县1110,峡江县1111,新干县1112,永丰县1113,泰和县1114,遂川县1115,万安县1116,安福县1117,永新县1118,章丘1119,平阴县1120,济阳县1121,商河县1122,临清市1123,阳谷县1124,莘县1125,茌平县1126,东阿县1127,冠县1128,高唐县1129,乐陵市1130,禹城市1131,陵县1132,平原县1133,夏津县1134,武城县1135,齐河县1136,临邑县1137,宁津县1138,庆云县1139,垦利县1140,利津县1141,广饶县1142,桓台县1143,高青县1144,沂源县1145,安丘市1146,昌邑市1147,高密市1148,青州市1149,诸城市1150,寿光市1151,临朐县1152,昌乐县1153,栖霞市1154,海阳市1155,龙口市1156,莱阳市1157,莱州市1158,蓬莱市1159,招远市1160,长岛县1161,荣成1162,乳山市1163,文登1169,五莲县1170,莒县1171,郯城县1172,苍山县1173,莒南县1174,沂水县1175,蒙阴县1176,平邑县1177,费县1178,沂南县1179,临沭县1180,滕州市1181,曲阜市1182,兖州市1183,邹城市1184,微山县1185,鱼台县1186,金乡县1187,嘉祥县1188,汶上县1189,泗水县1190,梁山县1191,新泰市1192,肥城市1193,宁阳县1194,东平县1195,惠民县1196,阳信县1197,无棣县1198,沾化县1199,博兴县1200,邹平县1201,曹县1202,定陶县1203,成武县1204,单县1205,巨野县1206,郓城县1207,鄄城县1208,东明县1209,新郑市1210,登封市1211,新密市1212,巩义市1213,荥阳市1214,中牟县1215,义马市1216,灵宝市1217,渑池县1218,陕县1219,卢氏县1220,偃师市1221,孟津县1222,新安县1223,栾川县1224,嵩县1225,汝阳县1226,宜阳县1227,洛宁县1228,伊川县1229,孟州市1230,沁阳市1231,修武县1232,博爱县1233,武陟县1234,温县1235,卫辉市1236,辉县市1237,新乡县1238,获嘉县1239,原阳县1240,延津县1241,封丘县1242,长垣县1243,浚县1244,淇县1245,林州市1246,安阳县1247,汤阴县1248,滑县1249,内黄县1250,清丰县1251,南乐县1252,范县1253,台前县1254,濮阳县1255,杞县1256,通许县1257,尉氏县1258,开封县1259,兰考县1260,永城市1261,虞城县1262,民权县1263,宁陵县1264,睢县1265,夏邑县1266,柘城县1267,禹州市1268,长葛市1269,许昌县1270,鄢陵县1271,襄城县1272,舞阳县1273,临颍县1274,舞钢市1275,汝州市1276,宝丰县1277,叶县1278,鲁山县1279,郏县1280,邓州市1281,南召县1282,方城县1283,西峡县1284,镇平县1285,内乡县1286,淅川县1287,社旗县1289,新野县1290,桐柏县1291,息县1292,淮滨县1293,光山县1294,固始县1295,商城县1296,罗山县1297,新县1298,项城市1299,扶沟县1300,西华县1301,商水县1302,太康县1303,鹿邑县1304,郸城县1305,淮阳县1306,沈丘县1307,确山县1308,泌阳县1309,遂平县1310,西平县1311,上蔡县1312,汝南县1313,平舆县1314,新蔡县1315,正阳县1316,丹江口市1317,郧县1318,竹山县1319,房县1320,郧西县1321,竹溪县1322,老河口市1323,枣阳市1324,宜城市1325,南漳县1326,谷城县1327,保康县1328,钟祥市1329,沙洋县1330,京山县1331,应城市1332,安陆市1333,汉川市1334,孝昌县1335,大悟县1336,云梦县1337,麻城市1338,武穴市1339,红安县1340,罗田县1341,英山县1342,浠水县1343,蕲春县1344,黄梅县1345,团风县1346,大冶市1347,阳新县1348,赤壁市1349,嘉鱼县1350,通城县1351,崇阳县1352,通山县1353,石首市1354,洪湖市1355,松滋市1356,江陵县1357,公安县1358,监利县1359,枝江市1360,宜都市1361,当阳市1362,远安县1363,兴山县1364,秭归县1365,长阳土家族自治县1366,五峰土家族自治县1367,广水市1368,恩施市1369,利川市1370,建始县1371,巴东县1372,宣恩县1373,咸丰县1374,来凤县1375,鹤峰县1376,浏阳1379,宁乡县1380,慈利县1381,桑植县1382,津市市1383,安乡县1384,汉寿县1385,澧县1386,临澧县1387,桃源县1388,石门县1389,沅江市1390,南县1391,桃江县1392,安化县1393,汨罗市1394,临湘市1395,岳阳县1396,华容县1397,湘阴县1398,平江县1399,醴陵市1400,株洲县1401,攸县1402,茶陵县1403,炎陵县1404,湘乡市1405,韶山市1406,湘潭县1407,常宁市1408,耒阳市1409,衡阳县1410,衡南县1411,衡山县1412,衡东县1413,祁东县1414,资兴市1415,桂阳县1416,永兴县1417,宜章县1418,嘉禾县1419,临武县1420,汝城县1421,桂东县1422,安仁县1423,东安县1424,道县1425,宁远县1426,江永县1427,蓝山县1428,新田县1429,双牌县1430,祁阳县1431,江华瑶族自治县1432,武冈市1433,邵东县1434,邵阳县1435,新邵县1436,隆回县1437,洞口县1438,绥宁县1439,新宁县1440,城步苗族自治县1441,洪江市1442,沅陵县1443,辰溪县1444,溆浦县1445,中方县1446,会同县1447,麻阳苗族自治县1448,新晃侗族自治县1449,芷江侗族自治县1450,靖州苗族侗族自治县1451,通道侗族自治县1452,冷水江市1453,涟源市1454,双峰县1455,新化县1456,吉首市1457,泸溪县1458,凤凰县1459,花垣县1460,保靖县1461,古丈县1462,永顺县1463,龙山县1464,英德市1465,连州市1466,佛冈县1467,阳山县1469,连山壮族瑶族自治县1470,连南瑶族自治县1471,乐昌市1472,南雄市1473,始兴县1474,仁化县1475,翁源县1476,新丰县1477,乳源瑶族自治县1478,紫金县1479,龙川县1480,连平县1481,和平县1482,东源县1483,兴宁市1484,梅县1485,大埔县1486,丰顺县1487,五华县1488,平远县1489,蕉岭县1491,饶平县1492,南澳县1493,普宁市1495,揭西县1496,惠来县1497,陆丰市1498,海丰县1499,陆河县1500,博罗县1501,惠东县1502,龙门县1503,恩平市1504,台山市1505,开平市1506,鹤山市1507,高要市1508,四会市1509,广宁县1510,怀集县1511,封开县1512,德庆县1513,罗定市1514,云安县1515,新兴县1516,郁南县1517,阳春市1518,阳西县1519,阳东县1520,化州市1521,信宜市1522,高州市1523,电白县1524,吴川市1525,廉江市1526,雷州市1527,遂溪县1528,徐闻县1529,武鸣县1530,横县1531,宾阳县1532,上林县1533,隆安县1534,马山县1535,阳朔县1537,灵川县1538,全州县1539,兴安县1540,永福县1541,灌阳县1542,资源县1543,平乐县1544,荔浦县1545,龙胜各族自治县1546,恭城瑶族自治县1547,柳江县'''print city_code.split("\n")class ShopBindHandle(object):def __init__(self,base_page_url="http://www.dianping.com/search/keyword/{}/0_{}/p{}",city=u"北京",wifi_id=None,brand_name=None):self.base_page_url = base_page_urlself.city=cityself.wifi_id = wifi_idself.city_code = self.get_city_code()self.brand_name=brand_name#参数为两个字典，比如：{"longitude":111,"latitude":322},{"longitude":115551,"latitude":322333}def calc_distance(self,geo1,geo2):Lat_A, Lng_A, Lat_B, Lng_B = geo1.get("latitude"),geo1.get("latitude"), geo2.get("latitude"),geo2.get("latitude")ra = 6378.140 # 赤道半径 (km)rb = 6356.755 # 极半径 (km)flatten = (ra - rb) / ra # 地球扁率rad_lat_A = radians(Lat_A)rad_lng_A = radians(Lng_A)rad_lat_B = radians(Lat_B)rad_lng_B = radians(Lng_B)pA = atan(rb / ra * tan(rad_lat_A))pB = atan(rb / ra * tan(rad_lat_B))xx = acos(sin(pA) * sin(pB) + cos(pA) * cos(pB) * cos(rad_lng_A - rad_lng_B))c1 = (sin(xx) - xx) * (sin(pA) + sin(pB)) ** 2 / cos(xx / 2) ** 2c2 = (sin(xx) + xx) * (sin(pA) - sin(pB)) ** 2 / sin(xx / 2) ** 2dr = flatten / 8 * (c1 - c2)distance = ra * (xx + dr)return distance#得到城市对应的编号，如：(上海,1)def get_city_code(self):pass# 得到所有大众点评上面所有的brand_name相关的连接def get_brand_urls(self,brand_name):try:for i in xrange(1,1000):res = urllib2.urlopen(urllib2.Request(self.base_page_url.format(self.city_code,brand_name,i), None, header), timeout=10)for brand_shop_url in BeautifulSoup(res).findAll("a", attrs={"data-hippo-type": "shop"}):if brand_name in brand_shop_url.get_text():yield "http://www.dianping.com"+brand_shop_url.get("href")except:return# 得到大众点评上面对应的商店的id和geodef get_shop_id(self,shop_url):soup = BeautifulSoup(urllib2.urlopen(shop_url))_geo_text = soup.select_one("#aside script").get_text()_p = re.compile(r"(lng:.*,lat:.*)}").findall(_geo_text)[0]#[u'lng:121.47835,lat:31.22062']return {"shop_id":shop_url.split("/")[-1],"geo":{"longitude":_p.split(",")[0].split(":")[-1],"latitude":_p.split(",")[1].split(":")[-1]}}# print get_shop_id("http://www.dianping.com/shop/22680632")def get_all_shops(brand_name):for brand_url in get_brand_urls(brand_name):if not brand_url:continueyield get_shop_id(brand_url)# 得到唯一的shop_iddef get_bind(self,brand_name,geo):dis0=-1shop0=Nonefor shop in self.get_all_shops(brand_name):dis = self.calc_distance(shop.geo,geo)if dis > dis0:shop0=shopdis0 = disreturn shop0.shop_id,self.wifi_iddef get_shop_info(shop_url):res = urllib2.urlopen(shop_url)soup = BeautifulSoup(res.read())# basic_info = soup.find(id="basic-info")# brand_name = basic_info.h1.get_text().strip().split("\n")[0]# print brand_name# brief_info = basic_info.select(".brief-info span")# star = brief_info[0].attrs.get("title")# print star# judge = brief_info[1].get_text()# print judge# avg = brief_info[2].get_text()# print avg# taste = brief_info[3].get_text()# print taste# env = brief_info[4].get_text()# print env# serve = brief_info[5].get_text()# print serve# address = basic_info.select(".expand-info.address")[0].get_text().replace("\n","").replace(" ","")# print address# tel = basic_info.select(".expand-info.tel")[0].get_text().replace("\n","")# print tel# "other J-other Hide"# other = basic_info.select(".other .info")# brand_alias = other[0].get_text().replace("\n","")# print brand_alias# working_time = "".join(other[1].get_text().strip().split("\n")[0:-1]).replace(" ","")# print working_time# geo_text = soup.find(id="aside").find("script").get_text()# import re# p = re.compile(r"(lng:.*,lat:.*)}").findall(geo_text)#[u'lng:121.47835,lat:31.22062']# p = dict([(x.split(":")[0],x.split(":")[-1])for x in p[0].split(",")]) #{u'lat': u'31.22062', u'lng': u'121.47835'}# print pspeciality = soup.select_one("#shop-tabs script")#.get_text()# speciality = BeautifulSoup(speciality).find_all("a")# print speciality.contents[3]print specialityprint "ok"# get_shop_info("http://www.dianping.com/shop/24730133")def get_url():test_url="http://www.dianping.com/search/keyword/1/0_星巴克/p100"res = urllib2.urlopen(test_url)soup = BeautifulSoup(res.read())soup = soup.findAll("a", attrs={"data-hippo-type": "shop"})print soup# print get_url()# test_url="http://www.dianping.com/shop/24730133"# res = urllib2.urlopen(urllib2.Request(test_url, None, header))# print res.read()# print res.read()# soup = BeautifulSoup(res.read())# soup = soup.find_all("a")[4]# print soup.attrs.get("href")# print soup.get_text()# print res.headers# print res.header