[Pikapika]科技文献的脉络梳理-构建查询Wiki的数据结构

首先我们需要实现的是一个能够查询Wiki的数据结构，而有如下几种处理方式

直接的爬虫抓取及其优化
调用现成python库（基于wiki的api）
下载离线版wikipedia

直接使用爬虫抓取wikipedia

如果直接使用python的urllib库调用的话，由于网络延迟等问题，效率大概是3sec/search

这里粘贴一个爬取百度百科的代码，wiki可以肯定比百度百科慢，毕竟在墙外面……

# -*- coding:utf-8 -*-
import urllib2
import re
import cgi
import time
import requests
import random
import httplib
from bs4 import BeautifulSoup


ip_update = -1
ip_list = []

def get_ip_list():
    url = 'http://www.xicidaili.com/nn/';
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
    }
    web_data = requests.get(url, headers=headers)
    soup = BeautifulSoup(web_data.text, 'lxml')
    ips = soup.find_all('tr')
    global ip_list
    ip_list = []
    for i in range(1, len(ips)):
        ip_info = ips[i]
        tds = ip_info.find_all('td')
        ip_list.append(tds[1].text + ':' + tds[2].text)
    print "IP list updated ..."
    print ip_list

def get_random_ip():
    global ip_update
    global ip_list
    if (ip_update < 0):
        get_ip_list()
        ip_update = 100
    else:
        ip_update -= 1
    proxy_list = []
    for ip in ip_list:
        proxy_list.append('http://' + ip)
    proxy_ip = random.choice(proxy_list)
    return proxy_ip

def open_url(url,enable_proxy=True):
        #ip = get_random_ip()
        ip = '180.76.154.5:8888'
        print url
        print "Process url " + '...' + " with proxy " + ip
        proxy_handler = urllib2.ProxyHandler({"http" : ip})
        null_proxy_handler = urllib2.ProxyHandler({})
        if enable_proxy:
            opener = urllib2.build_opener(proxy_handler)
        else:
            opener = urllib2.build_opener(null_proxy_handler)
        urllib2.install_opener(opener)
        i_headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8'}
        req = urllib2.Request(url,headers=i_headers)
        time.sleep(0.5)
        try:
            print "Sleepp....."
            html = urllib2.urlopen(req)
            print "Sleepp....."
            if (not html.geturl() == url):
                print "Warning: URL redirected..."
            return html
        except httplib.BadStatusLine:
            print "ERROR"
            return None
        except urllib2.URLError:
            print "Error: URLERROR"
            return open_url(url,enable_proxy)



if __name__ == '__main__':
    fin = open('keywords_output.txt','r')
    lst = fin.read().split('\n')
    fout = open('keywords_ranking.txt','w')
    lst = lst[0:-2]
    for w in lst:
        res = open_url('https://baike.baidu.com/item/%s'%(w))
        if (res == None):
            fout.write("--\n")
            continue
        text = res.read()
        for ww in lst:
            print ww,len(re.findall(ww,text)),
            fout.write(str(len(re.findall(ww,text)))+'\t')
        fout.write('\n')
        time.sleep(1)
        print ;

# -*- coding:utf-8 -*-

import urllib2

import re

import cgi

import time

import requests

import random

import httplib

from bs4 import BeautifulSoup

ip_update = -1

ip_list = []

def get_ip_list():

url = 'http://www.xicidaili.com/nn/';

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'

}

web_data = requests.get(url, headers=headers)

soup = BeautifulSoup(web_data.text, 'lxml')

ips = soup.find_all('tr')

global ip_list

ip_list = []

for i in range(1, len(ips)):

ip_info = ips[i]

tds = ip_info.find_all('td')

ip_list.append(tds[1].text + ':' + tds[2].text)

print "IP list updated ..."

print ip_list

def get_random_ip():

global ip_update

global ip_list

if (ip_update < 0):

get_ip_list()

ip_update = 100

else:

ip_update -= 1

proxy_list = []

for ip in ip_list:

proxy_list.append('http://' + ip)

proxy_ip = random.choice(proxy_list)

return proxy_ip

def open_url(url,enable_proxy=True):

#ip = get_random_ip()

ip = '180.76.154.5:8888'

print url

print "Process url " + '...' + " with proxy " + ip

proxy_handler = urllib2.ProxyHandler({"http" : ip})

null_proxy_handler = urllib2.ProxyHandler({})

if enable_proxy:

opener = urllib2.build_opener(proxy_handler)

else:

opener = urllib2.build_opener(null_proxy_handler)

urllib2.install_opener(opener)

i_headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8'}

req = urllib2.Request(url,headers=i_headers)

time.sleep(0.5)

try:

print "Sleepp....."

html = urllib2.urlopen(req)

print "Sleepp....."

if (not html.geturl() == url):

print "Warning: URL redirected..."

return html

except httplib.BadStatusLine:

print "ERROR"

return None

except urllib2.URLError:

print "Error: URLERROR"

return open_url(url,enable_proxy)

if __name__ == '__main__':

fin = open('keywords_output.txt','r')

lst = fin.read().split('\n')

fout = open('keywords_ranking.txt','w')

lst = lst[0:-2]

for w in lst:

res = open_url('https://baike.baidu.com/item/%s'%(w))

if (res == None):

fout.write("--\n")

continue

text = res.read()

for ww in lst:

print ww,len(re.findall(ww,text)),

fout.write(str(len(re.findall(ww,text)))+'\t')

fout.write('\n')

time.sleep(1)

print ;

接着考虑使用更加高效的爬取框架scrapy

代码架构比较复杂，但是搜索的速度可以提升至1sec/search，其原理大概是通过异步的方式解决网络延迟的问题，但是基础的长达3sec的网络延迟还是必定会存在的，不过对于我们的分析还是过于慢了

Python库实现wiki查询

python有一个叫做wikipedia的库，其核心是通过wikipedia的公开api实现查询。还是很慢……

离线Wiki数据查询

离线数据下载主页：https://en.wikipedia.org/wiki/Wikipedia:Database_download

具体来说我下载的离线数据在这里：https://dumps.wikimedia.org/other/static_html_dumps/current/zh/

其中html.lst文件列举了所有文件名，wikipedia-zh-html.tar.7z文件就是实际的数据集，解压后11G，最棒的是解压后发现居然已经帮我把trie树建出来了！

但是，问题来了，虽然在个人的mac上面文件名非常正常，但是把它弄到ubuntu下面就完全不可看，大概就是一堆问号的情况。事实上这只是ssh上去的显示问题，还是可以用程序访问的。

不过速度嘛……

直接使用爬虫抓取wikipedia

Python库实现wiki查询

离线Wiki数据查询

发表评论 取消回复

发表评论取消回复