from mechanize import Browser,LinkNotFoundError
from ClientCookie import BaseHandler
from StringIO import StringIO
# import tidy
#
import sys
import re
from time import gmtime, strftime
#
# The following two line is specific to the site you want to crawl
# it provides some capabilities to your crawler for it to be able
# to understand the meaning of the data it is crawling ;
# as an example for knowing the age of the crawled resource
#
from datetime import date
# from my_parser import parsed_resource
#
"""
Let's declare some customized pre-processors.
These are useful when the HTML you are crawling through is not clean enough for mechanize.
When you crawl through bad HTML, mechanize often raises errors.
So either you tidy it with a strict tidy module (see TidyProcessor)
or you tidy some errors you identified "by hand" (see MyProcessor).
Note that because the tidy module is quite strict on HTML, it may change the whole
structure of the page you are dealing with. As an example, in bad HTML, you may encounter
nested forms or forms nested in tables or tables nested in forms. Tidying them may produce
unintended results such as closing the form too early or making it empty. This is the reason
you may have to use MyProcessor instead of TidyProcessor.
"""
#
class FakeResponse:
def __init__(self, resp, nudata):
self._resp = resp
self._sio = StringIO(nudata)
#
def __getattr__(self, name):
try:
return getattr(self._sio, name)
except AttributeError:
return getattr(self._resp, name)
#
class TidyProcessor(BaseHandler):
def http_response(self, request, response):
options = dict(output_xhtml=1,
add_xml_decl=1,
indent=1,
output_encoding='utf8',
input_encoding='latin1',
force_output=1
)
r = tidy.parseString(response.read(), **options)
return FakeResponse(response, str(r))
https_response = http_response
#
class MyProcessor(BaseHandler):
def http_response(self, request, response):
r = response.read()
r = r.replace('"image""','"image"')
r = r.replace('"','"')
return FakeResponse(response, r)
https_response = http_response
#
# Open a browser and optionally choose a customized HTML pre-processor
b = Browser()
b.add_handler(MyProcessor())
#
""""
Let's declare some utility methods that will enhance mechanize browsing capabilities
"""
#
def find(response,searchst):
response.seek(0)
lr = response.read()
return re.search(searchst, lr, re.I)
#
def save_response(response,kw='file'):
"""Saves last response to timestamped file"""
name = strftime("%Y%m%d%H%M%S_",gmtime())
name = name + kw + '.html'
f = open('./'+name,'w')
response.seek(0)
f.write(response.read())
f.close
return "Response saved as %s" % name
#
"""
Hereafter is the only (and somewhat big) script that is specific to the site you want to crawl.
"""
#
def dvspot_crawl():
"""
Here starts the browsing session.
For every move, I could have put as a comment an equivalent PBP command line.
PBP is a nice scripting layer on top of mechanize.
But it does not allow looping or conditional browsing.
So I preferred scripting directly with mechanize instead of using PBP
and then adding an additional layer of scripting on top of it.
"""
#
MAX_NR_OF_ITEMS_PER_SESSION = 500
#
# Go to home page
#
b.open("http://www.dvspot.com/reviews/cameraList.php?listall=1&start=0")
#
# Navigate through the paginated list of cameras
#
next_page = 0
while next_page == 0:
#
# Display and save details of every listed item
#
url = b.geturl()
next_element = 0
while next_element >= 0:
try:
response1 = b.follow_link(url_regex=re.compile(r"cameraDetail"), nr=next_element)
next_element = next_element + 1
print save_response(response1,"dvspot_camera_"+str(next_element))
b.open(url)
# if you crawled too many items, stop crawling
if next_element*next_page > MAX_NR_OF_ITEMS_PER_SESSION:
next_element = -1
next_page = -1
except LinkNotFoundError:
# You reached the last item in this page
next_element = -1
#
try:
b.open(url)
response2 = b.follow_link(text_regex=re.compile(r"Next Page"), nr=0)
print "processing Next Page"
except LinkNotFoundError:
# You reached the last page of the listing of items
next_page = -1
#
return
#
#
#
if __name__ == '__main__':
#
""" Note that you may need to specify your proxy first.
On windows, you do :
set HTTP_PROXY=http://proxyname.bigcorp.com:8080
"""
#
dvspot_crawl()


But this is the perspective of a guy that’s researching session scraping techniques and has yet to use either package, so ymmv.

Phil, if ever you remember why you thought that Mechanoid was better than Mechanize, please tell us !

– Sig

http://mechanoid.sourceforge.net/

–Phil.

Lixto Visual Wrapper

This component is responsible for locating and extracting desired data from the Web. While working with Lixto Visual Wrapper, an operator generates a wrapper program which converts relevant information from HTML to XML. Lixto Visual Wrapper generates a so-called XML companion for given Web pages.

http://www.lixto.com/show.php?page=vw_architecture&lg=EN

Cheers,
Roman