<\1', sr) # 2008-06-17 / 2008-10... if sdate=='2006-11-07' or sdate=='2006-11-08': sr = re.sub('\s*\s* ]*>", "", sr) sr = re.sub("", "", sr) if sdate=='2006-06-07' and re.search('/text/', url): sr = re.sub('(?!

', sr) # Make sure correction is before written answer question number - XXX right place? sr = re.sub('(\[\d+\])\s*((?:

)?)\s*(.*?)', r'\3 \1\2', sr) # split by sections hrsections = re.split('(?i)', sr) # this is the case for debates on 2003-03-13 page 30 # http://www.publications.parliament.uk/pa/cm200203/cmhansrd/vo030313/debtext/30313-32.htm if len(hrsections) == 1: if miscfuncs.IsNotQuiet(): print len(hrsections), 'page missing', url fout.write('

\n') urla = urla[1:] continue # Grr, missing footers ALL OVER THE PLACE now if len(hrsections) == 2: WriteCleanText(fout, hrsections[1], url, sdate) # write the body of the text for i in range(1,len(hrsections) - 1): WriteCleanText(fout, hrsections[i], url, sdate) # find the lead on with the footer footer = hrsections[-1] # the files are sectioned by the

tag into header, body and footer. nextsectionlink = re.findall('<\s*a\s+href\s*=\s*"?(.*?)"?\s*>next(?: section)?(?i)', footer) if len(nextsectionlink) > 1: raise Exception, "More than one Next Section!!!" if not nextsectionlink: urla = urla[1:] if urla and miscfuncs.IsNotQuiet(): print "Bridging the missing next section link at %s" % url else: currenturl = url url = urlparse.urljoin(url, nextsectionlink[0]) if len(urla) > 1 and urla[1] == url: urla = urla[1:] else: for uo in urla: if uo == url: print "previous URLs:\n" print string.join(urla, "\n") print "\nbad next url:\n" print url print "\ncurrent url:\n" print currenturl raise Exception, "Next Section misses out the urla list" urla[0] = url fout.close() # now we have the difficulty of pulling in the first link out of this silly index page def ExtractAllLinks(url, dgf, forcescrape): #print "ExtractAllLinks from", url request = urllib2.Request(url) if not forcescrape and dgf and os.path.exists(dgf): mtime = os.path.getmtime(dgf) mtime = time.gmtime(mtime) mtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT", mtime) request.add_header('If-Modified-Since', mtime) opener = urllib2.build_opener( DefaultErrorHandler() ) urx = opener.open(request) if hasattr(urx, 'status'): if urx.status == 304: return [] xlines = ''.join(urx.readlines()) urx.close() xlines = re.sub('^.*?(?is)', '', xlines) res = re.findall('(?is)', xlines) if not res: raise Exception, "No link found!!! %s\nURL: %s" % (xlines, url) urla = [] for iconti in res: uo = urlparse.urljoin(url, iconti) if (not urla) or (urla[-1] != uo): urla.append(uo) return urla def MakeDayMap(folder, typ): # make the output firectory if not os.path.isdir(pwcmdirs): os.mkdir(pwcmdirs) pwcmfolder = os.path.join(pwcmdirs, folder) if not os.path.isdir(pwcmfolder): os.mkdir(pwcmfolder) # the following is code copied from the lordspullgluepages # scan through the directory and make a mapping of all the copies for each lddaymap = { } for ldfile in os.listdir(pwcmfolder): mnums = re.match("%s(\d{4}-\d\d-\d\d)([a-z]*)\.html$" % typ, ldfile) if mnums: sdate = mnums.group(1) salpha = mnums.group(2) lddaymap.setdefault(sdate, []).append((AlphaStringToOrder(salpha), salpha, ldfile)) elif os.path.isfile(os.path.join(pwcmfolder, ldfile)): print "not recognized file:", ldfile, " in ", pwcmfolder return lddaymap, pwcmfolder def GetFileDayVersions(day, lddaymap, pwcmfolder, typ): # make the filename dgflatestalpha, dgflatest, dgflatestdayalpha = "", None, None if day in lddaymap: ldgf = max(lddaymap[day]) # uses alphastringtoorder dgflatestalpha = ldgf[1] dgflatest = os.path.join(pwcmfolder, ldgf[2]) dgflatestdayalpha = "%s%s" % (day, dgflatestalpha) dgfnextalpha = NextAlphaString(dgflatestalpha) ldgfnext = '%s%s%s.html' % (typ, day, dgfnextalpha) dgfnext = os.path.join(pwcmfolder, ldgfnext) dgfnextdayalpha = "%s%s" % (day, dgfnextalpha) assert not dgflatest or os.path.isfile(dgflatest) assert not os.path.isfile(dgfnext) return dgflatest, dgflatestdayalpha, dgfnext, dgfnextdayalpha def readPageX(filename): if not filename or not os.path.isfile(filename): return {} hFile = open(filename) line= hFile.readline() hFile.close() lmap={} attributes= re.search('', line).group(0) for match in re.finditer('([^ =]+)="([^"]+)"', attributes): lmap[match.group(1)] = match.group(2) return lmap def CompareScrapedFiles(prevfile, nextfile): if not prevfile: return "DIFFERENT" hprevfile = open(prevfile) dprevfile = hprevfile.readlines() hprevfile.close() hnextfile = open(nextfile) dnextfile = hnextfile.readlines() hnextfile.close() if len(dprevfile) == len(dnextfile) and dprevfile[1:] == dnextfile[1:]: return "SAME" if len(dprevfile) < len(dnextfile) and dprevfile[1:] == dnextfile[1:len(dprevfile)]: return "EXTENSION" return "DIFFERENT" ############################## # For gluing together debates ############################## def PullGluePages(options, folder, typ): daymap, scrapedDataOutputPath = MakeDayMap(folder, typ) # loop through the index file previously made by createhansardindex for commonsIndexRecord in CommonsIndex().res: # implement date range if not re.search(typ, commonsIndexRecord.recordType, re.I): continue if commonsIndexRecord.date < options.datefrom or commonsIndexRecord.date > options.dateto: continue latestFilePath, latestFileStem, nextFilePath, nextFileStem = \ GetFileDayVersions(commonsIndexRecord.date, daymap, scrapedDataOutputPath, typ) try: # hansard index page urlx = commonsIndexRecord.url if commonsIndexRecord.recordType == 'Votes and Proceedings' or commonsIndexRecord.recordType == 'questionbook': urla = [urlx] else: urla = ExtractAllLinks(urlx, latestFilePath, options.forcescrape) # this checks the url at start of file if not urla: continue if miscfuncs.IsNotQuiet(): print commonsIndexRecord.date, (latestFilePath and 'RE-scraping' or 'scraping'), re.sub(".*?cmhansrd/", "", urlx) # now we take out the local pointer and start the gluing GlueByNext(tempfilename, urla, urlx, commonsIndexRecord.date) except Exception, e: options.anyerrors = True if options.quietc: print e print "\tERROR! %s failed to scrape on %s, quietly moving to next day" % (typ, commonsIndexRecord.date) continue else: raise if CompareScrapedFiles(latestFilePath, tempfilename) == "SAME": if miscfuncs.IsNotQuiet(): print " matched with:", latestFilePath continue # before we copy over the file from tempfilename to nextFilePath, copy over the patch if there is one. ReplicatePatchToNewScrapedVersion(folder, latestFileStem, latestFilePath, nextFilePath, nextFileStem) # now commit the file os.rename(tempfilename, nextFilePath) # make the message print commonsIndexRecord.date, (latestFilePath and 'RE-scraped' or 'scraped'), re.sub(".*?cmpages/", "", nextFilePath) def ReplicatePatchToNewScrapedVersion(folderName, latestFileStem, latestFilePath, nextFilePath, nextFileStem): if not latestFilePath: return # check that the patch file for the 'next' version has not yet been created lpatchfilenext, lorgfilenext = GenPatchFileNames(folderName, nextFileStem)[:2] assert lorgfilenext == nextFilePath # patchtool should give same name we are using if os.path.isfile(lpatchfilenext): print " *****Warning: patchfile already present for newly scraped file:", lpatchfilenext assert False # patchfile already present for newly scraped file # now find the patch file and copy it in, verifying we know what we're doing lpatchfile, lorgfile, tmpfile = GenPatchFileNames(folderName, latestFileStem)[:3] assert lorgfile == latestFilePath # if there's an old patch, apply the patch to the old file if os.path.isfile(lpatchfile): shutil.copyfile(tempfilename, tmpfile) status = os.system("patch --quiet %s < %s" % (tmpfile, lpatchfile)) if status == 0: #print "Patchfile still applies, copying over ", lpatchfile, "=>", lpatchfilenext #print " There you go..." shutil.copyfile(lpatchfile, lpatchfilenext) else: print " Could not apply old patch file to this, status=", status def PullGlueToday(forcescrape): # Fetch 'Today in the Commons' index page frontpagedata = fetchTextFromUrl(TodayInTheCommonsIndexPageUrl) link01url = re.search("Go to Full Report", frontpagedata).group(1) pageurl = urlparse.urljoin(TodayInTheCommonsIndexPageUrl, link01url) preparedDateMatch = re.search("

Prepared: (\d+:\d+) on (\d+ [a-zA-Z]+ \d+)

", frontpagedata) preparedDateTime = mx.DateTime.DateTimeFrom(preparedDateMatch.group(1) + " " + preparedDateMatch.group(2)) spreparedDateTime = "%s" % preparedDateTime # convert to string (can't find the real way to do it) # extract the date from the browse links lower down headingDateMatch = re.search('''(?x)

Browse\sReport\sBy\sSection

\s*

House\sof\sCommons

\s* ([^<]*)

''', frontpagedata) headingDateTime = mx.DateTime.DateTimeFrom(headingDateMatch.group(1)) sdate = headingDateTime.date assert sdate <= preparedDateTime.date # prepared date must come after date from heading # make files which we will copy into lddaymap, pwcmfolder = MakeDayMap("debates", "debates") dgflatest, dgflatestdayalpha, dgfnext, dgfnextdayalpha = GetFileDayVersions(sdate, lddaymap, pwcmfolder, "debates") # See if we actually want to proceed with scraping, or if there already exists a 'printed' version # in which case we avoid replacing it with the 'today' version latestScrapedFileMetaData = readPageX(dgflatest) if latestScrapedFileMetaData.get('type')=='printed': print "'Printed' version of hansard for today has already been scraped. Skipping scrape of 'Today' version" return None if not forcescrape and latestScrapedFileMetaData.get('prepareddatetime') == spreparedDateTime: if miscfuncs.IsNotQuiet(): print "Prepared datetime", spreparedDateTime, "already done" return None tempFileHandle = open(tempfilename, "w") tempFileHandle.write('\n' % (TodayInTheCommonsIndexPageUrl, time.strftime('%Y-%m-%d', time.gmtime()), time.strftime('%X', time.gmtime()), spreparedDateTime)) GlueByToday(tempFileHandle, pageurl) tempFileHandle.close() comp = CompareScrapedFiles(dgflatest, tempfilename) # now commit the file if comp == 'DIFFERENT': print "writing: ", dgfnext os.rename(tempfilename, dgfnext) return sdate elif comp == 'EXTENSION': print "OVER-writing: ", dgflatest shutil.copyfile(tempfilename, dgflatest) os.remove(tempfilename) return sdate else: assert comp == 'SAME' print "download exactly the same: ", dgflatest return None def GlueByToday(outputFileHandle, pageurl): pagenumber=1 while pageurl: assert pagenumber==int(re.search('(\d+)\.htm$', pageurl).group(1)) preparedDateTime, nextLink, body = ScrapeTodayPage(pageurl) if miscfuncs.IsNotQuiet(): print "Processed [%s] which was prepared [%s]" % (pageurl, preparedDateTime) now = time.gmtime() outputFileHandle.write('\n' % (pageurl, preparedDateTime) ) outputFileHandle.write(body) outputFileHandle.write('\n') if nextLink: pageurl = urlparse.urljoin(pageurl, nextLink) else: pageurl = None pagenumber += 1 def ScrapeTodayPage(pageurl): raw_html = fetchTextFromUrl(pageurl) matches = re.search('''(?sx) Prepared:\s (\d+:\d+)\son\s(\d+\s[a-zA-Z]+\s\d+) .* (?: Next # link to next page | ) .* (.*) # main body text of page

\s* (?:|) ''', raw_html ) if not matches: print re.findall('', raw_html) print re.findall('', raw_html) print re.findall('(\d+:\d+)\son\s(\d+\s[a-zA-Z]+\s\d+)', raw_html) print re.findall('(?:Next|)', raw_html) print raw_html[:200] preparedDateTime = mx.DateTime.DateTimeFrom(matches.group(1) + " " + matches.group(2)) nextLink = matches.group(3) sbody = matches.group(4) # remove junk links inserted for left hand panel body = re.sub('Back to top', '', sbody) return preparedDateTime, nextLink, body def fetchTextFromUrl(url): ur = urllib.urlopen(url) frontpagedata = ur.read() ur.close(); return frontpagedata

', '', sr) sr = re.sub('\s+', '', sr) # To cope with post 2006-09; need a better way of doing this! sr = re.sub('

Browse\sReport\sBy\sSection