import sys import os.path import re import string # returns a dict of date, list of files xmldir = "C:/publicwhip/pwdata/scrapedxml" xmldir = "" xmldir = "/home/fawkes/pwdata/scrapedxml" typefiles = ("debates", ) typefiles = ("debates", "westminhall", "wms") def FindAllFiles(): fildays = {} for subtyp in typefiles: dxmldir = os.path.join(xmldir, subtyp) dfils = os.listdir(dxmldir) for dfil in dfils: h = re.search("(\d{4}-\d{2}-\d{2}).*\.xml$", dfil) if h: fildays.setdefault(h.group(1), []).append(os.path.join(dxmldir, dfil)) # separate the days into month batches daykeys = fildays.keys() daykeys.sort() monthlists = [] for daykey in daykeys: if (not monthlists) or (monthlists[-1][-1][:7] != daykey[:7]): monthlists.append([daykey]) else: monthlists[-1].append(daykey) print "number of monthlists", len(monthlists) #monthlists = monthlists[0:3] return fildays, monthlists # the object we use to search the countries def MakeCountriesSearch(lcountries): clist = [ ] for c in re.split("\n", lcountries): lc = string.strip(c) if lc: clist.append(lc) print "number of countries", len(clist) #clist = clist[30:120] rece = re.compile(string.join("|")) reclist = map(re.compile, clist) return rece, reclist, clist # different ways of summing the lists def SBool(r): return (r and 1 or 0) def SumBool(s, r): return s + SBool(r) def SumBLists(countthis, recsg): if not countthis: return map(SBool, recsg) return map(SumBool, countthis, recsg) def Sum(s, r): return s + r def SumLists(countthis, recsg): if not countthis: return recsg return map(Sum, countthis, recsg) def SumZero(recsg): return [ 0 for r in recsg ] # do all the things of a paragraph def MakeCountFile(rece, reclist, clist, fils): countperpara = None numparas = 0 countperspee = None numspees = 0 for fil in fils: fin = open(fil) countthisspee = None for para in fin.readlines(): if re.match("\s*<(?:p|\S*heading|speech|ques|reply)", para): # don't count speech titles as paragraphs if re.match("\s*