', table_content, flags=re.IGNORECASE|re.DOTALL)
for row in rows:
#collumn = re.findall('
]*>(.*?)
', row, flags=re.IGNORECASE|re.DOTALL)
bid = re.search('href="\/Ejendomspraesentation.aspx\?bid=([^"]*)"', row, flags=re.IGNORECASE|re.DOTALL)
if bid:
bid = int(bid.group(1))
result.append(bid)
return result
def extract_waitinglist_placements(self, bids, sleep=1):
result = {}
for bid in bids:
log.debug("Requesting placement on building #%u.", bid)
data = {
'buildingId': bid
}
headers = {
'Content-Type': 'application/json; charset=UTF-8'
}
response = self.session.post(self.URL_placement, data=json.dumps(data), headers=headers)
if response:
response = response.json()
if response["d"] and response["d"]["WaitPlacement"]:
result[str(bid)] = int(response["d"]["WaitPlacement"])
log.debug("It was %u.", result[str(bid)])
else:
raise RuntimeError("Error reading a placement: Error in JSON structure.")
else:
raise RuntimeError("Error reading a placement.")
time.sleep(sleep)
return result
def write_data(data):
fieldnames_temp = set()
try:
output_file = open(args.output,'r')
reader = csv.DictReader(output_file, delimiter=',')
all_data = list(reader)
# Extract the fieldnames
if reader.fieldnames:
for name in reader.fieldnames:
if name != "date":
fieldnames_temp.add(str(name))
output_file.close()
except IOError:
log.info("There was no existing data in the datafile.")
all_data = list()
# Do a union over all elements of the list.
fieldnames = ["date"]
fieldnames_temp |= set(data.keys())
fieldnames.extend(list(fieldnames_temp))
# Insert the date as the first field.
data["date"] = datetime.datetime.now().date().isoformat()
# Add this datapoint as new data.
all_data.append(data)
output_file = open(args.output,'wb')
writer = csv.DictWriter(output_file, delimiter=',', fieldnames=fieldnames)
writer.writeheader()
writer.writerows(all_data)
output_file.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--username', help='Your username on findbolig.nu.', required=True)
parser.add_argument('-p', '--password', help='Your password on findbolig.nu.', required=True)
parser.add_argument('-o', '--output', help='The output file.', required=True)
parser.add_argument('-l', '--log', help='Set the log level to debug.', default='WARNING')
args = parser.parse_args()
numeric_level = getattr(logging, args.log.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % args.log)
logging.basicConfig(format='%(levelname)s\t%(message)s', level=numeric_level)
print "findbolig.nu venteliste extractor v.0.1\n"
client = FindBoligNuClient()
success = client.login(args.username, args.password)
if not success:
log.error("Couldn't login using the credentials provided.")
sys.exit(-2)
# Fetch bids for all buildings on the whishlist.
venteliste_bids = client.extract_waitinglist_references()
# Iterate the list of bids and return a dict of the placements.
venteliste_placements = client.extract_waitinglist_placements(venteliste_bids, 0)
# Append to the datafile.
write_data(venteliste_placements)