This is another chunk of code I recently uncovered, it was a python library for working with the banlist.nl webservice. The idea was that people who wanted to do interesting things with their banlists, especially administrators of user lists or hosting bots could use this to make their life easier. Included is also a script to transfer bans from banlist.nl to a GHost++ server instance. It is written in python3 and made available under the apache license. You can download the package
#!/usr/bin/python3 # # Copyright [2010] [Alexander Knaust] # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. '''This is a library of functions for interacting with banlist.nl, and the banlist system in general, designed with ghost++ in mind, although kept as extensible as possible. With this you can download online banlists as well as share your banlist with others using the banlist.nl servers. I recommend using it with httplib2 because it will significantly decrease the load on the server as well as the load on your connection It is designed to be Python 3.x compatible ''' import hashlib import re import xml.dom import xml.dom.minidom from datetime import datetime __version__ = "0.1" OLD_LOGIN = "http://www.banlist.nl/banlist_login.php?" OLD_UPLOAD = "http://www.banlist.nl/banlist_upload.php" OLD_DOWNLOAD = "http://www.banlist.nl/banlist_download.php?list=" NEW_UPLOAD = "http://www.banlist.nl/bl_upload.php"; NEW_SERVER = "http://www.banlist.nl/bl_server.php?"; ANON_DOWNLOAD = "http://www.banlist.nl/show.php?user="; #info types for the parse_localxml SEPERATE_INFOS=0 ALL_BANS=1 ALL_INFOS=2 #i've included THE official DotA shared banlists here, as well as the VIP lists #for Northrend and Azeroth (list the members of the NSG/ASG) just for fun. #these are all links to xml documents that point to the raw data, and have #some metadata attached... you can make your own at http://wc3banlist.de/xmlgen.php NORTHREND_AH_LIST = 'http://banlist.nl/files/approved_northrend.xml' NORTHREND_XML_LIST = 'http://banlist.eu/xml.php' NORTHREND_VIP_LIST = 'http://banlist.eu/vipxml.php' AZEROTH_AH_LIST = 'http://banlist.us/download.php?id=AAH' AZEROTH_XML_LIST = 'http://banlist.us/download.php?id=ASGXML' AZEROTH_VIP_LIST = 'http://banlist.us/download.php?id=VIP' LORDAERON_AH_LIST = 'http://banlist.nl/files/approved_lordaeron.xml' KALIMDOR_AH_LIST = 'http://banlist.nl/files/approved_kalimdor.xml' #lol? no ah's there... #python 2/3 indiscrepencies try: import urllib.request as urllib2 import urllib.parse as urlparse from io import StringIO except ImportError: from StringIO import StringIO import urllib2 import urllib as urlparse #httplib2 is much preferred because of caching and compression support #however if you really are going to run without it... _has_httplib2 = False try: import httplib2 _has_httplib2 = True http_con = httplib2.Http('.cache') except ImportError: print('You should get httplib2, its better...') class PyBlnlError(Exception): '''Is the parent of all the errors here''' pass class LoginError(PyBlnlError): pass class UploadError(PyBlnlError): pass class DownloadError(PyBlnlError, ValueError): pass class Banlist(list): '''A class to represent banlists, infolists and even viplists, it inherits list, because thats what they are... basically it is a list with a few extra attributes, type safety, and a customized sort Only accepts tuples of lengths 2/3 as elements ''' INFOLIST = 0 BANLIST = 1 VIPLIST = 2 def __init__ ( self, banner='banlist.nl', realm='Northrend', type=BANLIST, has_dates=False ): '''Initialize a banlist, checking to make sure the values are good Keyword Arguments: banner --the person this banlist belongs to type --the type of the banlist (BANLIST, INFOLIST, VIPLIST) has_dates --whether or not the banlist contains extra dates, (tuples length 3) ''' if not isinstance(has_dates, bool): raise ValueError('has_dates should be a boolean') if type not in (self.INFOLIST, self.BANLIST, self.VIPLIST): raise ValueError('type must be Banlist.INFOLIST or Banlist.BANLIST') self.banner = banner self.realm = realm self.type = type self.has_dates = has_dates def append( self, ban ): '''Overrides list.append for type saftey''' if not isinstance(ban, tuple): raise ValueError("Can't add {0} as a ban, because its not a tuple".format(ban)) else: if (len(ban) !=2 and not self.has_dates ) or (len(ban)!=3 and self.has_dates): raise ValueError("Can't add {0} as a ban, because it is not the right length".format(ban)) else: list.append(self, ban) def insert( self, i, ban ): '''Overrides list.insert for type saftey''' if not isinstance(ban, tuple): raise ValueError("Can't add {0} as a ban, because its not a tuple".format(ban)) else: if (len(ban) !=2 and not self.has_dates ) or (len(ban)!=3 and self.has_dates): raise ValueError("Can't add {0} as a ban, because it is not the right length".format(ban)) else: list.insert(self,i , ban) def extend( self, other ): '''Overrides list.extend for type safety''' if not isinstance(other, Banlist): raise ValueError('Can only extend with other Banlist objects!') elif other.has_dates != self.has_dates: raise ValueError('Cannot combine Banlist with dates, and banlist without') else: if other.type != self.type: print('WARNING : Combining two different types of banlists!') #yea man... list.extend(self, other) def sort(self): '''Overrides list.sort() to force sorting by first key (banned name)''' list.sort(self, lambda a,b: cmp(a[0],b[0])) ######################################################################## ## End of Classes ## ######################################################################## def _fetch_url( url, credentials = None ): '''Returns a stream given a url string, is a seperate function to account for httplib2/urllib.request differences Keyword Arguments: url --a url string pointing to what to get credentials --a dictionary of key:credential to fetch with POST ''' if _has_httplib2: if credentials is not None: headers = {'Content-type': 'application/x-www-form-urlencoded'} response, content = http_con.request(url,'POST',headers=headers, body=urlparse.urlencode(credentials)) else: response, content = http_con.request(url) else: if credentials is not None: content = urllib2.urlopen(url, urlparse.urlencode(credentials)).read() else: content = urllib2.urlopen(url).read() return StringIO(str(content)) #i don't think it uses a standard encoding... def lookup_realm( banlist_realm ): '''Turns a realm w/e (works at least for xml 'type's) into a server (europe.battle.net, etc...) Keyword Arguments: banlist_realm --a string that represents the realm (should have 'northrend','azeroth', etc. in it) returns --a server string in the form of europe.battle.net... etc. (None if no-match) ''' if 'northrend' in banlist_realm.lower(): return 'europe.battle.net' elif 'azeroth' in banlist_realm.lower(): return 'useast.battle.net' elif 'lordaeron' in banlist_realm.lower(): return 'uswest.battle.net' elif 'kalimdor' in banlist_realm.lower(): return 'asia.battle.net' else: raise ValueError('Could not determine realm of {0}'.format(banlist_realm)) def _get_session_key(user, passwd ,unhashed = False ): ''' Returns the session key required for talking to banlist.nl using the older (banlist.nl account) scripts given the url, name, and passwd Keyword Arguments : login_url --the base url for connecting to banlist.nl user --a banlist username passwd --the usernames password (md5 hashed) unhashed --whether the passwd passed is already hashed returns --a string with the session ID ''' hashed_pw = hashlib.md5(bytes(passwd)).hexdigest() if unhashed else passwd #construct the login url (old scripts specific afaik) user_login_url = OLD_LOGIN + "login=" + user + "&pass=" + hashed_pw sess_key_page = _fetch_url( user_login_url ) sessid = sess_key_page.readline() if sessid.find('Error') >= 0: raise LoginError ( 'Incorrect Username/Password (Login Rejected)' ) return sessid def upload_bans_old ( username, passwd, bans, unhashed = False ): '''Uploads bans for a user to www.banlist.nl. Bans should be a Banlist object raises LoginException if the Login Data is incorrect. Will replace all bans they have uploaded Note: due to a serverside restriction to prevent server hammering, bans can only be uploaded every 5 minutes Note: it also strips all whitespace from banname and reason Keyword Arguments : username --the banlist.nl username to upload the bans to (not forum) passwd --the user's passwd (md5 hashed) bans --a Banlist object with bans to upload unhashed --whether the password is hashed or not (for testing purposes mainly) ''' #retreive the session id so we can upload the bans sessid = _get_session_key (username, passwd, unhashed ) #construct the banlist string to upload from the tuples, #banlist = 'gingerbreadman' + "\t" + 'wolves' + "\t0" + "\r\n" + 'muffinman' + "\t" + 'snakes are good' + "\t0" banlist = '' for banned, reason in bans: banlist += banned.strip() + '\t' + reason.strip() + '\t0\r\n' banlist = banlist.rstrip() #strip the last line break reply = _fetch_url(OLD_UPLOAD, {'PHPSESSID' : sessid, 'bans' : banlist}).readline() if reply: raise UploadError(reply) def download_bans_old( username, passwd, list_type, extract_dates=False, backdate_format='%d-%m-%Y', date_format ='%Y-%m-%d', unhashed = False): '''Uses the old banlist.nl scripts to download a banlist(s) Keyword Arguments: username --the user to login as passwd --the user's password (md5 hashed) list_type -- Either 'others' or 'own' (fetches buddiesbanlist from banlist.nl) extract_dates --whether to extract the dates or not backdate_format --the format override for dates matching DD-DD-DDDD date_format --the format override for dates matching DDDD-DD-DD unhashed --whether the password is hashed returns --a list of Banlist objects, or a Banlist object if list_type = 'own' ''' if ( list_type != 'others' and list_type != 'own' ): raise DownloadError("list_type should be 'others' or 'own'") sessid = _get_session_key( username, passwd, unhashed ) #get the list of bans ban_file = _fetch_url(OLD_DOWNLOAD + list_type, dict([('PHPSESSID', sessid),])) #if list type is others, we are going to be making a list of banlists #each with a given username, and all the bans that user has if list_type is 'others': list_of_banlists = list() cur_banlist = Banlist(has_dates=extract_dates) for line in ban_file.readlines(): banned, reason, username = re.split(r'\t',line)[:3] if username != cur_banlist.banner: list_of_banlists.append(cur_banlist) cur_banlist = Banlist(username) if extract_dates: date_banned = _extract_date(reason, date_format=date_format, backdate_format=backdate_format) cur_banlist.append((banned, reason, date_banned)) else: cur_banlist.append((banned, reason)) return list_of_banlists #if it is own, we are downloading our own banlist, so the banlist's banner = username else: banlist = Banlist(username,has_dates=extract_dates) for line in ban_file.readlines(): banned, reason = re.split(r'\t',line)[:2] if extract_dates: date_banned = _extract_date(reason, date_format=date_format, backdate_format=backdate_format) banlist.append((banned, reason, date_banned)) else: banlist.append((banned, reason)) return banlist def download_bans_anon ( username, extract_dates=False, backdate_format='%d-%m-%Y', date_format='%Y-%m-%d' ): '''Fetch the bans for a specific user anonymously (used by xml lists) Keyword Arguments: username --The banlist.nl name of the person whose bans we are fetching extract_dates --whether to extract the dates or not backdate_format --the format override for dates matching DD-DD-DDDD date_format --the format override for dates matching DDDD-DD-DD returns --a Banlist ''' banlist = Banlist(username,has_dates=extract_dates) fetched_bans = _fetch_url(ANON_DOWNLOAD + username.strip()) textbans = fetched_bans.readlines() if re.match(r"^---ERROR--- User \S* not found!$", textbans[0] ): raise DownloadError ('User {0} was not found (does not exist)'.format(username)) else: for line in textbans: banned, reason = line.split( None, 1 ) if extract_dates: date_banned = _extract_date(reason, date_format=date_format, backdate_format=backdate_format) banlist.append((banned.strip(), reason.strip(), date_banned)) else: banlist.append((banned.strip(), reason.strip())) return banlist def upload_bans_new ( username, passwd, ban_tuples, unhashed = False ): ''' Uploads bans using the newer banlist.nl scripts Keyword Arguments : username --the user's FORUM username passwd --hashed user's passwd up_url --the upload url ''' hashed_pw = hashlib.md5(bytes(passwd, "utf-8")).hexdigest() if unhashed else passwd #construct the banlist string to upload from the tuples, #banlist = 'gingerbreadman' + "\t" + 'wolves' + "\t0" + "\r\n" + 'muffinman' + "\t" + 'snakes are good' + "\t0" banlist = '' for banned, reason in ban_tuples: banlist += banned.strip() + "\t0\t0\t" + reason.strip() + '\n' banlist = banlist.rstrip() #strip the last line break post_data = urllib.parse.urlencode() reply = _fetch_url(NEW_UPLOAD, {'username' : username, 'password' : hashed_pw, 'bans' : banlist}).readline() #this uploads the bans if reply: raise UploadError(str(reply, 'utf-8')) def parse_online_xml (url): '''Given an online xml resource (for a viplist or xml banlist) it returns a dictionary with all the important information about the list i.e. Dictionary with keys {title, realm, type, data} Keyword Arguments: url --a url to an xml document (in serverlist format) see above returns --a dictionary (string:string) ''' metadata = dict() xmldoc = xml.dom.minidom.parse(_fetch_url(url)) metadata['title'] = xmldoc.getElementsByTagName('title')[0].firstChild.nodeValue metadata['realm'] = xmldoc.getElementsByTagName('realm')[0].firstChild.nodeValue metadata['type'] = xmldoc.getElementsByTagName('type')[0].firstChild.nodeValue metadata['data'] = xmldoc.getElementsByTagName('data')[0].firstChild.nodeValue return metadata #I thought it was best to keep these close to the function that uses them... _back_date_match = re.compile ( r'(\d{2,2})\D(\d{2,2})\D(\d{4,4})' ) _date_match = re.compile ( r'(\d{4,4})\D(\d{2,2})\D(\d{2,2})' ) def _extract_date ( reason, backdate_format='%d-%m-%Y', date_format='%Y-%m-%d', default_date=None ): ''' Attempts to extract the dates from a ban reason, using Date.strptime if it can't then it returns default_date (which defaults to Datetime.now). It uses two regex matchers, backdate and date, backdate picks up dates with the format DD-DD-DDDD (anything as seperator) date picks up DDDD-DD-DD Keyword Arguments : reason --a string representing a ban reason (with or without dates) date_format --a date format override for strptime() default_date --the date to return for no match, defaults to NOW return: returns a Datetime object ''' if default_date is None: default_date = datetime.today() #check to see if there is a ban at all if _date_match.search(reason) is None and _back_date_match.search(reason) is None: date_banned = default_date else: if _date_match.search(reason) is None: #should be a backdate try: d_b = '-'.join(_back_date_match.search(reason).groups()) date_banned = datetime.strptime(d_b, backdate_format) except: date_banned = default_date pass else: try: d_b = '-'.join(_date_match.search(reason).groups()) date_banned = datetime.strptime(d_b, date_format) except: date_banned = datetime.today() pass return date_banned def download_bans_xml ( url, strip_banner=True, extract_dates=True, raw_url=False, backdate_format='%d-%m-%Y', date_format='%Y-%m-%d'): '''Downloads a banlist from an online xml list (which is an xml document with a link to the raw data as text. Note : Extracting dates is a problem, because it requires that everyone on the list use a consistent date format, which is not always the case. It tries to make a deduction based on the realm, but defaults to the current date. (getting the gamename is near-impossible) Keyword Arguments: url --the url of the XML/text file to parse and download from strip_banner --Whether or not to remove the (banned by xxx) if present extract_dates --Whether to try to get the date from the ban or not (recommend not to) see note above raw_url --Whether the url is an xml( default) or a direct link to the textual data you should specify date_format if extract_dates is true backdate_format --overrides automatic guessing of DD-DD-DDDD dates date_format --override automatic guessing of DDDD-DD-DD dates returns --a list of banlists (one per banner) ''' banlist = list() #first step is to go through the short xml file, and gather the data if not raw_url: metadata = parse_online_xml(url) if metadata['type'] != 'banlist': raise ValueError('Not a valid banlist-xml') xml_data = _fetch_url(metadata['data']) if extract_dates: if not date_format: #our sophisticated guess at how the date is shaped xD mdy = metadata['realm'].find('US')>-1 backdate_format = '%m-%d-%Y' if mdy else '%d-%m-%Y' else: xml_data = _fetch_url(url) if extract_dates: if not backdate_format: backdate_format = '%d-%m-%Y' #regex matchers/subbers precompiled banner_match = re.compile ( r'\(banned by (\w+).*' ) #now for the fun part...although its called xml_data its really just plaintext c =1; list_of_banlists = list() cur_banlist = Banlist(has_dates=extract_dates) if raw_url else Banlist(has_dates=extract_dates, realm=metadata['realm']) for line in xml_data: if line.strip()!="" and len(line.split(None , 1))>1: banned = line.split(' ', 1)[0].strip().lower() #get the name of the banned guy if banned != '-0000000info': #some XML lists include this as a header... reason = line.split(' ', 1)[1].strip() if banner_match.search(reason) != None: banmatch = banner_match.search(reason) banner = banmatch.group(1) #get the person who is banning! if strip_banner: #strip (banned by xxx) reason = reason[:reason.rindex(banmatch.group(0))] else: banner = 'banlist.nl' #yeaman if banner != cur_banlist.banner: if len(cur_banlist)>0: list_of_banlists.append(cur_banlist) #add this banlist to the list, start a new one if raw_url: cur_banlist = Banlist(banner, has_dates=extract_dates) else: cur_banlist = Banlist(banner, has_dates=extract_dates, realm=metadata['realm']) #lets have fun with the dates... if extract_dates: date_banned = _extract_date(reason, backdate_format) cur_banlist.append((banned, reason, date_banned)) else: cur_banlist.append((banned,reason)) c+=1 list_of_banlists.append(cur_banlist) #add the last one and return return list_of_banlists def download_viplist ( url , raw_url=False ): '''Downloads the contents of a viplist, these are useful for automated updating of xml lists, and automated updating of other things, each vip has a name (forum) rank and a banlist.nl account Keyword Arguments: url --the url of an xml/text file with the viplist raw_url --Whether or not it is xml (default) or not xml returns --a list of tuples (vip, rank, banlist.nl acc) ''' viplist = list() if not raw_url: metadata = parse_online_xml(url) if metadata['type'] != 'viplist': raise ValueError('{0} is not a viplist-xml'.format(url)) vipdata = _fetch_url(metadata['data']) else: vipdata = _fetch_url(url) #will match #ilaggoodly Northrend Approved Host, Banlist account: ilagNorth vip_matcher = re.compile('^(\S+)\s([^,]+), Banlist account: (\S+)$') for line in vipdata: viplist.append(tuple(vip_matcher.match(line).groups())) return viplist def convert_local_xml( path, banner='banlist.nl',realm='Northrend', pull_dates=True, infos=SEPERATE_INFOS, ignore_deleted=False): '''Parses a local bans from wc3banlist, either exported or a localdb.xml and returns a banlist object or (banlist, infolist) Keyword Arguments: path --the place where the bans are located banner --the banner who's name will go in the banlist objects pull_dates --whether to keep the dates from the bans in the Banlist object infos --What to do with infos, (keep seperate, turn all bans to infos, or turn all infos to bans) returns: a banlist object, or two banlist objects (bans, infos) ''' banlist_date_format = '%Y-%m-%d' def _get_xmlbans( blist, type ): '''A sub-function that gets the bans out of an infolist or banlist''' try: #this can happen with exported lists, since they don't both types topelement = doc.getElementsByTagName(type+'list')[0] except IndexError: return for ban in topelement.getElementsByTagName(type): banned = ban.getAttribute('nick') deleted = ban.getAttribute('deleted') reason = ban.getElementsByTagName('comment')[0].firstChild.nodeValue date_banned = datetime.strptime(ban.getElementsByTagName('date')[0].firstChild.nodeValue, banlist_date_format) if not ignore_deleted: if deleted =="0": if pull_dates: blist.append((banned, reason, date_banned)) else: blist.append((banned, reason)) else: if pull_dates: blist.append((banned, reason, date_banned)) else: blist.append((banned, reason)) with open(path, 'r', encoding='utf-8') as xmlfile: doc = xml.dom.minidom.parse(xmlfile) banlist = Banlist(banner, realm, has_dates=pull_dates) infolist = Banlist(banner, realm, has_dates=pull_dates, type=Banlist.INFOLIST) if infos<=SEPERATE_INFOS: _get_xmlbans(banlist, 'ban') _get_xmlbans(infolist, 'info') return banlist, infolist elif infos == ALL_BANS: _get_xmlbans(banlist, 'ban') _get_xmlbans(banlist, 'info') return banlist else: _get_xmlbans(infolist, 'ban') _get_xmlbans(infolist, 'info') return infolist def banlist_to_localxml( banlist, filename='convertedbans.xml' ): '''Converts a Banlist object into a wc3banlist style exported xml list, that you can import into wc3banlist. The lastchange dates will be set to the dates in the banlist if it has any, else it will be set to now Keyword Arguments: banlist --a Banlist object to be converted ''' if not isinstance(banlist, Banlist): raise ValueError('banlist should be a Banlist object') type = 'ban' if banlist.type == Banlist.BANLIST else 'info' doc = xml.dom.minidom.Document() banlist_date_format = '%Y-%m-%d' listtype = doc.createElement(type+'list') doc.appendChild(listtype) #an example exported banlist #<?xml version="1.0" encoding="UTF-8"?> #<banlist> #<ban nick="ilaggoodly" deleted="0"> #<comment>leaver</comment> #<lastchange> #<date>2001-01-01</date> #<time>00:05:26</time> #</lastchange> #</ban> #</banlist> for ban in banlist: bantag = doc.createElement(type) listtype.appendChild(bantag) bantag.setAttributeNode(doc.createAttribute('deleted')) bantag.setAttributeNode(doc.createAttribute('nick')) bantag.setAttribute('deleted','0') bantag.setAttribute('nick', ban[0]) commenttag = doc.createElement('comment') commenttag.appendChild(doc.createTextNode(ban[1])) bantag.appendChild(commenttag) if banlist.has_dates: date_banned = datetime.strftime(ban[2], banlist_date_format) else: date_banned = datetime.strftime(datetime.today(), banlist_date_format) lastchange = doc.createElement('lastchange') datetag = doc.createElement('date') timetag = doc.createElement('time') datetag.appendChild(doc.createTextNode(date_banned)) timetag.appendChild(doc.createTextNode('00:00:00')) lastchange.appendChild(datetag) lastchange.appendChild(timetag) bantag.appendChild(lastchange) #write out our xml (it writes everything on one line, because #i discovered that if you try to pass a newline paramater #wc3banlist can no longer read it .... with open(filename,'w') as outfile: doc.writexml(outfile, encoding="UTF-8") if __name__=='__main__': user = 'snakeattack' password = '420505' my_banlist = download_bans_old(user, password,'own', unhashed=True) #upload_bans_old(user, password, my_banlist, unhashed=True) download_bans_anon('ilaggoodly',extract_dates=True) download_viplist(NORTHREND_VIP_LIST) lol = download_bans_xml(NORTHREND_XML_LIST) print(lol[0])