MaxMind(R) GeoIP Lite CSV Import

Use this script to import the Maxmind GeoIP lite CSV datasets into your database. This takes at least 200MB of RAM; the resulting database will be ~400MB. Stick in the same directory as the models. Make sure to set DEBUG=False to prevent running out of memory during import.

# Copyright (c) 2007, Justin Bronn# All rights reserved.## Released under New BSD License#""" These scripts are used to import the MaxMind(R) GeoIP Lite CSV files. In order to save memory during import ensure DEBUG=False in your settings."""frommodelsimportCountry,CountryBlock,Location,LocationBlockfromdjango.contrib.gis.geosimportPointfromcsvimportreaderimportsysdefcountry_import(csv_file):fh=open(csv_file)table=reader(fh)header=table.next()forstartip,endip,ipfrom,ipto,country,country_nameintable:cntry,created=Country.objects.get_or_create(name=country_name,code=country)ifcreated:print'Created: %s'%cntryblock=CountryBlock(ipto=ipto,ipfrom=ipfrom,startip=startip,endip=endip,country=cntry)block.save()fh.close()deltabledeflocation_import(loc_csv,block_csv):ifloc_csv:# First, importing from the Location CSV file.fh=open(loc_csv)table=reader(fh)header=table.next()# Caching the countries table in memorycountries=dict((m.code,m)forminCountry.objects.all())i=0forlocid,cntry,reg,cty,postal,lat,lon,dma,areaintable:pnt=Point(float(lon),float(lat))# The points for the countries are in the first 244 entries --# pulling these out and updating the pointsifint(locid)<244:try:country=Country.objects.get(code=cntry)country.point=pntexcept:country=Country(code=cntry,point=pnt)country.save()countries[cntry]=country# updating the country dictionaryelse:country=countries[cntry]# region and cityregion=reg.decode('UTF-8','ignore')city=cty.decode('UTF-8','ignore')# Constructing the Locationloc=Location(locid=locid,country=country,region=region,city=city,postalcode=postal,point=pnt,dmacode=dma,areacode=area)loc.save()i+=1ifi%10000==0:print'Saved %d Locations so far ...'%ifh.close()deltabledelcountriesifblock_csv:# Second, importing from the Location IP block CSV filefh=open(block_csv)table=reader(fh)header=table.next()# This will take a little bit... and ~200+MB of RAMprint'Caching Location table...',sys.stdout.flush()locations=dict((m.locid,m)forminLocation.objects.all())print'DONE.'i=0foripfrom,ipto,locidintable:loc=locations[int(locid)]# pulling location from our cached table (less expensive than Location.objects.get())loc_block=LocationBlock(location=loc,ipfrom=ipfrom,ipto=ipto)loc_block.save()i+=1ifi%10000==0:print'Saved %d Location Blocks so far ...'%ifh.close()deltabledellocations